From fb4aae080bcd8181d3d9aa4dc9c5088b0ddd3149 Mon Sep 17 00:00:00 2001 From: Joe Schr <8218910+TheJoeSchr@users.noreply.github.com> Date: Fri, 3 Jan 2025 16:21:54 +0100 Subject: [PATCH 1/8] refactor: Modify stacked_imbalance to return list of prices instead of max price --- freqtrade/data/converter/orderflow.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/freqtrade/data/converter/orderflow.py b/freqtrade/data/converter/orderflow.py index a51394ce9..5af63dfb8 100644 --- a/freqtrade/data/converter/orderflow.py +++ b/freqtrade/data/converter/orderflow.py @@ -269,16 +269,18 @@ def stacked_imbalance( int_series.groupby((int_series != int_series.shift()).cumsum()).cumcount() + 1 ) - max_stacked_imbalance_idx = stacked.index[stacked >= stacked_imbalance_range] - stacked_imbalance_price = np.nan - if not max_stacked_imbalance_idx.empty: - idx = ( - max_stacked_imbalance_idx[0] + stacked_imbalance_idx = stacked.index[stacked >= stacked_imbalance_range] + stacked_imbalance_prices = [] + + if not stacked_imbalance_idx.empty: + indices = ( + stacked_imbalance_idx if not should_reverse - else np.flipud(max_stacked_imbalance_idx)[0] + else np.flipud(stacked_imbalance_idx) ) - stacked_imbalance_price = imbalance.index[idx] - return stacked_imbalance_price + stacked_imbalance_prices = [float(imbalance.index[idx]) for idx in indices] + + return stacked_imbalance_prices if stacked_imbalance_prices else [np.nan] def stacked_imbalance_ask(df: pd.DataFrame, stacked_imbalance_range: int): From 558957887238ae508680e5fefac8fd6de9d479a1 Mon Sep 17 00:00:00 2001 From: Joe Schr <8218910+TheJoeSchr@users.noreply.github.com> Date: Fri, 3 Jan 2025 16:25:15 +0100 Subject: [PATCH 2/8] test: Update tests for stacked imbalances returning lists --- tests/data/test_converter_orderflow.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/data/test_converter_orderflow.py b/tests/data/test_converter_orderflow.py index 656c1eab3..acef1ef4f 100644 --- a/tests/data/test_converter_orderflow.py +++ b/tests/data/test_converter_orderflow.py @@ -1,4 +1,3 @@ -import numpy as np import pandas as pd import pytest @@ -185,24 +184,24 @@ def test_public_trades_mock_populate_dataframe_with_trades__check_orderflow( assert results["max_delta"] == 17.298 # Assert that stacked imbalances are NaN (not applicable in this test) - assert np.isnan(results["stacked_imbalances_bid"]) - assert np.isnan(results["stacked_imbalances_ask"]) + assert results["stacked_imbalances_bid"] == [np.nan] + assert results["stacked_imbalances_ask"] == [np.nan] # Repeat assertions for the third from last row results = df.iloc[-2] assert pytest.approx(results["delta"]) == -20.862 assert pytest.approx(results["min_delta"]) == -54.559999 assert 82.842 == results["max_delta"] - assert 234.99 == results["stacked_imbalances_bid"] - assert 234.96 == results["stacked_imbalances_ask"] + assert results["stacked_imbalances_bid"] == [234.99] + assert results["stacked_imbalances_ask"] == [234.96] # Repeat assertions for the last row results = df.iloc[-1] assert pytest.approx(results["delta"]) == -49.302 assert results["min_delta"] == -70.222 assert pytest.approx(results["max_delta"]) == 11.213 - assert np.isnan(results["stacked_imbalances_bid"]) - assert np.isnan(results["stacked_imbalances_ask"]) + assert results["stacked_imbalances_bid"] == [np.nan] + assert results["stacked_imbalances_ask"] == [np.nan] def test_public_trades_trades_mock_populate_dataframe_with_trades__check_trades( @@ -358,7 +357,8 @@ def test_public_trades_binned_big_sample_list(public_trades_list): assert 197.512 == df["bid_amount"].iloc[0] # total bid amount assert 88.98 == df["ask_amount"].iloc[0] # total ask amount assert 26 == df["ask"].iloc[0] # ask price - assert -108.532 == pytest.approx(df["delta"].iloc[0]) # delta (bid amount - ask amount) + # delta (bid amount - ask amount) + assert -108.532 == pytest.approx(df["delta"].iloc[0]) assert 3 == df["bid"].iloc[-1] # bid price assert 50.659 == df["bid_amount"].iloc[-1] # total bid amount @@ -534,7 +534,8 @@ def test_analyze_with_orderflow( assert col in df1.columns, f"Column {col} not found in df.columns" if col not in ("stacked_imbalances_bid", "stacked_imbalances_ask"): - assert df1[col].count() == 5, f"Column {col} has {df1[col].count()} non-NaN values" + assert df1[col].count() == 5, f"Column {col} has { + df1[col].count()} non-NaN values" assert len(strategy._cached_grouped_trades_per_pair[pair]) == 5 @@ -552,7 +553,8 @@ def test_analyze_with_orderflow( assert "open" in df2.columns assert spy.call_count == 0 for col in ORDERFLOW_ADDED_COLUMNS: - assert col in df2.columns, f"Round2: Column {col} not found in df.columns" + assert col in df2.columns, f"Round2: Column { + col} not found in df.columns" if col not in ("stacked_imbalances_bid", "stacked_imbalances_ask"): assert ( From ea96abecd06b753f0780b1875dc92c4b884308db Mon Sep 17 00:00:00 2001 From: Joe Schr <8218910+TheJoeSchr@users.noreply.github.com> Date: Fri, 3 Jan 2025 16:27:18 +0100 Subject: [PATCH 3/8] test: Add comprehensive test for stacked_imbalances with multiple price entries --- tests/data/test_converter_orderflow.py | 36 ++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/data/test_converter_orderflow.py b/tests/data/test_converter_orderflow.py index acef1ef4f..50b47b7d5 100644 --- a/tests/data/test_converter_orderflow.py +++ b/tests/data/test_converter_orderflow.py @@ -7,6 +7,7 @@ from freqtrade.data.converter.orderflow import ( ORDERFLOW_ADDED_COLUMNS, timeframe_to_DateOffset, trades_to_volumeprofile_with_total_delta_bid_ask, + stacked_imbalance, ) from freqtrade.data.converter.trade_converter import trades_list_to_df from freqtrade.data.dataprovider import DataProvider @@ -569,6 +570,41 @@ def test_analyze_with_orderflow( assert isinstance(lastval_of2, dict) +def test_stacked_imbalances_multiple_prices(): + """Test that stacked imbalances correctly returns multiple price levels when present""" + # Test with empty result + df_no_stacks = pd.DataFrame( + { + 'bid_imbalance': [False, False, True, False], + 'ask_imbalance': [False, True, False, False] + }, + index=[234.95, 234.96, 234.97, 234.98] + ) + no_stacks = stacked_imbalance(df_no_stacks, "bid", stacked_imbalance_range=2, should_reverse=False) + assert no_stacks == [np.nan] + + # Create a sample DataFrame with known imbalances + df = pd.DataFrame( + { + 'bid_imbalance': [True, True, True, False, False, True, True, False], + 'ask_imbalance': [False, False, True, True, True, False, False, True] + }, + index=[234.95, 234.96, 234.97, 234.98, 234.99, 235.00, 235.01, 235.02] + ) + # Test bid imbalances (should return prices in ascending order) + bid_prices = stacked_imbalance(df, "bid", stacked_imbalance_range=2, should_reverse=False) + assert bid_prices == [234.95, 234.96, 234.97, 235.00, 235.01] + + # Test ask imbalances (should return prices in descending order) + ask_prices = stacked_imbalance(df, "ask", stacked_imbalance_range=2, should_reverse=True) + assert ask_prices == [235.02, 234.99, 234.98, 234.97] + + # Test with higher stacked_imbalance_range + bid_prices_higher = stacked_imbalance(df, "bid", stacked_imbalance_range=3, should_reverse=False) + assert bid_prices_higher == [234.95, 234.96, 234.97] + + + def test_timeframe_to_DateOffset(): assert timeframe_to_DateOffset("1s") == pd.DateOffset(seconds=1) assert timeframe_to_DateOffset("1m") == pd.DateOffset(minutes=1) From 11976f11b0c0e9a9c462689fa41d76eae426da93 Mon Sep 17 00:00:00 2001 From: Joe Schr <8218910+TheJoeSchr@users.noreply.github.com> Date: Fri, 3 Jan 2025 16:30:19 +0100 Subject: [PATCH 4/8] fix: Improve stacked imbalance detection in orderflow converter --- freqtrade/data/converter/orderflow.py | 25 ++++++++++++++----------- tests/data/test_converter_orderflow.py | 16 ++++++++-------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/freqtrade/data/converter/orderflow.py b/freqtrade/data/converter/orderflow.py index 5af63dfb8..6d2df7d3d 100644 --- a/freqtrade/data/converter/orderflow.py +++ b/freqtrade/data/converter/orderflow.py @@ -265,20 +265,23 @@ def stacked_imbalance( """ imbalance = df[f"{label}_imbalance"] int_series = pd.Series(np.where(imbalance, 1, 0)) - stacked = int_series * ( - int_series.groupby((int_series != int_series.shift()).cumsum()).cumcount() + 1 - ) - - stacked_imbalance_idx = stacked.index[stacked >= stacked_imbalance_range] - stacked_imbalance_prices = [] + # Group consecutive True values and get their counts + groups = (int_series != int_series.shift()).cumsum() + counts = int_series.groupby(groups).cumsum() - if not stacked_imbalance_idx.empty: - indices = ( - stacked_imbalance_idx + # Find indices where count meets or exceeds the range requirement + valid_indices = counts[counts >= stacked_imbalance_range].index + + stacked_imbalance_prices = [] + if not valid_indices.empty: + # Get all prices from valid indices from beginning of the range + valid_prices = [imbalance.index.values[idx-(stacked_imbalance_range-1)] for idx in valid_indices] + # Sort prices according to direction + stacked_imbalance_prices = ( + sorted(valid_prices) if not should_reverse - else np.flipud(stacked_imbalance_idx) + else sorted(valid_prices, reverse=True) ) - stacked_imbalance_prices = [float(imbalance.index[idx]) for idx in indices] return stacked_imbalance_prices if stacked_imbalance_prices else [np.nan] diff --git a/tests/data/test_converter_orderflow.py b/tests/data/test_converter_orderflow.py index 50b47b7d5..792daea45 100644 --- a/tests/data/test_converter_orderflow.py +++ b/tests/data/test_converter_orderflow.py @@ -193,8 +193,8 @@ def test_public_trades_mock_populate_dataframe_with_trades__check_orderflow( assert pytest.approx(results["delta"]) == -20.862 assert pytest.approx(results["min_delta"]) == -54.559999 assert 82.842 == results["max_delta"] - assert results["stacked_imbalances_bid"] == [234.99] - assert results["stacked_imbalances_ask"] == [234.96] + assert results["stacked_imbalances_bid"] == [234.97] + assert results["stacked_imbalances_ask"] == [234.94] # Repeat assertions for the last row results = df.iloc[-1] @@ -586,22 +586,22 @@ def test_stacked_imbalances_multiple_prices(): # Create a sample DataFrame with known imbalances df = pd.DataFrame( { - 'bid_imbalance': [True, True, True, False, False, True, True, False], - 'ask_imbalance': [False, False, True, True, True, False, False, True] + 'bid_imbalance': [True, True, True, False, False, True, True, False, True], + 'ask_imbalance': [False, False, True, True, True, False, False, True, True] }, - index=[234.95, 234.96, 234.97, 234.98, 234.99, 235.00, 235.01, 235.02] + index=[234.95, 234.96, 234.97, 234.98, 234.99, 235.00, 235.01, 235.02, 235.03] ) # Test bid imbalances (should return prices in ascending order) bid_prices = stacked_imbalance(df, "bid", stacked_imbalance_range=2, should_reverse=False) - assert bid_prices == [234.95, 234.96, 234.97, 235.00, 235.01] + assert bid_prices == [234.95, 234.96, 235.00] # Test ask imbalances (should return prices in descending order) ask_prices = stacked_imbalance(df, "ask", stacked_imbalance_range=2, should_reverse=True) - assert ask_prices == [235.02, 234.99, 234.98, 234.97] + assert ask_prices == [235.02, 234.98, 234.97] # Test with higher stacked_imbalance_range bid_prices_higher = stacked_imbalance(df, "bid", stacked_imbalance_range=3, should_reverse=False) - assert bid_prices_higher == [234.95, 234.96, 234.97] + assert bid_prices_higher == [234.95] From 12adbeb7f3cbbfa68deff4e550a449b964e424a3 Mon Sep 17 00:00:00 2001 From: Joe Schr <8218910+TheJoeSchr@users.noreply.github.com> Date: Fri, 3 Jan 2025 18:23:36 +0100 Subject: [PATCH 5/8] fix: don't sort stacked imbalances, return empty list if no found... ... also removes helper functions `stacked_imbalance_bid` & `stacked_imbalance_ask` --- freqtrade/data/converter/orderflow.py | 37 +++++++---------------- tests/data/test_converter_orderflow.py | 41 +++++++++++++------------- 2 files changed, 31 insertions(+), 47 deletions(-) diff --git a/freqtrade/data/converter/orderflow.py b/freqtrade/data/converter/orderflow.py index 6d2df7d3d..2d0ada211 100644 --- a/freqtrade/data/converter/orderflow.py +++ b/freqtrade/data/converter/orderflow.py @@ -164,12 +164,12 @@ def populate_dataframe_with_trades( dataframe.at[index, "imbalances"] = imbalances.to_dict(orient="index") stacked_imbalance_range = config_orderflow["stacked_imbalance_range"] - dataframe.at[index, "stacked_imbalances_bid"] = stacked_imbalance_bid( - imbalances, stacked_imbalance_range=stacked_imbalance_range + dataframe.at[index, "stacked_imbalances_bid"] = stacked_imbalance( + imbalances, label="bid", stacked_imbalance_range=stacked_imbalance_range ) - dataframe.at[index, "stacked_imbalances_ask"] = stacked_imbalance_ask( - imbalances, stacked_imbalance_range=stacked_imbalance_range + dataframe.at[index, "stacked_imbalances_ask"] = stacked_imbalance( + imbalances, label="ask", stacked_imbalance_range=stacked_imbalance_range ) bid = np.where( @@ -256,9 +256,7 @@ def trades_orderflow_to_imbalances(df: pd.DataFrame, imbalance_ratio: int, imbal return dataframe -def stacked_imbalance( - df: pd.DataFrame, label: str, stacked_imbalance_range: int, should_reverse: bool -): +def stacked_imbalance(df: pd.DataFrame, label: str, stacked_imbalance_range: int): """ y * (y.groupby((y != y.shift()).cumsum()).cumcount() + 1) https://stackoverflow.com/questions/27626542/counting-consecutive-positive-values-in-python-pandas-array @@ -268,27 +266,14 @@ def stacked_imbalance( # Group consecutive True values and get their counts groups = (int_series != int_series.shift()).cumsum() counts = int_series.groupby(groups).cumsum() - + # Find indices where count meets or exceeds the range requirement valid_indices = counts[counts >= stacked_imbalance_range].index - + stacked_imbalance_prices = [] if not valid_indices.empty: # Get all prices from valid indices from beginning of the range - valid_prices = [imbalance.index.values[idx-(stacked_imbalance_range-1)] for idx in valid_indices] - # Sort prices according to direction - stacked_imbalance_prices = ( - sorted(valid_prices) - if not should_reverse - else sorted(valid_prices, reverse=True) - ) - - return stacked_imbalance_prices if stacked_imbalance_prices else [np.nan] - - -def stacked_imbalance_ask(df: pd.DataFrame, stacked_imbalance_range: int): - return stacked_imbalance(df, "ask", stacked_imbalance_range, should_reverse=True) - - -def stacked_imbalance_bid(df: pd.DataFrame, stacked_imbalance_range: int): - return stacked_imbalance(df, "bid", stacked_imbalance_range, should_reverse=False) + stacked_imbalance_prices = [ + imbalance.index.values[idx - (stacked_imbalance_range - 1)] for idx in valid_indices + ] + return stacked_imbalance_prices if stacked_imbalance_prices else [] diff --git a/tests/data/test_converter_orderflow.py b/tests/data/test_converter_orderflow.py index 792daea45..b097c93d9 100644 --- a/tests/data/test_converter_orderflow.py +++ b/tests/data/test_converter_orderflow.py @@ -5,9 +5,9 @@ from freqtrade.constants import DEFAULT_TRADES_COLUMNS from freqtrade.data.converter import populate_dataframe_with_trades from freqtrade.data.converter.orderflow import ( ORDERFLOW_ADDED_COLUMNS, + stacked_imbalance, timeframe_to_DateOffset, trades_to_volumeprofile_with_total_delta_bid_ask, - stacked_imbalance, ) from freqtrade.data.converter.trade_converter import trades_list_to_df from freqtrade.data.dataprovider import DataProvider @@ -185,8 +185,8 @@ def test_public_trades_mock_populate_dataframe_with_trades__check_orderflow( assert results["max_delta"] == 17.298 # Assert that stacked imbalances are NaN (not applicable in this test) - assert results["stacked_imbalances_bid"] == [np.nan] - assert results["stacked_imbalances_ask"] == [np.nan] + assert results["stacked_imbalances_bid"] == [] + assert results["stacked_imbalances_ask"] == [] # Repeat assertions for the third from last row results = df.iloc[-2] @@ -201,8 +201,8 @@ def test_public_trades_mock_populate_dataframe_with_trades__check_orderflow( assert pytest.approx(results["delta"]) == -49.302 assert results["min_delta"] == -70.222 assert pytest.approx(results["max_delta"]) == 11.213 - assert results["stacked_imbalances_bid"] == [np.nan] - assert results["stacked_imbalances_ask"] == [np.nan] + assert results["stacked_imbalances_bid"] == [] + assert results["stacked_imbalances_ask"] == [] def test_public_trades_trades_mock_populate_dataframe_with_trades__check_trades( @@ -575,34 +575,33 @@ def test_stacked_imbalances_multiple_prices(): # Test with empty result df_no_stacks = pd.DataFrame( { - 'bid_imbalance': [False, False, True, False], - 'ask_imbalance': [False, True, False, False] + "bid_imbalance": [False, False, True, False], + "ask_imbalance": [False, True, False, False], }, - index=[234.95, 234.96, 234.97, 234.98] + index=[234.95, 234.96, 234.97, 234.98], ) - no_stacks = stacked_imbalance(df_no_stacks, "bid", stacked_imbalance_range=2, should_reverse=False) - assert no_stacks == [np.nan] - + no_stacks = stacked_imbalance(df_no_stacks, "bid", stacked_imbalance_range=2) + assert no_stacks == [] + # Create a sample DataFrame with known imbalances df = pd.DataFrame( { - 'bid_imbalance': [True, True, True, False, False, True, True, False, True], - 'ask_imbalance': [False, False, True, True, True, False, False, True, True] + "bid_imbalance": [True, True, True, False, False, True, True, False, True], + "ask_imbalance": [False, False, True, True, True, False, False, True, True], }, - index=[234.95, 234.96, 234.97, 234.98, 234.99, 235.00, 235.01, 235.02, 235.03] + index=[234.95, 234.96, 234.97, 234.98, 234.99, 235.00, 235.01, 235.02, 235.03], ) # Test bid imbalances (should return prices in ascending order) - bid_prices = stacked_imbalance(df, "bid", stacked_imbalance_range=2, should_reverse=False) + bid_prices = stacked_imbalance(df, "bid", stacked_imbalance_range=2) assert bid_prices == [234.95, 234.96, 235.00] - + # Test ask imbalances (should return prices in descending order) - ask_prices = stacked_imbalance(df, "ask", stacked_imbalance_range=2, should_reverse=True) - assert ask_prices == [235.02, 234.98, 234.97] - + ask_prices = stacked_imbalance(df, "ask", stacked_imbalance_range=2) + assert ask_prices == [234.97, 234.98, 235.02] + # Test with higher stacked_imbalance_range - bid_prices_higher = stacked_imbalance(df, "bid", stacked_imbalance_range=3, should_reverse=False) + bid_prices_higher = stacked_imbalance(df, "bid", stacked_imbalance_range=3) assert bid_prices_higher == [234.95] - def test_timeframe_to_DateOffset(): From fd7272ba6359ac9fb54078d5f4dc690e9f7f11b6 Mon Sep 17 00:00:00 2001 From: Joe Schr <8218910+TheJoeSchr@users.noreply.github.com> Date: Fri, 3 Jan 2025 18:47:03 +0100 Subject: [PATCH 6/8] chore: fix unterminated string literal in test runner --- tests/data/test_converter_orderflow.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/data/test_converter_orderflow.py b/tests/data/test_converter_orderflow.py index b097c93d9..31d9c0457 100644 --- a/tests/data/test_converter_orderflow.py +++ b/tests/data/test_converter_orderflow.py @@ -535,8 +535,7 @@ def test_analyze_with_orderflow( assert col in df1.columns, f"Column {col} not found in df.columns" if col not in ("stacked_imbalances_bid", "stacked_imbalances_ask"): - assert df1[col].count() == 5, f"Column {col} has { - df1[col].count()} non-NaN values" + assert df1[col].count() == 5, f"Column {col} has {df1[col].count()} non-NaN values" assert len(strategy._cached_grouped_trades_per_pair[pair]) == 5 @@ -554,8 +553,7 @@ def test_analyze_with_orderflow( assert "open" in df2.columns assert spy.call_count == 0 for col in ORDERFLOW_ADDED_COLUMNS: - assert col in df2.columns, f"Round2: Column { - col} not found in df.columns" + assert col in df2.columns, f"Round2: Column {col} not found in df.columns" if col not in ("stacked_imbalances_bid", "stacked_imbalances_ask"): assert ( From 7c148a01238c0866aa465f4dddd13a6bd6105c0f Mon Sep 17 00:00:00 2001 From: Joe Schr <8218910+TheJoeSchr@users.noreply.github.com> Date: Sat, 4 Jan 2025 23:48:55 +0100 Subject: [PATCH 7/8] fix: remove superfluous territory return statement --- freqtrade/data/converter/orderflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/data/converter/orderflow.py b/freqtrade/data/converter/orderflow.py index 2d0ada211..4b4923cfb 100644 --- a/freqtrade/data/converter/orderflow.py +++ b/freqtrade/data/converter/orderflow.py @@ -276,4 +276,4 @@ def stacked_imbalance(df: pd.DataFrame, label: str, stacked_imbalance_range: int stacked_imbalance_prices = [ imbalance.index.values[idx - (stacked_imbalance_range - 1)] for idx in valid_indices ] - return stacked_imbalance_prices if stacked_imbalance_prices else [] + return stacked_imbalance_prices From 5f5e51326f0ca1017ead4e85df3e63147ba9b28f Mon Sep 17 00:00:00 2001 From: Joe Schr <8218910+TheJoeSchr@users.noreply.github.com> Date: Sat, 4 Jan 2025 23:51:38 +0100 Subject: [PATCH 8/8] chore: modify orderflow docs re. stacked imbalance changes --- docs/advanced-orderflow.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/advanced-orderflow.md b/docs/advanced-orderflow.md index 9769b8e92..4561b35a3 100644 --- a/docs/advanced-orderflow.md +++ b/docs/advanced-orderflow.md @@ -70,8 +70,8 @@ dataframe["delta"] # Difference between ask and bid volume. dataframe["min_delta"] # Minimum delta within the candle dataframe["max_delta"] # Maximum delta within the candle dataframe["total_trades"] # Total number of trades -dataframe["stacked_imbalances_bid"] # Price level of stacked bid imbalance -dataframe["stacked_imbalances_ask"] # Price level of stacked ask imbalance +dataframe["stacked_imbalances_bid"] # List of price levels of stacked bid imbalance range beginnings +dataframe["stacked_imbalances_ask"] # List of price levels of stacked ask imbalance range beginnings ``` You can access these columns in your strategy code for further analysis. Here's an example: