From 362f0895f464ae8a1a813dba47734d149fe87976 Mon Sep 17 00:00:00 2001 From: James Lin Date: Fri, 25 Apr 2025 16:13:33 -0400 Subject: [PATCH 1/6] skip trade-related columns --- freqtrade/freqai/data_kitchen.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 6cb52eb29..f7ca2c7a9 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -709,6 +709,11 @@ class FreqaiDataKitchen: skip_columns = [ (f"{s}_{suffix}") for s in ["date", "open", "high", "low", "close", "volume"] ] + for s in ['trades', 'orderflow', 'imbalances', 'stacked_imbalances_bid', + 'stacked_imbalances_ask', 'max_delta', 'min_delta', 'bid', 'ask', + 'delta', 'total_trades']: + if s in dataframe.columns and f"{s}_{suffix}" in df_to_merge.columns: + skip_columns.append(f"{s}_{suffix}") dataframe = dataframe.drop(columns=skip_columns) return dataframe From 12728c0fccabcf2a44effc44a9cd76ff07889218 Mon Sep 17 00:00:00 2001 From: James Lin Date: Sat, 26 Apr 2025 10:18:39 -0400 Subject: [PATCH 2/6] fix --- freqtrade/freqai/data_kitchen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index f7ca2c7a9..322ac0baf 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -712,7 +712,7 @@ class FreqaiDataKitchen: for s in ['trades', 'orderflow', 'imbalances', 'stacked_imbalances_bid', 'stacked_imbalances_ask', 'max_delta', 'min_delta', 'bid', 'ask', 'delta', 'total_trades']: - if s in dataframe.columns and f"{s}_{suffix}" in df_to_merge.columns: + if s in dataframe.columns and f"{s}_{suffix}" in dataframe.columns: skip_columns.append(f"{s}_{suffix}") dataframe = dataframe.drop(columns=skip_columns) return dataframe From 5ee7f8037de480b40b162dbe178b4af4cddf6860 Mon Sep 17 00:00:00 2001 From: James Lin Date: Sat, 26 Apr 2025 10:20:09 -0400 Subject: [PATCH 3/6] fix --- freqtrade/freqai/data_kitchen.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 322ac0baf..eac1426fb 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -709,9 +709,11 @@ class FreqaiDataKitchen: skip_columns = [ (f"{s}_{suffix}") for s in ["date", "open", "high", "low", "close", "volume"] ] - for s in ['trades', 'orderflow', 'imbalances', 'stacked_imbalances_bid', - 'stacked_imbalances_ask', 'max_delta', 'min_delta', 'bid', 'ask', - 'delta', 'total_trades']: + for s in [ + 'trades', 'orderflow', 'imbalances', 'stacked_imbalances_bid', + 'stacked_imbalances_ask', 'max_delta', 'min_delta', 'bid', 'ask', + 'delta', 'total_trades' + ]: if s in dataframe.columns and f"{s}_{suffix}" in dataframe.columns: skip_columns.append(f"{s}_{suffix}") dataframe = dataframe.drop(columns=skip_columns) From 0a4f82bc22794a33cf0047741bd3a1ed10c9d578 Mon Sep 17 00:00:00 2001 From: James Lin Date: Sat, 26 Apr 2025 10:24:53 -0400 Subject: [PATCH 4/6] update columns from ORDERFLOW_ADDED_COLUMNS --- freqtrade/freqai/data_kitchen.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index eac1426fb..4b49c3495 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -17,7 +17,7 @@ from sklearn.model_selection import train_test_split from freqtrade.configuration import TimeRange from freqtrade.constants import DOCS_LINK, Config -from freqtrade.data.converter import reduce_dataframe_footprint +from freqtrade.data.converter import ORDERFLOW_ADDED_COLUMNS, reduce_dataframe_footprint from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds from freqtrade.strategy import merge_informative_pair @@ -709,11 +709,7 @@ class FreqaiDataKitchen: skip_columns = [ (f"{s}_{suffix}") for s in ["date", "open", "high", "low", "close", "volume"] ] - for s in [ - 'trades', 'orderflow', 'imbalances', 'stacked_imbalances_bid', - 'stacked_imbalances_ask', 'max_delta', 'min_delta', 'bid', 'ask', - 'delta', 'total_trades' - ]: + for s in ORDERFLOW_ADDED_COLUMNS: if s in dataframe.columns and f"{s}_{suffix}" in dataframe.columns: skip_columns.append(f"{s}_{suffix}") dataframe = dataframe.drop(columns=skip_columns) From 1d99b7d0f96ea4dfded2d10b308d7ee8eeb14303 Mon Sep 17 00:00:00 2001 From: James Lin Date: Tue, 29 Apr 2025 09:18:07 -0400 Subject: [PATCH 5/6] move ORDERFLOW_ADDED_COLUMNS to constant --- freqtrade/constants.py | 13 +++++++++++++ freqtrade/data/converter/orderflow.py | 16 +--------------- freqtrade/freqai/data_kitchen.py | 4 ++-- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index a5c07ae1f..807b0888b 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -71,6 +71,19 @@ DEFAULT_DATAFRAME_COLUMNS = ["date", "open", "high", "low", "close", "volume"] # it has wide consequences for stored trades files DEFAULT_TRADES_COLUMNS = ["timestamp", "id", "type", "side", "price", "amount", "cost"] DEFAULT_ORDERFLOW_COLUMNS = ["level", "bid", "ask", "delta"] +ORDERFLOW_ADDED_COLUMNS = [ + "trades", + "orderflow", + "imbalances", + "stacked_imbalances_bid", + "stacked_imbalances_ask", + "max_delta", + "min_delta", + "bid", + "ask", + "delta", + "total_trades", +] TRADES_DTYPES = { "timestamp": "int64", "id": "str", diff --git a/freqtrade/data/converter/orderflow.py b/freqtrade/data/converter/orderflow.py index 4b4923cfb..2b4a1fc61 100644 --- a/freqtrade/data/converter/orderflow.py +++ b/freqtrade/data/converter/orderflow.py @@ -9,26 +9,12 @@ from datetime import datetime import numpy as np import pandas as pd -from freqtrade.constants import DEFAULT_ORDERFLOW_COLUMNS, Config +from freqtrade.constants import DEFAULT_ORDERFLOW_COLUMNS, ORDERFLOW_ADDED_COLUMNS, Config from freqtrade.exceptions import DependencyException logger = logging.getLogger(__name__) -ORDERFLOW_ADDED_COLUMNS = [ - "trades", - "orderflow", - "imbalances", - "stacked_imbalances_bid", - "stacked_imbalances_ask", - "max_delta", - "min_delta", - "bid", - "ask", - "delta", - "total_trades", -] - def _init_dataframe_with_trades_columns(dataframe: pd.DataFrame): """ diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 4b49c3495..b518dbc6c 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -16,8 +16,8 @@ from pandas import DataFrame from sklearn.model_selection import train_test_split from freqtrade.configuration import TimeRange -from freqtrade.constants import DOCS_LINK, Config -from freqtrade.data.converter import ORDERFLOW_ADDED_COLUMNS, reduce_dataframe_footprint +from freqtrade.constants import DOCS_LINK, ORDERFLOW_ADDED_COLUMNS, Config +from freqtrade.data.converter import reduce_dataframe_footprint from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds from freqtrade.strategy import merge_informative_pair From a77618cc6b427a081b210a20fc7a633621a116f9 Mon Sep 17 00:00:00 2001 From: James Lin Date: Tue, 29 Apr 2025 12:09:13 -0400 Subject: [PATCH 6/6] fix --- freqtrade/freqai/data_kitchen.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index b518dbc6c..c03dbb276 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -709,9 +709,11 @@ class FreqaiDataKitchen: skip_columns = [ (f"{s}_{suffix}") for s in ["date", "open", "high", "low", "close", "volume"] ] + for s in ORDERFLOW_ADDED_COLUMNS: if s in dataframe.columns and f"{s}_{suffix}" in dataframe.columns: skip_columns.append(f"{s}_{suffix}") + dataframe = dataframe.drop(columns=skip_columns) return dataframe