From a8c887a5b03b2971c8415580deeb60149df5deab Mon Sep 17 00:00:00 2001 From: Briarion Date: Mon, 16 Mar 2026 08:46:57 +0300 Subject: [PATCH 1/2] fix: avoid DataFrame fragmentation in get_predictions_to_append Replace column-by-column DataFrame assignment with dict-based construction. The previous approach triggered pandas PerformanceWarning about DataFrame fragmentation when many prediction columns and their corresponding mean/std columns were added one at a time. Also add defensive key checks (label in self.data["labels_mean"]) to prevent KeyError when custom models produce prediction columns that don't have corresponding entries in labels_mean/labels_std. Co-Authored-By: Claude Opus 4.6 (1M context) --- freqtrade/freqai/data_kitchen.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index c39343ab2..327acce27 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -428,18 +428,24 @@ class FreqaiDataKitchen: Get backtest prediction from current backtest period """ - append_df = DataFrame() + # Build dict first and construct DataFrame once to avoid + # column-by-column assignment which causes DataFrame fragmentation + # and PerformanceWarning on large prediction sets. + append_dict: dict[str, Any] = {} + for label in predictions.columns: - append_df[label] = predictions[label] - if append_df[label].dtype == object: + append_dict[label] = predictions[label] + if predictions[label].dtype == object: continue - if "labels_mean" in self.data: - append_df[f"{label}_mean"] = self.data["labels_mean"][label] - if "labels_std" in self.data: - append_df[f"{label}_std"] = self.data["labels_std"][label] + if "labels_mean" in self.data and label in self.data["labels_mean"]: + append_dict[f"{label}_mean"] = self.data["labels_mean"][label] + if "labels_std" in self.data and label in self.data["labels_std"]: + append_dict[f"{label}_std"] = self.data["labels_std"][label] for extra_col in self.data["extra_returns_per_train"]: - append_df[f"{extra_col}"] = self.data["extra_returns_per_train"][extra_col] + append_dict[f"{extra_col}"] = self.data["extra_returns_per_train"][extra_col] + + append_df = DataFrame(append_dict) append_df["do_predict"] = do_predict if self.freqai_config["feature_parameters"].get("DI_threshold", 0) > 0: From 9560ba2144e3e29127f8d6c1155f34668ba95d50 Mon Sep 17 00:00:00 2001 From: Briarion Date: Wed, 18 Mar 2026 08:05:22 +0300 Subject: [PATCH 2/2] fix: move do_predict and DI_values into dict before DataFrame construction Address review feedback: do_predict and DI_values were still assigned after DataFrame creation, partially defeating the fragmentation fix. Now all columns are collected in the dict before the single DataFrame() call, ensuring zero post-construction column assignments. Co-Authored-By: Claude Opus 4.6 (1M context) --- freqtrade/freqai/data_kitchen.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 327acce27..30826b174 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -445,11 +445,11 @@ class FreqaiDataKitchen: for extra_col in self.data["extra_returns_per_train"]: append_dict[f"{extra_col}"] = self.data["extra_returns_per_train"][extra_col] - append_df = DataFrame(append_dict) - - append_df["do_predict"] = do_predict + append_dict["do_predict"] = do_predict if self.freqai_config["feature_parameters"].get("DI_threshold", 0) > 0: - append_df["DI_values"] = self.DI_values + append_dict["DI_values"] = self.DI_values + + append_df = DataFrame(append_dict) user_cols = [col for col in dataframe_backtest.columns if col.startswith("%%")] cols = ["date"]