Merge pull request #9191 from freqtrade/fix/double-date-column

Bug: FreqAI fit_live_predictions()

freqtrade/freqai/data_drawer.py

@@ -282,15 +282,12 @@ class FreqaiDataDrawer:
         # historically made during downtime. The newest pred will get appeneded later in
         # append_model_predictions)
         new_pred.iloc[:, :] = np.nan
-        new_pred["date"] = dataframe["date"]
+        new_pred["date_pred"] = dataframe["date"]
-
         hist_preds = self.historic_predictions[pair].copy()
-        # rename date_pred column to date so that we can merge on date
-        hist_preds = hist_preds.rename(columns={"date_pred": "date"})
 
         # find the closest common date between new_pred and historic predictions
         # and cut off the new_pred dataframe at that date
-        common_dates = pd.merge(new_pred, hist_preds, on="date", how="inner")
+        common_dates = pd.merge(new_pred, hist_preds, on="date_pred", how="inner")
         if len(common_dates.index) > 0:
             new_pred = new_pred.iloc[len(common_dates):]
         else:
@@ -299,18 +296,12 @@ class FreqaiDataDrawer:
                            f"for more than {len(dataframe.index)} candles.")
 
         df_concat = pd.concat([hist_preds, new_pred], ignore_index=True, keys=hist_preds.keys())
-
         # remove last row because we will append that later in append_model_predictions()
         df_concat = df_concat.iloc[:-1]
         # any missing values will get zeroed out so users can see the exact
         # downtime in FreqUI
         df_concat = df_concat.fillna(0)
-
-        # rename date column back to date_pred
-        df_concat = df_concat.rename(columns={"date": "date_pred"})
-
         self.historic_predictions[pair] = df_concat
-
         self.model_return_values[pair] = df_concat.tail(len(dataframe.index)).reset_index(drop=True)
 
     def append_model_predictions(self, pair: str, predictions: DataFrame,

freqtrade/freqai/data_kitchen.py

@@ -244,6 +244,14 @@ class FreqaiDataKitchen:
                 f"{self.pair}: dropped {len(unfiltered_df) - len(filtered_df)} training points"
                 f" due to NaNs in populated dataset {len(unfiltered_df)}."
             )
+            if len(unfiltered_df) == 0 and not self.live:
+                raise OperationalException(
+                    f"{self.pair}: all training data dropped due to NaNs. "
+                    "You likely did not download enough training data prior "
+                    "to your backtest timerange. Hint:\n"
+                    "https://www.freqtrade.io/en/stable/freqai-running/"
+                    "#downloading-data-to-cover-the-full-backtest-period"
+                )
             if (1 - len(filtered_df) / len(unfiltered_df)) > 0.1 and self.live:
                 worst_indicator = str(unfiltered_df.count().idxmin())
                 logger.warning(

tests/freqai/test_freqai_datadrawer.py

@@ -181,7 +181,6 @@ def test_set_initial_return_values(mocker, freqai_conf):
 
     assert (hist_pred_df['date_pred'].iloc[-1] ==
             pd.Timestamp(end_x_plus_5) - pd.Timedelta(days=1))
-    assert 'date' not in hist_pred_df.columns
     assert 'date_pred' in hist_pred_df.columns
     assert hist_pred_df.shape[0] == 7  # Total rows: 5 from historic and 2 new zeros
 
@@ -236,7 +235,6 @@ def test_set_initial_return_values_warning(mocker, freqai_conf):
     model_return_df = freqai.dd.model_return_values[pair]
 
     assert hist_pred_df['date_pred'].iloc[-1] == pd.Timestamp(end_x_plus_5) - pd.Timedelta(days=1)
-    assert 'date' not in hist_pred_df.columns
     assert 'date_pred' in hist_pred_df.columns
     assert hist_pred_df.shape[0] == 9  # Total rows: 5 from historic and 4 new zeros