Merge pull request #13048 from freqtrade/dependabot/pip/develop/pandas-3.0.2

chore(deps): bump pandas from 2.3.3 to 3.0.2

freqtrade/data/history/datahandlers/featherdatahandler.py

@@ -1,6 +1,6 @@
 import logging
 
-from pandas import DataFrame, read_feather, to_datetime
+from pandas import DataFrame, read_feather
 from pyarrow import dataset
 
 from freqtrade.configuration import TimeRange
@@ -71,7 +71,7 @@ class FeatherDataHandler(IDataHandler):
                     "volume": "float",
                 }
             )
-            pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True)
+            pairdata["date"] = pairdata["date"].dt.as_unit("ms")
             return pairdata
         except Exception as e:
             logger.exception(

freqtrade/data/history/datahandlers/jsondatahandler.py

@@ -80,7 +80,7 @@ class JsonDataHandler(IDataHandler):
                 "volume": "float",
             }
         )
-        pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True)
+        pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True).dt.as_unit("ms")
         return pairdata
 
     def ohlcv_append(
@@ -104,6 +104,9 @@ class JsonDataHandler(IDataHandler):
         :param trading_mode: Trading mode to use (used to determine the filename)
         """
         filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
+        # Convert StringDtype columns to object to avoid NaN serialization issues
+        for col in data.select_dtypes(include="string").columns:
+            data[col] = data[col].astype(object).where(data[col].notna(), other=None)
         trades = data.values.tolist()
         misc.file_dump_json(filename, trades, is_zip=self._use_zip)
 

freqtrade/data/history/datahandlers/parquetdatahandler.py

@@ -1,6 +1,6 @@
 import logging
 
-from pandas import DataFrame, read_parquet, to_datetime
+from pandas import DataFrame, read_parquet
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
@@ -68,7 +68,7 @@ class ParquetDataHandler(IDataHandler):
                     "volume": "float",
                 }
             )
-            pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True)
+            pairdata["date"] = pairdata["date"].dt.as_unit("ms")
             return pairdata
         except Exception as e:
             logger.exception(

freqtrade/freqai/data_kitchen.py

@@ -24,8 +24,6 @@ from freqtrade.strategy import merge_informative_pair
 from freqtrade.strategy.interface import IStrategy
 
 
-pd.set_option("future.no_silent_downcasting", True)
-
 SECONDS_IN_DAY = 86400
 SECONDS_IN_HOUR = 3600
 
@@ -239,16 +237,14 @@ class FreqaiDataKitchen:
         filtered_df = filtered_df.replace([np.inf, -np.inf], np.nan)
 
         drop_index = pd.isnull(filtered_df).any(axis=1)  # get the rows that have NaNs,
-        drop_index = drop_index.replace(True, 1).replace(False, 0).infer_objects(copy=False)
+        drop_index = drop_index.replace(True, 1).replace(False, 0).infer_objects()
         if training_filter:
             # we don't care about total row number (total no. datapoints) in training, we only care
             # about removing any row with NaNs
             # if labels has multiple columns (user wants to train multiple modelEs), we detect here
             labels = unfiltered_df.filter(label_list or [], axis=1)
             drop_index_labels = pd.isnull(labels).any(axis=1)
-            drop_index_labels = (
+            drop_index_labels = drop_index_labels.replace(True, 1).replace(False, 0).infer_objects()
-                drop_index_labels.replace(True, 1).replace(False, 0).infer_objects(copy=False)
-            )
             dates = unfiltered_df["date"]
             filtered_df = filtered_df[
                 (drop_index == 0) & (drop_index_labels == 0)

pyproject.toml

@@ -40,7 +40,7 @@ dependencies = [
   "urllib3",
   "jsonschema",
   "numpy>2.0,<3.0",
-  "pandas>=2.2.0,<3.0",
+  "pandas>=2.2.0,<4.0",
   "TA-Lib<0.7",
   "ft-pandas-ta",
   "technical",

requirements.txt

@@ -1,5 +1,5 @@
 numpy==2.4.4
-pandas==2.3.3
+pandas==3.0.2
 bottleneck==1.6.0
 numexpr==2.14.1
 # Indicator libraries

tests/conftest.py

@@ -176,20 +176,20 @@ def generate_test_data(
 
     base = np.random.normal(base, 2, size=size)
     if timeframe == "1y":
-        date = pd.date_range(start, periods=size, freq="1YS", tz="UTC")
+        date = pd.date_range(start, periods=size, freq="1YS", tz="UTC", unit="ms")
     elif timeframe == "1M":
-        date = pd.date_range(start, periods=size, freq="1MS", tz="UTC")
+        date = pd.date_range(start, periods=size, freq="1MS", tz="UTC", unit="ms")
     elif timeframe == "3M":
-        date = pd.date_range(start, periods=size, freq="3MS", tz="UTC")
+        date = pd.date_range(start, periods=size, freq="3MS", tz="UTC", unit="ms")
     elif timeframe == "1w" or timeframe == "7d":
-        date = pd.date_range(start, periods=size, freq="1W-MON", tz="UTC")
+        date = pd.date_range(start, periods=size, freq="1W-MON", tz="UTC", unit="ms")
     else:
         tf_mins = timeframe_to_minutes(timeframe)
         if tf_mins >= 1:
-            date = pd.date_range(start, periods=size, freq=f"{tf_mins}min", tz="UTC")
+            date = pd.date_range(start, periods=size, freq=f"{tf_mins}min", tz="UTC", unit="ms")
         else:
             tf_secs = timeframe_to_seconds(timeframe)
-            date = pd.date_range(start, periods=size, freq=f"{tf_secs}s", tz="UTC")
+            date = pd.date_range(start, periods=size, freq=f"{tf_secs}s", tz="UTC", unit="ms")
     df = pd.DataFrame(
         {
             "date": date,

tests/data/test_converter.py

@@ -207,10 +207,13 @@ def test_ohlcv_to_dataframe_multi(timeframe):
 
     data1 = data.copy()
     if timeframe in ("1M", "3M", "1y"):
-        data1.loc[:, "date"] = data1.loc[:, "date"] + pd.to_timedelta("1w")
+        data1.loc[:, "date"] = data1.loc[:, "date"] + pd.to_timedelta("1W")
     else:
         # Shift by half a timeframe
-        data1.loc[:, "date"] = data1.loc[:, "date"] + (pd.to_timedelta(timeframe) / 2)
+        timeframe_f = (
+            timeframe.upper() if timeframe.endswith("d") or timeframe.endswith("w") else timeframe
+        )
+        data1.loc[:, "date"] = data1.loc[:, "date"] + (pd.to_timedelta(timeframe_f) / 2)
     df2 = ohlcv_to_dataframe(data1, timeframe, "UNITTEST/USDT")
 
     assert len(df2) == len(data) - 1

tests/optimize/test_backtesting.py

@@ -860,6 +860,9 @@ def test_backtest_one(default_conf, mocker, testdatadir) -> None:
             "funding_fees": [0.0, 0.0],
         }
     )
+    # TODO: pandas3 - create correctly above ?!?
+    expected["open_date"] = expected["open_date"].astype("datetime64[ms, UTC]")
+    expected["close_date"] = expected["close_date"].astype("datetime64[ms, UTC]")
     pd.testing.assert_frame_equal(results, expected)
     assert "orders" in results.columns
     data_pair = processed[pair]

tests/optimize/test_backtesting_adjust_position.py

@@ -83,6 +83,9 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) ->
             "funding_fees": [0.0, 0.0],
         }
     )
+    # TODO: pandas3 - create correctly above ?!?
+    expected["open_date"] = expected["open_date"].astype("datetime64[ms, UTC]")
+    expected["close_date"] = expected["close_date"].astype("datetime64[ms, UTC]")
     results_no = results.drop(columns=["orders"])
     pd.testing.assert_frame_equal(results_no, expected, check_exact=True)