diff --git a/freqtrade/data/history/datahandlers/featherdatahandler.py b/freqtrade/data/history/datahandlers/featherdatahandler.py index ef293d6b2..23d0b2d76 100644 --- a/freqtrade/data/history/datahandlers/featherdatahandler.py +++ b/freqtrade/data/history/datahandlers/featherdatahandler.py @@ -1,6 +1,6 @@ import logging -from pandas import DataFrame, read_feather, to_datetime +from pandas import DataFrame, read_feather from pyarrow import dataset from freqtrade.configuration import TimeRange @@ -71,7 +71,7 @@ class FeatherDataHandler(IDataHandler): "volume": "float", } ) - pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True) + pairdata["date"] = pairdata["date"].dt.as_unit("ms") return pairdata except Exception as e: logger.exception( diff --git a/freqtrade/data/history/datahandlers/jsondatahandler.py b/freqtrade/data/history/datahandlers/jsondatahandler.py index 332b687b4..a2bd3f6db 100644 --- a/freqtrade/data/history/datahandlers/jsondatahandler.py +++ b/freqtrade/data/history/datahandlers/jsondatahandler.py @@ -80,7 +80,7 @@ class JsonDataHandler(IDataHandler): "volume": "float", } ) - pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True) + pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True).dt.as_unit("ms") return pairdata def ohlcv_append( @@ -104,6 +104,9 @@ class JsonDataHandler(IDataHandler): :param trading_mode: Trading mode to use (used to determine the filename) """ filename = self._pair_trades_filename(self._datadir, pair, trading_mode) + # Convert StringDtype columns to object to avoid NaN serialization issues + for col in data.select_dtypes(include="string").columns: + data[col] = data[col].astype(object).where(data[col].notna(), other=None) trades = data.values.tolist() misc.file_dump_json(filename, trades, is_zip=self._use_zip) diff --git a/freqtrade/data/history/datahandlers/parquetdatahandler.py b/freqtrade/data/history/datahandlers/parquetdatahandler.py index 1813f9991..7a5cb39f1 100644 --- a/freqtrade/data/history/datahandlers/parquetdatahandler.py +++ b/freqtrade/data/history/datahandlers/parquetdatahandler.py @@ -1,6 +1,6 @@ import logging -from pandas import DataFrame, read_parquet, to_datetime +from pandas import DataFrame, read_parquet from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS @@ -68,7 +68,7 @@ class ParquetDataHandler(IDataHandler): "volume": "float", } ) - pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True) + pairdata["date"] = pairdata["date"].dt.as_unit("ms") return pairdata except Exception as e: logger.exception( diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 9f04e0ca4..16a3bc3c7 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -24,8 +24,6 @@ from freqtrade.strategy import merge_informative_pair from freqtrade.strategy.interface import IStrategy -pd.set_option("future.no_silent_downcasting", True) - SECONDS_IN_DAY = 86400 SECONDS_IN_HOUR = 3600 @@ -239,16 +237,14 @@ class FreqaiDataKitchen: filtered_df = filtered_df.replace([np.inf, -np.inf], np.nan) drop_index = pd.isnull(filtered_df).any(axis=1) # get the rows that have NaNs, - drop_index = drop_index.replace(True, 1).replace(False, 0).infer_objects(copy=False) + drop_index = drop_index.replace(True, 1).replace(False, 0).infer_objects() if training_filter: # we don't care about total row number (total no. datapoints) in training, we only care # about removing any row with NaNs # if labels has multiple columns (user wants to train multiple modelEs), we detect here labels = unfiltered_df.filter(label_list or [], axis=1) drop_index_labels = pd.isnull(labels).any(axis=1) - drop_index_labels = ( - drop_index_labels.replace(True, 1).replace(False, 0).infer_objects(copy=False) - ) + drop_index_labels = drop_index_labels.replace(True, 1).replace(False, 0).infer_objects() dates = unfiltered_df["date"] filtered_df = filtered_df[ (drop_index == 0) & (drop_index_labels == 0) diff --git a/pyproject.toml b/pyproject.toml index fd23460bb..c6024a3b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dependencies = [ "urllib3", "jsonschema", "numpy>2.0,<3.0", - "pandas>=2.2.0,<3.0", + "pandas>=2.2.0,<4.0", "TA-Lib<0.7", "ft-pandas-ta", "technical", diff --git a/requirements.txt b/requirements.txt index 2af13fdb7..80c924deb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ numpy==2.4.4 -pandas==2.3.3 +pandas==3.0.2 bottleneck==1.6.0 numexpr==2.14.1 # Indicator libraries diff --git a/tests/conftest.py b/tests/conftest.py index 46601ddfb..dc0860466 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -176,20 +176,20 @@ def generate_test_data( base = np.random.normal(base, 2, size=size) if timeframe == "1y": - date = pd.date_range(start, periods=size, freq="1YS", tz="UTC") + date = pd.date_range(start, periods=size, freq="1YS", tz="UTC", unit="ms") elif timeframe == "1M": - date = pd.date_range(start, periods=size, freq="1MS", tz="UTC") + date = pd.date_range(start, periods=size, freq="1MS", tz="UTC", unit="ms") elif timeframe == "3M": - date = pd.date_range(start, periods=size, freq="3MS", tz="UTC") + date = pd.date_range(start, periods=size, freq="3MS", tz="UTC", unit="ms") elif timeframe == "1w" or timeframe == "7d": - date = pd.date_range(start, periods=size, freq="1W-MON", tz="UTC") + date = pd.date_range(start, periods=size, freq="1W-MON", tz="UTC", unit="ms") else: tf_mins = timeframe_to_minutes(timeframe) if tf_mins >= 1: - date = pd.date_range(start, periods=size, freq=f"{tf_mins}min", tz="UTC") + date = pd.date_range(start, periods=size, freq=f"{tf_mins}min", tz="UTC", unit="ms") else: tf_secs = timeframe_to_seconds(timeframe) - date = pd.date_range(start, periods=size, freq=f"{tf_secs}s", tz="UTC") + date = pd.date_range(start, periods=size, freq=f"{tf_secs}s", tz="UTC", unit="ms") df = pd.DataFrame( { "date": date, diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 835f5a861..4946741aa 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -207,10 +207,13 @@ def test_ohlcv_to_dataframe_multi(timeframe): data1 = data.copy() if timeframe in ("1M", "3M", "1y"): - data1.loc[:, "date"] = data1.loc[:, "date"] + pd.to_timedelta("1w") + data1.loc[:, "date"] = data1.loc[:, "date"] + pd.to_timedelta("1W") else: # Shift by half a timeframe - data1.loc[:, "date"] = data1.loc[:, "date"] + (pd.to_timedelta(timeframe) / 2) + timeframe_f = ( + timeframe.upper() if timeframe.endswith("d") or timeframe.endswith("w") else timeframe + ) + data1.loc[:, "date"] = data1.loc[:, "date"] + (pd.to_timedelta(timeframe_f) / 2) df2 = ohlcv_to_dataframe(data1, timeframe, "UNITTEST/USDT") assert len(df2) == len(data) - 1 diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 70b591d54..95250e7a7 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -860,6 +860,9 @@ def test_backtest_one(default_conf, mocker, testdatadir) -> None: "funding_fees": [0.0, 0.0], } ) + # TODO: pandas3 - create correctly above ?!? + expected["open_date"] = expected["open_date"].astype("datetime64[ms, UTC]") + expected["close_date"] = expected["close_date"].astype("datetime64[ms, UTC]") pd.testing.assert_frame_equal(results, expected) assert "orders" in results.columns data_pair = processed[pair] diff --git a/tests/optimize/test_backtesting_adjust_position.py b/tests/optimize/test_backtesting_adjust_position.py index 33fd87d83..f698173c9 100644 --- a/tests/optimize/test_backtesting_adjust_position.py +++ b/tests/optimize/test_backtesting_adjust_position.py @@ -83,6 +83,9 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) -> "funding_fees": [0.0, 0.0], } ) + # TODO: pandas3 - create correctly above ?!? + expected["open_date"] = expected["open_date"].astype("datetime64[ms, UTC]") + expected["close_date"] = expected["close_date"].astype("datetime64[ms, UTC]") results_no = results.drop(columns=["orders"]) pd.testing.assert_frame_equal(results_no, expected, check_exact=True)