diff --git a/freqtrade/data/history/datahandlers/featherdatahandler.py b/freqtrade/data/history/datahandlers/featherdatahandler.py index ef293d6b2..23d0b2d76 100644 --- a/freqtrade/data/history/datahandlers/featherdatahandler.py +++ b/freqtrade/data/history/datahandlers/featherdatahandler.py @@ -1,6 +1,6 @@ import logging -from pandas import DataFrame, read_feather, to_datetime +from pandas import DataFrame, read_feather from pyarrow import dataset from freqtrade.configuration import TimeRange @@ -71,7 +71,7 @@ class FeatherDataHandler(IDataHandler): "volume": "float", } ) - pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True) + pairdata["date"] = pairdata["date"].dt.as_unit("ms") return pairdata except Exception as e: logger.exception( diff --git a/freqtrade/data/history/datahandlers/jsondatahandler.py b/freqtrade/data/history/datahandlers/jsondatahandler.py index 332b687b4..a2bd3f6db 100644 --- a/freqtrade/data/history/datahandlers/jsondatahandler.py +++ b/freqtrade/data/history/datahandlers/jsondatahandler.py @@ -80,7 +80,7 @@ class JsonDataHandler(IDataHandler): "volume": "float", } ) - pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True) + pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True).dt.as_unit("ms") return pairdata def ohlcv_append( @@ -104,6 +104,9 @@ class JsonDataHandler(IDataHandler): :param trading_mode: Trading mode to use (used to determine the filename) """ filename = self._pair_trades_filename(self._datadir, pair, trading_mode) + # Convert StringDtype columns to object to avoid NaN serialization issues + for col in data.select_dtypes(include="string").columns: + data[col] = data[col].astype(object).where(data[col].notna(), other=None) trades = data.values.tolist() misc.file_dump_json(filename, trades, is_zip=self._use_zip) diff --git a/freqtrade/data/history/datahandlers/parquetdatahandler.py b/freqtrade/data/history/datahandlers/parquetdatahandler.py index 1813f9991..7a5cb39f1 100644 --- a/freqtrade/data/history/datahandlers/parquetdatahandler.py +++ b/freqtrade/data/history/datahandlers/parquetdatahandler.py @@ -1,6 +1,6 @@ import logging -from pandas import DataFrame, read_parquet, to_datetime +from pandas import DataFrame, read_parquet from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS @@ -68,7 +68,7 @@ class ParquetDataHandler(IDataHandler): "volume": "float", } ) - pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True) + pairdata["date"] = pairdata["date"].dt.as_unit("ms") return pairdata except Exception as e: logger.exception( diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 9f04e0ca4..16a3bc3c7 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -24,8 +24,6 @@ from freqtrade.strategy import merge_informative_pair from freqtrade.strategy.interface import IStrategy -pd.set_option("future.no_silent_downcasting", True) - SECONDS_IN_DAY = 86400 SECONDS_IN_HOUR = 3600 @@ -239,16 +237,14 @@ class FreqaiDataKitchen: filtered_df = filtered_df.replace([np.inf, -np.inf], np.nan) drop_index = pd.isnull(filtered_df).any(axis=1) # get the rows that have NaNs, - drop_index = drop_index.replace(True, 1).replace(False, 0).infer_objects(copy=False) + drop_index = drop_index.replace(True, 1).replace(False, 0).infer_objects() if training_filter: # we don't care about total row number (total no. datapoints) in training, we only care # about removing any row with NaNs # if labels has multiple columns (user wants to train multiple modelEs), we detect here labels = unfiltered_df.filter(label_list or [], axis=1) drop_index_labels = pd.isnull(labels).any(axis=1) - drop_index_labels = ( - drop_index_labels.replace(True, 1).replace(False, 0).infer_objects(copy=False) - ) + drop_index_labels = drop_index_labels.replace(True, 1).replace(False, 0).infer_objects() dates = unfiltered_df["date"] filtered_df = filtered_df[ (drop_index == 0) & (drop_index_labels == 0) diff --git a/freqtrade/persistence/key_value_store.py b/freqtrade/persistence/key_value_store.py index ac3cedcd1..9010ad0e2 100644 --- a/freqtrade/persistence/key_value_store.py +++ b/freqtrade/persistence/key_value_store.py @@ -18,6 +18,7 @@ class ValueTypesEnum(StrEnum): INT = "int" +# must be < 50 characters to fit the database column KeyStoreKeys = Literal[ "bot_start_time", "startup_time", @@ -37,7 +38,7 @@ class _KeyValueStoreModel(ModelBase): id: Mapped[int] = mapped_column(primary_key=True) - key: Mapped[KeyStoreKeys] = mapped_column(String(25), nullable=False, index=True) + key: Mapped[KeyStoreKeys] = mapped_column(String(50), nullable=False, index=True) value_type: Mapped[ValueTypesEnum] = mapped_column(String(20), nullable=False) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 27970930f..83dfbea77 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -35,10 +35,12 @@ def get_last_sequence_ids(engine, sequence_name: str, table_back_name: str) -> i if engine.name == "postgresql": with engine.begin() as connection: - last_id = connection.execute(text(f"select nextval('{sequence_name}')")).fetchone()[0] + last_id = connection.execute( + text(f"""select nextval('"{sequence_name}"')""") + ).fetchone()[0] with engine.begin() as connection: connection.execute( - text(f"ALTER SEQUENCE {sequence_name} rename to {table_back_name}_id_seq_bak") + text(f'ALTER SEQUENCE "{sequence_name}" rename to "{table_back_name}_id_seq_bak"') ) return last_id @@ -88,9 +90,9 @@ def drop_index_on_table(engine, inspector, table_bak_name): # drop indexes on backup table in new session for index in inspector.get_indexes(table_bak_name): if engine.name == "mysql": - connection.execute(text(f"drop index {index['name']} on {table_bak_name}")) + connection.execute(text(f'drop index "{index["name"]}" on {table_bak_name}')) else: - connection.execute(text(f"drop index {index['name']}")) + connection.execute(text(f'drop index "{index["name"]}"')) def migrate_trades_and_orders_table( @@ -315,6 +317,31 @@ def migrate_pairlocks_table(decl_base, inspector, engine, pairlock_back_name: st set_sequence_ids(engine, pairlock_id=pairlock_id) +def migrate_kv_store_table(decl_base, inspector, engine, kv_store_back_name: str, cols: list): + # Schema migration necessary + with engine.begin() as connection: + connection.execute(text(f'alter table "KeyValueStore" rename to "{kv_store_back_name}"')) + + drop_index_on_table(engine, inspector, kv_store_back_name) + kv_store_id = get_last_sequence_ids(engine, "KeyValueStore_id_seq", kv_store_back_name) + + # let SQLAlchemy create the schema as required + decl_base.metadata.create_all(engine) + # Copy data back - following the correct schema + with engine.begin() as connection: + connection.execute( + text( + f"""insert into "KeyValueStore" + (id, key, value_type, string_value, datetime_value, float_value, int_value) + select id, key, value_type, string_value, datetime_value, float_value, int_value + from "{kv_store_back_name}" + """ + ) + ) + + set_sequence_ids(engine, kv_id=kv_store_id) + + def set_sqlite_to_wal(engine): if engine.name == "sqlite" and str(engine.url) != "sqlite://": # Set Mode to @@ -385,12 +412,15 @@ def check_migrate(engine: Engine, decl_base, previous_tables: list[str]) -> None cols_trades = inspector.get_columns("trades") cols_orders = inspector.get_columns("orders") cols_pairlocks = inspector.get_columns("pairlocks") + cols_kv_store = inspector.get_columns("KeyValueStore") tabs = get_table_names_for_table(inspector, "trades") table_back_name = get_backup_name(tabs, "trades_bak") order_tabs = get_table_names_for_table(inspector, "orders") order_table_bak_name = get_backup_name(order_tabs, "orders_bak") pairlock_tabs = get_table_names_for_table(inspector, "pairlocks") pairlock_table_bak_name = get_backup_name(pairlock_tabs, "pairlocks_bak") + kv_store_tabs = get_table_names_for_table(inspector, "KeyValueStore") + kv_store_back_name = get_backup_name(kv_store_tabs, "KeyValueStore_bak") # Check if migration necessary # Migrates both trades and orders table! @@ -421,6 +451,16 @@ def check_migrate(engine: Engine, decl_base, previous_tables: list[str]) -> None migrate_pairlocks_table( decl_base, inspector, engine, pairlock_table_bak_name, cols_pairlocks ) + if "KeyValueStore" in previous_tables: + key_column = next(filter(lambda x: x["name"] == "key", cols_kv_store), None) + # length of key column < 50, recreate table with correct length and migrate data + if key_column and getattr(key_column["type"], "length", -1) < 50: + migrating = True + logger.info( + f"Running database migration for KeyValueStore - backup: {kv_store_back_name}" + ) + migrate_kv_store_table(decl_base, inspector, engine, kv_store_back_name, cols_kv_store) + if "orders" not in previous_tables and "trades" in previous_tables: raise OperationalException( "Your database seems to be very old. " diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 8e88cc4e7..4d3b5372c 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -483,7 +483,7 @@ class Telegram(RPCHandler): profit_prefix = "Sub " cp_extra = ( f"*Final Profit:* `{format_pct(msg['final_profit_ratio'])} " - f"({msg['cumulative_profit']:.8f} {msg['quote_currency']}{cp_fiat})`\n" + f"({fmt_coin(msg['cumulative_profit'], msg['stake_currency'])}{cp_fiat})`\n" ) else: exit_wording = f"Partially {exit_wording.lower()}" @@ -832,7 +832,7 @@ class Telegram(RPCHandler): ): # Adding initial stoploss only if it is different from stoploss lines.append( - f"*Initial Stoploss:* `{r['initial_stop_loss_abs']:.8f}` " + f"*Initial Stoploss:* `{round_value(r['initial_stop_loss_abs'], 8)}` " f"`({format_pct(r['initial_stop_loss_ratio'])})`" ) diff --git a/pyproject.toml b/pyproject.toml index fd23460bb..c6024a3b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dependencies = [ "urllib3", "jsonschema", "numpy>2.0,<3.0", - "pandas>=2.2.0,<3.0", + "pandas>=2.2.0,<4.0", "TA-Lib<0.7", "ft-pandas-ta", "technical", diff --git a/requirements.txt b/requirements.txt index 2af13fdb7..80c924deb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ numpy==2.4.4 -pandas==2.3.3 +pandas==3.0.2 bottleneck==1.6.0 numexpr==2.14.1 # Indicator libraries diff --git a/tests/conftest.py b/tests/conftest.py index 46601ddfb..dc0860466 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -176,20 +176,20 @@ def generate_test_data( base = np.random.normal(base, 2, size=size) if timeframe == "1y": - date = pd.date_range(start, periods=size, freq="1YS", tz="UTC") + date = pd.date_range(start, periods=size, freq="1YS", tz="UTC", unit="ms") elif timeframe == "1M": - date = pd.date_range(start, periods=size, freq="1MS", tz="UTC") + date = pd.date_range(start, periods=size, freq="1MS", tz="UTC", unit="ms") elif timeframe == "3M": - date = pd.date_range(start, periods=size, freq="3MS", tz="UTC") + date = pd.date_range(start, periods=size, freq="3MS", tz="UTC", unit="ms") elif timeframe == "1w" or timeframe == "7d": - date = pd.date_range(start, periods=size, freq="1W-MON", tz="UTC") + date = pd.date_range(start, periods=size, freq="1W-MON", tz="UTC", unit="ms") else: tf_mins = timeframe_to_minutes(timeframe) if tf_mins >= 1: - date = pd.date_range(start, periods=size, freq=f"{tf_mins}min", tz="UTC") + date = pd.date_range(start, periods=size, freq=f"{tf_mins}min", tz="UTC", unit="ms") else: tf_secs = timeframe_to_seconds(timeframe) - date = pd.date_range(start, periods=size, freq=f"{tf_secs}s", tz="UTC") + date = pd.date_range(start, periods=size, freq=f"{tf_secs}s", tz="UTC", unit="ms") df = pd.DataFrame( { "date": date, diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 835f5a861..4946741aa 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -207,10 +207,13 @@ def test_ohlcv_to_dataframe_multi(timeframe): data1 = data.copy() if timeframe in ("1M", "3M", "1y"): - data1.loc[:, "date"] = data1.loc[:, "date"] + pd.to_timedelta("1w") + data1.loc[:, "date"] = data1.loc[:, "date"] + pd.to_timedelta("1W") else: # Shift by half a timeframe - data1.loc[:, "date"] = data1.loc[:, "date"] + (pd.to_timedelta(timeframe) / 2) + timeframe_f = ( + timeframe.upper() if timeframe.endswith("d") or timeframe.endswith("w") else timeframe + ) + data1.loc[:, "date"] = data1.loc[:, "date"] + (pd.to_timedelta(timeframe_f) / 2) df2 = ohlcv_to_dataframe(data1, timeframe, "UNITTEST/USDT") assert len(df2) == len(data) - 1 diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 70b591d54..95250e7a7 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -860,6 +860,9 @@ def test_backtest_one(default_conf, mocker, testdatadir) -> None: "funding_fees": [0.0, 0.0], } ) + # TODO: pandas3 - create correctly above ?!? + expected["open_date"] = expected["open_date"].astype("datetime64[ms, UTC]") + expected["close_date"] = expected["close_date"].astype("datetime64[ms, UTC]") pd.testing.assert_frame_equal(results, expected) assert "orders" in results.columns data_pair = processed[pair] diff --git a/tests/optimize/test_backtesting_adjust_position.py b/tests/optimize/test_backtesting_adjust_position.py index 33fd87d83..f698173c9 100644 --- a/tests/optimize/test_backtesting_adjust_position.py +++ b/tests/optimize/test_backtesting_adjust_position.py @@ -83,6 +83,9 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) -> "funding_fees": [0.0, 0.0], } ) + # TODO: pandas3 - create correctly above ?!? + expected["open_date"] = expected["open_date"].astype("datetime64[ms, UTC]") + expected["close_date"] = expected["close_date"].astype("datetime64[ms, UTC]") results_no = results.drop(columns=["orders"]) pd.testing.assert_frame_equal(results_no, expected, check_exact=True)