mohammadian7/freqtrade
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'freqtrade:develop' into fix-bitget-stoploss

Browse Source
This commit is contained in:
ABS
2026-04-19 23:15:35 +08:00
committed by GitHub
parent 284446d782 84535a1404
commit a02c84ecda
13 changed files with 77 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
freqtrade/data/history/datahandlers/featherdatahandler.py
+2 -2
View File
@@ -1,6 +1,6 @@
import logging
from pandas import DataFrame, read_feather, to_datetime
from pandas import DataFrame, read_feather
from pyarrow import dataset
from freqtrade.configuration import TimeRange
@@ -71,7 +71,7 @@ class FeatherDataHandler(IDataHandler):
"volume": "float",
}
)
pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True)
pairdata["date"] = pairdata["date"].dt.as_unit("ms")
return pairdata
except Exception as e:
logger.exception(
freqtrade/data/history/datahandlers/jsondatahandler.py
+4 -1
View File
@@ -80,7 +80,7 @@ class JsonDataHandler(IDataHandler):
"volume": "float",
}
)
pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True)
pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True).dt.as_unit("ms")
return pairdata
def ohlcv_append(
@@ -104,6 +104,9 @@ class JsonDataHandler(IDataHandler):
:param trading_mode: Trading mode to use (used to determine the filename)
"""
filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
# Convert StringDtype columns to object to avoid NaN serialization issues
for col in data.select_dtypes(include="string").columns:
data[col] = data[col].astype(object).where(data[col].notna(), other=None)
trades = data.values.tolist()
misc.file_dump_json(filename, trades, is_zip=self._use_zip)
freqtrade/data/history/datahandlers/parquetdatahandler.py
+2 -2
View File
@@ -1,6 +1,6 @@
import logging
from pandas import DataFrame, read_parquet, to_datetime
from pandas import DataFrame, read_parquet
from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
@@ -68,7 +68,7 @@ class ParquetDataHandler(IDataHandler):
"volume": "float",
}
)
pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True)
pairdata["date"] = pairdata["date"].dt.as_unit("ms")
return pairdata
except Exception as e:
logger.exception(
freqtrade/freqai/data_kitchen.py
+2 -6
View File
@@ -24,8 +24,6 @@ from freqtrade.strategy import merge_informative_pair
from freqtrade.strategy.interface import IStrategy
pd.set_option("future.no_silent_downcasting", True)
SECONDS_IN_DAY = 86400
SECONDS_IN_HOUR = 3600
@@ -239,16 +237,14 @@ class FreqaiDataKitchen:
filtered_df = filtered_df.replace([np.inf, -np.inf], np.nan)
drop_index = pd.isnull(filtered_df).any(axis=1) # get the rows that have NaNs,
drop_index = drop_index.replace(True, 1).replace(False, 0).infer_objects(copy=False)
drop_index = drop_index.replace(True, 1).replace(False, 0).infer_objects()
if training_filter:
# we don't care about total row number (total no. datapoints) in training, we only care
# about removing any row with NaNs
# if labels has multiple columns (user wants to train multiple modelEs), we detect here
labels = unfiltered_df.filter(label_list or [], axis=1)
drop_index_labels = pd.isnull(labels).any(axis=1)
drop_index_labels = (
drop_index_labels.replace(True, 1).replace(False, 0).infer_objects(copy=False)
)
drop_index_labels = drop_index_labels.replace(True, 1).replace(False, 0).infer_objects()
dates = unfiltered_df["date"]
filtered_df = filtered_df[
(drop_index == 0) & (drop_index_labels == 0)
freqtrade/persistence/key_value_store.py
+2 -1
View File
@@ -18,6 +18,7 @@ class ValueTypesEnum(StrEnum):
INT = "int"
# must be < 50 characters to fit the database column
KeyStoreKeys = Literal[
"bot_start_time",
"startup_time",
@@ -37,7 +38,7 @@ class _KeyValueStoreModel(ModelBase):
id: Mapped[int] = mapped_column(primary_key=True)
key: Mapped[KeyStoreKeys] = mapped_column(String(25), nullable=False, index=True)
key: Mapped[KeyStoreKeys] = mapped_column(String(50), nullable=False, index=True)
value_type: Mapped[ValueTypesEnum] = mapped_column(String(20), nullable=False)
freqtrade/persistence/migrations.py
+44 -4
View File
@@ -35,10 +35,12 @@ def get_last_sequence_ids(engine, sequence_name: str, table_back_name: str) -> i
if engine.name == "postgresql":
with engine.begin() as connection:
last_id = connection.execute(text(f"select nextval('{sequence_name}')")).fetchone()[0]
last_id = connection.execute(
text(f"""select nextval('"{sequence_name}"')""")
).fetchone()[0]
with engine.begin() as connection:
connection.execute(
text(f"ALTER SEQUENCE {sequence_name} rename to {table_back_name}_id_seq_bak")
text(f'ALTER SEQUENCE "{sequence_name}" rename to "{table_back_name}_id_seq_bak"')
)
return last_id
@@ -88,9 +90,9 @@ def drop_index_on_table(engine, inspector, table_bak_name):
# drop indexes on backup table in new session
for index in inspector.get_indexes(table_bak_name):
if engine.name == "mysql":
connection.execute(text(f"drop index {index['name']} on {table_bak_name}"))
connection.execute(text(f'drop index "{index["name"]}" on {table_bak_name}'))
else:
connection.execute(text(f"drop index {index['name']}"))
connection.execute(text(f'drop index "{index["name"]}"'))
def migrate_trades_and_orders_table(
@@ -315,6 +317,31 @@ def migrate_pairlocks_table(decl_base, inspector, engine, pairlock_back_name: st
set_sequence_ids(engine, pairlock_id=pairlock_id)
def migrate_kv_store_table(decl_base, inspector, engine, kv_store_back_name: str, cols: list):
# Schema migration necessary
with engine.begin() as connection:
connection.execute(text(f'alter table "KeyValueStore" rename to "{kv_store_back_name}"'))
drop_index_on_table(engine, inspector, kv_store_back_name)
kv_store_id = get_last_sequence_ids(engine, "KeyValueStore_id_seq", kv_store_back_name)
# let SQLAlchemy create the schema as required
decl_base.metadata.create_all(engine)
# Copy data back - following the correct schema
with engine.begin() as connection:
connection.execute(
text(
f"""insert into "KeyValueStore"
(id, key, value_type, string_value, datetime_value, float_value, int_value)
select id, key, value_type, string_value, datetime_value, float_value, int_value
from "{kv_store_back_name}"
"""
)
)
set_sequence_ids(engine, kv_id=kv_store_id)
def set_sqlite_to_wal(engine):
if engine.name == "sqlite" and str(engine.url) != "sqlite://":
# Set Mode to
@@ -385,12 +412,15 @@ def check_migrate(engine: Engine, decl_base, previous_tables: list[str]) -> None
cols_trades = inspector.get_columns("trades")
cols_orders = inspector.get_columns("orders")
cols_pairlocks = inspector.get_columns("pairlocks")
cols_kv_store = inspector.get_columns("KeyValueStore")
tabs = get_table_names_for_table(inspector, "trades")
table_back_name = get_backup_name(tabs, "trades_bak")
order_tabs = get_table_names_for_table(inspector, "orders")
order_table_bak_name = get_backup_name(order_tabs, "orders_bak")
pairlock_tabs = get_table_names_for_table(inspector, "pairlocks")
pairlock_table_bak_name = get_backup_name(pairlock_tabs, "pairlocks_bak")
kv_store_tabs = get_table_names_for_table(inspector, "KeyValueStore")
kv_store_back_name = get_backup_name(kv_store_tabs, "KeyValueStore_bak")
# Check if migration necessary
# Migrates both trades and orders table!
@@ -421,6 +451,16 @@ def check_migrate(engine: Engine, decl_base, previous_tables: list[str]) -> None
migrate_pairlocks_table(
decl_base, inspector, engine, pairlock_table_bak_name, cols_pairlocks
)
if "KeyValueStore" in previous_tables:
key_column = next(filter(lambda x: x["name"] == "key", cols_kv_store), None)
# length of key column < 50, recreate table with correct length and migrate data
if key_column and getattr(key_column["type"], "length", -1) < 50:
migrating = True
logger.info(
f"Running database migration for KeyValueStore - backup: {kv_store_back_name}"
)
migrate_kv_store_table(decl_base, inspector, engine, kv_store_back_name, cols_kv_store)
if "orders" not in previous_tables and "trades" in previous_tables:
raise OperationalException(
"Your database seems to be very old. "
freqtrade/rpc/telegram.py
+2 -2
View File
@@ -483,7 +483,7 @@ class Telegram(RPCHandler):
profit_prefix = "Sub "
cp_extra = (
f"*Final Profit:* `{format_pct(msg['final_profit_ratio'])} "
f"({msg['cumulative_profit']:.8f} {msg['quote_currency']}{cp_fiat})`\n"
f"({fmt_coin(msg['cumulative_profit'], msg['stake_currency'])}{cp_fiat})`\n"
)
else:
exit_wording = f"Partially {exit_wording.lower()}"
@@ -832,7 +832,7 @@ class Telegram(RPCHandler):
):
# Adding initial stoploss only if it is different from stoploss
lines.append(
f"*Initial Stoploss:* `{r['initial_stop_loss_abs']:.8f}` "
f"*Initial Stoploss:* `{round_value(r['initial_stop_loss_abs'], 8)}` "
f"`({format_pct(r['initial_stop_loss_ratio'])})`"
)
pyproject.toml
+1 -1
View File
@@ -40,7 +40,7 @@ dependencies = [
"urllib3",
"jsonschema",
"numpy>2.0,<3.0",
"pandas>=2.2.0,<3.0",
"pandas>=2.2.0,<4.0",
"TA-Lib<0.7",
"ft-pandas-ta",
"technical",
requirements.txt
+1 -1
View File
@@ -1,5 +1,5 @@
numpy==2.4.4
pandas==2.3.3
pandas==3.0.2
bottleneck==1.6.0
numexpr==2.14.1
# Indicator libraries
tests/conftest.py
+6 -6
View File
@@ -176,20 +176,20 @@ def generate_test_data(
base = np.random.normal(base, 2, size=size)
if timeframe == "1y":
date = pd.date_range(start, periods=size, freq="1YS", tz="UTC")
date = pd.date_range(start, periods=size, freq="1YS", tz="UTC", unit="ms")
elif timeframe == "1M":
date = pd.date_range(start, periods=size, freq="1MS", tz="UTC")
date = pd.date_range(start, periods=size, freq="1MS", tz="UTC", unit="ms")
elif timeframe == "3M":
date = pd.date_range(start, periods=size, freq="3MS", tz="UTC")
date = pd.date_range(start, periods=size, freq="3MS", tz="UTC", unit="ms")
elif timeframe == "1w" or timeframe == "7d":
date = pd.date_range(start, periods=size, freq="1W-MON", tz="UTC")
date = pd.date_range(start, periods=size, freq="1W-MON", tz="UTC", unit="ms")
else:
tf_mins = timeframe_to_minutes(timeframe)
if tf_mins >= 1:
date = pd.date_range(start, periods=size, freq=f"{tf_mins}min", tz="UTC")
date = pd.date_range(start, periods=size, freq=f"{tf_mins}min", tz="UTC", unit="ms")
else:
tf_secs = timeframe_to_seconds(timeframe)
date = pd.date_range(start, periods=size, freq=f"{tf_secs}s", tz="UTC")
date = pd.date_range(start, periods=size, freq=f"{tf_secs}s", tz="UTC", unit="ms")
df = pd.DataFrame(
{
"date": date,
tests/data/test_converter.py
+5 -2
View File
@@ -207,10 +207,13 @@ def test_ohlcv_to_dataframe_multi(timeframe):
data1 = data.copy()
if timeframe in ("1M", "3M", "1y"):
data1.loc[:, "date"] = data1.loc[:, "date"] + pd.to_timedelta("1w")
data1.loc[:, "date"] = data1.loc[:, "date"] + pd.to_timedelta("1W")
else:
# Shift by half a timeframe
data1.loc[:, "date"] = data1.loc[:, "date"] + (pd.to_timedelta(timeframe) / 2)
timeframe_f = (
timeframe.upper() if timeframe.endswith("d") or timeframe.endswith("w") else timeframe
)
data1.loc[:, "date"] = data1.loc[:, "date"] + (pd.to_timedelta(timeframe_f) / 2)
df2 = ohlcv_to_dataframe(data1, timeframe, "UNITTEST/USDT")
assert len(df2) == len(data) - 1
tests/optimize/test_backtesting.py
+3
View File
@@ -860,6 +860,9 @@ def test_backtest_one(default_conf, mocker, testdatadir) -> None:
"funding_fees": [0.0, 0.0],
}
)
# TODO: pandas3 - create correctly above ?!?
expected["open_date"] = expected["open_date"].astype("datetime64[ms, UTC]")
expected["close_date"] = expected["close_date"].astype("datetime64[ms, UTC]")
pd.testing.assert_frame_equal(results, expected)
assert "orders" in results.columns
data_pair = processed[pair]
tests/optimize/test_backtesting_adjust_position.py
+3
View File
@@ -83,6 +83,9 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) ->
"funding_fees": [0.0, 0.0],
}
)
# TODO: pandas3 - create correctly above ?!?
expected["open_date"] = expected["open_date"].astype("datetime64[ms, UTC]")
expected["close_date"] = expected["close_date"].astype("datetime64[ms, UTC]")
results_no = results.drop(columns=["orders"])
pd.testing.assert_frame_equal(results_no, expected, check_exact=True)