diff --git a/build_helpers/schema.json b/build_helpers/schema.json index 8bf56b2a1..5d07b4ad1 100644 --- a/build_helpers/schema.json +++ b/build_helpers/schema.json @@ -283,6 +283,10 @@ "month" ] }, + "skip_wallet_history_migration": { + "description": "Disable wallet history migration.", + "type": "boolean" + }, "hyperopt_path": { "description": "Specify additional lookup path for Hyperopt Loss functions.", "type": "string" diff --git a/docs/backtesting.md b/docs/backtesting.md index 6b86d9635..12455b367 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -211,58 +211,59 @@ A backtesting result will look like that: │ TOTAL │ │ 77 │ 0.22 │ 54.774 │ 5.48 │ 22:12:00 │ 67 0 10 87.0 │ └───────────┴─────────────┴────────┴──────────────┴─────────────────┴──────────────┴─────────────────┴────────────────────────┘ SUMMARY METRICS -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Metric ┃ Value ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ Backtesting from │ 2025-07-01 00:00:00 │ -│ Backtesting to │ 2025-08-01 00:00:00 │ -│ Trading Mode │ Isolated Futures │ -│ Max open trades │ 3 │ -│ │ │ -│ Total/Daily Avg Trades │ 77 / 2.48 │ -│ Starting balance │ 1000 USDT │ -│ Final balance │ 1054.774 USDT │ -│ Absolute profit │ 54.774 USDT │ -│ Total profit % │ 5.48% │ -│ CAGR % │ 87.36% │ -│ Sortino │ 2.48 │ -│ Sharpe │ 3.75 │ -│ Calmar │ 40.99 │ -│ SQN │ 0.69 │ -│ Profit factor │ 1.29 │ -│ Expectancy (Ratio) │ 0.71 (0.04) │ -│ Avg. daily profit │ 1.767 USDT │ -│ Avg. stake amount │ 345.016 USDT │ -│ Total trade volume │ 53316.954 USDT │ -│ │ │ -│ Long / Short trades │ 67 / 10 │ -│ Long / Short profit % │ 8.94% / -3.47% │ -│ Long / Short profit USDT │ 89.425 / -34.651 │ -│ │ │ -│ Best Pair │ LTC/USDT:USDT 5.62% │ -│ Worst Pair │ ADA/USDT:USDT -5.21% │ -│ Best trade │ ETC/USDT:USDT 2.00% │ -│ Worst trade │ ADA/USDT:USDT -10.17% │ -│ Best day │ 26.91 USDT │ -│ Worst day │ -47.741 USDT │ -│ Days win/draw/lose │ 20 / 6 / 5 │ -│ Min/Max/Avg. Duration Winners │ 0d 00:35 / 5d 18:15 / 0d 15:49 │ -│ Min/Max/Avg. Duration Losers │ 0d 10:40 / 17d 08:00 / 2d 17:00 │ -│ Max Consecutive Wins / Loss │ 36 / 3 │ -│ Rejected Entry signals │ 258 │ -│ Entry/Exit Timeouts │ 0 / 0 │ -│ │ │ -│ Min balance │ 1003.168 USDT │ -│ Max balance │ 1149.421 USDT │ -│ Max % of account underwater │ 8.23% │ -│ Absolute drawdown │ 94.647 USDT (8.23%) │ -│ Drawdown duration │ 9 days 08:50:00 │ -│ Profit at drawdown start │ 149.421 USDT │ -│ Profit at drawdown end │ 54.774 USDT │ -│ Drawdown start │ 2025-07-22 15:10:00 │ -│ Drawdown end │ 2025-08-01 00:00:00 │ -│ Market change │ 30.51% │ -└───────────────────────────────┴─────────────────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Metric ┃ Value ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ Backtesting from │ 2025-07-01 00:00:00 │ +│ Backtesting to │ 2025-08-01 00:00:00 │ +│ Trading Mode │ Isolated Futures │ +│ Max open trades │ 3 │ +│ │ │ +│ Total/Daily Avg Trades │ 77 / 2.48 │ +│ Starting balance │ 1000 USDT │ +│ Final balance │ 1054.669 USDT │ +│ Absolute profit │ 54.669 USDT │ +│ Total profit % │ 5.47% │ +│ CAGR % │ 87.14% │ +│ Sortino │ 2.46 │ +│ Sharpe │ 3.73 │ +│ Calmar │ 40.81 │ +│ SQN │ 0.69 │ +│ Profit factor │ 1.29 │ +│ Expectancy (Ratio) │ 0.71 (0.04) │ +│ Avg. daily profit │ 1.764 USDT │ +│ Avg. stake amount │ 345.251 USDT │ +│ Total trade volume │ 53352.96 USDT │ +│ │ │ +│ Long / Short trades │ 67 / 10 │ +│ Long / Short profit % │ 8.93% / -3.46% │ +│ Long / Short profit USDT │ 89.262 / -34.593 │ +│ │ │ +│ Best Pair │ LTC/USDT:USDT 5.62% │ +│ Worst Pair │ ADA/USDT:USDT -5.21% │ +│ Best trade │ ETC/USDT:USDT 2.00% │ +│ Worst trade │ ADA/USDT:USDT -10.17% │ +│ Best day │ 26.931 USDT │ +│ Worst day │ -47.741 USDT │ +│ Days win/draw/lose │ 20 / 6 / 5 │ +│ Min/Max/Avg. Duration Winners │ 0d 00:35 / 5d 18:15 / 0d 15:49 │ +│ Min/Max/Avg. Duration Losers │ 0d 10:40 / 17d 08:00 / 2d 17:00 │ +│ Max Consecutive Wins / Loss │ 36 / 3 │ +│ Rejected Entry signals │ 258 │ +│ Entry/Exit Timeouts │ 0 / 0 │ +│ │ │ +│ Min/Max balance realized │ 1003.168 USDT / 1149.577 USDT │ +│ Min/Max balance unrealized │ 1000 USDT / 1149.577 USDT │ +│ Min/Max balance dates │ 2025-07-01 00:05:00 / 2025-07-22 15:15:00 │ +│ Max % of account underwater │ 8.26% │ +│ Absolute drawdown │ 94.908 USDT (8.26%) │ +│ Drawdown duration │ 9 days 08:50:00 │ +│ Profit at drawdown start │ 149.577 USDT │ +│ Profit at drawdown end │ 54.669 USDT │ +│ Drawdown start │ 2025-07-22 15:10:00 │ +│ Drawdown end │ 2025-08-01 00:00:00 │ +│ Market change │ 30.51% │ +└───────────────────────────────┴───────────────────────────────────────────┘ Backtested 2025-07-01 00:00:00 -> 2025-08-01 00:00:00 | Max open trades : 3 STRATEGY SUMMARY @@ -329,54 +330,59 @@ The last element of the backtest report is the summary metrics table. It contains key metrics about the performance of your strategy on backtesting data. ``` -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Metric ┃ Value ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ Backtesting from │ 2025-07-01 00:00:00 │ -│ Backtesting to │ 2025-08-01 00:00:00 │ -│ Trading Mode │ Isolated Futures │ -│ Max open trades │ 3 │ -│ │ │ -│ Total/Daily Avg Trades │ 72 / 2.32 │ -│ Starting balance │ 1000 USDT │ -│ Final balance │ 1106.734 USDT │ -│ Absolute profit │ 106.734 USDT │ -│ Total profit % │ 10.67% │ -│ CAGR % │ 230.04% │ -│ Sortino │ 4.99 │ -│ Sharpe │ 8.00 │ -│ Calmar │ 77.76 │ -│ SQN │ 1.52 │ -│ Profit factor │ 1.79 │ -│ Expectancy (Ratio) │ 1.48 (0.07) │ -│ Avg. daily profit │ 3.443 USDT │ -│ Avg. stake amount │ 363.133 USDT │ -│ Total trade volume │ 52466.174 USDT │ -│ │ │ -│ Best Pair │ LTC/USDT:USDT 4.48% │ -│ Worst Pair │ ADA/USDT:USDT -1.78% │ -│ Best trade │ ETC/USDT:USDT 2.00% │ -│ Worst trade │ ADA/USDT:USDT -10.17% │ -│ Best day │ 23.535 USDT │ -│ Worst day │ -49.813 USDT │ -│ Days win/draw/lose │ 21 / 6 / 4 │ -│ Min/Max/Avg. Duration Winners │ 0d 00:35 / 5d 18:15 / 0d 15:30 │ -│ Min/Max/Avg. Duration Losers │ 0d 12:00 / 17d 08:00 / 3d 23:28 │ -│ Max Consecutive Wins / Loss │ 58 / 4 │ -│ Rejected Entry signals │ 254 │ -│ Entry/Exit Timeouts │ 0 / 0 │ -│ │ │ -│ Min balance │ 1003.168 USDT │ -│ Max balance │ 1209 USDT │ -│ Max % of account underwater │ 8.46% │ -│ Absolute drawdown │ 102.266 USDT (8.46%) │ -│ Drawdown duration │ 9 days 08:50:00 │ -│ Profit at drawdown start │ 209 USDT │ -│ Profit at drawdown end │ 106.734 USDT │ -│ Drawdown start │ 2025-07-22 15:10:00 │ -│ Drawdown end │ 2025-08-01 00:00:00 │ -│ Market change │ 30.51% │ -└───────────────────────────────┴─────────────────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Metric ┃ Value ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ Backtesting from │ 2025-07-01 00:00:00 │ +│ Backtesting to │ 2025-08-01 00:00:00 │ +│ Trading Mode │ Isolated Futures │ +│ Max open trades │ 3 │ +│ │ │ +│ Total/Daily Avg Trades │ 77 / 2.48 │ +│ Starting balance │ 1000 USDT │ +│ Final balance │ 1054.669 USDT │ +│ Absolute profit │ 54.669 USDT │ +│ Total profit % │ 5.47% │ +│ CAGR % │ 87.14% │ +│ Sortino │ 2.46 │ +│ Sharpe │ 3.73 │ +│ Calmar │ 40.81 │ +│ SQN │ 0.69 │ +│ Profit factor │ 1.29 │ +│ Expectancy (Ratio) │ 0.71 (0.04) │ +│ Avg. daily profit │ 1.764 USDT │ +│ Avg. stake amount │ 345.251 USDT │ +│ Total trade volume │ 53352.96 USDT │ +│ │ │ +│ Long / Short trades │ 67 / 10 │ +│ Long / Short profit % │ 8.93% / -3.46% │ +│ Long / Short profit USDT │ 89.262 / -34.593 │ +│ │ │ +│ Best Pair │ LTC/USDT:USDT 5.62% │ +│ Worst Pair │ ADA/USDT:USDT -5.21% │ +│ Best trade │ ETC/USDT:USDT 2.00% │ +│ Worst trade │ ADA/USDT:USDT -10.17% │ +│ Best day │ 26.931 USDT │ +│ Worst day │ -47.741 USDT │ +│ Days win/draw/lose │ 20 / 6 / 5 │ +│ Min/Max/Avg. Duration Winners │ 0d 00:35 / 5d 18:15 / 0d 15:49 │ +│ Min/Max/Avg. Duration Losers │ 0d 10:40 / 17d 08:00 / 2d 17:00 │ +│ Max Consecutive Wins / Loss │ 36 / 3 │ +│ Rejected Entry signals │ 258 │ +│ Entry/Exit Timeouts │ 0 / 0 │ +│ │ │ +│ Min/Max balance realized │ 1003.168 USDT / 1149.577 USDT │ +│ Min/Max balance unrealized │ 1000 USDT / 1149.577 USDT │ +│ Min/Max balance dates │ 2025-07-01 00:05:00 / 2025-07-22 15:15:00 │ +│ Max % of account underwater │ 8.26% │ +│ Absolute drawdown │ 94.908 USDT (8.26%) │ +│ Drawdown duration │ 9 days 08:50:00 │ +│ Profit at drawdown start │ 149.577 USDT │ +│ Profit at drawdown end │ 54.669 USDT │ +│ Drawdown start │ 2025-07-22 15:10:00 │ +│ Drawdown end │ 2025-08-01 00:00:00 │ +│ Market change │ 30.51% │ +└───────────────────────────────┴───────────────────────────────────────────┘ ``` - `Backtesting from` / `Backtesting to`: Backtesting range (usually defined with the `--timerange` option). @@ -409,7 +415,9 @@ It contains key metrics about the performance of your strategy on backtesting da - `Max Consecutive Wins / Loss`: Maximum consecutive wins/losses in a row. - `Rejected Entry signals`: Trade entry signals that could not be acted upon due to `max_open_trades` being reached. - `Entry/Exit Timeouts`: Entry/exit orders which did not fill (only applicable if custom pricing is used). -- `Min balance` / `Max balance`: Lowest and Highest Wallet balance during the backtest period. +- `Min/Max balance realized`: Lowest and Highest Wallet balance during the backtest period based on closed trades trades. +- `Min/Max balance unrealized`: Lowest and Highest Wallet balance during the backtest period - including capital tied in open trades. +- `Min/Max balance dates`: Dates when the minimum and maximum unrealized balance occurred. - `Max % of account underwater`: Maximum percentage your account has decreased from the top since the simulation started. Calculated as the maximum of `(Max Balance - Current Balance) / (Max Balance)`. - `Absolute drawdown`: Maximum absolute drawdown experienced, including percentage relative to the account calculated as `(Absolute Drawdown) / (DrawdownHigh + startingBalance)`.. - `Drawdown duration`: Duration of the largest drawdown period. diff --git a/docs/freq-ui.md b/docs/freq-ui.md index ff9758888..d561810f7 100644 --- a/docs/freq-ui.md +++ b/docs/freq-ui.md @@ -46,6 +46,23 @@ On this page, you can also interact with the bot by starting and stopping it and ![FreqUI - trade view](assets/freqUI-trade-pane-dark.png#only-dark) ![FreqUI - trade view](assets/freqUI-trade-pane-light.png#only-light) +### Dashboard + +The dashboard view provides an overview of the bot's performance and status. +If multiple bots are connected, the dashboard will show an overview of all connected bots, allowing you to easily switch between them or show just a subset of available bots. + +#### Wallet Balance + +New in freqtrade 2026.4: This shows the balance of the bot over time. + +Compared to the "cumulative Profit" chart, this chart will show the actual balance of the bot over time, including unrealized profit and losses, as well as deposits and withdrawals. + +Historic data has re-populated based on available exchange data - however is assumed to be best-effort and may not be 100% accurate. +More specifically, it won't cover deposits and withdrawals, and will assume a starting balance of current balance - profit/losses. + +For clarity - a "Capture start" marker line is shown on the chart, which indicates the point at which the migration to the new wallet balance tracking system happened. +Only beyond this point, the wallet balance is expected to be accurate. + ### Plot Configurator FreqUI Plots can be configured either via a `plot_config` configuration object in the strategy (which can be loaded via "from strategy" button) or via the UI. diff --git a/freqtrade/config_schema/config_schema.py b/freqtrade/config_schema/config_schema.py index fc2c42441..7fc885662 100644 --- a/freqtrade/config_schema/config_schema.py +++ b/freqtrade/config_schema/config_schema.py @@ -236,6 +236,10 @@ CONF_SCHEMA = { "type": "string", "enum": BACKTEST_CACHE_AGE, }, + "skip_wallet_history_migration": { + "description": "Disable wallet history migration.", + "type": "boolean", + }, # Hyperopt "hyperopt_path": { "description": "Specify additional lookup path for Hyperopt Loss functions.", diff --git a/freqtrade/data/btanalysis/__init__.py b/freqtrade/data/btanalysis/__init__.py index 2253a45ae..bd8b55df8 100644 --- a/freqtrade/data/btanalysis/__init__.py +++ b/freqtrade/data/btanalysis/__init__.py @@ -7,6 +7,7 @@ from .bt_fileutils import ( get_backtest_market_change, get_backtest_result, get_backtest_resultlist, + get_backtest_wallet_change, get_latest_backtest_filename, get_latest_hyperopt_file, get_latest_hyperopt_filename, diff --git a/freqtrade/data/btanalysis/bt_fileutils.py b/freqtrade/data/btanalysis/bt_fileutils.py index e1c0ea64c..a97d5bef3 100644 --- a/freqtrade/data/btanalysis/bt_fileutils.py +++ b/freqtrade/data/btanalysis/bt_fileutils.py @@ -312,6 +312,27 @@ def get_backtest_market_change(filename: Path, include_ts: bool = True) -> pd.Da return df +def get_backtest_wallet_change(filename: Path, strategy_name: str) -> pd.DataFrame | None: + """ + Read backtest wallet change file. + :param filename: Path to the backtest result zip file + :param strategy_name: Name of the strategy to load + :return: DataFrame with wallet change data + """ + if filename.suffix != ".zip": + return None + + try: + data = load_file_from_zip(filename, f"{filename.stem}_{strategy_name}_wallet.feather") + df = pd.read_feather(BytesIO(data)) + + df.loc[:, "__date_ts"] = df.loc[:, "date"].astype(np.int64) // 1000 // 1000 + return df + except ValueError: + pass + return None + + def find_existing_backtest_stats( dirname: Path | str, run_ids: dict[str, str], min_backtest_date: datetime | None = None ) -> dict[str, Any]: @@ -503,13 +524,16 @@ def load_backtest_analysis_data( return None -def trade_list_to_dataframe(trades: list[Trade] | list[LocalTrade]) -> pd.DataFrame: +def trade_list_to_dataframe( + trades: list[Trade] | list[LocalTrade], *, minified: bool = True +) -> pd.DataFrame: """ Convert list of Trade objects to pandas Dataframe :param trades: List of trade objects + :param minified: Whether to use minified version of trade JSON :return: Dataframe with BT_DATA_COLUMNS """ - df = pd.DataFrame.from_records([t.to_json(True) for t in trades], columns=BT_DATA_COLUMNS) + df = pd.DataFrame.from_records([t.to_json(minified) for t in trades], columns=BT_DATA_COLUMNS) if len(df) > 0: df["close_date"] = pd.to_datetime(df["close_timestamp"], unit="ms", utc=True) df["open_date"] = pd.to_datetime(df["open_timestamp"], unit="ms", utc=True) diff --git a/freqtrade/data/btanalysis/trade_parallelism.py b/freqtrade/data/btanalysis/trade_parallelism.py index eabdcf08a..8ed9702e0 100644 --- a/freqtrade/data/btanalysis/trade_parallelism.py +++ b/freqtrade/data/btanalysis/trade_parallelism.py @@ -1,9 +1,15 @@ import logging +from datetime import datetime import numpy as np import pandas as pd from freqtrade.constants import IntOrInf +from freqtrade.exchange import ( + timeframe_to_prev_date, + timeframe_to_resample_freq, +) +from freqtrade.util import dt_from_ts logger = logging.getLogger(__name__) @@ -58,3 +64,95 @@ def evaluate_result_multi( """ df_final = analyze_trade_parallelism(trades, timeframe) return df_final[df_final["open_trades"] > max_open_trades] + + +def balance_distribution_over_time( + trades: pd.DataFrame, + min_date: datetime, + max_date: datetime, + timeframe: str, + stake_currency: str, + start_balance: float, + pairlist: list[str], +) -> pd.DataFrame: + """ + Return a dataframe with stake_currency and the pairlist as columns + Each column will contain the amount of the currency at the given time + Columns added are: + - stake_currency: amount of stake currency + - : amount of base currency in the pair + - _leverage: leverage used for the pair at the time (NaN if no open trade) + - _is_short: 1 if the open trade is short, 0 if long (NaN if no open trade) + - _collateral: amount of stake currency used as collateral for open trades + :param trades: Trades Dataframe - can be loaded from backtest, or created + via trade_list_to_dataframe + :param timeframe: Frequency to use for the resulting dataframe + :param min_date: start date + :param max_date: End date (will be rounded down to timeframe) + :param stake_currency: The stake currency + :param start_balance: Starting balance in stake currency + :param pairlist: List of trading pairs to include in the dataframe + Can be obtained via trade_df["pair"].unique() + For pairs without trades, the column will be all zeros + :return: Dataframe with balance distribution over time + """ + min_date_res = timeframe_to_prev_date(timeframe, min_date) + max_date_res = timeframe_to_prev_date(timeframe, max_date) + index = pd.date_range(min_date_res, max_date_res, freq=timeframe_to_resample_freq(timeframe)) + pairs_lev = [f"{pair}_leverage" for pair in pairlist] + pairs_is_short = [f"{pair}_is_short" for pair in pairlist] + pairs_collateral = [f"{pair}_collateral" for pair in pairlist] + pairs_lev += pairs_is_short + + df = pd.DataFrame( + index=index, columns=[stake_currency] + pairlist + pairs_lev + pairs_collateral, dtype=float + ) + # Initialize variables to starting values + df[stake_currency] = float(start_balance) + df[pairlist + pairs_collateral] = 0.0 + df[pairs_lev] = np.nan + + for trade in trades.sort_values(by=["open_date"]).itertuples(): + pair = trade.pair + end_date = trade.close_date if trade.close_date is not pd.NaT else None + # Exclude open orders - these won't have order_filled_timestamp set. + df.loc[trade.open_date : end_date, f"{pair}_leverage"] = trade.leverage + df.loc[trade.open_date : end_date, f"{pair}_is_short"] = 1 if trade.is_short else 0 + orders = [o for o in trade.orders if o["order_filled_timestamp"]] + current_position = 0 + current_collateral = 0 + for order in sorted(orders, key=lambda x: x["order_filled_timestamp"]): + filled_at = pd.Timestamp(dt_from_ts(order["order_filled_timestamp"])) + real_amount = order.get("filled", order["amount"]) + stake = order["safe_price"] * real_amount + stake_no_lev = stake / trade.leverage + if order["ft_is_entry"]: + # Entry order: lock collateral and pay fee + # For both long and short: balance decreases by collateral + fee + fee_open = stake * trade.fee_open + current_position += real_amount + current_collateral += stake_no_lev + df.loc[filled_at:end_date, pair] += real_amount + df.loc[filled_at:end_date, f"{pair}_collateral"] += stake_no_lev + df.loc[filled_at:, stake_currency] -= stake_no_lev + fee_open + else: + # Exit order: release collateral and realize profit/loss + fee_close = stake * trade.fee_close + if trade.is_short: + # For SHORT + df.loc[filled_at:, stake_currency] += ( + current_collateral * (1 + trade.leverage) - stake + ) - fee_close + else: + # For LONG + df.loc[filled_at:, stake_currency] += ( + stake - current_collateral * (trade.leverage - 1) - fee_close + ) + df.loc[filled_at:end_date, pair] -= real_amount + df.loc[filled_at:end_date, f"{pair}_collateral"] -= stake_no_lev + current_position -= real_amount + current_collateral -= stake_no_lev + + # Round to avoid floating point issues + df = df.round(14) + return df diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 24c45d048..360008dc6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -172,6 +172,7 @@ class FreqtradeBot(LoggingMixin): self._schedule.every().day.at(t).do(update) self._schedule.every().day.at("00:02").do(self.exchange.ws_connection_reset) + self._schedule.every().day.at("00:07").do(self.wallets.record_wallet_state) self.strategy.ft_bot_start() # Initialize protections AFTER bot start - otherwise parameters are not loaded. @@ -238,7 +239,7 @@ class FreqtradeBot(LoggingMixin): Called on startup and after reloading the bot - triggers notifications and performs startup tasks """ - migrate_live_content(self.config, self.exchange) + migrate_live_content(self.config, self.exchange, self.wallets.get_starting_balance()) set_startup_time() self.rpc.startup_messages(self.config, self.pairlists, self.protections) diff --git a/freqtrade/ft_types/backtest_result_type.py b/freqtrade/ft_types/backtest_result_type.py index b253231a1..768d13517 100644 --- a/freqtrade/ft_types/backtest_result_type.py +++ b/freqtrade/ft_types/backtest_result_type.py @@ -55,6 +55,7 @@ class BacktestContentTypeIcomplete(TypedDict, total=False): backtest_start_time: int backtest_end_time: int run_id: str + wallet_summary: DataFrame class BacktestContentType(BacktestContentTypeIcomplete, total=True): diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index a95b29005..e9edb2569 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -51,6 +51,7 @@ from freqtrade.mixins import LoggingMixin from freqtrade.optimize.backtest_caching import get_strategy_run_id from freqtrade.optimize.bt_progress import BTProgress from freqtrade.optimize.optimize_reports import ( + convert_bt_wallet_collection, generate_backtest_stats, generate_rejected_signals, generate_trade_signal_candles, @@ -137,6 +138,7 @@ class Backtesting: } self.rejected_dict: dict[str, list] = {} self.starting_balance: float = 0.0 + self.wallet_captures: list = [] self._exchange_name = self.config["exchange"]["name"] self.__initial_backtest = exchange is None @@ -451,6 +453,7 @@ class Backtesting: self.replaced_entry_orders = 0 self.canceled_exit_orders = 0 self.replaced_exit_orders = 0 + self.wallet_captures = [] self.dataprovider.clear_cache() if enable_protections: self._load_protections(self.strategy) @@ -1603,6 +1606,7 @@ class Backtesting: pair_detail_cache: dict[str, list[tuple]] = {} pair_tradedir_cache: dict[str, LongShort | None] = {} pairs_with_open_trades = [t.pair for t in LocalTrade.bt_trades_open] + self._capture_wallet(current_time, self.strategy.config["stake_currency"], 1) for current_time_det, is_first, has_detail, idx, pair in self._time_pair_generator_det( current_time, pairs @@ -1627,6 +1631,7 @@ class Backtesting: ) trade_dir = self.check_for_trade_entry(row) pair_tradedir_cache[pair] = trade_dir + self._capture_wallet(current_time, pair.split("/")[0], row[OPEN_IDX]) else: # Detail candle - from cache. @@ -1680,6 +1685,15 @@ class Backtesting: yield current_time_det, pair, row, is_last_row, trade_dir self.progress.increment() + def _capture_wallet(self, current_time: datetime, currency: str, price: float) -> None: + """ + Capture the current wallet state. + """ + if self.dataprovider.runmode != RunMode.BACKTEST: + return + if total := self.wallets.get_total(currency): + self.wallet_captures.append((current_time, currency, price, total)) + def backtest( self, processed: dict, start_date: datetime, end_date: datetime ) -> BacktestContentTypeIcomplete: @@ -1739,6 +1753,7 @@ class Backtesting: "canceled_entry_orders": self.canceled_entry_orders, "replaced_entry_orders": self.replaced_entry_orders, "final_balance": self.wallets.get_total(self.strategy.config["stake_currency"]), + "wallet_summary": convert_bt_wallet_collection(self.wallet_captures), } def backtest_one_strategy( @@ -1867,6 +1882,11 @@ class Backtesting: dt_appendix, market_change_data=combined_res, analysis_results=self.analysis_results, + wallet_summary={ + s: x["wallet_summary"] + for s, x in self.all_bt_content.items() + if "wallet_summary" in x + }, strategy_files={s.get_strategy_name(): s.__file__ for s in self.strategylist}, ) diff --git a/freqtrade/optimize/optimize_reports/__init__.py b/freqtrade/optimize/optimize_reports/__init__.py index 5cf8e51ad..a41a8ebbf 100644 --- a/freqtrade/optimize/optimize_reports/__init__.py +++ b/freqtrade/optimize/optimize_reports/__init__.py @@ -12,6 +12,7 @@ from freqtrade.optimize.optimize_reports.bt_output import ( ) from freqtrade.optimize.optimize_reports.bt_storage import store_backtest_results from freqtrade.optimize.optimize_reports.optimize_reports import ( + convert_bt_wallet_collection, generate_all_periodic_breakdown_stats, generate_backtest_stats, generate_daily_stats, diff --git a/freqtrade/optimize/optimize_reports/bt_output.py b/freqtrade/optimize/optimize_reports/bt_output.py index 338fe5ca5..754ed6b9f 100644 --- a/freqtrade/optimize/optimize_reports/bt_output.py +++ b/freqtrade/optimize/optimize_reports/bt_output.py @@ -287,6 +287,24 @@ def text_table_add_metrics(strat_results: dict) -> None: if "trading_mode" in strat_results else [] ) + wallet_metrics: list[tuple[str, str]] = [] + if wallet_stats := strat_results.get("wallet_stats"): + wallet_metrics = [ + ( + "Min/Max balance realized", + f"{fmt_coin(strat_results['csum_min'], stake)} / " + f"{fmt_coin(strat_results['csum_max'], stake)}", + ), + ( + "Min/Max balance unrealized", + f"{fmt_coin(wallet_stats['low_balance'], stake)} / " + f"{fmt_coin(wallet_stats['high_balance'], stake)}", + ), + ( + "Min/Max balance dates", + f"{wallet_stats['low_date']} / {wallet_stats['high_date']}", + ), + ] # Newly added fields should be ignored if they are missing in strat_results. hyperopt-show # command stores these results and newer version of freqtrade must be able to handle old @@ -408,8 +426,7 @@ def text_table_add_metrics(strat_results: dict) -> None: ), *entry_adjustment_metrics, ("", ""), # Empty line to improve readability - ("Min balance", fmt_coin(strat_results["csum_min"], stake)), - ("Max balance", fmt_coin(strat_results["csum_max"], stake)), + *wallet_metrics, *drawdown_metrics, ("Market change", f"{strat_results['market_change']:.2%}"), ] diff --git a/freqtrade/optimize/optimize_reports/bt_storage.py b/freqtrade/optimize/optimize_reports/bt_storage.py index ef73d4721..c4f20e3e4 100644 --- a/freqtrade/optimize/optimize_reports/bt_storage.py +++ b/freqtrade/optimize/optimize_reports/bt_storage.py @@ -52,6 +52,7 @@ def store_backtest_results( dtappendix: str, *, market_change_data: DataFrame | None = None, + wallet_summary: dict[str, DataFrame] | None = None, analysis_results: dict[str, dict[str, DataFrame]] | None = None, strategy_files: dict[str, str] | None = None, ) -> Path: @@ -123,6 +124,15 @@ def store_backtest_results( market_change_buf.seek(0) zipf.writestr(market_change_name, market_change_buf.getvalue()) + # Add wallet summary if present + if wallet_summary is not None: + for strategy, df in wallet_summary.items(): + wallet_name = f"{base_filename.stem}_{strategy}_wallet.feather" + wallet_buf = BytesIO() + df.reset_index().to_feather(wallet_buf, compression_level=9, compression="lz4") + wallet_buf.seek(0) + zipf.writestr(wallet_name, wallet_buf.getvalue()) + # Add analysis results if present and running in backtest mode if ( config.get("export", "none") == "signals" diff --git a/freqtrade/optimize/optimize_reports/optimize_reports.py b/freqtrade/optimize/optimize_reports/optimize_reports.py index 57b7740d8..bea047202 100644 --- a/freqtrade/optimize/optimize_reports/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports/optimize_reports.py @@ -29,6 +29,48 @@ from freqtrade.util import decimals_per_coin, fmt_coin, format_duration, get_dry logger = logging.getLogger(__name__) +def convert_bt_wallet_collection(wallet_captures: list[tuple]) -> DataFrame: + """ + Convert the wallet capture list to a DataFrame. + Assumes the wallet_captures list contains tuples with the following structure: + (date, currency, price, balance). + """ + if len(wallet_captures) == 0: + return DataFrame() + return DataFrame( + wallet_captures, + columns=["date", "currency", "rate", "balance"], + ) + + +def generate_wallet_stats(wallet_df: DataFrame, stake_currency: str) -> dict[str, Any]: + """Generate wallet statistics from the wallet DataFrame.""" + if wallet_df is None or wallet_df.empty: + return {} + wallet_df.loc[:, "total_quote"] = wallet_df["rate"] * wallet_df["balance"] + # Group by date to get total wallet value at each timestamp + wallet = wallet_df.groupby("date")["total_quote"].sum().reset_index() + total_quote = wallet["total_quote"] + low_idx = total_quote.idxmin() + high_idx = total_quote.idxmax() + start_balance = wallet.iloc[0]["total_quote"] + end_balance = wallet.iloc[-1]["total_quote"] + high_balance = total_quote.loc[high_idx] + low_balance = total_quote.loc[low_idx] + low_date = wallet.loc[low_idx, "date"] + high_date = wallet.loc[high_idx, "date"] + return { + "start_balance": start_balance, + "end_balance": end_balance, + "high_balance": high_balance, + "low_balance": low_balance, + "low_date": low_date.strftime(DATETIME_PRINT_FORMAT), + "low_ts": int(low_date.timestamp() * 1000), + "high_date": high_date.strftime(DATETIME_PRINT_FORMAT), + "high_ts": int(high_date.timestamp() * 1000), + } + + def generate_trade_signal_candles( preprocessed_df: dict[str, DataFrame], bt_results: BacktestContentType, date_col: str ) -> dict[str, DataFrame]: @@ -592,6 +634,7 @@ def generate_strategy_stats( "sharpe": calculate_sharpe(results, min_date, max_date, start_balance), "calmar": calculate_calmar(results, min_date, max_date, start_balance), "sqn": calculate_sqn(results, start_balance), + "wallet_stats": generate_wallet_stats(content.get("wallet_summary"), stake_currency), "profit_factor": profit_factor, "backtest_start": min_date.strftime(DATETIME_PRINT_FORMAT), "backtest_start_ts": int(min_date.timestamp() * 1000), diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py index 3612544ee..4966c0b83 100644 --- a/freqtrade/persistence/__init__.py +++ b/freqtrade/persistence/__init__.py @@ -10,3 +10,4 @@ from freqtrade.persistence.usedb_context import ( disable_database_use, enable_database_use, ) +from freqtrade.persistence.wallet_history import WalletHistory diff --git a/freqtrade/persistence/key_value_store.py b/freqtrade/persistence/key_value_store.py index 310e82b4b..ac3cedcd1 100644 --- a/freqtrade/persistence/key_value_store.py +++ b/freqtrade/persistence/key_value_store.py @@ -22,6 +22,8 @@ KeyStoreKeys = Literal[ "bot_start_time", "startup_time", "binance_migration", + "wallet_history_migration", + "wallet_history_migration_date", ] diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 4d4808eeb..05905abfe 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -20,6 +20,7 @@ from freqtrade.persistence.key_value_store import _KeyValueStoreModel from freqtrade.persistence.migrations import check_migrate from freqtrade.persistence.pairlock import PairLock from freqtrade.persistence.trade_model import Order, Trade +from freqtrade.persistence.wallet_history import WalletHistory logger = logging.getLogger(__name__) @@ -91,6 +92,7 @@ def init_db(db_url: str) -> None: _CustomData.session = scoped_session( sessionmaker(bind=engine, autoflush=True), scopefunc=get_request_or_thread_id ) + WalletHistory.session = Trade.session previous_tables = inspect(engine).get_table_names() ModelBase.metadata.create_all(engine) diff --git a/freqtrade/persistence/wallet_history.py b/freqtrade/persistence/wallet_history.py new file mode 100644 index 000000000..2aef7000d --- /dev/null +++ b/freqtrade/persistence/wallet_history.py @@ -0,0 +1,50 @@ +from datetime import datetime +from typing import ClassVar + +from sqlalchemy import DateTime, Float, Integer, String, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from freqtrade.persistence.base import ModelBase, SessionType + + +class WalletHistory(ModelBase): + """ + Daily wallet state tracking with minimal fields + """ + + __tablename__ = "wallet_history" + session: ClassVar[SessionType] + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + timestamp: Mapped[datetime] = mapped_column(DateTime, nullable=False, index=True) + currency: Mapped[str] = mapped_column(String(25), nullable=False) + # Rate: price of 1 unit of `currency` quoted in `quote_currency`. + # e.g., USDT/ETH -> USDT per ETH + rate: Mapped[float] = mapped_column(Float, nullable=True) + # Quote currency for rate/total fields (e.g., 'USDT') + quote_currency: Mapped[str] = mapped_column(String(25), nullable=False) + + # Balance in `currency` units + balance: Mapped[float] = mapped_column(Float, nullable=False) + + # Canonical total wallet equity/value denominated in `quote_currency` (if available) + # For futures positions, collateral + PnL is used to compute this value. + total_quote: Mapped[float] = mapped_column(Float, nullable=True) + # Total position value in `quote_currency` - including leverage + total_position_value: Mapped[float] = mapped_column(Float, nullable=True) + collateral: Mapped[float] = mapped_column(Float, nullable=True) + leverage: Mapped[float] = mapped_column(Float, nullable=False, default=1.0) + + bot_managed: Mapped[bool] = mapped_column(nullable=False, default=True) + + __table_args__ = ( + # Ensure one record per currency per day + UniqueConstraint("timestamp", "currency", name="unique_wallet_daily"), + ) + + def __repr__(self) -> str: + return ( + f"WalletHistory(timestamp={self.timestamp}, currency={self.currency}, " + f"rate={self.rate}, total_quote={self.total_quote}, " + f"balance={self.balance}, leverage={self.leverage})" + ) diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index 428fea1c9..afbd44101 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -16,10 +16,11 @@ from freqtrade.data.btanalysis import ( get_backtest_market_change, get_backtest_result, get_backtest_resultlist, + get_backtest_wallet_change, load_and_merge_backtest_result, update_backtest_metadata, ) -from freqtrade.enums import BacktestState +from freqtrade.enums import BacktestState, RunMode from freqtrade.exceptions import ConfigurationError, DependencyException, OperationalException from freqtrade.ft_types import get_BacktestResultType_default from freqtrade.misc import deep_merge_dicts, is_file_in_dir @@ -29,6 +30,7 @@ from freqtrade.rpc.api_server.api_schemas import ( BacktestMetadataUpdate, BacktestRequest, BacktestResponse, + WalletHistoryResponse, ) from freqtrade.rpc.api_server.deps import get_config, verify_strategy from freqtrade.rpc.api_server.webserver_bgwork import ApiBG @@ -106,6 +108,11 @@ def __run_backtest_bg(btconfig: Config): ApiBG.bt["bt"].results, datetime.now().strftime("%Y-%m-%d_%H-%M-%S"), market_change_data=combined_res, + wallet_summary={ + s: x["wallet_summary"] + for s, x in ApiBG.bt["bt"].all_bt_content.items() + if "wallet_summary" in x + }, strategy_files={ s.get_strategy_name(): s.__file__ for s in ApiBG.bt["bt"].strategylist }, @@ -137,6 +144,7 @@ async def api_start_backtest( verify_strategy(bt_settings.strategy) btconfig = deepcopy(config) + btconfig["runmode"] = RunMode.BACKTEST remove_exchange_credentials(btconfig["exchange"], True) settings = dict(bt_settings) if settings.get("freqai", None) is not None: @@ -354,3 +362,29 @@ def api_get_backtest_market_change(file: str, config=Depends(get_config)): "data": df.values.tolist(), "length": len(df), } + + +@router.get( + "/backtest/history/{file}/{strategy}/wallet", + response_model=WalletHistoryResponse, + tags=["webserver", "backtest"], +) +def api_get_backtest_wallet(file: str, strategy: str, config=Depends(get_config)): + bt_results_base: Path = config["user_data_dir"] / "backtest_results" + file_abs = (bt_results_base / file).with_suffix(".zip") + # Ensure file is in backtest_results directory + if not is_file_in_dir(file_abs, bt_results_base): + raise HTTPException(status_code=400, detail="Unable to retrieve wallet history.") + + results = get_backtest_wallet_change(file_abs, strategy) + if results is None: + raise HTTPException(status_code=404, detail="Unable to retrieve wallet history.") + # Consolidate the wallet to the base currency + results.loc[:, "total_quote"] = results["rate"] * results["balance"] + results = results.groupby(["date", "__date_ts"]).agg({"total_quote": "sum"}).reset_index() + + return { + "columns": results.columns.tolist(), + "data": results.values.tolist(), + "length": len(results), + } diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 7952d5724..722438ae3 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -679,6 +679,15 @@ class BacktestMarketChange(BaseModel): data: list[list[Any]] +class WalletHistoryResponse(BaseModel): + columns: list[str] + length: int + data: list[list[Any]] + # start date of the effectively captured data + # Before this date, it's based on a reconstructed wallet history + capture_start_ts: int | None = None + + class MarketRequest(ExchangeModePayloadMixin, BaseModel): base: str | None = None quote: str | None = None diff --git a/freqtrade/rpc/api_server/api_trading.py b/freqtrade/rpc/api_server/api_trading.py index 3ec7a08b3..0bae0eefb 100644 --- a/freqtrade/rpc/api_server/api_trading.py +++ b/freqtrade/rpc/api_server/api_trading.py @@ -31,6 +31,7 @@ from freqtrade.rpc.api_server.api_schemas import ( ResultMsg, Stats, StatusMsg, + WalletHistoryResponse, WhitelistResponse, ) from freqtrade.rpc.api_server.deps import get_config, get_rpc @@ -104,6 +105,22 @@ def stats(rpc: RPC = Depends(get_rpc)): return rpc._rpc_stats() +@router.get( + "/historic_balance", + response_model=WalletHistoryResponse, + tags=["info"], +) +def api_get_wallet_history(rpc: RPC = Depends(get_rpc)): + results, capture_date_ts = rpc._rpc_get_historic_balance() + + return { + "columns": results.columns.tolist(), + "data": results.values.tolist(), + "length": len(results), + "capture_start_ts": capture_date_ts, + } + + @router.get("/daily", response_model=DailyWeeklyMonthly, tags=["Trading-info"]) def daily( timescale: int = Query(7, ge=1, description="Number of days to fetch data for"), diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index d25bda78e..64283898f 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -69,7 +69,8 @@ logger = logging.getLogger(__name__) # 2.45: Add price to forceexit endpoint # 2.46: Add prepend_data to download-data endpoint # 2.47: Add Strategy parameters -API_VERSION = 2.47 +# 2.48: add /backtest/history/wallets endpoint +API_VERSION = 2.48 # Public API, requires no auth. router_public = APIRouter() diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 37b8dfa6d..0c5f46f01 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -12,7 +12,7 @@ import psutil from dateutil.relativedelta import relativedelta from dateutil.tz import tzlocal from numpy import inf, int64, isnan, mean, nan -from pandas import DataFrame, NaT +from pandas import DataFrame, NaT, read_sql from sqlalchemy import func, select from freqtrade import __version__ @@ -785,6 +785,26 @@ class RPC: "bot_start_date": format_date(bot_start), } + def _rpc_get_historic_balance(self) -> tuple[DataFrame, int]: + """ + Returns the historic balance of the bot + :return: DataFrame with the balance history and the timestamp of the migration + """ + results = read_sql("wallet_history", con=Trade.session.bind, parse_dates=["timestamp"]) + + results = results.rename({"timestamp": "date"}, axis=1) + results.loc[:, "__date_ts"] = results.loc[:, "date"].astype("int64") // 1000 // 1000 + # Exclude non-bot managed for now + results_filtered = results.loc[results["bot_managed"]] + + results_final = ( + results_filtered.groupby(["date", "__date_ts"]) + .agg({"total_quote": "sum"}) + .reset_index() + ) + hist = KeyValueStore.get_datetime_value("wallet_history_migration_date") + return results_final, dt_ts_def(hist, 0) + def __balance_get_est_stake( self, coin: str, stake_currency: str, amount: float, balance: Wallet ) -> tuple[float, float]: diff --git a/freqtrade/util/migrations/__init__.py b/freqtrade/util/migrations/__init__.py index 20aafb04b..90f866075 100644 --- a/freqtrade/util/migrations/__init__.py +++ b/freqtrade/util/migrations/__init__.py @@ -1,8 +1,10 @@ +from freqtrade.constants import Config from freqtrade.exchange import Exchange from freqtrade.util.migrations.funding_rate_mig import migrate_funding_fee_timeframe +from freqtrade.util.migrations.migrate_wallet_history import migrate_wallet_history -def migrate_data(config, exchange: Exchange | None = None) -> None: +def migrate_data(config: Config, exchange: Exchange | None = None) -> None: """ Migrate persisted data from old formats to new formats """ @@ -10,10 +12,9 @@ def migrate_data(config, exchange: Exchange | None = None) -> None: migrate_funding_fee_timeframe(config, exchange) -def migrate_live_content(config, exchange: Exchange | None = None) -> None: +def migrate_live_content(config: Config, exchange: Exchange, starting_balance: float) -> None: """ Migrate database content from old formats to new formats Used for dry/live mode. """ - # Currently not used - pass + migrate_wallet_history(config, exchange, starting_balance) diff --git a/freqtrade/util/migrations/migrate_wallet_history.py b/freqtrade/util/migrations/migrate_wallet_history.py new file mode 100644 index 000000000..dbc887c56 --- /dev/null +++ b/freqtrade/util/migrations/migrate_wallet_history.py @@ -0,0 +1,214 @@ +import logging + +import numpy as np +import pandas as pd + +from freqtrade.constants import Config +from freqtrade.data.btanalysis.bt_fileutils import trade_list_to_dataframe +from freqtrade.data.btanalysis.trade_parallelism import balance_distribution_over_time +from freqtrade.exchange import Exchange +from freqtrade.exchange.exchange_utils_timeframe import timeframe_to_prev_date +from freqtrade.persistence import KeyValueStore, Trade, WalletHistory +from freqtrade.util import dt_now, dt_ts + + +logger = logging.getLogger(__name__) + + +def migrate_wallet_history(config: Config, exchange: Exchange, starting_balance: float): + if config.get("skip_wallet_history_migration") or not exchange.get_option( + "ohlcv_has_history", True + ): + # we can't fill up wallet history without ohlcv history + return + if KeyValueStore.get_int_value("wallet_history_migration"): + logger.debug("Wallet history migration already completed.") + return + logger.info("Starting wallet history migration...") + _migrate_wallet_history(config, exchange, starting_balance) + logger.info("Wallet history migration completed.") + KeyValueStore.store_value("wallet_history_migration", 1) + KeyValueStore.store_value("wallet_history_migration_date", dt_now()) + + +def _migrate_wallet_history(config: Config, exchange: Exchange, starting_balance: float): + # Prepare balance distribution data with OHLCV rates + balance_dist, pairlist_valid = _prepare_balance_distribution(config, exchange, starting_balance) + if not balance_dist.empty and pairlist_valid: + _create_wallet_history_entries( + config, exchange, balance_dist, pairlist_valid, config["stake_currency"] + ) + + +def _prepare_balance_distribution( + config: Config, exchange: Exchange, starting_balance: float +) -> tuple[pd.DataFrame, list[str]]: + trade_df = trade_list_to_dataframe(Trade.get_trades_proxy(), minified=False) + if trade_df.empty: + # no trades, nothing to do + return pd.DataFrame(), [] + pairlist = list(trade_df["pair"].unique()) + timeframe = "1d" + stake_currency = config["stake_currency"] + min_date = timeframe_to_prev_date(timeframe, KeyValueStore.get_datetime_value("bot_start_time")) + balance_dist = balance_distribution_over_time( + trade_df, + min_date=min_date, + max_date=dt_now(), + start_balance=starting_balance, + stake_currency=stake_currency, + timeframe=timeframe, + pairlist=pairlist, + ) + pairlist_valid = [p for p in pairlist if p in exchange.markets] + pairlist_invalid = set(pairlist) - set(pairlist_valid) + if pairlist_invalid: + logger.warning( + f"The following trading pairs from the trade history are not available on the exchange " + f"and will be skipped during wallet history migration: {', '.join(pairlist_invalid)}" + ) + + logger.info("Wallet History migration: Fetching OHLCV data ...") + data = exchange.refresh_latest_ohlcv( + [(p, timeframe, config["candle_type_def"]) for p in pairlist_valid], + since_ms=dt_ts(min_date), + cache=False, + drop_incomplete=False, + ) + logger.info( + "Wallet History migration: Done fetching OHLCV data for wallet history migration..." + ) + + dfs = [] + # Combine all dataframes into one using the open rate + for p, x in data.items(): + x = x.set_index("date", drop=True) + col = f"{p[0]}_open" + x[col] = x["open"] + dfs.append(x[[col]]) + + if not dfs: + logger.warning( + "No OHLCV data available for the trading pairs; skipping wallet history migration." + ) + return pd.DataFrame(), [] + merged = pd.concat(dfs, axis=1) + + balance_dist = balance_dist.join(merged, how="left") + df_value = pd.DataFrame( + index=balance_dist.index, columns=[f"{p}_value" for p in pairlist_valid], dtype=float + ) + for p in pairlist_valid: + # df_value[f"{p}_value"] = balance_dist[f"{p}_open"] * balance_dist[p] + # Identical calculation to rpc and wallets.py + df_value[f"{p}_value"] = np.where( + balance_dist[f"{p}_is_short"] == 0, + (balance_dist[f"{p}_open"] * balance_dist[p]) + - balance_dist[f"{p}_collateral"] * (balance_dist[f"{p}_leverage"] - 1), + ( + balance_dist[f"{p}_collateral"] * (1 + balance_dist[f"{p}_leverage"]) + - balance_dist[f"{p}_open"] * balance_dist[p] + ), + ) + balance_dist = pd.concat([balance_dist, df_value], axis=1) + + # Aggregate total value at each point in time + balance_dist["total_value"] = balance_dist[ + [f"{p}_value" for p in pairlist_valid] + [stake_currency] + ].sum(axis=1) + + return balance_dist, pairlist_valid + + +def _create_wallet_history_entries( + config: Config, + exchange: Exchange, + balance_dist: pd.DataFrame, + pairlist_valid: list[str], + stake_currency: str, +): + is_futures = config["trading_mode"] == "futures" + # Precompute column indices for faster tuple-based iteration + # Assume the first column is the index (date) + stake_idx = balance_dist.columns.get_loc(stake_currency) + pair_balance_idx = {pair: balance_dist.columns.get_loc(pair) + 1 for pair in pairlist_valid} + pair_leverage_idx = { + pair: balance_dist.columns.get_loc(f"{pair}_leverage") + 1 for pair in pairlist_valid + } + pair_collateral_idx = { + pair: balance_dist.columns.get_loc(f"{pair}_collateral") + 1 for pair in pairlist_valid + } + pair_is_short_idx = { + pair: balance_dist.columns.get_loc(f"{pair}_is_short") + 1 for pair in pairlist_valid + } + pair_rate_idx = { + pair: balance_dist.columns.get_loc(f"{pair}_open") + 1 for pair in pairlist_valid + } + # Convert balance_dist to WalletHistory entries + wallet_entries = [] + for row in balance_dist.itertuples(index=True, name=None): + date = row[0] + + # Add stake currency entry + stake_balance = row[stake_idx + 1] + if not pd.isna(stake_balance): + wallet_entries.append( + WalletHistory( + timestamp=date, + currency=stake_currency, + rate=1.0, # Stake currency price is always 1.0 + balance=stake_balance, + total_quote=stake_balance, + quote_currency=stake_currency, + leverage=1.0, + bot_managed=True, + ) + ) + + # Add entries for each trading pair + for pair in pairlist_valid: + base_currency = exchange.get_pair_base_currency(pair) + balance = row[pair_balance_idx[pair]] + leverage = row[pair_leverage_idx[pair]] + # Only add entry if balance is not empty/NaN + if not pd.isna(balance) and balance > 0: + rate_value = row[pair_rate_idx[pair]] + rate = rate_value if not pd.isna(rate_value) else None + + total_quote = balance * rate if rate else None + collateral: float | None = None + if is_futures: + collateral = row[pair_collateral_idx[pair]] + is_short = row[pair_is_short_idx[pair]] + if collateral is not None and not pd.isna(collateral): + # Same formula than in rpc's _rpc_balance + total_quote = ( + (rate * balance - collateral * (leverage - 1)) + if is_short == 0 + else (collateral * (1 + leverage) - rate * balance) + ) + wallet_entries.append( + WalletHistory( + timestamp=date, + currency=base_currency, + quote_currency=stake_currency, + rate=rate, + balance=balance, + total_quote=total_quote, + leverage=leverage if not pd.isna(leverage) else 1.0, + bot_managed=True, + total_position_value=balance * rate if is_futures and rate else None, + # collateral=collateral, + ) + ) + + # Save entries to database + if wallet_entries: + try: + # Use bulk_save_objects for better performance + WalletHistory.session.bulk_save_objects(wallet_entries) + WalletHistory.session.commit() + logger.info(f"Successfully created {len(wallet_entries)} wallet balance records") + except Exception as e: + WalletHistory.session.rollback() + logger.error(f"Error saving wallet balance records: {e}") diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 56c32adb6..e3ac1288b 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -10,8 +10,8 @@ from freqtrade.enums import RunMode, TradingMode from freqtrade.exceptions import DependencyException from freqtrade.exchange import Exchange from freqtrade.misc import safe_value_fallback -from freqtrade.persistence import LocalTrade, Trade -from freqtrade.util.datetime_helpers import dt_now +from freqtrade.persistence import LocalTrade, Trade, WalletHistory +from freqtrade.util import dt_floor_day, dt_now logger = logging.getLogger(__name__) @@ -445,3 +445,71 @@ class Wallets: logger.debug(msg) else: logger.info(msg) + + def record_wallet_state(self) -> None: + """Record daily wallet totals to database""" + if self._is_backtest: + # only record in live mode. + return + timestamp = dt_floor_day(dt_now()) + + # Record total balances for all currencies + wallet_records = [] + position_collaterals = 0.0 + open_assets: dict[str, Trade] = {t.safe_base_currency: t for t in Trade.get_open_trades()} + for pos in self.get_all_positions().values(): + base = self._exchange.get_pair_base_currency(pos.symbol) + rate = self._exchange.get_conversion_rate(base, self._stake_currency) + total_quote = None + leverage = pos.leverage or 1.0 + if rate: + # Same formula than in rpc's _rpc_balance + total_quote = ( + rate * pos.position - pos.collateral * (leverage - 1) + if pos.side == "long" + else pos.collateral * (1 + leverage) - rate * pos.position + ) + + position_record = WalletHistory( + timestamp=timestamp, + currency=pos.symbol, + quote_currency=self._stake_currency, + rate=rate, + balance=pos.position, + total_quote=total_quote, + total_position_value=rate * pos.position if rate else None, + collateral=pos.collateral, + leverage=leverage, + bot_managed=base in open_assets, + ) + position_collaterals += pos.collateral + wallet_records.append(position_record) + + for wallet in self.get_all_balances().values(): + # TODO: (needs decision) exclude minimal balances? + rate = self._exchange.get_conversion_rate(wallet.currency, self._stake_currency) + is_bot_managed = ( + self._stake_currency == wallet.currency or wallet.currency in open_assets + ) + balance = wallet.total - ( + position_collaterals if wallet.currency == self._stake_currency else 0 + ) + total_quote = rate * balance if rate else None + + wallet_record = WalletHistory( + timestamp=timestamp, + currency=wallet.currency, + quote_currency=self._stake_currency, + rate=rate, + balance=balance, + leverage=1.0, + total_quote=total_quote, + bot_managed=is_bot_managed, + ) + wallet_records.append(wallet_record) + try: + WalletHistory.session.bulk_save_objects(wallet_records) + WalletHistory.session.commit() + except Exception as e: + WalletHistory.session.rollback() + logger.error(f"Error saving wallet balance records: {e}") diff --git a/tests/conftest.py b/tests/conftest.py index abd15a6a1..93d34fe18 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -169,10 +169,12 @@ def generate_trades_history(n_rows, start_date: datetime | None = None, days=5): return df -def generate_test_data(timeframe: str, size: int, start: str = "2020-07-05", random_seed=42): +def generate_test_data( + timeframe: str, size: int, start: str = "2020-07-05", random_seed=42, base=20 +): np.random.seed(random_seed) - base = np.random.normal(20, 2, size=size) + base = np.random.normal(base, 2, size=size) if timeframe == "1y": date = pd.date_range(start, periods=size, freq="1YS", tz="UTC") elif timeframe == "1M": diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index c869e6a92..82ff56c3a 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -10,8 +10,9 @@ from freqtrade.configuration import TimeRange from freqtrade.constants import LAST_BT_RESULT_FN from freqtrade.data.btanalysis import ( BT_DATA_COLUMNS, - analyze_trade_parallelism, extract_trades_of_period, + get_backtest_market_change, + get_backtest_wallet_change, get_latest_backtest_filename, get_latest_hyperopt_file, load_backtest_data, @@ -209,17 +210,6 @@ def test_extract_trades_of_period(testdatadir): assert trades1.iloc[-1].close_date == datetime(2017, 11, 14, 15, 25, 0, tzinfo=UTC) -def test_analyze_trade_parallelism(testdatadir): - filename = testdatadir / "backtest_results/backtest-result.json" - bt_data = load_backtest_data(filename) - - res = analyze_trade_parallelism(bt_data, "5m") - assert isinstance(res, DataFrame) - assert "open_trades" in res.columns - assert res["open_trades"].max() == 3 - assert res["open_trades"].min() == 0 - - def test_load_trades(default_conf, mocker): db_mock = mocker.patch( "freqtrade.data.btanalysis.bt_fileutils.load_trades_from_db", MagicMock() @@ -649,3 +639,56 @@ def test_load_file_from_zip(tmp_path): with pytest.raises(ValueError, match=r"File .* not found in zip.*"): load_file_from_zip(zip_file, "testfile55.txt") + + +def test_get_backtest_market_change(tmp_path): + df = DataFrame( + { + "date": [dt_utc(2020, 1, 1), dt_utc(2020, 1, 2)], + "price": [100.0, 110.0], + } + ) + feather_file = tmp_path / "backtest-result_market_change.feather" + df.to_feather(feather_file) + + direct_df = get_backtest_market_change(feather_file) + assert isinstance(direct_df, DataFrame) + assert "__date_ts" in direct_df.columns + assert direct_df.loc[0, "__date_ts"] == int(df.loc[0, "date"].timestamp() * 1000) + + no_ts_df = get_backtest_market_change(feather_file, include_ts=False) + assert "__date_ts" not in no_ts_df.columns + + zip_file = tmp_path / "backtest-result.zip" + with ZipFile(zip_file, "w") as zipf: + zipf.write(feather_file, arcname=f"{zip_file.stem}_market_change.feather") + + zipped_df = get_backtest_market_change(zip_file) + assert isinstance(zipped_df, DataFrame) + assert zipped_df.loc[0, "__date_ts"] == int(df.loc[0, "date"].timestamp() * 1000) + assert list(zipped_df["price"]) == [100.0, 110.0] + + +def test_get_backtest_wallet_change(tmp_path): + df = DataFrame( + { + "date": [dt_utc(2020, 1, 1), dt_utc(2020, 1, 2)], + "balance": [1.0, 1.1], + "rate": [1.0, 1.1], + } + ) + wallet_feather = tmp_path / "backtest-result_TestStrategy_wallet.feather" + df.to_feather(wallet_feather) + + zip_file = tmp_path / "backtest-result.zip" + with ZipFile(zip_file, "w") as zipf: + zipf.write(wallet_feather, arcname=wallet_feather.name) + + wallet_df = get_backtest_wallet_change(zip_file, "TestStrategy") + assert isinstance(wallet_df, DataFrame) + assert "__date_ts" in wallet_df.columns + assert wallet_df.loc[0, "__date_ts"] == int(df.loc[0, "date"].timestamp() * 1000) + assert list(wallet_df["balance"]) == [1.0, 1.1] + + assert get_backtest_wallet_change(tmp_path / "backtest-result.feather", "TestStrategy") is None + assert get_backtest_wallet_change(zip_file, "UnknownStrategy") is None diff --git a/tests/data/test_trade_parallelism.py b/tests/data/test_trade_parallelism.py new file mode 100644 index 000000000..266c59446 --- /dev/null +++ b/tests/data/test_trade_parallelism.py @@ -0,0 +1,210 @@ +from datetime import timedelta + +import pytest +from pandas import DataFrame, Timestamp + +from freqtrade.data.btanalysis import ( + analyze_trade_parallelism, + load_backtest_data, +) +from freqtrade.data.btanalysis.trade_parallelism import balance_distribution_over_time +from freqtrade.util import dt_utc + + +def test_analyze_trade_parallelism(testdatadir): + filename = testdatadir / "backtest_results/backtest-result.json" + bt_data = load_backtest_data(filename) + + res = analyze_trade_parallelism(bt_data, "5m") + assert isinstance(res, DataFrame) + assert "open_trades" in res.columns + assert res["open_trades"].max() == 3 + assert res["open_trades"].min() == 0 + + +@pytest.mark.parametrize("is_short", [False, True]) +def test_balance_distribution_over_time(is_short): + """ + Test balance_distribution_over_time for both long and short trades. + """ + # Create a minimal trades DataFrame with 4 trades over time + # Base dates for trades + start_date = dt_utc(2023, 1, 1) + base_date = start_date + timedelta(hours=15) + stake_currency = "USDT" + start_balance = 1000.0 + fee = 0.001 # 0.1% fee + + # Create trades spanning different time periods + trades_data = { + "pair": ["BTC/USDT", "ETH/USDT", "XRP/USDT", "LTC/USDT"], + "stake_amount": [100.0, 150.0, 80.0, 120.0], + "open_date": [ + base_date, + base_date + timedelta(hours=2), + base_date + timedelta(hours=5), + base_date + timedelta(hours=8), + ], + "close_date": [ + base_date + timedelta(hours=3), + base_date + timedelta(hours=6), + base_date + timedelta(hours=9), + base_date + timedelta(hours=12), + ], + "open_rate": [40000.0, 2000.0, 0.5, 100.0], + "close_rate": [41000.0, 2100.0, 0.52, 105.0], + "fee_open": [fee, fee, fee, fee], + "fee_close": [fee, fee, fee, fee], + "is_short": [is_short, is_short, is_short, is_short], + "leverage": [1.0, 1.0, 1.0, 1.0], + "orders": [ + # Trade 1: BTC/USDT - entry at 40000, exit at 41000 + [ + { + "amount": 0.0025, # 100 / 40000 + "filled": 0.0025, + "safe_price": 40000.0, + "ft_order_side": "sell" if is_short else "buy", + "order_filled_timestamp": int(base_date.timestamp() * 1000), + "ft_is_entry": True, + }, + { + "amount": 0.0025, + "filled": 0.0025, + "safe_price": 41000.0, + "ft_order_side": "buy" if is_short else "sell", + "order_filled_timestamp": int( + (base_date + timedelta(hours=3)).timestamp() * 1000 + ), + "ft_is_entry": False, + }, + ], + # Trade 2: ETH/USDT - entry at 2000, exit at 2100 + [ + { + "amount": 0.075, # 150 / 2000 + "filled": 0.075, + "safe_price": 2000.0, + "ft_order_side": "sell" if is_short else "buy", + "order_filled_timestamp": int( + (base_date + timedelta(hours=2)).timestamp() * 1000 + ), + "ft_is_entry": True, + }, + { + "amount": 0.075, + "filled": 0.075, + "safe_price": 2100.0, + "ft_order_side": "buy" if is_short else "sell", + "order_filled_timestamp": int( + (base_date + timedelta(hours=6)).timestamp() * 1000 + ), + "ft_is_entry": False, + }, + ], + # Trade 3: XRP/USDT - entry at 0.5, exit at 0.52 + [ + { + "amount": 160.0, # 80 / 0.5 + "filled": 160.0, + "safe_price": 0.5, + "ft_order_side": "sell" if is_short else "buy", + "order_filled_timestamp": int( + (base_date + timedelta(hours=5)).timestamp() * 1000 + ), + "ft_is_entry": True, + }, + { + "amount": 160.0, + "filled": 160.0, + "safe_price": 0.52, + "ft_order_side": "buy" if is_short else "sell", + "order_filled_timestamp": int( + (base_date + timedelta(hours=9)).timestamp() * 1000 + ), + "ft_is_entry": False, + }, + ], + # Trade 4: LTC/USDT - entry at 100, exit at 105 + [ + { + "amount": 1.2, # 120 / 100 + "filled": 1.2, + "safe_price": 100.0, + "ft_order_side": "sell" if is_short else "buy", + "order_filled_timestamp": int( + (base_date + timedelta(hours=8)).timestamp() * 1000 + ), + "ft_is_entry": True, + }, + { + "amount": 1.2, + "filled": 1.2, + "safe_price": 105.0, + "ft_order_side": "buy" if is_short else "sell", + "order_filled_timestamp": int( + (base_date + timedelta(hours=12)).timestamp() * 1000 + ), + "ft_is_entry": False, + }, + ], + ], + } + + trades_df = DataFrame(trades_data) + pairlist = ["BTC/USDT", "ETH/USDT", "XRP/USDT", "LTC/USDT"] + + min_date = start_date + max_date = start_date + timedelta(hours=35) + + result = balance_distribution_over_time( + trades=trades_df, + min_date=min_date, + max_date=max_date, + timeframe="1h", + stake_currency=stake_currency, + start_balance=start_balance, + pairlist=pairlist, + ) + + # Verify basic structure + assert isinstance(result, DataFrame) + assert stake_currency in result.columns + for pair in pairlist: + assert pair in result.columns + assert f"{pair}_leverage" in result.columns + assert f"{pair}_is_short" in result.columns + assert f"{pair}_collateral" in result.columns + + # Verify the index is a DatetimeIndex + assert isinstance(result.index, Timestamp.__class__.__bases__[0]) + + # Verify we have entries over the full time period (36h) + assert len(result) == 36 + + # First trade opens 15h after the start date + assert result.iloc[0][stake_currency] == 1000 + expected_first_balance = start_balance - (100.0 + 100.0 * fee) + assert result.iloc[15][stake_currency] == pytest.approx(expected_first_balance) + + # Check that pair columns have non-zero values during trade periods + # Trade 1 (BTC/USDT) is open from hour 15 to hour 18 + # At hour 16, BTC/USDT should have position + btc_during_trade = result.loc[base_date + timedelta(hours=1), "BTC/USDT"] + assert btc_during_trade > 0, "Trade should have positive position during open period" + + # After Trade 1 closes at hour 3, BTC/USDT position should be 0 + btc_after_close = result.loc[base_date + timedelta(hours=4) :, "BTC/USDT"] + assert all(btc_after_close == 0), "Position should be 0 after trade closes" + + # Final stake currency should reflect all trades' cash flows minus fees + final_balance = result.iloc[-1][stake_currency] + + # Verify the balance changed (trades had effect) + assert final_balance != start_balance, "Balance should change after trading" + + # Since all exit prices > entry prices, exits return more cash than entries spent + # This means final balance > start balance for long trades and < start balance for short trades + assert (final_balance > start_balance) if not is_short else (final_balance < start_balance), ( + "Balance increases for long and decreases for short trades" + ) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index e051e6b3c..70b591d54 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -757,10 +757,12 @@ def test_backtest__check_trade_exit(default_conf, mocker) -> None: def test_backtest_one(default_conf, mocker, testdatadir) -> None: default_conf["use_exit_signal"] = False default_conf["max_open_trades"] = 10 + default_conf["runmode"] = RunMode.BACKTEST patch_exchange(mocker) mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=0.00001) mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=float("inf")) + mocker.patch(f"{EXMS}.get_pair_base_currency", lambda _, x: x.split("/")[0]) backtesting = Backtesting(default_conf) backtesting._set_strategy(backtesting.strategylist[0]) pair = "UNITTEST/BTC" @@ -875,13 +877,23 @@ def test_backtest_one(default_conf, mocker, testdatadir) -> None: ln1.iloc[0]["low"], 6 ) < round(t["close_rate"], 6) < round(ln1.iloc[0]["high"], 6) + wallet_summary = result["wallet_summary"] + assert isinstance(wallet_summary, pd.DataFrame) + assert len(wallet_summary) == 255 + unique_currencies = wallet_summary["currency"].value_counts() + assert unique_currencies["BTC"] == 200 + assert unique_currencies["UNITTEST"] == 55 + @pytest.mark.parametrize("use_detail", [True, False]) def test_backtest_one_detail(default_conf_usdt, mocker, testdatadir, use_detail) -> None: default_conf_usdt["use_exit_signal"] = False + default_conf_usdt["runmode"] = RunMode.BACKTEST patch_exchange(mocker) mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=0.00001) mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=float("inf")) + mocker.patch(f"{EXMS}.get_pair_base_currency", lambda _, x: x.split("/")[0]) + default_conf_usdt["unfilledtimeout"] = { "entry": 11, "exit": 30, @@ -968,6 +980,12 @@ def test_backtest_one_detail(default_conf_usdt, mocker, testdatadir, use_detail) ) assert late_entry > 0 + wallet_summary = result["wallet_summary"] + assert isinstance(wallet_summary, pd.DataFrame) + assert len(wallet_summary) == 591 if use_detail else 597 + unique_currencies = wallet_summary["currency"].value_counts() + assert unique_currencies["USDT"] == 576 + assert unique_currencies["XRP"] == 15 if use_detail else 21 @pytest.mark.parametrize( diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 1247d4c02..63a1d6efd 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -7,8 +7,10 @@ import logging import time from copy import deepcopy from datetime import UTC, datetime, timedelta +from io import BytesIO from pathlib import Path from unittest.mock import ANY, MagicMock, PropertyMock, patch +from zipfile import ZipFile import pandas as pd import pytest @@ -1434,6 +1436,41 @@ def test_api_stats(botclient, mocker, ticker, fee, markets, is_short): assert "draws" in rc.json()["durations"] +@pytest.mark.parametrize("is_short", [True, False]) +def test_api_historic_balance(botclient, mocker, ticker, fee, markets, is_short): + ftbot, client = botclient + patch_get_signal(ftbot, enter_long=not is_short, enter_short=is_short) + mocker.patch.multiple( + EXMS, + get_balances=MagicMock(return_value=ticker), + fetch_ticker=ticker, + get_fee=fee, + markets=PropertyMock(return_value=markets), + ) + + rc = client_get(client, f"{BASE_URI}/historic_balance") + assert_response(rc, 200) + resp = rc.json() + assert "columns" in resp + assert "data" in resp + assert "length" in resp + assert "capture_start_ts" in resp + assert resp["length"] == 0 + + ftbot.wallets.record_wallet_state() + + rc = client_get(client, f"{BASE_URI}/historic_balance") + assert_response(rc, 200) + resp1 = rc.json() + assert "columns" in resp1 + assert "data" in resp1 + assert "length" in resp1 + assert "capture_start_ts" in resp1 + assert resp1["length"] == 1 + assert "__date_ts" in resp1["columns"] + assert "total_quote" in resp1["columns"] + + def test_api_performance(botclient, fee): ftbot, client = botclient patch_get_signal(ftbot) @@ -3275,7 +3312,7 @@ def test_api_patch_backtest_history_entry(botclient, tmp_path: Path): assert fileres[CURRENT_TEST_STRATEGY]["notes"] == "FooBar" -def test_api_patch_backtest_market_change(botclient, tmp_path: Path): +def test_api_backtest_market_change(botclient, tmp_path: Path): ftbot, client = botclient # Create a temporary directory and file @@ -3313,6 +3350,55 @@ def test_api_patch_backtest_market_change(botclient, tmp_path: Path): ] +def test_api_backtest_wallets(botclient, tmp_path: Path): + ftbot, client = botclient + + # Create a temporary directory and file + bt_results_base = tmp_path / "backtest_results" + bt_results_base.mkdir() + zip_file = bt_results_base / "backtest_15.zip" + with ZipFile(zip_file, "w") as zipf: + wallet_df = pd.DataFrame( + { + "date": [ + "2018-01-01T00:00:00Z", + "2018-01-01T00:00:00Z", + "2018-01-01T00:05:00Z", + "2018-01-01T00:05:00Z", + ], + "currency": ["ETH", "BTC", "ETH", "BTC"], + "rate": [2000, 60_000, 2001, 60_001], + "balance": [0.5, 0.25, 0.5, 0.25], + } + ) + wallet_df["date"] = pd.to_datetime(wallet_df["date"]) + wallet_buf = BytesIO() + wallet_df.reset_index().to_feather(wallet_buf, compression_level=9, compression="lz4") + wallet_buf.seek(0) + zipf.writestr("backtest_15_SampleStrategy_wallet.feather", wallet_buf.read()) + + # Wrong basedirectory + rc = client_get(client, f"{BASE_URI}/backtest/history/randomFile.json/SampleStrategy/wallet") + assert_response(rc, 503) + + ftbot.config["user_data_dir"] = tmp_path + ftbot.config["runmode"] = RunMode.WEBSERVER + + # Nonexisting file - fails "is_file_in_dir" check + rc = client_get(client, f"{BASE_URI}/backtest/history/randomFile.json/SampleStrategy/wallet") + assert_response(rc, 400) + + rc = client_get(client, f"{BASE_URI}/backtest/history/backtest_15/SampleStrategy/wallet") + assert_response(rc, 200) + result = rc.json() + assert result["length"] == 2 + assert result["columns"] == ["date", "__date_ts", "total_quote"] + assert result["data"] == [ + ["2018-01-01T00:00:00Z", 1514764800000, 16000.0], + ["2018-01-01T00:05:00Z", 1514765100000, 16000.75], + ] + + def test_health(botclient): _ftbot, client = botclient diff --git a/tests/test_wallets.py b/tests/test_wallets.py index a7f83ebf0..20861108a 100644 --- a/tests/test_wallets.py +++ b/tests/test_wallets.py @@ -7,12 +7,14 @@ from sqlalchemy import select from freqtrade.constants import UNLIMITED_STAKE_AMOUNT from freqtrade.exceptions import DependencyException -from freqtrade.persistence import Trade +from freqtrade.persistence import Trade, WalletHistory +from freqtrade.wallets import PositionWallet, Wallet from tests.conftest import ( EXMS, create_mock_trades, create_mock_trades_usdt, get_patched_freqtradebot, + log_has_re, patch_wallet, ) @@ -607,3 +609,81 @@ def test_dry_run_wallet_initialization(mocker, default_conf_usdt, config, wallet pytest.approx(freqtrade.wallets._wallets[stake_currency].free) == wallets[stake_currency]["free"] - 100.0 ) + + +@pytest.mark.usefixtures("init_persistence") +def test_record_wallet_state_stores_wallet_history(mocker, default_conf_usdt): + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) + stake_currency = default_conf_usdt["stake_currency"] + freqtrade.wallets._wallets = { + stake_currency: Wallet(stake_currency, free=100.0, used=50, total=150), + "BTC": Wallet("BTC", free=2.0, used=1.0, total=3.0), + } + freqtrade.wallets._positions = { + "ETH/USDT:USDT": PositionWallet( + symbol="ETH/USDT:USDT", + position=0.8, + collateral=1.0, + leverage=3.0, + side="long", + ) + } + + conversion_rates = {stake_currency: 1.0, "BTC": 70000, "ETH": 2500.1} + mocker.patch.object( + freqtrade.exchange, + "get_conversion_rate", + side_effect=lambda currency, *args, **kwargs: conversion_rates.get(currency, 1.0), + ) + mocker.patch( + "freqtrade.persistence.trade_model.Trade.get_open_trades", + return_value=[ + MagicMock(pair="ETH/USDT:USDT", safe_base_currency="ETH"), + ], + ) + + freqtrade.wallets.record_wallet_state() + + wallet_entries = WalletHistory.session.query(WalletHistory).all() + assert len(wallet_entries) == 3 + assert "total_quote" in repr(wallet_entries[0]) + assert "WalletHistory(" in repr(wallet_entries[0]) + + records_by_currency = {entry.currency: entry for entry in wallet_entries} + assert records_by_currency[stake_currency].balance == 149 + assert records_by_currency[stake_currency].rate == 1.0 + assert records_by_currency["BTC"].rate == 70000 + assert records_by_currency["BTC"].balance == 3 + assert not records_by_currency["BTC"].bot_managed + assert records_by_currency["ETH/USDT:USDT"].balance == 0.8 + assert records_by_currency["ETH/USDT:USDT"].rate == 2500.1 + assert records_by_currency["ETH/USDT:USDT"].bot_managed is True + + +@pytest.mark.usefixtures("init_persistence") +def test_record_wallet_state_stores_wallet_history_error(mocker, default_conf, caplog): + freqtrade = get_patched_freqtradebot(mocker, default_conf) + stake_currency = default_conf["stake_currency"] + freqtrade.wallets._wallets = { + stake_currency: Wallet(stake_currency, free=1.0, used=0.5, total=1.5), + "ETH": Wallet("ETH", free=2.0, used=1.0, total=3.0), + } + freqtrade.wallets._positions = { + "ETH/BTC": PositionWallet( + symbol="ETH/BTC", + position=0.8, + collateral=1.0, + leverage=3.0, + side="long", + ) + } + + # Mock bulk_save_objects to raise an exception + mocker.patch.object( + WalletHistory.session, "bulk_save_objects", side_effect=Exception("DB Error") + ) + freqtrade.wallets.record_wallet_state() + + assert log_has_re(r"Error saving wallet balance records: .*", caplog) + wallet_entries = WalletHistory.session.query(WalletHistory).all() + assert len(wallet_entries) == 0 diff --git a/tests/util/test_historic_wallets_migration.py b/tests/util/test_historic_wallets_migration.py new file mode 100644 index 000000000..e4a078201 --- /dev/null +++ b/tests/util/test_historic_wallets_migration.py @@ -0,0 +1,500 @@ +from datetime import datetime, timedelta +from unittest.mock import MagicMock + +import pandas as pd +import pytest + +from freqtrade.enums import CandleType +from freqtrade.persistence import KeyValueStore, Order, Trade, WalletHistory +from freqtrade.util import dt_now, dt_utc +from freqtrade.util.migrations.migrate_wallet_history import ( + _migrate_wallet_history, + _prepare_balance_distribution, + migrate_wallet_history, +) +from tests.conftest import EXMS, generate_test_data, get_patched_exchange, log_has_re + + +def create_closed_mock_trade(fee, pair: str, open_date: datetime, close_date: datetime): + """Create a closed trade for wallet history testing.""" + trade = Trade( + pair=pair, + stake_amount=100.0, + amount=10.0, + amount_requested=10.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=10.0, + close_rate=11.0, + close_profit=0.1, + close_profit_abs=9.5, + exchange="binance", + is_open=False, + strategy="TestStrategy", + timeframe=5, + open_date=open_date, + close_date=close_date, + is_short=False, + ) + order_entry = Order( + ft_order_side="buy", + ft_pair=pair, + ft_is_open=False, + ft_amount=10.0, + ft_price=10.0, + order_id=f"order_{pair}_entry", + status="closed", + symbol=pair, + order_type="limit", + side="buy", + price=10.0, + average=10.0, + amount=10.0, + filled=10.0, + remaining=0.0, + order_date=open_date, + order_filled_date=open_date, + ) + + order_exit = Order( + ft_order_side="sell", + ft_pair=pair, + ft_is_open=False, + ft_amount=10.0, + ft_price=11.0, + order_id=f"order_{pair}_exit", + status="closed", + symbol=pair, + order_type="limit", + side="sell", + price=11.0, + average=11.0, + amount=10.0, + filled=10.0, + remaining=0.0, + order_date=close_date, + order_filled_date=close_date, + ) + + trade.orders.append(order_entry) + trade.orders.append(order_exit) + return trade + + +@pytest.mark.usefixtures("init_persistence") +def test_migrate_wallet_history_skips_when_no_ohlcv_history(mocker, default_conf_usdt): + """Test that migration is skipped when exchange doesn't support OHLCV history.""" + exchange = MagicMock() + exchange.get_option.return_value = False # ohlcv_has_history = False + + migrate_mock = mocker.patch( + "freqtrade.util.migrations.migrate_wallet_history._migrate_wallet_history" + ) + + migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + + # Should return early without setting the migration flag + assert KeyValueStore.get_int_value("wallet_history_migration") is None + assert not migrate_mock.called + + +@pytest.mark.usefixtures("init_persistence") +def test_migrate_wallet_history_skips_when_already_migrated(mocker, default_conf_usdt): + """Test that migration is skipped if already completed.""" + exchange = MagicMock() + exchange.get_option.return_value = True + + migrate_mock = mocker.patch( + "freqtrade.util.migrations.migrate_wallet_history._migrate_wallet_history" + ) + + # Set migration as already completed + KeyValueStore.store_value("wallet_history_migration", 1) + + migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + # Should not call any migration logic + assert KeyValueStore.get_int_value("wallet_history_migration") == 1 + assert not migrate_mock.called + + +@pytest.mark.usefixtures("init_persistence") +def test_migrate_wallet_history_no_trades(default_conf_usdt): + """Test migration with no trades in database.""" + exchange = MagicMock() + exchange.get_option.return_value = True + + # Set bot_start_time + KeyValueStore.store_value("bot_start_time", dt_now() - timedelta(days=5)) + + migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + + # Should complete migration (flag set) but no wallet entries + assert KeyValueStore.get_int_value("wallet_history_migration") == 1 + assert WalletHistory.session.query(WalletHistory).count() == 0 + + +@pytest.mark.usefixtures("init_persistence") +def test_migrate_wallet_history_with_trades(default_conf_usdt, fee, time_machine, markets): + """Test migration with trades creates wallet history entries.""" + start_time = dt_utc(2024, 1, 10, 12, 0, 0) + time_machine.move_to(start_time, tick=False) + + # Bot started 10 days ago + bot_start = start_time - timedelta(days=10) + KeyValueStore.store_value("bot_start_time", bot_start) + + # Create mock trades with dates within the range + trade_open = start_time - timedelta(days=5) + trade_close = start_time - timedelta(days=3) + trade1 = create_closed_mock_trade( + fee, + "ETH/USDT", + open_date=trade_open, + close_date=trade_close, + ) + Trade.session.add(trade1) + Trade.commit() + + # Generate mock OHLCV data starting from bot_start + candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT) + ohlcv_df = generate_test_data("1d", size=15, start=bot_start.strftime("%Y-%m-%d")) + ohlcv_data = {("ETH/USDT", "1d", candle_type): ohlcv_df} + + exchange = MagicMock() + exchange.get_option.return_value = True + exchange.markets = markets + exchange.refresh_latest_ohlcv.return_value = ohlcv_data + exchange.get_pair_base_currency = MagicMock(side_effect=lambda pair: markets.get(pair)["base"]) + + migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + + # Should complete migration + assert KeyValueStore.get_int_value("wallet_history_migration") == 1 + + # Should have created wallet history entries + wallet_entries = WalletHistory.session.query(WalletHistory).all() + assert len(wallet_entries) > 0 + + +@pytest.mark.usefixtures("init_persistence") +def test_migrate_wallet_history_with_multiple_pairs(default_conf_usdt, fee, time_machine, markets): + """Test migration with multiple trading pairs.""" + start_time = dt_utc(2024, 1, 15, 12, 0, 0) + time_machine.move_to(start_time, tick=False) + + # Bot started 15 days ago + bot_start = start_time - timedelta(days=15) + KeyValueStore.store_value("bot_start_time", bot_start) + + # Create mock trades for multiple pairs within the date range + trade1 = create_closed_mock_trade( + fee, + "ETH/USDT", + open_date=start_time - timedelta(days=10), + close_date=start_time - timedelta(days=6), + ) + trade2 = create_closed_mock_trade( + fee, + "BTC/USDT", + open_date=start_time - timedelta(days=7), + close_date=start_time - timedelta(days=5), + ) + Trade.session.add(trade1) + Trade.session.add(trade2) + Trade.commit() + + # Generate mock OHLCV data for both pairs starting from bot_start + candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT) + ohlcv_data = {} + ohlcv_data[("ETH/USDT", "1d", candle_type)] = generate_test_data( + "1d", size=20, start=bot_start.strftime("%Y-%m-%d"), base=1500 + ) + + ohlcv_data[("BTC/USDT", "1d", candle_type)] = generate_test_data( + "1d", size=20, start=bot_start.strftime("%Y-%m-%d"), base=30000 + ) + + exchange = MagicMock() + exchange.get_option.return_value = True + exchange.markets = markets + exchange.refresh_latest_ohlcv.return_value = ohlcv_data + exchange.get_pair_base_currency = MagicMock(side_effect=lambda pair: markets.get(pair)["base"]) + + migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + + # Should complete migration + assert KeyValueStore.get_int_value("wallet_history_migration") == 1 + + # Should have wallet history entries + wallet_entries = WalletHistory.session.query(WalletHistory).all() + assert len(wallet_entries) > 0 + + # Check that stake currency (USDT) entries exist + usdt_entries = [e for e in wallet_entries if e.currency == "USDT"] + assert len(usdt_entries) > 0 + assert len(wallet_entries) > len(usdt_entries) + + # Stake currency should have price = 1.0 + for entry in usdt_entries: + assert entry.rate == 1.0 + + eth_entries = [e for e in wallet_entries if e.currency == "ETH"] + btc_entries = [e for e in wallet_entries if e.currency == "BTC"] + assert len(eth_entries) == 4 + assert len(btc_entries) == 2 + assert all(entry.rate and entry.rate > 1400 and entry.rate < 1600 for entry in eth_entries) + assert all(entry.rate and entry.rate > 29000 and entry.rate < 31000 for entry in btc_entries) + assert all(entry.balance == 10 for entry in btc_entries) + + +@pytest.mark.usefixtures("init_persistence") +def test_migrate_wallet_history_pair_not_in_markets( + default_conf_usdt, caplog, fee, time_machine, markets +): + """Test migration handles pairs that are not in exchange markets.""" + start_time = dt_utc(2024, 1, 10, 12, 0, 0) + time_machine.move_to(start_time, tick=False) + + # Bot started 10 days ago + bot_start = start_time - timedelta(days=10) + KeyValueStore.store_value("bot_start_time", bot_start) + + # Create a trade with a pair that won't be in markets + trade1 = create_closed_mock_trade( + fee, + "UNKNOWN/USDT", + open_date=start_time - timedelta(days=5), + close_date=start_time - timedelta(days=3), + ) + Trade.session.add(trade1) + Trade.commit() + + exchange = MagicMock() + exchange.get_option.return_value = True + exchange.markets = markets + exchange.refresh_latest_ohlcv.return_value = {} + + migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + assert log_has_re("No OHLCV data available for .*", caplog) + + +@pytest.mark.usefixtures("init_persistence") +def test_migrate_wallet_history_stores_migration_date( + default_conf_usdt, fee, time_machine, markets +): + """Test that migration stores the migration date.""" + start_time = dt_utc(2024, 1, 10, 12, 0, 0) + time_machine.move_to(start_time, tick=False) + + # Bot started 10 days ago + bot_start = start_time - timedelta(days=10) + KeyValueStore.store_value("bot_start_time", bot_start) + + # Create a trade + trade1 = create_closed_mock_trade( + fee, + "ETH/USDT", + open_date=start_time - timedelta(days=5), + close_date=start_time - timedelta(days=3), + ) + Trade.session.add(trade1) + Trade.commit() + + candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT) + ohlcv_data = { + ("ETH/USDT", "1d", candle_type): generate_test_data( + "1d", size=15, start=bot_start.strftime("%Y-%m-%d") + ) + } + + exchange = MagicMock() + exchange.get_option.return_value = True + exchange.markets = markets + exchange.refresh_latest_ohlcv.return_value = ohlcv_data + + migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + + # Check migration date is stored + migration_date = KeyValueStore.get_datetime_value("wallet_history_migration_date") + assert migration_date is not None + + +@pytest.mark.usefixtures("init_persistence") +def test_internal_migrate_wallet_history_empty_trades(default_conf_usdt, time_machine): + """Test _migrate_wallet_history returns early when no trades exist.""" + start_time = dt_utc(2024, 1, 1, 12, 0, 0) + time_machine.move_to(start_time, tick=False) + + # Set bot_start_time + KeyValueStore.store_value("bot_start_time", start_time - timedelta(days=5)) + + exchange = MagicMock() + exchange.get_option.return_value = True + exchange.markets = {} + exchange.refresh_latest_ohlcv.return_value = {} + + # Call internal function directly with no trades + _migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + + # refresh_latest_ohlcv should not be called when there are no trades + exchange.refresh_latest_ohlcv.assert_not_called() + + +@pytest.mark.usefixtures("init_persistence") +def test_migrate_wallet_history_with_patched_exchange(mocker, default_conf_usdt, fee, time_machine): + """Test migration using get_patched_exchange helper.""" + start_time = dt_utc(2024, 1, 10, 12, 0, 0) + time_machine.move_to(start_time, tick=False) + + # Bot started 10 days ago + bot_start = start_time - timedelta(days=10) + KeyValueStore.store_value("bot_start_time", bot_start) + + # Create a trade + trade1 = create_closed_mock_trade( + fee, + "ETH/USDT", + open_date=start_time - timedelta(days=5), + close_date=start_time - timedelta(days=3), + ) + Trade.session.add(trade1) + Trade.commit() + + # Generate mock OHLCV data starting from bot_start + candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT) + ohlcv_df = generate_test_data("1d", size=15, start=bot_start.strftime("%Y-%m-%d")) + ohlcv_data = {("ETH/USDT", "1d", candle_type): ohlcv_df} + + # Mock exchange methods + mocker.patch.multiple( + EXMS, + get_option=MagicMock(return_value=True), + refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), + ) + + exchange = get_patched_exchange(mocker, default_conf_usdt) + + migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + + # Should complete migration + assert KeyValueStore.get_int_value("wallet_history_migration") == 1 + + +@pytest.mark.usefixtures("init_persistence") +def test_migrate_wallet_history_db_error_handling( + mocker, default_conf_usdt, fee, time_machine, markets +): + """Test that database errors are handled gracefully.""" + start_time = dt_utc(2024, 1, 10, 12, 0, 0) + time_machine.move_to(start_time, tick=False) + + # Bot started 10 days ago + bot_start = start_time - timedelta(days=10) + KeyValueStore.store_value("bot_start_time", bot_start) + + # Create a trade + trade1 = create_closed_mock_trade( + fee, + "ETH/USDT", + open_date=start_time - timedelta(days=5), + close_date=start_time - timedelta(days=3), + ) + Trade.session.add(trade1) + Trade.commit() + + candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT) + ohlcv_data = { + ("ETH/USDT", "1d", candle_type): generate_test_data( + "1d", size=15, start=bot_start.strftime("%Y-%m-%d") + ) + } + + exchange = MagicMock() + exchange.get_option.return_value = True + exchange.markets = markets + exchange.refresh_latest_ohlcv.return_value = ohlcv_data + + # Mock bulk_save_objects to raise an exception + mocker.patch.object( + WalletHistory.session, "bulk_save_objects", side_effect=Exception("DB Error") + ) + + # Should not raise exception, but handle error gracefully + migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + + # Migration flag should still be set even after error in _migrate + assert KeyValueStore.get_int_value("wallet_history_migration") == 1 + + +@pytest.mark.usefixtures("init_persistence") +def test__prepare_balance_distribution(default_conf_usdt, fee, time_machine, markets): + """Test migration with multiple trading pairs.""" + start_time = dt_utc(2024, 1, 15, 12, 0, 0) + time_machine.move_to(start_time, tick=False) + + # Bot started 15 days ago + bot_start = start_time - timedelta(days=15) + KeyValueStore.store_value("bot_start_time", bot_start) + + # Create mock trades for multiple pairs within the date range + trade1 = create_closed_mock_trade( + fee, + "ETH/USDT", + open_date=start_time - timedelta(days=10), + close_date=start_time - timedelta(days=6), + ) + trade2 = create_closed_mock_trade( + fee, + "BTC/USDT", + open_date=start_time - timedelta(days=7), + close_date=start_time - timedelta(days=5), + ) + Trade.session.add(trade1) + Trade.session.add(trade2) + Trade.commit() + + # Generate mock OHLCV data for both pairs starting from bot_start + candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT) + ohlcv_data = {} + ohlcv_data[("ETH/USDT", "1d", candle_type)] = generate_test_data( + "1d", size=20, start=bot_start.strftime("%Y-%m-%d"), base=1500 + ) + + ohlcv_data[("BTC/USDT", "1d", candle_type)] = generate_test_data( + "1d", size=20, start=bot_start.strftime("%Y-%m-%d"), base=30000 + ) + + exchange = MagicMock() + exchange.get_option.return_value = True + exchange.markets = markets + exchange.refresh_latest_ohlcv.return_value = ohlcv_data + + balance_dist, pairlist_valid = _prepare_balance_distribution( + default_conf_usdt, exchange, 1000.0 + ) + assert not balance_dist.empty + assert len(pairlist_valid) == 2 + assert "ETH/USDT" in pairlist_valid + assert "BTC/USDT" in pairlist_valid + + assert len(balance_dist) == 16 # 16 days from bot_start to now + assert balance_dist["USDT"].iloc[0] == 1000.0 + assert pd.isna(balance_dist["USDT"]).sum() == 0 + + assert all( + col in balance_dist.columns + for col in [ + "USDT", + "ETH/USDT", + "ETH/USDT_collateral", + "ETH/USDT_leverage", + "BTC/USDT", + "BTC/USDT_collateral", + "BTC/USDT_leverage", + "ETH/USDT_open", + "BTC/USDT_open", + "ETH/USDT_value", + "BTC/USDT_value", + "total_value", + ] + )