Merge pull request #13019 from freqtrade/feat/capture_wallets
Capture wallet balance
This commit is contained in:
Matthias
@@ -283,6 +283,10 @@
|
||||
"month"
|
||||
]
|
||||
},
|
||||
"skip_wallet_history_migration": {
|
||||
"description": "Disable wallet history migration.",
|
||||
"type": "boolean"
|
||||
},
|
||||
"hyperopt_path": {
|
||||
"description": "Specify additional lookup path for Hyperopt Loss functions.",
|
||||
"type": "string"
|
||||
|
||||
+109
-101
@@ -211,58 +211,59 @@ A backtesting result will look like that:
|
||||
│ TOTAL │ │ 77 │ 0.22 │ 54.774 │ 5.48 │ 22:12:00 │ 67 0 10 87.0 │
|
||||
└───────────┴─────────────┴────────┴──────────────┴─────────────────┴──────────────┴─────────────────┴────────────────────────┘
|
||||
SUMMARY METRICS
|
||||
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
┃ Metric ┃ Value ┃
|
||||
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
|
||||
│ Backtesting from │ 2025-07-01 00:00:00 │
|
||||
│ Backtesting to │ 2025-08-01 00:00:00 │
|
||||
│ Trading Mode │ Isolated Futures │
|
||||
│ Max open trades │ 3 │
|
||||
│ │ │
|
||||
│ Total/Daily Avg Trades │ 77 / 2.48 │
|
||||
│ Starting balance │ 1000 USDT │
|
||||
│ Final balance │ 1054.774 USDT │
|
||||
│ Absolute profit │ 54.774 USDT │
|
||||
│ Total profit % │ 5.48% │
|
||||
│ CAGR % │ 87.36% │
|
||||
│ Sortino │ 2.48 │
|
||||
│ Sharpe │ 3.75 │
|
||||
│ Calmar │ 40.99 │
|
||||
│ SQN │ 0.69 │
|
||||
│ Profit factor │ 1.29 │
|
||||
│ Expectancy (Ratio) │ 0.71 (0.04) │
|
||||
│ Avg. daily profit │ 1.767 USDT │
|
||||
│ Avg. stake amount │ 345.016 USDT │
|
||||
│ Total trade volume │ 53316.954 USDT │
|
||||
│ │ │
|
||||
│ Long / Short trades │ 67 / 10 │
|
||||
│ Long / Short profit % │ 8.94% / -3.47% │
|
||||
│ Long / Short profit USDT │ 89.425 / -34.651 │
|
||||
│ │ │
|
||||
│ Best Pair │ LTC/USDT:USDT 5.62% │
|
||||
│ Worst Pair │ ADA/USDT:USDT -5.21% │
|
||||
│ Best trade │ ETC/USDT:USDT 2.00% │
|
||||
│ Worst trade │ ADA/USDT:USDT -10.17% │
|
||||
│ Best day │ 26.91 USDT │
|
||||
│ Worst day │ -47.741 USDT │
|
||||
│ Days win/draw/lose │ 20 / 6 / 5 │
|
||||
│ Min/Max/Avg. Duration Winners │ 0d 00:35 / 5d 18:15 / 0d 15:49 │
|
||||
│ Min/Max/Avg. Duration Losers │ 0d 10:40 / 17d 08:00 / 2d 17:00 │
|
||||
│ Max Consecutive Wins / Loss │ 36 / 3 │
|
||||
│ Rejected Entry signals │ 258 │
|
||||
│ Entry/Exit Timeouts │ 0 / 0 │
|
||||
│ │ │
|
||||
│ Min balance │ 1003.168 USDT │
|
||||
│ Max balance │ 1149.421 USDT │
|
||||
│ Max % of account underwater │ 8.23% │
|
||||
│ Absolute drawdown │ 94.647 USDT (8.23%) │
|
||||
│ Drawdown duration │ 9 days 08:50:00 │
|
||||
│ Profit at drawdown start │ 149.421 USDT │
|
||||
│ Profit at drawdown end │ 54.774 USDT │
|
||||
│ Drawdown start │ 2025-07-22 15:10:00 │
|
||||
│ Drawdown end │ 2025-08-01 00:00:00 │
|
||||
│ Market change │ 30.51% │
|
||||
└───────────────────────────────┴─────────────────────────────────┘
|
||||
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
┃ Metric ┃ Value ┃
|
||||
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
|
||||
│ Backtesting from │ 2025-07-01 00:00:00 │
|
||||
│ Backtesting to │ 2025-08-01 00:00:00 │
|
||||
│ Trading Mode │ Isolated Futures │
|
||||
│ Max open trades │ 3 │
|
||||
│ │ │
|
||||
│ Total/Daily Avg Trades │ 77 / 2.48 │
|
||||
│ Starting balance │ 1000 USDT │
|
||||
│ Final balance │ 1054.669 USDT │
|
||||
│ Absolute profit │ 54.669 USDT │
|
||||
│ Total profit % │ 5.47% │
|
||||
│ CAGR % │ 87.14% │
|
||||
│ Sortino │ 2.46 │
|
||||
│ Sharpe │ 3.73 │
|
||||
│ Calmar │ 40.81 │
|
||||
│ SQN │ 0.69 │
|
||||
│ Profit factor │ 1.29 │
|
||||
│ Expectancy (Ratio) │ 0.71 (0.04) │
|
||||
│ Avg. daily profit │ 1.764 USDT │
|
||||
│ Avg. stake amount │ 345.251 USDT │
|
||||
│ Total trade volume │ 53352.96 USDT │
|
||||
│ │ │
|
||||
│ Long / Short trades │ 67 / 10 │
|
||||
│ Long / Short profit % │ 8.93% / -3.46% │
|
||||
│ Long / Short profit USDT │ 89.262 / -34.593 │
|
||||
│ │ │
|
||||
│ Best Pair │ LTC/USDT:USDT 5.62% │
|
||||
│ Worst Pair │ ADA/USDT:USDT -5.21% │
|
||||
│ Best trade │ ETC/USDT:USDT 2.00% │
|
||||
│ Worst trade │ ADA/USDT:USDT -10.17% │
|
||||
│ Best day │ 26.931 USDT │
|
||||
│ Worst day │ -47.741 USDT │
|
||||
│ Days win/draw/lose │ 20 / 6 / 5 │
|
||||
│ Min/Max/Avg. Duration Winners │ 0d 00:35 / 5d 18:15 / 0d 15:49 │
|
||||
│ Min/Max/Avg. Duration Losers │ 0d 10:40 / 17d 08:00 / 2d 17:00 │
|
||||
│ Max Consecutive Wins / Loss │ 36 / 3 │
|
||||
│ Rejected Entry signals │ 258 │
|
||||
│ Entry/Exit Timeouts │ 0 / 0 │
|
||||
│ │ │
|
||||
│ Min/Max balance realized │ 1003.168 USDT / 1149.577 USDT │
|
||||
│ Min/Max balance unrealized │ 1000 USDT / 1149.577 USDT │
|
||||
│ Min/Max balance dates │ 2025-07-01 00:05:00 / 2025-07-22 15:15:00 │
|
||||
│ Max % of account underwater │ 8.26% │
|
||||
│ Absolute drawdown │ 94.908 USDT (8.26%) │
|
||||
│ Drawdown duration │ 9 days 08:50:00 │
|
||||
│ Profit at drawdown start │ 149.577 USDT │
|
||||
│ Profit at drawdown end │ 54.669 USDT │
|
||||
│ Drawdown start │ 2025-07-22 15:10:00 │
|
||||
│ Drawdown end │ 2025-08-01 00:00:00 │
|
||||
│ Market change │ 30.51% │
|
||||
└───────────────────────────────┴───────────────────────────────────────────┘
|
||||
|
||||
Backtested 2025-07-01 00:00:00 -> 2025-08-01 00:00:00 | Max open trades : 3
|
||||
STRATEGY SUMMARY
|
||||
@@ -329,54 +330,59 @@ The last element of the backtest report is the summary metrics table.
|
||||
It contains key metrics about the performance of your strategy on backtesting data.
|
||||
|
||||
```
|
||||
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
┃ Metric ┃ Value ┃
|
||||
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
|
||||
│ Backtesting from │ 2025-07-01 00:00:00 │
|
||||
│ Backtesting to │ 2025-08-01 00:00:00 │
|
||||
│ Trading Mode │ Isolated Futures │
|
||||
│ Max open trades │ 3 │
|
||||
│ │ │
|
||||
│ Total/Daily Avg Trades │ 72 / 2.32 │
|
||||
│ Starting balance │ 1000 USDT │
|
||||
│ Final balance │ 1106.734 USDT │
|
||||
│ Absolute profit │ 106.734 USDT │
|
||||
│ Total profit % │ 10.67% │
|
||||
│ CAGR % │ 230.04% │
|
||||
│ Sortino │ 4.99 │
|
||||
│ Sharpe │ 8.00 │
|
||||
│ Calmar │ 77.76 │
|
||||
│ SQN │ 1.52 │
|
||||
│ Profit factor │ 1.79 │
|
||||
│ Expectancy (Ratio) │ 1.48 (0.07) │
|
||||
│ Avg. daily profit │ 3.443 USDT │
|
||||
│ Avg. stake amount │ 363.133 USDT │
|
||||
│ Total trade volume │ 52466.174 USDT │
|
||||
│ │ │
|
||||
│ Best Pair │ LTC/USDT:USDT 4.48% │
|
||||
│ Worst Pair │ ADA/USDT:USDT -1.78% │
|
||||
│ Best trade │ ETC/USDT:USDT 2.00% │
|
||||
│ Worst trade │ ADA/USDT:USDT -10.17% │
|
||||
│ Best day │ 23.535 USDT │
|
||||
│ Worst day │ -49.813 USDT │
|
||||
│ Days win/draw/lose │ 21 / 6 / 4 │
|
||||
│ Min/Max/Avg. Duration Winners │ 0d 00:35 / 5d 18:15 / 0d 15:30 │
|
||||
│ Min/Max/Avg. Duration Losers │ 0d 12:00 / 17d 08:00 / 3d 23:28 │
|
||||
│ Max Consecutive Wins / Loss │ 58 / 4 │
|
||||
│ Rejected Entry signals │ 254 │
|
||||
│ Entry/Exit Timeouts │ 0 / 0 │
|
||||
│ │ │
|
||||
│ Min balance │ 1003.168 USDT │
|
||||
│ Max balance │ 1209 USDT │
|
||||
│ Max % of account underwater │ 8.46% │
|
||||
│ Absolute drawdown │ 102.266 USDT (8.46%) │
|
||||
│ Drawdown duration │ 9 days 08:50:00 │
|
||||
│ Profit at drawdown start │ 209 USDT │
|
||||
│ Profit at drawdown end │ 106.734 USDT │
|
||||
│ Drawdown start │ 2025-07-22 15:10:00 │
|
||||
│ Drawdown end │ 2025-08-01 00:00:00 │
|
||||
│ Market change │ 30.51% │
|
||||
└───────────────────────────────┴─────────────────────────────────┘
|
||||
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
┃ Metric ┃ Value ┃
|
||||
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
|
||||
│ Backtesting from │ 2025-07-01 00:00:00 │
|
||||
│ Backtesting to │ 2025-08-01 00:00:00 │
|
||||
│ Trading Mode │ Isolated Futures │
|
||||
│ Max open trades │ 3 │
|
||||
│ │ │
|
||||
│ Total/Daily Avg Trades │ 77 / 2.48 │
|
||||
│ Starting balance │ 1000 USDT │
|
||||
│ Final balance │ 1054.669 USDT │
|
||||
│ Absolute profit │ 54.669 USDT │
|
||||
│ Total profit % │ 5.47% │
|
||||
│ CAGR % │ 87.14% │
|
||||
│ Sortino │ 2.46 │
|
||||
│ Sharpe │ 3.73 │
|
||||
│ Calmar │ 40.81 │
|
||||
│ SQN │ 0.69 │
|
||||
│ Profit factor │ 1.29 │
|
||||
│ Expectancy (Ratio) │ 0.71 (0.04) │
|
||||
│ Avg. daily profit │ 1.764 USDT │
|
||||
│ Avg. stake amount │ 345.251 USDT │
|
||||
│ Total trade volume │ 53352.96 USDT │
|
||||
│ │ │
|
||||
│ Long / Short trades │ 67 / 10 │
|
||||
│ Long / Short profit % │ 8.93% / -3.46% │
|
||||
│ Long / Short profit USDT │ 89.262 / -34.593 │
|
||||
│ │ │
|
||||
│ Best Pair │ LTC/USDT:USDT 5.62% │
|
||||
│ Worst Pair │ ADA/USDT:USDT -5.21% │
|
||||
│ Best trade │ ETC/USDT:USDT 2.00% │
|
||||
│ Worst trade │ ADA/USDT:USDT -10.17% │
|
||||
│ Best day │ 26.931 USDT │
|
||||
│ Worst day │ -47.741 USDT │
|
||||
│ Days win/draw/lose │ 20 / 6 / 5 │
|
||||
│ Min/Max/Avg. Duration Winners │ 0d 00:35 / 5d 18:15 / 0d 15:49 │
|
||||
│ Min/Max/Avg. Duration Losers │ 0d 10:40 / 17d 08:00 / 2d 17:00 │
|
||||
│ Max Consecutive Wins / Loss │ 36 / 3 │
|
||||
│ Rejected Entry signals │ 258 │
|
||||
│ Entry/Exit Timeouts │ 0 / 0 │
|
||||
│ │ │
|
||||
│ Min/Max balance realized │ 1003.168 USDT / 1149.577 USDT │
|
||||
│ Min/Max balance unrealized │ 1000 USDT / 1149.577 USDT │
|
||||
│ Min/Max balance dates │ 2025-07-01 00:05:00 / 2025-07-22 15:15:00 │
|
||||
│ Max % of account underwater │ 8.26% │
|
||||
│ Absolute drawdown │ 94.908 USDT (8.26%) │
|
||||
│ Drawdown duration │ 9 days 08:50:00 │
|
||||
│ Profit at drawdown start │ 149.577 USDT │
|
||||
│ Profit at drawdown end │ 54.669 USDT │
|
||||
│ Drawdown start │ 2025-07-22 15:10:00 │
|
||||
│ Drawdown end │ 2025-08-01 00:00:00 │
|
||||
│ Market change │ 30.51% │
|
||||
└───────────────────────────────┴───────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
- `Backtesting from` / `Backtesting to`: Backtesting range (usually defined with the `--timerange` option).
|
||||
@@ -409,7 +415,9 @@ It contains key metrics about the performance of your strategy on backtesting da
|
||||
- `Max Consecutive Wins / Loss`: Maximum consecutive wins/losses in a row.
|
||||
- `Rejected Entry signals`: Trade entry signals that could not be acted upon due to `max_open_trades` being reached.
|
||||
- `Entry/Exit Timeouts`: Entry/exit orders which did not fill (only applicable if custom pricing is used).
|
||||
- `Min balance` / `Max balance`: Lowest and Highest Wallet balance during the backtest period.
|
||||
- `Min/Max balance realized`: Lowest and Highest Wallet balance during the backtest period based on closed trades trades.
|
||||
- `Min/Max balance unrealized`: Lowest and Highest Wallet balance during the backtest period - including capital tied in open trades.
|
||||
- `Min/Max balance dates`: Dates when the minimum and maximum unrealized balance occurred.
|
||||
- `Max % of account underwater`: Maximum percentage your account has decreased from the top since the simulation started. Calculated as the maximum of `(Max Balance - Current Balance) / (Max Balance)`.
|
||||
- `Absolute drawdown`: Maximum absolute drawdown experienced, including percentage relative to the account calculated as `(Absolute Drawdown) / (DrawdownHigh + startingBalance)`..
|
||||
- `Drawdown duration`: Duration of the largest drawdown period.
|
||||
|
||||
@@ -46,6 +46,23 @@ On this page, you can also interact with the bot by starting and stopping it and
|
||||
|
||||
|
||||
|
||||
### Dashboard
|
||||
|
||||
The dashboard view provides an overview of the bot's performance and status.
|
||||
If multiple bots are connected, the dashboard will show an overview of all connected bots, allowing you to easily switch between them or show just a subset of available bots.
|
||||
|
||||
#### Wallet Balance
|
||||
|
||||
New in freqtrade 2026.4: This shows the balance of the bot over time.
|
||||
|
||||
Compared to the "cumulative Profit" chart, this chart will show the actual balance of the bot over time, including unrealized profit and losses, as well as deposits and withdrawals.
|
||||
|
||||
Historic data has re-populated based on available exchange data - however is assumed to be best-effort and may not be 100% accurate.
|
||||
More specifically, it won't cover deposits and withdrawals, and will assume a starting balance of current balance - profit/losses.
|
||||
|
||||
For clarity - a "Capture start" marker line is shown on the chart, which indicates the point at which the migration to the new wallet balance tracking system happened.
|
||||
Only beyond this point, the wallet balance is expected to be accurate.
|
||||
|
||||
### Plot Configurator
|
||||
|
||||
FreqUI Plots can be configured either via a `plot_config` configuration object in the strategy (which can be loaded via "from strategy" button) or via the UI.
|
||||
|
||||
@@ -236,6 +236,10 @@ CONF_SCHEMA = {
|
||||
"type": "string",
|
||||
"enum": BACKTEST_CACHE_AGE,
|
||||
},
|
||||
"skip_wallet_history_migration": {
|
||||
"description": "Disable wallet history migration.",
|
||||
"type": "boolean",
|
||||
},
|
||||
# Hyperopt
|
||||
"hyperopt_path": {
|
||||
"description": "Specify additional lookup path for Hyperopt Loss functions.",
|
||||
|
||||
@@ -7,6 +7,7 @@ from .bt_fileutils import (
|
||||
get_backtest_market_change,
|
||||
get_backtest_result,
|
||||
get_backtest_resultlist,
|
||||
get_backtest_wallet_change,
|
||||
get_latest_backtest_filename,
|
||||
get_latest_hyperopt_file,
|
||||
get_latest_hyperopt_filename,
|
||||
|
||||
@@ -312,6 +312,27 @@ def get_backtest_market_change(filename: Path, include_ts: bool = True) -> pd.Da
|
||||
return df
|
||||
|
||||
|
||||
def get_backtest_wallet_change(filename: Path, strategy_name: str) -> pd.DataFrame | None:
|
||||
"""
|
||||
Read backtest wallet change file.
|
||||
:param filename: Path to the backtest result zip file
|
||||
:param strategy_name: Name of the strategy to load
|
||||
:return: DataFrame with wallet change data
|
||||
"""
|
||||
if filename.suffix != ".zip":
|
||||
return None
|
||||
|
||||
try:
|
||||
data = load_file_from_zip(filename, f"{filename.stem}_{strategy_name}_wallet.feather")
|
||||
df = pd.read_feather(BytesIO(data))
|
||||
|
||||
df.loc[:, "__date_ts"] = df.loc[:, "date"].astype(np.int64) // 1000 // 1000
|
||||
return df
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def find_existing_backtest_stats(
|
||||
dirname: Path | str, run_ids: dict[str, str], min_backtest_date: datetime | None = None
|
||||
) -> dict[str, Any]:
|
||||
@@ -503,13 +524,16 @@ def load_backtest_analysis_data(
|
||||
return None
|
||||
|
||||
|
||||
def trade_list_to_dataframe(trades: list[Trade] | list[LocalTrade]) -> pd.DataFrame:
|
||||
def trade_list_to_dataframe(
|
||||
trades: list[Trade] | list[LocalTrade], *, minified: bool = True
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Convert list of Trade objects to pandas Dataframe
|
||||
:param trades: List of trade objects
|
||||
:param minified: Whether to use minified version of trade JSON
|
||||
:return: Dataframe with BT_DATA_COLUMNS
|
||||
"""
|
||||
df = pd.DataFrame.from_records([t.to_json(True) for t in trades], columns=BT_DATA_COLUMNS)
|
||||
df = pd.DataFrame.from_records([t.to_json(minified) for t in trades], columns=BT_DATA_COLUMNS)
|
||||
if len(df) > 0:
|
||||
df["close_date"] = pd.to_datetime(df["close_timestamp"], unit="ms", utc=True)
|
||||
df["open_date"] = pd.to_datetime(df["open_timestamp"], unit="ms", utc=True)
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from freqtrade.constants import IntOrInf
|
||||
from freqtrade.exchange import (
|
||||
timeframe_to_prev_date,
|
||||
timeframe_to_resample_freq,
|
||||
)
|
||||
from freqtrade.util import dt_from_ts
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -58,3 +64,95 @@ def evaluate_result_multi(
|
||||
"""
|
||||
df_final = analyze_trade_parallelism(trades, timeframe)
|
||||
return df_final[df_final["open_trades"] > max_open_trades]
|
||||
|
||||
|
||||
def balance_distribution_over_time(
|
||||
trades: pd.DataFrame,
|
||||
min_date: datetime,
|
||||
max_date: datetime,
|
||||
timeframe: str,
|
||||
stake_currency: str,
|
||||
start_balance: float,
|
||||
pairlist: list[str],
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Return a dataframe with stake_currency and the pairlist as columns
|
||||
Each column will contain the amount of the currency at the given time
|
||||
Columns added are:
|
||||
- stake_currency: amount of stake currency
|
||||
- <pair>: amount of base currency in the pair
|
||||
- <pair>_leverage: leverage used for the pair at the time (NaN if no open trade)
|
||||
- <pair>_is_short: 1 if the open trade is short, 0 if long (NaN if no open trade)
|
||||
- <pair>_collateral: amount of stake currency used as collateral for open trades
|
||||
:param trades: Trades Dataframe - can be loaded from backtest, or created
|
||||
via trade_list_to_dataframe
|
||||
:param timeframe: Frequency to use for the resulting dataframe
|
||||
:param min_date: start date
|
||||
:param max_date: End date (will be rounded down to timeframe)
|
||||
:param stake_currency: The stake currency
|
||||
:param start_balance: Starting balance in stake currency
|
||||
:param pairlist: List of trading pairs to include in the dataframe
|
||||
Can be obtained via trade_df["pair"].unique()
|
||||
For pairs without trades, the column will be all zeros
|
||||
:return: Dataframe with balance distribution over time
|
||||
"""
|
||||
min_date_res = timeframe_to_prev_date(timeframe, min_date)
|
||||
max_date_res = timeframe_to_prev_date(timeframe, max_date)
|
||||
index = pd.date_range(min_date_res, max_date_res, freq=timeframe_to_resample_freq(timeframe))
|
||||
pairs_lev = [f"{pair}_leverage" for pair in pairlist]
|
||||
pairs_is_short = [f"{pair}_is_short" for pair in pairlist]
|
||||
pairs_collateral = [f"{pair}_collateral" for pair in pairlist]
|
||||
pairs_lev += pairs_is_short
|
||||
|
||||
df = pd.DataFrame(
|
||||
index=index, columns=[stake_currency] + pairlist + pairs_lev + pairs_collateral, dtype=float
|
||||
)
|
||||
# Initialize variables to starting values
|
||||
df[stake_currency] = float(start_balance)
|
||||
df[pairlist + pairs_collateral] = 0.0
|
||||
df[pairs_lev] = np.nan
|
||||
|
||||
for trade in trades.sort_values(by=["open_date"]).itertuples():
|
||||
pair = trade.pair
|
||||
end_date = trade.close_date if trade.close_date is not pd.NaT else None
|
||||
# Exclude open orders - these won't have order_filled_timestamp set.
|
||||
df.loc[trade.open_date : end_date, f"{pair}_leverage"] = trade.leverage
|
||||
df.loc[trade.open_date : end_date, f"{pair}_is_short"] = 1 if trade.is_short else 0
|
||||
orders = [o for o in trade.orders if o["order_filled_timestamp"]]
|
||||
current_position = 0
|
||||
current_collateral = 0
|
||||
for order in sorted(orders, key=lambda x: x["order_filled_timestamp"]):
|
||||
filled_at = pd.Timestamp(dt_from_ts(order["order_filled_timestamp"]))
|
||||
real_amount = order.get("filled", order["amount"])
|
||||
stake = order["safe_price"] * real_amount
|
||||
stake_no_lev = stake / trade.leverage
|
||||
if order["ft_is_entry"]:
|
||||
# Entry order: lock collateral and pay fee
|
||||
# For both long and short: balance decreases by collateral + fee
|
||||
fee_open = stake * trade.fee_open
|
||||
current_position += real_amount
|
||||
current_collateral += stake_no_lev
|
||||
df.loc[filled_at:end_date, pair] += real_amount
|
||||
df.loc[filled_at:end_date, f"{pair}_collateral"] += stake_no_lev
|
||||
df.loc[filled_at:, stake_currency] -= stake_no_lev + fee_open
|
||||
else:
|
||||
# Exit order: release collateral and realize profit/loss
|
||||
fee_close = stake * trade.fee_close
|
||||
if trade.is_short:
|
||||
# For SHORT
|
||||
df.loc[filled_at:, stake_currency] += (
|
||||
current_collateral * (1 + trade.leverage) - stake
|
||||
) - fee_close
|
||||
else:
|
||||
# For LONG
|
||||
df.loc[filled_at:, stake_currency] += (
|
||||
stake - current_collateral * (trade.leverage - 1) - fee_close
|
||||
)
|
||||
df.loc[filled_at:end_date, pair] -= real_amount
|
||||
df.loc[filled_at:end_date, f"{pair}_collateral"] -= stake_no_lev
|
||||
current_position -= real_amount
|
||||
current_collateral -= stake_no_lev
|
||||
|
||||
# Round to avoid floating point issues
|
||||
df = df.round(14)
|
||||
return df
|
||||
|
||||
@@ -172,6 +172,7 @@ class FreqtradeBot(LoggingMixin):
|
||||
self._schedule.every().day.at(t).do(update)
|
||||
|
||||
self._schedule.every().day.at("00:02").do(self.exchange.ws_connection_reset)
|
||||
self._schedule.every().day.at("00:07").do(self.wallets.record_wallet_state)
|
||||
|
||||
self.strategy.ft_bot_start()
|
||||
# Initialize protections AFTER bot start - otherwise parameters are not loaded.
|
||||
@@ -238,7 +239,7 @@ class FreqtradeBot(LoggingMixin):
|
||||
Called on startup and after reloading the bot - triggers notifications and
|
||||
performs startup tasks
|
||||
"""
|
||||
migrate_live_content(self.config, self.exchange)
|
||||
migrate_live_content(self.config, self.exchange, self.wallets.get_starting_balance())
|
||||
set_startup_time()
|
||||
|
||||
self.rpc.startup_messages(self.config, self.pairlists, self.protections)
|
||||
|
||||
@@ -55,6 +55,7 @@ class BacktestContentTypeIcomplete(TypedDict, total=False):
|
||||
backtest_start_time: int
|
||||
backtest_end_time: int
|
||||
run_id: str
|
||||
wallet_summary: DataFrame
|
||||
|
||||
|
||||
class BacktestContentType(BacktestContentTypeIcomplete, total=True):
|
||||
|
||||
@@ -51,6 +51,7 @@ from freqtrade.mixins import LoggingMixin
|
||||
from freqtrade.optimize.backtest_caching import get_strategy_run_id
|
||||
from freqtrade.optimize.bt_progress import BTProgress
|
||||
from freqtrade.optimize.optimize_reports import (
|
||||
convert_bt_wallet_collection,
|
||||
generate_backtest_stats,
|
||||
generate_rejected_signals,
|
||||
generate_trade_signal_candles,
|
||||
@@ -137,6 +138,7 @@ class Backtesting:
|
||||
}
|
||||
self.rejected_dict: dict[str, list] = {}
|
||||
self.starting_balance: float = 0.0
|
||||
self.wallet_captures: list = []
|
||||
|
||||
self._exchange_name = self.config["exchange"]["name"]
|
||||
self.__initial_backtest = exchange is None
|
||||
@@ -451,6 +453,7 @@ class Backtesting:
|
||||
self.replaced_entry_orders = 0
|
||||
self.canceled_exit_orders = 0
|
||||
self.replaced_exit_orders = 0
|
||||
self.wallet_captures = []
|
||||
self.dataprovider.clear_cache()
|
||||
if enable_protections:
|
||||
self._load_protections(self.strategy)
|
||||
@@ -1603,6 +1606,7 @@ class Backtesting:
|
||||
pair_detail_cache: dict[str, list[tuple]] = {}
|
||||
pair_tradedir_cache: dict[str, LongShort | None] = {}
|
||||
pairs_with_open_trades = [t.pair for t in LocalTrade.bt_trades_open]
|
||||
self._capture_wallet(current_time, self.strategy.config["stake_currency"], 1)
|
||||
|
||||
for current_time_det, is_first, has_detail, idx, pair in self._time_pair_generator_det(
|
||||
current_time, pairs
|
||||
@@ -1627,6 +1631,7 @@ class Backtesting:
|
||||
)
|
||||
trade_dir = self.check_for_trade_entry(row)
|
||||
pair_tradedir_cache[pair] = trade_dir
|
||||
self._capture_wallet(current_time, pair.split("/")[0], row[OPEN_IDX])
|
||||
|
||||
else:
|
||||
# Detail candle - from cache.
|
||||
@@ -1680,6 +1685,15 @@ class Backtesting:
|
||||
yield current_time_det, pair, row, is_last_row, trade_dir
|
||||
self.progress.increment()
|
||||
|
||||
def _capture_wallet(self, current_time: datetime, currency: str, price: float) -> None:
|
||||
"""
|
||||
Capture the current wallet state.
|
||||
"""
|
||||
if self.dataprovider.runmode != RunMode.BACKTEST:
|
||||
return
|
||||
if total := self.wallets.get_total(currency):
|
||||
self.wallet_captures.append((current_time, currency, price, total))
|
||||
|
||||
def backtest(
|
||||
self, processed: dict, start_date: datetime, end_date: datetime
|
||||
) -> BacktestContentTypeIcomplete:
|
||||
@@ -1739,6 +1753,7 @@ class Backtesting:
|
||||
"canceled_entry_orders": self.canceled_entry_orders,
|
||||
"replaced_entry_orders": self.replaced_entry_orders,
|
||||
"final_balance": self.wallets.get_total(self.strategy.config["stake_currency"]),
|
||||
"wallet_summary": convert_bt_wallet_collection(self.wallet_captures),
|
||||
}
|
||||
|
||||
def backtest_one_strategy(
|
||||
@@ -1867,6 +1882,11 @@ class Backtesting:
|
||||
dt_appendix,
|
||||
market_change_data=combined_res,
|
||||
analysis_results=self.analysis_results,
|
||||
wallet_summary={
|
||||
s: x["wallet_summary"]
|
||||
for s, x in self.all_bt_content.items()
|
||||
if "wallet_summary" in x
|
||||
},
|
||||
strategy_files={s.get_strategy_name(): s.__file__ for s in self.strategylist},
|
||||
)
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ from freqtrade.optimize.optimize_reports.bt_output import (
|
||||
)
|
||||
from freqtrade.optimize.optimize_reports.bt_storage import store_backtest_results
|
||||
from freqtrade.optimize.optimize_reports.optimize_reports import (
|
||||
convert_bt_wallet_collection,
|
||||
generate_all_periodic_breakdown_stats,
|
||||
generate_backtest_stats,
|
||||
generate_daily_stats,
|
||||
|
||||
@@ -287,6 +287,24 @@ def text_table_add_metrics(strat_results: dict) -> None:
|
||||
if "trading_mode" in strat_results
|
||||
else []
|
||||
)
|
||||
wallet_metrics: list[tuple[str, str]] = []
|
||||
if wallet_stats := strat_results.get("wallet_stats"):
|
||||
wallet_metrics = [
|
||||
(
|
||||
"Min/Max balance realized",
|
||||
f"{fmt_coin(strat_results['csum_min'], stake)} / "
|
||||
f"{fmt_coin(strat_results['csum_max'], stake)}",
|
||||
),
|
||||
(
|
||||
"Min/Max balance unrealized",
|
||||
f"{fmt_coin(wallet_stats['low_balance'], stake)} / "
|
||||
f"{fmt_coin(wallet_stats['high_balance'], stake)}",
|
||||
),
|
||||
(
|
||||
"Min/Max balance dates",
|
||||
f"{wallet_stats['low_date']} / {wallet_stats['high_date']}",
|
||||
),
|
||||
]
|
||||
|
||||
# Newly added fields should be ignored if they are missing in strat_results. hyperopt-show
|
||||
# command stores these results and newer version of freqtrade must be able to handle old
|
||||
@@ -408,8 +426,7 @@ def text_table_add_metrics(strat_results: dict) -> None:
|
||||
),
|
||||
*entry_adjustment_metrics,
|
||||
("", ""), # Empty line to improve readability
|
||||
("Min balance", fmt_coin(strat_results["csum_min"], stake)),
|
||||
("Max balance", fmt_coin(strat_results["csum_max"], stake)),
|
||||
*wallet_metrics,
|
||||
*drawdown_metrics,
|
||||
("Market change", f"{strat_results['market_change']:.2%}"),
|
||||
]
|
||||
|
||||
@@ -52,6 +52,7 @@ def store_backtest_results(
|
||||
dtappendix: str,
|
||||
*,
|
||||
market_change_data: DataFrame | None = None,
|
||||
wallet_summary: dict[str, DataFrame] | None = None,
|
||||
analysis_results: dict[str, dict[str, DataFrame]] | None = None,
|
||||
strategy_files: dict[str, str] | None = None,
|
||||
) -> Path:
|
||||
@@ -123,6 +124,15 @@ def store_backtest_results(
|
||||
market_change_buf.seek(0)
|
||||
zipf.writestr(market_change_name, market_change_buf.getvalue())
|
||||
|
||||
# Add wallet summary if present
|
||||
if wallet_summary is not None:
|
||||
for strategy, df in wallet_summary.items():
|
||||
wallet_name = f"{base_filename.stem}_{strategy}_wallet.feather"
|
||||
wallet_buf = BytesIO()
|
||||
df.reset_index().to_feather(wallet_buf, compression_level=9, compression="lz4")
|
||||
wallet_buf.seek(0)
|
||||
zipf.writestr(wallet_name, wallet_buf.getvalue())
|
||||
|
||||
# Add analysis results if present and running in backtest mode
|
||||
if (
|
||||
config.get("export", "none") == "signals"
|
||||
|
||||
@@ -29,6 +29,48 @@ from freqtrade.util import decimals_per_coin, fmt_coin, format_duration, get_dry
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def convert_bt_wallet_collection(wallet_captures: list[tuple]) -> DataFrame:
|
||||
"""
|
||||
Convert the wallet capture list to a DataFrame.
|
||||
Assumes the wallet_captures list contains tuples with the following structure:
|
||||
(date, currency, price, balance).
|
||||
"""
|
||||
if len(wallet_captures) == 0:
|
||||
return DataFrame()
|
||||
return DataFrame(
|
||||
wallet_captures,
|
||||
columns=["date", "currency", "rate", "balance"],
|
||||
)
|
||||
|
||||
|
||||
def generate_wallet_stats(wallet_df: DataFrame, stake_currency: str) -> dict[str, Any]:
|
||||
"""Generate wallet statistics from the wallet DataFrame."""
|
||||
if wallet_df is None or wallet_df.empty:
|
||||
return {}
|
||||
wallet_df.loc[:, "total_quote"] = wallet_df["rate"] * wallet_df["balance"]
|
||||
# Group by date to get total wallet value at each timestamp
|
||||
wallet = wallet_df.groupby("date")["total_quote"].sum().reset_index()
|
||||
total_quote = wallet["total_quote"]
|
||||
low_idx = total_quote.idxmin()
|
||||
high_idx = total_quote.idxmax()
|
||||
start_balance = wallet.iloc[0]["total_quote"]
|
||||
end_balance = wallet.iloc[-1]["total_quote"]
|
||||
high_balance = total_quote.loc[high_idx]
|
||||
low_balance = total_quote.loc[low_idx]
|
||||
low_date = wallet.loc[low_idx, "date"]
|
||||
high_date = wallet.loc[high_idx, "date"]
|
||||
return {
|
||||
"start_balance": start_balance,
|
||||
"end_balance": end_balance,
|
||||
"high_balance": high_balance,
|
||||
"low_balance": low_balance,
|
||||
"low_date": low_date.strftime(DATETIME_PRINT_FORMAT),
|
||||
"low_ts": int(low_date.timestamp() * 1000),
|
||||
"high_date": high_date.strftime(DATETIME_PRINT_FORMAT),
|
||||
"high_ts": int(high_date.timestamp() * 1000),
|
||||
}
|
||||
|
||||
|
||||
def generate_trade_signal_candles(
|
||||
preprocessed_df: dict[str, DataFrame], bt_results: BacktestContentType, date_col: str
|
||||
) -> dict[str, DataFrame]:
|
||||
@@ -592,6 +634,7 @@ def generate_strategy_stats(
|
||||
"sharpe": calculate_sharpe(results, min_date, max_date, start_balance),
|
||||
"calmar": calculate_calmar(results, min_date, max_date, start_balance),
|
||||
"sqn": calculate_sqn(results, start_balance),
|
||||
"wallet_stats": generate_wallet_stats(content.get("wallet_summary"), stake_currency),
|
||||
"profit_factor": profit_factor,
|
||||
"backtest_start": min_date.strftime(DATETIME_PRINT_FORMAT),
|
||||
"backtest_start_ts": int(min_date.timestamp() * 1000),
|
||||
|
||||
@@ -10,3 +10,4 @@ from freqtrade.persistence.usedb_context import (
|
||||
disable_database_use,
|
||||
enable_database_use,
|
||||
)
|
||||
from freqtrade.persistence.wallet_history import WalletHistory
|
||||
|
||||
@@ -22,6 +22,8 @@ KeyStoreKeys = Literal[
|
||||
"bot_start_time",
|
||||
"startup_time",
|
||||
"binance_migration",
|
||||
"wallet_history_migration",
|
||||
"wallet_history_migration_date",
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ from freqtrade.persistence.key_value_store import _KeyValueStoreModel
|
||||
from freqtrade.persistence.migrations import check_migrate
|
||||
from freqtrade.persistence.pairlock import PairLock
|
||||
from freqtrade.persistence.trade_model import Order, Trade
|
||||
from freqtrade.persistence.wallet_history import WalletHistory
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -91,6 +92,7 @@ def init_db(db_url: str) -> None:
|
||||
_CustomData.session = scoped_session(
|
||||
sessionmaker(bind=engine, autoflush=True), scopefunc=get_request_or_thread_id
|
||||
)
|
||||
WalletHistory.session = Trade.session
|
||||
|
||||
previous_tables = inspect(engine).get_table_names()
|
||||
ModelBase.metadata.create_all(engine)
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
from datetime import datetime
|
||||
from typing import ClassVar
|
||||
|
||||
from sqlalchemy import DateTime, Float, Integer, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from freqtrade.persistence.base import ModelBase, SessionType
|
||||
|
||||
|
||||
class WalletHistory(ModelBase):
|
||||
"""
|
||||
Daily wallet state tracking with minimal fields
|
||||
"""
|
||||
|
||||
__tablename__ = "wallet_history"
|
||||
session: ClassVar[SessionType]
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
timestamp: Mapped[datetime] = mapped_column(DateTime, nullable=False, index=True)
|
||||
currency: Mapped[str] = mapped_column(String(25), nullable=False)
|
||||
# Rate: price of 1 unit of `currency` quoted in `quote_currency`.
|
||||
# e.g., USDT/ETH -> USDT per ETH
|
||||
rate: Mapped[float] = mapped_column(Float, nullable=True)
|
||||
# Quote currency for rate/total fields (e.g., 'USDT')
|
||||
quote_currency: Mapped[str] = mapped_column(String(25), nullable=False)
|
||||
|
||||
# Balance in `currency` units
|
||||
balance: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
|
||||
# Canonical total wallet equity/value denominated in `quote_currency` (if available)
|
||||
# For futures positions, collateral + PnL is used to compute this value.
|
||||
total_quote: Mapped[float] = mapped_column(Float, nullable=True)
|
||||
# Total position value in `quote_currency` - including leverage
|
||||
total_position_value: Mapped[float] = mapped_column(Float, nullable=True)
|
||||
collateral: Mapped[float] = mapped_column(Float, nullable=True)
|
||||
leverage: Mapped[float] = mapped_column(Float, nullable=False, default=1.0)
|
||||
|
||||
bot_managed: Mapped[bool] = mapped_column(nullable=False, default=True)
|
||||
|
||||
__table_args__ = (
|
||||
# Ensure one record per currency per day
|
||||
UniqueConstraint("timestamp", "currency", name="unique_wallet_daily"),
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"WalletHistory(timestamp={self.timestamp}, currency={self.currency}, "
|
||||
f"rate={self.rate}, total_quote={self.total_quote}, "
|
||||
f"balance={self.balance}, leverage={self.leverage})"
|
||||
)
|
||||
@@ -16,10 +16,11 @@ from freqtrade.data.btanalysis import (
|
||||
get_backtest_market_change,
|
||||
get_backtest_result,
|
||||
get_backtest_resultlist,
|
||||
get_backtest_wallet_change,
|
||||
load_and_merge_backtest_result,
|
||||
update_backtest_metadata,
|
||||
)
|
||||
from freqtrade.enums import BacktestState
|
||||
from freqtrade.enums import BacktestState, RunMode
|
||||
from freqtrade.exceptions import ConfigurationError, DependencyException, OperationalException
|
||||
from freqtrade.ft_types import get_BacktestResultType_default
|
||||
from freqtrade.misc import deep_merge_dicts, is_file_in_dir
|
||||
@@ -29,6 +30,7 @@ from freqtrade.rpc.api_server.api_schemas import (
|
||||
BacktestMetadataUpdate,
|
||||
BacktestRequest,
|
||||
BacktestResponse,
|
||||
WalletHistoryResponse,
|
||||
)
|
||||
from freqtrade.rpc.api_server.deps import get_config, verify_strategy
|
||||
from freqtrade.rpc.api_server.webserver_bgwork import ApiBG
|
||||
@@ -106,6 +108,11 @@ def __run_backtest_bg(btconfig: Config):
|
||||
ApiBG.bt["bt"].results,
|
||||
datetime.now().strftime("%Y-%m-%d_%H-%M-%S"),
|
||||
market_change_data=combined_res,
|
||||
wallet_summary={
|
||||
s: x["wallet_summary"]
|
||||
for s, x in ApiBG.bt["bt"].all_bt_content.items()
|
||||
if "wallet_summary" in x
|
||||
},
|
||||
strategy_files={
|
||||
s.get_strategy_name(): s.__file__ for s in ApiBG.bt["bt"].strategylist
|
||||
},
|
||||
@@ -137,6 +144,7 @@ async def api_start_backtest(
|
||||
verify_strategy(bt_settings.strategy)
|
||||
|
||||
btconfig = deepcopy(config)
|
||||
btconfig["runmode"] = RunMode.BACKTEST
|
||||
remove_exchange_credentials(btconfig["exchange"], True)
|
||||
settings = dict(bt_settings)
|
||||
if settings.get("freqai", None) is not None:
|
||||
@@ -354,3 +362,29 @@ def api_get_backtest_market_change(file: str, config=Depends(get_config)):
|
||||
"data": df.values.tolist(),
|
||||
"length": len(df),
|
||||
}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/backtest/history/{file}/{strategy}/wallet",
|
||||
response_model=WalletHistoryResponse,
|
||||
tags=["webserver", "backtest"],
|
||||
)
|
||||
def api_get_backtest_wallet(file: str, strategy: str, config=Depends(get_config)):
|
||||
bt_results_base: Path = config["user_data_dir"] / "backtest_results"
|
||||
file_abs = (bt_results_base / file).with_suffix(".zip")
|
||||
# Ensure file is in backtest_results directory
|
||||
if not is_file_in_dir(file_abs, bt_results_base):
|
||||
raise HTTPException(status_code=400, detail="Unable to retrieve wallet history.")
|
||||
|
||||
results = get_backtest_wallet_change(file_abs, strategy)
|
||||
if results is None:
|
||||
raise HTTPException(status_code=404, detail="Unable to retrieve wallet history.")
|
||||
# Consolidate the wallet to the base currency
|
||||
results.loc[:, "total_quote"] = results["rate"] * results["balance"]
|
||||
results = results.groupby(["date", "__date_ts"]).agg({"total_quote": "sum"}).reset_index()
|
||||
|
||||
return {
|
||||
"columns": results.columns.tolist(),
|
||||
"data": results.values.tolist(),
|
||||
"length": len(results),
|
||||
}
|
||||
|
||||
@@ -679,6 +679,15 @@ class BacktestMarketChange(BaseModel):
|
||||
data: list[list[Any]]
|
||||
|
||||
|
||||
class WalletHistoryResponse(BaseModel):
|
||||
columns: list[str]
|
||||
length: int
|
||||
data: list[list[Any]]
|
||||
# start date of the effectively captured data
|
||||
# Before this date, it's based on a reconstructed wallet history
|
||||
capture_start_ts: int | None = None
|
||||
|
||||
|
||||
class MarketRequest(ExchangeModePayloadMixin, BaseModel):
|
||||
base: str | None = None
|
||||
quote: str | None = None
|
||||
|
||||
@@ -31,6 +31,7 @@ from freqtrade.rpc.api_server.api_schemas import (
|
||||
ResultMsg,
|
||||
Stats,
|
||||
StatusMsg,
|
||||
WalletHistoryResponse,
|
||||
WhitelistResponse,
|
||||
)
|
||||
from freqtrade.rpc.api_server.deps import get_config, get_rpc
|
||||
@@ -104,6 +105,22 @@ def stats(rpc: RPC = Depends(get_rpc)):
|
||||
return rpc._rpc_stats()
|
||||
|
||||
|
||||
@router.get(
|
||||
"/historic_balance",
|
||||
response_model=WalletHistoryResponse,
|
||||
tags=["info"],
|
||||
)
|
||||
def api_get_wallet_history(rpc: RPC = Depends(get_rpc)):
|
||||
results, capture_date_ts = rpc._rpc_get_historic_balance()
|
||||
|
||||
return {
|
||||
"columns": results.columns.tolist(),
|
||||
"data": results.values.tolist(),
|
||||
"length": len(results),
|
||||
"capture_start_ts": capture_date_ts,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/daily", response_model=DailyWeeklyMonthly, tags=["Trading-info"])
|
||||
def daily(
|
||||
timescale: int = Query(7, ge=1, description="Number of days to fetch data for"),
|
||||
|
||||
@@ -69,7 +69,8 @@ logger = logging.getLogger(__name__)
|
||||
# 2.45: Add price to forceexit endpoint
|
||||
# 2.46: Add prepend_data to download-data endpoint
|
||||
# 2.47: Add Strategy parameters
|
||||
API_VERSION = 2.47
|
||||
# 2.48: add /backtest/history/wallets endpoint
|
||||
API_VERSION = 2.48
|
||||
|
||||
# Public API, requires no auth.
|
||||
router_public = APIRouter()
|
||||
|
||||
+21
-1
@@ -12,7 +12,7 @@ import psutil
|
||||
from dateutil.relativedelta import relativedelta
|
||||
from dateutil.tz import tzlocal
|
||||
from numpy import inf, int64, isnan, mean, nan
|
||||
from pandas import DataFrame, NaT
|
||||
from pandas import DataFrame, NaT, read_sql
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from freqtrade import __version__
|
||||
@@ -785,6 +785,26 @@ class RPC:
|
||||
"bot_start_date": format_date(bot_start),
|
||||
}
|
||||
|
||||
def _rpc_get_historic_balance(self) -> tuple[DataFrame, int]:
|
||||
"""
|
||||
Returns the historic balance of the bot
|
||||
:return: DataFrame with the balance history and the timestamp of the migration
|
||||
"""
|
||||
results = read_sql("wallet_history", con=Trade.session.bind, parse_dates=["timestamp"])
|
||||
|
||||
results = results.rename({"timestamp": "date"}, axis=1)
|
||||
results.loc[:, "__date_ts"] = results.loc[:, "date"].astype("int64") // 1000 // 1000
|
||||
# Exclude non-bot managed for now
|
||||
results_filtered = results.loc[results["bot_managed"]]
|
||||
|
||||
results_final = (
|
||||
results_filtered.groupby(["date", "__date_ts"])
|
||||
.agg({"total_quote": "sum"})
|
||||
.reset_index()
|
||||
)
|
||||
hist = KeyValueStore.get_datetime_value("wallet_history_migration_date")
|
||||
return results_final, dt_ts_def(hist, 0)
|
||||
|
||||
def __balance_get_est_stake(
|
||||
self, coin: str, stake_currency: str, amount: float, balance: Wallet
|
||||
) -> tuple[float, float]:
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
from freqtrade.constants import Config
|
||||
from freqtrade.exchange import Exchange
|
||||
from freqtrade.util.migrations.funding_rate_mig import migrate_funding_fee_timeframe
|
||||
from freqtrade.util.migrations.migrate_wallet_history import migrate_wallet_history
|
||||
|
||||
|
||||
def migrate_data(config, exchange: Exchange | None = None) -> None:
|
||||
def migrate_data(config: Config, exchange: Exchange | None = None) -> None:
|
||||
"""
|
||||
Migrate persisted data from old formats to new formats
|
||||
"""
|
||||
@@ -10,10 +12,9 @@ def migrate_data(config, exchange: Exchange | None = None) -> None:
|
||||
migrate_funding_fee_timeframe(config, exchange)
|
||||
|
||||
|
||||
def migrate_live_content(config, exchange: Exchange | None = None) -> None:
|
||||
def migrate_live_content(config: Config, exchange: Exchange, starting_balance: float) -> None:
|
||||
"""
|
||||
Migrate database content from old formats to new formats
|
||||
Used for dry/live mode.
|
||||
"""
|
||||
# Currently not used
|
||||
pass
|
||||
migrate_wallet_history(config, exchange, starting_balance)
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from freqtrade.constants import Config
|
||||
from freqtrade.data.btanalysis.bt_fileutils import trade_list_to_dataframe
|
||||
from freqtrade.data.btanalysis.trade_parallelism import balance_distribution_over_time
|
||||
from freqtrade.exchange import Exchange
|
||||
from freqtrade.exchange.exchange_utils_timeframe import timeframe_to_prev_date
|
||||
from freqtrade.persistence import KeyValueStore, Trade, WalletHistory
|
||||
from freqtrade.util import dt_now, dt_ts
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def migrate_wallet_history(config: Config, exchange: Exchange, starting_balance: float):
|
||||
if config.get("skip_wallet_history_migration") or not exchange.get_option(
|
||||
"ohlcv_has_history", True
|
||||
):
|
||||
# we can't fill up wallet history without ohlcv history
|
||||
return
|
||||
if KeyValueStore.get_int_value("wallet_history_migration"):
|
||||
logger.debug("Wallet history migration already completed.")
|
||||
return
|
||||
logger.info("Starting wallet history migration...")
|
||||
_migrate_wallet_history(config, exchange, starting_balance)
|
||||
logger.info("Wallet history migration completed.")
|
||||
KeyValueStore.store_value("wallet_history_migration", 1)
|
||||
KeyValueStore.store_value("wallet_history_migration_date", dt_now())
|
||||
|
||||
|
||||
def _migrate_wallet_history(config: Config, exchange: Exchange, starting_balance: float):
|
||||
# Prepare balance distribution data with OHLCV rates
|
||||
balance_dist, pairlist_valid = _prepare_balance_distribution(config, exchange, starting_balance)
|
||||
if not balance_dist.empty and pairlist_valid:
|
||||
_create_wallet_history_entries(
|
||||
config, exchange, balance_dist, pairlist_valid, config["stake_currency"]
|
||||
)
|
||||
|
||||
|
||||
def _prepare_balance_distribution(
|
||||
config: Config, exchange: Exchange, starting_balance: float
|
||||
) -> tuple[pd.DataFrame, list[str]]:
|
||||
trade_df = trade_list_to_dataframe(Trade.get_trades_proxy(), minified=False)
|
||||
if trade_df.empty:
|
||||
# no trades, nothing to do
|
||||
return pd.DataFrame(), []
|
||||
pairlist = list(trade_df["pair"].unique())
|
||||
timeframe = "1d"
|
||||
stake_currency = config["stake_currency"]
|
||||
min_date = timeframe_to_prev_date(timeframe, KeyValueStore.get_datetime_value("bot_start_time"))
|
||||
balance_dist = balance_distribution_over_time(
|
||||
trade_df,
|
||||
min_date=min_date,
|
||||
max_date=dt_now(),
|
||||
start_balance=starting_balance,
|
||||
stake_currency=stake_currency,
|
||||
timeframe=timeframe,
|
||||
pairlist=pairlist,
|
||||
)
|
||||
pairlist_valid = [p for p in pairlist if p in exchange.markets]
|
||||
pairlist_invalid = set(pairlist) - set(pairlist_valid)
|
||||
if pairlist_invalid:
|
||||
logger.warning(
|
||||
f"The following trading pairs from the trade history are not available on the exchange "
|
||||
f"and will be skipped during wallet history migration: {', '.join(pairlist_invalid)}"
|
||||
)
|
||||
|
||||
logger.info("Wallet History migration: Fetching OHLCV data ...")
|
||||
data = exchange.refresh_latest_ohlcv(
|
||||
[(p, timeframe, config["candle_type_def"]) for p in pairlist_valid],
|
||||
since_ms=dt_ts(min_date),
|
||||
cache=False,
|
||||
drop_incomplete=False,
|
||||
)
|
||||
logger.info(
|
||||
"Wallet History migration: Done fetching OHLCV data for wallet history migration..."
|
||||
)
|
||||
|
||||
dfs = []
|
||||
# Combine all dataframes into one using the open rate
|
||||
for p, x in data.items():
|
||||
x = x.set_index("date", drop=True)
|
||||
col = f"{p[0]}_open"
|
||||
x[col] = x["open"]
|
||||
dfs.append(x[[col]])
|
||||
|
||||
if not dfs:
|
||||
logger.warning(
|
||||
"No OHLCV data available for the trading pairs; skipping wallet history migration."
|
||||
)
|
||||
return pd.DataFrame(), []
|
||||
merged = pd.concat(dfs, axis=1)
|
||||
|
||||
balance_dist = balance_dist.join(merged, how="left")
|
||||
df_value = pd.DataFrame(
|
||||
index=balance_dist.index, columns=[f"{p}_value" for p in pairlist_valid], dtype=float
|
||||
)
|
||||
for p in pairlist_valid:
|
||||
# df_value[f"{p}_value"] = balance_dist[f"{p}_open"] * balance_dist[p]
|
||||
# Identical calculation to rpc and wallets.py
|
||||
df_value[f"{p}_value"] = np.where(
|
||||
balance_dist[f"{p}_is_short"] == 0,
|
||||
(balance_dist[f"{p}_open"] * balance_dist[p])
|
||||
- balance_dist[f"{p}_collateral"] * (balance_dist[f"{p}_leverage"] - 1),
|
||||
(
|
||||
balance_dist[f"{p}_collateral"] * (1 + balance_dist[f"{p}_leverage"])
|
||||
- balance_dist[f"{p}_open"] * balance_dist[p]
|
||||
),
|
||||
)
|
||||
balance_dist = pd.concat([balance_dist, df_value], axis=1)
|
||||
|
||||
# Aggregate total value at each point in time
|
||||
balance_dist["total_value"] = balance_dist[
|
||||
[f"{p}_value" for p in pairlist_valid] + [stake_currency]
|
||||
].sum(axis=1)
|
||||
|
||||
return balance_dist, pairlist_valid
|
||||
|
||||
|
||||
def _create_wallet_history_entries(
|
||||
config: Config,
|
||||
exchange: Exchange,
|
||||
balance_dist: pd.DataFrame,
|
||||
pairlist_valid: list[str],
|
||||
stake_currency: str,
|
||||
):
|
||||
is_futures = config["trading_mode"] == "futures"
|
||||
# Precompute column indices for faster tuple-based iteration
|
||||
# Assume the first column is the index (date)
|
||||
stake_idx = balance_dist.columns.get_loc(stake_currency)
|
||||
pair_balance_idx = {pair: balance_dist.columns.get_loc(pair) + 1 for pair in pairlist_valid}
|
||||
pair_leverage_idx = {
|
||||
pair: balance_dist.columns.get_loc(f"{pair}_leverage") + 1 for pair in pairlist_valid
|
||||
}
|
||||
pair_collateral_idx = {
|
||||
pair: balance_dist.columns.get_loc(f"{pair}_collateral") + 1 for pair in pairlist_valid
|
||||
}
|
||||
pair_is_short_idx = {
|
||||
pair: balance_dist.columns.get_loc(f"{pair}_is_short") + 1 for pair in pairlist_valid
|
||||
}
|
||||
pair_rate_idx = {
|
||||
pair: balance_dist.columns.get_loc(f"{pair}_open") + 1 for pair in pairlist_valid
|
||||
}
|
||||
# Convert balance_dist to WalletHistory entries
|
||||
wallet_entries = []
|
||||
for row in balance_dist.itertuples(index=True, name=None):
|
||||
date = row[0]
|
||||
|
||||
# Add stake currency entry
|
||||
stake_balance = row[stake_idx + 1]
|
||||
if not pd.isna(stake_balance):
|
||||
wallet_entries.append(
|
||||
WalletHistory(
|
||||
timestamp=date,
|
||||
currency=stake_currency,
|
||||
rate=1.0, # Stake currency price is always 1.0
|
||||
balance=stake_balance,
|
||||
total_quote=stake_balance,
|
||||
quote_currency=stake_currency,
|
||||
leverage=1.0,
|
||||
bot_managed=True,
|
||||
)
|
||||
)
|
||||
|
||||
# Add entries for each trading pair
|
||||
for pair in pairlist_valid:
|
||||
base_currency = exchange.get_pair_base_currency(pair)
|
||||
balance = row[pair_balance_idx[pair]]
|
||||
leverage = row[pair_leverage_idx[pair]]
|
||||
# Only add entry if balance is not empty/NaN
|
||||
if not pd.isna(balance) and balance > 0:
|
||||
rate_value = row[pair_rate_idx[pair]]
|
||||
rate = rate_value if not pd.isna(rate_value) else None
|
||||
|
||||
total_quote = balance * rate if rate else None
|
||||
collateral: float | None = None
|
||||
if is_futures:
|
||||
collateral = row[pair_collateral_idx[pair]]
|
||||
is_short = row[pair_is_short_idx[pair]]
|
||||
if collateral is not None and not pd.isna(collateral):
|
||||
# Same formula than in rpc's _rpc_balance
|
||||
total_quote = (
|
||||
(rate * balance - collateral * (leverage - 1))
|
||||
if is_short == 0
|
||||
else (collateral * (1 + leverage) - rate * balance)
|
||||
)
|
||||
wallet_entries.append(
|
||||
WalletHistory(
|
||||
timestamp=date,
|
||||
currency=base_currency,
|
||||
quote_currency=stake_currency,
|
||||
rate=rate,
|
||||
balance=balance,
|
||||
total_quote=total_quote,
|
||||
leverage=leverage if not pd.isna(leverage) else 1.0,
|
||||
bot_managed=True,
|
||||
total_position_value=balance * rate if is_futures and rate else None,
|
||||
# collateral=collateral,
|
||||
)
|
||||
)
|
||||
|
||||
# Save entries to database
|
||||
if wallet_entries:
|
||||
try:
|
||||
# Use bulk_save_objects for better performance
|
||||
WalletHistory.session.bulk_save_objects(wallet_entries)
|
||||
WalletHistory.session.commit()
|
||||
logger.info(f"Successfully created {len(wallet_entries)} wallet balance records")
|
||||
except Exception as e:
|
||||
WalletHistory.session.rollback()
|
||||
logger.error(f"Error saving wallet balance records: {e}")
|
||||
+70
-2
@@ -10,8 +10,8 @@ from freqtrade.enums import RunMode, TradingMode
|
||||
from freqtrade.exceptions import DependencyException
|
||||
from freqtrade.exchange import Exchange
|
||||
from freqtrade.misc import safe_value_fallback
|
||||
from freqtrade.persistence import LocalTrade, Trade
|
||||
from freqtrade.util.datetime_helpers import dt_now
|
||||
from freqtrade.persistence import LocalTrade, Trade, WalletHistory
|
||||
from freqtrade.util import dt_floor_day, dt_now
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -445,3 +445,71 @@ class Wallets:
|
||||
logger.debug(msg)
|
||||
else:
|
||||
logger.info(msg)
|
||||
|
||||
def record_wallet_state(self) -> None:
|
||||
"""Record daily wallet totals to database"""
|
||||
if self._is_backtest:
|
||||
# only record in live mode.
|
||||
return
|
||||
timestamp = dt_floor_day(dt_now())
|
||||
|
||||
# Record total balances for all currencies
|
||||
wallet_records = []
|
||||
position_collaterals = 0.0
|
||||
open_assets: dict[str, Trade] = {t.safe_base_currency: t for t in Trade.get_open_trades()}
|
||||
for pos in self.get_all_positions().values():
|
||||
base = self._exchange.get_pair_base_currency(pos.symbol)
|
||||
rate = self._exchange.get_conversion_rate(base, self._stake_currency)
|
||||
total_quote = None
|
||||
leverage = pos.leverage or 1.0
|
||||
if rate:
|
||||
# Same formula than in rpc's _rpc_balance
|
||||
total_quote = (
|
||||
rate * pos.position - pos.collateral * (leverage - 1)
|
||||
if pos.side == "long"
|
||||
else pos.collateral * (1 + leverage) - rate * pos.position
|
||||
)
|
||||
|
||||
position_record = WalletHistory(
|
||||
timestamp=timestamp,
|
||||
currency=pos.symbol,
|
||||
quote_currency=self._stake_currency,
|
||||
rate=rate,
|
||||
balance=pos.position,
|
||||
total_quote=total_quote,
|
||||
total_position_value=rate * pos.position if rate else None,
|
||||
collateral=pos.collateral,
|
||||
leverage=leverage,
|
||||
bot_managed=base in open_assets,
|
||||
)
|
||||
position_collaterals += pos.collateral
|
||||
wallet_records.append(position_record)
|
||||
|
||||
for wallet in self.get_all_balances().values():
|
||||
# TODO: (needs decision) exclude minimal balances?
|
||||
rate = self._exchange.get_conversion_rate(wallet.currency, self._stake_currency)
|
||||
is_bot_managed = (
|
||||
self._stake_currency == wallet.currency or wallet.currency in open_assets
|
||||
)
|
||||
balance = wallet.total - (
|
||||
position_collaterals if wallet.currency == self._stake_currency else 0
|
||||
)
|
||||
total_quote = rate * balance if rate else None
|
||||
|
||||
wallet_record = WalletHistory(
|
||||
timestamp=timestamp,
|
||||
currency=wallet.currency,
|
||||
quote_currency=self._stake_currency,
|
||||
rate=rate,
|
||||
balance=balance,
|
||||
leverage=1.0,
|
||||
total_quote=total_quote,
|
||||
bot_managed=is_bot_managed,
|
||||
)
|
||||
wallet_records.append(wallet_record)
|
||||
try:
|
||||
WalletHistory.session.bulk_save_objects(wallet_records)
|
||||
WalletHistory.session.commit()
|
||||
except Exception as e:
|
||||
WalletHistory.session.rollback()
|
||||
logger.error(f"Error saving wallet balance records: {e}")
|
||||
|
||||
+4
-2
@@ -169,10 +169,12 @@ def generate_trades_history(n_rows, start_date: datetime | None = None, days=5):
|
||||
return df
|
||||
|
||||
|
||||
def generate_test_data(timeframe: str, size: int, start: str = "2020-07-05", random_seed=42):
|
||||
def generate_test_data(
|
||||
timeframe: str, size: int, start: str = "2020-07-05", random_seed=42, base=20
|
||||
):
|
||||
np.random.seed(random_seed)
|
||||
|
||||
base = np.random.normal(20, 2, size=size)
|
||||
base = np.random.normal(base, 2, size=size)
|
||||
if timeframe == "1y":
|
||||
date = pd.date_range(start, periods=size, freq="1YS", tz="UTC")
|
||||
elif timeframe == "1M":
|
||||
|
||||
@@ -10,8 +10,9 @@ from freqtrade.configuration import TimeRange
|
||||
from freqtrade.constants import LAST_BT_RESULT_FN
|
||||
from freqtrade.data.btanalysis import (
|
||||
BT_DATA_COLUMNS,
|
||||
analyze_trade_parallelism,
|
||||
extract_trades_of_period,
|
||||
get_backtest_market_change,
|
||||
get_backtest_wallet_change,
|
||||
get_latest_backtest_filename,
|
||||
get_latest_hyperopt_file,
|
||||
load_backtest_data,
|
||||
@@ -209,17 +210,6 @@ def test_extract_trades_of_period(testdatadir):
|
||||
assert trades1.iloc[-1].close_date == datetime(2017, 11, 14, 15, 25, 0, tzinfo=UTC)
|
||||
|
||||
|
||||
def test_analyze_trade_parallelism(testdatadir):
|
||||
filename = testdatadir / "backtest_results/backtest-result.json"
|
||||
bt_data = load_backtest_data(filename)
|
||||
|
||||
res = analyze_trade_parallelism(bt_data, "5m")
|
||||
assert isinstance(res, DataFrame)
|
||||
assert "open_trades" in res.columns
|
||||
assert res["open_trades"].max() == 3
|
||||
assert res["open_trades"].min() == 0
|
||||
|
||||
|
||||
def test_load_trades(default_conf, mocker):
|
||||
db_mock = mocker.patch(
|
||||
"freqtrade.data.btanalysis.bt_fileutils.load_trades_from_db", MagicMock()
|
||||
@@ -649,3 +639,56 @@ def test_load_file_from_zip(tmp_path):
|
||||
|
||||
with pytest.raises(ValueError, match=r"File .* not found in zip.*"):
|
||||
load_file_from_zip(zip_file, "testfile55.txt")
|
||||
|
||||
|
||||
def test_get_backtest_market_change(tmp_path):
|
||||
df = DataFrame(
|
||||
{
|
||||
"date": [dt_utc(2020, 1, 1), dt_utc(2020, 1, 2)],
|
||||
"price": [100.0, 110.0],
|
||||
}
|
||||
)
|
||||
feather_file = tmp_path / "backtest-result_market_change.feather"
|
||||
df.to_feather(feather_file)
|
||||
|
||||
direct_df = get_backtest_market_change(feather_file)
|
||||
assert isinstance(direct_df, DataFrame)
|
||||
assert "__date_ts" in direct_df.columns
|
||||
assert direct_df.loc[0, "__date_ts"] == int(df.loc[0, "date"].timestamp() * 1000)
|
||||
|
||||
no_ts_df = get_backtest_market_change(feather_file, include_ts=False)
|
||||
assert "__date_ts" not in no_ts_df.columns
|
||||
|
||||
zip_file = tmp_path / "backtest-result.zip"
|
||||
with ZipFile(zip_file, "w") as zipf:
|
||||
zipf.write(feather_file, arcname=f"{zip_file.stem}_market_change.feather")
|
||||
|
||||
zipped_df = get_backtest_market_change(zip_file)
|
||||
assert isinstance(zipped_df, DataFrame)
|
||||
assert zipped_df.loc[0, "__date_ts"] == int(df.loc[0, "date"].timestamp() * 1000)
|
||||
assert list(zipped_df["price"]) == [100.0, 110.0]
|
||||
|
||||
|
||||
def test_get_backtest_wallet_change(tmp_path):
|
||||
df = DataFrame(
|
||||
{
|
||||
"date": [dt_utc(2020, 1, 1), dt_utc(2020, 1, 2)],
|
||||
"balance": [1.0, 1.1],
|
||||
"rate": [1.0, 1.1],
|
||||
}
|
||||
)
|
||||
wallet_feather = tmp_path / "backtest-result_TestStrategy_wallet.feather"
|
||||
df.to_feather(wallet_feather)
|
||||
|
||||
zip_file = tmp_path / "backtest-result.zip"
|
||||
with ZipFile(zip_file, "w") as zipf:
|
||||
zipf.write(wallet_feather, arcname=wallet_feather.name)
|
||||
|
||||
wallet_df = get_backtest_wallet_change(zip_file, "TestStrategy")
|
||||
assert isinstance(wallet_df, DataFrame)
|
||||
assert "__date_ts" in wallet_df.columns
|
||||
assert wallet_df.loc[0, "__date_ts"] == int(df.loc[0, "date"].timestamp() * 1000)
|
||||
assert list(wallet_df["balance"]) == [1.0, 1.1]
|
||||
|
||||
assert get_backtest_wallet_change(tmp_path / "backtest-result.feather", "TestStrategy") is None
|
||||
assert get_backtest_wallet_change(zip_file, "UnknownStrategy") is None
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
from datetime import timedelta
|
||||
|
||||
import pytest
|
||||
from pandas import DataFrame, Timestamp
|
||||
|
||||
from freqtrade.data.btanalysis import (
|
||||
analyze_trade_parallelism,
|
||||
load_backtest_data,
|
||||
)
|
||||
from freqtrade.data.btanalysis.trade_parallelism import balance_distribution_over_time
|
||||
from freqtrade.util import dt_utc
|
||||
|
||||
|
||||
def test_analyze_trade_parallelism(testdatadir):
|
||||
filename = testdatadir / "backtest_results/backtest-result.json"
|
||||
bt_data = load_backtest_data(filename)
|
||||
|
||||
res = analyze_trade_parallelism(bt_data, "5m")
|
||||
assert isinstance(res, DataFrame)
|
||||
assert "open_trades" in res.columns
|
||||
assert res["open_trades"].max() == 3
|
||||
assert res["open_trades"].min() == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_short", [False, True])
|
||||
def test_balance_distribution_over_time(is_short):
|
||||
"""
|
||||
Test balance_distribution_over_time for both long and short trades.
|
||||
"""
|
||||
# Create a minimal trades DataFrame with 4 trades over time
|
||||
# Base dates for trades
|
||||
start_date = dt_utc(2023, 1, 1)
|
||||
base_date = start_date + timedelta(hours=15)
|
||||
stake_currency = "USDT"
|
||||
start_balance = 1000.0
|
||||
fee = 0.001 # 0.1% fee
|
||||
|
||||
# Create trades spanning different time periods
|
||||
trades_data = {
|
||||
"pair": ["BTC/USDT", "ETH/USDT", "XRP/USDT", "LTC/USDT"],
|
||||
"stake_amount": [100.0, 150.0, 80.0, 120.0],
|
||||
"open_date": [
|
||||
base_date,
|
||||
base_date + timedelta(hours=2),
|
||||
base_date + timedelta(hours=5),
|
||||
base_date + timedelta(hours=8),
|
||||
],
|
||||
"close_date": [
|
||||
base_date + timedelta(hours=3),
|
||||
base_date + timedelta(hours=6),
|
||||
base_date + timedelta(hours=9),
|
||||
base_date + timedelta(hours=12),
|
||||
],
|
||||
"open_rate": [40000.0, 2000.0, 0.5, 100.0],
|
||||
"close_rate": [41000.0, 2100.0, 0.52, 105.0],
|
||||
"fee_open": [fee, fee, fee, fee],
|
||||
"fee_close": [fee, fee, fee, fee],
|
||||
"is_short": [is_short, is_short, is_short, is_short],
|
||||
"leverage": [1.0, 1.0, 1.0, 1.0],
|
||||
"orders": [
|
||||
# Trade 1: BTC/USDT - entry at 40000, exit at 41000
|
||||
[
|
||||
{
|
||||
"amount": 0.0025, # 100 / 40000
|
||||
"filled": 0.0025,
|
||||
"safe_price": 40000.0,
|
||||
"ft_order_side": "sell" if is_short else "buy",
|
||||
"order_filled_timestamp": int(base_date.timestamp() * 1000),
|
||||
"ft_is_entry": True,
|
||||
},
|
||||
{
|
||||
"amount": 0.0025,
|
||||
"filled": 0.0025,
|
||||
"safe_price": 41000.0,
|
||||
"ft_order_side": "buy" if is_short else "sell",
|
||||
"order_filled_timestamp": int(
|
||||
(base_date + timedelta(hours=3)).timestamp() * 1000
|
||||
),
|
||||
"ft_is_entry": False,
|
||||
},
|
||||
],
|
||||
# Trade 2: ETH/USDT - entry at 2000, exit at 2100
|
||||
[
|
||||
{
|
||||
"amount": 0.075, # 150 / 2000
|
||||
"filled": 0.075,
|
||||
"safe_price": 2000.0,
|
||||
"ft_order_side": "sell" if is_short else "buy",
|
||||
"order_filled_timestamp": int(
|
||||
(base_date + timedelta(hours=2)).timestamp() * 1000
|
||||
),
|
||||
"ft_is_entry": True,
|
||||
},
|
||||
{
|
||||
"amount": 0.075,
|
||||
"filled": 0.075,
|
||||
"safe_price": 2100.0,
|
||||
"ft_order_side": "buy" if is_short else "sell",
|
||||
"order_filled_timestamp": int(
|
||||
(base_date + timedelta(hours=6)).timestamp() * 1000
|
||||
),
|
||||
"ft_is_entry": False,
|
||||
},
|
||||
],
|
||||
# Trade 3: XRP/USDT - entry at 0.5, exit at 0.52
|
||||
[
|
||||
{
|
||||
"amount": 160.0, # 80 / 0.5
|
||||
"filled": 160.0,
|
||||
"safe_price": 0.5,
|
||||
"ft_order_side": "sell" if is_short else "buy",
|
||||
"order_filled_timestamp": int(
|
||||
(base_date + timedelta(hours=5)).timestamp() * 1000
|
||||
),
|
||||
"ft_is_entry": True,
|
||||
},
|
||||
{
|
||||
"amount": 160.0,
|
||||
"filled": 160.0,
|
||||
"safe_price": 0.52,
|
||||
"ft_order_side": "buy" if is_short else "sell",
|
||||
"order_filled_timestamp": int(
|
||||
(base_date + timedelta(hours=9)).timestamp() * 1000
|
||||
),
|
||||
"ft_is_entry": False,
|
||||
},
|
||||
],
|
||||
# Trade 4: LTC/USDT - entry at 100, exit at 105
|
||||
[
|
||||
{
|
||||
"amount": 1.2, # 120 / 100
|
||||
"filled": 1.2,
|
||||
"safe_price": 100.0,
|
||||
"ft_order_side": "sell" if is_short else "buy",
|
||||
"order_filled_timestamp": int(
|
||||
(base_date + timedelta(hours=8)).timestamp() * 1000
|
||||
),
|
||||
"ft_is_entry": True,
|
||||
},
|
||||
{
|
||||
"amount": 1.2,
|
||||
"filled": 1.2,
|
||||
"safe_price": 105.0,
|
||||
"ft_order_side": "buy" if is_short else "sell",
|
||||
"order_filled_timestamp": int(
|
||||
(base_date + timedelta(hours=12)).timestamp() * 1000
|
||||
),
|
||||
"ft_is_entry": False,
|
||||
},
|
||||
],
|
||||
],
|
||||
}
|
||||
|
||||
trades_df = DataFrame(trades_data)
|
||||
pairlist = ["BTC/USDT", "ETH/USDT", "XRP/USDT", "LTC/USDT"]
|
||||
|
||||
min_date = start_date
|
||||
max_date = start_date + timedelta(hours=35)
|
||||
|
||||
result = balance_distribution_over_time(
|
||||
trades=trades_df,
|
||||
min_date=min_date,
|
||||
max_date=max_date,
|
||||
timeframe="1h",
|
||||
stake_currency=stake_currency,
|
||||
start_balance=start_balance,
|
||||
pairlist=pairlist,
|
||||
)
|
||||
|
||||
# Verify basic structure
|
||||
assert isinstance(result, DataFrame)
|
||||
assert stake_currency in result.columns
|
||||
for pair in pairlist:
|
||||
assert pair in result.columns
|
||||
assert f"{pair}_leverage" in result.columns
|
||||
assert f"{pair}_is_short" in result.columns
|
||||
assert f"{pair}_collateral" in result.columns
|
||||
|
||||
# Verify the index is a DatetimeIndex
|
||||
assert isinstance(result.index, Timestamp.__class__.__bases__[0])
|
||||
|
||||
# Verify we have entries over the full time period (36h)
|
||||
assert len(result) == 36
|
||||
|
||||
# First trade opens 15h after the start date
|
||||
assert result.iloc[0][stake_currency] == 1000
|
||||
expected_first_balance = start_balance - (100.0 + 100.0 * fee)
|
||||
assert result.iloc[15][stake_currency] == pytest.approx(expected_first_balance)
|
||||
|
||||
# Check that pair columns have non-zero values during trade periods
|
||||
# Trade 1 (BTC/USDT) is open from hour 15 to hour 18
|
||||
# At hour 16, BTC/USDT should have position
|
||||
btc_during_trade = result.loc[base_date + timedelta(hours=1), "BTC/USDT"]
|
||||
assert btc_during_trade > 0, "Trade should have positive position during open period"
|
||||
|
||||
# After Trade 1 closes at hour 3, BTC/USDT position should be 0
|
||||
btc_after_close = result.loc[base_date + timedelta(hours=4) :, "BTC/USDT"]
|
||||
assert all(btc_after_close == 0), "Position should be 0 after trade closes"
|
||||
|
||||
# Final stake currency should reflect all trades' cash flows minus fees
|
||||
final_balance = result.iloc[-1][stake_currency]
|
||||
|
||||
# Verify the balance changed (trades had effect)
|
||||
assert final_balance != start_balance, "Balance should change after trading"
|
||||
|
||||
# Since all exit prices > entry prices, exits return more cash than entries spent
|
||||
# This means final balance > start balance for long trades and < start balance for short trades
|
||||
assert (final_balance > start_balance) if not is_short else (final_balance < start_balance), (
|
||||
"Balance increases for long and decreases for short trades"
|
||||
)
|
||||
@@ -757,10 +757,12 @@ def test_backtest__check_trade_exit(default_conf, mocker) -> None:
|
||||
def test_backtest_one(default_conf, mocker, testdatadir) -> None:
|
||||
default_conf["use_exit_signal"] = False
|
||||
default_conf["max_open_trades"] = 10
|
||||
default_conf["runmode"] = RunMode.BACKTEST
|
||||
|
||||
patch_exchange(mocker)
|
||||
mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=0.00001)
|
||||
mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=float("inf"))
|
||||
mocker.patch(f"{EXMS}.get_pair_base_currency", lambda _, x: x.split("/")[0])
|
||||
backtesting = Backtesting(default_conf)
|
||||
backtesting._set_strategy(backtesting.strategylist[0])
|
||||
pair = "UNITTEST/BTC"
|
||||
@@ -875,13 +877,23 @@ def test_backtest_one(default_conf, mocker, testdatadir) -> None:
|
||||
ln1.iloc[0]["low"], 6
|
||||
) < round(t["close_rate"], 6) < round(ln1.iloc[0]["high"], 6)
|
||||
|
||||
wallet_summary = result["wallet_summary"]
|
||||
assert isinstance(wallet_summary, pd.DataFrame)
|
||||
assert len(wallet_summary) == 255
|
||||
unique_currencies = wallet_summary["currency"].value_counts()
|
||||
assert unique_currencies["BTC"] == 200
|
||||
assert unique_currencies["UNITTEST"] == 55
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_detail", [True, False])
|
||||
def test_backtest_one_detail(default_conf_usdt, mocker, testdatadir, use_detail) -> None:
|
||||
default_conf_usdt["use_exit_signal"] = False
|
||||
default_conf_usdt["runmode"] = RunMode.BACKTEST
|
||||
patch_exchange(mocker)
|
||||
mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=0.00001)
|
||||
mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=float("inf"))
|
||||
mocker.patch(f"{EXMS}.get_pair_base_currency", lambda _, x: x.split("/")[0])
|
||||
|
||||
default_conf_usdt["unfilledtimeout"] = {
|
||||
"entry": 11,
|
||||
"exit": 30,
|
||||
@@ -968,6 +980,12 @@ def test_backtest_one_detail(default_conf_usdt, mocker, testdatadir, use_detail)
|
||||
)
|
||||
|
||||
assert late_entry > 0
|
||||
wallet_summary = result["wallet_summary"]
|
||||
assert isinstance(wallet_summary, pd.DataFrame)
|
||||
assert len(wallet_summary) == 591 if use_detail else 597
|
||||
unique_currencies = wallet_summary["currency"].value_counts()
|
||||
assert unique_currencies["USDT"] == 576
|
||||
assert unique_currencies["XRP"] == 15 if use_detail else 21
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -7,8 +7,10 @@ import logging
|
||||
import time
|
||||
from copy import deepcopy
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from unittest.mock import ANY, MagicMock, PropertyMock, patch
|
||||
from zipfile import ZipFile
|
||||
|
||||
import pandas as pd
|
||||
import pytest
|
||||
@@ -1434,6 +1436,41 @@ def test_api_stats(botclient, mocker, ticker, fee, markets, is_short):
|
||||
assert "draws" in rc.json()["durations"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_short", [True, False])
|
||||
def test_api_historic_balance(botclient, mocker, ticker, fee, markets, is_short):
|
||||
ftbot, client = botclient
|
||||
patch_get_signal(ftbot, enter_long=not is_short, enter_short=is_short)
|
||||
mocker.patch.multiple(
|
||||
EXMS,
|
||||
get_balances=MagicMock(return_value=ticker),
|
||||
fetch_ticker=ticker,
|
||||
get_fee=fee,
|
||||
markets=PropertyMock(return_value=markets),
|
||||
)
|
||||
|
||||
rc = client_get(client, f"{BASE_URI}/historic_balance")
|
||||
assert_response(rc, 200)
|
||||
resp = rc.json()
|
||||
assert "columns" in resp
|
||||
assert "data" in resp
|
||||
assert "length" in resp
|
||||
assert "capture_start_ts" in resp
|
||||
assert resp["length"] == 0
|
||||
|
||||
ftbot.wallets.record_wallet_state()
|
||||
|
||||
rc = client_get(client, f"{BASE_URI}/historic_balance")
|
||||
assert_response(rc, 200)
|
||||
resp1 = rc.json()
|
||||
assert "columns" in resp1
|
||||
assert "data" in resp1
|
||||
assert "length" in resp1
|
||||
assert "capture_start_ts" in resp1
|
||||
assert resp1["length"] == 1
|
||||
assert "__date_ts" in resp1["columns"]
|
||||
assert "total_quote" in resp1["columns"]
|
||||
|
||||
|
||||
def test_api_performance(botclient, fee):
|
||||
ftbot, client = botclient
|
||||
patch_get_signal(ftbot)
|
||||
@@ -3275,7 +3312,7 @@ def test_api_patch_backtest_history_entry(botclient, tmp_path: Path):
|
||||
assert fileres[CURRENT_TEST_STRATEGY]["notes"] == "FooBar"
|
||||
|
||||
|
||||
def test_api_patch_backtest_market_change(botclient, tmp_path: Path):
|
||||
def test_api_backtest_market_change(botclient, tmp_path: Path):
|
||||
ftbot, client = botclient
|
||||
|
||||
# Create a temporary directory and file
|
||||
@@ -3313,6 +3350,55 @@ def test_api_patch_backtest_market_change(botclient, tmp_path: Path):
|
||||
]
|
||||
|
||||
|
||||
def test_api_backtest_wallets(botclient, tmp_path: Path):
|
||||
ftbot, client = botclient
|
||||
|
||||
# Create a temporary directory and file
|
||||
bt_results_base = tmp_path / "backtest_results"
|
||||
bt_results_base.mkdir()
|
||||
zip_file = bt_results_base / "backtest_15.zip"
|
||||
with ZipFile(zip_file, "w") as zipf:
|
||||
wallet_df = pd.DataFrame(
|
||||
{
|
||||
"date": [
|
||||
"2018-01-01T00:00:00Z",
|
||||
"2018-01-01T00:00:00Z",
|
||||
"2018-01-01T00:05:00Z",
|
||||
"2018-01-01T00:05:00Z",
|
||||
],
|
||||
"currency": ["ETH", "BTC", "ETH", "BTC"],
|
||||
"rate": [2000, 60_000, 2001, 60_001],
|
||||
"balance": [0.5, 0.25, 0.5, 0.25],
|
||||
}
|
||||
)
|
||||
wallet_df["date"] = pd.to_datetime(wallet_df["date"])
|
||||
wallet_buf = BytesIO()
|
||||
wallet_df.reset_index().to_feather(wallet_buf, compression_level=9, compression="lz4")
|
||||
wallet_buf.seek(0)
|
||||
zipf.writestr("backtest_15_SampleStrategy_wallet.feather", wallet_buf.read())
|
||||
|
||||
# Wrong basedirectory
|
||||
rc = client_get(client, f"{BASE_URI}/backtest/history/randomFile.json/SampleStrategy/wallet")
|
||||
assert_response(rc, 503)
|
||||
|
||||
ftbot.config["user_data_dir"] = tmp_path
|
||||
ftbot.config["runmode"] = RunMode.WEBSERVER
|
||||
|
||||
# Nonexisting file - fails "is_file_in_dir" check
|
||||
rc = client_get(client, f"{BASE_URI}/backtest/history/randomFile.json/SampleStrategy/wallet")
|
||||
assert_response(rc, 400)
|
||||
|
||||
rc = client_get(client, f"{BASE_URI}/backtest/history/backtest_15/SampleStrategy/wallet")
|
||||
assert_response(rc, 200)
|
||||
result = rc.json()
|
||||
assert result["length"] == 2
|
||||
assert result["columns"] == ["date", "__date_ts", "total_quote"]
|
||||
assert result["data"] == [
|
||||
["2018-01-01T00:00:00Z", 1514764800000, 16000.0],
|
||||
["2018-01-01T00:05:00Z", 1514765100000, 16000.75],
|
||||
]
|
||||
|
||||
|
||||
def test_health(botclient):
|
||||
_ftbot, client = botclient
|
||||
|
||||
|
||||
+81
-1
@@ -7,12 +7,14 @@ from sqlalchemy import select
|
||||
|
||||
from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
|
||||
from freqtrade.exceptions import DependencyException
|
||||
from freqtrade.persistence import Trade
|
||||
from freqtrade.persistence import Trade, WalletHistory
|
||||
from freqtrade.wallets import PositionWallet, Wallet
|
||||
from tests.conftest import (
|
||||
EXMS,
|
||||
create_mock_trades,
|
||||
create_mock_trades_usdt,
|
||||
get_patched_freqtradebot,
|
||||
log_has_re,
|
||||
patch_wallet,
|
||||
)
|
||||
|
||||
@@ -607,3 +609,81 @@ def test_dry_run_wallet_initialization(mocker, default_conf_usdt, config, wallet
|
||||
pytest.approx(freqtrade.wallets._wallets[stake_currency].free)
|
||||
== wallets[stake_currency]["free"] - 100.0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("init_persistence")
|
||||
def test_record_wallet_state_stores_wallet_history(mocker, default_conf_usdt):
|
||||
freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt)
|
||||
stake_currency = default_conf_usdt["stake_currency"]
|
||||
freqtrade.wallets._wallets = {
|
||||
stake_currency: Wallet(stake_currency, free=100.0, used=50, total=150),
|
||||
"BTC": Wallet("BTC", free=2.0, used=1.0, total=3.0),
|
||||
}
|
||||
freqtrade.wallets._positions = {
|
||||
"ETH/USDT:USDT": PositionWallet(
|
||||
symbol="ETH/USDT:USDT",
|
||||
position=0.8,
|
||||
collateral=1.0,
|
||||
leverage=3.0,
|
||||
side="long",
|
||||
)
|
||||
}
|
||||
|
||||
conversion_rates = {stake_currency: 1.0, "BTC": 70000, "ETH": 2500.1}
|
||||
mocker.patch.object(
|
||||
freqtrade.exchange,
|
||||
"get_conversion_rate",
|
||||
side_effect=lambda currency, *args, **kwargs: conversion_rates.get(currency, 1.0),
|
||||
)
|
||||
mocker.patch(
|
||||
"freqtrade.persistence.trade_model.Trade.get_open_trades",
|
||||
return_value=[
|
||||
MagicMock(pair="ETH/USDT:USDT", safe_base_currency="ETH"),
|
||||
],
|
||||
)
|
||||
|
||||
freqtrade.wallets.record_wallet_state()
|
||||
|
||||
wallet_entries = WalletHistory.session.query(WalletHistory).all()
|
||||
assert len(wallet_entries) == 3
|
||||
assert "total_quote" in repr(wallet_entries[0])
|
||||
assert "WalletHistory(" in repr(wallet_entries[0])
|
||||
|
||||
records_by_currency = {entry.currency: entry for entry in wallet_entries}
|
||||
assert records_by_currency[stake_currency].balance == 149
|
||||
assert records_by_currency[stake_currency].rate == 1.0
|
||||
assert records_by_currency["BTC"].rate == 70000
|
||||
assert records_by_currency["BTC"].balance == 3
|
||||
assert not records_by_currency["BTC"].bot_managed
|
||||
assert records_by_currency["ETH/USDT:USDT"].balance == 0.8
|
||||
assert records_by_currency["ETH/USDT:USDT"].rate == 2500.1
|
||||
assert records_by_currency["ETH/USDT:USDT"].bot_managed is True
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("init_persistence")
|
||||
def test_record_wallet_state_stores_wallet_history_error(mocker, default_conf, caplog):
|
||||
freqtrade = get_patched_freqtradebot(mocker, default_conf)
|
||||
stake_currency = default_conf["stake_currency"]
|
||||
freqtrade.wallets._wallets = {
|
||||
stake_currency: Wallet(stake_currency, free=1.0, used=0.5, total=1.5),
|
||||
"ETH": Wallet("ETH", free=2.0, used=1.0, total=3.0),
|
||||
}
|
||||
freqtrade.wallets._positions = {
|
||||
"ETH/BTC": PositionWallet(
|
||||
symbol="ETH/BTC",
|
||||
position=0.8,
|
||||
collateral=1.0,
|
||||
leverage=3.0,
|
||||
side="long",
|
||||
)
|
||||
}
|
||||
|
||||
# Mock bulk_save_objects to raise an exception
|
||||
mocker.patch.object(
|
||||
WalletHistory.session, "bulk_save_objects", side_effect=Exception("DB Error")
|
||||
)
|
||||
freqtrade.wallets.record_wallet_state()
|
||||
|
||||
assert log_has_re(r"Error saving wallet balance records: .*", caplog)
|
||||
wallet_entries = WalletHistory.session.query(WalletHistory).all()
|
||||
assert len(wallet_entries) == 0
|
||||
|
||||
@@ -0,0 +1,500 @@
|
||||
from datetime import datetime, timedelta
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pandas as pd
|
||||
import pytest
|
||||
|
||||
from freqtrade.enums import CandleType
|
||||
from freqtrade.persistence import KeyValueStore, Order, Trade, WalletHistory
|
||||
from freqtrade.util import dt_now, dt_utc
|
||||
from freqtrade.util.migrations.migrate_wallet_history import (
|
||||
_migrate_wallet_history,
|
||||
_prepare_balance_distribution,
|
||||
migrate_wallet_history,
|
||||
)
|
||||
from tests.conftest import EXMS, generate_test_data, get_patched_exchange, log_has_re
|
||||
|
||||
|
||||
def create_closed_mock_trade(fee, pair: str, open_date: datetime, close_date: datetime):
|
||||
"""Create a closed trade for wallet history testing."""
|
||||
trade = Trade(
|
||||
pair=pair,
|
||||
stake_amount=100.0,
|
||||
amount=10.0,
|
||||
amount_requested=10.0,
|
||||
fee_open=fee.return_value,
|
||||
fee_close=fee.return_value,
|
||||
open_rate=10.0,
|
||||
close_rate=11.0,
|
||||
close_profit=0.1,
|
||||
close_profit_abs=9.5,
|
||||
exchange="binance",
|
||||
is_open=False,
|
||||
strategy="TestStrategy",
|
||||
timeframe=5,
|
||||
open_date=open_date,
|
||||
close_date=close_date,
|
||||
is_short=False,
|
||||
)
|
||||
order_entry = Order(
|
||||
ft_order_side="buy",
|
||||
ft_pair=pair,
|
||||
ft_is_open=False,
|
||||
ft_amount=10.0,
|
||||
ft_price=10.0,
|
||||
order_id=f"order_{pair}_entry",
|
||||
status="closed",
|
||||
symbol=pair,
|
||||
order_type="limit",
|
||||
side="buy",
|
||||
price=10.0,
|
||||
average=10.0,
|
||||
amount=10.0,
|
||||
filled=10.0,
|
||||
remaining=0.0,
|
||||
order_date=open_date,
|
||||
order_filled_date=open_date,
|
||||
)
|
||||
|
||||
order_exit = Order(
|
||||
ft_order_side="sell",
|
||||
ft_pair=pair,
|
||||
ft_is_open=False,
|
||||
ft_amount=10.0,
|
||||
ft_price=11.0,
|
||||
order_id=f"order_{pair}_exit",
|
||||
status="closed",
|
||||
symbol=pair,
|
||||
order_type="limit",
|
||||
side="sell",
|
||||
price=11.0,
|
||||
average=11.0,
|
||||
amount=10.0,
|
||||
filled=10.0,
|
||||
remaining=0.0,
|
||||
order_date=close_date,
|
||||
order_filled_date=close_date,
|
||||
)
|
||||
|
||||
trade.orders.append(order_entry)
|
||||
trade.orders.append(order_exit)
|
||||
return trade
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("init_persistence")
|
||||
def test_migrate_wallet_history_skips_when_no_ohlcv_history(mocker, default_conf_usdt):
|
||||
"""Test that migration is skipped when exchange doesn't support OHLCV history."""
|
||||
exchange = MagicMock()
|
||||
exchange.get_option.return_value = False # ohlcv_has_history = False
|
||||
|
||||
migrate_mock = mocker.patch(
|
||||
"freqtrade.util.migrations.migrate_wallet_history._migrate_wallet_history"
|
||||
)
|
||||
|
||||
migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
|
||||
|
||||
# Should return early without setting the migration flag
|
||||
assert KeyValueStore.get_int_value("wallet_history_migration") is None
|
||||
assert not migrate_mock.called
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("init_persistence")
|
||||
def test_migrate_wallet_history_skips_when_already_migrated(mocker, default_conf_usdt):
|
||||
"""Test that migration is skipped if already completed."""
|
||||
exchange = MagicMock()
|
||||
exchange.get_option.return_value = True
|
||||
|
||||
migrate_mock = mocker.patch(
|
||||
"freqtrade.util.migrations.migrate_wallet_history._migrate_wallet_history"
|
||||
)
|
||||
|
||||
# Set migration as already completed
|
||||
KeyValueStore.store_value("wallet_history_migration", 1)
|
||||
|
||||
migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
|
||||
# Should not call any migration logic
|
||||
assert KeyValueStore.get_int_value("wallet_history_migration") == 1
|
||||
assert not migrate_mock.called
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("init_persistence")
|
||||
def test_migrate_wallet_history_no_trades(default_conf_usdt):
|
||||
"""Test migration with no trades in database."""
|
||||
exchange = MagicMock()
|
||||
exchange.get_option.return_value = True
|
||||
|
||||
# Set bot_start_time
|
||||
KeyValueStore.store_value("bot_start_time", dt_now() - timedelta(days=5))
|
||||
|
||||
migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
|
||||
|
||||
# Should complete migration (flag set) but no wallet entries
|
||||
assert KeyValueStore.get_int_value("wallet_history_migration") == 1
|
||||
assert WalletHistory.session.query(WalletHistory).count() == 0
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("init_persistence")
|
||||
def test_migrate_wallet_history_with_trades(default_conf_usdt, fee, time_machine, markets):
|
||||
"""Test migration with trades creates wallet history entries."""
|
||||
start_time = dt_utc(2024, 1, 10, 12, 0, 0)
|
||||
time_machine.move_to(start_time, tick=False)
|
||||
|
||||
# Bot started 10 days ago
|
||||
bot_start = start_time - timedelta(days=10)
|
||||
KeyValueStore.store_value("bot_start_time", bot_start)
|
||||
|
||||
# Create mock trades with dates within the range
|
||||
trade_open = start_time - timedelta(days=5)
|
||||
trade_close = start_time - timedelta(days=3)
|
||||
trade1 = create_closed_mock_trade(
|
||||
fee,
|
||||
"ETH/USDT",
|
||||
open_date=trade_open,
|
||||
close_date=trade_close,
|
||||
)
|
||||
Trade.session.add(trade1)
|
||||
Trade.commit()
|
||||
|
||||
# Generate mock OHLCV data starting from bot_start
|
||||
candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT)
|
||||
ohlcv_df = generate_test_data("1d", size=15, start=bot_start.strftime("%Y-%m-%d"))
|
||||
ohlcv_data = {("ETH/USDT", "1d", candle_type): ohlcv_df}
|
||||
|
||||
exchange = MagicMock()
|
||||
exchange.get_option.return_value = True
|
||||
exchange.markets = markets
|
||||
exchange.refresh_latest_ohlcv.return_value = ohlcv_data
|
||||
exchange.get_pair_base_currency = MagicMock(side_effect=lambda pair: markets.get(pair)["base"])
|
||||
|
||||
migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
|
||||
|
||||
# Should complete migration
|
||||
assert KeyValueStore.get_int_value("wallet_history_migration") == 1
|
||||
|
||||
# Should have created wallet history entries
|
||||
wallet_entries = WalletHistory.session.query(WalletHistory).all()
|
||||
assert len(wallet_entries) > 0
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("init_persistence")
|
||||
def test_migrate_wallet_history_with_multiple_pairs(default_conf_usdt, fee, time_machine, markets):
|
||||
"""Test migration with multiple trading pairs."""
|
||||
start_time = dt_utc(2024, 1, 15, 12, 0, 0)
|
||||
time_machine.move_to(start_time, tick=False)
|
||||
|
||||
# Bot started 15 days ago
|
||||
bot_start = start_time - timedelta(days=15)
|
||||
KeyValueStore.store_value("bot_start_time", bot_start)
|
||||
|
||||
# Create mock trades for multiple pairs within the date range
|
||||
trade1 = create_closed_mock_trade(
|
||||
fee,
|
||||
"ETH/USDT",
|
||||
open_date=start_time - timedelta(days=10),
|
||||
close_date=start_time - timedelta(days=6),
|
||||
)
|
||||
trade2 = create_closed_mock_trade(
|
||||
fee,
|
||||
"BTC/USDT",
|
||||
open_date=start_time - timedelta(days=7),
|
||||
close_date=start_time - timedelta(days=5),
|
||||
)
|
||||
Trade.session.add(trade1)
|
||||
Trade.session.add(trade2)
|
||||
Trade.commit()
|
||||
|
||||
# Generate mock OHLCV data for both pairs starting from bot_start
|
||||
candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT)
|
||||
ohlcv_data = {}
|
||||
ohlcv_data[("ETH/USDT", "1d", candle_type)] = generate_test_data(
|
||||
"1d", size=20, start=bot_start.strftime("%Y-%m-%d"), base=1500
|
||||
)
|
||||
|
||||
ohlcv_data[("BTC/USDT", "1d", candle_type)] = generate_test_data(
|
||||
"1d", size=20, start=bot_start.strftime("%Y-%m-%d"), base=30000
|
||||
)
|
||||
|
||||
exchange = MagicMock()
|
||||
exchange.get_option.return_value = True
|
||||
exchange.markets = markets
|
||||
exchange.refresh_latest_ohlcv.return_value = ohlcv_data
|
||||
exchange.get_pair_base_currency = MagicMock(side_effect=lambda pair: markets.get(pair)["base"])
|
||||
|
||||
migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
|
||||
|
||||
# Should complete migration
|
||||
assert KeyValueStore.get_int_value("wallet_history_migration") == 1
|
||||
|
||||
# Should have wallet history entries
|
||||
wallet_entries = WalletHistory.session.query(WalletHistory).all()
|
||||
assert len(wallet_entries) > 0
|
||||
|
||||
# Check that stake currency (USDT) entries exist
|
||||
usdt_entries = [e for e in wallet_entries if e.currency == "USDT"]
|
||||
assert len(usdt_entries) > 0
|
||||
assert len(wallet_entries) > len(usdt_entries)
|
||||
|
||||
# Stake currency should have price = 1.0
|
||||
for entry in usdt_entries:
|
||||
assert entry.rate == 1.0
|
||||
|
||||
eth_entries = [e for e in wallet_entries if e.currency == "ETH"]
|
||||
btc_entries = [e for e in wallet_entries if e.currency == "BTC"]
|
||||
assert len(eth_entries) == 4
|
||||
assert len(btc_entries) == 2
|
||||
assert all(entry.rate and entry.rate > 1400 and entry.rate < 1600 for entry in eth_entries)
|
||||
assert all(entry.rate and entry.rate > 29000 and entry.rate < 31000 for entry in btc_entries)
|
||||
assert all(entry.balance == 10 for entry in btc_entries)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("init_persistence")
|
||||
def test_migrate_wallet_history_pair_not_in_markets(
|
||||
default_conf_usdt, caplog, fee, time_machine, markets
|
||||
):
|
||||
"""Test migration handles pairs that are not in exchange markets."""
|
||||
start_time = dt_utc(2024, 1, 10, 12, 0, 0)
|
||||
time_machine.move_to(start_time, tick=False)
|
||||
|
||||
# Bot started 10 days ago
|
||||
bot_start = start_time - timedelta(days=10)
|
||||
KeyValueStore.store_value("bot_start_time", bot_start)
|
||||
|
||||
# Create a trade with a pair that won't be in markets
|
||||
trade1 = create_closed_mock_trade(
|
||||
fee,
|
||||
"UNKNOWN/USDT",
|
||||
open_date=start_time - timedelta(days=5),
|
||||
close_date=start_time - timedelta(days=3),
|
||||
)
|
||||
Trade.session.add(trade1)
|
||||
Trade.commit()
|
||||
|
||||
exchange = MagicMock()
|
||||
exchange.get_option.return_value = True
|
||||
exchange.markets = markets
|
||||
exchange.refresh_latest_ohlcv.return_value = {}
|
||||
|
||||
migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
|
||||
assert log_has_re("No OHLCV data available for .*", caplog)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("init_persistence")
|
||||
def test_migrate_wallet_history_stores_migration_date(
|
||||
default_conf_usdt, fee, time_machine, markets
|
||||
):
|
||||
"""Test that migration stores the migration date."""
|
||||
start_time = dt_utc(2024, 1, 10, 12, 0, 0)
|
||||
time_machine.move_to(start_time, tick=False)
|
||||
|
||||
# Bot started 10 days ago
|
||||
bot_start = start_time - timedelta(days=10)
|
||||
KeyValueStore.store_value("bot_start_time", bot_start)
|
||||
|
||||
# Create a trade
|
||||
trade1 = create_closed_mock_trade(
|
||||
fee,
|
||||
"ETH/USDT",
|
||||
open_date=start_time - timedelta(days=5),
|
||||
close_date=start_time - timedelta(days=3),
|
||||
)
|
||||
Trade.session.add(trade1)
|
||||
Trade.commit()
|
||||
|
||||
candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT)
|
||||
ohlcv_data = {
|
||||
("ETH/USDT", "1d", candle_type): generate_test_data(
|
||||
"1d", size=15, start=bot_start.strftime("%Y-%m-%d")
|
||||
)
|
||||
}
|
||||
|
||||
exchange = MagicMock()
|
||||
exchange.get_option.return_value = True
|
||||
exchange.markets = markets
|
||||
exchange.refresh_latest_ohlcv.return_value = ohlcv_data
|
||||
|
||||
migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
|
||||
|
||||
# Check migration date is stored
|
||||
migration_date = KeyValueStore.get_datetime_value("wallet_history_migration_date")
|
||||
assert migration_date is not None
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("init_persistence")
|
||||
def test_internal_migrate_wallet_history_empty_trades(default_conf_usdt, time_machine):
|
||||
"""Test _migrate_wallet_history returns early when no trades exist."""
|
||||
start_time = dt_utc(2024, 1, 1, 12, 0, 0)
|
||||
time_machine.move_to(start_time, tick=False)
|
||||
|
||||
# Set bot_start_time
|
||||
KeyValueStore.store_value("bot_start_time", start_time - timedelta(days=5))
|
||||
|
||||
exchange = MagicMock()
|
||||
exchange.get_option.return_value = True
|
||||
exchange.markets = {}
|
||||
exchange.refresh_latest_ohlcv.return_value = {}
|
||||
|
||||
# Call internal function directly with no trades
|
||||
_migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
|
||||
|
||||
# refresh_latest_ohlcv should not be called when there are no trades
|
||||
exchange.refresh_latest_ohlcv.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("init_persistence")
|
||||
def test_migrate_wallet_history_with_patched_exchange(mocker, default_conf_usdt, fee, time_machine):
|
||||
"""Test migration using get_patched_exchange helper."""
|
||||
start_time = dt_utc(2024, 1, 10, 12, 0, 0)
|
||||
time_machine.move_to(start_time, tick=False)
|
||||
|
||||
# Bot started 10 days ago
|
||||
bot_start = start_time - timedelta(days=10)
|
||||
KeyValueStore.store_value("bot_start_time", bot_start)
|
||||
|
||||
# Create a trade
|
||||
trade1 = create_closed_mock_trade(
|
||||
fee,
|
||||
"ETH/USDT",
|
||||
open_date=start_time - timedelta(days=5),
|
||||
close_date=start_time - timedelta(days=3),
|
||||
)
|
||||
Trade.session.add(trade1)
|
||||
Trade.commit()
|
||||
|
||||
# Generate mock OHLCV data starting from bot_start
|
||||
candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT)
|
||||
ohlcv_df = generate_test_data("1d", size=15, start=bot_start.strftime("%Y-%m-%d"))
|
||||
ohlcv_data = {("ETH/USDT", "1d", candle_type): ohlcv_df}
|
||||
|
||||
# Mock exchange methods
|
||||
mocker.patch.multiple(
|
||||
EXMS,
|
||||
get_option=MagicMock(return_value=True),
|
||||
refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
|
||||
)
|
||||
|
||||
exchange = get_patched_exchange(mocker, default_conf_usdt)
|
||||
|
||||
migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
|
||||
|
||||
# Should complete migration
|
||||
assert KeyValueStore.get_int_value("wallet_history_migration") == 1
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("init_persistence")
|
||||
def test_migrate_wallet_history_db_error_handling(
|
||||
mocker, default_conf_usdt, fee, time_machine, markets
|
||||
):
|
||||
"""Test that database errors are handled gracefully."""
|
||||
start_time = dt_utc(2024, 1, 10, 12, 0, 0)
|
||||
time_machine.move_to(start_time, tick=False)
|
||||
|
||||
# Bot started 10 days ago
|
||||
bot_start = start_time - timedelta(days=10)
|
||||
KeyValueStore.store_value("bot_start_time", bot_start)
|
||||
|
||||
# Create a trade
|
||||
trade1 = create_closed_mock_trade(
|
||||
fee,
|
||||
"ETH/USDT",
|
||||
open_date=start_time - timedelta(days=5),
|
||||
close_date=start_time - timedelta(days=3),
|
||||
)
|
||||
Trade.session.add(trade1)
|
||||
Trade.commit()
|
||||
|
||||
candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT)
|
||||
ohlcv_data = {
|
||||
("ETH/USDT", "1d", candle_type): generate_test_data(
|
||||
"1d", size=15, start=bot_start.strftime("%Y-%m-%d")
|
||||
)
|
||||
}
|
||||
|
||||
exchange = MagicMock()
|
||||
exchange.get_option.return_value = True
|
||||
exchange.markets = markets
|
||||
exchange.refresh_latest_ohlcv.return_value = ohlcv_data
|
||||
|
||||
# Mock bulk_save_objects to raise an exception
|
||||
mocker.patch.object(
|
||||
WalletHistory.session, "bulk_save_objects", side_effect=Exception("DB Error")
|
||||
)
|
||||
|
||||
# Should not raise exception, but handle error gracefully
|
||||
migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
|
||||
|
||||
# Migration flag should still be set even after error in _migrate
|
||||
assert KeyValueStore.get_int_value("wallet_history_migration") == 1
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("init_persistence")
|
||||
def test__prepare_balance_distribution(default_conf_usdt, fee, time_machine, markets):
|
||||
"""Test migration with multiple trading pairs."""
|
||||
start_time = dt_utc(2024, 1, 15, 12, 0, 0)
|
||||
time_machine.move_to(start_time, tick=False)
|
||||
|
||||
# Bot started 15 days ago
|
||||
bot_start = start_time - timedelta(days=15)
|
||||
KeyValueStore.store_value("bot_start_time", bot_start)
|
||||
|
||||
# Create mock trades for multiple pairs within the date range
|
||||
trade1 = create_closed_mock_trade(
|
||||
fee,
|
||||
"ETH/USDT",
|
||||
open_date=start_time - timedelta(days=10),
|
||||
close_date=start_time - timedelta(days=6),
|
||||
)
|
||||
trade2 = create_closed_mock_trade(
|
||||
fee,
|
||||
"BTC/USDT",
|
||||
open_date=start_time - timedelta(days=7),
|
||||
close_date=start_time - timedelta(days=5),
|
||||
)
|
||||
Trade.session.add(trade1)
|
||||
Trade.session.add(trade2)
|
||||
Trade.commit()
|
||||
|
||||
# Generate mock OHLCV data for both pairs starting from bot_start
|
||||
candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT)
|
||||
ohlcv_data = {}
|
||||
ohlcv_data[("ETH/USDT", "1d", candle_type)] = generate_test_data(
|
||||
"1d", size=20, start=bot_start.strftime("%Y-%m-%d"), base=1500
|
||||
)
|
||||
|
||||
ohlcv_data[("BTC/USDT", "1d", candle_type)] = generate_test_data(
|
||||
"1d", size=20, start=bot_start.strftime("%Y-%m-%d"), base=30000
|
||||
)
|
||||
|
||||
exchange = MagicMock()
|
||||
exchange.get_option.return_value = True
|
||||
exchange.markets = markets
|
||||
exchange.refresh_latest_ohlcv.return_value = ohlcv_data
|
||||
|
||||
balance_dist, pairlist_valid = _prepare_balance_distribution(
|
||||
default_conf_usdt, exchange, 1000.0
|
||||
)
|
||||
assert not balance_dist.empty
|
||||
assert len(pairlist_valid) == 2
|
||||
assert "ETH/USDT" in pairlist_valid
|
||||
assert "BTC/USDT" in pairlist_valid
|
||||
|
||||
assert len(balance_dist) == 16 # 16 days from bot_start to now
|
||||
assert balance_dist["USDT"].iloc[0] == 1000.0
|
||||
assert pd.isna(balance_dist["USDT"]).sum() == 0
|
||||
|
||||
assert all(
|
||||
col in balance_dist.columns
|
||||
for col in [
|
||||
"USDT",
|
||||
"ETH/USDT",
|
||||
"ETH/USDT_collateral",
|
||||
"ETH/USDT_leverage",
|
||||
"BTC/USDT",
|
||||
"BTC/USDT_collateral",
|
||||
"BTC/USDT_leverage",
|
||||
"ETH/USDT_open",
|
||||
"BTC/USDT_open",
|
||||
"ETH/USDT_value",
|
||||
"BTC/USDT_value",
|
||||
"total_value",
|
||||
]
|
||||
)
|
||||
Reference in New Issue
Block a user