Merge pull request #13019 from freqtrade/feat/capture_wallets

Capture wallet balance
This commit is contained in:
Matthias
2026-04-07 07:16:34 +02:00
committed by GitHub
33 changed files with 1738 additions and 131 deletions
+4 -2
View File
@@ -169,10 +169,12 @@ def generate_trades_history(n_rows, start_date: datetime | None = None, days=5):
return df
def generate_test_data(timeframe: str, size: int, start: str = "2020-07-05", random_seed=42):
def generate_test_data(
timeframe: str, size: int, start: str = "2020-07-05", random_seed=42, base=20
):
np.random.seed(random_seed)
base = np.random.normal(20, 2, size=size)
base = np.random.normal(base, 2, size=size)
if timeframe == "1y":
date = pd.date_range(start, periods=size, freq="1YS", tz="UTC")
elif timeframe == "1M":
+55 -12
View File
@@ -10,8 +10,9 @@ from freqtrade.configuration import TimeRange
from freqtrade.constants import LAST_BT_RESULT_FN
from freqtrade.data.btanalysis import (
BT_DATA_COLUMNS,
analyze_trade_parallelism,
extract_trades_of_period,
get_backtest_market_change,
get_backtest_wallet_change,
get_latest_backtest_filename,
get_latest_hyperopt_file,
load_backtest_data,
@@ -209,17 +210,6 @@ def test_extract_trades_of_period(testdatadir):
assert trades1.iloc[-1].close_date == datetime(2017, 11, 14, 15, 25, 0, tzinfo=UTC)
def test_analyze_trade_parallelism(testdatadir):
filename = testdatadir / "backtest_results/backtest-result.json"
bt_data = load_backtest_data(filename)
res = analyze_trade_parallelism(bt_data, "5m")
assert isinstance(res, DataFrame)
assert "open_trades" in res.columns
assert res["open_trades"].max() == 3
assert res["open_trades"].min() == 0
def test_load_trades(default_conf, mocker):
db_mock = mocker.patch(
"freqtrade.data.btanalysis.bt_fileutils.load_trades_from_db", MagicMock()
@@ -649,3 +639,56 @@ def test_load_file_from_zip(tmp_path):
with pytest.raises(ValueError, match=r"File .* not found in zip.*"):
load_file_from_zip(zip_file, "testfile55.txt")
def test_get_backtest_market_change(tmp_path):
df = DataFrame(
{
"date": [dt_utc(2020, 1, 1), dt_utc(2020, 1, 2)],
"price": [100.0, 110.0],
}
)
feather_file = tmp_path / "backtest-result_market_change.feather"
df.to_feather(feather_file)
direct_df = get_backtest_market_change(feather_file)
assert isinstance(direct_df, DataFrame)
assert "__date_ts" in direct_df.columns
assert direct_df.loc[0, "__date_ts"] == int(df.loc[0, "date"].timestamp() * 1000)
no_ts_df = get_backtest_market_change(feather_file, include_ts=False)
assert "__date_ts" not in no_ts_df.columns
zip_file = tmp_path / "backtest-result.zip"
with ZipFile(zip_file, "w") as zipf:
zipf.write(feather_file, arcname=f"{zip_file.stem}_market_change.feather")
zipped_df = get_backtest_market_change(zip_file)
assert isinstance(zipped_df, DataFrame)
assert zipped_df.loc[0, "__date_ts"] == int(df.loc[0, "date"].timestamp() * 1000)
assert list(zipped_df["price"]) == [100.0, 110.0]
def test_get_backtest_wallet_change(tmp_path):
df = DataFrame(
{
"date": [dt_utc(2020, 1, 1), dt_utc(2020, 1, 2)],
"balance": [1.0, 1.1],
"rate": [1.0, 1.1],
}
)
wallet_feather = tmp_path / "backtest-result_TestStrategy_wallet.feather"
df.to_feather(wallet_feather)
zip_file = tmp_path / "backtest-result.zip"
with ZipFile(zip_file, "w") as zipf:
zipf.write(wallet_feather, arcname=wallet_feather.name)
wallet_df = get_backtest_wallet_change(zip_file, "TestStrategy")
assert isinstance(wallet_df, DataFrame)
assert "__date_ts" in wallet_df.columns
assert wallet_df.loc[0, "__date_ts"] == int(df.loc[0, "date"].timestamp() * 1000)
assert list(wallet_df["balance"]) == [1.0, 1.1]
assert get_backtest_wallet_change(tmp_path / "backtest-result.feather", "TestStrategy") is None
assert get_backtest_wallet_change(zip_file, "UnknownStrategy") is None
+210
View File
@@ -0,0 +1,210 @@
from datetime import timedelta
import pytest
from pandas import DataFrame, Timestamp
from freqtrade.data.btanalysis import (
analyze_trade_parallelism,
load_backtest_data,
)
from freqtrade.data.btanalysis.trade_parallelism import balance_distribution_over_time
from freqtrade.util import dt_utc
def test_analyze_trade_parallelism(testdatadir):
filename = testdatadir / "backtest_results/backtest-result.json"
bt_data = load_backtest_data(filename)
res = analyze_trade_parallelism(bt_data, "5m")
assert isinstance(res, DataFrame)
assert "open_trades" in res.columns
assert res["open_trades"].max() == 3
assert res["open_trades"].min() == 0
@pytest.mark.parametrize("is_short", [False, True])
def test_balance_distribution_over_time(is_short):
"""
Test balance_distribution_over_time for both long and short trades.
"""
# Create a minimal trades DataFrame with 4 trades over time
# Base dates for trades
start_date = dt_utc(2023, 1, 1)
base_date = start_date + timedelta(hours=15)
stake_currency = "USDT"
start_balance = 1000.0
fee = 0.001 # 0.1% fee
# Create trades spanning different time periods
trades_data = {
"pair": ["BTC/USDT", "ETH/USDT", "XRP/USDT", "LTC/USDT"],
"stake_amount": [100.0, 150.0, 80.0, 120.0],
"open_date": [
base_date,
base_date + timedelta(hours=2),
base_date + timedelta(hours=5),
base_date + timedelta(hours=8),
],
"close_date": [
base_date + timedelta(hours=3),
base_date + timedelta(hours=6),
base_date + timedelta(hours=9),
base_date + timedelta(hours=12),
],
"open_rate": [40000.0, 2000.0, 0.5, 100.0],
"close_rate": [41000.0, 2100.0, 0.52, 105.0],
"fee_open": [fee, fee, fee, fee],
"fee_close": [fee, fee, fee, fee],
"is_short": [is_short, is_short, is_short, is_short],
"leverage": [1.0, 1.0, 1.0, 1.0],
"orders": [
# Trade 1: BTC/USDT - entry at 40000, exit at 41000
[
{
"amount": 0.0025, # 100 / 40000
"filled": 0.0025,
"safe_price": 40000.0,
"ft_order_side": "sell" if is_short else "buy",
"order_filled_timestamp": int(base_date.timestamp() * 1000),
"ft_is_entry": True,
},
{
"amount": 0.0025,
"filled": 0.0025,
"safe_price": 41000.0,
"ft_order_side": "buy" if is_short else "sell",
"order_filled_timestamp": int(
(base_date + timedelta(hours=3)).timestamp() * 1000
),
"ft_is_entry": False,
},
],
# Trade 2: ETH/USDT - entry at 2000, exit at 2100
[
{
"amount": 0.075, # 150 / 2000
"filled": 0.075,
"safe_price": 2000.0,
"ft_order_side": "sell" if is_short else "buy",
"order_filled_timestamp": int(
(base_date + timedelta(hours=2)).timestamp() * 1000
),
"ft_is_entry": True,
},
{
"amount": 0.075,
"filled": 0.075,
"safe_price": 2100.0,
"ft_order_side": "buy" if is_short else "sell",
"order_filled_timestamp": int(
(base_date + timedelta(hours=6)).timestamp() * 1000
),
"ft_is_entry": False,
},
],
# Trade 3: XRP/USDT - entry at 0.5, exit at 0.52
[
{
"amount": 160.0, # 80 / 0.5
"filled": 160.0,
"safe_price": 0.5,
"ft_order_side": "sell" if is_short else "buy",
"order_filled_timestamp": int(
(base_date + timedelta(hours=5)).timestamp() * 1000
),
"ft_is_entry": True,
},
{
"amount": 160.0,
"filled": 160.0,
"safe_price": 0.52,
"ft_order_side": "buy" if is_short else "sell",
"order_filled_timestamp": int(
(base_date + timedelta(hours=9)).timestamp() * 1000
),
"ft_is_entry": False,
},
],
# Trade 4: LTC/USDT - entry at 100, exit at 105
[
{
"amount": 1.2, # 120 / 100
"filled": 1.2,
"safe_price": 100.0,
"ft_order_side": "sell" if is_short else "buy",
"order_filled_timestamp": int(
(base_date + timedelta(hours=8)).timestamp() * 1000
),
"ft_is_entry": True,
},
{
"amount": 1.2,
"filled": 1.2,
"safe_price": 105.0,
"ft_order_side": "buy" if is_short else "sell",
"order_filled_timestamp": int(
(base_date + timedelta(hours=12)).timestamp() * 1000
),
"ft_is_entry": False,
},
],
],
}
trades_df = DataFrame(trades_data)
pairlist = ["BTC/USDT", "ETH/USDT", "XRP/USDT", "LTC/USDT"]
min_date = start_date
max_date = start_date + timedelta(hours=35)
result = balance_distribution_over_time(
trades=trades_df,
min_date=min_date,
max_date=max_date,
timeframe="1h",
stake_currency=stake_currency,
start_balance=start_balance,
pairlist=pairlist,
)
# Verify basic structure
assert isinstance(result, DataFrame)
assert stake_currency in result.columns
for pair in pairlist:
assert pair in result.columns
assert f"{pair}_leverage" in result.columns
assert f"{pair}_is_short" in result.columns
assert f"{pair}_collateral" in result.columns
# Verify the index is a DatetimeIndex
assert isinstance(result.index, Timestamp.__class__.__bases__[0])
# Verify we have entries over the full time period (36h)
assert len(result) == 36
# First trade opens 15h after the start date
assert result.iloc[0][stake_currency] == 1000
expected_first_balance = start_balance - (100.0 + 100.0 * fee)
assert result.iloc[15][stake_currency] == pytest.approx(expected_first_balance)
# Check that pair columns have non-zero values during trade periods
# Trade 1 (BTC/USDT) is open from hour 15 to hour 18
# At hour 16, BTC/USDT should have position
btc_during_trade = result.loc[base_date + timedelta(hours=1), "BTC/USDT"]
assert btc_during_trade > 0, "Trade should have positive position during open period"
# After Trade 1 closes at hour 3, BTC/USDT position should be 0
btc_after_close = result.loc[base_date + timedelta(hours=4) :, "BTC/USDT"]
assert all(btc_after_close == 0), "Position should be 0 after trade closes"
# Final stake currency should reflect all trades' cash flows minus fees
final_balance = result.iloc[-1][stake_currency]
# Verify the balance changed (trades had effect)
assert final_balance != start_balance, "Balance should change after trading"
# Since all exit prices > entry prices, exits return more cash than entries spent
# This means final balance > start balance for long trades and < start balance for short trades
assert (final_balance > start_balance) if not is_short else (final_balance < start_balance), (
"Balance increases for long and decreases for short trades"
)
+18
View File
@@ -757,10 +757,12 @@ def test_backtest__check_trade_exit(default_conf, mocker) -> None:
def test_backtest_one(default_conf, mocker, testdatadir) -> None:
default_conf["use_exit_signal"] = False
default_conf["max_open_trades"] = 10
default_conf["runmode"] = RunMode.BACKTEST
patch_exchange(mocker)
mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=0.00001)
mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=float("inf"))
mocker.patch(f"{EXMS}.get_pair_base_currency", lambda _, x: x.split("/")[0])
backtesting = Backtesting(default_conf)
backtesting._set_strategy(backtesting.strategylist[0])
pair = "UNITTEST/BTC"
@@ -875,13 +877,23 @@ def test_backtest_one(default_conf, mocker, testdatadir) -> None:
ln1.iloc[0]["low"], 6
) < round(t["close_rate"], 6) < round(ln1.iloc[0]["high"], 6)
wallet_summary = result["wallet_summary"]
assert isinstance(wallet_summary, pd.DataFrame)
assert len(wallet_summary) == 255
unique_currencies = wallet_summary["currency"].value_counts()
assert unique_currencies["BTC"] == 200
assert unique_currencies["UNITTEST"] == 55
@pytest.mark.parametrize("use_detail", [True, False])
def test_backtest_one_detail(default_conf_usdt, mocker, testdatadir, use_detail) -> None:
default_conf_usdt["use_exit_signal"] = False
default_conf_usdt["runmode"] = RunMode.BACKTEST
patch_exchange(mocker)
mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=0.00001)
mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=float("inf"))
mocker.patch(f"{EXMS}.get_pair_base_currency", lambda _, x: x.split("/")[0])
default_conf_usdt["unfilledtimeout"] = {
"entry": 11,
"exit": 30,
@@ -968,6 +980,12 @@ def test_backtest_one_detail(default_conf_usdt, mocker, testdatadir, use_detail)
)
assert late_entry > 0
wallet_summary = result["wallet_summary"]
assert isinstance(wallet_summary, pd.DataFrame)
assert len(wallet_summary) == 591 if use_detail else 597
unique_currencies = wallet_summary["currency"].value_counts()
assert unique_currencies["USDT"] == 576
assert unique_currencies["XRP"] == 15 if use_detail else 21
@pytest.mark.parametrize(
+87 -1
View File
@@ -7,8 +7,10 @@ import logging
import time
from copy import deepcopy
from datetime import UTC, datetime, timedelta
from io import BytesIO
from pathlib import Path
from unittest.mock import ANY, MagicMock, PropertyMock, patch
from zipfile import ZipFile
import pandas as pd
import pytest
@@ -1434,6 +1436,41 @@ def test_api_stats(botclient, mocker, ticker, fee, markets, is_short):
assert "draws" in rc.json()["durations"]
@pytest.mark.parametrize("is_short", [True, False])
def test_api_historic_balance(botclient, mocker, ticker, fee, markets, is_short):
ftbot, client = botclient
patch_get_signal(ftbot, enter_long=not is_short, enter_short=is_short)
mocker.patch.multiple(
EXMS,
get_balances=MagicMock(return_value=ticker),
fetch_ticker=ticker,
get_fee=fee,
markets=PropertyMock(return_value=markets),
)
rc = client_get(client, f"{BASE_URI}/historic_balance")
assert_response(rc, 200)
resp = rc.json()
assert "columns" in resp
assert "data" in resp
assert "length" in resp
assert "capture_start_ts" in resp
assert resp["length"] == 0
ftbot.wallets.record_wallet_state()
rc = client_get(client, f"{BASE_URI}/historic_balance")
assert_response(rc, 200)
resp1 = rc.json()
assert "columns" in resp1
assert "data" in resp1
assert "length" in resp1
assert "capture_start_ts" in resp1
assert resp1["length"] == 1
assert "__date_ts" in resp1["columns"]
assert "total_quote" in resp1["columns"]
def test_api_performance(botclient, fee):
ftbot, client = botclient
patch_get_signal(ftbot)
@@ -3275,7 +3312,7 @@ def test_api_patch_backtest_history_entry(botclient, tmp_path: Path):
assert fileres[CURRENT_TEST_STRATEGY]["notes"] == "FooBar"
def test_api_patch_backtest_market_change(botclient, tmp_path: Path):
def test_api_backtest_market_change(botclient, tmp_path: Path):
ftbot, client = botclient
# Create a temporary directory and file
@@ -3313,6 +3350,55 @@ def test_api_patch_backtest_market_change(botclient, tmp_path: Path):
]
def test_api_backtest_wallets(botclient, tmp_path: Path):
ftbot, client = botclient
# Create a temporary directory and file
bt_results_base = tmp_path / "backtest_results"
bt_results_base.mkdir()
zip_file = bt_results_base / "backtest_15.zip"
with ZipFile(zip_file, "w") as zipf:
wallet_df = pd.DataFrame(
{
"date": [
"2018-01-01T00:00:00Z",
"2018-01-01T00:00:00Z",
"2018-01-01T00:05:00Z",
"2018-01-01T00:05:00Z",
],
"currency": ["ETH", "BTC", "ETH", "BTC"],
"rate": [2000, 60_000, 2001, 60_001],
"balance": [0.5, 0.25, 0.5, 0.25],
}
)
wallet_df["date"] = pd.to_datetime(wallet_df["date"])
wallet_buf = BytesIO()
wallet_df.reset_index().to_feather(wallet_buf, compression_level=9, compression="lz4")
wallet_buf.seek(0)
zipf.writestr("backtest_15_SampleStrategy_wallet.feather", wallet_buf.read())
# Wrong basedirectory
rc = client_get(client, f"{BASE_URI}/backtest/history/randomFile.json/SampleStrategy/wallet")
assert_response(rc, 503)
ftbot.config["user_data_dir"] = tmp_path
ftbot.config["runmode"] = RunMode.WEBSERVER
# Nonexisting file - fails "is_file_in_dir" check
rc = client_get(client, f"{BASE_URI}/backtest/history/randomFile.json/SampleStrategy/wallet")
assert_response(rc, 400)
rc = client_get(client, f"{BASE_URI}/backtest/history/backtest_15/SampleStrategy/wallet")
assert_response(rc, 200)
result = rc.json()
assert result["length"] == 2
assert result["columns"] == ["date", "__date_ts", "total_quote"]
assert result["data"] == [
["2018-01-01T00:00:00Z", 1514764800000, 16000.0],
["2018-01-01T00:05:00Z", 1514765100000, 16000.75],
]
def test_health(botclient):
_ftbot, client = botclient
+81 -1
View File
@@ -7,12 +7,14 @@ from sqlalchemy import select
from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
from freqtrade.exceptions import DependencyException
from freqtrade.persistence import Trade
from freqtrade.persistence import Trade, WalletHistory
from freqtrade.wallets import PositionWallet, Wallet
from tests.conftest import (
EXMS,
create_mock_trades,
create_mock_trades_usdt,
get_patched_freqtradebot,
log_has_re,
patch_wallet,
)
@@ -607,3 +609,81 @@ def test_dry_run_wallet_initialization(mocker, default_conf_usdt, config, wallet
pytest.approx(freqtrade.wallets._wallets[stake_currency].free)
== wallets[stake_currency]["free"] - 100.0
)
@pytest.mark.usefixtures("init_persistence")
def test_record_wallet_state_stores_wallet_history(mocker, default_conf_usdt):
freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt)
stake_currency = default_conf_usdt["stake_currency"]
freqtrade.wallets._wallets = {
stake_currency: Wallet(stake_currency, free=100.0, used=50, total=150),
"BTC": Wallet("BTC", free=2.0, used=1.0, total=3.0),
}
freqtrade.wallets._positions = {
"ETH/USDT:USDT": PositionWallet(
symbol="ETH/USDT:USDT",
position=0.8,
collateral=1.0,
leverage=3.0,
side="long",
)
}
conversion_rates = {stake_currency: 1.0, "BTC": 70000, "ETH": 2500.1}
mocker.patch.object(
freqtrade.exchange,
"get_conversion_rate",
side_effect=lambda currency, *args, **kwargs: conversion_rates.get(currency, 1.0),
)
mocker.patch(
"freqtrade.persistence.trade_model.Trade.get_open_trades",
return_value=[
MagicMock(pair="ETH/USDT:USDT", safe_base_currency="ETH"),
],
)
freqtrade.wallets.record_wallet_state()
wallet_entries = WalletHistory.session.query(WalletHistory).all()
assert len(wallet_entries) == 3
assert "total_quote" in repr(wallet_entries[0])
assert "WalletHistory(" in repr(wallet_entries[0])
records_by_currency = {entry.currency: entry for entry in wallet_entries}
assert records_by_currency[stake_currency].balance == 149
assert records_by_currency[stake_currency].rate == 1.0
assert records_by_currency["BTC"].rate == 70000
assert records_by_currency["BTC"].balance == 3
assert not records_by_currency["BTC"].bot_managed
assert records_by_currency["ETH/USDT:USDT"].balance == 0.8
assert records_by_currency["ETH/USDT:USDT"].rate == 2500.1
assert records_by_currency["ETH/USDT:USDT"].bot_managed is True
@pytest.mark.usefixtures("init_persistence")
def test_record_wallet_state_stores_wallet_history_error(mocker, default_conf, caplog):
freqtrade = get_patched_freqtradebot(mocker, default_conf)
stake_currency = default_conf["stake_currency"]
freqtrade.wallets._wallets = {
stake_currency: Wallet(stake_currency, free=1.0, used=0.5, total=1.5),
"ETH": Wallet("ETH", free=2.0, used=1.0, total=3.0),
}
freqtrade.wallets._positions = {
"ETH/BTC": PositionWallet(
symbol="ETH/BTC",
position=0.8,
collateral=1.0,
leverage=3.0,
side="long",
)
}
# Mock bulk_save_objects to raise an exception
mocker.patch.object(
WalletHistory.session, "bulk_save_objects", side_effect=Exception("DB Error")
)
freqtrade.wallets.record_wallet_state()
assert log_has_re(r"Error saving wallet balance records: .*", caplog)
wallet_entries = WalletHistory.session.query(WalletHistory).all()
assert len(wallet_entries) == 0
@@ -0,0 +1,500 @@
from datetime import datetime, timedelta
from unittest.mock import MagicMock
import pandas as pd
import pytest
from freqtrade.enums import CandleType
from freqtrade.persistence import KeyValueStore, Order, Trade, WalletHistory
from freqtrade.util import dt_now, dt_utc
from freqtrade.util.migrations.migrate_wallet_history import (
_migrate_wallet_history,
_prepare_balance_distribution,
migrate_wallet_history,
)
from tests.conftest import EXMS, generate_test_data, get_patched_exchange, log_has_re
def create_closed_mock_trade(fee, pair: str, open_date: datetime, close_date: datetime):
"""Create a closed trade for wallet history testing."""
trade = Trade(
pair=pair,
stake_amount=100.0,
amount=10.0,
amount_requested=10.0,
fee_open=fee.return_value,
fee_close=fee.return_value,
open_rate=10.0,
close_rate=11.0,
close_profit=0.1,
close_profit_abs=9.5,
exchange="binance",
is_open=False,
strategy="TestStrategy",
timeframe=5,
open_date=open_date,
close_date=close_date,
is_short=False,
)
order_entry = Order(
ft_order_side="buy",
ft_pair=pair,
ft_is_open=False,
ft_amount=10.0,
ft_price=10.0,
order_id=f"order_{pair}_entry",
status="closed",
symbol=pair,
order_type="limit",
side="buy",
price=10.0,
average=10.0,
amount=10.0,
filled=10.0,
remaining=0.0,
order_date=open_date,
order_filled_date=open_date,
)
order_exit = Order(
ft_order_side="sell",
ft_pair=pair,
ft_is_open=False,
ft_amount=10.0,
ft_price=11.0,
order_id=f"order_{pair}_exit",
status="closed",
symbol=pair,
order_type="limit",
side="sell",
price=11.0,
average=11.0,
amount=10.0,
filled=10.0,
remaining=0.0,
order_date=close_date,
order_filled_date=close_date,
)
trade.orders.append(order_entry)
trade.orders.append(order_exit)
return trade
@pytest.mark.usefixtures("init_persistence")
def test_migrate_wallet_history_skips_when_no_ohlcv_history(mocker, default_conf_usdt):
"""Test that migration is skipped when exchange doesn't support OHLCV history."""
exchange = MagicMock()
exchange.get_option.return_value = False # ohlcv_has_history = False
migrate_mock = mocker.patch(
"freqtrade.util.migrations.migrate_wallet_history._migrate_wallet_history"
)
migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
# Should return early without setting the migration flag
assert KeyValueStore.get_int_value("wallet_history_migration") is None
assert not migrate_mock.called
@pytest.mark.usefixtures("init_persistence")
def test_migrate_wallet_history_skips_when_already_migrated(mocker, default_conf_usdt):
"""Test that migration is skipped if already completed."""
exchange = MagicMock()
exchange.get_option.return_value = True
migrate_mock = mocker.patch(
"freqtrade.util.migrations.migrate_wallet_history._migrate_wallet_history"
)
# Set migration as already completed
KeyValueStore.store_value("wallet_history_migration", 1)
migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
# Should not call any migration logic
assert KeyValueStore.get_int_value("wallet_history_migration") == 1
assert not migrate_mock.called
@pytest.mark.usefixtures("init_persistence")
def test_migrate_wallet_history_no_trades(default_conf_usdt):
"""Test migration with no trades in database."""
exchange = MagicMock()
exchange.get_option.return_value = True
# Set bot_start_time
KeyValueStore.store_value("bot_start_time", dt_now() - timedelta(days=5))
migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
# Should complete migration (flag set) but no wallet entries
assert KeyValueStore.get_int_value("wallet_history_migration") == 1
assert WalletHistory.session.query(WalletHistory).count() == 0
@pytest.mark.usefixtures("init_persistence")
def test_migrate_wallet_history_with_trades(default_conf_usdt, fee, time_machine, markets):
"""Test migration with trades creates wallet history entries."""
start_time = dt_utc(2024, 1, 10, 12, 0, 0)
time_machine.move_to(start_time, tick=False)
# Bot started 10 days ago
bot_start = start_time - timedelta(days=10)
KeyValueStore.store_value("bot_start_time", bot_start)
# Create mock trades with dates within the range
trade_open = start_time - timedelta(days=5)
trade_close = start_time - timedelta(days=3)
trade1 = create_closed_mock_trade(
fee,
"ETH/USDT",
open_date=trade_open,
close_date=trade_close,
)
Trade.session.add(trade1)
Trade.commit()
# Generate mock OHLCV data starting from bot_start
candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT)
ohlcv_df = generate_test_data("1d", size=15, start=bot_start.strftime("%Y-%m-%d"))
ohlcv_data = {("ETH/USDT", "1d", candle_type): ohlcv_df}
exchange = MagicMock()
exchange.get_option.return_value = True
exchange.markets = markets
exchange.refresh_latest_ohlcv.return_value = ohlcv_data
exchange.get_pair_base_currency = MagicMock(side_effect=lambda pair: markets.get(pair)["base"])
migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
# Should complete migration
assert KeyValueStore.get_int_value("wallet_history_migration") == 1
# Should have created wallet history entries
wallet_entries = WalletHistory.session.query(WalletHistory).all()
assert len(wallet_entries) > 0
@pytest.mark.usefixtures("init_persistence")
def test_migrate_wallet_history_with_multiple_pairs(default_conf_usdt, fee, time_machine, markets):
"""Test migration with multiple trading pairs."""
start_time = dt_utc(2024, 1, 15, 12, 0, 0)
time_machine.move_to(start_time, tick=False)
# Bot started 15 days ago
bot_start = start_time - timedelta(days=15)
KeyValueStore.store_value("bot_start_time", bot_start)
# Create mock trades for multiple pairs within the date range
trade1 = create_closed_mock_trade(
fee,
"ETH/USDT",
open_date=start_time - timedelta(days=10),
close_date=start_time - timedelta(days=6),
)
trade2 = create_closed_mock_trade(
fee,
"BTC/USDT",
open_date=start_time - timedelta(days=7),
close_date=start_time - timedelta(days=5),
)
Trade.session.add(trade1)
Trade.session.add(trade2)
Trade.commit()
# Generate mock OHLCV data for both pairs starting from bot_start
candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT)
ohlcv_data = {}
ohlcv_data[("ETH/USDT", "1d", candle_type)] = generate_test_data(
"1d", size=20, start=bot_start.strftime("%Y-%m-%d"), base=1500
)
ohlcv_data[("BTC/USDT", "1d", candle_type)] = generate_test_data(
"1d", size=20, start=bot_start.strftime("%Y-%m-%d"), base=30000
)
exchange = MagicMock()
exchange.get_option.return_value = True
exchange.markets = markets
exchange.refresh_latest_ohlcv.return_value = ohlcv_data
exchange.get_pair_base_currency = MagicMock(side_effect=lambda pair: markets.get(pair)["base"])
migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
# Should complete migration
assert KeyValueStore.get_int_value("wallet_history_migration") == 1
# Should have wallet history entries
wallet_entries = WalletHistory.session.query(WalletHistory).all()
assert len(wallet_entries) > 0
# Check that stake currency (USDT) entries exist
usdt_entries = [e for e in wallet_entries if e.currency == "USDT"]
assert len(usdt_entries) > 0
assert len(wallet_entries) > len(usdt_entries)
# Stake currency should have price = 1.0
for entry in usdt_entries:
assert entry.rate == 1.0
eth_entries = [e for e in wallet_entries if e.currency == "ETH"]
btc_entries = [e for e in wallet_entries if e.currency == "BTC"]
assert len(eth_entries) == 4
assert len(btc_entries) == 2
assert all(entry.rate and entry.rate > 1400 and entry.rate < 1600 for entry in eth_entries)
assert all(entry.rate and entry.rate > 29000 and entry.rate < 31000 for entry in btc_entries)
assert all(entry.balance == 10 for entry in btc_entries)
@pytest.mark.usefixtures("init_persistence")
def test_migrate_wallet_history_pair_not_in_markets(
default_conf_usdt, caplog, fee, time_machine, markets
):
"""Test migration handles pairs that are not in exchange markets."""
start_time = dt_utc(2024, 1, 10, 12, 0, 0)
time_machine.move_to(start_time, tick=False)
# Bot started 10 days ago
bot_start = start_time - timedelta(days=10)
KeyValueStore.store_value("bot_start_time", bot_start)
# Create a trade with a pair that won't be in markets
trade1 = create_closed_mock_trade(
fee,
"UNKNOWN/USDT",
open_date=start_time - timedelta(days=5),
close_date=start_time - timedelta(days=3),
)
Trade.session.add(trade1)
Trade.commit()
exchange = MagicMock()
exchange.get_option.return_value = True
exchange.markets = markets
exchange.refresh_latest_ohlcv.return_value = {}
migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
assert log_has_re("No OHLCV data available for .*", caplog)
@pytest.mark.usefixtures("init_persistence")
def test_migrate_wallet_history_stores_migration_date(
default_conf_usdt, fee, time_machine, markets
):
"""Test that migration stores the migration date."""
start_time = dt_utc(2024, 1, 10, 12, 0, 0)
time_machine.move_to(start_time, tick=False)
# Bot started 10 days ago
bot_start = start_time - timedelta(days=10)
KeyValueStore.store_value("bot_start_time", bot_start)
# Create a trade
trade1 = create_closed_mock_trade(
fee,
"ETH/USDT",
open_date=start_time - timedelta(days=5),
close_date=start_time - timedelta(days=3),
)
Trade.session.add(trade1)
Trade.commit()
candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT)
ohlcv_data = {
("ETH/USDT", "1d", candle_type): generate_test_data(
"1d", size=15, start=bot_start.strftime("%Y-%m-%d")
)
}
exchange = MagicMock()
exchange.get_option.return_value = True
exchange.markets = markets
exchange.refresh_latest_ohlcv.return_value = ohlcv_data
migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
# Check migration date is stored
migration_date = KeyValueStore.get_datetime_value("wallet_history_migration_date")
assert migration_date is not None
@pytest.mark.usefixtures("init_persistence")
def test_internal_migrate_wallet_history_empty_trades(default_conf_usdt, time_machine):
"""Test _migrate_wallet_history returns early when no trades exist."""
start_time = dt_utc(2024, 1, 1, 12, 0, 0)
time_machine.move_to(start_time, tick=False)
# Set bot_start_time
KeyValueStore.store_value("bot_start_time", start_time - timedelta(days=5))
exchange = MagicMock()
exchange.get_option.return_value = True
exchange.markets = {}
exchange.refresh_latest_ohlcv.return_value = {}
# Call internal function directly with no trades
_migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
# refresh_latest_ohlcv should not be called when there are no trades
exchange.refresh_latest_ohlcv.assert_not_called()
@pytest.mark.usefixtures("init_persistence")
def test_migrate_wallet_history_with_patched_exchange(mocker, default_conf_usdt, fee, time_machine):
"""Test migration using get_patched_exchange helper."""
start_time = dt_utc(2024, 1, 10, 12, 0, 0)
time_machine.move_to(start_time, tick=False)
# Bot started 10 days ago
bot_start = start_time - timedelta(days=10)
KeyValueStore.store_value("bot_start_time", bot_start)
# Create a trade
trade1 = create_closed_mock_trade(
fee,
"ETH/USDT",
open_date=start_time - timedelta(days=5),
close_date=start_time - timedelta(days=3),
)
Trade.session.add(trade1)
Trade.commit()
# Generate mock OHLCV data starting from bot_start
candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT)
ohlcv_df = generate_test_data("1d", size=15, start=bot_start.strftime("%Y-%m-%d"))
ohlcv_data = {("ETH/USDT", "1d", candle_type): ohlcv_df}
# Mock exchange methods
mocker.patch.multiple(
EXMS,
get_option=MagicMock(return_value=True),
refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
)
exchange = get_patched_exchange(mocker, default_conf_usdt)
migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
# Should complete migration
assert KeyValueStore.get_int_value("wallet_history_migration") == 1
@pytest.mark.usefixtures("init_persistence")
def test_migrate_wallet_history_db_error_handling(
mocker, default_conf_usdt, fee, time_machine, markets
):
"""Test that database errors are handled gracefully."""
start_time = dt_utc(2024, 1, 10, 12, 0, 0)
time_machine.move_to(start_time, tick=False)
# Bot started 10 days ago
bot_start = start_time - timedelta(days=10)
KeyValueStore.store_value("bot_start_time", bot_start)
# Create a trade
trade1 = create_closed_mock_trade(
fee,
"ETH/USDT",
open_date=start_time - timedelta(days=5),
close_date=start_time - timedelta(days=3),
)
Trade.session.add(trade1)
Trade.commit()
candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT)
ohlcv_data = {
("ETH/USDT", "1d", candle_type): generate_test_data(
"1d", size=15, start=bot_start.strftime("%Y-%m-%d")
)
}
exchange = MagicMock()
exchange.get_option.return_value = True
exchange.markets = markets
exchange.refresh_latest_ohlcv.return_value = ohlcv_data
# Mock bulk_save_objects to raise an exception
mocker.patch.object(
WalletHistory.session, "bulk_save_objects", side_effect=Exception("DB Error")
)
# Should not raise exception, but handle error gracefully
migrate_wallet_history(default_conf_usdt, exchange, 1000.0)
# Migration flag should still be set even after error in _migrate
assert KeyValueStore.get_int_value("wallet_history_migration") == 1
@pytest.mark.usefixtures("init_persistence")
def test__prepare_balance_distribution(default_conf_usdt, fee, time_machine, markets):
"""Test migration with multiple trading pairs."""
start_time = dt_utc(2024, 1, 15, 12, 0, 0)
time_machine.move_to(start_time, tick=False)
# Bot started 15 days ago
bot_start = start_time - timedelta(days=15)
KeyValueStore.store_value("bot_start_time", bot_start)
# Create mock trades for multiple pairs within the date range
trade1 = create_closed_mock_trade(
fee,
"ETH/USDT",
open_date=start_time - timedelta(days=10),
close_date=start_time - timedelta(days=6),
)
trade2 = create_closed_mock_trade(
fee,
"BTC/USDT",
open_date=start_time - timedelta(days=7),
close_date=start_time - timedelta(days=5),
)
Trade.session.add(trade1)
Trade.session.add(trade2)
Trade.commit()
# Generate mock OHLCV data for both pairs starting from bot_start
candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT)
ohlcv_data = {}
ohlcv_data[("ETH/USDT", "1d", candle_type)] = generate_test_data(
"1d", size=20, start=bot_start.strftime("%Y-%m-%d"), base=1500
)
ohlcv_data[("BTC/USDT", "1d", candle_type)] = generate_test_data(
"1d", size=20, start=bot_start.strftime("%Y-%m-%d"), base=30000
)
exchange = MagicMock()
exchange.get_option.return_value = True
exchange.markets = markets
exchange.refresh_latest_ohlcv.return_value = ohlcv_data
balance_dist, pairlist_valid = _prepare_balance_distribution(
default_conf_usdt, exchange, 1000.0
)
assert not balance_dist.empty
assert len(pairlist_valid) == 2
assert "ETH/USDT" in pairlist_valid
assert "BTC/USDT" in pairlist_valid
assert len(balance_dist) == 16 # 16 days from bot_start to now
assert balance_dist["USDT"].iloc[0] == 1000.0
assert pd.isna(balance_dist["USDT"]).sum() == 0
assert all(
col in balance_dist.columns
for col in [
"USDT",
"ETH/USDT",
"ETH/USDT_collateral",
"ETH/USDT_leverage",
"BTC/USDT",
"BTC/USDT_collateral",
"BTC/USDT_leverage",
"ETH/USDT_open",
"BTC/USDT_open",
"ETH/USDT_value",
"BTC/USDT_value",
"total_value",
]
)