diff --git a/tests/util/test_historic_wallets_migration.py b/tests/util/test_historic_wallets_migration.py new file mode 100644 index 000000000..5c0b17876 --- /dev/null +++ b/tests/util/test_historic_wallets_migration.py @@ -0,0 +1,425 @@ +from datetime import datetime, timedelta +from unittest.mock import MagicMock + +import pytest + +from freqtrade.enums import CandleType +from freqtrade.persistence import Trade +from freqtrade.persistence.key_value_store import KeyValueStore +from freqtrade.persistence.trade_model import Order +from freqtrade.persistence.wallet_history import WalletHistory +from freqtrade.util.datetime_helpers import dt_now, dt_utc +from freqtrade.util.migrations.migrate_wallet_history import ( + _migrate_wallet_history, + migrate_wallet_history, +) +from tests.conftest import EXMS, generate_test_data, get_patched_exchange, log_has_re + + +def create_mock_trade_for_wallet(fee, pair: str, open_date: datetime, close_date: datetime): + """Create a closed trade for wallet history testing.""" + trade = Trade( + pair=pair, + stake_amount=100.0, + amount=10.0, + amount_requested=10.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=10.0, + close_rate=11.0, + close_profit=0.1, + close_profit_abs=9.5, + exchange="binance", + is_open=False, + strategy="TestStrategy", + timeframe=5, + open_date=open_date, + close_date=close_date, + is_short=False, + ) + order_entry = Order( + ft_order_side="buy", + ft_pair=pair, + ft_is_open=False, + ft_amount=10.0, + ft_price=10.0, + order_id=f"order_{pair}_entry", + status="closed", + symbol=pair, + order_type="limit", + side="buy", + price=10.0, + average=10.0, + amount=10.0, + filled=10.0, + remaining=0.0, + order_date=open_date, + order_filled_date=open_date, + ) + + order_exit = Order( + ft_order_side="sell", + ft_pair=pair, + ft_is_open=False, + ft_amount=10.0, + ft_price=11.0, + order_id=f"order_{pair}_exit", + status="closed", + symbol=pair, + order_type="limit", + side="sell", + price=11.0, + average=11.0, + amount=10.0, + filled=10.0, + remaining=0.0, + order_date=close_date, + order_filled_date=close_date, + ) + + trade.orders.append(order_entry) + trade.orders.append(order_exit) + return trade + + +@pytest.mark.usefixtures("init_persistence") +def test_migrate_wallet_history_skips_when_no_ohlcv_history(mocker, default_conf_usdt): + """Test that migration is skipped when exchange doesn't support OHLCV history.""" + exchange = MagicMock() + exchange.get_option.return_value = False # ohlcv_has_history = False + + migrate_mock = mocker.patch( + "freqtrade.util.migrations.migrate_wallet_history._migrate_wallet_history" + ) + + migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + + # Should return early without setting the migration flag + assert KeyValueStore.get_int_value("wallet_history_migration") is None + assert not migrate_mock.called + + +@pytest.mark.usefixtures("init_persistence") +def test_migrate_wallet_history_skips_when_already_migrated(mocker, default_conf_usdt): + """Test that migration is skipped if already completed.""" + exchange = MagicMock() + exchange.get_option.return_value = True + + migrate_mock = mocker.patch( + "freqtrade.util.migrations.migrate_wallet_history._migrate_wallet_history" + ) + + # Set migration as already completed + KeyValueStore.store_value("wallet_history_migration", 1) + + migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + # Should not call any migration logic + assert KeyValueStore.get_int_value("wallet_history_migration") == 1 + assert not migrate_mock.called + + +@pytest.mark.usefixtures("init_persistence") +def test_migrate_wallet_history_no_trades(default_conf_usdt): + """Test migration with no trades in database.""" + exchange = MagicMock() + exchange.get_option.return_value = True + + # Set bot_start_time + KeyValueStore.store_value("bot_start_time", dt_now() - timedelta(days=5)) + + migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + + # Should complete migration (flag set) but no wallet entries + assert KeyValueStore.get_int_value("wallet_history_migration") == 1 + assert WalletHistory.session.query(WalletHistory).count() == 0 + + +@pytest.mark.usefixtures("init_persistence") +def test_migrate_wallet_history_with_trades(default_conf_usdt, fee, time_machine, markets): + """Test migration with trades creates wallet history entries.""" + start_time = dt_utc(2024, 1, 10, 12, 0, 0) + time_machine.move_to(start_time, tick=False) + + # Bot started 10 days ago + bot_start = start_time - timedelta(days=10) + KeyValueStore.store_value("bot_start_time", bot_start) + + # Create mock trades with dates within the range + trade_open = start_time - timedelta(days=5) + trade_close = start_time - timedelta(days=3) + trade1 = create_mock_trade_for_wallet( + fee, + "ETH/USDT", + open_date=trade_open, + close_date=trade_close, + ) + Trade.session.add(trade1) + Trade.commit() + + # Generate mock OHLCV data starting from bot_start + candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT) + ohlcv_df = generate_test_data("1d", size=15, start=bot_start.strftime("%Y-%m-%d")) + ohlcv_data = {("ETH/USDT", "1d", candle_type): ohlcv_df} + + exchange = MagicMock() + exchange.get_option.return_value = True + exchange.markets = markets + exchange.refresh_latest_ohlcv.return_value = ohlcv_data + + migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + + # Should complete migration + assert KeyValueStore.get_int_value("wallet_history_migration") == 1 + + # Should have created wallet history entries + wallet_entries = WalletHistory.session.query(WalletHistory).all() + assert len(wallet_entries) > 0 + + +@pytest.mark.usefixtures("init_persistence") +def test_migrate_wallet_history_with_multiple_pairs(default_conf_usdt, fee, time_machine, markets): + """Test migration with multiple trading pairs.""" + start_time = dt_utc(2024, 1, 15, 12, 0, 0) + time_machine.move_to(start_time, tick=False) + + # Bot started 15 days ago + bot_start = start_time - timedelta(days=15) + KeyValueStore.store_value("bot_start_time", bot_start) + + # Create mock trades for multiple pairs within the date range + trade1 = create_mock_trade_for_wallet( + fee, + "ETH/USDT", + open_date=start_time - timedelta(days=10), + close_date=start_time - timedelta(days=6), + ) + trade2 = create_mock_trade_for_wallet( + fee, + "BTC/USDT", + open_date=start_time - timedelta(days=7), + close_date=start_time - timedelta(days=5), + ) + Trade.session.add(trade1) + Trade.session.add(trade2) + Trade.commit() + + # Generate mock OHLCV data for both pairs starting from bot_start + candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT) + ohlcv_data = {} + ohlcv_data[("ETH/USDT", "1d", candle_type)] = generate_test_data( + "1d", size=20, start=bot_start.strftime("%Y-%m-%d") + ) + + ohlcv_data[("BTC/USDT", "1d", candle_type)] = generate_test_data( + "1d", size=20, start=bot_start.strftime("%Y-%m-%d") + ) + + exchange = MagicMock() + exchange.get_option.return_value = True + exchange.markets = markets + exchange.refresh_latest_ohlcv.return_value = ohlcv_data + + migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + + # Should complete migration + assert KeyValueStore.get_int_value("wallet_history_migration") == 1 + + # Should have wallet history entries + wallet_entries = WalletHistory.session.query(WalletHistory).all() + assert len(wallet_entries) > 0 + + # Check that stake currency (USDT) entries exist + usdt_entries = [e for e in wallet_entries if e.currency == "USDT"] + assert len(usdt_entries) > 0 + assert len(wallet_entries) > len(usdt_entries) + + # Stake currency should have price = 1.0 + for entry in usdt_entries: + assert entry.price == 1.0 + + eth_entries = [e for e in wallet_entries if e.currency == "ETH"] + btc_entries = [e for e in wallet_entries if e.currency == "BTC"] + assert len(eth_entries) == 4 + assert len(btc_entries) == 2 + assert all(entry.price and entry.price != 1.0 for entry in eth_entries) + assert all(entry.price and entry.price != 1.0 for entry in btc_entries) + assert all(entry.balance == 10 for entry in btc_entries) + + +@pytest.mark.usefixtures("init_persistence") +def test_migrate_wallet_history_pair_not_in_markets( + default_conf_usdt, caplog, fee, time_machine, markets +): + """Test migration handles pairs that are not in exchange markets.""" + start_time = dt_utc(2024, 1, 10, 12, 0, 0) + time_machine.move_to(start_time, tick=False) + + # Bot started 10 days ago + bot_start = start_time - timedelta(days=10) + KeyValueStore.store_value("bot_start_time", bot_start) + + # Create a trade with a pair that won't be in markets + trade1 = create_mock_trade_for_wallet( + fee, + "UNKNOWN/USDT", + open_date=start_time - timedelta(days=5), + close_date=start_time - timedelta(days=3), + ) + Trade.session.add(trade1) + Trade.commit() + + exchange = MagicMock() + exchange.get_option.return_value = True + exchange.markets = markets + exchange.refresh_latest_ohlcv.return_value = {} + + migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + assert log_has_re("No OHLCV data available for .*", caplog) + + +@pytest.mark.usefixtures("init_persistence") +def test_migrate_wallet_history_stores_migration_date( + default_conf_usdt, fee, time_machine, markets +): + """Test that migration stores the migration date.""" + start_time = dt_utc(2024, 1, 10, 12, 0, 0) + time_machine.move_to(start_time, tick=False) + + # Bot started 10 days ago + bot_start = start_time - timedelta(days=10) + KeyValueStore.store_value("bot_start_time", bot_start) + + # Create a trade + trade1 = create_mock_trade_for_wallet( + fee, + "ETH/USDT", + open_date=start_time - timedelta(days=5), + close_date=start_time - timedelta(days=3), + ) + Trade.session.add(trade1) + Trade.commit() + + candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT) + ohlcv_data = { + ("ETH/USDT", "1d", candle_type): generate_test_data( + "1d", size=15, start=bot_start.strftime("%Y-%m-%d") + ) + } + + exchange = MagicMock() + exchange.get_option.return_value = True + exchange.markets = markets + exchange.refresh_latest_ohlcv.return_value = ohlcv_data + + migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + + # Check migration date is stored + migration_date = KeyValueStore.get_datetime_value("wallet_history_migration_date") + assert migration_date is not None + + +@pytest.mark.usefixtures("init_persistence") +def test_internal_migrate_wallet_history_empty_trades(default_conf_usdt, time_machine): + """Test _migrate_wallet_history returns early when no trades exist.""" + start_time = dt_utc(2024, 1, 1, 12, 0, 0) + time_machine.move_to(start_time, tick=False) + + # Set bot_start_time + KeyValueStore.store_value("bot_start_time", start_time - timedelta(days=5)) + + exchange = MagicMock() + exchange.get_option.return_value = True + exchange.markets = {} + exchange.refresh_latest_ohlcv.return_value = {} + + # Call internal function directly with no trades + _migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + + # refresh_latest_ohlcv should not be called when there are no trades + exchange.refresh_latest_ohlcv.assert_not_called() + + +@pytest.mark.usefixtures("init_persistence") +def test_migrate_wallet_history_with_patched_exchange(mocker, default_conf_usdt, fee, time_machine): + """Test migration using get_patched_exchange helper.""" + start_time = dt_utc(2024, 1, 10, 12, 0, 0) + time_machine.move_to(start_time, tick=False) + + # Bot started 10 days ago + bot_start = start_time - timedelta(days=10) + KeyValueStore.store_value("bot_start_time", bot_start) + + # Create a trade + trade1 = create_mock_trade_for_wallet( + fee, + "ETH/USDT", + open_date=start_time - timedelta(days=5), + close_date=start_time - timedelta(days=3), + ) + Trade.session.add(trade1) + Trade.commit() + + # Generate mock OHLCV data starting from bot_start + candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT) + ohlcv_df = generate_test_data("1d", size=15, start=bot_start.strftime("%Y-%m-%d")) + ohlcv_data = {("ETH/USDT", "1d", candle_type): ohlcv_df} + + # Mock exchange methods + mocker.patch.multiple( + EXMS, + get_option=MagicMock(return_value=True), + refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), + ) + + exchange = get_patched_exchange(mocker, default_conf_usdt) + + migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + + # Should complete migration + assert KeyValueStore.get_int_value("wallet_history_migration") == 1 + + +@pytest.mark.usefixtures("init_persistence") +def test_migrate_wallet_history_db_error_handling( + mocker, default_conf_usdt, fee, time_machine, markets +): + """Test that database errors are handled gracefully.""" + start_time = dt_utc(2024, 1, 10, 12, 0, 0) + time_machine.move_to(start_time, tick=False) + + # Bot started 10 days ago + bot_start = start_time - timedelta(days=10) + KeyValueStore.store_value("bot_start_time", bot_start) + + # Create a trade + trade1 = create_mock_trade_for_wallet( + fee, + "ETH/USDT", + open_date=start_time - timedelta(days=5), + close_date=start_time - timedelta(days=3), + ) + Trade.session.add(trade1) + Trade.commit() + + candle_type = default_conf_usdt.get("candle_type_def", CandleType.SPOT) + ohlcv_data = { + ("ETH/USDT", "1d", candle_type): generate_test_data( + "1d", size=15, start=bot_start.strftime("%Y-%m-%d") + ) + } + + exchange = MagicMock() + exchange.get_option.return_value = True + exchange.markets = markets + exchange.refresh_latest_ohlcv.return_value = ohlcv_data + + # Mock bulk_save_objects to raise an exception + mocker.patch.object( + WalletHistory.session, "bulk_save_objects", side_effect=Exception("DB Error") + ) + + # Should not raise exception, but handle error gracefully + migrate_wallet_history(default_conf_usdt, exchange, 1000.0) + + # Migration flag should still be set even after error in _migrate + assert KeyValueStore.get_int_value("wallet_history_migration") == 1