diff --git a/freqtrade/data/btanalysis/trade_parallelism.py b/freqtrade/data/btanalysis/trade_parallelism.py index eabdcf08a..b68692cf9 100644 --- a/freqtrade/data/btanalysis/trade_parallelism.py +++ b/freqtrade/data/btanalysis/trade_parallelism.py @@ -1,9 +1,16 @@ import logging +from datetime import datetime import numpy as np import pandas as pd from freqtrade.constants import IntOrInf +from freqtrade.exchange.exchange_utils_timeframe import ( + timeframe_to_next_date, + timeframe_to_prev_date, + timeframe_to_resample_freq, +) +from freqtrade.util.datetime_helpers import dt_from_ts logger = logging.getLogger(__name__) @@ -58,3 +65,40 @@ def evaluate_result_multi( """ df_final = analyze_trade_parallelism(trades, timeframe) return df_final[df_final["open_trades"] > max_open_trades] + + +def balance_distribution_over_time( + trades: pd.DataFrame, + min_date: datetime, + max_date: datetime, + timeframe: str, + stake_currency: str, + start_balance: float, + pairlist: list[str], +) -> pd.DataFrame: + """ + Return a dataframe with stake_currency and the pairlist as columns + Each column will contain the amount of the currency at the given time + """ + min_date_res = timeframe_to_prev_date(timeframe, min_date) + max_date_res = timeframe_to_next_date(timeframe, max_date) + index = pd.date_range(min_date_res, max_date_res, freq=timeframe_to_resample_freq(timeframe)) + df = pd.DataFrame(index=index) + df[stake_currency] = float(start_balance) + df[pairlist] = 0.0 + for trade in trades.sort_values(by=["open_date"]).itertuples(): + for order in sorted(trade.orders, key=lambda x: x["order_filled_timestamp"]): + filled_at = pd.Timestamp(dt_from_ts(order["order_filled_timestamp"])) + real_amount = order["amount"] / trade.leverage + stake = order["safe_price"] * real_amount + if order["ft_is_entry"]: + fee = stake * trade.fee_open + df.loc[filled_at:, trade.pair] += real_amount + df.loc[filled_at:, stake_currency] -= stake + fee + else: + fee = stake * trade.fee_close + df.loc[filled_at:, trade.pair] -= real_amount + df.loc[filled_at:, stake_currency] += stake - fee + + df = df.round(14) + return df diff --git a/freqtrade/util/migrations/__init__.py b/freqtrade/util/migrations/__init__.py index 20aafb04b..0ed6c97da 100644 --- a/freqtrade/util/migrations/__init__.py +++ b/freqtrade/util/migrations/__init__.py @@ -1,5 +1,6 @@ from freqtrade.exchange import Exchange from freqtrade.util.migrations.funding_rate_mig import migrate_funding_fee_timeframe +from freqtrade.util.migrations.migrate_wallet_history import migrate_wallet_history def migrate_data(config, exchange: Exchange | None = None) -> None: @@ -10,10 +11,9 @@ def migrate_data(config, exchange: Exchange | None = None) -> None: migrate_funding_fee_timeframe(config, exchange) -def migrate_live_content(config, exchange: Exchange | None = None) -> None: +def migrate_live_content(config, exchange: Exchange) -> None: """ Migrate database content from old formats to new formats Used for dry/live mode. """ - # Currently not used - pass + migrate_wallet_history(config, exchange) diff --git a/freqtrade/util/migrations/migrate_wallet_history.py b/freqtrade/util/migrations/migrate_wallet_history.py new file mode 100644 index 000000000..ef3596e14 --- /dev/null +++ b/freqtrade/util/migrations/migrate_wallet_history.py @@ -0,0 +1,99 @@ +import pandas as pd + +from freqtrade.constants import Config +from freqtrade.data.btanalysis.bt_fileutils import trade_list_to_dataframe +from freqtrade.data.btanalysis.trade_parallelism import balance_distribution_over_time +from freqtrade.exchange import Exchange +from freqtrade.exchange.exchange_utils_timeframe import timeframe_to_prev_date +from freqtrade.persistence.key_value_store import KeyValueStore +from freqtrade.persistence.trade_model import Trade +from freqtrade.persistence.wallet_history import WalletBalance +from freqtrade.util.datetime_helpers import dt_now, dt_ts + + +def migrate_wallet_history(config: Config, exchange: Exchange): + if not exchange.get_option("ohlcv_has_history", True): + # we can't fill up wallet history without ohlcv history + return + trade_df = trade_list_to_dataframe(Trade.get_trades_proxy()) + if trade_df.empty: + # no trades, nothing to do + return + starting_balance = 1000 # wallets.get_starting_balance() + pairlist = list(trade_df["pair"].unique()) + timeframe = "1d" + stake_currency = config["stake_currency"] + min_date = timeframe_to_prev_date(timeframe, KeyValueStore.get_datetime_value("bot_start_time")) + balance_dist = balance_distribution_over_time( + trade_df, + min_date=min_date, + max_date=dt_now(), + start_balance=starting_balance, + stake_currency=stake_currency, + timeframe=timeframe, + pairlist=pairlist, + ) + + data = exchange.refresh_latest_ohlcv( + [(p, timeframe, config["candle_type_def"]) for p in pairlist], + since_ms=dt_ts(min_date), + cache=False, + drop_incomplete=False, + ) + + dfs = [] + # Combine all dataframes into one using the open rate + for p, x in data.items(): + x = x.set_index("date", drop=True) + col = f"{p[0]}_open" + x[col] = x["open"] + dfs.append(x[[col]]) + + merged = pd.concat(dfs, axis=1) + + balance_dist = balance_dist.join(merged, how="left") + for p in pairlist: + balance_dist[f"{p}_value"] = balance_dist[f"{p}_open"] * balance_dist[p] + + balance_dist["total_value"] = balance_dist[ + [f"{p}_value" for p in pairlist] + [stake_currency] + ].sum(axis=1) + + # Convert balance_dist to WalletBalance entries + wallet_entries = [] + for date, row in balance_dist.iterrows(): + # Add stake currency entry + if not pd.isna(row[stake_currency]): + wallet_entries.append( + WalletBalance( + timestamp=date, + currency=stake_currency, + price=1.0, # Stake currency price is always 1.0 + balance=row[stake_currency], + ) + ) + + # Add entries for each trading pair + for pair in pairlist: + base_currency = pair.split("/")[0] + # Only add entry if balance is not empty/NaN + if not pd.isna(row[pair]) and row[pair] > 0: + price_col = f"{pair}_open" + price = row[price_col] if not pd.isna(row[price_col]) else None + + wallet_entries.append( + WalletBalance( + timestamp=date, currency=base_currency, price=price, balance=row[pair] + ) + ) + + # Save entries to database + if wallet_entries: + try: + # Use bulk_save_objects for better performance + WalletBalance.session.bulk_save_objects(wallet_entries) + WalletBalance.session.commit() + print(f"Successfully created {len(wallet_entries)} wallet balance records") + except Exception as e: + WalletBalance.session.rollback() + print(f"Error saving wallet balance records: {e}")