From 92e64927b01b3c7d295ea953a140caa76d4e0bc5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 26 Nov 2024 07:02:54 +0100 Subject: [PATCH 01/19] feat: allow object as dry-run balance --- build_helpers/schema.json | 13 +++++++++++-- freqtrade/configuration/config_schema.py | 4 +++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/build_helpers/schema.json b/build_helpers/schema.json index e12b0bf0d..73c06ba46 100644 --- a/build_helpers/schema.json +++ b/build_helpers/schema.json @@ -102,8 +102,17 @@ }, "dry_run_wallet": { "description": "Initial wallet balance for dry run mode.", - "type": "number", - "default": 1000 + "type": [ + "number", + "object" + ], + "default": 1000, + "patternProperties": { + "^[a-zA-Z0-9]+$": { + "type": "number" + } + }, + "additionalProperties": false }, "cancel_open_orders_on_exit": { "description": "Cancel open orders when exiting.", diff --git a/freqtrade/configuration/config_schema.py b/freqtrade/configuration/config_schema.py index 30f1f6f28..d31069e7b 100644 --- a/freqtrade/configuration/config_schema.py +++ b/freqtrade/configuration/config_schema.py @@ -85,8 +85,10 @@ CONF_SCHEMA = { }, "dry_run_wallet": { "description": "Initial wallet balance for dry run mode.", - "type": "number", + "type": ["number", "object"], "default": DRY_RUN_WALLET, + "patternProperties": {r"^[a-zA-Z0-9]+$": {"type": "number"}}, + "additionalProperties": False, }, "cancel_open_orders_on_exit": { "description": "Cancel open orders when exiting.", From 5b0be7e1a9f94a31d1336dcb30f280c712d442c3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 26 Nov 2024 07:03:54 +0100 Subject: [PATCH 02/19] feat: support dict like dry_run_wallet --- freqtrade/wallets.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 2b26fb8b6..f2cff8792 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -41,7 +41,14 @@ class Wallets: self._exchange = exchange self._wallets: dict[str, Wallet] = {} self._positions: dict[str, PositionWallet] = {} - self._start_cap = config["dry_run_wallet"] + self._start_cap: dict[str, float] = {} + self._stake_currency = config["stake_currency"] + + if isinstance(_start_cap := config["dry_run_wallet"], float | int): + self._start_cap[self._stake_currency] = _start_cap + else: + self._start_cap = _start_cap + self._last_wallet_refresh: datetime | None = None self.update() @@ -112,7 +119,7 @@ class Wallets: _wallets[curr] = Wallet(curr, trade.amount - pending, pending, trade.amount) - current_stake = self._start_cap + tot_profit - tot_in_trades + current_stake = self._start_cap[self._stake_currency] + tot_profit - tot_in_trades total_stake = current_stake + used_stake else: tot_in_trades = 0 @@ -129,12 +136,13 @@ class Wallets: collateral=collateral, side=position.trade_direction, ) - current_stake = self._start_cap + tot_profit - tot_in_trades + current_stake = self._start_cap[self._stake_currency] + tot_profit - tot_in_trades + used_stake = tot_in_trades total_stake = current_stake + tot_in_trades - _wallets[self._config["stake_currency"]] = Wallet( - currency=self._config["stake_currency"], + _wallets[self._stake_currency] = Wallet( + currency=self._stake_currency, free=current_stake, used=used_stake, total=total_stake, From b4b6de4e0da6598ce458db9f5115a3ac143e2eb9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 26 Nov 2024 07:11:26 +0100 Subject: [PATCH 03/19] test: update test ... --- tests/test_wallets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_wallets.py b/tests/test_wallets.py index 2c12d27b5..3b0a88bad 100644 --- a/tests/test_wallets.py +++ b/tests/test_wallets.py @@ -168,7 +168,7 @@ def test_get_trade_stake_amount_unlimited_amount( assert result == 0 freqtrade.config["dry_run_wallet"] = 200 - freqtrade.wallets._start_cap = 200 + freqtrade.wallets._start_cap["BTC"] = 200 result = freqtrade.wallets.get_trade_stake_amount("XRP/USDT", 3) assert round(result, 4) == round(result2, 4) From 15c1a8ee0bf98eab4909796ddd941fa9f7edbadb Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 26 Nov 2024 07:13:05 +0100 Subject: [PATCH 04/19] chore: reduce dict lookups, reuse attribute --- freqtrade/wallets.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index f2cff8792..ec79e25e3 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -252,7 +252,7 @@ class Wallets: else: tot_profit = Trade.get_total_closed_profit() open_stakes = Trade.total_open_trades_stakes() - available_balance = self.get_free(self._config["stake_currency"]) + available_balance = self.get_free(self._stake_currency) return available_balance - tot_profit + open_stakes def get_total_stake_amount(self): @@ -272,9 +272,9 @@ class Wallets: # Ensure % is used from the overall balance # Otherwise we'd risk lowering stakes with each open trade. # (tied up + current free) * ratio) - tied up - available_amount = ( - val_tied_up + self.get_free(self._config["stake_currency"]) - ) * self._config["tradable_balance_ratio"] + available_amount = (val_tied_up + self.get_free(self._stake_currency)) * self._config[ + "tradable_balance_ratio" + ] return available_amount def get_available_stake_amount(self) -> float: @@ -285,7 +285,7 @@ class Wallets: ( + free amount) * tradable_balance_ratio - """ - free = self.get_free(self._config["stake_currency"]) + free = self.get_free(self._stake_currency) return min(self.get_total_stake_amount() - Trade.total_open_trades_stakes(), free) def _calculate_unlimited_stake_amount( @@ -344,8 +344,8 @@ class Wallets: if edge: stake_amount = edge.stake_amount( pair, - self.get_free(self._config["stake_currency"]), - self.get_total(self._config["stake_currency"]), + self.get_free(self._stake_currency), + self.get_total(self._stake_currency), val_tied_up, ) else: From 3fc259bb9b57995889feb0489f8fa00f1bc52ffa Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 26 Nov 2024 07:23:57 +0100 Subject: [PATCH 05/19] feat: add non-trading balance to wallet --- freqtrade/wallets.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index ec79e25e3..39b7070ec 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -116,8 +116,14 @@ class Wallets: for o in trade.open_orders if o.amount and o.ft_order_side == trade.exit_side ) + curr_wallet_bal = self._start_cap.get(curr, 0) - _wallets[curr] = Wallet(curr, trade.amount - pending, pending, trade.amount) + _wallets[curr] = Wallet( + curr, + curr_wallet_bal + trade.amount - pending, + pending, + trade.amount + curr_wallet_bal, + ) current_stake = self._start_cap[self._stake_currency] + tot_profit - tot_in_trades total_stake = current_stake + used_stake @@ -147,6 +153,11 @@ class Wallets: used=used_stake, total=total_stake, ) + for currency in self._start_cap: + if currency not in _wallets: + bal = self._start_cap[currency] + _wallets[currency] = Wallet(currency, bal, 0, bal) + self._wallets = _wallets self._positions = _positions From 37aba6f7d5a96670a776a49ddb35089240b07e27 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 26 Nov 2024 20:04:09 +0100 Subject: [PATCH 06/19] feat: Allow fetch_tickers from different marketsegment --- freqtrade/exchange/binance.py | 10 ++++++++-- freqtrade/exchange/exchange.py | 23 ++++++++++++++++++----- freqtrade/exchange/kraken.py | 10 ++++++++-- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 64b597fe8..151736e1b 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -52,8 +52,14 @@ class Binance(Exchange): (TradingMode.FUTURES, MarginMode.ISOLATED) ] - def get_tickers(self, symbols: list[str] | None = None, *, cached: bool = False) -> Tickers: - tickers = super().get_tickers(symbols=symbols, cached=cached) + def get_tickers( + self, + symbols: list[str] | None = None, + *, + cached: bool = False, + market_type: TradingMode | None = None, + ) -> Tickers: + tickers = super().get_tickers(symbols=symbols, cached=cached, market_type=market_type) if self.trading_mode == TradingMode.FUTURES: # Binance's future result has no bid/ask values. # Therefore we must fetch that from fetch_bids_asks and combine the two results. diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index a3006c99b..e7be5394b 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -201,7 +201,7 @@ class Exchange: self._cache_lock = Lock() # Cache for 10 minutes ... - self._fetch_tickers_cache: TTLCache = TTLCache(maxsize=2, ttl=60 * 10) + self._fetch_tickers_cache: TTLCache = TTLCache(maxsize=4, ttl=60 * 10) # Cache values for 300 to avoid frequent polling of the exchange for prices # Caching only applies to RPC methods, so prices for open trades are still # refreshed once every iteration. @@ -1801,24 +1801,37 @@ class Exchange: raise OperationalException(e) from e @retrier - def get_tickers(self, symbols: list[str] | None = None, *, cached: bool = False) -> Tickers: + def get_tickers( + self, + symbols: list[str] | None = None, + *, + cached: bool = False, + market_type: TradingMode | None = None, + ) -> Tickers: """ :param symbols: List of symbols to fetch :param cached: Allow cached result + :param market_type: Market type to fetch - either spot or futures. :return: fetch_tickers result """ tickers: Tickers if not self.exchange_has("fetchTickers"): return {} + cache_key = f"fetch_tickers_{market_type}" if market_type else "fetch_tickers" if cached: with self._cache_lock: - tickers = self._fetch_tickers_cache.get("fetch_tickers") # type: ignore + tickers = self._fetch_tickers_cache.get(cache_key) # type: ignore if tickers: return tickers try: - tickers = self._api.fetch_tickers(symbols) + # Re-map futures to swap + market_types = { + TradingMode.FUTURES: "swap", + } + params = {"type": market_types.get(market_type, market_type)} if market_type else {} + tickers = self._api.fetch_tickers(symbols, params) with self._cache_lock: - self._fetch_tickers_cache["fetch_tickers"] = tickers + self._fetch_tickers_cache[cache_key] = tickers return tickers except ccxt.NotSupported as e: raise OperationalException( diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index b2035fcef..53fce867b 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -50,11 +50,17 @@ class Kraken(Exchange): return parent_check and market.get("darkpool", False) is False - def get_tickers(self, symbols: list[str] | None = None, *, cached: bool = False) -> Tickers: + def get_tickers( + self, + symbols: list[str] | None = None, + *, + cached: bool = False, + market_type: TradingMode | None = None, + ) -> Tickers: # Only fetch tickers for current stake currency # Otherwise the request for kraken becomes too large. symbols = list(self.get_markets(quote_currencies=[self._config["stake_currency"]])) - return super().get_tickers(symbols=symbols, cached=cached) + return super().get_tickers(symbols=symbols, cached=cached, market_type=market_type) @retrier def get_balances(self) -> CcxtBalances: From 2bb111605c9daed19ae61200eb3b9cfb4dd31c35 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 26 Nov 2024 20:06:28 +0100 Subject: [PATCH 07/19] feat: update rpc_balance to fetch spot tickers Happens if there is no market for the given pair --- freqtrade/rpc/rpc.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index afa8baea2..d4c05d2d2 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -7,7 +7,7 @@ from abc import abstractmethod from collections.abc import Generator, Sequence from datetime import date, datetime, timedelta, timezone from math import isnan -from typing import Any, cast +from typing import Any import psutil from dateutil.relativedelta import relativedelta @@ -682,12 +682,23 @@ class RPC: est_bot_stake = amount else: pair = self._freqtrade.exchange.get_valid_pair_combination(coin, stake_currency) - rate: float | None = cast(Ticker, tickers.get(pair, {})).get("last", None) - if rate: - if pair.startswith(stake_currency) and not pair.endswith(stake_currency): - rate = 1.0 / rate - est_stake = rate * balance.total - est_bot_stake = rate * amount + ticker: Ticker | None = tickers.get(pair, None) + if not ticker: + tickers_spot: Tickers = self._freqtrade.exchange.get_tickers( + cached=True, + market_type=TradingMode.SPOT + if self._config.get("trading_mode", TradingMode.SPOT) != TradingMode.SPOT + else TradingMode.FUTURES, + ) + ticker = tickers_spot.get(pair, None) + + if ticker: + rate: float | None = ticker.get("last", None) + if rate: + if pair.startswith(stake_currency) and not pair.endswith(stake_currency): + rate = 1.0 / rate + est_stake = rate * balance.total + est_bot_stake = rate * amount return est_stake, est_bot_stake From e6e193f2520a824276649d7c11f90413d1154d3f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 26 Nov 2024 20:17:56 +0100 Subject: [PATCH 08/19] test: assert tickers is called a 2nd time if necessary --- tests/rpc/test_rpc.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index dd8c1bb9a..efdc18977 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -530,6 +530,13 @@ def test_rpc_balance_handle(default_conf_usdt, mocker, tickers): "total": 5.0, "used": 4.0, }, + # Invalid coin not in tickers list. + # This triggers a 2nd call to get_tickers + "NotACoin": { + "free": 0.0, + "total": 2.0, + "used": 0.0, + }, "USDT": { "free": 50.0, "total": 100.0, @@ -590,8 +597,10 @@ def test_rpc_balance_handle(default_conf_usdt, mocker, tickers): assert pytest.approx(result["total"]) == 2824.83464 assert pytest.approx(result["value"]) == 2824.83464 * 1.2 - assert tickers.call_count == 1 + assert tickers.call_count == 2 assert tickers.call_args_list[0][1]["cached"] is True + # Testing futures - so we should get spot tickers + assert tickers.call_args_list[1][1]["market_type"] == "spot" assert "USD" == result["symbol"] assert result["currencies"] == [ { @@ -622,6 +631,20 @@ def test_rpc_balance_handle(default_conf_usdt, mocker, tickers): "is_bot_managed": False, "is_position": False, }, + { + "currency": "NotACoin", + "balance": 2.0, + "bot_owned": 0, + "est_stake": 0, + "est_stake_bot": 0, + "free": 0.0, + "is_bot_managed": False, + "is_position": False, + "position": 0, + "side": "long", + "stake": "USDT", + "used": 0.0, + }, { "currency": "USDT", "free": 50.0, From 7369331e2dbdc527ac24e9791e9815d418ba8fc9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 26 Nov 2024 20:35:40 +0100 Subject: [PATCH 09/19] tests: add test for multi-pair dry-run wallets --- tests/test_wallets.py | 60 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/test_wallets.py b/tests/test_wallets.py index 3b0a88bad..d5e49438a 100644 --- a/tests/test_wallets.py +++ b/tests/test_wallets.py @@ -451,3 +451,63 @@ def test_check_exit_amount_futures(mocker, default_conf, fee): assert freqtrade.wallets.check_exit_amount(trade) is False assert total_mock.call_count == 0 assert update_mock.call_count == 1 + + +@pytest.mark.parametrize( + "config,wallets", + [ + ( + {"stake_currency": "USDT", "dry_run_wallet": 1000.0}, + {"USDT": {"currency": "USDT", "free": 1000.0, "used": 0.0, "total": 1000.0}}, + ), + ( + {"stake_currency": "USDT", "dry_run_wallet": {"USDT": 1000.0, "BTC": 0.1, "ETH": 2.0}}, + { + "USDT": {"currency": "USDT", "free": 1000.0, "used": 0.0, "total": 1000.0}, + "BTC": {"currency": "BTC", "free": 0.1, "used": 0.0, "total": 0.1}, + "ETH": {"currency": "ETH", "free": 2.0, "used": 0.0, "total": 2.0}, + }, + ), + ], +) +def test_dry_run_wallet_initialization(mocker, default_conf_usdt, config, wallets): + default_conf_usdt.update(config) + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) + + # Verify each wallet matches the expected values + for currency, expected_wallet in wallets.items(): + wallet = freqtrade.wallets._wallets[currency] + assert wallet.currency == expected_wallet["currency"] + assert wallet.free == expected_wallet["free"] + assert wallet.used == expected_wallet["used"] + assert wallet.total == expected_wallet["total"] + + # Verify no extra wallets were created + assert len(freqtrade.wallets._wallets) == len(wallets) + + # Create a trade and verify the new currency is added to the wallets + mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=0.0) + mocker.patch(f"{EXMS}.get_rate", return_value=2.22) + mocker.patch( + f"{EXMS}.fetch_ticker", + return_value={ + "bid": 0.20, + "ask": 0.22, + "last": 0.22, + }, + ) + freqtrade.execute_entry("NEO/USDT", 100.0) + + # Update wallets and verify NEO is now included + freqtrade.wallets.update() + assert "NEO" in freqtrade.wallets._wallets + + assert freqtrade.wallets._wallets["NEO"].total == 45.04504504 # 100 USDT / 0.22 + assert freqtrade.wallets._wallets["NEO"].used == 0.0 + assert freqtrade.wallets._wallets["NEO"].free == 45.04504504 + + # Verify USDT wallet was reduced by trade amount + assert ( + pytest.approx(freqtrade.wallets._wallets["USDT"].total) == wallets["USDT"]["total"] - 100.0 + ) + assert len(freqtrade.wallets._wallets) == len(wallets) + 1 # Original wallets + NEO From 671821aeb3ec1a4fdee4c94b8b9931fb323ee976 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 27 Nov 2024 06:48:07 +0100 Subject: [PATCH 10/19] docs: Add documentation for dry-run-wallet as dict --- docs/configuration.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 9c46b7546..cfce38176 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -168,7 +168,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `timeframe` | The timeframe to use (e.g `1m`, `5m`, `15m`, `30m`, `1h` ...). Usually missing in configuration, and specified in the strategy. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** String | `fiat_display_currency` | Fiat currency used to show your profits. [More information below](#what-values-can-be-used-for-fiat_display_currency).
**Datatype:** String | `dry_run` | **Required.** Define if the bot must be in Dry Run or production mode.
*Defaults to `true`.*
**Datatype:** Boolean -| `dry_run_wallet` | Define the starting amount in stake currency for the simulated wallet used by the bot running in Dry Run mode.
*Defaults to `1000`.*
**Datatype:** Float +| `dry_run_wallet` | Define the starting amount in stake currency for the simulated wallet used by the bot running in Dry Run mode. [More information below](#dry-run-wallet)
*Defaults to `1000`.*
**Datatype:** Float or Dict | `cancel_open_orders_on_exit` | Cancel open orders when the `/stop` RPC command is issued, `Ctrl+C` is pressed or the bot dies unexpectedly. When set to `true`, this allows you to use `/stop` to cancel unfilled and partially filled orders in the event of a market crash. It does not impact open positions.
*Defaults to `false`.*
**Datatype:** Boolean | `process_only_new_candles` | Enable processing of indicators only when new candles arrive. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `true`.*
**Datatype:** Boolean | `minimal_roi` | **Required.** Set the threshold as ratio the bot will use to exit a trade. [More information below](#understand-minimal_roi). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict @@ -323,6 +323,25 @@ To limit this calculation in case of large stoploss values, the calculated minim !!! Warning Since the limits on exchanges are usually stable and are not updated often, some pairs can show pretty high minimum limits, simply because the price increased a lot since the last limit adjustment by the exchange. Freqtrade adjusts the stake-amount to this value, unless it's > 30% more than the calculated/desired stake-amount - in which case the trade is rejected. +#### Dry-run wallet + +When running in dry-run mode, the bot will use a simulated wallet to execute trades. The starting balance of this wallet is defined by `dry_run_wallet` (defaults to 1000). +For more complex scenarios, you can also assign a dictionary to `dry_run_wallet` to define the starting balance for each currency. + +```json +"dry_run_wallet": { + "BTC": 0.01, + "ETH": 2, + "USDT": 1000 +} +``` + +Command line options (`--dry-run-wallet`) can be used to override the configuration value, but only for the float value, not for the dictionary. If you'd like to use the dictionary, please adjust the configuration file. + +!!! Note + Balances not in stake-currency will not be used for trading, but are shown as part of the wallet balance. + On Cross-margin exchanges, the wallet balance may be used to calculate the available collateral for trading. + #### Tradable balance By default, the bot assumes that the `complete amount - 1%` is at it's disposal, and when using [dynamic stake amount](#dynamic-stake-amount), it will split the complete balance into `max_open_trades` buckets per trade. From 09308e568d1b8c40b71979b55f4417f48d3c593a Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 27 Nov 2024 07:14:11 +0100 Subject: [PATCH 11/19] fix: increase code reliability by not relying on stake-currency to be in the dict --- freqtrade/wallets.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 39b7070ec..9864d0604 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -125,7 +125,9 @@ class Wallets: trade.amount + curr_wallet_bal, ) - current_stake = self._start_cap[self._stake_currency] + tot_profit - tot_in_trades + current_stake = ( + self._start_cap.get(self._stake_currency, 0) + tot_profit - tot_in_trades + ) total_stake = current_stake + used_stake else: tot_in_trades = 0 @@ -142,7 +144,9 @@ class Wallets: collateral=collateral, side=position.trade_direction, ) - current_stake = self._start_cap[self._stake_currency] + tot_profit - tot_in_trades + current_stake = ( + self._start_cap.get(self._stake_currency, 0) + tot_profit - tot_in_trades + ) used_stake = tot_in_trades total_stake = current_stake + tot_in_trades From c082e5f6a6f5271bf26ba3ed4bc0ec94ed8d2c0b Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 3 Dec 2024 07:26:47 +0100 Subject: [PATCH 12/19] feat: add dry_run_wallet helper --- freqtrade/util/__init__.py | 2 ++ freqtrade/util/dry_run_wallet.py | 12 ++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 freqtrade/util/dry_run_wallet.py diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index 7a4d4d119..a0b618b11 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -11,6 +11,7 @@ from freqtrade.util.datetime_helpers import ( format_ms_time, shorten_date, ) +from freqtrade.util.dry_run_wallet import get_dry_run_wallet from freqtrade.util.formatters import decimals_per_coin, fmt_coin, fmt_coin2, round_value from freqtrade.util.ft_precise import FtPrecise from freqtrade.util.measure_time import MeasureTime @@ -35,6 +36,7 @@ __all__ = [ "dt_utc", "format_date", "format_ms_time", + "get_dry_run_wallet", "FtPrecise", "PeriodicCache", "shorten_date", diff --git a/freqtrade/util/dry_run_wallet.py b/freqtrade/util/dry_run_wallet.py new file mode 100644 index 000000000..f994f0d4c --- /dev/null +++ b/freqtrade/util/dry_run_wallet.py @@ -0,0 +1,12 @@ +from pytest import Config + + +def get_dry_run_wallet(config: Config) -> int | float: + """ + Return dry-run wallet balance in stake currency from configuration. + This setup also supports dictionary mode for dry-run-wallet. + """ + if isinstance(_start_cap := config["dry_run_wallet"], float | int): + return _start_cap + else: + return _start_cap.get("stake_currency") From 7a8971b9b6051a20978355048f1929a03b0e46d9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 3 Dec 2024 07:27:45 +0100 Subject: [PATCH 13/19] feat: use get_dry_run_wallet helper --- freqtrade/commands/optimize_commands.py | 3 ++- freqtrade/optimize/analysis/lookahead_helpers.py | 4 ++-- freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py | 3 ++- .../hyperopt_loss/hyperopt_loss_max_drawdown_relative.py | 3 ++- .../optimize/hyperopt_loss/hyperopt_loss_multi_metric.py | 3 ++- .../optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py | 3 ++- freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py | 3 ++- freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py | 3 ++- freqtrade/optimize/optimize_reports/optimize_reports.py | 4 ++-- freqtrade/plot/plotting.py | 3 ++- 10 files changed, 20 insertions(+), 12 deletions(-) diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index ea75f3bd1..788bca489 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -4,6 +4,7 @@ from typing import Any from freqtrade import constants from freqtrade.enums import RunMode from freqtrade.exceptions import ConfigurationError, OperationalException +from freqtrade.util import get_dry_run_wallet logger = logging.getLogger(__name__) @@ -26,7 +27,7 @@ def setup_optimize_configuration(args: dict[str, Any], method: RunMode) -> dict[ RunMode.HYPEROPT: "hyperoptimization", } if method in no_unlimited_runmodes.keys(): - wallet_size = config["dry_run_wallet"] * config["tradable_balance_ratio"] + wallet_size = get_dry_run_wallet(config) * config["tradable_balance_ratio"] # tradable_balance_ratio if ( config["stake_amount"] != constants.UNLIMITED_STAKE_AMOUNT diff --git a/freqtrade/optimize/analysis/lookahead_helpers.py b/freqtrade/optimize/analysis/lookahead_helpers.py index d664f9635..631a9549f 100644 --- a/freqtrade/optimize/analysis/lookahead_helpers.py +++ b/freqtrade/optimize/analysis/lookahead_helpers.py @@ -10,7 +10,7 @@ from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.optimize.analysis.lookahead import LookaheadAnalysis from freqtrade.resolvers import StrategyResolver -from freqtrade.util import print_rich_table +from freqtrade.util import get_dry_run_wallet, print_rich_table logger = logging.getLogger(__name__) @@ -163,7 +163,7 @@ class LookaheadAnalysisSubFunctions: config["max_open_trades"] = len(config["pairs"]) min_dry_run_wallet = 1000000000 - if config["dry_run_wallet"] < min_dry_run_wallet: + if get_dry_run_wallet(config) < min_dry_run_wallet: logger.info( "Dry run wallet was not set to 1 billion, pushing it up there " "just to avoid false positives" diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py index f22d59e50..6072629c4 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py @@ -12,6 +12,7 @@ from pandas import DataFrame from freqtrade.constants import Config from freqtrade.data.metrics import calculate_calmar from freqtrade.optimize.hyperopt import IHyperOptLoss +from freqtrade.util import get_dry_run_wallet class CalmarHyperOptLoss(IHyperOptLoss): @@ -36,7 +37,7 @@ class CalmarHyperOptLoss(IHyperOptLoss): Uses Calmar Ratio calculation. """ - starting_balance = config["dry_run_wallet"] + starting_balance = get_dry_run_wallet(config) calmar_ratio = calculate_calmar(results, min_date, max_date, starting_balance) # print(expected_returns_mean, max_drawdown, calmar_ratio) return -calmar_ratio diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown_relative.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown_relative.py index ee7088d75..4bbbcf7d3 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown_relative.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown_relative.py @@ -10,6 +10,7 @@ from pandas import DataFrame from freqtrade.constants import Config from freqtrade.data.metrics import calculate_underwater from freqtrade.optimize.hyperopt import IHyperOptLoss +from freqtrade.util import get_dry_run_wallet class MaxDrawDownRelativeHyperOptLoss(IHyperOptLoss): @@ -31,7 +32,7 @@ class MaxDrawDownRelativeHyperOptLoss(IHyperOptLoss): total_profit = results["profit_abs"].sum() try: drawdown_df = calculate_underwater( - results, value_col="profit_abs", starting_balance=config["dry_run_wallet"] + results, value_col="profit_abs", starting_balance=get_dry_run_wallet(config) ) max_drawdown = abs(min(drawdown_df["drawdown"])) relative_drawdown = max(drawdown_df["drawdown_relative"]) diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_multi_metric.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_multi_metric.py index de8d117d6..dd5fa4a17 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_multi_metric.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_multi_metric.py @@ -36,6 +36,7 @@ from pandas import DataFrame from freqtrade.constants import Config from freqtrade.data.metrics import calculate_expectancy, calculate_max_drawdown from freqtrade.optimize.hyperopt import IHyperOptLoss +from freqtrade.util import get_dry_run_wallet # smaller numbers penalize drawdowns more severely @@ -83,7 +84,7 @@ class MultiMetricHyperOptLoss(IHyperOptLoss): # Calculate drawdown try: drawdown = calculate_max_drawdown( - results, starting_balance=config["dry_run_wallet"], value_col="profit_abs" + results, starting_balance=get_dry_run_wallet(config), value_col="profit_abs" ) relative_account_drawdown = drawdown.relative_account_drawdown except ValueError: diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py index 61e2a6d32..5230408a9 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py @@ -13,6 +13,7 @@ from pandas import DataFrame from freqtrade.constants import Config from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.optimize.hyperopt import IHyperOptLoss +from freqtrade.util import get_dry_run_wallet # smaller numbers penalize drawdowns more severely @@ -26,7 +27,7 @@ class ProfitDrawDownHyperOptLoss(IHyperOptLoss): try: drawdown = calculate_max_drawdown( - results, starting_balance=config["dry_run_wallet"], value_col="profit_abs" + results, starting_balance=get_dry_run_wallet(config), value_col="profit_abs" ) relative_account_drawdown = drawdown.relative_account_drawdown except ValueError: diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py index 2c7042a8a..4806ddb7f 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py @@ -12,6 +12,7 @@ from pandas import DataFrame from freqtrade.constants import Config from freqtrade.data.metrics import calculate_sharpe from freqtrade.optimize.hyperopt import IHyperOptLoss +from freqtrade.util.dry_run_wallet import get_dry_run_wallet class SharpeHyperOptLoss(IHyperOptLoss): @@ -36,7 +37,7 @@ class SharpeHyperOptLoss(IHyperOptLoss): Uses Sharpe Ratio calculation. """ - starting_balance = config["dry_run_wallet"] + starting_balance = get_dry_run_wallet(config) sharp_ratio = calculate_sharpe(results, min_date, max_date, starting_balance) # print(expected_returns_mean, up_stdev, sharp_ratio) return -sharp_ratio diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py index 32ff0c73f..c01be889d 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py @@ -12,6 +12,7 @@ from pandas import DataFrame from freqtrade.constants import Config from freqtrade.data.metrics import calculate_sortino from freqtrade.optimize.hyperopt import IHyperOptLoss +from freqtrade.util import get_dry_run_wallet class SortinoHyperOptLoss(IHyperOptLoss): @@ -36,7 +37,7 @@ class SortinoHyperOptLoss(IHyperOptLoss): Uses Sortino Ratio calculation. """ - starting_balance = config["dry_run_wallet"] + starting_balance = get_dry_run_wallet(config) sortino_ratio = calculate_sortino(results, min_date, max_date, starting_balance) # print(expected_returns_mean, down_stdev, sortino_ratio) return -sortino_ratio diff --git a/freqtrade/optimize/optimize_reports/optimize_reports.py b/freqtrade/optimize/optimize_reports/optimize_reports.py index 8f1f0140a..532abed39 100644 --- a/freqtrade/optimize/optimize_reports/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports/optimize_reports.py @@ -18,7 +18,7 @@ from freqtrade.data.metrics import ( calculate_sortino, ) from freqtrade.ft_types import BacktestResultType -from freqtrade.util import decimals_per_coin, fmt_coin +from freqtrade.util import decimals_per_coin, fmt_coin, get_dry_run_wallet logger = logging.getLogger(__name__) @@ -373,7 +373,7 @@ def generate_strategy_stats( return {} config = content["config"] max_open_trades = min(config["max_open_trades"], len(pairlist)) - start_balance = config["dry_run_wallet"] + start_balance = get_dry_run_wallet(config) stake_currency = config["stake_currency"] pair_results = generate_pair_metrics( diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index cf86f070d..8ec7c7bae 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -28,6 +28,7 @@ from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.strategy import IStrategy from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper +from freqtrade.util import get_dry_run_wallet logger = logging.getLogger(__name__) @@ -706,7 +707,7 @@ def plot_profit(config: Config) -> None: trades, config["timeframe"], config.get("stake_currency", ""), - config.get("available_capital", config["dry_run_wallet"]), + config.get("available_capital", get_dry_run_wallet(config)), ) store_plot_file( fig, From b0b73bf166d7545f361524ca3b714f582c50de79 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 3 Dec 2024 19:15:57 +0100 Subject: [PATCH 14/19] feat: add starting_balance as argument to hyperopt_loss_function --- freqtrade/optimize/hyperopt/hyperopt_optimizer.py | 2 ++ freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py | 3 +-- .../optimize/hyperopt_loss/hyperopt_loss_interface.py | 1 + .../hyperopt_loss/hyperopt_loss_max_drawdown_relative.py | 7 ++++--- .../optimize/hyperopt_loss/hyperopt_loss_multi_metric.py | 4 ++-- .../hyperopt_loss/hyperopt_loss_profit_drawdown.py | 7 ++++--- freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py | 6 +----- freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py | 6 +----- tests/optimize/test_hyperoptloss.py | 1 + 9 files changed, 17 insertions(+), 20 deletions(-) diff --git a/freqtrade/optimize/hyperopt/hyperopt_optimizer.py b/freqtrade/optimize/hyperopt/hyperopt_optimizer.py index bd234aa57..c8d18d224 100644 --- a/freqtrade/optimize/hyperopt/hyperopt_optimizer.py +++ b/freqtrade/optimize/hyperopt/hyperopt_optimizer.py @@ -28,6 +28,7 @@ from freqtrade.optimize.hyperopt_loss.hyperopt_loss_interface import IHyperOptLo from freqtrade.optimize.hyperopt_tools import HyperoptStateContainer, HyperoptTools from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver +from freqtrade.util.dry_run_wallet import get_dry_run_wallet # Suppress scikit-learn FutureWarnings from skopt @@ -363,6 +364,7 @@ class HyperOptimizer: config=self.config, processed=processed, backtest_stats=strat_stats, + starting_balance=get_dry_run_wallet(self.config), ) return { "loss": loss, diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py index 6072629c4..2a04d4070 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py @@ -12,7 +12,6 @@ from pandas import DataFrame from freqtrade.constants import Config from freqtrade.data.metrics import calculate_calmar from freqtrade.optimize.hyperopt import IHyperOptLoss -from freqtrade.util import get_dry_run_wallet class CalmarHyperOptLoss(IHyperOptLoss): @@ -29,6 +28,7 @@ class CalmarHyperOptLoss(IHyperOptLoss): min_date: datetime, max_date: datetime, config: Config, + starting_balance: float, *args, **kwargs, ) -> float: @@ -37,7 +37,6 @@ class CalmarHyperOptLoss(IHyperOptLoss): Uses Calmar Ratio calculation. """ - starting_balance = get_dry_run_wallet(config) calmar_ratio = calculate_calmar(results, min_date, max_date, starting_balance) # print(expected_returns_mean, max_drawdown, calmar_ratio) return -calmar_ratio diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_interface.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_interface.py index a48fee731..ddc96d6a0 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_interface.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_interface.py @@ -31,6 +31,7 @@ class IHyperOptLoss(ABC): config: Config, processed: dict[str, DataFrame], backtest_stats: dict[str, Any], + starting_balance: float, **kwargs, ) -> float: """ diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown_relative.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown_relative.py index 4bbbcf7d3..a753263a3 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown_relative.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown_relative.py @@ -10,7 +10,6 @@ from pandas import DataFrame from freqtrade.constants import Config from freqtrade.data.metrics import calculate_underwater from freqtrade.optimize.hyperopt import IHyperOptLoss -from freqtrade.util import get_dry_run_wallet class MaxDrawDownRelativeHyperOptLoss(IHyperOptLoss): @@ -22,7 +21,9 @@ class MaxDrawDownRelativeHyperOptLoss(IHyperOptLoss): """ @staticmethod - def hyperopt_loss_function(results: DataFrame, config: Config, *args, **kwargs) -> float: + def hyperopt_loss_function( + results: DataFrame, config: Config, starting_balance: float, *args, **kwargs + ) -> float: """ Objective function. @@ -32,7 +33,7 @@ class MaxDrawDownRelativeHyperOptLoss(IHyperOptLoss): total_profit = results["profit_abs"].sum() try: drawdown_df = calculate_underwater( - results, value_col="profit_abs", starting_balance=get_dry_run_wallet(config) + results, value_col="profit_abs", starting_balance=starting_balance ) max_drawdown = abs(min(drawdown_df["drawdown"])) relative_drawdown = max(drawdown_df["drawdown_relative"]) diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_multi_metric.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_multi_metric.py index dd5fa4a17..7a10d279b 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_multi_metric.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_multi_metric.py @@ -36,7 +36,6 @@ from pandas import DataFrame from freqtrade.constants import Config from freqtrade.data.metrics import calculate_expectancy, calculate_max_drawdown from freqtrade.optimize.hyperopt import IHyperOptLoss -from freqtrade.util import get_dry_run_wallet # smaller numbers penalize drawdowns more severely @@ -59,6 +58,7 @@ class MultiMetricHyperOptLoss(IHyperOptLoss): results: DataFrame, trade_count: int, config: Config, + starting_balance: float, **kwargs, ) -> float: total_profit = results["profit_abs"].sum() @@ -84,7 +84,7 @@ class MultiMetricHyperOptLoss(IHyperOptLoss): # Calculate drawdown try: drawdown = calculate_max_drawdown( - results, starting_balance=get_dry_run_wallet(config), value_col="profit_abs" + results, starting_balance=starting_balance, value_col="profit_abs" ) relative_account_drawdown = drawdown.relative_account_drawdown except ValueError: diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py index 5230408a9..2ab581520 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py @@ -13,7 +13,6 @@ from pandas import DataFrame from freqtrade.constants import Config from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.optimize.hyperopt import IHyperOptLoss -from freqtrade.util import get_dry_run_wallet # smaller numbers penalize drawdowns more severely @@ -22,12 +21,14 @@ DRAWDOWN_MULT = 0.075 class ProfitDrawDownHyperOptLoss(IHyperOptLoss): @staticmethod - def hyperopt_loss_function(results: DataFrame, config: Config, *args, **kwargs) -> float: + def hyperopt_loss_function( + results: DataFrame, config: Config, starting_balance: float, *args, **kwargs + ) -> float: total_profit = results["profit_abs"].sum() try: drawdown = calculate_max_drawdown( - results, starting_balance=get_dry_run_wallet(config), value_col="profit_abs" + results, starting_balance=starting_balance, value_col="profit_abs" ) relative_account_drawdown = drawdown.relative_account_drawdown except ValueError: diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py index 4806ddb7f..20e4ee2b6 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py @@ -9,10 +9,8 @@ from datetime import datetime from pandas import DataFrame -from freqtrade.constants import Config from freqtrade.data.metrics import calculate_sharpe from freqtrade.optimize.hyperopt import IHyperOptLoss -from freqtrade.util.dry_run_wallet import get_dry_run_wallet class SharpeHyperOptLoss(IHyperOptLoss): @@ -25,10 +23,9 @@ class SharpeHyperOptLoss(IHyperOptLoss): @staticmethod def hyperopt_loss_function( results: DataFrame, - trade_count: int, min_date: datetime, max_date: datetime, - config: Config, + starting_balance: float, *args, **kwargs, ) -> float: @@ -37,7 +34,6 @@ class SharpeHyperOptLoss(IHyperOptLoss): Uses Sharpe Ratio calculation. """ - starting_balance = get_dry_run_wallet(config) sharp_ratio = calculate_sharpe(results, min_date, max_date, starting_balance) # print(expected_returns_mean, up_stdev, sharp_ratio) return -sharp_ratio diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py index c01be889d..935d038e5 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py @@ -9,10 +9,8 @@ from datetime import datetime from pandas import DataFrame -from freqtrade.constants import Config from freqtrade.data.metrics import calculate_sortino from freqtrade.optimize.hyperopt import IHyperOptLoss -from freqtrade.util import get_dry_run_wallet class SortinoHyperOptLoss(IHyperOptLoss): @@ -25,10 +23,9 @@ class SortinoHyperOptLoss(IHyperOptLoss): @staticmethod def hyperopt_loss_function( results: DataFrame, - trade_count: int, min_date: datetime, max_date: datetime, - config: Config, + starting_balance: float, *args, **kwargs, ) -> float: @@ -37,7 +34,6 @@ class SortinoHyperOptLoss(IHyperOptLoss): Uses Sortino Ratio calculation. """ - starting_balance = get_dry_run_wallet(config) sortino_ratio = calculate_sortino(results, min_date, max_date, starting_balance) # print(expected_returns_mean, down_stdev, sortino_ratio) return -sortino_ratio diff --git a/tests/optimize/test_hyperoptloss.py b/tests/optimize/test_hyperoptloss.py index 53de37a0e..9634bd117 100644 --- a/tests/optimize/test_hyperoptloss.py +++ b/tests/optimize/test_hyperoptloss.py @@ -116,6 +116,7 @@ def test_loss_functions_better_profits(default_conf, hyperopt_results, lossfunct config=default_conf, processed=None, backtest_stats={"profit_total": hyperopt_results["profit_abs"].sum()}, + starting_balance=default_conf["dry_run_wallet"], ) over = hl.hyperopt_loss_function( results_over, From 18305a5bf6b163b6d7609ee4ff953f9a3b2dfcce Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 3 Dec 2024 19:16:48 +0100 Subject: [PATCH 15/19] docs: update hyperopt docs to include new argument --- docs/advanced-hyperopt.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index 480b20daf..b12d8f5b6 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -39,6 +39,7 @@ class SuperDuperHyperOptLoss(IHyperOptLoss): config: Config, processed: dict[str, DataFrame], backtest_stats: dict[str, Any], + starting_balance: float, **kwargs, ) -> float: """ @@ -70,6 +71,7 @@ Currently, the arguments are: * `config`: Config object used (Note: Not all strategy-related parameters will be updated here if they are part of a hyperopt space). * `processed`: Dict of Dataframes with the pair as keys containing the data used for backtesting. * `backtest_stats`: Backtesting statistics using the same format as the backtesting file "strategy" substructure. Available fields can be seen in `generate_strategy_stats()` in `optimize_reports.py`. +* `starting_balance`: Starting balance used for backtesting. This function needs to return a floating point number (`float`). Smaller numbers will be interpreted as better results. The parameters and balancing for this is up to you. From ad8f62128778d2ddf6191485755f5c5356754c8f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 3 Dec 2024 19:17:01 +0100 Subject: [PATCH 16/19] tests: Improve hyperopt loss tests --- tests/optimize/test_hyperoptloss.py | 84 +++++++++++++++++++++++++---- 1 file changed, 73 insertions(+), 11 deletions(-) diff --git a/tests/optimize/test_hyperoptloss.py b/tests/optimize/test_hyperoptloss.py index 9634bd117..8f1b1c786 100644 --- a/tests/optimize/test_hyperoptloss.py +++ b/tests/optimize/test_hyperoptloss.py @@ -39,13 +39,34 @@ def test_loss_calculation_prefer_correct_trade_count(hyperopt_conf, hyperopt_res hyperopt_conf.update({"hyperopt_loss": "ShortTradeDurHyperOptLoss"}) hl = HyperOptLossResolver.load_hyperoptloss(hyperopt_conf) correct = hl.hyperopt_loss_function( - hyperopt_results, 600, datetime(2019, 1, 1), datetime(2019, 5, 1) + results=hyperopt_results, + trade_count=600, + min_date=datetime(2019, 1, 1), + max_date=datetime(2019, 5, 1), + config=hyperopt_conf, + processed=None, + backtest_stats={"profit_total": hyperopt_results["profit_abs"].sum()}, + starting_balance=hyperopt_conf["dry_run_wallet"], ) over = hl.hyperopt_loss_function( - hyperopt_results, 600 + 100, datetime(2019, 1, 1), datetime(2019, 5, 1) + results=hyperopt_results, + trade_count=600 + 100, + min_date=datetime(2019, 1, 1), + max_date=datetime(2019, 5, 1), + config=hyperopt_conf, + processed=None, + backtest_stats={"profit_total": hyperopt_results["profit_abs"].sum()}, + starting_balance=hyperopt_conf["dry_run_wallet"], ) under = hl.hyperopt_loss_function( - hyperopt_results, 600 - 100, datetime(2019, 1, 1), datetime(2019, 5, 1) + results=hyperopt_results, + trade_count=600 - 100, + min_date=datetime(2019, 1, 1), + max_date=datetime(2019, 5, 1), + config=hyperopt_conf, + processed=None, + backtest_stats={"profit_total": hyperopt_results["profit_abs"].sum()}, + starting_balance=hyperopt_conf["dry_run_wallet"], ) assert over > correct assert under > correct @@ -58,9 +79,25 @@ def test_loss_calculation_prefer_shorter_trades(hyperopt_conf, hyperopt_results) hyperopt_conf.update({"hyperopt_loss": "ShortTradeDurHyperOptLoss"}) hl = HyperOptLossResolver.load_hyperoptloss(hyperopt_conf) longer = hl.hyperopt_loss_function( - hyperopt_results, 100, datetime(2019, 1, 1), datetime(2019, 5, 1) + results=hyperopt_results, + trade_count=100, + min_date=datetime(2019, 1, 1), + max_date=datetime(2019, 5, 1), + config=hyperopt_conf, + processed=None, + backtest_stats={"profit_total": hyperopt_results["profit_abs"].sum()}, + starting_balance=hyperopt_conf["dry_run_wallet"], + ) + shorter = hl.hyperopt_loss_function( + results=resultsb, + trade_count=100, + min_date=datetime(2019, 1, 1), + max_date=datetime(2019, 5, 1), + config=hyperopt_conf, + processed=None, + backtest_stats={"profit_total": resultsb["profit_abs"].sum()}, + starting_balance=hyperopt_conf["dry_run_wallet"], ) - shorter = hl.hyperopt_loss_function(resultsb, 100, datetime(2019, 1, 1), datetime(2019, 5, 1)) assert shorter < longer @@ -73,11 +110,34 @@ def test_loss_calculation_has_limited_profit(hyperopt_conf, hyperopt_results) -> hyperopt_conf.update({"hyperopt_loss": "ShortTradeDurHyperOptLoss"}) hl = HyperOptLossResolver.load_hyperoptloss(hyperopt_conf) correct = hl.hyperopt_loss_function( - hyperopt_results, 600, datetime(2019, 1, 1), datetime(2019, 5, 1) + results=hyperopt_results, + trade_count=600, + min_date=datetime(2019, 1, 1), + max_date=datetime(2019, 5, 1), + config=hyperopt_conf, + processed=None, + backtest_stats={"profit_total": hyperopt_results["profit_abs"].sum()}, + starting_balance=hyperopt_conf["dry_run_wallet"], + ) + over = hl.hyperopt_loss_function( + results=results_over, + trade_count=600, + min_date=datetime(2019, 1, 1), + max_date=datetime(2019, 5, 1), + config=hyperopt_conf, + processed=None, + backtest_stats={"profit_total": results_over["profit_abs"].sum()}, + starting_balance=hyperopt_conf["dry_run_wallet"], ) - over = hl.hyperopt_loss_function(results_over, 600, datetime(2019, 1, 1), datetime(2019, 5, 1)) under = hl.hyperopt_loss_function( - results_under, 600, datetime(2019, 1, 1), datetime(2019, 5, 1) + results=results_under, + trade_count=600, + min_date=datetime(2019, 1, 1), + max_date=datetime(2019, 5, 1), + config=hyperopt_conf, + processed=None, + backtest_stats={"profit_total": results_under["profit_abs"].sum()}, + starting_balance=hyperopt_conf["dry_run_wallet"], ) assert over < correct assert under > correct @@ -109,7 +169,7 @@ def test_loss_functions_better_profits(default_conf, hyperopt_results, lossfunct default_conf.update({"hyperopt_loss": lossfunction}) hl = HyperOptLossResolver.load_hyperoptloss(default_conf) correct = hl.hyperopt_loss_function( - hyperopt_results, + results=hyperopt_results, trade_count=len(hyperopt_results), min_date=datetime(2019, 1, 1), max_date=datetime(2019, 5, 1), @@ -119,22 +179,24 @@ def test_loss_functions_better_profits(default_conf, hyperopt_results, lossfunct starting_balance=default_conf["dry_run_wallet"], ) over = hl.hyperopt_loss_function( - results_over, + results=results_over, trade_count=len(results_over), min_date=datetime(2019, 1, 1), max_date=datetime(2019, 5, 1), config=default_conf, processed=None, backtest_stats={"profit_total": results_over["profit_abs"].sum()}, + starting_balance=default_conf["dry_run_wallet"], ) under = hl.hyperopt_loss_function( - results_under, + results=results_under, trade_count=len(results_under), min_date=datetime(2019, 1, 1), max_date=datetime(2019, 5, 1), config=default_conf, processed=None, backtest_stats={"profit_total": results_under["profit_abs"].sum()}, + starting_balance=default_conf["dry_run_wallet"], ) assert over < correct assert under > correct From fe834f00a2b0aeb02e1bfc895279d258ce9bb41c Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 3 Dec 2024 19:37:14 +0100 Subject: [PATCH 17/19] fix: import causing long startup time --- freqtrade/commands/optimize_commands.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index 788bca489..9695a313b 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -4,7 +4,6 @@ from typing import Any from freqtrade import constants from freqtrade.enums import RunMode from freqtrade.exceptions import ConfigurationError, OperationalException -from freqtrade.util import get_dry_run_wallet logger = logging.getLogger(__name__) @@ -18,7 +17,7 @@ def setup_optimize_configuration(args: dict[str, Any], method: RunMode) -> dict[ :return: Configuration """ from freqtrade.configuration import setup_utils_configuration - from freqtrade.util import fmt_coin + from freqtrade.util import fmt_coin, get_dry_run_wallet config = setup_utils_configuration(args, method) From ebae0a7248ca37d3af8519bce45d1c5b3b84a2af Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 3 Dec 2024 19:44:26 +0100 Subject: [PATCH 18/19] chore: improve typing of new functionality --- freqtrade/optimize/optimize_reports/optimize_reports.py | 6 +++--- freqtrade/util/dry_run_wallet.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/optimize_reports/optimize_reports.py b/freqtrade/optimize/optimize_reports/optimize_reports.py index 532abed39..23119077b 100644 --- a/freqtrade/optimize/optimize_reports/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports/optimize_reports.py @@ -69,7 +69,7 @@ def generate_rejected_signals( def _generate_result_line( - result: DataFrame, starting_balance: int, first_column: str | list[str] + result: DataFrame, starting_balance: float, first_column: str | list[str] ) -> dict: """ Generate one result dict, with "first_column" as key. @@ -111,7 +111,7 @@ def _generate_result_line( def generate_pair_metrics( pairlist: list[str], stake_currency: str, - starting_balance: int, + starting_balance: float, results: DataFrame, skip_nan: bool = False, ) -> list[dict]: @@ -144,7 +144,7 @@ def generate_pair_metrics( def generate_tag_metrics( tag_type: Literal["enter_tag", "exit_reason"] | list[Literal["enter_tag", "exit_reason"]], - starting_balance: int, + starting_balance: float, results: DataFrame, skip_nan: bool = False, ) -> list[dict]: diff --git a/freqtrade/util/dry_run_wallet.py b/freqtrade/util/dry_run_wallet.py index f994f0d4c..c904db6b3 100644 --- a/freqtrade/util/dry_run_wallet.py +++ b/freqtrade/util/dry_run_wallet.py @@ -1,4 +1,4 @@ -from pytest import Config +from freqtrade.constants import Config def get_dry_run_wallet(config: Config) -> int | float: From 98e0a5f10101c365cac05311317e9018fcf6aaf6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Dec 2024 15:51:37 +0100 Subject: [PATCH 19/19] chore: remove unused arguments in loss functions --- freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py | 3 --- .../hyperopt_loss/hyperopt_loss_max_drawdown_relative.py | 3 +-- freqtrade/optimize/hyperopt_loss/hyperopt_loss_multi_metric.py | 2 -- .../optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py | 3 +-- 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py index 2a04d4070..4f1a82e1e 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py @@ -9,7 +9,6 @@ from datetime import datetime from pandas import DataFrame -from freqtrade.constants import Config from freqtrade.data.metrics import calculate_calmar from freqtrade.optimize.hyperopt import IHyperOptLoss @@ -24,10 +23,8 @@ class CalmarHyperOptLoss(IHyperOptLoss): @staticmethod def hyperopt_loss_function( results: DataFrame, - trade_count: int, min_date: datetime, max_date: datetime, - config: Config, starting_balance: float, *args, **kwargs, diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown_relative.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown_relative.py index a753263a3..3cd578cb4 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown_relative.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown_relative.py @@ -7,7 +7,6 @@ Hyperoptimization. from pandas import DataFrame -from freqtrade.constants import Config from freqtrade.data.metrics import calculate_underwater from freqtrade.optimize.hyperopt import IHyperOptLoss @@ -22,7 +21,7 @@ class MaxDrawDownRelativeHyperOptLoss(IHyperOptLoss): @staticmethod def hyperopt_loss_function( - results: DataFrame, config: Config, starting_balance: float, *args, **kwargs + results: DataFrame, starting_balance: float, *args, **kwargs ) -> float: """ Objective function. diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_multi_metric.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_multi_metric.py index 7a10d279b..adffdfb0b 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_multi_metric.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_multi_metric.py @@ -33,7 +33,6 @@ TARGET_TRADE_AMOUNT variable sets the minimum number of trades required to avoid import numpy as np from pandas import DataFrame -from freqtrade.constants import Config from freqtrade.data.metrics import calculate_expectancy, calculate_max_drawdown from freqtrade.optimize.hyperopt import IHyperOptLoss @@ -57,7 +56,6 @@ class MultiMetricHyperOptLoss(IHyperOptLoss): def hyperopt_loss_function( results: DataFrame, trade_count: int, - config: Config, starting_balance: float, **kwargs, ) -> float: diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py index 2ab581520..af1b33dfb 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py @@ -10,7 +10,6 @@ individual needs. from pandas import DataFrame -from freqtrade.constants import Config from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.optimize.hyperopt import IHyperOptLoss @@ -22,7 +21,7 @@ DRAWDOWN_MULT = 0.075 class ProfitDrawDownHyperOptLoss(IHyperOptLoss): @staticmethod def hyperopt_loss_function( - results: DataFrame, config: Config, starting_balance: float, *args, **kwargs + results: DataFrame, starting_balance: float, *args, **kwargs ) -> float: total_profit = results["profit_abs"].sum()