diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1b344d872..8fa4bf9ee 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: # stages: [push] - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.10.0" + rev: "v1.10.1" hooks: - id: mypy exclude: build_helpers @@ -31,7 +31,7 @@ repos: - repo: https://github.com/charliermarsh/ruff-pre-commit # Ruff version. - rev: 'v0.4.10' + rev: 'v0.5.0' hooks: - id: ruff diff --git a/build_helpers/TA_Lib-0.4.31-cp310-cp310-win_amd64.whl b/build_helpers/TA_Lib-0.4.31-cp310-cp310-win_amd64.whl deleted file mode 100644 index 4ce492a40..000000000 Binary files a/build_helpers/TA_Lib-0.4.31-cp310-cp310-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.31-cp311-cp311-linux_armv7l.whl b/build_helpers/TA_Lib-0.4.31-cp311-cp311-linux_armv7l.whl deleted file mode 100644 index a664f12e6..000000000 Binary files a/build_helpers/TA_Lib-0.4.31-cp311-cp311-linux_armv7l.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.31-cp311-cp311-win_amd64.whl b/build_helpers/TA_Lib-0.4.31-cp311-cp311-win_amd64.whl deleted file mode 100644 index e5b8cb4ef..000000000 Binary files a/build_helpers/TA_Lib-0.4.31-cp311-cp311-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.31-cp312-cp312-win_amd64.whl b/build_helpers/TA_Lib-0.4.31-cp312-cp312-win_amd64.whl deleted file mode 100644 index 79596cf0c..000000000 Binary files a/build_helpers/TA_Lib-0.4.31-cp312-cp312-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.31-cp39-cp39-win_amd64.whl b/build_helpers/TA_Lib-0.4.31-cp39-cp39-win_amd64.whl deleted file mode 100644 index 01d3c626e..000000000 Binary files a/build_helpers/TA_Lib-0.4.31-cp39-cp39-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.32-cp310-cp310-win_amd64.whl b/build_helpers/TA_Lib-0.4.32-cp310-cp310-win_amd64.whl new file mode 100644 index 000000000..d31d63db8 Binary files /dev/null and b/build_helpers/TA_Lib-0.4.32-cp310-cp310-win_amd64.whl differ diff --git a/build_helpers/TA_Lib-0.4.32-cp311-cp311-linux_armv7l.whl b/build_helpers/TA_Lib-0.4.32-cp311-cp311-linux_armv7l.whl new file mode 100644 index 000000000..e69bdd97d Binary files /dev/null and b/build_helpers/TA_Lib-0.4.32-cp311-cp311-linux_armv7l.whl differ diff --git a/build_helpers/TA_Lib-0.4.32-cp311-cp311-win_amd64.whl b/build_helpers/TA_Lib-0.4.32-cp311-cp311-win_amd64.whl new file mode 100644 index 000000000..e992ece01 Binary files /dev/null and b/build_helpers/TA_Lib-0.4.32-cp311-cp311-win_amd64.whl differ diff --git a/build_helpers/TA_Lib-0.4.32-cp312-cp312-win_amd64.whl b/build_helpers/TA_Lib-0.4.32-cp312-cp312-win_amd64.whl new file mode 100644 index 000000000..98845c409 Binary files /dev/null and b/build_helpers/TA_Lib-0.4.32-cp312-cp312-win_amd64.whl differ diff --git a/build_helpers/TA_Lib-0.4.31-cp39-cp39-linux_armv7l.whl b/build_helpers/TA_Lib-0.4.32-cp39-cp39-linux_armv7l.whl similarity index 54% rename from build_helpers/TA_Lib-0.4.31-cp39-cp39-linux_armv7l.whl rename to build_helpers/TA_Lib-0.4.32-cp39-cp39-linux_armv7l.whl index bceb21773..03bc79df8 100644 Binary files a/build_helpers/TA_Lib-0.4.31-cp39-cp39-linux_armv7l.whl and b/build_helpers/TA_Lib-0.4.32-cp39-cp39-linux_armv7l.whl differ diff --git a/build_helpers/TA_Lib-0.4.32-cp39-cp39-win_amd64.whl b/build_helpers/TA_Lib-0.4.32-cp39-cp39-win_amd64.whl new file mode 100644 index 000000000..f0c46dafe Binary files /dev/null and b/build_helpers/TA_Lib-0.4.32-cp39-cp39-win_amd64.whl differ diff --git a/docs/configuration.md b/docs/configuration.md index e2501cf48..54329c505 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -650,9 +650,9 @@ Once you will be happy with your bot performance running in the Dry-run mode, yo * API-keys may or may not be provided. Only Read-Only operations (i.e. operations that do not alter account state) on the exchange are performed in dry-run mode. * Wallets (`/balance`) are simulated based on `dry_run_wallet`. * Orders are simulated, and will not be posted to the exchange. -* Market orders fill based on orderbook volume the moment the order is placed. +* Market orders fill based on orderbook volume the moment the order is placed, with a maximum slippage of 5%. * Limit orders fill once the price reaches the defined level - or time out based on `unfilledtimeout` settings. -* Limit orders will be converted to market orders if they cross the price by more than 1%. +* Limit orders will be converted to market orders if they cross the price by more than 1%, and will be filled immediately based regular market order rules (see point about Market orders above). * In combination with `stoploss_on_exchange`, the stop_loss price is assumed to be filled. * Open orders (not trades, which are stored in the database) are kept open after bot restarts, with the assumption that they were not filled while being offline. diff --git a/docs/freqai-running.md b/docs/freqai-running.md index 553a8b698..1b721b658 100644 --- a/docs/freqai-running.md +++ b/docs/freqai-running.md @@ -73,11 +73,11 @@ Backtesting mode requires [downloading the necessary data](#downloading-data-to- --- -### Saving prediction data +### Saving backtesting prediction data To allow for tweaking your strategy (**not** the features!), FreqAI will automatically save the predictions during backtesting so that they can be reused for future backtests and live runs using the same `identifier` model. This provides a performance enhancement geared towards enabling **high-level hyperopting** of entry/exit criteria. -An additional directory called `backtesting_predictions`, which contains all the predictions stored in `hdf` format, will be created in the `unique-id` folder. +An additional directory called `backtesting_predictions`, which contains all the predictions stored in `feather` format, will be created in the `unique-id` folder. To change your **features**, you **must** set a new `identifier` in the config to signal to FreqAI to train new models. @@ -89,7 +89,6 @@ FreqAI allow you to reuse live historic predictions through the backtest paramet The `--timerange` parameter must not be informed, as it will be automatically calculated through the data in the historic predictions file. - ### Downloading data to cover the full backtest period For live/dry deployments, FreqAI will download the necessary data automatically. However, to use backtesting functionality, you need to download the necessary data using `download-data` (details [here](data-download.md#data-downloading)). You need to pay careful attention to understanding how much *additional* data needs to be downloaded to ensure that there is a sufficient amount of training data *before* the start of the backtesting time range. The amount of additional data can be roughly estimated by moving the start date of the time range backwards by `train_period_days` and the `startup_candle_count` (see the [parameter table](freqai-parameter-table.md) for detailed descriptions of these parameters) from the beginning of the desired backtesting time range. diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 5c4e6df5d..d9c41c38b 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,6 +1,6 @@ """Freqtrade bot""" -__version__ = "2024.6-dev" +__version__ = "2024.7-dev" if "dev" in __version__: from pathlib import Path diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index c527a80d6..0c93af78a 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -2,10 +2,10 @@ This module contains the argument manager class """ -import argparse +from argparse import ArgumentParser, Namespace, _ArgumentGroup from functools import partial from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union from freqtrade.commands.cli_options import AVAILABLE_CLI_OPTIONS from freqtrade.constants import DEFAULT_CONFIG @@ -226,6 +226,19 @@ ARGS_ANALYZE_ENTRIES_EXITS = [ "analysis_csv_path", ] + +ARGS_STRATEGY_UPDATER = ["strategy_list", "strategy_path", "recursive_strategy_search"] + +ARGS_LOOKAHEAD_ANALYSIS = [ + a + for a in ARGS_BACKTEST + if a + not in ("position_stacking", "use_max_market_positions", "backtest_cache", "backtest_breakdown") +] + ["minimum_trade_amount", "targeted_trade_amount", "lookahead_analysis_exportfilename"] + +ARGS_RECURSIVE_ANALYSIS = ["timeframe", "timerange", "dataformat_ohlcv", "pairs", "startup_candle"] + +# Command level configs - keep at the bottom of the above definitions NO_CONF_REQURIED = [ "convert-data", "convert-trade-data", @@ -248,14 +261,6 @@ NO_CONF_REQURIED = [ NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] -ARGS_STRATEGY_UPDATER = ["strategy_list", "strategy_path", "recursive_strategy_search"] - -ARGS_LOOKAHEAD_ANALYSIS = [ - a for a in ARGS_BACKTEST if a not in ("position_stacking", "use_max_market_positions", "cache") -] + ["minimum_trade_amount", "targeted_trade_amount", "lookahead_analysis_exportfilename"] - -ARGS_RECURSIVE_ANALYSIS = ["timeframe", "timerange", "dataformat_ohlcv", "pairs", "startup_candle"] - class Arguments: """ @@ -264,7 +269,7 @@ class Arguments: def __init__(self, args: Optional[List[str]]) -> None: self.args = args - self._parsed_arg: Optional[argparse.Namespace] = None + self._parsed_arg: Optional[Namespace] = None def get_parsed_arg(self) -> Dict[str, Any]: """ @@ -277,7 +282,7 @@ class Arguments: return vars(self._parsed_arg) - def _parse_args(self) -> argparse.Namespace: + def _parse_args(self) -> Namespace: """ Parses given arguments and returns an argparse Namespace instance. """ @@ -306,7 +311,9 @@ class Arguments: return parsed_arg - def _build_args(self, optionlist, parser): + def _build_args( + self, optionlist: List[str], parser: Union[ArgumentParser, _ArgumentGroup] + ) -> None: for val in optionlist: opt = AVAILABLE_CLI_OPTIONS[val] parser.add_argument(*opt.cli, dest=val, **opt.kwargs) @@ -317,16 +324,16 @@ class Arguments: :return: None """ # Build shared arguments (as group Common Options) - _common_parser = argparse.ArgumentParser(add_help=False) + _common_parser = ArgumentParser(add_help=False) group = _common_parser.add_argument_group("Common arguments") self._build_args(optionlist=ARGS_COMMON, parser=group) - _strategy_parser = argparse.ArgumentParser(add_help=False) + _strategy_parser = ArgumentParser(add_help=False) strategy_group = _strategy_parser.add_argument_group("Strategy arguments") self._build_args(optionlist=ARGS_STRATEGY, parser=strategy_group) # Build main command - self.parser = argparse.ArgumentParser( + self.parser = ArgumentParser( prog="freqtrade", description="Free, open source crypto trading bot" ) self._build_args(optionlist=["version"], parser=self.parser) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 257166f9c..3a542226c 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -45,7 +45,8 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: "name": exchange["name"], **valid_entry, "supported": "Official" if exchange["supported"] else "", - "trade_modes": ", ".join( + "trade_modes": ("DEX: " if exchange["dex"] else "") + + ", ".join( (f"{a['margin_mode']} " if a["margin_mode"] else "") + a["trading_mode"] for a in exchange["trade_modes"] ), diff --git a/freqtrade/data/__init__.py b/freqtrade/data/__init__.py index f716abfc5..492155031 100644 --- a/freqtrade/data/__init__.py +++ b/freqtrade/data/__init__.py @@ -2,5 +2,8 @@ Module to handle data operations for freqtrade """ +from freqtrade.data import converter + + # limit what's imported when using `from freqtrade.data import *` __all__ = ["converter"] diff --git a/freqtrade/exchange/binance_leverage_tiers.json b/freqtrade/exchange/binance_leverage_tiers.json index be9f68d98..ae797c9e5 100644 --- a/freqtrade/exchange/binance_leverage_tiers.json +++ b/freqtrade/exchange/binance_leverage_tiers.json @@ -8437,7 +8437,7 @@ } } ], - "BTC/USDT:USDT-240628": [ + "BTC/USDT:USDT-240927": [ { "tier": 1.0, "currency": "USDT", @@ -8567,7 +8567,7 @@ } } ], - "BTC/USDT:USDT-240927": [ + "BTC/USDT:USDT-241227": [ { "tier": 1.0, "currency": "USDT", @@ -13805,7 +13805,7 @@ } } ], - "ETH/USDT:USDT-240628": [ + "ETH/USDT:USDT-240927": [ { "tier": 1.0, "currency": "USDT", @@ -13935,7 +13935,7 @@ } } ], - "ETH/USDT:USDT-240927": [ + "ETH/USDT:USDT-241227": [ { "tier": 1.0, "currency": "USDT", diff --git a/freqtrade/exchange/check_exchange.py b/freqtrade/exchange/check_exchange.py index 73b1c8a97..6d82bae04 100644 --- a/freqtrade/exchange/check_exchange.py +++ b/freqtrade/exchange/check_exchange.py @@ -47,7 +47,7 @@ def check_exchange(config: Config, check_for_bad: bool = True) -> bool: f'{", ".join(available_exchanges())}' ) - valid, reason = validate_exchange(exchange) + valid, reason, _ = validate_exchange(exchange) if not valid: if check_for_bad: raise OperationalException( diff --git a/freqtrade/exchange/exchange_utils.py b/freqtrade/exchange/exchange_utils.py index dcae1ab3b..15555248b 100644 --- a/freqtrade/exchange/exchange_utils.py +++ b/freqtrade/exchange/exchange_utils.py @@ -53,16 +53,19 @@ def available_exchanges(ccxt_module: Optional[CcxtModuleType] = None) -> List[st return [x for x in exchanges if validate_exchange(x)[0]] -def validate_exchange(exchange: str) -> Tuple[bool, str]: +def validate_exchange(exchange: str) -> Tuple[bool, str, bool]: """ returns: can_use, reason with Reason including both missing and missing_opt """ ex_mod = getattr(ccxt, exchange.lower())() + + if not ex_mod or not ex_mod.has: + return False, "", False + result = True reason = "" - if not ex_mod or not ex_mod.has: - return False, "" + is_dex = getattr(ex_mod, "dex", False) missing = [ k for k, v in EXCHANGE_HAS_REQUIRED.items() @@ -81,18 +84,19 @@ def validate_exchange(exchange: str) -> Tuple[bool, str]: if missing_opt: reason += f"{'. ' if reason else ''}missing opt: {', '.join(missing_opt)}. " - return result, reason + return result, reason, is_dex def _build_exchange_list_entry( exchange_name: str, exchangeClasses: Dict[str, Any] ) -> ValidExchangesType: - valid, comment = validate_exchange(exchange_name) + valid, comment, is_dex = validate_exchange(exchange_name) result: ValidExchangesType = { "name": exchange_name, "valid": valid, "supported": exchange_name.lower() in SUPPORTED_EXCHANGES, "comment": comment, + "dex": is_dex, "trade_modes": [{"trading_mode": "spot", "margin_mode": ""}], } if resolved := exchangeClasses.get(exchange_name.lower()): diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 9a33fe430..23e2779a0 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -33,7 +33,7 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool = if log: logger.info(f'dumping json to "{filename}"') - with gzip.open(filename, "w") as fpz: + with gzip.open(filename, "wt", encoding="utf-8") as fpz: rapidjson.dump(data, fpz, default=str, number_mode=rapidjson.NM_NATIVE) else: if log: @@ -60,7 +60,7 @@ def file_dump_joblib(filename: Path, data: Any, log: bool = True) -> None: logger.debug(f'done joblib dump to "{filename}"') -def json_load(datafile: Union[gzip.GzipFile, TextIO]) -> Any: +def json_load(datafile: TextIO) -> Any: """ load data with rapidjson Use this to have a consistent experience, @@ -77,7 +77,7 @@ def file_load_json(file: Path): # Try gzip file first, otherwise regular json file. if gzipfile.is_file(): logger.debug(f"Loading historical data from file {gzipfile}") - with gzip.open(gzipfile) as datafile: + with gzip.open(gzipfile, "rt", encoding="utf-8") as datafile: pairdata = json_load(datafile) elif file.is_file(): logger.debug(f"Loading historical data from file {file}") diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 03b026744..a71ec170d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -217,8 +217,6 @@ class Backtesting: raise OperationalException( "VolumePairList not allowed for backtesting. Please use StaticPairList instead." ) - if "PerformanceFilter" in self.pairlists.name_list: - raise OperationalException("PerformanceFilter not allowed for backtesting.") if len(self.strategylist) > 1 and "PrecisionFilter" in self.pairlists.name_list: raise OperationalException( diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 917dad45c..88f0d23d8 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -13,7 +13,7 @@ from freqtrade.constants import ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.misc import plural -from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting from freqtrade.util import PeriodicCache, dt_floor_day, dt_now, dt_ts @@ -21,6 +21,8 @@ logger = logging.getLogger(__name__) class AgeFilter(IPairList): + supports_backtesting = SupportsBacktesting.NO + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) diff --git a/freqtrade/plugins/pairlist/FullTradesFilter.py b/freqtrade/plugins/pairlist/FullTradesFilter.py index 13586611d..caa69cb1e 100644 --- a/freqtrade/plugins/pairlist/FullTradesFilter.py +++ b/freqtrade/plugins/pairlist/FullTradesFilter.py @@ -7,13 +7,15 @@ from typing import List from freqtrade.exchange.types import Tickers from freqtrade.persistence import Trade -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, SupportsBacktesting logger = logging.getLogger(__name__) class FullTradesFilter(IPairList): + supports_backtesting = SupportsBacktesting.NO_ACTION + @property def needstickers(self) -> bool: """ diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index a2e70e649..e84700f8f 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -5,6 +5,7 @@ PairList Handler base class import logging from abc import ABC, abstractmethod from copy import deepcopy +from enum import Enum from typing import Any, Dict, List, Literal, Optional, TypedDict, Union from freqtrade.constants import Config @@ -51,8 +52,20 @@ PairlistParameter = Union[ ] +class SupportsBacktesting(str, Enum): + """ + Enum to indicate if a Pairlist Handler supports backtesting. + """ + + YES = "yes" + NO = "no" + NO_ACTION = "no_action" + BIASED = "biased" + + class IPairList(LoggingMixin, ABC): is_pairlist_generator = False + supports_backtesting: SupportsBacktesting = SupportsBacktesting.NO def __init__( self, diff --git a/freqtrade/plugins/pairlist/MarketCapPairList.py b/freqtrade/plugins/pairlist/MarketCapPairList.py index 648766e20..677abed4b 100644 --- a/freqtrade/plugins/pairlist/MarketCapPairList.py +++ b/freqtrade/plugins/pairlist/MarketCapPairList.py @@ -11,7 +11,7 @@ from cachetools import TTLCache from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting from freqtrade.util.coin_gecko import FtCoinGeckoApi @@ -20,6 +20,7 @@ logger = logging.getLogger(__name__) class MarketCapPairList(IPairList): is_pairlist_generator = True + supports_backtesting = SupportsBacktesting.BIASED def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) diff --git a/freqtrade/plugins/pairlist/OffsetFilter.py b/freqtrade/plugins/pairlist/OffsetFilter.py index bd981358e..5defaaf60 100644 --- a/freqtrade/plugins/pairlist/OffsetFilter.py +++ b/freqtrade/plugins/pairlist/OffsetFilter.py @@ -7,13 +7,15 @@ from typing import Dict, List from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting logger = logging.getLogger(__name__) class OffsetFilter(IPairList): + supports_backtesting = SupportsBacktesting.YES + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index c10ae7394..77a2caf56 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -9,13 +9,15 @@ import pandas as pd from freqtrade.exchange.types import Tickers from freqtrade.persistence import Trade -from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting logger = logging.getLogger(__name__) class PerformanceFilter(IPairList): + supports_backtesting = SupportsBacktesting.NO_ACTION + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) diff --git a/freqtrade/plugins/pairlist/PrecisionFilter.py b/freqtrade/plugins/pairlist/PrecisionFilter.py index b2f767a67..660ff8fea 100644 --- a/freqtrade/plugins/pairlist/PrecisionFilter.py +++ b/freqtrade/plugins/pairlist/PrecisionFilter.py @@ -8,13 +8,15 @@ from typing import Optional from freqtrade.exceptions import OperationalException from freqtrade.exchange import ROUND_UP from freqtrade.exchange.types import Ticker -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, SupportsBacktesting logger = logging.getLogger(__name__) class PrecisionFilter(IPairList): + supports_backtesting = SupportsBacktesting.BIASED + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) diff --git a/freqtrade/plugins/pairlist/PriceFilter.py b/freqtrade/plugins/pairlist/PriceFilter.py index d651533ce..3da7d8443 100644 --- a/freqtrade/plugins/pairlist/PriceFilter.py +++ b/freqtrade/plugins/pairlist/PriceFilter.py @@ -7,13 +7,15 @@ from typing import Dict, Optional from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Ticker -from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting logger = logging.getLogger(__name__) class PriceFilter(IPairList): + supports_backtesting = SupportsBacktesting.BIASED + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) diff --git a/freqtrade/plugins/pairlist/ProducerPairList.py b/freqtrade/plugins/pairlist/ProducerPairList.py index 09a0c49d2..18af7a734 100644 --- a/freqtrade/plugins/pairlist/ProducerPairList.py +++ b/freqtrade/plugins/pairlist/ProducerPairList.py @@ -9,7 +9,7 @@ from typing import Dict, List, Optional from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting logger = logging.getLogger(__name__) @@ -31,6 +31,7 @@ class ProducerPairList(IPairList): """ is_pairlist_generator = True + supports_backtesting = SupportsBacktesting.NO def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index 26fadb9ae..317aad20b 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -16,7 +16,7 @@ from freqtrade import __version__ from freqtrade.configuration.load_config import CONFIG_PARSE_MODE from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist @@ -25,6 +25,8 @@ logger = logging.getLogger(__name__) class RemotePairList(IPairList): is_pairlist_generator = True + # Potential winner bias + supports_backtesting = SupportsBacktesting.BIASED def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index 59ac1ac7c..3882ec8a8 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -9,7 +9,7 @@ from typing import Dict, List, Literal from freqtrade.enums import RunMode from freqtrade.exchange import timeframe_to_seconds from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting from freqtrade.util.periodic_cache import PeriodicCache @@ -19,6 +19,8 @@ ShuffleValues = Literal["candle", "iteration"] class ShuffleFilter(IPairList): + supports_backtesting = SupportsBacktesting.YES + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) diff --git a/freqtrade/plugins/pairlist/SpreadFilter.py b/freqtrade/plugins/pairlist/SpreadFilter.py index 736903abd..00109abb3 100644 --- a/freqtrade/plugins/pairlist/SpreadFilter.py +++ b/freqtrade/plugins/pairlist/SpreadFilter.py @@ -7,13 +7,15 @@ from typing import Dict, Optional from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Ticker -from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting logger = logging.getLogger(__name__) class SpreadFilter(IPairList): + supports_backtesting = SupportsBacktesting.NO + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) diff --git a/freqtrade/plugins/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py index 922d0fd94..c4f322353 100644 --- a/freqtrade/plugins/pairlist/StaticPairList.py +++ b/freqtrade/plugins/pairlist/StaticPairList.py @@ -9,7 +9,7 @@ from copy import deepcopy from typing import Dict, List from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting logger = logging.getLogger(__name__) @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) class StaticPairList(IPairList): is_pairlist_generator = True + supports_backtesting = SupportsBacktesting.YES def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index f5af2d0a7..2d11e45ef 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -15,7 +15,7 @@ from freqtrade.constants import ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.misc import plural -from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting from freqtrade.util import dt_floor_day, dt_now, dt_ts @@ -27,6 +27,8 @@ class VolatilityFilter(IPairList): Filters pairs by volatility """ + supports_backtesting = SupportsBacktesting.NO + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index ea172f140..7cc91f743 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -14,7 +14,7 @@ from freqtrade.constants import ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting from freqtrade.util import dt_now, format_ms_time @@ -26,6 +26,7 @@ SORT_VALUES = ["quoteVolume"] class VolumePairList(IPairList): is_pairlist_generator = True + supports_backtesting = SupportsBacktesting.NO def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 473e003b6..175e5b18a 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -13,7 +13,7 @@ from freqtrade.constants import ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.misc import plural -from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting from freqtrade.util import dt_floor_day, dt_now, dt_ts @@ -21,6 +21,8 @@ logger = logging.getLogger(__name__) class RangeStabilityFilter(IPairList): + supports_backtesting = SupportsBacktesting.NO + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index a6afd5e64..803a60d18 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -11,10 +11,11 @@ from cachetools import TTLCache, cached from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import CandleType +from freqtrade.enums.runmode import RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.mixins import LoggingMixin -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, SupportsBacktesting from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import PairListResolver @@ -57,9 +58,44 @@ class PairListManager(LoggingMixin): f"{invalid}." ) + self._check_backtest() + refresh_period = config.get("pairlist_refresh_period", 3600) LoggingMixin.__init__(self, logger, refresh_period) + def _check_backtest(self) -> None: + if self._config["runmode"] not in (RunMode.BACKTEST, RunMode.EDGE, RunMode.HYPEROPT): + return + + pairlist_errors: List[str] = [] + noaction_pairlists: List[str] = [] + biased_pairlists: List[str] = [] + for pairlist_handler in self._pairlist_handlers: + if pairlist_handler.supports_backtesting == SupportsBacktesting.NO: + pairlist_errors.append(pairlist_handler.name) + if pairlist_handler.supports_backtesting == SupportsBacktesting.NO_ACTION: + noaction_pairlists.append(pairlist_handler.name) + if pairlist_handler.supports_backtesting == SupportsBacktesting.BIASED: + biased_pairlists.append(pairlist_handler.name) + + if noaction_pairlists: + logger.warning( + f"Pairlist Handlers {', '.join(noaction_pairlists)} do not generate " + "any changes during backtesting. While it's safe to leave them enabled, they will " + "not behave like in dry/live modes. " + ) + + if biased_pairlists: + logger.warning( + f"Pairlist Handlers {', '.join(biased_pairlists)} will introduce a lookahead bias " + "to your backtest results, as they use today's data - which inheritly suffers from " + "'winner bias'." + ) + if pairlist_errors: + raise OperationalException( + f"Pairlist Handlers {', '.join(pairlist_errors)} do not support backtesting." + ) + @property def whitelist(self) -> List[str]: """The current whitelist""" diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 39137b605..990ce4d82 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1787,7 +1787,7 @@ class Telegram(RPCHandler): "_Bot Control_\n" "------------\n" "*/start:* `Starts the trader`\n" - "*/stop:* Stops the trader\n" + "*/stop:* `Stops the trader`\n" "*/stopentry:* `Stops entering, but handles open trades gracefully` \n" "*/forceexit |all:* `Instantly exits the given trade or all trades, " "regardless of profit`\n" @@ -1820,7 +1820,7 @@ class Telegram(RPCHandler): "that represents the current market direction. If no direction is provided `" "`the currently set market direction will be output.` \n" "*/list_custom_data :* `List custom_data for Trade ID & Key combo.`\n" - "`If no Key is supplied it will list all key-value pairs found for that Trade ID.`" + "`If no Key is supplied it will list all key-value pairs found for that Trade ID.`\n" "_Statistics_\n" "------------\n" "*/status |[table]:* `Lists all open trades`\n" diff --git a/freqtrade/types/valid_exchanges_type.py b/freqtrade/types/valid_exchanges_type.py index c01149455..9de05b964 100644 --- a/freqtrade/types/valid_exchanges_type.py +++ b/freqtrade/types/valid_exchanges_type.py @@ -14,4 +14,5 @@ class ValidExchangesType(TypedDict): valid: bool supported: bool comment: str + dex: bool trade_modes: List[TradeModeType] diff --git a/ft_client/freqtrade_client/__init__.py b/ft_client/freqtrade_client/__init__.py index 60b5d1164..6c0da7eae 100644 --- a/ft_client/freqtrade_client/__init__.py +++ b/ft_client/freqtrade_client/__init__.py @@ -1,7 +1,7 @@ from freqtrade_client.ft_rest_client import FtRestClient -__version__ = "2024.6-dev" +__version__ = "2024.7-dev" if "dev" in __version__: from pathlib import Path diff --git a/ft_client/requirements.txt b/ft_client/requirements.txt index 35406c1d0..5e6856e92 100644 --- a/ft_client/requirements.txt +++ b/ft_client/requirements.txt @@ -1,3 +1,3 @@ # Requirements for freqtrade client library requests==2.32.3 -python-rapidjson==1.17 +python-rapidjson==1.18 diff --git a/pyproject.toml b/pyproject.toml index 55ff28db6..a0dc7aa60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -144,6 +144,7 @@ extend-select = [ # "TCH", # flake8-type-checking "PTH", # flake8-use-pathlib # "RUF", # ruff + "ASYNC", # flake8-async ] extend-ignore = [ diff --git a/requirements-dev.txt b/requirements-dev.txt index f06316438..79ab31061 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,8 +7,8 @@ -r docs/requirements-docs.txt coveralls==4.0.1 -ruff==0.4.10 -mypy==1.10.0 +ruff==0.5.0 +mypy==1.10.1 pre-commit==3.7.1 pytest==8.2.2 pytest-asyncio==0.23.7 @@ -18,7 +18,7 @@ pytest-random-order==1.1.1 pytest-xdist==3.6.1 isort==5.13.2 # For datetime mocking -time-machine==2.14.1 +time-machine==2.14.2 # Convert jupyter notebooks to markdown documents nbconvert==7.16.4 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index ffde73b84..30ae0fac1 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -2,7 +2,8 @@ -r requirements.txt # Required for hyperopt -scipy==1.13.1 +scipy==1.14.0; python_version >= "3.10" +scipy==1.13.1; python_version < "3.10" scikit-learn==1.5.0 ft-scikit-optimize==0.9.2 filelock==3.15.4 diff --git a/requirements.txt b/requirements.txt index 5dec7bb5d..170e5d805 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ bottleneck==1.4.0 numexpr==2.10.1 pandas-ta==0.3.14b -ccxt==4.3.50 +ccxt==4.3.54 cryptography==42.0.8 aiohttp==3.9.5 SQLAlchemy==2.0.31 @@ -16,7 +16,7 @@ cachetools==5.3.3 requests==2.32.3 urllib3==2.2.2 jsonschema==4.22.0 -TA-Lib==0.4.31 +TA-Lib==0.4.32 technical==1.4.3 tabulate==0.9.0 pycoingecko==3.1.0 @@ -30,7 +30,7 @@ pyarrow==16.1.0; platform_machine != 'armv7l' py_find_1st==1.1.6 # Load ticker files 30% faster -python-rapidjson==1.17 +python-rapidjson==1.18 # Properly format api responses orjson==3.10.5 @@ -42,7 +42,7 @@ fastapi==0.111.0 pydantic==2.7.4 uvicorn==0.30.1 pyjwt==2.8.0 -aiofiles==23.2.1 +aiofiles==24.1.0 psutil==6.0.0 # Support for colorized terminal output diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 6e182e6e8..e9829a8cc 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -429,7 +429,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> backtesting.start() -def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> None: +def test_backtesting_no_pair_left(default_conf, mocker) -> None: mocker.patch(f"{EXMS}.exchange_has", MagicMock(return_value=True)) mocker.patch( "freqtrade.data.history.history_utils.load_pair_history", @@ -449,13 +449,6 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> with pytest.raises(OperationalException, match="No pair in whitelist."): Backtesting(default_conf) - default_conf["pairlists"] = [{"method": "VolumePairList", "number_assets": 5}] - with pytest.raises( - OperationalException, - match=r"VolumePairList not allowed for backtesting\..*StaticPairList.*", - ): - Backtesting(default_conf) - default_conf.update( { "pairlists": [{"method": "StaticPairList"}], @@ -469,7 +462,7 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> Backtesting(default_conf) -def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, tickers) -> None: +def test_backtesting_pairlist_list(default_conf, mocker, tickers) -> None: mocker.patch(f"{EXMS}.exchange_has", MagicMock(return_value=True)) mocker.patch(f"{EXMS}.get_tickers", tickers) mocker.patch(f"{EXMS}.price_to_precision", lambda s, x, y: y) @@ -495,12 +488,6 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti ): Backtesting(default_conf) - default_conf["pairlists"] = [{"method": "StaticPairList"}, {"method": "PerformanceFilter"}] - with pytest.raises( - OperationalException, match="PerformanceFilter not allowed for backtesting." - ): - Backtesting(default_conf) - default_conf["pairlists"] = [ {"method": "StaticPairList"}, {"method": "PrecisionFilter"}, diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index f6c58a1e7..37ebdc58b 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -38,6 +38,7 @@ TESTABLE_PAIRLISTS = [p for p in AVAILABLE_PAIRLISTS if p not in ["RemotePairLis @pytest.fixture(scope="function") def whitelist_conf(default_conf): + default_conf["runmode"] = "dry_run" default_conf["stake_currency"] = "BTC" default_conf["exchange"]["pair_whitelist"] = [ "ETH/BTC", @@ -68,6 +69,7 @@ def whitelist_conf(default_conf): @pytest.fixture(scope="function") def whitelist_conf_2(default_conf): + default_conf["runmode"] = "dry_run" default_conf["stake_currency"] = "BTC" default_conf["exchange"]["pair_whitelist"] = [ "ETH/BTC", @@ -94,6 +96,7 @@ def whitelist_conf_2(default_conf): @pytest.fixture(scope="function") def whitelist_conf_agefilter(default_conf): + default_conf["runmode"] = "dry_run" default_conf["stake_currency"] = "BTC" default_conf["exchange"]["pair_whitelist"] = [ "ETH/BTC", @@ -773,7 +776,7 @@ def test_VolumePairList_whitelist_gen( whitelist_result, caplog, ) -> None: - whitelist_conf["runmode"] = "backtest" + whitelist_conf["runmode"] = "util_exchange" whitelist_conf["pairlists"] = pairlists whitelist_conf["stake_currency"] = base_currency @@ -2387,3 +2390,65 @@ def test_MarketCapPairList_exceptions(mocker, default_conf_usdt): OperationalException, match="This filter only support marketcap rank up to 250." ): PairListManager(exchange, default_conf_usdt) + + +@pytest.mark.parametrize( + "pairlists,expected_error,expected_warning", + [ + ( + [{"method": "StaticPairList"}], + None, # Error + None, # Warning + ), + ( + [{"method": "VolumePairList", "number_assets": 10}], + "VolumePairList", # Error + None, # Warning + ), + ( + [{"method": "MarketCapPairList", "number_assets": 10}], + None, # Error + r"MarketCapPairList.*lookahead.*", # Warning + ), + ( + [{"method": "StaticPairList"}, {"method": "FullTradesFilter"}], + None, # Error + r"FullTradesFilter do not generate.*", # Warning + ), + ( # combi, fails and warns + [ + {"method": "VolumePairList", "number_assets": 10}, + {"method": "MarketCapPairList", "number_assets": 10}, + ], + "VolumePairList", # Error + r"MarketCapPairList.*lookahead.*", # Warning + ), + ], +) +def test_backtesting_modes( + mocker, default_conf_usdt, pairlists, expected_error, expected_warning, caplog, markets, tickers +): + default_conf_usdt["runmode"] = "dry_run" + default_conf_usdt["pairlists"] = pairlists + + mocker.patch.multiple( + EXMS, + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers, + ) + exchange = get_patched_exchange(mocker, default_conf_usdt) + + # Dry run mode - works always + PairListManager(exchange, default_conf_usdt) + + default_conf_usdt["runmode"] = "backtest" + if expected_error: + with pytest.raises(OperationalException, match=f"Pairlist Handlers {expected_error}.*"): + PairListManager(exchange, default_conf_usdt) + + if not expected_error: + PairListManager(exchange, default_conf_usdt) + + if expected_warning: + assert log_has_re(f"Pairlist Handlers {expected_warning}", caplog) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index ffb2408f1..673f98e05 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -2154,6 +2154,7 @@ def test_api_exchanges(botclient): "valid": True, "supported": True, "comment": "", + "dex": False, "trade_modes": [ {"trading_mode": "spot", "margin_mode": ""}, {"trading_mode": "futures", "margin_mode": "isolated"}, @@ -2165,6 +2166,16 @@ def test_api_exchanges(botclient): "name": "mexc", "valid": True, "supported": False, + "dex": False, + "comment": "", + "trade_modes": [{"trading_mode": "spot", "margin_mode": ""}], + } + waves = [x for x in response["exchanges"] if x["name"] == "wavesexchange"][0] + assert waves == { + "name": "wavesexchange", + "valid": True, + "supported": False, + "dex": True, "comment": "", "trade_modes": [{"trading_mode": "spot", "margin_mode": ""}], }