Merge remote-tracking branch 'upstream/develop' into feature/fetch-public-trades

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
         # stages: [push]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.10.0"
+    rev: "v1.10.1"
     hooks:
       - id: mypy
         exclude: build_helpers
@@ -31,7 +31,7 @@ repos:
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.4.10'
+    rev: 'v0.5.0'
     hooks:
       - id: ruff
 

docs/configuration.md

@@ -650,9 +650,9 @@ Once you will be happy with your bot performance running in the Dry-run mode, yo
 * API-keys may or may not be provided. Only Read-Only operations (i.e. operations that do not alter account state) on the exchange are performed in dry-run mode.
 * Wallets (`/balance`) are simulated based on `dry_run_wallet`.
 * Orders are simulated, and will not be posted to the exchange.
-* Market orders fill based on orderbook volume the moment the order is placed.
+* Market orders fill based on orderbook volume the moment the order is placed, with a maximum slippage of 5%.
 * Limit orders fill once the price reaches the defined level - or time out based on `unfilledtimeout` settings.
-* Limit orders will be converted to market orders if they cross the price by more than 1%.
+* Limit orders will be converted to market orders if they cross the price by more than 1%, and will be filled immediately based regular market order rules (see point about Market orders above).
 * In combination with `stoploss_on_exchange`, the stop_loss price is assumed to be filled.
 * Open orders (not trades, which are stored in the database) are kept open after bot restarts, with the assumption that they were not filled while being offline.
 

docs/freqai-running.md

@@ -73,11 +73,11 @@ Backtesting mode requires [downloading the necessary data](#downloading-data-to-
 
 ---
 
-### Saving prediction data
+### Saving backtesting prediction data
 
 To allow for tweaking your strategy (**not** the features!), FreqAI will automatically save the predictions during backtesting so that they can be reused for future backtests and live runs using the same `identifier` model. This provides a performance enhancement geared towards enabling **high-level hyperopting** of entry/exit criteria.
 
-An additional directory called `backtesting_predictions`, which contains all the predictions stored in `hdf` format, will be created in the `unique-id` folder.
+An additional directory called `backtesting_predictions`, which contains all the predictions stored in `feather` format, will be created in the `unique-id` folder.
 
 To change your **features**, you **must** set a new `identifier` in the config to signal to FreqAI to train new models.
 
@@ -89,7 +89,6 @@ FreqAI allow you to reuse live historic predictions through the backtest paramet
 
 The `--timerange` parameter must not be informed, as it will be automatically calculated through the data in the historic predictions file.
 
-
 ### Downloading data to cover the full backtest period
 
 For live/dry deployments, FreqAI will download the necessary data automatically. However, to use backtesting functionality, you need to download the necessary data using `download-data` (details [here](data-download.md#data-downloading)). You need to pay careful attention to understanding how much *additional* data needs to be downloaded to ensure that there is a sufficient amount of training data *before* the start of the backtesting time range. The amount of additional data can be roughly estimated by moving the start date of the time range backwards by `train_period_days` and the `startup_candle_count` (see the [parameter table](freqai-parameter-table.md) for detailed descriptions of these parameters) from the beginning of the desired backtesting time range. 

freqtrade/__init__.py

@@ -1,6 +1,6 @@
 """Freqtrade bot"""
 
-__version__ = "2024.6-dev"
+__version__ = "2024.7-dev"
 
 if "dev" in __version__:
     from pathlib import Path

freqtrade/commands/arguments.py

@@ -2,10 +2,10 @@
 This module contains the argument manager class
 """
 
-import argparse
+from argparse import ArgumentParser, Namespace, _ArgumentGroup
 from functools import partial
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 from freqtrade.commands.cli_options import AVAILABLE_CLI_OPTIONS
 from freqtrade.constants import DEFAULT_CONFIG
@@ -226,6 +226,19 @@ ARGS_ANALYZE_ENTRIES_EXITS = [
     "analysis_csv_path",
 ]
 
+
+ARGS_STRATEGY_UPDATER = ["strategy_list", "strategy_path", "recursive_strategy_search"]
+
+ARGS_LOOKAHEAD_ANALYSIS = [
+    a
+    for a in ARGS_BACKTEST
+    if a
+    not in ("position_stacking", "use_max_market_positions", "backtest_cache", "backtest_breakdown")
+] + ["minimum_trade_amount", "targeted_trade_amount", "lookahead_analysis_exportfilename"]
+
+ARGS_RECURSIVE_ANALYSIS = ["timeframe", "timerange", "dataformat_ohlcv", "pairs", "startup_candle"]
+
+# Command level configs - keep at the bottom of the above definitions
 NO_CONF_REQURIED = [
     "convert-data",
     "convert-trade-data",
@@ -248,14 +261,6 @@ NO_CONF_REQURIED = [
 
 NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"]
 
-ARGS_STRATEGY_UPDATER = ["strategy_list", "strategy_path", "recursive_strategy_search"]
-
-ARGS_LOOKAHEAD_ANALYSIS = [
-    a for a in ARGS_BACKTEST if a not in ("position_stacking", "use_max_market_positions", "cache")
-] + ["minimum_trade_amount", "targeted_trade_amount", "lookahead_analysis_exportfilename"]
-
-ARGS_RECURSIVE_ANALYSIS = ["timeframe", "timerange", "dataformat_ohlcv", "pairs", "startup_candle"]
-
 
 class Arguments:
     """
@@ -264,7 +269,7 @@ class Arguments:
 
     def __init__(self, args: Optional[List[str]]) -> None:
         self.args = args
-        self._parsed_arg: Optional[argparse.Namespace] = None
+        self._parsed_arg: Optional[Namespace] = None
 
     def get_parsed_arg(self) -> Dict[str, Any]:
         """
@@ -277,7 +282,7 @@ class Arguments:
 
         return vars(self._parsed_arg)
 
-    def _parse_args(self) -> argparse.Namespace:
+    def _parse_args(self) -> Namespace:
         """
         Parses given arguments and returns an argparse Namespace instance.
         """
@@ -306,7 +311,9 @@ class Arguments:
 
         return parsed_arg
 
-    def _build_args(self, optionlist, parser):
+    def _build_args(
+        self, optionlist: List[str], parser: Union[ArgumentParser, _ArgumentGroup]
+    ) -> None:
         for val in optionlist:
             opt = AVAILABLE_CLI_OPTIONS[val]
             parser.add_argument(*opt.cli, dest=val, **opt.kwargs)
@@ -317,16 +324,16 @@ class Arguments:
         :return: None
         """
         # Build shared arguments (as group Common Options)
-        _common_parser = argparse.ArgumentParser(add_help=False)
+        _common_parser = ArgumentParser(add_help=False)
         group = _common_parser.add_argument_group("Common arguments")
         self._build_args(optionlist=ARGS_COMMON, parser=group)
 
-        _strategy_parser = argparse.ArgumentParser(add_help=False)
+        _strategy_parser = ArgumentParser(add_help=False)
         strategy_group = _strategy_parser.add_argument_group("Strategy arguments")
         self._build_args(optionlist=ARGS_STRATEGY, parser=strategy_group)
 
         # Build main command
-        self.parser = argparse.ArgumentParser(
+        self.parser = ArgumentParser(
             prog="freqtrade", description="Free, open source crypto trading bot"
         )
         self._build_args(optionlist=["version"], parser=self.parser)

freqtrade/commands/list_commands.py

@@ -45,7 +45,8 @@ def start_list_exchanges(args: Dict[str, Any]) -> None:
                 "name": exchange["name"],
                 **valid_entry,
                 "supported": "Official" if exchange["supported"] else "",
-                "trade_modes": ", ".join(
+                "trade_modes": ("DEX: " if exchange["dex"] else "")
+                + ", ".join(
                     (f"{a['margin_mode']} " if a["margin_mode"] else "") + a["trading_mode"]
                     for a in exchange["trade_modes"]
                 ),

freqtrade/data/__init__.py

@@ -2,5 +2,8 @@
 Module to handle data operations for freqtrade
 """
 
+from freqtrade.data import converter
+
+
 # limit what's imported when using `from freqtrade.data import *`
 __all__ = ["converter"]

freqtrade/exchange/binance_leverage_tiers.json

@@ -8437,7 +8437,7 @@
       }
     }
   ],
-  "BTC/USDT:USDT-240628": [
+  "BTC/USDT:USDT-240927": [
     {
       "tier": 1.0,
       "currency": "USDT",
@@ -8567,7 +8567,7 @@
       }
     }
   ],
-  "BTC/USDT:USDT-240927": [
+  "BTC/USDT:USDT-241227": [
     {
       "tier": 1.0,
       "currency": "USDT",
@@ -13805,7 +13805,7 @@
       }
     }
   ],
-  "ETH/USDT:USDT-240628": [
+  "ETH/USDT:USDT-240927": [
     {
       "tier": 1.0,
       "currency": "USDT",
@@ -13935,7 +13935,7 @@
       }
     }
   ],
-  "ETH/USDT:USDT-240927": [
+  "ETH/USDT:USDT-241227": [
     {
       "tier": 1.0,
       "currency": "USDT",

freqtrade/exchange/check_exchange.py

@@ -47,7 +47,7 @@ def check_exchange(config: Config, check_for_bad: bool = True) -> bool:
             f'{", ".join(available_exchanges())}'
         )
 
-    valid, reason = validate_exchange(exchange)
+    valid, reason, _ = validate_exchange(exchange)
     if not valid:
         if check_for_bad:
             raise OperationalException(

freqtrade/exchange/exchange_utils.py

@@ -53,16 +53,19 @@ def available_exchanges(ccxt_module: Optional[CcxtModuleType] = None) -> List[st
     return [x for x in exchanges if validate_exchange(x)[0]]
 
 
-def validate_exchange(exchange: str) -> Tuple[bool, str]:
+def validate_exchange(exchange: str) -> Tuple[bool, str, bool]:
     """
     returns: can_use, reason
         with Reason including both missing and missing_opt
     """
     ex_mod = getattr(ccxt, exchange.lower())()
+
+    if not ex_mod or not ex_mod.has:
+        return False, "", False
+
     result = True
     reason = ""
-    if not ex_mod or not ex_mod.has:
+    is_dex = getattr(ex_mod, "dex", False)
-        return False, ""
     missing = [
         k
         for k, v in EXCHANGE_HAS_REQUIRED.items()
@@ -81,18 +84,19 @@ def validate_exchange(exchange: str) -> Tuple[bool, str]:
     if missing_opt:
         reason += f"{'. ' if reason else ''}missing opt: {', '.join(missing_opt)}. "
 
-    return result, reason
+    return result, reason, is_dex
 
 
 def _build_exchange_list_entry(
     exchange_name: str, exchangeClasses: Dict[str, Any]
 ) -> ValidExchangesType:
-    valid, comment = validate_exchange(exchange_name)
+    valid, comment, is_dex = validate_exchange(exchange_name)
     result: ValidExchangesType = {
         "name": exchange_name,
         "valid": valid,
         "supported": exchange_name.lower() in SUPPORTED_EXCHANGES,
         "comment": comment,
+        "dex": is_dex,
         "trade_modes": [{"trading_mode": "spot", "margin_mode": ""}],
     }
     if resolved := exchangeClasses.get(exchange_name.lower()):

freqtrade/misc.py

@@ -33,7 +33,7 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool =
         if log:
             logger.info(f'dumping json to "{filename}"')
 
-        with gzip.open(filename, "w") as fpz:
+        with gzip.open(filename, "wt", encoding="utf-8") as fpz:
             rapidjson.dump(data, fpz, default=str, number_mode=rapidjson.NM_NATIVE)
     else:
         if log:
@@ -60,7 +60,7 @@ def file_dump_joblib(filename: Path, data: Any, log: bool = True) -> None:
     logger.debug(f'done joblib dump to "{filename}"')
 
 
-def json_load(datafile: Union[gzip.GzipFile, TextIO]) -> Any:
+def json_load(datafile: TextIO) -> Any:
     """
     load data with rapidjson
     Use this to have a consistent experience,
@@ -77,7 +77,7 @@ def file_load_json(file: Path):
     # Try gzip file first, otherwise regular json file.
     if gzipfile.is_file():
         logger.debug(f"Loading historical data from file {gzipfile}")
-        with gzip.open(gzipfile) as datafile:
+        with gzip.open(gzipfile, "rt", encoding="utf-8") as datafile:
             pairdata = json_load(datafile)
     elif file.is_file():
         logger.debug(f"Loading historical data from file {file}")

freqtrade/optimize/backtesting.py

@@ -217,8 +217,6 @@ class Backtesting:
             raise OperationalException(
                 "VolumePairList not allowed for backtesting. Please use StaticPairList instead."
             )
-        if "PerformanceFilter" in self.pairlists.name_list:
-            raise OperationalException("PerformanceFilter not allowed for backtesting.")
 
         if len(self.strategylist) > 1 and "PrecisionFilter" in self.pairlists.name_list:
             raise OperationalException(

freqtrade/plugins/pairlist/AgeFilter.py

@@ -13,7 +13,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.misc import plural
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import PeriodicCache, dt_floor_day, dt_now, dt_ts
 
 
@@ -21,6 +21,8 @@ logger = logging.getLogger(__name__)
 
 
 class AgeFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/FullTradesFilter.py

@@ -7,13 +7,15 @@ from typing import List
 
 from freqtrade.exchange.types import Tickers
 from freqtrade.persistence import Trade
-from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.plugins.pairlist.IPairList import IPairList, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class FullTradesFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO_ACTION
+
     @property
     def needstickers(self) -> bool:
         """

freqtrade/plugins/pairlist/IPairList.py

@@ -5,6 +5,7 @@ PairList Handler base class
 import logging
 from abc import ABC, abstractmethod
 from copy import deepcopy
+from enum import Enum
 from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
 
 from freqtrade.constants import Config
@@ -51,8 +52,20 @@ PairlistParameter = Union[
 ]
 
 
+class SupportsBacktesting(str, Enum):
+    """
+    Enum to indicate if a Pairlist Handler supports backtesting.
+    """
+
+    YES = "yes"
+    NO = "no"
+    NO_ACTION = "no_action"
+    BIASED = "biased"
+
+
 class IPairList(LoggingMixin, ABC):
     is_pairlist_generator = False
+    supports_backtesting: SupportsBacktesting = SupportsBacktesting.NO
 
     def __init__(
         self,

freqtrade/plugins/pairlist/MarketCapPairList.py

@@ -11,7 +11,7 @@ from cachetools import TTLCache
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util.coin_gecko import FtCoinGeckoApi
 
 
@@ -20,6 +20,7 @@ logger = logging.getLogger(__name__)
 
 class MarketCapPairList(IPairList):
     is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.BIASED
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/OffsetFilter.py

@@ -7,13 +7,15 @@ from typing import Dict, List
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class OffsetFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.YES
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/PerformanceFilter.py

@@ -9,13 +9,15 @@ import pandas as pd
 
 from freqtrade.exchange.types import Tickers
 from freqtrade.persistence import Trade
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class PerformanceFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO_ACTION
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/PrecisionFilter.py

@@ -8,13 +8,15 @@ from typing import Optional
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import ROUND_UP
 from freqtrade.exchange.types import Ticker
-from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.plugins.pairlist.IPairList import IPairList, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class PrecisionFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.BIASED
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/PriceFilter.py

@@ -7,13 +7,15 @@ from typing import Dict, Optional
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Ticker
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class PriceFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.BIASED
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/ProducerPairList.py

@@ -9,7 +9,7 @@ from typing import Dict, List, Optional
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
@@ -31,6 +31,7 @@ class ProducerPairList(IPairList):
     """
 
     is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.NO
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/RemotePairList.py

@@ -16,7 +16,7 @@ from freqtrade import __version__
 from freqtrade.configuration.load_config import CONFIG_PARSE_MODE
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 
 
@@ -25,6 +25,8 @@ logger = logging.getLogger(__name__)
 
 class RemotePairList(IPairList):
     is_pairlist_generator = True
+    # Potential winner bias
+    supports_backtesting = SupportsBacktesting.BIASED
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/ShuffleFilter.py

@@ -9,7 +9,7 @@ from typing import Dict, List, Literal
 from freqtrade.enums import RunMode
 from freqtrade.exchange import timeframe_to_seconds
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util.periodic_cache import PeriodicCache
 
 
@@ -19,6 +19,8 @@ ShuffleValues = Literal["candle", "iteration"]
 
 
 class ShuffleFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.YES
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/SpreadFilter.py

@@ -7,13 +7,15 @@ from typing import Dict, Optional
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Ticker
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class SpreadFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/StaticPairList.py

@@ -9,7 +9,7 @@ from copy import deepcopy
 from typing import Dict, List
 
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
@@ -17,6 +17,7 @@ logger = logging.getLogger(__name__)
 
 class StaticPairList(IPairList):
     is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.YES
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/VolatilityFilter.py

@@ -15,7 +15,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.misc import plural
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import dt_floor_day, dt_now, dt_ts
 
 
@@ -27,6 +27,8 @@ class VolatilityFilter(IPairList):
     Filters pairs by volatility
     """
 
+    supports_backtesting = SupportsBacktesting.NO
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/VolumePairList.py

@@ -14,7 +14,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import dt_now, format_ms_time
 
 
@@ -26,6 +26,7 @@ SORT_VALUES = ["quoteVolume"]
 
 class VolumePairList(IPairList):
     is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.NO
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/rangestabilityfilter.py

@@ -13,7 +13,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.misc import plural
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import dt_floor_day, dt_now, dt_ts
 
 
@@ -21,6 +21,8 @@ logger = logging.getLogger(__name__)
 
 
 class RangeStabilityFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlistmanager.py

@@ -11,10 +11,11 @@ from cachetools import TTLCache, cached
 from freqtrade.constants import Config, ListPairsWithTimeframes
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.enums import CandleType
+from freqtrade.enums.runmode import RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.mixins import LoggingMixin
-from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.plugins.pairlist.IPairList import IPairList, SupportsBacktesting
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.resolvers import PairListResolver
 
@@ -57,9 +58,44 @@ class PairListManager(LoggingMixin):
                 f"{invalid}."
             )
 
+        self._check_backtest()
+
         refresh_period = config.get("pairlist_refresh_period", 3600)
         LoggingMixin.__init__(self, logger, refresh_period)
 
+    def _check_backtest(self) -> None:
+        if self._config["runmode"] not in (RunMode.BACKTEST, RunMode.EDGE, RunMode.HYPEROPT):
+            return
+
+        pairlist_errors: List[str] = []
+        noaction_pairlists: List[str] = []
+        biased_pairlists: List[str] = []
+        for pairlist_handler in self._pairlist_handlers:
+            if pairlist_handler.supports_backtesting == SupportsBacktesting.NO:
+                pairlist_errors.append(pairlist_handler.name)
+            if pairlist_handler.supports_backtesting == SupportsBacktesting.NO_ACTION:
+                noaction_pairlists.append(pairlist_handler.name)
+            if pairlist_handler.supports_backtesting == SupportsBacktesting.BIASED:
+                biased_pairlists.append(pairlist_handler.name)
+
+        if noaction_pairlists:
+            logger.warning(
+                f"Pairlist Handlers {', '.join(noaction_pairlists)} do not generate "
+                "any changes during backtesting. While it's safe to leave them enabled, they will "
+                "not behave like in dry/live modes. "
+            )
+
+        if biased_pairlists:
+            logger.warning(
+                f"Pairlist Handlers {', '.join(biased_pairlists)} will introduce a lookahead bias "
+                "to your backtest results, as they use today's data - which inheritly suffers from "
+                "'winner bias'."
+            )
+        if pairlist_errors:
+            raise OperationalException(
+                f"Pairlist Handlers {', '.join(pairlist_errors)} do not support backtesting."
+            )
+
     @property
     def whitelist(self) -> List[str]:
         """The current whitelist"""

freqtrade/rpc/telegram.py

@@ -1787,7 +1787,7 @@ class Telegram(RPCHandler):
             "_Bot Control_\n"
             "------------\n"
             "*/start:* `Starts the trader`\n"
-            "*/stop:* Stops the trader\n"
+            "*/stop:* `Stops the trader`\n"
             "*/stopentry:* `Stops entering, but handles open trades gracefully` \n"
             "*/forceexit <trade_id>|all:* `Instantly exits the given trade or all trades, "
             "regardless of profit`\n"
@@ -1820,7 +1820,7 @@ class Telegram(RPCHandler):
             "that represents the current market direction. If no direction is provided `"
             "`the currently set market direction will be output.` \n"
             "*/list_custom_data <trade_id> <key>:* `List custom_data for Trade ID & Key combo.`\n"
-            "`If no Key is supplied it will list all key-value pairs found for that Trade ID.`"
+            "`If no Key is supplied it will list all key-value pairs found for that Trade ID.`\n"
             "_Statistics_\n"
             "------------\n"
             "*/status <trade_id>|[table]:* `Lists all open trades`\n"

freqtrade/types/valid_exchanges_type.py

@@ -14,4 +14,5 @@ class ValidExchangesType(TypedDict):
     valid: bool
     supported: bool
     comment: str
+    dex: bool
     trade_modes: List[TradeModeType]

ft_client/freqtrade_client/__init__.py

@@ -1,7 +1,7 @@
 from freqtrade_client.ft_rest_client import FtRestClient
 
 
-__version__ = "2024.6-dev"
+__version__ = "2024.7-dev"
 
 if "dev" in __version__:
     from pathlib import Path

ft_client/requirements.txt

@@ -1,3 +1,3 @@
 # Requirements for freqtrade client library
 requests==2.32.3
-python-rapidjson==1.17
+python-rapidjson==1.18

pyproject.toml

@@ -144,6 +144,7 @@ extend-select = [
   # "TCH",  # flake8-type-checking
   "PTH",    # flake8-use-pathlib
   # "RUF",    # ruff
+  "ASYNC",  # flake8-async
 ]
 
 extend-ignore = [

requirements-dev.txt

@@ -7,8 +7,8 @@
 -r docs/requirements-docs.txt
 
 coveralls==4.0.1
-ruff==0.4.10
+ruff==0.5.0
-mypy==1.10.0
+mypy==1.10.1
 pre-commit==3.7.1
 pytest==8.2.2
 pytest-asyncio==0.23.7
@@ -18,7 +18,7 @@ pytest-random-order==1.1.1
 pytest-xdist==3.6.1
 isort==5.13.2
 # For datetime mocking
-time-machine==2.14.1
+time-machine==2.14.2
 
 # Convert jupyter notebooks to markdown documents
 nbconvert==7.16.4

requirements-hyperopt.txt

@@ -2,7 +2,8 @@
 -r requirements.txt
 
 # Required for hyperopt
-scipy==1.13.1
+scipy==1.14.0; python_version >= "3.10"
+scipy==1.13.1; python_version < "3.10"
 scikit-learn==1.5.0
 ft-scikit-optimize==0.9.2
 filelock==3.15.4

requirements.txt

@@ -4,7 +4,7 @@ bottleneck==1.4.0
 numexpr==2.10.1
 pandas-ta==0.3.14b
 
-ccxt==4.3.50
+ccxt==4.3.54
 cryptography==42.0.8
 aiohttp==3.9.5
 SQLAlchemy==2.0.31
@@ -16,7 +16,7 @@ cachetools==5.3.3
 requests==2.32.3
 urllib3==2.2.2
 jsonschema==4.22.0
-TA-Lib==0.4.31
+TA-Lib==0.4.32
 technical==1.4.3
 tabulate==0.9.0
 pycoingecko==3.1.0
@@ -30,7 +30,7 @@ pyarrow==16.1.0; platform_machine != 'armv7l'
 py_find_1st==1.1.6
 
 # Load ticker files 30% faster
-python-rapidjson==1.17
+python-rapidjson==1.18
 # Properly format api responses
 orjson==3.10.5
 
@@ -42,7 +42,7 @@ fastapi==0.111.0
 pydantic==2.7.4
 uvicorn==0.30.1
 pyjwt==2.8.0
-aiofiles==23.2.1
+aiofiles==24.1.0
 psutil==6.0.0
 
 # Support for colorized terminal output

tests/optimize/test_backtesting.py

@@ -429,7 +429,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) ->
         backtesting.start()
 
 
-def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> None:
+def test_backtesting_no_pair_left(default_conf, mocker) -> None:
     mocker.patch(f"{EXMS}.exchange_has", MagicMock(return_value=True))
     mocker.patch(
         "freqtrade.data.history.history_utils.load_pair_history",
@@ -449,13 +449,6 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) ->
     with pytest.raises(OperationalException, match="No pair in whitelist."):
         Backtesting(default_conf)
 
-    default_conf["pairlists"] = [{"method": "VolumePairList", "number_assets": 5}]
-    with pytest.raises(
-        OperationalException,
-        match=r"VolumePairList not allowed for backtesting\..*StaticPairList.*",
-    ):
-        Backtesting(default_conf)
-
     default_conf.update(
         {
             "pairlists": [{"method": "StaticPairList"}],
@@ -469,7 +462,7 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) ->
         Backtesting(default_conf)
 
 
-def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, tickers) -> None:
+def test_backtesting_pairlist_list(default_conf, mocker, tickers) -> None:
     mocker.patch(f"{EXMS}.exchange_has", MagicMock(return_value=True))
     mocker.patch(f"{EXMS}.get_tickers", tickers)
     mocker.patch(f"{EXMS}.price_to_precision", lambda s, x, y: y)
@@ -495,12 +488,6 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti
     ):
         Backtesting(default_conf)
 
-    default_conf["pairlists"] = [{"method": "StaticPairList"}, {"method": "PerformanceFilter"}]
-    with pytest.raises(
-        OperationalException, match="PerformanceFilter not allowed for backtesting."
-    ):
-        Backtesting(default_conf)
-
     default_conf["pairlists"] = [
         {"method": "StaticPairList"},
         {"method": "PrecisionFilter"},

tests/plugins/test_pairlist.py

@@ -38,6 +38,7 @@ TESTABLE_PAIRLISTS = [p for p in AVAILABLE_PAIRLISTS if p not in ["RemotePairLis
 
 @pytest.fixture(scope="function")
 def whitelist_conf(default_conf):
+    default_conf["runmode"] = "dry_run"
     default_conf["stake_currency"] = "BTC"
     default_conf["exchange"]["pair_whitelist"] = [
         "ETH/BTC",
@@ -68,6 +69,7 @@ def whitelist_conf(default_conf):
 
 @pytest.fixture(scope="function")
 def whitelist_conf_2(default_conf):
+    default_conf["runmode"] = "dry_run"
     default_conf["stake_currency"] = "BTC"
     default_conf["exchange"]["pair_whitelist"] = [
         "ETH/BTC",
@@ -94,6 +96,7 @@ def whitelist_conf_2(default_conf):
 
 @pytest.fixture(scope="function")
 def whitelist_conf_agefilter(default_conf):
+    default_conf["runmode"] = "dry_run"
     default_conf["stake_currency"] = "BTC"
     default_conf["exchange"]["pair_whitelist"] = [
         "ETH/BTC",
@@ -773,7 +776,7 @@ def test_VolumePairList_whitelist_gen(
     whitelist_result,
     caplog,
 ) -> None:
-    whitelist_conf["runmode"] = "backtest"
+    whitelist_conf["runmode"] = "util_exchange"
     whitelist_conf["pairlists"] = pairlists
     whitelist_conf["stake_currency"] = base_currency
 
@@ -2387,3 +2390,65 @@ def test_MarketCapPairList_exceptions(mocker, default_conf_usdt):
         OperationalException, match="This filter only support marketcap rank up to 250."
     ):
         PairListManager(exchange, default_conf_usdt)
+
+
+@pytest.mark.parametrize(
+    "pairlists,expected_error,expected_warning",
+    [
+        (
+            [{"method": "StaticPairList"}],
+            None,  # Error
+            None,  # Warning
+        ),
+        (
+            [{"method": "VolumePairList", "number_assets": 10}],
+            "VolumePairList",  # Error
+            None,  # Warning
+        ),
+        (
+            [{"method": "MarketCapPairList", "number_assets": 10}],
+            None,  # Error
+            r"MarketCapPairList.*lookahead.*",  # Warning
+        ),
+        (
+            [{"method": "StaticPairList"}, {"method": "FullTradesFilter"}],
+            None,  # Error
+            r"FullTradesFilter do not generate.*",  # Warning
+        ),
+        (  # combi, fails and warns
+            [
+                {"method": "VolumePairList", "number_assets": 10},
+                {"method": "MarketCapPairList", "number_assets": 10},
+            ],
+            "VolumePairList",  # Error
+            r"MarketCapPairList.*lookahead.*",  # Warning
+        ),
+    ],
+)
+def test_backtesting_modes(
+    mocker, default_conf_usdt, pairlists, expected_error, expected_warning, caplog, markets, tickers
+):
+    default_conf_usdt["runmode"] = "dry_run"
+    default_conf_usdt["pairlists"] = pairlists
+
+    mocker.patch.multiple(
+        EXMS,
+        markets=PropertyMock(return_value=markets),
+        exchange_has=MagicMock(return_value=True),
+        get_tickers=tickers,
+    )
+    exchange = get_patched_exchange(mocker, default_conf_usdt)
+
+    # Dry run mode - works always
+    PairListManager(exchange, default_conf_usdt)
+
+    default_conf_usdt["runmode"] = "backtest"
+    if expected_error:
+        with pytest.raises(OperationalException, match=f"Pairlist Handlers {expected_error}.*"):
+            PairListManager(exchange, default_conf_usdt)
+
+    if not expected_error:
+        PairListManager(exchange, default_conf_usdt)
+
+    if expected_warning:
+        assert log_has_re(f"Pairlist Handlers {expected_warning}", caplog)

tests/rpc/test_rpc_apiserver.py

@@ -2154,6 +2154,7 @@ def test_api_exchanges(botclient):
         "valid": True,
         "supported": True,
         "comment": "",
+        "dex": False,
         "trade_modes": [
             {"trading_mode": "spot", "margin_mode": ""},
             {"trading_mode": "futures", "margin_mode": "isolated"},
@@ -2165,6 +2166,16 @@ def test_api_exchanges(botclient):
         "name": "mexc",
         "valid": True,
         "supported": False,
+        "dex": False,
+        "comment": "",
+        "trade_modes": [{"trading_mode": "spot", "margin_mode": ""}],
+    }
+    waves = [x for x in response["exchanges"] if x["name"] == "wavesexchange"][0]
+    assert waves == {
+        "name": "wavesexchange",
+        "valid": True,
+        "supported": False,
+        "dex": True,
         "comment": "",
         "trade_modes": [{"trading_mode": "spot", "margin_mode": ""}],
     }