Merge branch 'develop' into ci/ccxt.pro

2024-07-03 13:12:51 +02:00
parent 2bc5756326 d8eb6e59fa
commit 7209b2e71a
66 changed files with 1327 additions and 434 deletions
@@ -55,7 +55,7 @@ jobs:

    - name: Installation - *nix
      run: |
-        python -m pip install --upgrade pip wheel
+        python -m pip install --upgrade "pip<=24.0" wheel
        export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
        export TA_LIBRARY_PATH=${HOME}/dependencies/lib
        export TA_INCLUDE_PATH=${HOME}/dependencies/include
@@ -192,7 +192,7 @@ jobs:

    - name: Installation (python)
      run: |
-        python -m pip install --upgrade pip wheel
+        python -m pip install --upgrade "pip<=24.0" wheel
        export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
        export TA_LIBRARY_PATH=${HOME}/dependencies/lib
        export TA_INCLUDE_PATH=${HOME}/dependencies/include
@@ -422,7 +422,7 @@ jobs:

    - name: Installation - *nix
      run: |
-        python -m pip install --upgrade pip wheel
+        python -m pip install --upgrade "pip<=24.0" wheel
        export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
        export TA_LIBRARY_PATH=${HOME}/dependencies/lib
        export TA_INCLUDE_PATH=${HOME}/dependencies/include
@@ -9,17 +9,17 @@ repos:
        # stages: [push]

  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.10.0"
+    rev: "v1.10.1"
    hooks:
      - id: mypy
        exclude: build_helpers
        additional_dependencies:
          - types-cachetools==5.3.0.7
          - types-filelock==3.2.7
-          - types-requests==2.32.0.20240602
+          - types-requests==2.32.0.20240622
          - types-tabulate==0.9.0.20240106
          - types-python-dateutil==2.9.0.20240316
-          - SQLAlchemy==2.0.30
+          - SQLAlchemy==2.0.31
        # stages: [push]

  - repo: https://github.com/pycqa/isort
@@ -31,7 +31,7 @@ repos:

  - repo: https://github.com/charliermarsh/ruff-pre-commit
    # Ruff version.
-    rev: 'v0.4.9'
+    rev: 'v0.5.0'
    hooks:
      - id: ruff

@@ -25,7 +25,7 @@ FROM base as python-deps
 RUN  apt-get update \
  && apt-get -y install build-essential libssl-dev git libffi-dev libgfortran5 pkg-config cmake gcc \
  && apt-get clean \
-  && pip install --upgrade pip wheel
+  && pip install --upgrade "pip<=24.0" wheel

 # Install TA-lib
 COPY build_helpers/* /tmp/
@@ -35,7 +35,7 @@ ENV LD_LIBRARY_PATH /usr/local/lib
 # Install dependencies
 COPY --chown=ftuser:ftuser requirements.txt requirements-hyperopt.txt /freqtrade/
 USER ftuser
-RUN  pip install --user --no-cache-dir numpy \
+RUN  pip install --user --no-cache-dir "numpy<2.0" \
  && pip install --user --no-cache-dir -r requirements-hyperopt.txt

 # Copy dependencies to runtime-image
@@ -1,6 +1,6 @@
 # vendored Wheels compiled via https://github.com/xmatthias/ta-lib-python/tree/ta_bundled_040

-python -m pip install --upgrade pip wheel
+python -m pip install --upgrade "pip<=24.0" wheel

 $pyv = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"

@@ -17,7 +17,7 @@ RUN mkdir /freqtrade \
  && chown ftuser:ftuser /freqtrade \
  # Allow sudoers
  && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers \
-  && pip install --upgrade pip
+  && pip install --upgrade "pip<=24.0"

 WORKDIR /freqtrade

@@ -650,9 +650,9 @@ Once you will be happy with your bot performance running in the Dry-run mode, yo
 * API-keys may or may not be provided. Only Read-Only operations (i.e. operations that do not alter account state) on the exchange are performed in dry-run mode.
 * Wallets (`/balance`) are simulated based on `dry_run_wallet`.
 * Orders are simulated, and will not be posted to the exchange.
-* Market orders fill based on orderbook volume the moment the order is placed.
+* Market orders fill based on orderbook volume the moment the order is placed, with a maximum slippage of 5%.
 * Limit orders fill once the price reaches the defined level - or time out based on `unfilledtimeout` settings.
-* Limit orders will be converted to market orders if they cross the price by more than 1%.
+* Limit orders will be converted to market orders if they cross the price by more than 1%, and will be filled immediately based regular market order rules (see point about Market orders above).
 * In combination with `stoploss_on_exchange`, the stop_loss price is assumed to be filled.
 * Open orders (not trades, which are stored in the database) are kept open after bot restarts, with the assumption that they were not filled while being offline.

@@ -373,7 +373,7 @@ Filters low-value coins which would not allow setting stoplosses.

 Namely, pairs are blacklisted if a variance of one percent or more in the stop price would be caused by precision rounding on the exchange, i.e. `rounded(stop_price) <= rounded(stop_price * 0.99)`. The idea is to avoid coins with a value VERY close to their lower trading boundary, not allowing setting of proper stoploss.

-!!! Tip "PerformanceFilter is pointless for futures trading"
+!!! Tip "PrecisionFilter is pointless for futures trading"
    The above does not apply to shorts. And for longs, in theory the trade will be liquidated first.

 !!! Warning "Backtesting"
@@ -2,6 +2,14 @@

 This page explains how to plot prices, indicators and profits.

+!!! Warning "Deprecated"
+    The commands described in this page (`plot-dataframe`, `plot-profit`) should be considered deprecated and are in maintenance mode.
+    This is mostly for the performance problems even medium sized plots can cause, but also because "store a file and open it in a browser" isn't very intuitive from a UI perspective.
+
+    While there are no immediate plans to remove them, they are not actively maintained - and may be removed short-term should major changes be required to keep them working.
+    
+    Please use [FreqUI](freq-ui.md) for plotting needs, which doesn't struggle with the same performance problems.
+
 ## Installation / Setup

 Plotting modules use the Plotly library. You can install / upgrade this by running the following command:
@@ -165,7 +165,9 @@ E.g. If the `current_rate` is 200 USD, then returning `0.02` will set the stoplo
 During backtesting, `current_rate` (and `current_profit`) are provided against the candle's high (or low for short trades) - while the resulting stoploss is evaluated against the candle's low (or high for short trades).

 The absolute value of the return value is used (the sign is ignored), so returning `0.05` or `-0.05` have the same result, a stoploss 5% below the current price.
-Returning None will be interpreted as "no desire to change", and is the only safe way to return when you'd like to not modify the stoploss.
+Returning `None` will be interpreted as "no desire to change", and is the only safe way to return when you'd like to not modify the stoploss.
+`NaN` and `inf` values are considered invalid and will be ignored (identical to `None`).
+

 Stoploss on exchange works similar to `trailing_stop`, and the stoploss on exchange is updated as configured in `stoploss_on_exchange_interval` ([More details about stoploss on exchange](stoploss.md#stop-loss-on-exchangefreqtrade)).

@@ -1,6 +1,6 @@
 """Freqtrade bot"""

-__version__ = "2024.6-dev"
+__version__ = "2024.7-dev"

 if "dev" in __version__:
    from pathlib import Path
@@ -2,10 +2,10 @@
 This module contains the argument manager class
 """

-import argparse
+from argparse import ArgumentParser, Namespace, _ArgumentGroup
 from functools import partial
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union

 from freqtrade.commands.cli_options import AVAILABLE_CLI_OPTIONS
 from freqtrade.constants import DEFAULT_CONFIG
@@ -226,6 +226,19 @@ ARGS_ANALYZE_ENTRIES_EXITS = [
    "analysis_csv_path",
 ]

+
+ARGS_STRATEGY_UPDATER = ["strategy_list", "strategy_path", "recursive_strategy_search"]
+
+ARGS_LOOKAHEAD_ANALYSIS = [
+    a
+    for a in ARGS_BACKTEST
+    if a
+    not in ("position_stacking", "use_max_market_positions", "backtest_cache", "backtest_breakdown")
+] + ["minimum_trade_amount", "targeted_trade_amount", "lookahead_analysis_exportfilename"]
+
+ARGS_RECURSIVE_ANALYSIS = ["timeframe", "timerange", "dataformat_ohlcv", "pairs", "startup_candle"]
+
+# Command level configs - keep at the bottom of the above definitions
 NO_CONF_REQURIED = [
    "convert-data",
    "convert-trade-data",
@@ -248,14 +261,6 @@ NO_CONF_REQURIED = [

 NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"]

-ARGS_STRATEGY_UPDATER = ["strategy_list", "strategy_path", "recursive_strategy_search"]
-
-ARGS_LOOKAHEAD_ANALYSIS = [
-    a for a in ARGS_BACKTEST if a not in ("position_stacking", "use_max_market_positions", "cache")
-] + ["minimum_trade_amount", "targeted_trade_amount", "lookahead_analysis_exportfilename"]
-
-ARGS_RECURSIVE_ANALYSIS = ["timeframe", "timerange", "dataformat_ohlcv", "pairs", "startup_candle"]
-

 class Arguments:
    """
@@ -264,7 +269,7 @@ class Arguments:

    def __init__(self, args: Optional[List[str]]) -> None:
        self.args = args
-        self._parsed_arg: Optional[argparse.Namespace] = None
+        self._parsed_arg: Optional[Namespace] = None

    def get_parsed_arg(self) -> Dict[str, Any]:
        """
@@ -277,7 +282,7 @@ class Arguments:

        return vars(self._parsed_arg)

-    def _parse_args(self) -> argparse.Namespace:
+    def _parse_args(self) -> Namespace:
        """
        Parses given arguments and returns an argparse Namespace instance.
        """
@@ -306,7 +311,9 @@ class Arguments:

        return parsed_arg

-    def _build_args(self, optionlist, parser):
+    def _build_args(
+        self, optionlist: List[str], parser: Union[ArgumentParser, _ArgumentGroup]
+    ) -> None:
        for val in optionlist:
            opt = AVAILABLE_CLI_OPTIONS[val]
            parser.add_argument(*opt.cli, dest=val, **opt.kwargs)
@@ -317,16 +324,16 @@ class Arguments:
        :return: None
        """
        # Build shared arguments (as group Common Options)
-        _common_parser = argparse.ArgumentParser(add_help=False)
+        _common_parser = ArgumentParser(add_help=False)
        group = _common_parser.add_argument_group("Common arguments")
        self._build_args(optionlist=ARGS_COMMON, parser=group)

-        _strategy_parser = argparse.ArgumentParser(add_help=False)
+        _strategy_parser = ArgumentParser(add_help=False)
        strategy_group = _strategy_parser.add_argument_group("Strategy arguments")
        self._build_args(optionlist=ARGS_STRATEGY, parser=strategy_group)

        # Build main command
-        self.parser = argparse.ArgumentParser(
+        self.parser = ArgumentParser(
            prog="freqtrade", description="Free, open source crypto trading bot"
        )
        self._build_args(optionlist=["version"], parser=self.parser)
@@ -2,5 +2,8 @@
 Module to handle data operations for freqtrade
 """

+from freqtrade.data import converter
+
+
 # limit what's imported when using `from freqtrade.data import *`
 __all__ = ["converter"]
@@ -618,6 +618,11 @@ def download_data_main(config: Config) -> None:
    # Start downloading
    try:
        if config.get("download_trades"):
+            if not exchange.get_option("trades_has_history", True):
+                raise OperationalException(
+                    f"Trade history not available for {exchange.name}. "
+                    "You cannot use --dl-trades for this exchange."
+                )
            pairs_not_available = refresh_backtest_trades_data(
                exchange,
                pairs=expanded_pairs,
@@ -28,6 +28,7 @@ class Binance(Exchange):
        "ohlcv_candle_limit": 1000,
        "trades_pagination": "id",
        "trades_pagination_arg": "fromId",
+        "trades_has_history": True,
        "l2_limit_range": [5, 10, 20, 50, 100, 500, 1000],
        "ws.enabled": True,
    }
@@ -20,4 +20,5 @@ class Bingx(Exchange):
        "stoploss_on_exchange": True,
        "stoploss_order_types": {"limit": "limit", "market": "market"},
        "order_time_in_force": ["GTC", "IOC", "PO"],
+        "trades_has_history": False,  # Endpoint doesn't seem to support pagination
    }
@@ -18,4 +18,5 @@ class Bitmart(Exchange):
    _ft_has: Dict = {
        "stoploss_on_exchange": False,  # Bitmart API does not support stoploss orders
        "ohlcv_candle_limit": 200,
+        "trades_has_history": False,  # Endpoint doesn't seem to support pagination
    }
@@ -34,6 +34,7 @@ class Bybit(Exchange):
        "ohlcv_has_history": True,
        "order_time_in_force": ["GTC", "FOK", "IOC", "PO"],
        "ws.enabled": True,
+        "trades_has_history": False,  # Endpoint doesn't support pagination
    }
    _ft_has_futures: Dict = {
        "ohlcv_has_history": True,
@@ -125,6 +125,7 @@ class Exchange:
        "tickers_have_price": True,
        "trades_pagination": "time",  # Possible are "time" or "id"
        "trades_pagination_arg": "since",
+        "trades_has_history": False,
        "l2_limit_range": None,
        "l2_limit_range_required": True,  # Allow Empty L2 limit (kucoin)
        "mark_ohlcv_price": "mark",
@@ -31,6 +31,7 @@ class Gate(Exchange):
        "stop_price_param": "stopPrice",
        "stop_price_prop": "stopPrice",
        "marketOrderRequiresPrice": True,
+        "trades_has_history": False,  # Endpoint would support this - but ccxt doesn't.
    }

    _ft_has_futures: Dict = {
@@ -28,6 +28,7 @@ class Htx(Exchange):
            "1w": 500,
            "1M": 500,
        },
+        "trades_has_history": False,  # Endpoint doesn't have a "since" parameter
    }

    def _get_stop_params(self, side: BuySell, ordertype: str, stop_price: float) -> Dict:
@@ -31,6 +31,7 @@ class Kraken(Exchange):
        "trades_pagination": "id",
        "trades_pagination_arg": "since",
        "trades_pagination_overlap": False,
+        "trades_has_history": True,
        "mark_ohlcv_timeframe": "4h",
    }

@@ -33,6 +33,7 @@ class Okx(Exchange):
        "funding_fee_timeframe": "8h",
        "stoploss_order_types": {"limit": "limit"},
        "stoploss_on_exchange": True,
+        "trades_has_history": False,  # Endpoint doesn't have a "since" parameter
    }
    _ft_has_futures: Dict = {
        "tickers_have_quoteVolume": False,
@@ -33,7 +33,7 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool =
        if log:
            logger.info(f'dumping json to "{filename}"')

-        with gzip.open(filename, "w") as fpz:
+        with gzip.open(filename, "wt", encoding="utf-8") as fpz:
            rapidjson.dump(data, fpz, default=str, number_mode=rapidjson.NM_NATIVE)
    else:
        if log:
@@ -60,7 +60,7 @@ def file_dump_joblib(filename: Path, data: Any, log: bool = True) -> None:
    logger.debug(f'done joblib dump to "{filename}"')


-def json_load(datafile: Union[gzip.GzipFile, TextIO]) -> Any:
+def json_load(datafile: TextIO) -> Any:
    """
    load data with rapidjson
    Use this to have a consistent experience,
@@ -77,7 +77,7 @@ def file_load_json(file: Path):
    # Try gzip file first, otherwise regular json file.
    if gzipfile.is_file():
        logger.debug(f"Loading historical data from file {gzipfile}")
-        with gzip.open(gzipfile) as datafile:
+        with gzip.open(gzipfile, "rt", encoding="utf-8") as datafile:
            pairdata = json_load(datafile)
    elif file.is_file():
        logger.debug(f"Loading historical data from file {file}")
@@ -217,8 +217,6 @@ class Backtesting:
            raise OperationalException(
                "VolumePairList not allowed for backtesting. Please use StaticPairList instead."
            )
-        if "PerformanceFilter" in self.pairlists.name_list:
-            raise OperationalException("PerformanceFilter not allowed for backtesting.")

        if len(self.strategylist) > 1 and "PrecisionFilter" in self.pairlists.name_list:
            raise OperationalException(
@@ -13,7 +13,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.misc import plural
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import PeriodicCache, dt_floor_day, dt_now, dt_ts


@@ -21,6 +21,8 @@ logger = logging.getLogger(__name__)


 class AgeFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO
+
    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)

@@ -7,13 +7,15 @@ from typing import List

 from freqtrade.exchange.types import Tickers
 from freqtrade.persistence import Trade
-from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.plugins.pairlist.IPairList import IPairList, SupportsBacktesting


 logger = logging.getLogger(__name__)


 class FullTradesFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO_ACTION
+
    @property
    def needstickers(self) -> bool:
        """
@@ -5,6 +5,7 @@ PairList Handler base class
 import logging
 from abc import ABC, abstractmethod
 from copy import deepcopy
+from enum import Enum
 from typing import Any, Dict, List, Literal, Optional, TypedDict, Union

 from freqtrade.constants import Config
@@ -51,8 +52,20 @@ PairlistParameter = Union[
 ]


+class SupportsBacktesting(str, Enum):
+    """
+    Enum to indicate if a Pairlist Handler supports backtesting.
+    """
+
+    YES = "yes"
+    NO = "no"
+    NO_ACTION = "no_action"
+    BIASED = "biased"
+
+
 class IPairList(LoggingMixin, ABC):
    is_pairlist_generator = False
+    supports_backtesting: SupportsBacktesting = SupportsBacktesting.NO

    def __init__(
        self,
@@ -11,7 +11,7 @@ from cachetools import TTLCache

 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util.coin_gecko import FtCoinGeckoApi


@@ -20,6 +20,7 @@ logger = logging.getLogger(__name__)

 class MarketCapPairList(IPairList):
    is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.BIASED

    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)
@@ -7,13 +7,15 @@ from typing import Dict, List

 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting


 logger = logging.getLogger(__name__)


 class OffsetFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.YES
+
    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)

@@ -9,13 +9,15 @@ import pandas as pd

 from freqtrade.exchange.types import Tickers
 from freqtrade.persistence import Trade
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting


 logger = logging.getLogger(__name__)


 class PerformanceFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO_ACTION
+
    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)

@@ -8,13 +8,15 @@ from typing import Optional
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import ROUND_UP
 from freqtrade.exchange.types import Ticker
-from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.plugins.pairlist.IPairList import IPairList, SupportsBacktesting


 logger = logging.getLogger(__name__)


 class PrecisionFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.BIASED
+
    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)

@@ -7,13 +7,15 @@ from typing import Dict, Optional

 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Ticker
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting


 logger = logging.getLogger(__name__)


 class PriceFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.BIASED
+
    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)

@@ -9,7 +9,7 @@ from typing import Dict, List, Optional

 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting


 logger = logging.getLogger(__name__)
@@ -31,6 +31,7 @@ class ProducerPairList(IPairList):
    """

    is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.NO

    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)
@@ -16,7 +16,7 @@ from freqtrade import __version__
 from freqtrade.configuration.load_config import CONFIG_PARSE_MODE
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist


@@ -25,6 +25,8 @@ logger = logging.getLogger(__name__)

 class RemotePairList(IPairList):
    is_pairlist_generator = True
+    # Potential winner bias
+    supports_backtesting = SupportsBacktesting.BIASED

    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)
@@ -9,7 +9,7 @@ from typing import Dict, List, Literal
 from freqtrade.enums import RunMode
 from freqtrade.exchange import timeframe_to_seconds
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util.periodic_cache import PeriodicCache


@@ -19,6 +19,8 @@ ShuffleValues = Literal["candle", "iteration"]


 class ShuffleFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.YES
+
    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)

@@ -7,13 +7,15 @@ from typing import Dict, Optional

 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Ticker
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting


 logger = logging.getLogger(__name__)


 class SpreadFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO
+
    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)

@@ -9,7 +9,7 @@ from copy import deepcopy
 from typing import Dict, List

 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting


 logger = logging.getLogger(__name__)
@@ -17,6 +17,7 @@ logger = logging.getLogger(__name__)

 class StaticPairList(IPairList):
    is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.YES

    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)
@@ -15,7 +15,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.misc import plural
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import dt_floor_day, dt_now, dt_ts


@@ -27,6 +27,8 @@ class VolatilityFilter(IPairList):
    Filters pairs by volatility
    """

+    supports_backtesting = SupportsBacktesting.NO
+
    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)

@@ -14,7 +14,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import dt_now, format_ms_time


@@ -26,6 +26,7 @@ SORT_VALUES = ["quoteVolume"]

 class VolumePairList(IPairList):
    is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.NO

    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)
@@ -13,7 +13,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.misc import plural
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import dt_floor_day, dt_now, dt_ts


@@ -21,6 +21,8 @@ logger = logging.getLogger(__name__)


 class RangeStabilityFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO
+
    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)

@@ -11,10 +11,11 @@ from cachetools import TTLCache, cached
 from freqtrade.constants import Config, ListPairsWithTimeframes
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.enums import CandleType
+from freqtrade.enums.runmode import RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.mixins import LoggingMixin
-from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.plugins.pairlist.IPairList import IPairList, SupportsBacktesting
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.resolvers import PairListResolver

@@ -57,9 +58,44 @@ class PairListManager(LoggingMixin):
                f"{invalid}."
            )

+        self._check_backtest()
+
        refresh_period = config.get("pairlist_refresh_period", 3600)
        LoggingMixin.__init__(self, logger, refresh_period)

+    def _check_backtest(self) -> None:
+        if self._config["runmode"] not in (RunMode.BACKTEST, RunMode.EDGE, RunMode.HYPEROPT):
+            return
+
+        pairlist_errors: List[str] = []
+        noaction_pairlists: List[str] = []
+        biased_pairlists: List[str] = []
+        for pairlist_handler in self._pairlist_handlers:
+            if pairlist_handler.supports_backtesting == SupportsBacktesting.NO:
+                pairlist_errors.append(pairlist_handler.name)
+            if pairlist_handler.supports_backtesting == SupportsBacktesting.NO_ACTION:
+                noaction_pairlists.append(pairlist_handler.name)
+            if pairlist_handler.supports_backtesting == SupportsBacktesting.BIASED:
+                biased_pairlists.append(pairlist_handler.name)
+
+        if noaction_pairlists:
+            logger.warning(
+                f"Pairlist Handlers {', '.join(noaction_pairlists)} do not generate "
+                "any changes during backtesting. While it's safe to leave them enabled, they will "
+                "not behave like in dry/live modes. "
+            )
+
+        if biased_pairlists:
+            logger.warning(
+                f"Pairlist Handlers {', '.join(biased_pairlists)} will introduce a lookahead bias "
+                "to your backtest results, as they use today's data - which inheritly suffers from "
+                "'winner bias'."
+            )
+        if pairlist_errors:
+            raise OperationalException(
+                f"Pairlist Handlers {', '.join(pairlist_errors)} do not support backtesting."
+            )
+
    @property
    def whitelist(self) -> List[str]:
        """The current whitelist"""
@@ -1787,7 +1787,7 @@ class Telegram(RPCHandler):
            "_Bot Control_\n"
            "------------\n"
            "*/start:* `Starts the trader`\n"
-            "*/stop:* Stops the trader\n"
+            "*/stop:* `Stops the trader`\n"
            "*/stopentry:* `Stops entering, but handles open trades gracefully` \n"
            "*/forceexit <trade_id>|all:* `Instantly exits the given trade or all trades, "
            "regardless of profit`\n"
@@ -1820,7 +1820,7 @@ class Telegram(RPCHandler):
            "that represents the current market direction. If no direction is provided `"
            "`the currently set market direction will be output.` \n"
            "*/list_custom_data <trade_id> <key>:* `List custom_data for Trade ID & Key combo.`\n"
-            "`If no Key is supplied it will list all key-value pairs found for that Trade ID.`"
+            "`If no Key is supplied it will list all key-value pairs found for that Trade ID.`\n"
            "_Statistics_\n"
            "------------\n"
            "*/status <trade_id>|[table]:* `Lists all open trades`\n"
@@ -6,6 +6,7 @@ This module defines the interface to apply for strategies
 import logging
 from abc import ABC, abstractmethod
 from datetime import datetime, timedelta, timezone
+from math import isinf, isnan
 from typing import Dict, List, Optional, Tuple, Union

 from pandas import DataFrame
@@ -1423,7 +1424,9 @@ class IStrategy(ABC, HyperStrategyMixin):
                after_fill=after_fill,
            )
            # Sanity check - error cases will return None
-            if stop_loss_value_custom:
+            if stop_loss_value_custom and not (
+                isnan(stop_loss_value_custom) or isinf(stop_loss_value_custom)
+            ):
                stop_loss_value = stop_loss_value_custom
                trade.adjust_stop_loss(
                    bound or current_rate, stop_loss_value, allow_refresh=after_fill
@@ -1,7 +1,7 @@
 from freqtrade_client.ft_rest_client import FtRestClient


-__version__ = "2024.6-dev"
+__version__ = "2024.7-dev"

 if "dev" in __version__:
    from pathlib import Path
@@ -1,3 +1,3 @@
 # Requirements for freqtrade client library
 requests==2.32.3
-python-rapidjson==1.17
+python-rapidjson==1.18
@@ -144,6 +144,7 @@ extend-select = [
  # "TCH",  # flake8-type-checking
  "PTH",    # flake8-use-pathlib
  # "RUF",    # ruff
+  "ASYNC",  # flake8-async
 ]

 extend-ignore = [
@@ -7,8 +7,8 @@
 -r docs/requirements-docs.txt

 coveralls==4.0.1
-ruff==0.4.9
-mypy==1.10.0
+ruff==0.5.0
+mypy==1.10.1
 pre-commit==3.7.1
 pytest==8.2.2
 pytest-asyncio==0.23.7
@@ -19,7 +19,7 @@ pytest-timeout==2.3.1
 pytest-xdist==3.6.1
 isort==5.13.2
 # For datetime mocking
-time-machine==2.14.1
+time-machine==2.14.2

 # Convert jupyter notebooks to markdown documents
 nbconvert==7.16.4
@@ -27,6 +27,6 @@ nbconvert==7.16.4
 # mypy types
 types-cachetools==5.3.0.7
 types-filelock==3.2.7
-types-requests==2.32.0.20240602
+types-requests==2.32.0.20240622
 types-tabulate==0.9.0.20240106
 types-python-dateutil==2.9.0.20240316
@@ -2,7 +2,8 @@
 -r requirements.txt

 # Required for hyperopt
-scipy==1.13.1
+scipy==1.14.0; python_version >= "3.10"
+scipy==1.13.1; python_version < "3.10"
 scikit-learn==1.5.0
 ft-scikit-optimize==0.9.2
-filelock==3.15.1
+filelock==3.15.4
@@ -1,13 +1,13 @@
 numpy==1.26.4
 pandas==2.2.2
-bottleneck==1.3.8
-numexpr==2.10.0
+bottleneck==1.4.0
+numexpr==2.10.1
 pandas-ta==0.3.14b

-ccxt==4.3.46
+ccxt==4.3.54
 cryptography==42.0.8
 aiohttp==3.9.5
-SQLAlchemy==2.0.30
+SQLAlchemy==2.0.31
 python-telegram-bot==21.3
 # can't be hard-pinned due to telegram-bot pinning httpx with ~
 httpx>=0.24.1
@@ -16,7 +16,7 @@ cachetools==5.3.3
 requests==2.32.3
 urllib3==2.2.2
 jsonschema==4.22.0
-TA-Lib==0.4.31
+TA-Lib==0.4.32
 technical==1.4.3
 tabulate==0.9.0
 pycoingecko==3.1.0
@@ -30,7 +30,7 @@ pyarrow==16.1.0; platform_machine != 'armv7l'
 py_find_1st==1.1.6

 # Load ticker files 30% faster
-python-rapidjson==1.17
+python-rapidjson==1.18
 # Properly format api responses
 orjson==3.10.5

@@ -42,8 +42,8 @@ fastapi==0.111.0
 pydantic==2.7.4
 uvicorn==0.30.1
 pyjwt==2.8.0
-aiofiles==23.2.1
-psutil==5.9.8
+aiofiles==24.1.0
+psutil==6.0.0

 # Support for colorized terminal output
 colorama==0.4.6
@@ -49,7 +49,7 @@ function updateenv() {
    source .venv/bin/activate
    SYS_ARCH=$(uname -m)
    echo "pip install in-progress. Please wait..."
-    ${PYTHON} -m pip install --upgrade pip wheel setuptools
+    ${PYTHON} -m pip install --upgrade "pip<=24.0" wheel setuptools
    REQUIREMENTS_HYPEROPT=""
    REQUIREMENTS_PLOT=""
    REQUIREMENTS_FREQAI=""
@@ -83,6 +83,12 @@ def test_download_data_main_trades(mocker):
    assert dl_mock.call_count == 1
    assert convert_mock.call_count == 1

+    # Exchange that doesn't support historic downloads
+    config["exchange"]["name"] = "bybit"
+    with pytest.raises(OperationalException, match=r"Trade history not available for .*"):
+        config
+        download_data_main(config)
+

 def test_download_data_main_data_invalid(mocker):
    patch_exchange(mocker, id="kraken")
@@ -429,7 +429,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) ->
        backtesting.start()


-def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> None:
+def test_backtesting_no_pair_left(default_conf, mocker) -> None:
    mocker.patch(f"{EXMS}.exchange_has", MagicMock(return_value=True))
    mocker.patch(
        "freqtrade.data.history.history_utils.load_pair_history",
@@ -449,13 +449,6 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) ->
    with pytest.raises(OperationalException, match="No pair in whitelist."):
        Backtesting(default_conf)

-    default_conf["pairlists"] = [{"method": "VolumePairList", "number_assets": 5}]
-    with pytest.raises(
-        OperationalException,
-        match=r"VolumePairList not allowed for backtesting\..*StaticPairList.*",
-    ):
-        Backtesting(default_conf)
-
    default_conf.update(
        {
            "pairlists": [{"method": "StaticPairList"}],
@@ -469,7 +462,7 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) ->
        Backtesting(default_conf)


-def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, tickers) -> None:
+def test_backtesting_pairlist_list(default_conf, mocker, tickers) -> None:
    mocker.patch(f"{EXMS}.exchange_has", MagicMock(return_value=True))
    mocker.patch(f"{EXMS}.get_tickers", tickers)
    mocker.patch(f"{EXMS}.price_to_precision", lambda s, x, y: y)
@@ -495,12 +488,6 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti
    ):
        Backtesting(default_conf)

-    default_conf["pairlists"] = [{"method": "StaticPairList"}, {"method": "PerformanceFilter"}]
-    with pytest.raises(
-        OperationalException, match="PerformanceFilter not allowed for backtesting."
-    ):
-        Backtesting(default_conf)
-
    default_conf["pairlists"] = [
        {"method": "StaticPairList"},
        {"method": "PrecisionFilter"},
@@ -38,6 +38,7 @@ TESTABLE_PAIRLISTS = [p for p in AVAILABLE_PAIRLISTS if p not in ["RemotePairLis

@pytest.fixture(scope="function")
 def whitelist_conf(default_conf):
+    default_conf["runmode"] = "dry_run"
    default_conf["stake_currency"] = "BTC"
    default_conf["exchange"]["pair_whitelist"] = [
        "ETH/BTC",
@@ -68,6 +69,7 @@ def whitelist_conf(default_conf):

@pytest.fixture(scope="function")
 def whitelist_conf_2(default_conf):
+    default_conf["runmode"] = "dry_run"
    default_conf["stake_currency"] = "BTC"
    default_conf["exchange"]["pair_whitelist"] = [
        "ETH/BTC",
@@ -94,6 +96,7 @@ def whitelist_conf_2(default_conf):

@pytest.fixture(scope="function")
 def whitelist_conf_agefilter(default_conf):
+    default_conf["runmode"] = "dry_run"
    default_conf["stake_currency"] = "BTC"
    default_conf["exchange"]["pair_whitelist"] = [
        "ETH/BTC",
@@ -773,7 +776,7 @@ def test_VolumePairList_whitelist_gen(
    whitelist_result,
    caplog,
 ) -> None:
-    whitelist_conf["runmode"] = "backtest"
+    whitelist_conf["runmode"] = "util_exchange"
    whitelist_conf["pairlists"] = pairlists
    whitelist_conf["stake_currency"] = base_currency

@@ -2387,3 +2390,65 @@ def test_MarketCapPairList_exceptions(mocker, default_conf_usdt):
        OperationalException, match="This filter only support marketcap rank up to 250."
    ):
        PairListManager(exchange, default_conf_usdt)
+
+
+@pytest.mark.parametrize(
+    "pairlists,expected_error,expected_warning",
+    [
+        (
+            [{"method": "StaticPairList"}],
+            None,  # Error
+            None,  # Warning
+        ),
+        (
+            [{"method": "VolumePairList", "number_assets": 10}],
+            "VolumePairList",  # Error
+            None,  # Warning
+        ),
+        (
+            [{"method": "MarketCapPairList", "number_assets": 10}],
+            None,  # Error
+            r"MarketCapPairList.*lookahead.*",  # Warning
+        ),
+        (
+            [{"method": "StaticPairList"}, {"method": "FullTradesFilter"}],
+            None,  # Error
+            r"FullTradesFilter do not generate.*",  # Warning
+        ),
+        (  # combi, fails and warns
+            [
+                {"method": "VolumePairList", "number_assets": 10},
+                {"method": "MarketCapPairList", "number_assets": 10},
+            ],
+            "VolumePairList",  # Error
+            r"MarketCapPairList.*lookahead.*",  # Warning
+        ),
+    ],
+)
+def test_backtesting_modes(
+    mocker, default_conf_usdt, pairlists, expected_error, expected_warning, caplog, markets, tickers
+):
+    default_conf_usdt["runmode"] = "dry_run"
+    default_conf_usdt["pairlists"] = pairlists
+
+    mocker.patch.multiple(
+        EXMS,
+        markets=PropertyMock(return_value=markets),
+        exchange_has=MagicMock(return_value=True),
+        get_tickers=tickers,
+    )
+    exchange = get_patched_exchange(mocker, default_conf_usdt)
+
+    # Dry run mode - works always
+    PairListManager(exchange, default_conf_usdt)
+
+    default_conf_usdt["runmode"] = "backtest"
+    if expected_error:
+        with pytest.raises(OperationalException, match=f"Pairlist Handlers {expected_error}.*"):
+            PairListManager(exchange, default_conf_usdt)
+
+    if not expected_error:
+        PairListManager(exchange, default_conf_usdt)
+
+    if expected_warning:
+        assert log_has_re(f"Pairlist Handlers {expected_warning}", caplog)
@@ -1,5 +1,6 @@
 # pragma pylint: disable=missing-docstring, C0103
 import logging
+import math
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from unittest.mock import MagicMock
@@ -458,55 +459,66 @@ def test_min_roi_reached3(default_conf, fee) -> None:
            ExitType.TRAILING_STOP_LOSS,
            None,
        ),
-        (0.01, 0.96, ExitType.NONE, None, True, False, 0.05, 1, ExitType.NONE, None),
-        (0.05, 1, ExitType.NONE, None, True, False, -0.01, 1, ExitType.TRAILING_STOP_LOSS, None),
+        (0.01, 0.96, ExitType.NONE, None, True, False, 0.05, 0.998, ExitType.NONE, None),
+        (
+            0.05,
+            0.998,
+            ExitType.NONE,
+            None,
+            True,
+            False,
+            -0.01,
+            0.998,
+            ExitType.TRAILING_STOP_LOSS,
+            None,
+        ),
        # Default custom case - trails with 10%
-        (0.05, 0.95, ExitType.NONE, None, False, True, -0.02, 0.95, ExitType.NONE, None),
+        (0.05, 0.945, ExitType.NONE, None, False, True, -0.02, 0.945, ExitType.NONE, None),
        (
            0.05,
-            0.95,
+            0.945,
            ExitType.NONE,
            None,
            False,
            True,
            -0.06,
-            0.95,
+            0.945,
            ExitType.TRAILING_STOP_LOSS,
            None,
        ),
        (
            0.05,
-            1,
+            0.998,
            ExitType.NONE,
            None,
            False,
            True,
            -0.06,
-            1,
+            0.998,
            ExitType.TRAILING_STOP_LOSS,
            lambda **kwargs: -0.05,
        ),
        (
            0.05,
-            1,
+            0.998,
            ExitType.NONE,
            None,
            False,
            True,
            0.09,
-            1.04,
+            1.036,
            ExitType.NONE,
            lambda **kwargs: -0.05,
        ),
        (
            0.05,
-            0.95,
+            0.945,
            ExitType.NONE,
            None,
            False,
            True,
            0.09,
-            0.98,
+            0.981,
            ExitType.NONE,
            lambda current_profit, **kwargs: (
                -0.1 if current_profit < 0.6 else -(current_profit * 2)
@@ -525,6 +537,19 @@ def test_min_roi_reached3(default_conf, fee) -> None:
            ExitType.NONE,
            lambda **kwargs: None,
        ),
+        # Error case - Returning inf.
+        (
+            0.05,
+            0.9,
+            ExitType.NONE,
+            None,
+            False,
+            True,
+            0.09,
+            0.9,
+            ExitType.NONE,
+            lambda **kwargs: math.inf,
+        ),
    ],
 )
 def test_ft_stoploss_reached(
@@ -552,6 +577,8 @@ def test_ft_stoploss_reached(
        exchange="binance",
        open_rate=1,
        liquidation_price=liq,
+        price_precision=4,
+        precision_mode=2,
    )
    trade.adjust_min_max_rates(trade.open_rate, trade.open_rate)
    strategy.trailing_stop = trailing
@@ -577,7 +604,7 @@ def test_ft_stoploss_reached(
        assert sl_flag.exit_flag is False
    else:
        assert sl_flag.exit_flag is True
-    assert round(trade.stop_loss, 2) == adjusted
+    assert round(trade.stop_loss, 3) == adjusted
    current_rate2 = trade.open_rate * (1 + profit2)

    sl_flag = strategy.ft_stoploss_reached(
@@ -593,7 +620,7 @@ def test_ft_stoploss_reached(
        assert sl_flag.exit_flag is False
    else:
        assert sl_flag.exit_flag is True
-    assert round(trade.stop_loss, 2) == adjusted2
+    assert round(trade.stop_loss, 3) == adjusted2

    strategy.custom_stoploss = original_stopvalue