Merge pull request #9876 from freqtrade/new_release

New release 2024.2

.github/dependabot.yml

@@ -10,8 +10,17 @@ updates:
   directory: "/"
   schedule:
     interval: weekly
+    time: "03:00"
+    timezone: "Etc/UTC"
   open-pull-requests-limit: 15
   target-branch: develop
+  groups:
+    types:
+      patterns:
+        - "types-*"
+    pytest:
+      patterns:
+        - "pytest*"
 
 - package-ecosystem: "github-actions"
   directory: "/"

.github/workflows/ci.yml

@@ -325,7 +325,7 @@ jobs:
     - uses: actions/setup-python@v5
       with:
         python-version: "3.10"
-    - uses: pre-commit/action@v3.0.0
+    - uses: pre-commit/action@v3.0.1
 
   docs-check:
     runs-on: ubuntu-22.04

.github/workflows/pre-commit-update.yml

@@ -30,12 +30,13 @@ jobs:
       - name: Run pre-commit
         run: pre-commit run --all-files
 
-      - uses: peter-evans/create-pull-request@v5
+      - uses: peter-evans/create-pull-request@v6
         with:
           token: ${{ secrets.REPO_SCOPED_TOKEN }}
           add-paths: .pre-commit-config.yaml
           labels: |
             Tech maintenance
+            Dependencies
           branch: update/pre-commit-hooks
           title: Update pre-commit hooks
           commit-message: "chore: update pre-commit hooks"

.pre-commit-config.yaml

@@ -16,10 +16,10 @@ repos:
         additional_dependencies:
           - types-cachetools==5.3.0.7
           - types-filelock==3.2.7
-          - types-requests==2.31.0.20240125
+          - types-requests==2.31.0.20240218
           - types-tabulate==0.9.0.20240106
           - types-python-dateutil==2.8.19.20240106
-          - SQLAlchemy==2.0.25
+          - SQLAlchemy==2.0.27
         # stages: [push]
 
   - repo: https://github.com/pycqa/isort
@@ -31,7 +31,7 @@ repos:
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.1.14'
+    rev: 'v0.2.2'
     hooks:
       - id: ruff
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11.7-slim-bookworm as base
+FROM python:3.11.8-slim-bookworm as base
 
 # Setup env
 ENV LANG C.UTF-8

docker/Dockerfile.armhf

@@ -1,4 +1,4 @@
-FROM python:3.11.7-slim-bookworm as base
+FROM python:3.11.8-slim-bookworm as base
 
 # Setup env
 ENV LANG C.UTF-8

docs/configuration.md

@@ -14,7 +14,7 @@ You can specify a different configuration file used by the bot with the `-c/--co
 If you used the [Quick start](docker_quickstart.md#docker-quick-start) method for installing
 the bot, the installation script should have already created the default configuration file (`config.json`) for you.
 
-If the default configuration file is not created we recommend to use `freqtrade new-config --config config.json` to generate a basic configuration file.
+If the default configuration file is not created we recommend to use `freqtrade new-config --config user_data/config.json` to generate a basic configuration file.
 
 The Freqtrade configuration file is to be written in JSON format.
 

docs/includes/pairlists.md

@@ -6,7 +6,7 @@ In your configuration, you can use Static Pairlist (defined by the [`StaticPairL
 
 Additionally, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter), [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist.
 
-If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You should always configure either `StaticPairList` or `VolumePairList` as the starting Pairlist Handler.
+If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You can define either `StaticPairList`, `VolumePairList`, `ProducerPairList`, `RemotePairList` or `MarketCapPairList` as the starting Pairlist Handler.
 
 Inactive markets are always removed from the resulting pairlist. Explicitly blacklisted pairs (those in the `pair_blacklist` configuration setting) are also always removed from the resulting pairlist.
 
@@ -24,6 +24,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged
 * [`VolumePairList`](#volume-pair-list)
 * [`ProducerPairList`](#producerpairlist)
 * [`RemotePairList`](#remotepairlist)
+* [`MarketCapPairList`](#marketcappairlist)
 * [`AgeFilter`](#agefilter)
 * [`FullTradesFilter`](#fulltradesfilter)
 * [`OffsetFilter`](#offsetfilter)
@@ -67,7 +68,7 @@ When used in the leading position of the chain of Pairlist Handlers, the `pair_w
 
 The `refresh_period` setting allows to define the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes).
 The pairlist cache (`refresh_period`) on `VolumePairList` is only applicable to generating pairlists.
-Filtering instances (not the first position in the list) will not apply any cache and will always use up-to-date data.
+Filtering instances (not the first position in the list) will not apply any cache (beyond caching candles for the duration of the candle in advanced mode) and will always use up-to-date data.
 
 `VolumePairList` is per default based on the ticker data from exchange, as reported by the ccxt library:
 
@@ -200,7 +201,7 @@ The RemotePairList is defined in the pairlists section of the configuration sett
 
 The optional `mode` option specifies if the pairlist should be used as a `blacklist` or as a `whitelist`. The default value is "whitelist".
 
-The optional `processing_mode` option in the RemotePairList configuration determines how the retrieved pairlist is processed. It can have two values: "filter" or "append".
+The optional `processing_mode` option in the RemotePairList configuration determines how the retrieved pairlist is processed. It can have two values: "filter" or "append". The default value is "filter".
 
 In "filter" mode, the retrieved pairlist is used as a filter. Only the pairs present in both the original pairlist and the retrieved pairlist are included in the final pairlist. Other pairs are filtered out.
 
@@ -264,6 +265,25 @@ The optional `bearer_token` will be included in the requests Authorization Heade
 !!! Note
     In case of a server error the last received pairlist will be kept if `keep_pairlist_on_failure` is set to true, when set to false a empty pairlist is returned.
 
+#### MarketCapPairList
+
+`MarketCapPairList` employs sorting/filtering of pairs by their marketcap rank based of CoinGecko. It will only recognize coins up to the coin placed at rank 250. The returned pairlist will be sorted based of their marketcap ranks.
+
+```json
+"pairlists": [
+    {
+        "method": "MarketCapPairList",
+        "number_assets": 20,
+        "max_rank": 50,
+        "refresh_period": 86400
+    }
+]
+```
+
+`number_assets` defines the maximum number of pairs returned by the pairlist. `max_rank` will determine the maximum rank used in creating/filtering the pairlist. It's expected that some coins within the top `max_rank` marketcap will not be included in the resulting pairlist since not all pairs will have active trading pairs in your preferred market/stake/exchange combination.
+
+`refresh_period` setting defines the period (in seconds) at which the marketcap rank data will be refreshed. Defaults to 86,400s (1 day). The pairlist cache (`refresh_period`) is applicable on both generating pairlists (first position in the list) and filtering instances (not the first position in the list).
+
 #### AgeFilter
 
 Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`) or more than `max_days_listed` days (defaults `None` mean infinity).

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
 markdown==3.5.2
 mkdocs==1.5.3
-mkdocs-material==9.5.6
+mkdocs-material==9.5.11
 mdx_truly_sane_lists==1.3
 pymdown-extensions==10.7
 jinja2==3.1.3

docs/sql_cheatsheet.md

@@ -109,7 +109,7 @@ Freqtrade does not depend or install any additional database driver. Please refe
 The following systems have been tested and are known to work with freqtrade:
 
 * sqlite (default)
-* PostgreSQL)
+* PostgreSQL
 * MariaDB
 
 !!! Warning

docs/strategy-callbacks.md

@@ -767,6 +767,7 @@ This callback is **not** called when there is an open order (either buy or sell)
 `adjust_trade_position()` is called very frequently for the duration of a trade, so you must keep your implementation as performant as possible.
 
 Position adjustments will always be applied in the direction of the trade, so a positive value will always increase your position (negative values will decrease your position), no matter if it's a long or short trade.
+Adjustment orders can be assigned with a tag by returning a 2 element Tuple, with the first element being the adjustment amount, and the 2nd element the tag (e.g. `return 250, 'increase_favorable_conditions'`).
 
 Modifications to leverage are not possible, and the stake-amount returned is assumed to be before applying leverage.
 
@@ -790,7 +791,7 @@ Returning a value more than the above (so remaining stake_amount would become ne
     If you wish to buy additional orders with DCA, then make sure to leave enough funds in the wallet for that.
     Using 'unlimited' stake amount with DCA orders requires you to also implement the `custom_stake_amount()` callback to avoid allocating all funds to the initial order.
 
-!!! Warning
+!!! Warning "Stoploss calculation"
     Stoploss is still calculated from the initial opening price, not averaged price.
     Regular stoploss rules still apply (cannot move down).
 
@@ -800,6 +801,11 @@ Returning a value more than the above (so remaining stake_amount would become ne
     During backtesting this callback is called for each candle in `timeframe` or `timeframe_detail`, so run-time performance will be affected.
     This can also cause deviating results between live and backtesting, since backtesting can adjust the trade only once per candle, whereas live could adjust the trade multiple times per candle.
 
+!!! Warning "Performance with many position adjustments"
+    Position adjustments can be a good approach to increase a strategy's output - but it can also have drawbacks if using this feature extensively.  
+    Each of the orders will be attached to the trade object for the duration of the trade - hence increasing memory usage.
+    Trades with long duration and 10s or even 100ds of position adjustments are therefore not recommended, and should be closed at regular intervals to not affect performance.
+
 ``` python
 from freqtrade.persistence import Trade
 
@@ -833,7 +839,8 @@ class DigDeeperStrategy(IStrategy):
                               min_stake: Optional[float], max_stake: float,
                               current_entry_rate: float, current_exit_rate: float,
                               current_entry_profit: float, current_exit_profit: float,
-                              **kwargs) -> Optional[float]:
+                              **kwargs
+                              ) -> Union[Optional[float], Tuple[Optional[float], Optional[str]]]:
         """
         Custom trade adjustment logic, returning the stake amount that a trade should be
         increased or decreased.
@@ -859,11 +866,12 @@ class DigDeeperStrategy(IStrategy):
         :return float: Stake amount to adjust your trade,
                        Positive values to increase position, Negative values to decrease position.
                        Return None for no action.
+                       Optionally, return a tuple with a 2nd element with an order reason
         """
 
         if current_profit > 0.05 and trade.nr_of_successful_exits == 0:
             # Take half of the profit at +5%
-            return -(trade.stake_amount / 2)
+            return -(trade.stake_amount / 2), 'half_profit_5%'
 
         if current_profit > -0.05:
             return None
@@ -891,7 +899,7 @@ class DigDeeperStrategy(IStrategy):
             stake_amount = filled_entries[0].stake_amount
             # This then calculates current safety order size
             stake_amount = stake_amount * (1 + (count_of_entries * 0.25))
-            return stake_amount
+            return stake_amount, '1/3rd_increase'
         except Exception as exception:
             return None
 

docs/utils.md

@@ -54,7 +54,7 @@ optional arguments:
 ### Create config examples
 
 ```
-$ freqtrade new-config --config config_binance.json
+$ freqtrade new-config --config user_data/config_binance.json
 
 ? Do you want to enable Dry-run (simulated trades)?  Yes
 ? Please insert your stake currency: BTC
@@ -990,11 +990,7 @@ options:
   -h, --help            show this help message and exit
   --strategy-list STRATEGY_LIST [STRATEGY_LIST ...]
                         Provide a space-separated list of strategies to
-                        backtest. Please note that timeframe needs to be set
-                        either in config or via command line. When using this
-                        together with `--export trades`, the strategy-name is
-                        injected into the filename (so `backtest-data.json`
-                        becomes `backtest-data-SampleStrategy.json`
+                        be converted.
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ Freqtrade bot """
-__version__ = '2024.1'
+__version__ = '2024.2'
 
 if 'dev' in __version__:
     from pathlib import Path

freqtrade/constants.py

@@ -33,9 +33,10 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
                          'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss',
                          'ProfitDrawDownHyperOptLoss']
 AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', 'RemotePairList',
-                       'AgeFilter', "FullTradesFilter", 'OffsetFilter', 'PerformanceFilter',
-                       'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
-                       'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
+                       'MarketCapPairList', 'AgeFilter', "FullTradesFilter", 'OffsetFilter',
+                       'PerformanceFilter', 'PrecisionFilter', 'PriceFilter',
+                       'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter',
+                       'VolatilityFilter']
 AVAILABLE_PROTECTIONS = ['CooldownPeriod',
                          'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard']
 AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5', 'feather', 'parquet']

freqtrade/data/converter/trade_converter_kraken.py

@@ -8,6 +8,7 @@ from freqtrade.data.converter.trade_converter import (trades_convert_types,
                                                       trades_df_remove_duplicates)
 from freqtrade.data.history.idatahandler import get_datahandler
 from freqtrade.exceptions import OperationalException
+from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
 
 
@@ -38,12 +39,22 @@ def import_kraken_trades_from_csv(config: Config, convert_to: str):
     }
     logger.info(f"Found csv files for {', '.join(data_symbols)}.")
 
+    if pairs_raw := config.get('pairs'):
+        pairs = expand_pairlist(pairs_raw, [m[0] for m in markets])
+        markets = {m for m in markets if m[0] in pairs}
+        if not markets:
+            logger.info(f"No data found for pairs {', '.join(pairs_raw)}.")
+            return
+    logger.info(f"Converting pairs: {', '.join(m[0] for m in markets)}.")
+
     for pair, name in markets:
+        logger.debug(f"Converting pair {pair}, files */{name}.csv")
         dfs = []
         # Load and combine all csv files for this pair
         for f in tradesdir.rglob(f"{name}.csv"):
             df = pd.read_csv(f, names=KRAKEN_CSV_TRADE_COLUMNS)
-            dfs.append(df)
+            if not df.empty:
+                dfs.append(df)
 
         # Load existing trades data
         if not dfs:
@@ -52,17 +63,18 @@ def import_kraken_trades_from_csv(config: Config, convert_to: str):
             continue
 
         trades = pd.concat(dfs, ignore_index=True)
+        del dfs
 
         trades.loc[:, 'timestamp'] = trades['timestamp'] * 1e3
         trades.loc[:, 'cost'] = trades['price'] * trades['amount']
         for col in DEFAULT_TRADES_COLUMNS:
             if col not in trades.columns:
-                trades[col] = ''
-
+                trades.loc[:, col] = ''
         trades = trades[DEFAULT_TRADES_COLUMNS]
         trades = trades_convert_types(trades)
 
         trades_df = trades_df_remove_duplicates(trades)
+        del trades
         logger.info(f"{pair}: {len(trades_df)} trades, from "
                     f"{trades_df['date'].min():{DATETIME_PRINT_FORMAT}} to "
                     f"{trades_df['date'].max():{DATETIME_PRINT_FORMAT}}")

freqtrade/data/metrics.py

@@ -143,8 +143,10 @@ def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date'
         starting_balance=starting_balance
     )
 
-    idxmin = max_drawdown_df['drawdown_relative'].idxmax() if relative \
-        else max_drawdown_df['drawdown'].idxmin()
+    idxmin = (
+        max_drawdown_df['drawdown_relative'].idxmax()
+        if relative else max_drawdown_df['drawdown'].idxmin()
+    )
     if idxmin == 0:
         raise ValueError("No losing trade, therefore no drawdown.")
     high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col]
@@ -191,6 +193,9 @@ def calculate_cagr(days_passed: int, starting_balance: float, final_balance: flo
     :param final_balance: Final balance to calculate CAGR against
     :return: CAGR
     """
+    if final_balance < 0:
+        # With leveraged trades, final_balance can become negative.
+        return 0
     return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1
 
 

freqtrade/exchange/bybit.py

@@ -25,6 +25,7 @@ class Bybit(Exchange):
     officially supported by the Freqtrade development team. So some features
     may still not work as expected.
     """
+    unified_account = False
 
     _ft_has: Dict = {
         "ohlcv_candle_limit": 1000,
@@ -82,9 +83,20 @@ class Bybit(Exchange):
         Must be overridden in child methods if required.
         """
         try:
-            if self.trading_mode == TradingMode.FUTURES and not self._config['dry_run']:
-                position_mode = self._api.set_position_mode(False)
-                self._log_exchange_response('set_position_mode', position_mode)
+            if not self._config['dry_run']:
+                if self.trading_mode == TradingMode.FUTURES:
+                    position_mode = self._api.set_position_mode(False)
+                    self._log_exchange_response('set_position_mode', position_mode)
+                is_unified = self._api.is_unified_enabled()
+                # Returns a tuple of bools, first for margin, second for Account
+                if is_unified and len(is_unified) > 1 and is_unified[1]:
+                    self.unified_account = True
+                    logger.info("Bybit: Unified account.")
+                    raise OperationalException("Bybit: Unified account is not supported. "
+                                               "Please use a standard (sub)account.")
+                else:
+                    self.unified_account = False
+                    logger.info("Bybit: Standard account.")
         except ccxt.DDoSProtection as e:
             raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:

freqtrade/exchange/common.py

@@ -2,7 +2,7 @@ import asyncio
 import logging
 import time
 from functools import wraps
-from typing import Any, Callable, Optional, TypeVar, cast, overload
+from typing import Any, Callable, Dict, List, Optional, TypeVar, cast, overload
 
 from freqtrade.constants import ExchangeConfig
 from freqtrade.exceptions import DDosProtection, RetryableOrderError, TemporaryError
@@ -60,16 +60,17 @@ SUPPORTED_EXCHANGES = [
     'okx',
 ]
 
-EXCHANGE_HAS_REQUIRED = [
+# either the main, or replacement methods (array) is required
+EXCHANGE_HAS_REQUIRED: Dict[str, List[str]] = {
     # Required / private
-    'fetchOrder',
-    'cancelOrder',
-    'createOrder',
-    'fetchBalance',
+    'fetchOrder': ['fetchOpenOrder', 'fetchClosedOrder'],
+    'cancelOrder': [],
+    'createOrder': [],
+    'fetchBalance': [],
 
     # Public endpoints
-    'fetchOHLCV',
-]
+    'fetchOHLCV': [],
+}
 
 EXCHANGE_HAS_OPTIONAL = [
     # Private
@@ -86,6 +87,7 @@ EXCHANGE_HAS_OPTIONAL = [
     # 'fetchPositions',  # Futures trading
     # 'fetchLeverageTiers',  # Futures initialization
     # 'fetchMarketLeverageTiers',  # Futures initialization
+    # 'fetchOpenOrder', 'fetchClosedOrder',  # replacement for fetchOrder
     # 'fetchOpenOrders', 'fetchClosedOrders',  # 'fetchOrders',  # Refinding balance...
 ]
 

freqtrade/exchange/exchange.py

@@ -23,7 +23,7 @@ from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHAN
                                  BuySell, Config, EntryExit, ExchangeConfig,
                                  ListPairsWithTimeframes, MakerTaker, OBLiteral, PairWithTimeframe)
 from freqtrade.data.converter import clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_dict_to_list
-from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, PriceType, TradingMode
+from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, PriceType, RunMode, TradingMode
 from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError,
                                   InvalidOrderException, OperationalException, PricingError,
                                   RetryableOrderError, TemporaryError)
@@ -43,6 +43,7 @@ from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.util import dt_from_ts, dt_now
 from freqtrade.util.datetime_helpers import dt_humanize, dt_ts
+from freqtrade.util.periodic_cache import PeriodicCache
 
 
 logger = logging.getLogger(__name__)
@@ -131,6 +132,7 @@ class Exchange:
 
         # Holds candles
         self._klines: Dict[PairWithTimeframe, DataFrame] = {}
+        self._expiring_candle_cache: Dict[Tuple[str, int], PeriodicCache] = {}
 
         # Holds all open sell orders for dry_run
         self._dry_run_open_orders: Dict[str, Any] = {}
@@ -595,7 +597,11 @@ class Exchange:
             raise OperationalException(
                 f"Invalid timeframe '{timeframe}'. This exchange supports: {self.timeframes}")
 
-        if timeframe and timeframe_to_minutes(timeframe) < 1:
+        if (
+            timeframe
+            and self._config['runmode'] != RunMode.UTIL_EXCHANGE
+            and timeframe_to_minutes(timeframe) < 1
+        ):
             raise OperationalException("Timeframes < 1m are currently not supported by Freqtrade.")
 
     def validate_ordertypes(self, order_types: Dict) -> None:
@@ -1238,7 +1244,7 @@ class Exchange:
                 f'Insufficient funds to create {ordertype} {side} order on market {pair}. '
                 f'Tried to {side} amount {amount} at rate {limit_rate} with '
                 f'stop-price {stop_price_norm}. Message: {e}') from e
-        except (ccxt.InvalidOrder, ccxt.BadRequest) as e:
+        except (ccxt.InvalidOrder, ccxt.BadRequest, ccxt.OperationRejected) as e:
             # Errors:
             # `Order would trigger immediately.`
             raise InvalidOrderException(
@@ -1254,11 +1260,43 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
+    def fetch_order_emulated(self, order_id: str, pair: str, params: Dict) -> Dict:
+        """
+        Emulated fetch_order if the exchange doesn't support fetch_order, but requires separate
+        calls for open and closed orders.
+        """
+        try:
+            order = self._api.fetch_open_order(order_id, pair, params=params)
+            self._log_exchange_response('fetch_open_order', order)
+            order = self._order_contracts_to_amount(order)
+            return order
+        except ccxt.OrderNotFound:
+            try:
+                order = self._api.fetch_closed_order(order_id, pair, params=params)
+                self._log_exchange_response('fetch_closed_order', order)
+                order = self._order_contracts_to_amount(order)
+                return order
+            except ccxt.OrderNotFound as e:
+                raise RetryableOrderError(
+                    f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e
+        except ccxt.InvalidOrder as e:
+            raise InvalidOrderException(
+                f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
+        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
+            raise TemporaryError(
+                f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e
+        except ccxt.BaseError as e:
+            raise OperationalException(e) from e
+
     @retrier(retries=API_FETCH_ORDER_RETRY_COUNT)
     def fetch_order(self, order_id: str, pair: str, params: Dict = {}) -> Dict:
         if self._config['dry_run']:
             return self.fetch_dry_run_order(order_id)
         try:
+            if not self.exchange_has('fetchOrder'):
+                return self.fetch_order_emulated(order_id, pair, params)
             order = self._api.fetch_order(order_id, pair, params=params)
             self._log_exchange_response('fetch_order', order)
             order = self._order_contracts_to_amount(order)
@@ -2120,6 +2158,39 @@ class Exchange:
 
         return results_df
 
+    def refresh_ohlcv_with_cache(
+        self,
+        pairs: List[PairWithTimeframe],
+        since_ms: int
+    ) -> Dict[PairWithTimeframe, DataFrame]:
+        """
+        Refresh ohlcv data for all pairs in needed_pairs if necessary.
+        Caches data with expiring per timeframe.
+        Should only be used for pairlists which need "on time" expirarion, and no longer cache.
+        """
+
+        timeframes = {p[1] for p in pairs}
+        for timeframe in timeframes:
+            if (timeframe, since_ms) not in self._expiring_candle_cache:
+                timeframe_in_sec = timeframe_to_seconds(timeframe)
+                # Initialise cache
+                self._expiring_candle_cache[(timeframe, since_ms)] = PeriodicCache(
+                    ttl=timeframe_in_sec, maxsize=1000)
+
+        # Get candles from cache
+        candles = {
+            c: self._expiring_candle_cache[(c[1], since_ms)].get(c, None) for c in pairs
+            if c in self._expiring_candle_cache[(c[1], since_ms)]
+        }
+        pairs_to_download = [p for p in pairs if p not in candles]
+        if pairs_to_download:
+            candles = self.refresh_latest_ohlcv(
+                pairs_to_download, since_ms=since_ms, cache=False
+            )
+            for c, val in candles.items():
+                self._expiring_candle_cache[(c[1], since_ms)][c] = val
+        return candles
+
     def _now_is_time_to_refresh(self, pair: str, timeframe: str, candle_type: CandleType) -> bool:
         # Timeframe in seconds
         interval_in_sec = timeframe_to_seconds(timeframe)
@@ -2681,7 +2752,7 @@ class Exchange:
             self._log_exchange_response('set_leverage', res)
         except ccxt.DDoSProtection as e:
             raise DDosProtection(e) from e
-        except (ccxt.BadRequest, ccxt.InsufficientFunds) as e:
+        except (ccxt.BadRequest, ccxt.OperationRejected, ccxt.InsufficientFunds) as e:
             if not accept_fail:
                 raise TemporaryError(
                     f'Could not set leverage due to {e.__class__.__name__}. Message: {e}') from e
@@ -2723,7 +2794,7 @@ class Exchange:
             self._log_exchange_response('set_margin_mode', res)
         except ccxt.DDoSProtection as e:
             raise DDosProtection(e) from e
-        except ccxt.BadRequest as e:
+        except (ccxt.BadRequest, ccxt.OperationRejected) as e:
             if not accept_fail:
                 raise TemporaryError(
                     f'Could not set margin mode due to {e.__class__.__name__}. Message: {e}') from e

freqtrade/exchange/exchange_utils.py

@@ -40,21 +40,34 @@ def available_exchanges(ccxt_module: Optional[CcxtModuleType] = None) -> List[st
 
 
 def validate_exchange(exchange: str) -> Tuple[bool, str]:
+    """
+    returns: can_use, reason
+        with Reason including both missing and missing_opt
+    """
     ex_mod = getattr(ccxt, exchange.lower())()
+    result = True
+    reason = ''
     if not ex_mod or not ex_mod.has:
         return False, ''
-    missing = [k for k in EXCHANGE_HAS_REQUIRED if ex_mod.has.get(k) is not True]
+    missing = [
+        k for k, v in EXCHANGE_HAS_REQUIRED.items()
+        if ex_mod.has.get(k) is not True
+        and not (all(ex_mod.has.get(x) for x in v))
+    ]
     if missing:
-        return False, f"missing: {', '.join(missing)}"
+        result = False
+        reason += f"missing: {', '.join(missing)}"
 
     missing_opt = [k for k in EXCHANGE_HAS_OPTIONAL if not ex_mod.has.get(k)]
 
     if exchange.lower() in BAD_EXCHANGES:
-        return False, BAD_EXCHANGES.get(exchange.lower(), '')
-    if missing_opt:
-        return True, f"missing opt: {', '.join(missing_opt)}"
+        result = False
+        reason = BAD_EXCHANGES.get(exchange.lower(), '')
 
-    return True, ''
+    if missing_opt:
+        reason += f"{'. ' if reason else ''}missing opt: {', '.join(missing_opt)}. "
+
+    return result, reason
 
 
 def _build_exchange_list_entry(

freqtrade/freqai/utils.py

@@ -118,10 +118,12 @@ def plot_feature_importance(model: Any, pair: str, dk: FreqaiDataKitchen,
         mdl = models[label]
         if "catboost.core" in str(mdl.__class__):
             feature_importance = mdl.get_feature_importance()
-        elif "lightgbm.sklearn" or "xgb" in str(mdl.__class__):
+        elif "lightgbm.sklearn" in str(mdl.__class__):
+            feature_importance = mdl.feature_importances_
+        elif "xgb" in str(mdl.__class__):
             feature_importance = mdl.feature_importances_
         else:
-            logger.info('Model type not support for generating feature importances.')
+            logger.info('Model type does not support generating feature importances.')
             return
 
         # Data preparation

freqtrade/freqtradebot.py

@@ -82,7 +82,6 @@ class FreqtradeBot(LoggingMixin):
 
         PairLocks.timeframe = self.config['timeframe']
 
-        self.pairlists = PairListManager(self.exchange, self.config)
         self.trading_mode: TradingMode = self.config.get('trading_mode', TradingMode.SPOT)
         self.last_process: Optional[datetime] = None
 
@@ -129,8 +128,9 @@ class FreqtradeBot(LoggingMixin):
                 self.update_funding_fees()
                 self.wallets.update()
 
-            # TODO: This would be more efficient if scheduled in utc time, and performed at each
-            # TODO: funding interval, specified by funding_fee_times on the exchange classes
+            # This would be more efficient if scheduled in utc time, and performed at each
+            # funding interval, specified by funding_fee_times on the exchange classes
+            # However, this reduces the precision - and might therefore lead to problems.
             for time_slot in range(0, 24):
                 for minutes in [1, 31]:
                     t = str(time(time_slot, minutes, 2))
@@ -432,10 +432,6 @@ class FreqtradeBot(LoggingMixin):
             try:
                 fo = self.exchange.fetch_order_or_stoploss_order(order.order_id, order.ft_pair,
                                                                  order.ft_order_side == 'stoploss')
-                if order.ft_order_side == 'stoploss':
-                    if fo and fo['status'] == 'open':
-                        # Assume this as the open stoploss order
-                        trade.stoploss_order_id = order.order_id
                 if fo:
                     logger.info(f"Found {order} for trade {trade}.")
                     self.update_trade_state(trade, order.order_id, fo,
@@ -645,8 +641,7 @@ class FreqtradeBot(LoggingMixin):
         max_entry_stake = self.exchange.get_max_pair_stake_amount(trade.pair, current_entry_rate)
         stake_available = self.wallets.get_available_stake_amount()
         logger.debug(f"Calling adjust_trade_position for pair {trade.pair}")
-        stake_amount = strategy_safe_wrapper(self.strategy.adjust_trade_position,
-                                             default_retval=None, supress_error=True)(
+        stake_amount, order_tag = self.strategy._adjust_trade_position_internal(
             trade=trade,
             current_time=datetime.now(timezone.utc), current_rate=current_entry_rate,
             current_profit=current_entry_profit, min_stake=min_entry_stake,
@@ -665,7 +660,8 @@ class FreqtradeBot(LoggingMixin):
                 else:
                     logger.debug("Max adjustment entries is set to unlimited.")
             self.execute_entry(trade.pair, stake_amount, price=current_entry_rate,
-                               trade=trade, is_short=trade.is_short, mode='pos_adjust')
+                               trade=trade, is_short=trade.is_short, mode='pos_adjust',
+                               enter_tag=order_tag)
 
         if stake_amount is not None and stake_amount < 0.0:
             # We should decrease our position
@@ -684,7 +680,7 @@ class FreqtradeBot(LoggingMixin):
                 return
 
             self.execute_trade_exit(trade, current_exit_rate, exit_check=ExitCheckTuple(
-                exit_type=ExitType.PARTIAL_EXIT), sub_trade_amt=amount)
+                exit_type=ExitType.PARTIAL_EXIT), sub_trade_amt=amount, exit_tag=order_tag)
 
     def _check_depth_of_market(self, pair: str, conf: Dict, side: SignalDirection) -> bool:
         """
@@ -706,7 +702,7 @@ class FreqtradeBot(LoggingMixin):
         delta = f"Delta: {bids_ask_delta}"
 
         logger.info(
-            f"{bids}, {asks}, {delta}, Direction: {side.value}"
+            f"{bids}, {asks}, {delta}, Direction: {side.value} "
             f"Bid Price: {order_book['bids'][0][0]}, Ask Price: {order_book['asks'][0][0]}, "
             f"Immediate Bid Quantity: {order_book['bids'][0][1]}, "
             f"Immediate Ask Quantity: {order_book['asks'][0][1]}."
@@ -782,6 +778,7 @@ class FreqtradeBot(LoggingMixin):
             leverage=leverage
         )
         order_obj = Order.parse_from_ccxt_object(order, pair, side, amount, enter_limit_requested)
+        order_obj.ft_order_tag = enter_tag
         order_id = order['id']
         order_status = order.get('status')
         logger.info(f"Order {order_id} was created for {pair} and status is {order_status}.")
@@ -894,17 +891,15 @@ class FreqtradeBot(LoggingMixin):
 
     def cancel_stoploss_on_exchange(self, trade: Trade) -> Trade:
         # First cancelling stoploss on exchange ...
-        if trade.stoploss_order_id:
+        for oslo in trade.open_sl_orders:
             try:
-                logger.info(f"Cancelling stoploss on exchange for {trade}")
+                logger.info(f"Cancelling stoploss on exchange for {trade} "
+                            f"order: {oslo.order_id}")
                 co = self.exchange.cancel_stoploss_order_with_result(
-                    trade.stoploss_order_id, trade.pair, trade.amount)
-                self.update_trade_state(trade, trade.stoploss_order_id, co, stoploss_order=True)
-
-                # Reset stoploss order id.
-                trade.stoploss_order_id = None
+                    oslo.order_id, trade.pair, trade.amount)
+                self.update_trade_state(trade, oslo.order_id, co, stoploss_order=True)
             except InvalidOrderException:
-                logger.exception(f"Could not cancel stoploss order {trade.stoploss_order_id} "
+                logger.exception(f"Could not cancel stoploss order {oslo.order_id} "
                                  f"for pair {trade.pair}")
         return trade
 
@@ -992,7 +987,7 @@ class FreqtradeBot(LoggingMixin):
 
         return enter_limit_requested, stake_amount, leverage
 
-    def _notify_enter(self, trade: Trade, order: Order, order_type: str,
+    def _notify_enter(self, trade: Trade, order: Order, order_type: Optional[str],
                       fill: bool = False, sub_trade: bool = False) -> None:
         """
         Sends rpc notification when a entry order occurred.
@@ -1016,7 +1011,7 @@ class FreqtradeBot(LoggingMixin):
             'direction': 'Short' if trade.is_short else 'Long',
             'limit': open_rate,  # Deprecated (?)
             'open_rate': open_rate,
-            'order_type': order_type,
+            'order_type': order_type or 'unknown',
             'stake_amount': trade.stake_amount,
             'stake_currency': self.config['stake_currency'],
             'base_currency': self.exchange.get_pair_base_currency(trade.pair),
@@ -1079,7 +1074,7 @@ class FreqtradeBot(LoggingMixin):
 
             if (
                 not trade.has_open_orders
-                and not trade.stoploss_order_id
+                and not trade.has_open_sl_orders
                 and not self.wallets.check_exit_amount(trade)
             ):
                 logger.warning(
@@ -1189,8 +1184,6 @@ class FreqtradeBot(LoggingMixin):
             order_obj = Order.parse_from_ccxt_object(stoploss_order, trade.pair, 'stoploss',
                                                      trade.amount, stop_price)
             trade.orders.append(order_obj)
-            trade.stoploss_order_id = str(stoploss_order['id'])
-            trade.stoploss_last_update = datetime.now(timezone.utc)
             return True
         except InsufficientFundsError as e:
             logger.warning(f"Unable to place stoploss order {e}.")
@@ -1198,13 +1191,11 @@ class FreqtradeBot(LoggingMixin):
             self.handle_insufficient_funds(trade)
 
         except InvalidOrderException as e:
-            trade.stoploss_order_id = None
             logger.error(f'Unable to place a stoploss order on exchange. {e}')
             logger.warning('Exiting the trade forcefully')
             self.emergency_exit(trade, stop_price)
 
         except ExchangeError:
-            trade.stoploss_order_id = None
             logger.exception('Unable to place a stoploss order on exchange.')
         return False
 
@@ -1218,27 +1209,28 @@ class FreqtradeBot(LoggingMixin):
         """
 
         logger.debug('Handling stoploss on exchange %s ...', trade)
-        stoploss_order = None
 
-        try:
-            # First we check if there is already a stoploss on exchange
-            stoploss_order = self.exchange.fetch_stoploss_order(
-                trade.stoploss_order_id, trade.pair) if trade.stoploss_order_id else None
-        except InvalidOrderException as exception:
-            logger.warning('Unable to fetch stoploss order: %s', exception)
+        stoploss_orders = []
+        for slo in trade.open_sl_orders:
+            stoploss_order = None
+            try:
+                # First we check if there is already a stoploss on exchange
+                stoploss_order = self.exchange.fetch_stoploss_order(
+                    slo.order_id, trade.pair) if slo.order_id else None
+            except InvalidOrderException as exception:
+                logger.warning('Unable to fetch stoploss order: %s', exception)
 
-        if stoploss_order:
-            self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order,
-                                    stoploss_order=True)
+            if stoploss_order:
+                stoploss_orders.append(stoploss_order)
+                self.update_trade_state(trade, slo.order_id, stoploss_order,
+                                        stoploss_order=True)
 
-        # We check if stoploss order is fulfilled
-        if stoploss_order and stoploss_order['status'] in ('closed', 'triggered'):
-            trade.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value
-            self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order,
-                                    stoploss_order=True)
-            self._notify_exit(trade, "stoploss", True)
-            self.handle_protections(trade.pair, trade.trade_direction)
-            return True
+            # We check if stoploss order is fulfilled
+            if stoploss_order and stoploss_order['status'] in ('closed', 'triggered'):
+                trade.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value
+                self._notify_exit(trade, "stoploss", True)
+                self.handle_protections(trade.pair, trade.trade_direction)
+                return True
 
         if trade.has_open_orders or not trade.is_open:
             # Trade has an open Buy or Sell order, Stoploss-handling can't happen in this case
@@ -1247,7 +1239,7 @@ class FreqtradeBot(LoggingMixin):
             return False
 
         # If enter order is fulfilled but there is no stoploss, we add a stoploss on exchange
-        if not stoploss_order:
+        if len(stoploss_orders) == 0:
             stop_price = trade.stoploss_or_liquidation
             if self.edge:
                 stoploss = self.edge.get_stoploss(pair=trade.pair)
@@ -1261,27 +1253,7 @@ class FreqtradeBot(LoggingMixin):
                 # in which case the trade will be closed - which we must check below.
                 return False
 
-        # If stoploss order is canceled for some reason we add it again
-        if (trade.is_open
-                and stoploss_order
-                and stoploss_order['status'] in ('canceled', 'cancelled')):
-            if self.create_stoploss_order(trade=trade, stop_price=trade.stoploss_or_liquidation):
-                return False
-            else:
-                logger.warning('Stoploss order was cancelled, but unable to recreate one.')
-
-        # Finally we check if stoploss on exchange should be moved up because of trailing.
-        # Triggered Orders are now real orders - so don't replace stoploss anymore
-        if (
-            trade.is_open and stoploss_order
-            and stoploss_order.get('status_stop') != 'triggered'
-            and (self.config.get('trailing_stop', False)
-                 or self.config.get('use_custom_stoploss', False))
-        ):
-            # if trailing stoploss is enabled we check if stoploss value has changed
-            # in which case we cancel stoploss order and put another one with new
-            # value immediately
-            self.handle_trailing_stoploss_on_exchange(trade, stoploss_order)
+        self.manage_trade_stoploss_orders(trade, stoploss_orders)
 
         return False
 
@@ -1317,6 +1289,42 @@ class FreqtradeBot(LoggingMixin):
                     logger.warning(f"Could not create trailing stoploss order "
                                    f"for pair {trade.pair}.")
 
+    def manage_trade_stoploss_orders(self, trade: Trade, stoploss_orders: List[Dict]):
+        """
+        Perform required actions acording to existing stoploss orders of trade
+        :param trade: Corresponding Trade
+        :param stoploss_orders: Current on exchange stoploss orders
+        :return: None
+        """
+        # If all stoploss orderd are canceled for some reason we add it again
+        canceled_sl_orders = [o for o in stoploss_orders
+                              if o['status'] in ('canceled', 'cancelled')]
+        if (
+            trade.is_open and
+            len(stoploss_orders) > 0 and
+            len(stoploss_orders) == len(canceled_sl_orders)
+        ):
+            if self.create_stoploss_order(trade=trade, stop_price=trade.stoploss_or_liquidation):
+                return False
+            else:
+                logger.warning('All Stoploss orders are cancelled, but unable to recreate one.')
+
+        active_sl_orders = [o for o in stoploss_orders if o not in canceled_sl_orders]
+        if len(active_sl_orders) > 0:
+            last_active_sl_order = active_sl_orders[-1]
+            # Finally we check if stoploss on exchange should be moved up because of trailing.
+            # Triggered Orders are now real orders - so don't replace stoploss anymore
+            if (trade.is_open and
+                    last_active_sl_order.get('status_stop') != 'triggered' and
+                    (self.config.get('trailing_stop', False) or
+                     self.config.get('use_custom_stoploss', False))):
+                # if trailing stoploss is enabled we check if stoploss value has changed
+                # in which case we cancel stoploss order and put another one with new
+                # value immediately
+                self.handle_trailing_stoploss_on_exchange(trade, last_active_sl_order)
+
+        return
+
     def manage_open_orders(self) -> None:
         """
         Management of open orders on exchange. Unfilled orders might be cancelled if timeout
@@ -1753,6 +1761,7 @@ class FreqtradeBot(LoggingMixin):
             return False
 
         order_obj = Order.parse_from_ccxt_object(order, trade.pair, trade.exit_side, amount, limit)
+        order_obj.ft_order_tag = exit_reason
         trade.orders.append(order_obj)
 
         trade.exit_order_status = ''
@@ -1767,7 +1776,7 @@ class FreqtradeBot(LoggingMixin):
 
         return True
 
-    def _notify_exit(self, trade: Trade, order_type: str, fill: bool = False,
+    def _notify_exit(self, trade: Trade, order_type: Optional[str], fill: bool = False,
                      sub_trade: bool = False, order: Optional[Order] = None) -> None:
         """
         Sends rpc notification when a sell occurred.
@@ -1799,7 +1808,7 @@ class FreqtradeBot(LoggingMixin):
             'gain': gain,
             'limit': order_rate,  # Deprecated
             'order_rate': order_rate,
-            'order_type': order_type,
+            'order_type': order_type or 'unknown',
             'amount': amount,
             'open_rate': trade.open_rate,
             'close_rate': order_rate,

freqtrade/optimize/backtesting.py

@@ -23,7 +23,7 @@ from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType
                              TradingMode)
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.exchange import (amount_to_contract_precision, price_to_precision,
-                                timeframe_to_minutes, timeframe_to_seconds)
+                                timeframe_to_seconds)
 from freqtrade.exchange.exchange import Exchange
 from freqtrade.mixins import LoggingMixin
 from freqtrade.optimize.backtest_caching import get_strategy_run_id
@@ -117,8 +117,9 @@ class Backtesting:
             raise OperationalException("Timeframe needs to be set in either "
                                        "configuration or as cli argument `--timeframe 5m`")
         self.timeframe = str(self.config.get('timeframe'))
-        self.timeframe_min = timeframe_to_minutes(self.timeframe)
-        self.timeframe_td = timedelta(minutes=self.timeframe_min)
+        self.timeframe_secs = timeframe_to_seconds(self.timeframe)
+        self.timeframe_min = self.timeframe_secs // 60
+        self.timeframe_td = timedelta(seconds=self.timeframe_secs)
         self.disable_database_use()
         self.init_backtest_detail()
         self.pairlists = PairListManager(self.exchange, self.config, self.dataprovider)
@@ -185,13 +186,14 @@ class Backtesting:
         # Load detail timeframe if specified
         self.timeframe_detail = str(self.config.get('timeframe_detail', ''))
         if self.timeframe_detail:
-            self.timeframe_detail_min = timeframe_to_minutes(self.timeframe_detail)
-            if self.timeframe_min <= self.timeframe_detail_min:
+            timeframe_detail_secs = timeframe_to_seconds(self.timeframe_detail)
+            self.timeframe_detail_td = timedelta(seconds=timeframe_detail_secs)
+            if self.timeframe_secs <= timeframe_detail_secs:
                 raise OperationalException(
                     "Detail timeframe must be smaller than strategy timeframe.")
 
         else:
-            self.timeframe_detail_min = 0
+            self.timeframe_detail_td = timedelta(seconds=0)
         self.detail_data: Dict[str, DataFrame] = {}
         self.futures_data: Dict[str, DataFrame] = {}
 
@@ -199,7 +201,7 @@ class Backtesting:
 
         self.prepare_backtest(False)
 
-        self.wallets = Wallets(self.config, self.exchange, log=False)
+        self.wallets = Wallets(self.config, self.exchange, is_backtest=True)
 
         self.progress = BTProgress()
         self.abort = False
@@ -537,14 +539,14 @@ class Backtesting:
         min_stake = self.exchange.get_min_pair_stake_amount(trade.pair, current_rate, -0.1)
         max_stake = self.exchange.get_max_pair_stake_amount(trade.pair, current_rate)
         stake_available = self.wallets.get_available_stake_amount()
-        stake_amount = strategy_safe_wrapper(self.strategy.adjust_trade_position,
-                                             default_retval=None, supress_error=True)(
+        stake_amount, order_tag = self.strategy._adjust_trade_position_internal(
             trade=trade,  # type: ignore[arg-type]
             current_time=current_time, current_rate=current_rate,
             current_profit=current_profit, min_stake=min_stake,
             max_stake=min(max_stake, stake_available),
             current_entry_rate=current_rate, current_exit_rate=current_rate,
-            current_entry_profit=current_profit, current_exit_profit=current_profit)
+            current_entry_profit=current_profit, current_exit_profit=current_profit
+        )
 
         # Check if we should increase our position
         if stake_amount is not None and stake_amount > 0.0:
@@ -554,7 +556,8 @@ class Backtesting:
                 check_adjust_entry = (entry_count <= self.strategy.max_entry_position_adjustment)
             if check_adjust_entry:
                 pos_trade = self._enter_trade(
-                    trade.pair, row, 'short' if trade.is_short else 'long', stake_amount, trade)
+                    trade.pair, row, 'short' if trade.is_short else 'long', stake_amount, trade,
+                    entry_tag1=order_tag)
                 if pos_trade is not None:
                     self.wallets.update()
                     return pos_trade
@@ -569,7 +572,7 @@ class Backtesting:
             if min_stake and remaining != 0 and remaining < min_stake:
                 # Remaining stake is too low to be sold.
                 return trade
-            exit_ = ExitCheckTuple(ExitType.PARTIAL_EXIT)
+            exit_ = ExitCheckTuple(ExitType.PARTIAL_EXIT, order_tag)
             pos_trade = self._get_exit_for_signal(trade, row, exit_, current_time, amount)
             if pos_trade is not None:
                 order = pos_trade.orders[-1]
@@ -681,11 +684,11 @@ class Backtesting:
 
             trade.exit_reason = exit_reason
 
-            return self._exit_trade(trade, row, close_rate, amount_)
+            return self._exit_trade(trade, row, close_rate, amount_, exit_reason)
         return None
 
-    def _exit_trade(self, trade: LocalTrade, sell_row: Tuple,
-                    close_rate: float, amount: Optional[float] = None) -> Optional[LocalTrade]:
+    def _exit_trade(self, trade: LocalTrade, sell_row: Tuple, close_rate: float,
+                    amount: float, exit_reason: Optional[str]) -> Optional[LocalTrade]:
         self.order_id_counter += 1
         exit_candle_time = sell_row[DATE_IDX].to_pydatetime()
         order_type = self.strategy.order_types['exit']
@@ -712,6 +715,7 @@ class Backtesting:
             filled=0,
             remaining=amount,
             cost=amount * close_rate,
+            ft_order_tag=exit_reason,
         )
         order._trade_bt = trade
         trade.orders.append(order)
@@ -835,7 +839,9 @@ class Backtesting:
                      stake_amount: Optional[float] = None,
                      trade: Optional[LocalTrade] = None,
                      requested_rate: Optional[float] = None,
-                     requested_stake: Optional[float] = None) -> Optional[LocalTrade]:
+                     requested_stake: Optional[float] = None,
+                     entry_tag1: Optional[str] = None
+                     ) -> Optional[LocalTrade]:
         """
         :param trade: Trade to adjust - initial entry if None
         :param requested_rate: Adjusted entry rate
@@ -843,7 +849,7 @@ class Backtesting:
         """
 
         current_time = row[DATE_IDX].to_pydatetime()
-        entry_tag = row[ENTER_TAG_IDX] if len(row) >= ENTER_TAG_IDX + 1 else None
+        entry_tag = entry_tag1 or (row[ENTER_TAG_IDX] if len(row) >= ENTER_TAG_IDX + 1 else None)
         # let's call the custom entry price, using the open price as default price
         order_type = self.strategy.order_types['entry']
         pos_adjust = trade is not None and requested_rate is None
@@ -944,6 +950,7 @@ class Backtesting:
                 filled=0,
                 remaining=amount,
                 cost=amount * propose_rate + trade.fee_open,
+                ft_order_tag=entry_tag,
             )
             order._trade_bt = trade
             trade.orders.append(order)
@@ -963,7 +970,8 @@ class Backtesting:
                     # Ignore trade if entry-order did not fill yet
                     continue
                 exit_row = data[pair][-1]
-                self._exit_trade(trade, exit_row, exit_row[OPEN_IDX], trade.amount)
+                self._exit_trade(trade, exit_row, exit_row[OPEN_IDX], trade.amount,
+                                 ExitType.FORCE_EXIT.value)
                 trade.orders[-1].close_bt_order(exit_row[DATE_IDX].to_pydatetime(), trade)
 
                 trade.close_date = exit_row[DATE_IDX].to_pydatetime()
@@ -1262,7 +1270,7 @@ class Backtesting:
                         open_trade_count_start = self.backtest_loop(
                             det_row, pair, current_time_det, end_date,
                             open_trade_count_start, trade_dir, is_first)
-                        current_time_det += timedelta(minutes=self.timeframe_detail_min)
+                        current_time_det += self.timeframe_detail_td
                         is_first = False
                 else:
                     self.dataprovider._set_dataframe_max_date(current_time)

freqtrade/persistence/migrations.py

@@ -1,7 +1,7 @@
 import logging
 from typing import List, Optional
 
-from sqlalchemy import inspect, select, text, tuple_, update
+from sqlalchemy import inspect, select, text, update
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.persistence.trade_model import Order, Trade
@@ -91,8 +91,6 @@ def migrate_trades_and_orders_table(
     is_stop_loss_trailing = get_column_def(
         cols, 'is_stop_loss_trailing',
         f'coalesce({stop_loss_pct}, 0.0) <> coalesce({initial_stop_loss_pct}, 0.0)')
-    stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null')
-    stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null')
     max_rate = get_column_def(cols, 'max_rate', '0.0')
     min_rate = get_column_def(cols, 'min_rate', 'null')
     exit_reason = get_column_def(cols, 'sell_reason', get_column_def(cols, 'exit_reason', 'null'))
@@ -160,7 +158,7 @@ def migrate_trades_and_orders_table(
             open_rate_requested, close_rate, close_rate_requested, close_profit,
             stake_amount, amount, amount_requested, open_date, close_date,
             stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
-            is_stop_loss_trailing, stoploss_order_id, stoploss_last_update,
+            is_stop_loss_trailing,
             max_rate, min_rate, exit_reason, exit_order_status, strategy, enter_tag,
             timeframe, open_trade_value, close_profit_abs,
             trading_mode, leverage, liquidation_price, is_short,
@@ -180,7 +178,6 @@ def migrate_trades_and_orders_table(
             {initial_stop_loss} initial_stop_loss,
             {initial_stop_loss_pct} initial_stop_loss_pct,
             {is_stop_loss_trailing} is_stop_loss_trailing,
-            {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
             {max_rate} max_rate, {min_rate} min_rate,
             case when {exit_reason} = 'sell_signal' then 'exit_signal'
                  when {exit_reason} = 'custom_sell' then 'custom_exit'
@@ -223,6 +220,7 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List):
     ft_amount = get_column_def(cols_order, 'ft_amount', 'coalesce(amount, 0.0)')
     ft_price = get_column_def(cols_order, 'ft_price', 'coalesce(price, 0.0)')
     ft_cancel_reason = get_column_def(cols_order, 'ft_cancel_reason', 'null')
+    ft_order_tag = get_column_def(cols_order, 'ft_order_tag', 'null')
 
     # sqlite does not support literals for booleans
     with engine.begin() as connection:
@@ -230,13 +228,14 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List):
             insert into orders (id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
             status, symbol, order_type, side, price, amount, filled, average, remaining, cost,
             stop_price, order_date, order_filled_date, order_update_date, ft_fee_base, funding_fee,
-            ft_amount, ft_price, ft_cancel_reason
+            ft_amount, ft_price, ft_cancel_reason, ft_order_tag
             )
             select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
             status, symbol, order_type, side, price, amount, filled, {average} average, remaining,
             cost, {stop_price} stop_price, order_date, order_filled_date,
             order_update_date, {ft_fee_base} ft_fee_base, {funding_fee} funding_fee,
-            {ft_amount} ft_amount, {ft_price} ft_price, {ft_cancel_reason} ft_cancel_reason
+            {ft_amount} ft_amount, {ft_price} ft_price, {ft_cancel_reason} ft_cancel_reason,
+            {ft_order_tag} ft_order_tag
             from {table_back_name}
             """))
 
@@ -277,6 +276,8 @@ def fix_old_dry_orders(engine):
     with engine.begin() as connection:
 
         # Update current dry-run Orders where
+        # - stoploss order is Open (will be replaced eventually)
+        # 2nd query:
         # - current Order is open
         # - current Trade is closed
         # - current Order trade_id not equal to current Trade.id
@@ -284,11 +285,6 @@ def fix_old_dry_orders(engine):
 
         stmt = update(Order).where(
             Order.ft_is_open.is_(True),
-            tuple_(Order.ft_trade_id, Order.order_id).not_in(
-                select(
-                    Trade.id, Trade.stoploss_order_id
-                ).where(Trade.stoploss_order_id.is_not(None))
-                  ),
             Order.ft_order_side == 'stoploss',
             Order.order_id.like('dry%'),
 
@@ -331,8 +327,8 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
     # if ('orders' not in previous_tables
     # or not has_column(cols_orders, 'funding_fee')):
     migrating = False
-    # if not has_column(cols_orders, 'ft_cancel_reason'):
-    if not has_column(cols_trades, 'funding_fee_running'):
+    # if not has_column(cols_trades, 'funding_fee_running'):
+    if not has_column(cols_orders, 'ft_order_tag'):
         migrating = True
         logger.info(f"Running database migration for trades - "
                     f"backup: {table_back_name}, {order_table_bak_name}")

freqtrade/persistence/trade_model.py

@@ -23,7 +23,7 @@ from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, amount_to_contract_precisi
 from freqtrade.leverage import interest
 from freqtrade.misc import safe_value_fallback
 from freqtrade.persistence.base import ModelBase, SessionType
-from freqtrade.util import FtPrecise, dt_from_ts, dt_now, dt_ts
+from freqtrade.util import FtPrecise, dt_from_ts, dt_now, dt_ts, dt_ts_none
 
 
 logger = logging.getLogger(__name__)
@@ -73,8 +73,7 @@ class Order(ModelBase):
     order_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
     status: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
     symbol: Mapped[Optional[str]] = mapped_column(String(25), nullable=True)
-    # TODO: type: order_type type is Optional[str]
-    order_type: Mapped[str] = mapped_column(String(50), nullable=True)
+    order_type: Mapped[Optional[str]] = mapped_column(String(50), nullable=True)
     side: Mapped[str] = mapped_column(String(25), nullable=True)
     price: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
     average: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
@@ -89,6 +88,8 @@ class Order(ModelBase):
     funding_fee: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
 
     ft_fee_base: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
+    ft_order_tag: Mapped[Optional[str]] = mapped_column(String(CUSTOM_TAG_MAX_LENGTH),
+                                                        nullable=True)
 
     @property
     def order_date_utc(self) -> datetime:
@@ -175,6 +176,8 @@ class Order(ModelBase):
         order_date = safe_value_fallback(order, 'timestamp')
         if order_date:
             self.order_date = datetime.fromtimestamp(order_date / 1000, tz=timezone.utc)
+        elif not self.order_date:
+            self.order_date = dt_now()
 
         self.ft_is_open = True
         if self.status in NON_OPEN_EXCHANGE_STATES:
@@ -212,13 +215,17 @@ class Order(ModelBase):
         return order
 
     def to_json(self, entry_side: str, minified: bool = False) -> Dict[str, Any]:
+        """
+        :param minified: If True, only return a subset of the data is returned.
+                         Only used for backtesting.
+        """
         resp = {
             'amount': self.safe_amount,
             'safe_price': self.safe_price,
             'ft_order_side': self.ft_order_side,
-            'order_filled_timestamp': int(self.order_filled_date.replace(
-                tzinfo=timezone.utc).timestamp() * 1000) if self.order_filled_date else None,
+            'order_filled_timestamp': dt_ts_none(self.order_filled_utc),
             'ft_is_entry': self.ft_order_side == entry_side,
+            'ft_order_tag': self.ft_order_tag,
         }
         if not minified:
             resp.update({
@@ -369,10 +376,6 @@ class LocalTrade:
     # percentage value of the initial stop loss
     initial_stop_loss_pct: Optional[float] = None
     is_stop_loss_trailing: bool = False
-    # stoploss order id which is on exchange
-    stoploss_order_id: Optional[str] = None
-    # last update time of the stoploss order on exchange
-    stoploss_last_update: Optional[datetime] = None
     # absolute value of the highest reached price
     max_rate: Optional[float] = None
     # Lowest price reached
@@ -456,14 +459,25 @@ class LocalTrade:
             return self.open_date_utc
         return max([self.open_date_utc, dt_last_filled])
 
+    @property
+    def date_entry_fill_utc(self) -> Optional[datetime]:
+        """ Date of the first filled order"""
+        orders = self.select_filled_orders(self.entry_side)
+        if (
+            orders
+            and len(filled_date := [o.order_filled_utc for o in orders if o.order_filled_utc])
+        ):
+            return min(filled_date)
+        return None
+
     @property
     def open_date_utc(self):
         return self.open_date.replace(tzinfo=timezone.utc)
 
     @property
     def stoploss_last_update_utc(self):
-        if self.stoploss_last_update:
-            return self.stoploss_last_update.replace(tzinfo=timezone.utc)
+        if self.has_open_sl_orders:
+            return max(o.order_date_utc for o in self.open_sl_orders)
         return None
 
     @property
@@ -519,7 +533,7 @@ class LocalTrade:
         return [o for o in self.orders if o.ft_is_open and o.ft_order_side != 'stoploss']
 
     @property
-    def has_open_orders(self) -> int:
+    def has_open_orders(self) -> bool:
         """
         True if there are open orders for this trade excluding stoploss orders
         """
@@ -529,6 +543,37 @@ class LocalTrade:
         ]
         return len(open_orders_wo_sl) > 0
 
+    @property
+    def open_sl_orders(self) -> List[Order]:
+        """
+        All open stoploss orders for this trade
+        """
+        return [
+            o for o in self.orders
+            if o.ft_order_side in ['stoploss'] and o.ft_is_open
+        ]
+
+    @property
+    def has_open_sl_orders(self) -> bool:
+        """
+        True if there are open stoploss orders for this trade
+        """
+        open_sl_orders = [
+            o for o in self.orders
+            if o.ft_order_side in ['stoploss'] and o.ft_is_open
+        ]
+        return len(open_sl_orders) > 0
+
+    @property
+    def sl_orders(self) -> List[Order]:
+        """
+        All stoploss orders for this trade
+        """
+        return [
+            o for o in self.orders
+            if o.ft_order_side in ['stoploss']
+        ]
+
     @property
     def open_orders_ids(self) -> List[str]:
         open_orders_ids_wo_sl = [
@@ -558,6 +603,11 @@ class LocalTrade:
         )
 
     def to_json(self, minified: bool = False) -> Dict[str, Any]:
+        """
+        :param minified: If True, only return a subset of the data is returned.
+                         Only used for backtesting.
+        :return: Dictionary with trade data
+        """
         filled_or_open_orders = self.select_filled_or_open_orders()
         orders_json = [order.to_json(self.entry_side, minified) for order in filled_or_open_orders]
 
@@ -584,15 +634,17 @@ class LocalTrade:
             'fee_close_currency': self.fee_close_currency,
 
             'open_date': self.open_date.strftime(DATETIME_PRINT_FORMAT),
-            'open_timestamp': int(self.open_date.replace(tzinfo=timezone.utc).timestamp() * 1000),
+            'open_timestamp': dt_ts_none(self.open_date_utc),
+            'open_fill_date': (self.date_entry_fill_utc.strftime(DATETIME_PRINT_FORMAT)
+                               if self.date_entry_fill_utc else None),
+            'open_fill_timestamp': dt_ts_none(self.date_entry_fill_utc),
             'open_rate': self.open_rate,
             'open_rate_requested': self.open_rate_requested,
             'open_trade_value': round(self.open_trade_value, 8),
 
             'close_date': (self.close_date.strftime(DATETIME_PRINT_FORMAT)
                            if self.close_date else None),
-            'close_timestamp': int(self.close_date.replace(
-                tzinfo=timezone.utc).timestamp() * 1000) if self.close_date else None,
+            'close_timestamp': dt_ts_none(self.close_date_utc),
             'realized_profit': self.realized_profit or 0.0,
             # Close-profit corresponds to relative realized_profit ratio
             'realized_profit_ratio': self.close_profit or None,
@@ -616,11 +668,9 @@ class LocalTrade:
             'stop_loss_abs': self.stop_loss,
             'stop_loss_ratio': self.stop_loss_pct if self.stop_loss_pct else None,
             'stop_loss_pct': (self.stop_loss_pct * 100) if self.stop_loss_pct else None,
-            'stoploss_order_id': self.stoploss_order_id,
-            'stoploss_last_update': (self.stoploss_last_update.strftime(DATETIME_PRINT_FORMAT)
-                                     if self.stoploss_last_update else None),
-            'stoploss_last_update_timestamp': int(self.stoploss_last_update.replace(
-                tzinfo=timezone.utc).timestamp() * 1000) if self.stoploss_last_update else None,
+            'stoploss_last_update': (self.stoploss_last_update_utc.strftime(DATETIME_PRINT_FORMAT)
+                                     if self.stoploss_last_update_utc else None),
+            'stoploss_last_update_timestamp': dt_ts_none(self.stoploss_last_update_utc),
             'initial_stop_loss_abs': self.initial_stop_loss,
             'initial_stop_loss_ratio': (self.initial_stop_loss_pct
                                         if self.initial_stop_loss_pct else None),
@@ -764,6 +814,7 @@ class LocalTrade:
             order.funding_fee = self.funding_fee_running
             # Reset running funding fees
             self.funding_fee_running = 0.0
+        order_type = order.order_type.upper() if order.order_type else None
 
         if order.ft_order_side == self.entry_side:
             # Update open rate and actual amount
@@ -771,21 +822,20 @@ class LocalTrade:
             self.amount = order.safe_amount_after_fee
             if self.is_open:
                 payment = "SELL" if self.is_short else "BUY"
-                logger.info(f'{order.order_type.upper()}_{payment} has been fulfilled for {self}.')
+                logger.info(f'{order_type}_{payment} has been fulfilled for {self}.')
 
             self.recalc_trade_from_orders()
         elif order.ft_order_side == self.exit_side:
             if self.is_open:
                 payment = "BUY" if self.is_short else "SELL"
                 # * On margin shorts, you buy a little bit more than the amount (amount + interest)
-                logger.info(f'{order.order_type.upper()}_{payment} has been fulfilled for {self}.')
+                logger.info(f'{order_type}_{payment} has been fulfilled for {self}.')
 
         elif order.ft_order_side == 'stoploss' and order.status not in ('open', ):
-            self.stoploss_order_id = None
             self.close_rate_requested = self.stop_loss
             self.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value
             if self.is_open and order.safe_filled > 0:
-                logger.info(f'{order.order_type.upper()} is hit for {self}.')
+                logger.info(f'{order_type} is hit for {self}.')
         else:
             raise ValueError(f'Unknown order type: {order.order_type}')
 
@@ -1358,11 +1408,6 @@ class LocalTrade:
             exit_order_status=data["exit_order_status"],
             stop_loss=data["stop_loss_abs"],
             stop_loss_pct=data["stop_loss_ratio"],
-            stoploss_order_id=data["stoploss_order_id"],
-            stoploss_last_update=(
-                datetime.fromtimestamp(data["stoploss_last_update_timestamp"] // 1000,
-                                       tz=timezone.utc)
-                if data["stoploss_last_update_timestamp"] else None),
             initial_stop_loss=data["initial_stop_loss_abs"],
             initial_stop_loss_pct=data["initial_stop_loss_ratio"],
             min_rate=data["min_rate"],
@@ -1400,6 +1445,7 @@ class LocalTrade:
                 ft_price=order["price"],
                 remaining=order["remaining"],
                 funding_fee=order.get("funding_fee", None),
+                ft_order_tag=order.get("ft_order_tag", None),
             )
             trade.orders.append(order_obj)
 
@@ -1468,11 +1514,6 @@ class Trade(ModelBase, LocalTrade):
         Float(), nullable=True)  # type: ignore
     is_stop_loss_trailing: Mapped[bool] = mapped_column(
         nullable=False, default=False)  # type: ignore
-    # stoploss order id which is on exchange
-    stoploss_order_id: Mapped[Optional[str]] = mapped_column(
-        String(255), nullable=True, index=True)  # type: ignore
-    # last update time of the stoploss order on exchange
-    stoploss_last_update: Mapped[Optional[datetime]] = mapped_column(nullable=True)  # type: ignore
     # absolute value of the highest reached price
     max_rate: Mapped[Optional[float]] = mapped_column(
         Float(), nullable=True, default=0.0)  # type: ignore

freqtrade/plugins/pairlist/MarketCapPairList.py

@@ -0,0 +1,157 @@
+"""
+Market Cap PairList provider
+
+Provides dynamic pair list based on Market Cap
+"""
+import logging
+from typing import Any, Dict, List
+
+from cachetools import TTLCache
+from pycoingecko import CoinGeckoAPI
+
+from freqtrade.constants import Config
+from freqtrade.exceptions import OperationalException
+from freqtrade.exchange.types import Tickers
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+
+
+logger = logging.getLogger(__name__)
+
+
+class MarketCapPairList(IPairList):
+
+    is_pairlist_generator = True
+
+    def __init__(self, exchange, pairlistmanager,
+                 config: Config, pairlistconfig: Dict[str, Any],
+                 pairlist_pos: int) -> None:
+        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
+
+        if 'number_assets' not in self._pairlistconfig:
+            raise OperationalException(
+                '`number_assets` not specified. Please check your configuration '
+                'for "pairlist.config.number_assets"')
+
+        self._stake_currency = config['stake_currency']
+        self._number_assets = self._pairlistconfig['number_assets']
+        self._max_rank = self._pairlistconfig.get('max_rank', 30)
+        self._refresh_period = self._pairlistconfig.get('refresh_period', 86400)
+        self._marketcap_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
+        self._def_candletype = self._config['candle_type_def']
+        self._coingekko: CoinGeckoAPI = CoinGeckoAPI()
+
+        if self._max_rank > 250:
+            raise OperationalException(
+                "This filter only support marketcap rank up to 250."
+            )
+
+    @property
+    def needstickers(self) -> bool:
+        """
+        Boolean property defining if tickers are necessary.
+        If no Pairlist requires tickers, an empty Dict is passed
+        as tickers argument to filter_pairlist
+        """
+        return False
+
+    def short_desc(self) -> str:
+        """
+        Short whitelist method description - used for startup-messages
+        """
+        num = self._number_assets
+        rank = self._max_rank
+        msg = f"{self.name} - {num} pairs placed within top {rank} market cap."
+        return msg
+
+    @staticmethod
+    def description() -> str:
+        return "Provides pair list based on CoinGecko's market cap rank."
+
+    @staticmethod
+    def available_parameters() -> Dict[str, PairlistParameter]:
+        return {
+            "number_assets": {
+                "type": "number",
+                "default": 30,
+                "description": "Number of assets",
+                "help": "Number of assets to use from the pairlist",
+            },
+            "max_rank": {
+                "type": "number",
+                "default": 30,
+                "description": "Max rank of assets",
+                "help": "Maximum rank of assets to use from the pairlist",
+            },
+            "refresh_period": {
+                "type": "number",
+                "default": 86400,
+                "description": "Refresh period",
+                "help": "Refresh period in seconds",
+            }
+        }
+
+    def gen_pairlist(self, tickers: Tickers) -> List[str]:
+        """
+        Generate the pairlist
+        :param tickers: Tickers (from exchange.get_tickers). May be cached.
+        :return: List of pairs
+        """
+        # Generate dynamic whitelist
+        # Must always run if this pairlist is the first in the list.
+        pairlist = self._marketcap_cache.get('pairlist_mc')
+        if pairlist:
+            # Item found - no refresh necessary
+            return pairlist.copy()
+        else:
+            # Use fresh pairlist
+            # Check if pair quote currency equals to the stake currency.
+            _pairlist = [k for k in self._exchange.get_markets(
+                quote_currencies=[self._stake_currency],
+                tradable_only=True, active_only=True).keys()]
+            # No point in testing for blacklisted pairs...
+            _pairlist = self.verify_blacklist(_pairlist, logger.info)
+
+            pairlist = self.filter_pairlist(_pairlist, tickers)
+            self._marketcap_cache['pairlist_mc'] = pairlist.copy()
+
+        return pairlist
+
+    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
+        """
+        Filters and sorts pairlist and returns the whitelist again.
+        Called on each bot iteration - please use internal caching if necessary
+        :param pairlist: pairlist to filter or sort
+        :param tickers: Tickers (from exchange.get_tickers). May be cached.
+        :return: new whitelist
+        """
+        marketcap_list = self._marketcap_cache.get('marketcap')
+
+        if marketcap_list is None:
+            data = self._coingekko.get_coins_markets(vs_currency='usd', order='market_cap_desc',
+                                                     per_page='250', page='1', sparkline='false',
+                                                     locale='en')
+            if data:
+                marketcap_list = [row['symbol'] for row in data]
+                self._marketcap_cache['marketcap'] = marketcap_list
+
+        if marketcap_list:
+            filtered_pairlist = []
+
+            market = self._config['trading_mode']
+            pair_format = f"{self._stake_currency.upper()}"
+            if (market == 'futures'):
+                pair_format += f":{self._stake_currency.upper()}"
+
+            top_marketcap = marketcap_list[:self._max_rank:]
+
+            for mc_pair in top_marketcap:
+                test_pair = f"{mc_pair.upper()}/{pair_format}"
+                if test_pair in pairlist:
+                    filtered_pairlist.append(test_pair)
+                    if len(filtered_pairlist) == self._number_assets:
+                        break
+
+            if len(filtered_pairlist) > 0:
+                return filtered_pairlist
+
+        return pairlist

freqtrade/plugins/pairlist/VolatilityFilter.py

@@ -103,11 +103,7 @@ class VolatilityFilter(IPairList):
             (p, '1d', self._def_candletype) for p in pairlist if p not in self._pair_cache]
 
         since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days))
-        # Get all candles
-        candles = {}
-        if needed_pairs:
-            candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms,
-                                                          cache=False)
+        candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms=since_ms)
 
         if self._enabled:
             for p in deepcopy(pairlist):
@@ -125,8 +121,7 @@ class VolatilityFilter(IPairList):
         :return: True if the pair can stay, false if it should be removed
         """
         # Check symbol in cache
-        cached_res = self._pair_cache.get(pair, None)
-        if cached_res is not None:
+        if (cached_res := self._pair_cache.get(pair, None)) is not None:
             return cached_res
 
         result = False

freqtrade/plugins/pairlist/VolumePairList.py

@@ -229,12 +229,8 @@ class VolumePairList(IPairList):
                 if p not in self._pair_cache
             ]
 
-            # Get all candles
-            candles = {}
-            if needed_pairs:
-                candles = self._exchange.refresh_latest_ohlcv(
-                    needed_pairs, since_ms=since_ms, cache=False
-                )
+            candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms)
+
             for i, p in enumerate(filtered_tickers):
                 contract_size = self._exchange.markets[p['symbol']].get('contractSize', 1.0) or 1.0
                 pair_candles = candles[

freqtrade/plugins/pairlist/rangestabilityfilter.py

@@ -100,12 +100,8 @@ class RangeStabilityFilter(IPairList):
         needed_pairs: ListPairsWithTimeframes = [
             (p, '1d', self._def_candletype) for p in pairlist if p not in self._pair_cache]
 
-        since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days - 1))
-        # Get all candles
-        candles = {}
-        if needed_pairs:
-            candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms,
-                                                          cache=False)
+        since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days + 1))
+        candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms=since_ms)
 
         if self._enabled:
             for p in deepcopy(pairlist):
@@ -123,8 +119,7 @@ class RangeStabilityFilter(IPairList):
         :return: True if the pair can stay, false if it should be removed
         """
         # Check symbol in cache
-        cached_res = self._pair_cache.get(pair, None)
-        if cached_res is not None:
+        if (cached_res := self._pair_cache.get(pair, None)) is not None:
             return cached_res
 
         result = True

freqtrade/rpc/api_server/api_schemas.py

@@ -261,6 +261,7 @@ class OrderSchema(BaseModel):
     order_timestamp: Optional[int] = None
     order_filled_timestamp: Optional[int] = None
     ft_fee_base: Optional[float] = None
+    ft_order_tag: Optional[str] = None
 
 
 class TradeSchema(BaseModel):
@@ -287,6 +288,8 @@ class TradeSchema(BaseModel):
 
     open_date: str
     open_timestamp: int
+    open_fill_date: Optional[str]
+    open_fill_timestamp: Optional[int]
     open_rate: float
     open_rate_requested: Optional[float] = None
     open_trade_value: float
@@ -314,7 +317,6 @@ class TradeSchema(BaseModel):
     stop_loss_abs: Optional[float] = None
     stop_loss_ratio: Optional[float] = None
     stop_loss_pct: Optional[float] = None
-    stoploss_order_id: Optional[str] = None
     stoploss_last_update: Optional[str] = None
     stoploss_last_update_timestamp: Optional[int] = None
     initial_stop_loss_abs: Optional[float] = None
@@ -397,7 +399,7 @@ class ForceEnterPayload(BaseModel):
 
 
 class ForceExitPayload(BaseModel):
-    tradeid: str
+    tradeid: Union[str, int]
     ordertype: Optional[OrderTypeValues] = None
     amount: Optional[float] = None
 

freqtrade/rpc/api_server/api_v1.py

@@ -215,7 +215,7 @@ def force_entry(payload: ForceEnterPayload, rpc: RPC = Depends(get_rpc)):
 @router.post('/forcesell', response_model=ResultMsg, tags=['trading'])
 def forceexit(payload: ForceExitPayload, rpc: RPC = Depends(get_rpc)):
     ordertype = payload.ordertype.value if payload.ordertype else None
-    return rpc._rpc_force_exit(payload.tradeid, ordertype, amount=payload.amount)
+    return rpc._rpc_force_exit(str(payload.tradeid), ordertype, amount=payload.amount)
 
 
 @router.get('/blacklist', response_model=BlacklistResponse, tags=['info', 'pairlist'])

freqtrade/rpc/rpc.py

@@ -979,15 +979,16 @@ class RPC:
                 except (ExchangeError):
                     pass
 
-            # cancel stoploss on exchange ...
+            # cancel stoploss on exchange orders ...
             if (self._freqtrade.strategy.order_types.get('stoploss_on_exchange')
-                    and trade.stoploss_order_id):
-                try:
-                    self._freqtrade.exchange.cancel_stoploss_order(trade.stoploss_order_id,
-                                                                   trade.pair)
-                    c_count += 1
-                except (ExchangeError):
-                    pass
+                    and trade.has_open_sl_orders):
+
+                for oslo in trade.open_sl_orders:
+                    try:
+                        self._freqtrade.exchange.cancel_stoploss_order(oslo.order_id, trade.pair)
+                        c_count += 1
+                    except (ExchangeError):
+                        pass
 
             trade.delete()
             self._freqtrade.wallets.update()

freqtrade/rpc/telegram.py

@@ -353,7 +353,7 @@ class Telegram(RPCHandler):
         message += f"*Amount:* `{round_value(msg['amount'], 8)}`\n"
         message += f"*Direction:* `{msg['direction']}"
         if msg.get('leverage') and msg.get('leverage', 1.0) != 1.0:
-            message += f" ({msg['leverage']:.1g}x)"
+            message += f" ({msg['leverage']:.3g}x)"
         message += "`\n"
         message += f"*Open Rate:* `{fmt_coin(msg['open_rate'], msg['quote_currency'])}`\n"
         if msg['type'] == RPCMessageType.ENTRY and msg['current_rate']:
@@ -371,7 +371,7 @@ class Telegram(RPCHandler):
             microsecond=0) - msg['open_date'].replace(microsecond=0)
         duration_min = duration.total_seconds() / 60
 
-        leverage_text = (f" ({msg['leverage']:.1g}x)"
+        leverage_text = (f" ({msg['leverage']:.3g}x)"
                          if msg.get('leverage') and msg.get('leverage', 1.0) != 1.0
                          else "")
 
@@ -1364,7 +1364,7 @@ class Telegram(RPCHandler):
     @authorized_only
     async def _enter_tag_performance(self, update: Update, context: CallbackContext) -> None:
         """
-        Handler for /buys PAIR .
+        Handler for /entries PAIR .
         Shows a performance statistic from finished trades
         :param bot: telegram bot
         :param update: message update
@@ -1375,28 +1375,28 @@ class Telegram(RPCHandler):
             pair = context.args[0]
 
         trades = self._rpc._rpc_enter_tag_performance(pair)
-        output = "<b>Entry Tag Performance:</b>\n"
+        output = "*Entry Tag Performance:*\n"
         for i, trade in enumerate(trades):
             stat_line = (
-                f"{i + 1}.\t <code>{trade['enter_tag']}\t"
+                f"{i + 1}.\t `{trade['enter_tag']}\t"
                 f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} "
                 f"({trade['profit_ratio']:.2%}) "
-                f"({trade['count']})</code>\n")
+                f"({trade['count']})`\n")
 
             if len(output + stat_line) >= MAX_MESSAGE_LENGTH:
-                await self._send_msg(output, parse_mode=ParseMode.HTML)
+                await self._send_msg(output, parse_mode=ParseMode.MARKDOWN)
                 output = stat_line
             else:
                 output += stat_line
 
-        await self._send_msg(output, parse_mode=ParseMode.HTML,
+        await self._send_msg(output, parse_mode=ParseMode.MARKDOWN,
                              reload_able=True, callback_path="update_enter_tag_performance",
                              query=update.callback_query)
 
     @authorized_only
     async def _exit_reason_performance(self, update: Update, context: CallbackContext) -> None:
         """
-        Handler for /sells.
+        Handler for /exits.
         Shows a performance statistic from finished trades
         :param bot: telegram bot
         :param update: message update
@@ -1407,21 +1407,21 @@ class Telegram(RPCHandler):
             pair = context.args[0]
 
         trades = self._rpc._rpc_exit_reason_performance(pair)
-        output = "<b>Exit Reason Performance:</b>\n"
+        output = "*Exit Reason Performance:*\n"
         for i, trade in enumerate(trades):
             stat_line = (
-                f"{i + 1}.\t <code>{trade['exit_reason']}\t"
+                f"{i + 1}.\t `{trade['exit_reason']}\t"
                 f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} "
                 f"({trade['profit_ratio']:.2%}) "
-                f"({trade['count']})</code>\n")
+                f"({trade['count']})`\n")
 
             if len(output + stat_line) >= MAX_MESSAGE_LENGTH:
-                await self._send_msg(output, parse_mode=ParseMode.HTML)
+                await self._send_msg(output, parse_mode=ParseMode.MARKDOWN)
                 output = stat_line
             else:
                 output += stat_line
 
-        await self._send_msg(output, parse_mode=ParseMode.HTML,
+        await self._send_msg(output, parse_mode=ParseMode.MARKDOWN,
                              reload_able=True, callback_path="update_exit_reason_performance",
                              query=update.callback_query)
 
@@ -1439,21 +1439,21 @@ class Telegram(RPCHandler):
             pair = context.args[0]
 
         trades = self._rpc._rpc_mix_tag_performance(pair)
-        output = "<b>Mix Tag Performance:</b>\n"
+        output = "*Mix Tag Performance:*\n"
         for i, trade in enumerate(trades):
             stat_line = (
-                f"{i + 1}.\t <code>{trade['mix_tag']}\t"
+                f"{i + 1}.\t `{trade['mix_tag']}\t"
                 f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} "
                 f"({trade['profit_ratio']:.2%}) "
-                f"({trade['count']})</code>\n")
+                f"({trade['count']})`\n")
 
             if len(output + stat_line) >= MAX_MESSAGE_LENGTH:
-                await self._send_msg(output, parse_mode=ParseMode.HTML)
+                await self._send_msg(output, parse_mode=ParseMode.MARKDOWN)
                 output = stat_line
             else:
                 output += stat_line
 
-        await self._send_msg(output, parse_mode=ParseMode.HTML,
+        await self._send_msg(output, parse_mode=ParseMode.MARKDOWN,
                              reload_able=True, callback_path="update_mix_tag_performance",
                              query=update.callback_query)
 
@@ -1676,8 +1676,8 @@ class Telegram(RPCHandler):
             "         *table :* `will display trades in a table`\n"
             "                `pending buy orders are marked with an asterisk (*)`\n"
             "                `pending sell orders are marked with a double asterisk (**)`\n"
-            "*/buys <pair|none>:* `Shows the enter_tag performance`\n"
-            "*/sells <pair|none>:* `Shows the exit reason performance`\n"
+            "*/entries <pair|none>:* `Shows the enter_tag performance`\n"
+            "*/exits <pair|none>:* `Shows the exit reason performance`\n"
             "*/mix_tags <pair|none>:* `Shows combined entry tag + exit reason performance`\n"
             "*/trades [limit]:* `Lists last closed trades (limited to 10 by default)`\n"
             "*/profit [<n>]:* `Lists cumulative profit from all finished trades, "
@@ -1777,13 +1777,9 @@ class Telegram(RPCHandler):
         msg += f"\nUpdated: {datetime.now().ctime()}"
         if not query.message:
             return
-        chat_id = query.message.chat_id
-        message_id = query.message.message_id
 
         try:
-            await self._app.bot.edit_message_text(
-                chat_id=chat_id,
-                message_id=message_id,
+            await query.edit_message_text(
                 text=msg,
                 parse_mode=parse_mode,
                 reply_markup=reply_markup

freqtrade/strategy/interface.py

@@ -511,7 +511,8 @@ class IStrategy(ABC, HyperStrategyMixin):
                               min_stake: Optional[float], max_stake: float,
                               current_entry_rate: float, current_exit_rate: float,
                               current_entry_profit: float, current_exit_profit: float,
-                              **kwargs) -> Optional[float]:
+                              **kwargs
+                              ) -> Union[Optional[float], Tuple[Optional[float], Optional[str]]]:
         """
         Custom trade adjustment logic, returning the stake amount that a trade should be
         increased or decreased.
@@ -537,6 +538,7 @@ class IStrategy(ABC, HyperStrategyMixin):
         :return float: Stake amount to adjust your trade,
                        Positive values to increase position, Negative values to decrease position.
                        Return None for no action.
+                       Optionally, return a tuple with a 2nd element with an order reason
         """
         return None
 
@@ -725,6 +727,36 @@ class IStrategy(ABC, HyperStrategyMixin):
 
     _ft_stop_uses_after_fill = False
 
+    def _adjust_trade_position_internal(
+            self, trade: Trade, current_time: datetime,
+            current_rate: float, current_profit: float,
+            min_stake: Optional[float], max_stake: float,
+            current_entry_rate: float, current_exit_rate: float,
+            current_entry_profit: float, current_exit_profit: float,
+            **kwargs
+    ) -> Tuple[Optional[float], str]:
+        """
+        wrapper around adjust_trade_position to handle the return value
+        """
+        resp = strategy_safe_wrapper(self.adjust_trade_position,
+                                     default_retval=(None, ''), supress_error=True)(
+            trade=trade, current_time=current_time,
+            current_rate=current_rate, current_profit=current_profit,
+            min_stake=min_stake, max_stake=max_stake,
+            current_entry_rate=current_entry_rate, current_exit_rate=current_exit_rate,
+            current_entry_profit=current_entry_profit, current_exit_profit=current_exit_profit,
+            **kwargs
+        )
+        order_tag = ''
+        if isinstance(resp, tuple):
+            if len(resp) >= 1:
+                stake_amount = resp[0]
+            if len(resp) > 1:
+                order_tag = resp[1] or ''
+        else:
+            stake_amount = resp
+        return stake_amount, order_tag
+
     def __informative_pairs_freqai(self) -> ListPairsWithTimeframes:
         """
         Create informative-pairs needed for FreqAI

freqtrade/util/__init__.py

@@ -1,6 +1,6 @@
 from freqtrade.util.datetime_helpers import (dt_floor_day, dt_from_ts, dt_humanize, dt_now, dt_ts,
-                                             dt_ts_def, dt_utc, format_date, format_ms_time,
-                                             shorten_date)
+                                             dt_ts_def, dt_ts_none, dt_utc, format_date,
+                                             format_ms_time, shorten_date)
 from freqtrade.util.formatters import decimals_per_coin, fmt_coin, round_value
 from freqtrade.util.ft_precise import FtPrecise
 from freqtrade.util.periodic_cache import PeriodicCache
@@ -14,6 +14,7 @@ __all__ = [
     'dt_now',
     'dt_ts',
     'dt_ts_def',
+    'dt_ts_none',
     'dt_utc',
     'format_date',
     'format_ms_time',

freqtrade/util/datetime_helpers.py

@@ -31,13 +31,23 @@ def dt_ts(dt: Optional[datetime] = None) -> int:
 def dt_ts_def(dt: Optional[datetime], default: int = 0) -> int:
     """
     Return dt in ms as a timestamp in UTC.
-    If dt is None, return the current datetime in UTC.
+    If dt is None, return the given default.
     """
     if dt:
         return int(dt.timestamp() * 1000)
     return default
 
 
+def dt_ts_none(dt: Optional[datetime]) -> Optional[int]:
+    """
+    Return dt in ms as a timestamp in UTC.
+    If dt is None, return the given default.
+    """
+    if dt:
+        return int(dt.timestamp() * 1000)
+    return None
+
+
 def dt_floor_day(dt: datetime) -> datetime:
     """Return the floor of the day for the given datetime."""
     return dt.replace(hour=0, minute=0, second=0, microsecond=0)

freqtrade/wallets.py

@@ -36,9 +36,9 @@ class PositionWallet(NamedTuple):
 
 class Wallets:
 
-    def __init__(self, config: Config, exchange: Exchange, log: bool = True) -> None:
+    def __init__(self, config: Config, exchange: Exchange, is_backtest: bool = False) -> None:
         self._config = config
-        self._log = log
+        self._is_backtest = is_backtest
         self._exchange = exchange
         self._wallets: Dict[str, Wallet] = {}
         self._positions: Dict[str, PositionWallet] = {}
@@ -78,11 +78,11 @@ class Wallets:
         _wallets = {}
         _positions = {}
         open_trades = Trade.get_trades_proxy(is_open=True)
-        # If not backtesting...
-        # TODO: potentially remove the ._log workaround to determine backtest mode.
-        if self._log:
+        if not self._is_backtest:
+            # Live / Dry-run mode
             tot_profit = Trade.get_total_closed_profit()
         else:
+            # Backtest mode
             tot_profit = LocalTrade.total_profit
         tot_profit += sum(trade.realized_profit for trade in open_trades)
         tot_in_trades = sum(trade.stake_amount for trade in open_trades)
@@ -177,7 +177,7 @@ class Wallets:
                 self._update_live()
             else:
                 self._update_dry()
-            if self._log:
+            if not self._is_backtest:
                 logger.info('Wallets synced.')
             self._last_wallet_refresh = dt_now()
 
@@ -341,19 +341,19 @@ class Wallets:
             max_allowed_stake = min(max_allowed_stake, max_stake_amount - trade_amount)
 
         if min_stake_amount is not None and min_stake_amount > max_allowed_stake:
-            if self._log:
+            if not self._is_backtest:
                 logger.warning("Minimum stake amount > available balance. "
                                f"{min_stake_amount} > {max_allowed_stake}")
             return 0
         if min_stake_amount is not None and stake_amount < min_stake_amount:
-            if self._log:
+            if not self._is_backtest:
                 logger.info(
                     f"Stake amount for pair {pair} is too small "
                     f"({stake_amount} < {min_stake_amount}), adjusting to {min_stake_amount}."
                 )
             if stake_amount * 1.3 < min_stake_amount:
                 # Top-cap stake-amount adjustments to +30%.
-                if self._log:
+                if not self._is_backtest:
                     logger.info(
                         f"Adjusted stake amount for pair {pair} is more than 30% bigger than "
                         f"the desired stake amount of ({stake_amount:.8f} * 1.3 = "
@@ -363,7 +363,7 @@ class Wallets:
             stake_amount = min_stake_amount
 
         if stake_amount > max_allowed_stake:
-            if self._log:
+            if not self._is_backtest:
                 logger.info(
                     f"Stake amount for pair {pair} is too big "
                     f"({stake_amount} > {max_allowed_stake}), adjusting to {max_allowed_stake}."

pyproject.toml

@@ -115,6 +115,8 @@ ignore = ["freqtrade/vendor/**"]
 line-length = 100
 extend-exclude = [".env", ".venv"]
 target-version = "py38"
+
+[tool.ruff.lint]
 # Exclude UP036 as it's causing the "exit if < 3.9" to fail.
 extend-select = [
   "C90",    # mccabe
@@ -132,16 +134,17 @@ extend-select = [
   # "TCH",  # flake8-type-checking
   "PTH",    # flake8-use-pathlib
 ]
+
 extend-ignore = [
   "E241",  # Multiple spaces after comma
   "E272",  # Multiple spaces before keyword
   "E221",  # Multiple spaces before operator
 ]
 
-[tool.ruff.mccabe]
+[tool.ruff.lint.mccabe]
 max-complexity = 12
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "tests/*" = ["S"]
 
 [tool.flake8]

requirements-dev.txt

@@ -7,11 +7,11 @@
 -r docs/requirements-docs.txt
 
 coveralls==3.3.1
-ruff==0.1.14
+ruff==0.2.2
 mypy==1.8.0
-pre-commit==3.6.0
-pytest==7.4.4
-pytest-asyncio==0.23.4
+pre-commit==3.6.2
+pytest==8.0.2
+pytest-asyncio==0.23.5
 pytest-cov==4.1.0
 pytest-mock==3.12.0
 pytest-random-order==1.1.1
@@ -21,11 +21,11 @@ isort==5.13.2
 time-machine==2.13.0
 
 # Convert jupyter notebooks to markdown documents
-nbconvert==7.14.2
+nbconvert==7.16.1
 
 # mypy types
 types-cachetools==5.3.0.7
 types-filelock==3.2.7
-types-requests==2.31.0.20240125
+types-requests==2.31.0.20240218
 types-tabulate==0.9.0.20240106
 types-python-dateutil==2.8.19.20240106

requirements-freqai-rl.txt

@@ -8,4 +8,4 @@ gymnasium==0.29.1; python_version < '3.12'
 stable_baselines3==2.2.1; python_version < '3.12'
 sb3_contrib>=2.0.0a9; python_version < '3.12'
 # Progress bar for stable-baselines3 and sb3-contrib
-tqdm==4.66.1
+tqdm==4.66.2

requirements-freqai.txt

@@ -3,10 +3,10 @@
 -r requirements-plot.txt
 
 # Required for freqai
-scikit-learn==1.4.0
+scikit-learn==1.4.1.post1
 joblib==1.3.2
 catboost==1.2.2; 'arm' not in platform_machine and python_version < '3.12'
-lightgbm==4.2.0
+lightgbm==4.3.0
 xgboost==2.0.3
-tensorboard==2.15.1
+tensorboard==2.16.2
 datasieve==0.1.7

requirements-hyperopt.txt

@@ -3,6 +3,6 @@
 
 # Required for hyperopt
 scipy==1.12.0
-scikit-learn==1.4.0
+scikit-learn==1.4.1.post1
 ft-scikit-optimize==0.9.2
 filelock==3.13.1

requirements-plot.txt

@@ -1,4 +1,4 @@
 # Include all requirements to run the bot.
 -r requirements.txt
 
-plotly==5.18.0
+plotly==5.19.0

requirements.txt

@@ -1,21 +1,21 @@
-numpy==1.26.3
+numpy==1.26.4
 pandas==2.1.4
 pandas-ta==0.3.14b
 
-ccxt==4.2.25
-cryptography==42.0.1
-aiohttp==3.9.2
-SQLAlchemy==2.0.25
-python-telegram-bot==20.7
+ccxt==4.2.51
+cryptography==42.0.5
+aiohttp==3.9.3
+SQLAlchemy==2.0.27
+python-telegram-bot==20.8
 # can't be hard-pinned due to telegram-bot pinning httpx with ~
 httpx>=0.24.1
 arrow==1.3.0
 cachetools==5.3.2
 requests==2.31.0
-urllib3==2.1.0
+urllib3==2.2.1
 jsonschema==4.21.1
 TA-Lib==0.4.28
-technical==1.4.2
+technical==1.4.3
 tabulate==0.9.0
 pycoingecko==3.1.0
 jinja2==3.1.3
@@ -30,15 +30,15 @@ py_find_1st==1.1.6
 # Load ticker files 30% faster
 python-rapidjson==1.14
 # Properly format api responses
-orjson==3.9.12
+orjson==3.9.15
 
 # Notify systemd
 sdnotify==0.3.2
 
 # API Server
-fastapi==0.109.0
-pydantic==2.5.3
-uvicorn==0.27.0
+fastapi==0.110.0
+pydantic==2.6.2
+uvicorn==0.27.1
 pyjwt==2.8.0
 aiofiles==23.2.1
 psutil==5.9.8
@@ -50,6 +50,7 @@ questionary==2.0.1
 prompt-toolkit==3.0.36
 # Extensions to datetime library
 python-dateutil==2.8.2
+pytz==2024.1
 
 #Futures
 schedule==1.2.1

setup.py

@@ -35,21 +35,21 @@ hdf5 = [
 
 develop = [
     'coveralls',
+    'isort',
     'mypy',
-    'ruff',
     'pre-commit',
-    'pytest',
     'pytest-asyncio',
     'pytest-cov',
     'pytest-mock',
     'pytest-random-order',
-    'isort',
+    'pytest',
+    'ruff',
     'time-machine',
     'types-cachetools',
     'types-filelock',
+    'types-python-dateutil'
     'types-requests',
     'types-tabulate',
-    'types-python-dateutil'
 ]
 
 jupyter = [
@@ -70,14 +70,17 @@ setup(
     ],
     install_requires=[
         # from requirements.txt
-        'ccxt>=4.2.15',
+        'ccxt>=4.2.47',
         'SQLAlchemy>=2.0.6',
         'python-telegram-bot>=20.1',
         'arrow>=1.0.0',
         'cachetools',
         'requests',
+        'httpx>=0.24.1',
         'urllib3',
         'jsonschema',
+        'numpy',
+        'pandas',
         'TA-Lib',
         'pandas-ta',
         'technical',
@@ -86,30 +89,28 @@ setup(
         'py_find_1st',
         'python-rapidjson',
         'orjson',
-        'sdnotify',
         'colorama',
         'jinja2',
         'questionary',
         'prompt-toolkit',
-        'numpy',
-        'pandas',
         'joblib>=1.2.0',
         'rich',
         'pyarrow; platform_machine != "armv7l"',
         'fastapi',
         'pydantic>=2.2.0',
+        'pyjwt',
+        'websockets',
         'uvicorn',
         'psutil',
-        'pyjwt',
-        'aiofiles',
         'schedule',
-        'websockets',
         'janus',
         'ast-comments',
+        'aiofiles',
         'aiohttp',
         'cryptography',
-        'httpx>=0.24.1',
+        'sdnotify',
         'python-dateutil',
+        'pytz',
         'packaging',
     ],
     extras_require={

tests/conftest.py

@@ -3342,7 +3342,7 @@ def leverage_tiers():
                 'maintAmt': 386950.0
             },
         ],
-        "ADA/BUSD:BUSD": [
+        "ADA/USDT:USDT": [
             {
                 "minNotional": 0,
                 "maxNotional": 100000,
@@ -3386,7 +3386,7 @@ def leverage_tiers():
                 "maintAmt": 1527500.0
             },
         ],
-        'BNB/BUSD:BUSD': [
+        'XRP/USDT:USDT': [
             {
                 "minNotional": 0,       # stake(before leverage) = 0
                 "maxNotional": 100000,  # max stake(before leverage) = 5000

tests/conftest_trades.py

@@ -266,7 +266,6 @@ def mock_trade_5(fee, is_short: bool):
         exchange='binance',
         strategy='SampleStrategy',
         enter_tag='TEST1',
-        stoploss_order_id=f'prod_stoploss_{direc(is_short)}_3455',
         timeframe=5,
         is_short=is_short,
         stop_loss_pct=0.10,

tests/conftest_trades_usdt.py

@@ -282,7 +282,6 @@ def mock_trade_usdt_5(fee, is_short: bool):
         open_rate=2.0,
         exchange='binance',
         strategy='SampleStrategy',
-        stoploss_order_id=f'prod_stoploss_3455_{direc(is_short)}',
         timeframe=5,
         is_short=is_short,
     )

tests/data/test_btanalysis.py

@@ -455,6 +455,13 @@ def test_calculate_max_drawdown2():
     with pytest.raises(ValueError, match='No losing trade, therefore no drawdown.'):
         calculate_max_drawdown(df, date_col='open_date', value_col='profit')
 
+    df1 = DataFrame(zip(values[:5], dates[:5]), columns=['profit', 'open_date'])
+    df1.loc[:, 'profit'] = df1['profit'] * -1
+    # No winning trade ...
+    drawdown, hdate, ldate, hval, lval, drawdown_rel = calculate_max_drawdown(
+        df1, date_col='open_date', value_col='profit')
+    assert drawdown == 0.043965
+
 
 @pytest.mark.parametrize('profits,relative,highd,lowd,result,result_rel', [
     ([0.0, -500.0, 500.0, 10000.0, -1000.0], False, 3, 4, 1000.0, 0.090909),

tests/data/test_trade_converter_kraken.py

@@ -34,6 +34,7 @@ def test_import_kraken_trades_from_csv(testdatadir, tmp_path, caplog, default_co
 
     import_kraken_trades_from_csv(default_conf_usdt, 'feather')
     assert log_has("Found csv files for BCHEUR.", caplog)
+    assert log_has("Converting pairs: BCH/EUR.", caplog)
     assert log_has_re(r"BCH/EUR: 340 trades.* 2023-01-01.* 2023-01-02.*", caplog)
 
     assert dstfile.is_file()
@@ -48,3 +49,10 @@ def test_import_kraken_trades_from_csv(testdatadir, tmp_path, caplog, default_co
                                                             tzinfo=timezone.utc)
     # ID is not filled
     assert len(trades.loc[trades['id'] != '']) == 0
+
+    caplog.clear()
+    default_conf_usdt['pairs'] = ['XRP/EUR']
+    # Filtered to non-existing pair
+    import_kraken_trades_from_csv(default_conf_usdt, 'feather')
+    assert log_has("Found csv files for BCHEUR.", caplog)
+    assert log_has("No data found for pairs XRP/EUR.", caplog)

tests/exchange/test_binance.py

@@ -596,10 +596,10 @@ async def test__async_get_historic_ohlcv_binance(default_conf, mocker, caplog, c
 
 
 @pytest.mark.parametrize('pair,nominal_value,mm_ratio,amt', [
-    ("BNB/BUSD:BUSD", 0.0, 0.025, 0),
+    ("XRP/USDT:USDT", 0.0, 0.025, 0),
     ("BNB/USDT:USDT", 100.0, 0.0065, 0),
     ("BTC/USDT:USDT", 170.30, 0.004, 0),
-    ("BNB/BUSD:BUSD", 999999.9, 0.1, 27500.0),
+    ("XRP/USDT:USDT", 999999.9, 0.1, 27500.0),
     ("BNB/USDT:USDT", 5000000.0, 0.15, 233035.0),
     ("BTC/USDT:USDT", 600000000, 0.5, 1.997038E8),
 ])

tests/exchange/test_bybit.py

@@ -1,20 +1,40 @@
 from datetime import datetime, timedelta, timezone
 from unittest.mock import MagicMock
 
+import pytest
+
 from freqtrade.enums.marginmode import MarginMode
 from freqtrade.enums.tradingmode import TradingMode
-from tests.conftest import EXMS, get_mock_coro, get_patched_exchange
+from freqtrade.exceptions import OperationalException
+from tests.conftest import EXMS, get_mock_coro, get_patched_exchange, log_has
 from tests.exchange.test_exchange import ccxt_exceptionhandlers
 
 
-def test_additional_exchange_init_bybit(default_conf, mocker):
+def test_additional_exchange_init_bybit(default_conf, mocker, caplog):
     default_conf['dry_run'] = False
     default_conf['trading_mode'] = TradingMode.FUTURES
     default_conf['margin_mode'] = MarginMode.ISOLATED
     api_mock = MagicMock()
     api_mock.set_position_mode = MagicMock(return_value={"dualSidePosition": False})
-    get_patched_exchange(mocker, default_conf, id="bybit", api_mock=api_mock)
+    api_mock.is_unified_enabled = MagicMock(return_value=[False, False])
+
+    exchange = get_patched_exchange(mocker, default_conf, id="bybit", api_mock=api_mock)
     assert api_mock.set_position_mode.call_count == 1
+    assert api_mock.is_unified_enabled.call_count == 1
+    assert exchange.unified_account is False
+
+    assert log_has("Bybit: Standard account.", caplog)
+
+    api_mock.set_position_mode.reset_mock()
+    api_mock.is_unified_enabled = MagicMock(return_value=[False, True])
+    with pytest.raises(OperationalException, match=r"Bybit: Unified account is not supported.*"):
+        get_patched_exchange(mocker, default_conf, id="bybit", api_mock=api_mock)
+    assert log_has("Bybit: Unified account.", caplog)
+    # exchange = get_patched_exchange(mocker, default_conf, id="bybit", api_mock=api_mock)
+    # assert api_mock.set_position_mode.call_count == 1
+    # assert api_mock.is_unified_enabled.call_count == 1
+    # assert exchange.unified_account is True
+
     ccxt_exceptionhandlers(mocker, default_conf, api_mock, 'bybit',
                            "additional_exchange_init", "set_position_mode")
 
@@ -111,6 +131,7 @@ def test_bybit_fetch_order_canceled_empty(default_conf_usdt, mocker):
         'amount': 20.0,
     })
 
+    mocker.patch(f"{EXMS}.exchange_has", return_value=True)
     exchange = get_patched_exchange(mocker, default_conf_usdt, api_mock, id='bybit')
 
     res = exchange.fetch_order('123', 'BTC/USDT')

tests/exchange/test_exchange.py

@@ -9,7 +9,7 @@ import ccxt
 import pytest
 from pandas import DataFrame
 
-from freqtrade.enums import CandleType, MarginMode, TradingMode
+from freqtrade.enums import CandleType, MarginMode, RunMode, TradingMode
 from freqtrade.exceptions import (DDosProtection, DependencyException, ExchangeError,
                                   InsufficientFundsError, InvalidOrderException,
                                   OperationalException, PricingError, TemporaryError)
@@ -796,7 +796,9 @@ def test_validate_timeframes_failed(default_conf, mocker):
 
     mocker.patch(f'{EXMS}._init_ccxt', MagicMock(return_value=api_mock))
     mocker.patch(f'{EXMS}._load_markets', MagicMock(return_value={}))
-    mocker.patch(f'{EXMS}.validate_pairs', MagicMock())
+    mocker.patch(f'{EXMS}.validate_pairs')
+    mocker.patch(f'{EXMS}.validate_stakecurrency')
+    mocker.patch(f'{EXMS}.validate_pricing')
     with pytest.raises(OperationalException,
                        match=r"Invalid timeframe '3m'. This exchange supports.*"):
         Exchange(default_conf)
@@ -806,6 +808,10 @@ def test_validate_timeframes_failed(default_conf, mocker):
                        match=r"Timeframes < 1m are currently not supported by Freqtrade."):
         Exchange(default_conf)
 
+    # Will not raise an exception in util mode.
+    default_conf['runmode'] = RunMode.UTIL_EXCHANGE
+    Exchange(default_conf)
+
 
 def test_validate_timeframes_emulated_ohlcv_1(default_conf, mocker):
     default_conf["timeframe"] = "3m"
@@ -2297,6 +2303,66 @@ def test_refresh_latest_ohlcv_cache(mocker, default_conf, candle_type, time_mach
     assert res[pair2].at[0, 'open']
 
 
+def test_refresh_ohlcv_with_cache(mocker, default_conf, time_machine) -> None:
+    start = datetime(2021, 8, 1, 0, 0, 0, 0, tzinfo=timezone.utc)
+    ohlcv = generate_test_data_raw('1h', 100, start.strftime('%Y-%m-%d'))
+    time_machine.move_to(start, tick=False)
+    pairs = [
+        ('ETH/BTC', '1d', CandleType.SPOT),
+        ('TKN/BTC', '1d', CandleType.SPOT),
+        ('LTC/BTC', '1d', CandleType.SPOT),
+        ('LTC/BTC', '5m', CandleType.SPOT),
+        ('LTC/BTC', '1h', CandleType.SPOT),
+    ]
+
+    ohlcv_data = {
+        p: ohlcv for p in pairs
+    }
+    ohlcv_mock = mocker.patch(f"{EXMS}.refresh_latest_ohlcv", return_value=ohlcv_data)
+    mocker.patch(f"{EXMS}.ohlcv_candle_limit", return_value=100)
+    exchange = get_patched_exchange(mocker, default_conf)
+
+    assert len(exchange._expiring_candle_cache) == 0
+
+    res = exchange.refresh_ohlcv_with_cache(pairs, start.timestamp())
+    assert ohlcv_mock.call_count == 1
+    assert ohlcv_mock.call_args_list[0][0][0] == pairs
+    assert len(ohlcv_mock.call_args_list[0][0][0]) == 5
+
+    assert len(res) == 5
+    # length of 3 - as we have 3 different timeframes
+    assert len(exchange._expiring_candle_cache) == 3
+
+    ohlcv_mock.reset_mock()
+    res = exchange.refresh_ohlcv_with_cache(pairs, start.timestamp())
+    assert ohlcv_mock.call_count == 0
+
+    # Expire 5m cache
+    time_machine.move_to(start + timedelta(minutes=6), tick=False)
+
+    ohlcv_mock.reset_mock()
+    res = exchange.refresh_ohlcv_with_cache(pairs, start.timestamp())
+    assert ohlcv_mock.call_count == 1
+    assert len(ohlcv_mock.call_args_list[0][0][0]) == 1
+
+    # Expire 5m and 1h cache
+    time_machine.move_to(start + timedelta(hours=2), tick=False)
+
+    ohlcv_mock.reset_mock()
+    res = exchange.refresh_ohlcv_with_cache(pairs, start.timestamp())
+    assert ohlcv_mock.call_count == 1
+    assert len(ohlcv_mock.call_args_list[0][0][0]) == 2
+
+    # Expire all caches
+    time_machine.move_to(start + timedelta(days=1, hours=2), tick=False)
+
+    ohlcv_mock.reset_mock()
+    res = exchange.refresh_ohlcv_with_cache(pairs, start.timestamp())
+    assert ohlcv_mock.call_count == 1
+    assert len(ohlcv_mock.call_args_list[0][0][0]) == 5
+    assert ohlcv_mock.call_args_list[0][0][0] == pairs
+
+
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
 async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_name):
     ohlcv = [
@@ -3171,6 +3237,7 @@ def test_is_cancel_order_result_suitable(mocker, default_conf, exchange_name, or
 def test_cancel_order_with_result(default_conf, mocker, exchange_name, corder,
                                   call_corder, call_forder):
     default_conf['dry_run'] = False
+    mocker.patch(f"{EXMS}.exchange_has", return_value=True)
     api_mock = MagicMock()
     api_mock.cancel_order = MagicMock(return_value=corder)
     api_mock.fetch_order = MagicMock(return_value={})
@@ -3184,6 +3251,7 @@ def test_cancel_order_with_result(default_conf, mocker, exchange_name, corder,
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
 def test_cancel_order_with_result_error(default_conf, mocker, exchange_name, caplog):
     default_conf['dry_run'] = False
+    mocker.patch(f"{EXMS}.exchange_has", return_value=True)
     api_mock = MagicMock()
     api_mock.cancel_order = MagicMock(side_effect=ccxt.InvalidOrder("Did not find order"))
     api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder("Did not find order"))
@@ -3281,6 +3349,7 @@ def test_fetch_order(default_conf, mocker, exchange_name, caplog):
     order.myid = 123
     order.symbol = 'TKN/BTC'
 
+    mocker.patch(f"{EXMS}.exchange_has", return_value=True)
     exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
     exchange._dry_run_open_orders['X'] = order
     assert exchange.fetch_order('X', 'TKN/BTC').myid == 123
@@ -3325,10 +3394,80 @@ def test_fetch_order(default_conf, mocker, exchange_name, caplog):
                            order_id='_', pair='TKN/BTC')
 
 
+@pytest.mark.usefixtures("init_persistence")
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+def test_fetch_order_emulated(default_conf, mocker, exchange_name, caplog):
+    default_conf['dry_run'] = True
+    default_conf['exchange']['log_responses'] = True
+    order = MagicMock()
+    order.myid = 123
+    order.symbol = 'TKN/BTC'
+
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+    mocker.patch(f'{EXMS}.exchange_has', return_value=False)
+    exchange._dry_run_open_orders['X'] = order
+    # Dry run - regular fetch_order behavior
+    assert exchange.fetch_order('X', 'TKN/BTC').myid == 123
+
+    with pytest.raises(InvalidOrderException, match=r'Tried to get an invalid dry-run-order.*'):
+        exchange.fetch_order('Y', 'TKN/BTC')
+
+    default_conf['dry_run'] = False
+    mocker.patch(f'{EXMS}.exchange_has', return_value=False)
+    api_mock = MagicMock()
+    api_mock.fetch_open_order = MagicMock(
+        return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'})
+    api_mock.fetch_closed_order = MagicMock(
+        return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'})
+    exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+    assert exchange.fetch_order(
+        'X', 'TKN/BTC') == {'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}
+    assert log_has(
+        ("API fetch_open_order: {\'id\': \'123\', \'amount\': 2, \'symbol\': \'TKN/BTC\'}"
+         ),
+        caplog
+    )
+    assert api_mock.fetch_open_order.call_count == 1
+    assert api_mock.fetch_closed_order.call_count == 0
+    caplog.clear()
+
+    # open_order doesn't find order
+    api_mock.fetch_open_order = MagicMock(side_effect=ccxt.OrderNotFound("Order not found"))
+    api_mock.fetch_closed_order = MagicMock(
+        return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'})
+    exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+    assert exchange.fetch_order(
+        'X', 'TKN/BTC') == {'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}
+    assert log_has(
+        ("API fetch_closed_order: {\'id\': \'123\', \'amount\': 2, \'symbol\': \'TKN/BTC\'}"
+         ),
+        caplog
+    )
+    assert api_mock.fetch_open_order.call_count == 1
+    assert api_mock.fetch_closed_order.call_count == 1
+    caplog.clear()
+
+    with pytest.raises(InvalidOrderException):
+        api_mock.fetch_open_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found"))
+        api_mock.fetch_closed_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found"))
+        exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+        exchange.fetch_order(order_id='_', pair='TKN/BTC')
+    assert api_mock.fetch_open_order.call_count == 1
+
+    api_mock.fetch_open_order = MagicMock(side_effect=ccxt.OrderNotFound("Order not found"))
+    exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+
+    ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name,
+                           'fetch_order_emulated', 'fetch_open_order',
+                           retries=1,
+                           order_id='_', pair='TKN/BTC', params={})
+
+
 @pytest.mark.usefixtures("init_persistence")
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
 def test_fetch_stoploss_order(default_conf, mocker, exchange_name):
     default_conf['dry_run'] = True
+    mocker.patch(f"{EXMS}.exchange_has", return_value=True)
     order = MagicMock()
     order.myid = 123
     exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
@@ -4963,8 +5102,8 @@ def test_get_maintenance_ratio_and_amt_exceptions(mocker, default_conf, leverage
 
 
 @pytest.mark.parametrize('pair,value,mmr,maintAmt', [
-    ('ADA/BUSD:BUSD', 500, 0.025, 0.0),
-    ('ADA/BUSD:BUSD', 20000000, 0.5, 1527500.0),
+    ('ADA/USDT:USDT', 500, 0.025, 0.0),
+    ('ADA/USDT:USDT', 20000000, 0.5, 1527500.0),
     ('ZEC/USDT:USDT', 500, 0.01, 0.0),
     ('ZEC/USDT:USDT', 20000000, 0.5, 654500.0),
 ])
@@ -4999,10 +5138,10 @@ def test_get_max_leverage_futures(default_conf, mocker, leverage_tiers):
 
     exchange._leverage_tiers = leverage_tiers
 
-    assert exchange.get_max_leverage("BNB/BUSD:BUSD", 1.0) == 20.0
+    assert exchange.get_max_leverage("XRP/USDT:USDT", 1.0) == 20.0
     assert exchange.get_max_leverage("BNB/USDT:USDT", 100.0) == 75.0
     assert exchange.get_max_leverage("BTC/USDT:USDT", 170.30) == 125.0
-    assert pytest.approx(exchange.get_max_leverage("BNB/BUSD:BUSD", 99999.9)) == 5.000005
+    assert pytest.approx(exchange.get_max_leverage("XRP/USDT:USDT", 99999.9)) == 5.000005
     assert pytest.approx(exchange.get_max_leverage("BNB/USDT:USDT", 1500)) == 33.333333333333333
     assert exchange.get_max_leverage("BTC/USDT:USDT", 300000000) == 2.0
     assert exchange.get_max_leverage("BTC/USDT:USDT", 600000000) == 1.0  # Last tier

tests/exchange/test_okx.py

@@ -196,7 +196,7 @@ def test_get_max_pair_stake_amount_okx(default_conf, mocker, leverage_tiers):
     exchange = get_patched_exchange(mocker, default_conf, id="okx")
     exchange._leverage_tiers = leverage_tiers
 
-    assert exchange.get_max_pair_stake_amount('BNB/BUSD:BUSD', 1.0) == 30000000
+    assert exchange.get_max_pair_stake_amount('XRP/USDT:USDT', 1.0) == 30000000
     assert exchange.get_max_pair_stake_amount('BNB/USDT:USDT', 1.0) == 50000000
     assert exchange.get_max_pair_stake_amount('BTC/USDT:USDT', 1.0) == 1000000000
     assert exchange.get_max_pair_stake_amount('BTC/USDT:USDT', 1.0, 10.0) == 100000000

tests/exchange_online/conftest.py

@@ -324,7 +324,8 @@ def get_futures_exchange(exchange_name, exchange_conf, class_mocker):
 
 
 @pytest.fixture(params=EXCHANGES, scope="class")
-def exchange(request, exchange_conf):
+def exchange(request, exchange_conf, class_mocker):
+    class_mocker.patch('freqtrade.exchange.bybit.Bybit.additional_exchange_init')
     yield from get_exchange(request.param, exchange_conf)
 
 

tests/exchange_online/test_ccxt_compat.py

@@ -12,6 +12,7 @@ import pytest
 from freqtrade.enums import CandleType
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date
 from freqtrade.exchange.exchange import timeframe_to_msecs
+from freqtrade.util import dt_floor_day, dt_now, dt_ts
 from tests.exchange_online.conftest import EXCHANGE_FIXTURE_TYPE, EXCHANGES
 
 
@@ -187,6 +188,25 @@ class TestCCXTExchange:
         now = datetime.now(timezone.utc) - timedelta(minutes=(timeframe_to_minutes(timeframe) * 2))
         assert exch.klines(pair_tf).iloc[-1]['date'] >= timeframe_to_prev_date(timeframe, now)
 
+    def test_ccxt_fetch_ohlcv_startdate(self, exchange: EXCHANGE_FIXTURE_TYPE):
+        """
+        Test that pair data starts at the provided startdate
+        """
+        exch, exchangename = exchange
+        pair = EXCHANGES[exchangename]['pair']
+        timeframe = '1d'
+
+        pair_tf = (pair, timeframe, CandleType.SPOT)
+        # last 5 days ...
+        since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=6))
+        ohlcv = exch.refresh_latest_ohlcv([pair_tf], since_ms=since_ms)
+        assert isinstance(ohlcv, dict)
+        assert len(ohlcv[pair_tf]) == len(exch.klines(pair_tf))
+        # Check if last-timeframe is within the last 2 intervals
+        now = datetime.now(timezone.utc) - timedelta(minutes=(timeframe_to_minutes(timeframe) * 2))
+        assert exch.klines(pair_tf).iloc[-1]['date'] >= timeframe_to_prev_date(timeframe, now)
+        assert exch.klines(pair_tf)['date'].astype(int).iloc[0] // 1e6 == since_ms
+
     def ccxt__async_get_candle_history(
             self, exchange, exchangename, pair, timeframe, candle_type, factor=0.9):
 

tests/freqtradebot/test_integration.py

@@ -49,7 +49,7 @@ def test_may_execute_exit_stoploss_on_exchange_multi(default_conf, ticker, fee,
     stoploss_order_closed['filled'] = stoploss_order_closed['amount']
 
     # Sell first trade based on stoploss, keep 2nd and 3rd trade open
-    stop_orders = [stoploss_order_closed, stoploss_order_open, stoploss_order_open]
+    stop_orders = [stoploss_order_closed, stoploss_order_open.copy(), stoploss_order_open.copy()]
     stoploss_order_mock = MagicMock(
         side_effect=stop_orders)
     # Sell 3rd trade (not called for the first trade)
@@ -100,9 +100,10 @@ def test_may_execute_exit_stoploss_on_exchange_multi(default_conf, ticker, fee,
         stop_order = stop_orders[idx]
         stop_order['id'] = f"stop{idx}"
         oobj = Order.parse_from_ccxt_object(stop_order, trade.pair, 'stoploss')
+        oobj.ft_is_open = True
 
         trade.orders.append(oobj)
-        trade.stoploss_order_id = f"stop{idx}"
+        assert len(trade.open_sl_orders) == 1
 
     n = freqtrade.exit_positions(trades)
     assert n == 2
@@ -113,6 +114,7 @@ def test_may_execute_exit_stoploss_on_exchange_multi(default_conf, ticker, fee,
 
     # Only order for 3rd trade needs to be cancelled
     assert cancel_order_mock.call_count == 1
+    assert stoploss_order_mock.call_count == 3
     # Wallets must be updated between stoploss cancellation and selling, and will be updated again
     # during update_trade_state
     assert wallets_mock.call_count == 4
@@ -536,7 +538,7 @@ def test_dca_order_adjust_entry_replace_fails(
     # Create DCA order for 2nd trade (so we have 2 open orders on 2 trades)
     # this 2nd order won't fill.
 
-    freqtrade.strategy.adjust_trade_position = MagicMock(return_value=20)
+    freqtrade.strategy.adjust_trade_position = MagicMock(return_value=(20, 'PeNF'))
 
     freqtrade.process()
 
@@ -627,12 +629,13 @@ def test_dca_exiting(default_conf_usdt, ticker_usdt, fee, mocker, caplog, levera
     assert log_has_re(
         r"Remaining amount of \d\.\d+.* would be smaller than the minimum of 10.", caplog)
 
-    freqtrade.strategy.adjust_trade_position = MagicMock(return_value=-20)
+    freqtrade.strategy.adjust_trade_position = MagicMock(return_value=(-20, 'PES'))
 
     freqtrade.process()
     trade = Trade.get_trades().first()
     assert len(trade.orders) == 2
     assert trade.orders[-1].ft_order_side == 'sell'
+    assert trade.orders[-1].ft_order_tag == 'PES'
     assert pytest.approx(trade.stake_amount) == 40.198
     assert pytest.approx(trade.amount) == 20.099 * leverage
     assert trade.open_rate == 2.0

tests/optimize/conftest.py

@@ -30,7 +30,7 @@ def hyperopt_conf(default_conf):
 
 
 @pytest.fixture(autouse=True)
-def backtesting_cleanup() -> None:
+def backtesting_cleanup():
     yield None
 
     Backtesting.cleanup()

tests/optimize/test_backtest_detail.py

@@ -900,7 +900,7 @@ TESTS = [
 
 
 @pytest.mark.parametrize("data", TESTS)
-def test_backtest_results(default_conf, fee, mocker, caplog, data: BTContainer) -> None:
+def test_backtest_results(default_conf, mocker, caplog, data: BTContainer) -> None:
     """
     run functional tests
     """

tests/optimize/test_backtesting.py

@@ -742,14 +742,18 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None:
          'orders': [
             [
                 {'amount': 0.00957442, 'safe_price': 0.104445, 'ft_order_side': 'buy',
-                 'order_filled_timestamp': 1517251200000, 'ft_is_entry': True},
+                 'order_filled_timestamp': 1517251200000, 'ft_is_entry': True,
+                 'ft_order_tag': ''},
                 {'amount': 0.00957442, 'safe_price': 0.10496853383458644, 'ft_order_side': 'sell',
-                 'order_filled_timestamp': 1517265300000, 'ft_is_entry': False}
+                 'order_filled_timestamp': 1517265300000, 'ft_is_entry': False,
+                 'ft_order_tag': 'roi'}
             ], [
                 {'amount': 0.0097064, 'safe_price': 0.10302485, 'ft_order_side': 'buy',
-                 'order_filled_timestamp': 1517283000000, 'ft_is_entry': True},
+                 'order_filled_timestamp': 1517283000000, 'ft_is_entry': True,
+                 'ft_order_tag': ''},
                 {'amount': 0.0097064, 'safe_price': 0.10354126528822055, 'ft_order_side': 'sell',
-                 'order_filled_timestamp': 1517285400000, 'ft_is_entry': False}
+                 'order_filled_timestamp': 1517285400000, 'ft_is_entry': False,
+                 'ft_order_tag': 'roi'}
             ]
          ]
          })

tests/optimize/test_backtesting_adjust_position.py

@@ -57,28 +57,30 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) ->
                                      ),
          'close_date': pd.to_datetime([dt_utc(2018, 1, 29, 22, 00, 0),
                                        dt_utc(2018, 1, 30, 4, 10, 0)], utc=True),
-         'open_rate': [0.10401764894444211, 0.10302485],
-         'close_rate': [0.10453904066847439, 0.103541],
+         'open_rate': [0.10401764891917063, 0.10302485],
+         'close_rate': [0.10453904064307624, 0.10354126528822055],
          'fee_open': [0.0025, 0.0025],
          'fee_close': [0.0025, 0.0025],
          'trade_duration': [200, 40],
          'profit_ratio': [0.0, 0.0],
          'profit_abs': [0.0, 0.0],
          'exit_reason': [ExitType.ROI.value, ExitType.ROI.value],
-         'initial_stop_loss_abs': [0.0940005, 0.09272236],
+         'initial_stop_loss_abs': [0.0940005, 0.092722365],
          'initial_stop_loss_ratio': [-0.1, -0.1],
-         'stop_loss_abs': [0.0940005, 0.09272236],
+         'stop_loss_abs': [0.0940005, 0.092722365],
          'stop_loss_ratio': [-0.1, -0.1],
          'min_rate': [0.10370188, 0.10300000000000001],
-         'max_rate': [0.10481985, 0.1038888],
+         'max_rate': [0.10481985, 0.10388887000000001],
          'is_open': [False, False],
          'enter_tag': ['', ''],
          'leverage': [1.0, 1.0],
          'is_short': [False, False],
          'open_timestamp': [1517251200000, 1517283000000],
-         'close_timestamp': [1517265300000, 1517285400000],
+         'close_timestamp': [1517263200000, 1517285400000],
          })
-    pd.testing.assert_frame_equal(results.drop(columns=['orders']), expected)
+    results_no = results.drop(columns=['orders'])
+    pd.testing.assert_frame_equal(results_no, expected, check_exact=True)
+
     data_pair = processed[pair]
     assert len(results.iloc[0]['orders']) == 6
     assert len(results.iloc[1]['orders']) == 2
@@ -148,7 +150,7 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker, levera
     assert pytest.approx(trade.amount) == 47.61904762 * leverage
     assert len(trade.orders) == 1
     # Increase position by 100
-    backtesting.strategy.adjust_trade_position = MagicMock(return_value=100)
+    backtesting.strategy.adjust_trade_position = MagicMock(return_value=(100, 'PartIncrease'))
 
     trade = backtesting._get_adjust_trade_entry_for_candle(trade, row, current_time)
 
@@ -156,6 +158,7 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker, levera
     assert pytest.approx(trade.stake_amount) == 200.0
     assert pytest.approx(trade.amount) == 95.23809524 * leverage
     assert len(trade.orders) == 2
+    assert trade.orders[-1].ft_order_tag == 'PartIncrease'
     assert pytest.approx(trade.liquidation_price) == (0.1038916 if leverage == 1 else 1.2127791)
 
     # Reduce by more than amount - no change to trade.
@@ -171,13 +174,14 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker, levera
     assert pytest.approx(trade.liquidation_price) == (0.1038916 if leverage == 1 else 1.2127791)
 
     # Reduce position by 50
-    backtesting.strategy.adjust_trade_position = MagicMock(return_value=-100)
+    backtesting.strategy.adjust_trade_position = MagicMock(return_value=(-100, 'partDecrease'))
     trade = backtesting._get_adjust_trade_entry_for_candle(trade, row, current_time)
 
     assert trade
     assert pytest.approx(trade.stake_amount) == 100.0
     assert pytest.approx(trade.amount) == 47.61904762 * leverage
     assert len(trade.orders) == 3
+    assert trade.orders[-1].ft_order_tag == 'partDecrease'
     assert trade.nr_of_successful_entries == 2
     assert trade.nr_of_successful_exits == 1
     assert pytest.approx(trade.liquidation_price) == (0.1038916 if leverage == 1 else 1.2127791)

tests/persistence/test_migrations.py

@@ -74,7 +74,7 @@ def test_init_dryrun_db(default_conf, tmpdir):
     assert Path(filename).is_file()
 
 
-def test_migrate_new(mocker, default_conf, fee, caplog):
+def test_migrate(mocker, default_conf, fee, caplog):
     """
     Test Database migration (starting with new pairformat)
     """
@@ -277,8 +277,6 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
     assert trade.exit_reason is None
     assert trade.strategy is None
     assert trade.timeframe == '5m'
-    assert trade.stoploss_order_id == 'dry_stop_order_id222'
-    assert trade.stoploss_last_update is None
     assert log_has("trying trades_bak1", caplog)
     assert log_has("trying trades_bak2", caplog)
     assert log_has("Running database migration for trades - backup: trades_bak2, orders_bak0",
@@ -294,9 +292,10 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
     assert orders[0].order_id == 'dry_buy_order'
     assert orders[0].ft_order_side == 'buy'
 
+    # All dry-run stoploss orders will be closed
     assert orders[-1].order_id == 'dry_stop_order_id222'
     assert orders[-1].ft_order_side == 'stoploss'
-    assert orders[-1].ft_is_open is True
+    assert orders[-1].ft_is_open is False
 
     assert orders[1].order_id == 'dry_buy_order22'
     assert orders[1].ft_order_side == 'buy'

tests/persistence/test_persistence.py

@@ -1400,6 +1400,8 @@ def test_to_json(fee):
         'is_open': None,
         'open_date': trade.open_date.strftime(DATETIME_PRINT_FORMAT),
         'open_timestamp': int(trade.open_date.timestamp() * 1000),
+        'open_fill_date': None,
+        'open_fill_timestamp': None,
         'close_date': None,
         'close_timestamp': None,
         'open_rate': 0.123,
@@ -1432,7 +1434,6 @@ def test_to_json(fee):
         'stop_loss_abs': None,
         'stop_loss_ratio': None,
         'stop_loss_pct': None,
-        'stoploss_order_id': None,
         'stoploss_last_update': None,
         'stoploss_last_update_timestamp': None,
         'initial_stop_loss_abs': None,
@@ -1487,6 +1488,8 @@ def test_to_json(fee):
         'quote_currency': 'BTC',
         'open_date': trade.open_date.strftime(DATETIME_PRINT_FORMAT),
         'open_timestamp': int(trade.open_date.timestamp() * 1000),
+        'open_fill_date': None,
+        'open_fill_timestamp': None,
         'close_date': trade.close_date.strftime(DATETIME_PRINT_FORMAT),
         'close_timestamp': int(trade.close_date.timestamp() * 1000),
         'open_rate': 0.123,
@@ -1500,7 +1503,6 @@ def test_to_json(fee):
         'stop_loss_abs': None,
         'stop_loss_pct': None,
         'stop_loss_ratio': None,
-        'stoploss_order_id': None,
         'stoploss_last_update': None,
         'stoploss_last_update_timestamp': None,
         'initial_stop_loss_abs': None,

tests/persistence/test_trade_fromjson.py

@@ -54,7 +54,6 @@ def test_trade_fromjson():
         "stop_loss_abs": 0.1981,
         "stop_loss_ratio": -0.216,
         "stop_loss_pct": -21.6,
-        "stoploss_order_id": null,
         "stoploss_last_update": "2022-10-18 09:13:42",
         "stoploss_last_update_timestamp": 1666077222000,
         "initial_stop_loss_abs": 0.1981,
@@ -224,7 +223,7 @@ def test_trade_serialize_load_back(fee):
         'realized_profit_ratio', 'close_profit_pct',
         'trade_duration_s', 'trade_duration',
         'profit_ratio', 'profit_pct', 'profit_abs', 'stop_loss_abs',
-        'initial_stop_loss_abs',
+        'initial_stop_loss_abs', 'open_fill_date', 'open_fill_timestamp',
         'orders',
     ]
     failed = []

tests/plugins/test_pairlist.py

@@ -18,6 +18,7 @@ from freqtrade.persistence import LocalTrade, Trade
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist
 from freqtrade.plugins.pairlistmanager import PairListManager
 from freqtrade.resolvers import PairListResolver
+from freqtrade.util.datetime_helpers import dt_now
 from tests.conftest import (EXMS, create_mock_trades_usdt, get_patched_exchange,
                             get_patched_freqtradebot, log_has, log_has_re, num_log_has)
 
@@ -620,13 +621,20 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
        "lookback_timeframe": "1d", "lookback_period": 6, "refresh_period": 86400}],
      "BTC", "binance", ['LTC/BTC', 'XRP/BTC', 'ETH/BTC', 'HOT/BTC', 'NEO/BTC']),
+    # VolumePairlist in range mode as filter.
+    # TKN/BTC is removed because it doesn't have enough candles
+    ([{"method": "VolumePairList", "number_assets": 5},
+      {"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
+       "lookback_timeframe": "1d", "lookback_period": 2, "refresh_period": 86400}],
+     "BTC", "binance", ['LTC/BTC', 'XRP/BTC', 'ETH/BTC', 'TKN/BTC', 'HOT/BTC']),
     # ftx data is already in Quote currency, therefore won't require conversion
     # ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
     #    "lookback_timeframe": "1d", "lookback_period": 1, "refresh_period": 86400}],
     #  "BTC", "ftx", ['HOT/BTC', 'LTC/BTC', 'ETH/BTC', 'TKN/BTC', 'XRP/BTC']),
 ])
-def test_VolumePairList_range(mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history,
-                              pairlists, base_currency, exchange, volumefilter_result) -> None:
+def test_VolumePairList_range(
+        mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history,
+        pairlists, base_currency, exchange, volumefilter_result, time_machine) -> None:
     whitelist_conf['pairlists'] = pairlists
     whitelist_conf['stake_currency'] = base_currency
     whitelist_conf['exchange']['name'] = exchange
@@ -685,23 +693,36 @@ def test_VolumePairList_range(mocker, whitelist_conf, shitcoinmarkets, tickers,
             get_tickers=tickers,
             markets=PropertyMock(return_value=shitcoinmarkets)
         )
-
+        start_dt = dt_now()
+        time_machine.move_to(start_dt)
         # remove ohlcv when looback_timeframe != 1d
         # to enforce fallback to ticker data
         if 'lookback_timeframe' in pairlists[0]:
             if pairlists[0]['lookback_timeframe'] != '1d':
-                ohlcv_data = []
+                ohlcv_data = {}
 
-        mocker.patch.multiple(
-            EXMS,
-            refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
-        )
+        ohclv_mock = mocker.patch(f"{EXMS}.refresh_latest_ohlcv", return_value=ohlcv_data)
 
         freqtrade.pairlists.refresh_pairlist()
         whitelist = freqtrade.pairlists.whitelist
+        assert ohclv_mock.call_count == 1
 
         assert isinstance(whitelist, list)
         assert whitelist == volumefilter_result
+        # Test caching
+        ohclv_mock.reset_mock()
+        freqtrade.pairlists.refresh_pairlist()
+        # in "filter" mode, caching is disabled.
+        assert ohclv_mock.call_count == 0
+        whitelist = freqtrade.pairlists.whitelist
+        assert whitelist == volumefilter_result
+
+        time_machine.move_to(start_dt + timedelta(days=2))
+        ohclv_mock.reset_mock()
+        freqtrade.pairlists.refresh_pairlist()
+        assert ohclv_mock.call_count == 1
+        whitelist = freqtrade.pairlists.whitelist
+        assert whitelist == volumefilter_result
 
 
 def test_PrecisionFilter_error(mocker, whitelist_conf) -> None:
@@ -1513,3 +1534,144 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None:
         pm.refresh_pairlist()
         assert pm.whitelist == []
         assert log_has_re(r'Whitelist with 0 pairs: \[]', caplog)
+
+
+@pytest.mark.parametrize('pairlists,trade_mode,result', [
+    ([
+        # Get 2 pairs
+        {"method": "StaticPairList", "allow_inactive": True},
+        {"method": "MarketCapPairList", "number_assets": 2}
+    ], 'spot', ['BTC/USDT', 'ETH/USDT']),
+    ([
+        # Get 6 pairs
+        {"method": "StaticPairList", "allow_inactive": True},
+        {"method": "MarketCapPairList", "number_assets": 6}
+    ], 'spot', ['BTC/USDT', 'ETH/USDT', 'XRP/USDT', 'ADA/USDT']),
+    ([
+        # Get 3 pairs within top 6 ranks
+        {"method": "StaticPairList", "allow_inactive": True},
+        {"method": "MarketCapPairList", "max_rank": 6, "number_assets": 3}
+    ], 'spot', ['BTC/USDT', 'ETH/USDT', 'XRP/USDT']),
+
+    ([
+        # Get 4 pairs within top 8 ranks
+        {"method": "StaticPairList", "allow_inactive": True},
+        {"method": "MarketCapPairList", "max_rank": 8, "number_assets": 4}
+    ], 'spot', ['BTC/USDT', 'ETH/USDT', 'XRP/USDT']),
+    ([
+        # MarketCapPairList as generator
+        {"method": "MarketCapPairList", "number_assets": 5}
+    ], 'spot', ['BTC/USDT', 'ETH/USDT', 'XRP/USDT']),
+    ([
+        # MarketCapPairList as generator - low max_rank
+        {"method": "MarketCapPairList", "max_rank": 2, "number_assets": 5}
+    ], 'spot', ['BTC/USDT', 'ETH/USDT']),
+    ([
+        # MarketCapPairList as generator - futures - low max_rank
+        {"method": "MarketCapPairList", "max_rank": 2, "number_assets": 5}
+    ], 'futures', ['ETH/USDT:USDT']),
+    ([
+        # MarketCapPairList as generator - futures - low number_assets
+        {"method": "MarketCapPairList", "number_assets": 2}
+    ], 'futures', ['ETH/USDT:USDT', 'ADA/USDT:USDT']),
+])
+def test_MarketCapPairList_filter(
+        mocker, default_conf_usdt, trade_mode, markets, pairlists, result
+):
+    test_value = [
+        {"symbol": "btc"},
+        {"symbol": "eth"},
+        {"symbol": "usdt"},
+        {"symbol": "bnb"},
+        {"symbol": "sol"},
+        {"symbol": "xrp"},
+        {"symbol": "usdc"},
+        {"symbol": "steth"},
+        {"symbol": "ada"},
+        {"symbol": "avax"},
+    ]
+
+    default_conf_usdt['trading_mode'] = trade_mode
+    if trade_mode == 'spot':
+        default_conf_usdt['exchange']['pair_whitelist'].extend(['BTC/USDT', 'ETC/USDT', 'ADA/USDT'])
+    default_conf_usdt['pairlists'] = pairlists
+    mocker.patch.multiple(EXMS,
+                          markets=PropertyMock(return_value=markets),
+                          exchange_has=MagicMock(return_value=True),
+                          )
+
+    mocker.patch("freqtrade.plugins.pairlist.MarketCapPairList.CoinGeckoAPI.get_coins_markets",
+                 return_value=test_value)
+
+    exchange = get_patched_exchange(mocker, default_conf_usdt)
+
+    pm = PairListManager(exchange, default_conf_usdt)
+    pm.refresh_pairlist()
+
+    assert pm.whitelist == result
+
+
+def test_MarketCapPairList_timing(mocker, default_conf_usdt, markets, time_machine):
+    test_value = [
+        {"symbol": "btc"},
+        {"symbol": "eth"},
+        {"symbol": "usdt"},
+        {"symbol": "bnb"},
+        {"symbol": "sol"},
+        {"symbol": "xrp"},
+        {"symbol": "usdc"},
+        {"symbol": "steth"},
+        {"symbol": "ada"},
+        {"symbol": "avax"},
+    ]
+
+    default_conf_usdt['trading_mode'] = 'spot'
+    default_conf_usdt['exchange']['pair_whitelist'].extend(['BTC/USDT', 'ETC/USDT', 'ADA/USDT'])
+    default_conf_usdt['pairlists'] = [{"method": "MarketCapPairList", "number_assets": 2}]
+
+    markets_mock = MagicMock(return_value=markets)
+    mocker.patch.multiple(EXMS,
+                          get_markets=markets_mock,
+                          exchange_has=MagicMock(return_value=True),
+                          )
+
+    mocker.patch("freqtrade.plugins.pairlist.MarketCapPairList.CoinGeckoAPI.get_coins_markets",
+                 return_value=test_value)
+
+    start_dt = dt_now()
+
+    exchange = get_patched_exchange(mocker, default_conf_usdt)
+    time_machine.move_to(start_dt)
+
+    pm = PairListManager(exchange, default_conf_usdt)
+    markets_mock.reset_mock()
+    pm.refresh_pairlist()
+    assert markets_mock.call_count == 3
+    markets_mock.reset_mock()
+
+    time_machine.move_to(start_dt + timedelta(hours=20))
+    pm.refresh_pairlist()
+    # Cached pairlist ...
+    assert markets_mock.call_count == 1
+
+    markets_mock.reset_mock()
+    time_machine.move_to(start_dt + timedelta(days=2))
+    pm.refresh_pairlist()
+    # No longer cached pairlist ...
+    assert markets_mock.call_count == 3
+
+
+def test_MarketCapPairList_exceptions(mocker, default_conf_usdt, markets, time_machine):
+
+    exchange = get_patched_exchange(mocker, default_conf_usdt)
+    default_conf_usdt['pairlists'] = [{"method": "MarketCapPairList"}]
+    with pytest.raises(OperationalException, match=r"`number_assets` not specified.*"):
+        # No number_assets
+        PairListManager(exchange, default_conf_usdt)
+
+    default_conf_usdt['pairlists'] = [{
+        "method": "MarketCapPairList", 'number_assets': 20, 'max_rank': 260
+    }]
+    with pytest.raises(OperationalException,
+                       match="This filter only support marketcap rank up to 250."):
+        PairListManager(exchange, default_conf_usdt)

tests/rpc/test_rpc.py

@@ -25,6 +25,8 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
         'quote_currency': 'BTC',
         'open_date': ANY,
         'open_timestamp': ANY,
+        'open_fill_date': ANY,
+        'open_fill_timestamp': ANY,
         'is_open': ANY,
         'fee_open': ANY,
         'fee_open_cost': ANY,
@@ -63,7 +65,6 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
         'stop_loss_abs': 9.89e-06,
         'stop_loss_pct': -10.0,
         'stop_loss_ratio': -0.1,
-        'stoploss_order_id': None,
         'stoploss_last_update': ANY,
         'stoploss_last_update_timestamp': ANY,
         'initial_stop_loss_abs': 9.89e-06,
@@ -99,7 +100,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
             'order_filled_timestamp': ANY, 'order_type': 'limit', 'price': 1.098e-05,
             'is_open': False, 'pair': 'ETH/BTC', 'order_id': ANY,
             'remaining': ANY, 'status': ANY, 'ft_is_entry': True, 'ft_fee_base': None,
-            'funding_fee': ANY,
+            'funding_fee': ANY, 'ft_order_tag': None,
         }],
     }
     mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock())
@@ -355,7 +356,6 @@ def test_rpc_delete_trade(mocker, default_conf, fee, markets, caplog, is_short):
         rpc._rpc_delete('200')
 
     trades = Trade.session.scalars(select(Trade)).all()
-    trades[2].stoploss_order_id = '102'
     trades[2].orders.append(
         Order(
             ft_order_side='stoploss',

tests/rpc/test_rpc_apiserver.py

@@ -180,7 +180,9 @@ def test_api_auth():
 
 def test_api_ws_auth(botclient):
     ftbot, client = botclient
-    def url(token): return f"/api/v1/message/ws?token={token}"
+
+    def url(token):
+        return f"/api/v1/message/ws?token={token}"
 
     bad_token = "bad-ws_token"
     with pytest.raises(WebSocketDisconnect):
@@ -1165,6 +1167,8 @@ def test_api_status(botclient, mocker, ticker, fee, markets, is_short,
         'current_rate': current_rate,
         'open_date': ANY,
         'open_timestamp': ANY,
+        'open_fill_date': ANY,
+        'open_fill_timestamp': ANY,
         'open_rate': 0.123,
         'pair': 'ETH/BTC',
         'base_currency': 'ETH',
@@ -1174,7 +1178,6 @@ def test_api_status(botclient, mocker, ticker, fee, markets, is_short,
         'stop_loss_abs': ANY,
         'stop_loss_pct': ANY,
         'stop_loss_ratio': ANY,
-        'stoploss_order_id': None,
         'stoploss_last_update': ANY,
         'stoploss_last_update_timestamp': ANY,
         'initial_stop_loss_abs': 0.0,
@@ -1369,6 +1372,8 @@ def test_api_force_entry(botclient, mocker, fee, endpoint):
         'close_rate': 0.265441,
         'open_date': ANY,
         'open_timestamp': ANY,
+        'open_fill_date': ANY,
+        'open_fill_timestamp': ANY,
         'open_rate': 0.245441,
         'pair': 'ETH/BTC',
         'base_currency': 'ETH',
@@ -1378,7 +1383,6 @@ def test_api_force_entry(botclient, mocker, fee, endpoint):
         'stop_loss_abs': None,
         'stop_loss_pct': None,
         'stop_loss_ratio': None,
-        'stoploss_order_id': None,
         'stoploss_last_update': None,
         'stoploss_last_update_timestamp': None,
         'initial_stop_loss_abs': None,

tests/rpc/test_rpc_telegram.py

@@ -1507,7 +1507,7 @@ async def test_telegram_entry_tag_performance_handle(
     await telegram._enter_tag_performance(update=update, context=context)
     assert msg_mock.call_count == 1
     assert 'Entry Tag Performance' in msg_mock.call_args_list[0][0][0]
-    assert '<code>TEST1\t3.987 USDT (5.00%) (1)</code>' in msg_mock.call_args_list[0][0][0]
+    assert '`TEST1\t3.987 USDT (5.00%) (1)`' in msg_mock.call_args_list[0][0][0]
 
     context.args = ['XRP/USDT']
     await telegram._enter_tag_performance(update=update, context=context)
@@ -1538,7 +1538,7 @@ async def test_telegram_exit_reason_performance_handle(
     await telegram._exit_reason_performance(update=update, context=context)
     assert msg_mock.call_count == 1
     assert 'Exit Reason Performance' in msg_mock.call_args_list[0][0][0]
-    assert '<code>roi\t2.842 USDT (10.00%) (1)</code>' in msg_mock.call_args_list[0][0][0]
+    assert '`roi\t2.842 USDT (10.00%) (1)`' in msg_mock.call_args_list[0][0][0]
     context.args = ['XRP/USDT']
 
     await telegram._exit_reason_performance(update=update, context=context)
@@ -1570,7 +1570,7 @@ async def test_telegram_mix_tag_performance_handle(default_conf_usdt, update, ti
     await telegram._mix_tag_performance(update=update, context=context)
     assert msg_mock.call_count == 1
     assert 'Mix Tag Performance' in msg_mock.call_args_list[0][0][0]
-    assert ('<code>TEST3 roi\t2.842 USDT (10.00%) (1)</code>'
+    assert ('`TEST3 roi\t2.842 USDT (10.00%) (1)`'
             in msg_mock.call_args_list[0][0][0])
 
     context.args = ['XRP/USDT']
@@ -2017,7 +2017,7 @@ def test_send_msg_enter_notification(default_conf, mocker, caplog, message_type,
     telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf)
 
     telegram.send_msg(msg)
-    leverage_text = f' ({leverage:.1g}x)' if leverage and leverage != 1.0 else ''
+    leverage_text = f' ({leverage:.3g}x)' if leverage and leverage != 1.0 else ''
 
     assert msg_mock.call_args[0][0] == (
         f'\N{LARGE BLUE CIRCLE} *Binance (dry):* New Trade (#1)\n'
@@ -2126,7 +2126,7 @@ def test_send_msg_entry_fill_notification(default_conf, mocker, message_type, en
         'amount': 1333.3333333333335,
         'open_date': dt_now() - timedelta(hours=1)
     })
-    leverage_text = f' ({leverage:.1g}x)' if leverage != 1.0 else ''
+    leverage_text = f' ({leverage:.3g}x)' if leverage != 1.0 else ''
     assert msg_mock.call_args[0][0] == (
         f'\N{CHECK MARK} *Binance (dry):* New Trade filled (#1)\n'
         f'*Pair:* `ETH/BTC`\n'
@@ -2365,7 +2365,7 @@ def test_send_msg_exit_fill_notification(default_conf, mocker, direction,
             'close_date': dt_now(),
         })
 
-        leverage_text = f' ({leverage:.1g}x)`\n' if leverage and leverage != 1.0 else '`\n'
+        leverage_text = f' ({leverage:.3g}x)`\n' if leverage and leverage != 1.0 else '`\n'
         assert msg_mock.call_args[0][0] == (
             '\N{WARNING SIGN} *Binance (dry):* Exited KEY/ETH (#1)\n'
             '*Profit:* `-57.41% (loss: -0.05746 ETH)`\n'
@@ -2458,7 +2458,7 @@ def test_send_msg_buy_notification_no_fiat(
         'open_date': dt_now() - timedelta(hours=1)
     })
 
-    leverage_text = f' ({leverage:.1g}x)' if leverage and leverage != 1.0 else ''
+    leverage_text = f' ({leverage:.3g}x)' if leverage and leverage != 1.0 else ''
     assert msg_mock.call_args[0][0] == (
         f'\N{LARGE BLUE CIRCLE} *Binance:* New Trade (#1)\n'
         '*Pair:* `ETH/BTC`\n'
@@ -2510,7 +2510,7 @@ def test_send_msg_exit_notification_no_fiat(
         'close_date': dt_now(),
     })
 
-    leverage_text = f' ({leverage:.1g}x)' if leverage and leverage != 1.0 else ''
+    leverage_text = f' ({leverage:.3g}x)' if leverage and leverage != 1.0 else ''
     assert msg_mock.call_args[0][0] == (
         '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n'
         '*Unrealized Profit:* `-57.41% (loss: -0.05746 ETH)`\n'
@@ -2557,22 +2557,22 @@ async def test_telegram__send_msg(default_conf, mocker, caplog) -> None:
 
     # Test update
     query = MagicMock()
+    query.edit_message_text = AsyncMock()
     await telegram._send_msg('test', callback_path="DeadBeef", query=query, reload_able=True)
-    edit_message_text = telegram._app.bot.edit_message_text
-    assert edit_message_text.call_count == 1
-    assert "Updated: " in edit_message_text.call_args_list[0][1]['text']
+    assert query.edit_message_text.call_count == 1
+    assert "Updated: " in query.edit_message_text.call_args_list[0][1]['text']
 
-    telegram._app.bot.edit_message_text = AsyncMock(side_effect=BadRequest("not modified"))
+    query.edit_message_text = AsyncMock(side_effect=BadRequest("not modified"))
     await telegram._send_msg('test', callback_path="DeadBeef", query=query)
-    assert telegram._app.bot.edit_message_text.call_count == 1
+    assert query.edit_message_text.call_count == 1
     assert not log_has_re(r"TelegramError: .*", caplog)
 
-    telegram._app.bot.edit_message_text = AsyncMock(side_effect=BadRequest(""))
+    query.edit_message_text = AsyncMock(side_effect=BadRequest(""))
     await telegram._send_msg('test2', callback_path="DeadBeef", query=query)
-    assert telegram._app.bot.edit_message_text.call_count == 1
+    assert query.edit_message_text.call_count == 1
     assert log_has_re(r"TelegramError: .*", caplog)
 
-    telegram._app.bot.edit_message_text = AsyncMock(side_effect=TelegramError("DeadBEEF"))
+    query.edit_message_text = AsyncMock(side_effect=TelegramError("DeadBEEF"))
     await telegram._send_msg('test3', callback_path="DeadBeef", query=query)
 
     assert log_has_re(r"TelegramError: DeadBEEF! Giving up.*", caplog)

tests/utils/test_datetime_helpers.py

@@ -3,8 +3,8 @@ from datetime import datetime, timedelta, timezone
 import pytest
 import time_machine
 
-from freqtrade.util import (dt_floor_day, dt_from_ts, dt_humanize, dt_now, dt_ts, dt_ts_def, dt_utc,
-                            format_date, format_ms_time, shorten_date)
+from freqtrade.util import (dt_floor_day, dt_from_ts, dt_humanize, dt_now, dt_ts, dt_ts_def,
+                            dt_ts_none, dt_utc, format_date, format_ms_time, shorten_date)
 
 
 def test_dt_now():
@@ -29,6 +29,13 @@ def test_dt_ts_def():
     assert dt_ts_def(datetime(2023, 5, 5, tzinfo=timezone.utc), 123) == 1683244800000
 
 
+def test_dt_ts_none():
+    assert dt_ts_none(None) is None
+    assert dt_ts_none(None) is None
+    assert dt_ts_none(datetime(2023, 5, 5, tzinfo=timezone.utc)) == 1683244800000
+    assert dt_ts_none(datetime(2023, 5, 5, tzinfo=timezone.utc)) == 1683244800000
+
+
 def test_dt_utc():
     assert dt_utc(2023, 5, 5) == datetime(2023, 5, 5, tzinfo=timezone.utc)
     assert dt_utc(2023, 5, 5, 0, 0, 0, 555500) == datetime(2023, 5, 5, 0, 0, 0, 555500,

tests/utils/test_periodiccache.py

@@ -5,7 +5,7 @@ from freqtrade.util import PeriodicCache
 
 def test_ttl_cache():
 
-    with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
+    with time_machine.travel("2021-09-01 05:00:00 +00:00", tick=False) as t:
 
         cache = PeriodicCache(5, ttl=60)
         cache1h = PeriodicCache(5, ttl=3600)