Merge branch 'freqtrade:develop' into bt-metrics

.github/dependabot.yml

@@ -10,8 +10,17 @@ updates:
   directory: "/"
   schedule:
     interval: weekly
+    time: "03:00"
+    timezone: "Etc/UTC"
   open-pull-requests-limit: 15
   target-branch: develop
+  groups:
+    types:
+      patterns:
+        - "types-*"
+    pytest:
+      patterns:
+        - "pytest*"
 
 - package-ecosystem: "github-actions"
   directory: "/"

.github/workflows/ci.yml

@@ -124,8 +124,11 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ "macos-latest", "macos-13" ]
+        os: [ "macos-latest", "macos-13", "macos-14" ]
         python-version: ["3.9", "3.10", "3.11", "3.12"]
+        exclude:
+          - os: "macos-14"
+            python-version: "3.9"
 
     steps:
     - uses: actions/checkout@v4
@@ -154,7 +157,7 @@ jobs:
       run: |
         cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd ..
 
-    - name: Installation - macOS
+    - name: Installation - macOS (Brew)
       run: |
         # brew update
         # TODO: Should be the brew upgrade
@@ -177,6 +180,9 @@ jobs:
         rm /usr/local/bin/python3.12-config || true
 
         brew install hdf5 c-blosc libomp
+
+    - name: Installation (python)
+      run: |
         python -m pip install --upgrade pip wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
         export TA_LIBRARY_PATH=${HOME}/dependencies/lib
@@ -325,7 +331,7 @@ jobs:
     - uses: actions/setup-python@v5
       with:
         python-version: "3.10"
-    - uses: pre-commit/action@v3.0.0
+    - uses: pre-commit/action@v3.0.1
 
   docs-check:
     runs-on: ubuntu-22.04
@@ -482,12 +488,12 @@ jobs:
         path: dist
 
     - name: Publish to PyPI (Test)
-      uses: pypa/gh-action-pypi-publish@v1.8.11
+      uses: pypa/gh-action-pypi-publish@v1.8.14
       with:
         repository-url: https://test.pypi.org/legacy/
 
     - name: Publish to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.8.11
+      uses: pypa/gh-action-pypi-publish@v1.8.14
 
 
   deploy-docker:

.github/workflows/pre-commit-update.yml

@@ -30,7 +30,7 @@ jobs:
       - name: Run pre-commit
         run: pre-commit run --all-files
 
-      - uses: peter-evans/create-pull-request@v5
+      - uses: peter-evans/create-pull-request@v6
         with:
           token: ${{ secrets.REPO_SCOPED_TOKEN }}
           add-paths: .pre-commit-config.yaml

.pre-commit-config.yaml

@@ -16,10 +16,10 @@ repos:
         additional_dependencies:
           - types-cachetools==5.3.0.7
           - types-filelock==3.2.7
-          - types-requests==2.31.0.20240125
+          - types-requests==2.31.0.20240311
           - types-tabulate==0.9.0.20240106
-          - types-python-dateutil==2.8.19.20240106
-          - SQLAlchemy==2.0.25
+          - types-python-dateutil==2.8.19.20240311
+          - SQLAlchemy==2.0.27
         # stages: [push]
 
   - repo: https://github.com/pycqa/isort
@@ -31,7 +31,7 @@ repos:
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.1.15'
+    rev: 'v0.3.0'
     hooks:
       - id: ruff
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11.7-slim-bookworm as base
+FROM python:3.11.8-slim-bookworm as base
 
 # Setup env
 ENV LANG C.UTF-8

docker/Dockerfile.armhf

@@ -1,4 +1,4 @@
-FROM python:3.11.7-slim-bookworm as base
+FROM python:3.11.8-slim-bookworm as base
 
 # Setup env
 ENV LANG C.UTF-8

docs/advanced-backtesting.md

@@ -109,12 +109,12 @@ automatically accessible by including them on the indicator-list, and these incl
 - **open_date     :** trade open datetime
 - **close_date    :** trade close datetime
 - **min_rate      :** minimum price seen throughout the position
-- **max_rate      :** maxiumum price seen throughout the position
+- **max_rate      :** maximum price seen throughout the position
 - **open          :** signal candle open price
 - **close         :** signal candle close price
 - **high          :** signal candle high price
 - **low           :** signal candle low price
-- **volume        :** signal candle volumne
+- **volume        :** signal candle volume
 - **profit_ratio  :** trade profit ratio
 - **profit_abs    :** absolute profit return of the trade 
 

docs/configuration.md

@@ -14,7 +14,7 @@ You can specify a different configuration file used by the bot with the `-c/--co
 If you used the [Quick start](docker_quickstart.md#docker-quick-start) method for installing
 the bot, the installation script should have already created the default configuration file (`config.json`) for you.
 
-If the default configuration file is not created we recommend to use `freqtrade new-config --config config.json` to generate a basic configuration file.
+If the default configuration file is not created we recommend to use `freqtrade new-config --config user_data/config.json` to generate a basic configuration file.
 
 The Freqtrade configuration file is to be written in JSON format.
 

docs/freqai-parameter-table.md

@@ -75,7 +75,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the
 | `rl_config` | A dictionary containing the control parameters for a Reinforcement Learning model. <br> **Datatype:** Dictionary.
 | `train_cycles` | Training time steps will be set based on the `train_cycles * number of training data points. <br> **Datatype:** Integer.
 | `max_trade_duration_candles`| Guides the agent training to keep trades below desired length. Example usage shown in `prediction_models/ReinforcementLearner.py` within the customizable `calculate_reward()` function. <br> **Datatype:** int.
-| `model_type` | Model string from stable_baselines3 or SBcontrib. Available strings include: `'TRPO', 'ARS', 'RecurrentPPO', 'MaskablePPO', 'PPO', 'A2C', 'DQN'`. User should ensure that `model_training_parameters` match those available to the corresponding stable_baselines3 model by visiting their documentaiton. [PPO doc](https://stable-baselines3.readthedocs.io/en/master/modules/ppo.html) (external website) <br> **Datatype:** string.
+| `model_type` | Model string from stable_baselines3 or SBcontrib. Available strings include: `'TRPO', 'ARS', 'RecurrentPPO', 'MaskablePPO', 'PPO', 'A2C', 'DQN'`. User should ensure that `model_training_parameters` match those available to the corresponding stable_baselines3 model by visiting their documentation. [PPO doc](https://stable-baselines3.readthedocs.io/en/master/modules/ppo.html) (external website) <br> **Datatype:** string.
 | `policy_type` | One of the available policy types from stable_baselines3 <br> **Datatype:** string.
 | `max_training_drawdown_pct` | The maximum drawdown that the agent is allowed to experience during training. <br> **Datatype:** float. <br> Default: 0.8
 | `cpu_count` | Number of threads/cpus to dedicate to the Reinforcement Learning training process (depending on if `ReinforcementLearning_multiproc` is selected or not). Recommended to leave this untouched, by default, this value is set to the total number of physical cores minus 1. <br> **Datatype:** int. 

docs/freqai-reinforcement-learning.md

@@ -142,7 +142,7 @@ Parameter details can be found [here](freqai-parameter-table.md), but in general
 As you begin to modify the strategy and the prediction model, you will quickly realize some important differences between the Reinforcement Learner and the Regressors/Classifiers. Firstly, the strategy does not set a target value (no labels!). Instead, you set the `calculate_reward()` function inside the `MyRLEnv` class (see below). A default `calculate_reward()` is provided inside `prediction_models/ReinforcementLearner.py` to demonstrate the necessary building blocks for creating rewards, but this is *not* designed for production. Users *must* create their own custom reinforcement learning model class or use a pre-built one from outside the Freqtrade source code and save it to `user_data/freqaimodels`. It is inside the `calculate_reward()` where creative theories about the market can be expressed. For example, you can reward your agent when it makes a winning trade, and penalize the agent when it makes a losing trade. Or perhaps, you wish to reward the agent for entering trades, and penalize the agent for sitting in trades too long. Below we show examples of how these rewards are all calculated:
 
 !!! note "Hint"
-    The best reward functions are ones that are continuously differentiable, and well scaled. In other words, adding a single large negative penalty to a rare event is not a good idea, and the neural net will not be able to learn that function. Instead, it is better to add a small negative penalty to a common event. This will help the agent learn faster. Not only this, but you can help improve the continuity of your rewards/penalties by having them scale with severity according to some linear/exponential functions. In other words, you'd slowly scale the penalty as the duration of the trade increases. This is better than a single large penalty occuring at a single point in time.
+    The best reward functions are ones that are continuously differentiable, and well scaled. In other words, adding a single large negative penalty to a rare event is not a good idea, and the neural net will not be able to learn that function. Instead, it is better to add a small negative penalty to a common event. This will help the agent learn faster. Not only this, but you can help improve the continuity of your rewards/penalties by having them scale with severity according to some linear/exponential functions. In other words, you'd slowly scale the penalty as the duration of the trade increases. This is better than a single large penalty occurring at a single point in time.
 
 ```python
 from freqtrade.freqai.prediction_models.ReinforcementLearner import ReinforcementLearner

docs/includes/pairlists.md

@@ -68,7 +68,7 @@ When used in the leading position of the chain of Pairlist Handlers, the `pair_w
 
 The `refresh_period` setting allows to define the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes).
 The pairlist cache (`refresh_period`) on `VolumePairList` is only applicable to generating pairlists.
-Filtering instances (not the first position in the list) will not apply any cache and will always use up-to-date data.
+Filtering instances (not the first position in the list) will not apply any cache (beyond caching candles for the duration of the candle in advanced mode) and will always use up-to-date data.
 
 `VolumePairList` is per default based on the ticker data from exchange, as reported by the ccxt library:
 
@@ -201,7 +201,7 @@ The RemotePairList is defined in the pairlists section of the configuration sett
 
 The optional `mode` option specifies if the pairlist should be used as a `blacklist` or as a `whitelist`. The default value is "whitelist".
 
-The optional `processing_mode` option in the RemotePairList configuration determines how the retrieved pairlist is processed. It can have two values: "filter" or "append".
+The optional `processing_mode` option in the RemotePairList configuration determines how the retrieved pairlist is processed. It can have two values: "filter" or "append". The default value is "filter".
 
 In "filter" mode, the retrieved pairlist is used as a filter. Only the pairs present in both the original pairlist and the retrieved pairlist are included in the final pairlist. Other pairs are filtered out.
 
@@ -450,6 +450,8 @@ If the trading range over the last 10 days is <1% or >99%, remove the pair from
 ]
 ```
 
+Adding `"sort_direction": "asc"` or `"sort_direction": "desc"` enables sorting for this pairlist.
+
 !!! Tip
     This Filter can be used to automatically remove stable coin pairs, which have a very low trading range, and are therefore extremely difficult to trade with profit.
     Additionally, it can also be used to automatically remove pairs with extreme high/low variance over a given amount of time.
@@ -460,7 +462,7 @@ Volatility is the degree of historical variation of a pairs over time, it is mea
 
 This filter removes pairs if the average volatility over a `lookback_days` days is below `min_volatility` or above `max_volatility`. Since this is a filter that requires additional data, the results are cached for `refresh_period`.
 
-This filter can be used to narrow down your pairs to a certain volatility or avoid very volatile pairs. 
+This filter can be used to narrow down your pairs to a certain volatility or avoid very volatile pairs.
 
 In the below example:
 If the volatility over the last 10 days is not in the range of 0.05-0.50, remove the pair from the whitelist. The filter is applied every 24h.
@@ -477,6 +479,8 @@ If the volatility over the last 10 days is not in the range of 0.05-0.50, remove
 ]
 ```
 
+Adding `"sort_direction": "asc"` or `"sort_direction": "desc"` enables sorting mode for this pairlist.
+
 ### Full example of Pairlist Handlers
 
 The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#pricefilter), filtering all assets where 1 price unit is > 1%. Then the [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) is applied and pairs are finally shuffled with the random seed set to some predefined value.

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
 markdown==3.5.2
 mkdocs==1.5.3
-mkdocs-material==9.5.6
+mkdocs-material==9.5.13
 mdx_truly_sane_lists==1.3
-pymdown-extensions==10.7
+pymdown-extensions==10.7.1
 jinja2==3.1.3

docs/sql_cheatsheet.md

@@ -109,7 +109,7 @@ Freqtrade does not depend or install any additional database driver. Please refe
 The following systems have been tested and are known to work with freqtrade:
 
 * sqlite (default)
-* PostgreSQL)
+* PostgreSQL
 * MariaDB
 
 !!! Warning

docs/strategy-advanced.md

@@ -11,34 +11,129 @@ The call sequence of the methods described here is covered under [bot execution
 !!! Tip
     Start off with a strategy template containing all available callback methods by running `freqtrade new-strategy --strategy MyAwesomeStrategy --template advanced`
 
-## Storing information
+## Storing information (Persistent)
 
-Storing information can be accomplished by creating a new dictionary within the strategy class.
+Freqtrade allows storing/retrieving user custom information associated with a specific trade in the database.
 
-The name of the variable can be chosen at will, but should be prefixed with `custom_` to avoid naming collisions with predefined strategy variables.
+Using a trade object, information can be stored using `trade.set_custom_data(key='my_key', value=my_value)` and retrieved using `trade.get_custom_data(key='my_key')`. Each data entry is associated with a trade and a user supplied key (of type `string`). This means that this can only be used in callbacks that also provide a trade object.
+
+For the data to be able to be stored within the database, freqtrade must serialized the data. This is done by converting the data to a JSON formatted string.
+Freqtrade will attempt to reverse this action on retrieval, so from a strategy perspective, this should not be relevant.
 
 ```python
+from freqtrade.persistence import Trade
+from datetime import timedelta
+
 class AwesomeStrategy(IStrategy):
-    # Create custom dictionary
-    custom_info = {}
 
-    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
-        # Check if the entry already exists
-        if not metadata["pair"] in self.custom_info:
-            # Create empty entry for this pair
-            self.custom_info[metadata["pair"]] = {}
+    def bot_loop_start(self, **kwargs) -> None:
+        for trade in Trade.get_open_order_trades():
+            fills = trade.select_filled_orders(trade.entry_side)
+            if trade.pair == 'ETH/USDT':
+                trade_entry_type = trade.get_custom_data(key='entry_type')
+                if trade_entry_type is None:
+                    trade_entry_type = 'breakout' if 'entry_1' in trade.enter_tag else 'dip'
+                elif fills > 1:
+                    trade_entry_type = 'buy_up'
+                trade.set_custom_data(key='entry_type', value=trade_entry_type)
+        return super().bot_loop_start(**kwargs)
 
-        if "crosstime" in self.custom_info[metadata["pair"]]:
-            self.custom_info[metadata["pair"]]["crosstime"] += 1
-        else:
-            self.custom_info[metadata["pair"]]["crosstime"] = 1
+    def adjust_entry_price(self, trade: Trade, order: Optional[Order], pair: str,
+                           current_time: datetime, proposed_rate: float, current_order_rate: float,
+                           entry_tag: Optional[str], side: str, **kwargs) -> float:
+        # Limit orders to use and follow SMA200 as price target for the first 10 minutes since entry trigger for BTC/USDT pair.
+        if (
+            pair == 'BTC/USDT' 
+            and entry_tag == 'long_sma200' 
+            and side == 'long' 
+            and (current_time - timedelta(minutes=10)) > trade.open_date_utc 
+            and order.filled == 0.0
+        ):
+            dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe)
+            current_candle = dataframe.iloc[-1].squeeze()
+            # store information about entry adjustment
+            existing_count = trade.get_custom_data('num_entry_adjustments', default=0)
+            if not existing_count:
+                existing_count = 1
+            else:
+                existing_count += 1
+            trade.set_custom_data(key='num_entry_adjustments', value=existing_count)
+
+            # adjust order price
+            return current_candle['sma_200']
+
+        # default: maintain existing order
+        return current_order_rate
+
+    def custom_exit(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs):
+
+        entry_adjustment_count = trade.get_custom_data(key='num_entry_adjustments')
+        trade_entry_type = trade.get_custom_data(key='entry_type')
+        if entry_adjustment_count is None:
+            if current_profit > 0.01 and (current_time - timedelta(minutes=100) > trade.open_date_utc):
+                return True, 'exit_1'
+        else
+            if entry_adjustment_count > 0 and if current_profit > 0.05:
+                return True, 'exit_2'
+            if trade_entry_type == 'breakout' and current_profit > 0.1:
+                return True, 'exit_3
+
+        return False, None
 ```
 
-!!! Warning
-    The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash.
+The above is a simple example - there are simpler ways to retrieve trade data like entry-adjustments.
 
 !!! Note
-    If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
+    It is recommended that simple data types are used `[bool, int, float, str]` to ensure no issues when serializing the data that needs to be stored.
+    Storing big junks of data may lead to unintended side-effects, like a database becoming big (and as a consequence, also slow).
+
+!!! Warning "Non-serializable data"
+    If supplied data cannot be serialized a warning is logged and the entry for the specified `key` will contain `None` as data.
+
+??? Note "All attributes"
+    custom-data has the following accessors through the Trade object (assumed as `trade` below):
+
+    * `trade.get_custom_data(key='something', default=0)` - Returns the actual value given in the type provided.
+    * `trade.get_custom_data_entry(key='something')` - Returns the entry - including metadata. The value is accessible via `.value` property.
+    * `trade.set_custom_data(key='something', value={'some': 'value'})` - set or update the corresponding key for this trade. Value must be serializable - and we recommend to keep the stored data relatively small.
+
+    "value" can be any type (both in setting and receiving) - but must be json serializable.
+
+## Storing information (Non-Persistent)
+
+!!! Warning "Deprecated"
+    This method of storing information is deprecated and we do advise against using non-persistent storage.  
+    Please use [Persistent Storage](#storing-information-persistent) instead.
+
+    It's content has therefore been collapsed.
+
+??? Abstract "Storing information"
+    Storing information can be accomplished by creating a new dictionary within the strategy class.
+
+    The name of the variable can be chosen at will, but should be prefixed with `custom_` to avoid naming collisions with predefined strategy variables.
+
+    ```python
+    class AwesomeStrategy(IStrategy):
+        # Create custom dictionary
+        custom_info = {}
+
+        def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+            # Check if the entry already exists
+            if not metadata["pair"] in self.custom_info:
+                # Create empty entry for this pair
+                self.custom_info[metadata["pair"]] = {}
+
+            if "crosstime" in self.custom_info[metadata["pair"]]:
+                self.custom_info[metadata["pair"]]["crosstime"] += 1
+            else:
+                self.custom_info[metadata["pair"]]["crosstime"] = 1
+    ```
+
+    !!! Warning
+        The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash.
+
+    !!! Note
+        If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
 
 ## Dataframe access
 

docs/strategy-callbacks.md

@@ -767,6 +767,7 @@ This callback is **not** called when there is an open order (either buy or sell)
 `adjust_trade_position()` is called very frequently for the duration of a trade, so you must keep your implementation as performant as possible.
 
 Position adjustments will always be applied in the direction of the trade, so a positive value will always increase your position (negative values will decrease your position), no matter if it's a long or short trade.
+Adjustment orders can be assigned with a tag by returning a 2 element Tuple, with the first element being the adjustment amount, and the 2nd element the tag (e.g. `return 250, 'increase_favorable_conditions'`).
 
 Modifications to leverage are not possible, and the stake-amount returned is assumed to be before applying leverage.
 
@@ -790,7 +791,7 @@ Returning a value more than the above (so remaining stake_amount would become ne
     If you wish to buy additional orders with DCA, then make sure to leave enough funds in the wallet for that.
     Using 'unlimited' stake amount with DCA orders requires you to also implement the `custom_stake_amount()` callback to avoid allocating all funds to the initial order.
 
-!!! Warning
+!!! Warning "Stoploss calculation"
     Stoploss is still calculated from the initial opening price, not averaged price.
     Regular stoploss rules still apply (cannot move down).
 
@@ -800,6 +801,11 @@ Returning a value more than the above (so remaining stake_amount would become ne
     During backtesting this callback is called for each candle in `timeframe` or `timeframe_detail`, so run-time performance will be affected.
     This can also cause deviating results between live and backtesting, since backtesting can adjust the trade only once per candle, whereas live could adjust the trade multiple times per candle.
 
+!!! Warning "Performance with many position adjustments"
+    Position adjustments can be a good approach to increase a strategy's output - but it can also have drawbacks if using this feature extensively.  
+    Each of the orders will be attached to the trade object for the duration of the trade - hence increasing memory usage.
+    Trades with long duration and 10s or even 100ds of position adjustments are therefore not recommended, and should be closed at regular intervals to not affect performance.
+
 ``` python
 from freqtrade.persistence import Trade
 
@@ -833,7 +839,8 @@ class DigDeeperStrategy(IStrategy):
                               min_stake: Optional[float], max_stake: float,
                               current_entry_rate: float, current_exit_rate: float,
                               current_entry_profit: float, current_exit_profit: float,
-                              **kwargs) -> Optional[float]:
+                              **kwargs
+                              ) -> Union[Optional[float], Tuple[Optional[float], Optional[str]]]:
         """
         Custom trade adjustment logic, returning the stake amount that a trade should be
         increased or decreased.
@@ -859,11 +866,12 @@ class DigDeeperStrategy(IStrategy):
         :return float: Stake amount to adjust your trade,
                        Positive values to increase position, Negative values to decrease position.
                        Return None for no action.
+                       Optionally, return a tuple with a 2nd element with an order reason
         """
 
         if current_profit > 0.05 and trade.nr_of_successful_exits == 0:
             # Take half of the profit at +5%
-            return -(trade.stake_amount / 2)
+            return -(trade.stake_amount / 2), 'half_profit_5%'
 
         if current_profit > -0.05:
             return None
@@ -891,7 +899,7 @@ class DigDeeperStrategy(IStrategy):
             stake_amount = filled_entries[0].stake_amount
             # This then calculates current safety order size
             stake_amount = stake_amount * (1 + (count_of_entries * 0.25))
-            return stake_amount
+            return stake_amount, '1/3rd_increase'
         except Exception as exception:
             return None
 

docs/strategy_analysis_example.md

@@ -19,7 +19,7 @@ from pathlib import Path
 project_root = "somedir/freqtrade"
 i=0
 try:
-    os.chdirdir(project_root)
+    os.chdir(project_root)
     assert Path('LICENSE').is_file()
 except:
     while i<4 and (not Path('LICENSE').is_file()):

docs/telegram-usage.md

@@ -59,7 +59,7 @@ For the Freqtrade configuration, you can then use the the full value (including
    "chat_id": "-1001332619709"
 ```
 !!! Warning "Using telegram groups"
-    When using telegram groups, you're giving every member of the telegram group access to your freqtrade bot and to all commands possible via telegram. Please make sure that you can trust everyone in the telegram group to avoid unpleasent surprises.
+    When using telegram groups, you're giving every member of the telegram group access to your freqtrade bot and to all commands possible via telegram. Please make sure that you can trust everyone in the telegram group to avoid unpleasant surprises.
 
 ## Control telegram noise
 
@@ -181,6 +181,7 @@ official commands. You can ask at any moment for help with `/help`.
 | `/locks` | Show currently locked pairs.
 | `/unlock <pair or lock_id>` | Remove the lock for this pair (or for this lock id).
 | `/marketdir [long | short | even | none]` | Updates the user managed variable that represents the current market direction. If no direction is provided, the currently set direction will be displayed.
+| `/list_custom_data <trade_id> [key]` | List custom_data for Trade ID & Key combination. If no Key is supplied it will list all key-value pairs found for that Trade ID.
 | **Modify Trade states** |
 | `/forceexit <trade_id> | /fx <tradeid>` | Instantly exits the given trade  (Ignoring `minimum_roi`).
 | `/forceexit all | /fx all` | Instantly exits all open trades (Ignoring `minimum_roi`).

docs/updating.md

@@ -6,7 +6,7 @@ To update your freqtrade installation, please use one of the below methods, corr
     Breaking changes / changed behavior will be documented in the changelog that is posted alongside every release.
     For the develop branch, please follow PR's to avoid being surprised by changes.
 
-## docker
+## Docker
 
 !!! Note "Legacy installations using the `master` image"
     We're switching from master to stable for the release Images - please adjust your docker-file and replace `freqtradeorg/freqtrade:master` with `freqtradeorg/freqtrade:stable`

docs/utils.md

@@ -54,7 +54,7 @@ optional arguments:
 ### Create config examples
 
 ```
-$ freqtrade new-config --config config_binance.json
+$ freqtrade new-config --config user_data/config_binance.json
 
 ? Do you want to enable Dry-run (simulated trades)?  Yes
 ? Please insert your stake currency: BTC
@@ -219,207 +219,49 @@ optional arguments:
   -a, --all         Print all exchanges known to the ccxt library.
 ```
 
-* Example: see exchanges available for the bot:
+Example: see exchanges available for the bot:
+
 ```
 $ freqtrade list-exchanges
 Exchanges available for Freqtrade:
-Exchange name    Valid    reason
----------------  -------  --------------------------------------------
-aax              True
-ascendex         True     missing opt: fetchMyTrades
-bequant          True
-bibox            True
-bigone           True
-binance          True
-binanceus        True
-bitbank          True     missing opt: fetchTickers
-bitcoincom       True
-bitfinex         True
-bitforex         True     missing opt: fetchMyTrades, fetchTickers
-bitget           True
-bithumb          True     missing opt: fetchMyTrades
-bitkk            True     missing opt: fetchMyTrades
-bitmart          True
-bitmax           True     missing opt: fetchMyTrades
-bitpanda         True
-bitvavo          True
-bitz             True     missing opt: fetchMyTrades
-btcalpha         True     missing opt: fetchTicker, fetchTickers
-btcmarkets       True     missing opt: fetchTickers
-buda             True     missing opt: fetchMyTrades, fetchTickers
-bw               True     missing opt: fetchMyTrades, fetchL2OrderBook
-bybit            True
-bytetrade        True
-cdax             True
-cex              True     missing opt: fetchMyTrades
-coinbaseprime    True     missing opt: fetchTickers
-coinbasepro      True     missing opt: fetchTickers
-coinex           True
-crex24           True
-deribit          True
-digifinex        True
-equos            True     missing opt: fetchTicker, fetchTickers
-eterbase         True
-fcoin            True     missing opt: fetchMyTrades, fetchTickers
-fcoinjp          True     missing opt: fetchMyTrades, fetchTickers
-gateio           True
-gemini           True
-gopax            True
-hbtc             True
-hitbtc           True
-huobijp          True
-huobipro         True
-idex             True
-kraken           True
-kucoin           True
-lbank            True     missing opt: fetchMyTrades
-mercado          True     missing opt: fetchTickers
-ndax             True     missing opt: fetchTickers
-novadax          True
-okcoin           True
-okex             True
-probit           True
-qtrade           True
-stex             True
-timex            True
-upbit            True     missing opt: fetchMyTrades
-vcc              True
-zb               True     missing opt: fetchMyTrades
-
+Exchange name       Supported    Markets                 Reason
+------------------  -----------  ----------------------  ------------------------------------------------------------------------
+binance             Official     spot, isolated futures
+bitmart             Official     spot
+bybit                            spot, isolated futures
+gate                Official     spot, isolated futures
+htx                 Official     spot
+huobi                            spot
+kraken              Official     spot
+okx                 Official     spot, isolated futures
 ```
 
+!!! info ""
+    Output reduced for clarity - supported and available exchanges may change over time.
+
 !!! Note "missing opt exchanges"
     Values with "missing opt:" might need special configuration (e.g. using orderbook if `fetchTickers` is missing) - but should in theory work (although we cannot guarantee they will).
 
-* Example: see all exchanges supported by the ccxt library (including 'bad' ones, i.e. those that are known to not work with Freqtrade):
+Example: see all exchanges supported by the ccxt library (including 'bad' ones, i.e. those that are known to not work with Freqtrade)
+
 ```
 $ freqtrade list-exchanges -a
 All exchanges supported by the ccxt library:
-Exchange name       Valid    reason
-------------------  -------  ---------------------------------------------------------------------------------------
-aax                 True
-aofex               False    missing: fetchOrder
-ascendex            True     missing opt: fetchMyTrades
-bequant             True
-bibox               True
-bigone              True
-binance             True
-binanceus           True
-bit2c               False    missing: fetchOrder, fetchOHLCV
-bitbank             True     missing opt: fetchTickers
-bitbay              False    missing: fetchOrder
-bitcoincom          True
-bitfinex            True
-bitfinex2           False    missing: fetchOrder
-bitflyer            False    missing: fetchOrder, fetchOHLCV
-bitforex            True     missing opt: fetchMyTrades, fetchTickers
-bitget              True
-bithumb             True     missing opt: fetchMyTrades
-bitkk               True     missing opt: fetchMyTrades
-bitmart             True
-bitmax              True     missing opt: fetchMyTrades
-bitmex              False    Various reasons.
-bitpanda            True
-bitso               False    missing: fetchOHLCV
-bitstamp            True     missing opt: fetchTickers
-bitstamp1           False    missing: fetchOrder, fetchOHLCV
-bitvavo             True
-bitz                True     missing opt: fetchMyTrades
-bl3p                False    missing: fetchOrder, fetchOHLCV
-bleutrade           False    missing: fetchOrder
-braziliex           False    missing: fetchOHLCV
-btcalpha            True     missing opt: fetchTicker, fetchTickers
-btcbox              False    missing: fetchOHLCV
-btcmarkets          True     missing opt: fetchTickers
-btctradeua          False    missing: fetchOrder, fetchOHLCV
-btcturk             False    missing: fetchOrder
-buda                True     missing opt: fetchMyTrades, fetchTickers
-bw                  True     missing opt: fetchMyTrades, fetchL2OrderBook
-bybit               True
-bytetrade           True
-cdax                True
-cex                 True     missing opt: fetchMyTrades
-chilebit            False    missing: fetchOrder, fetchOHLCV
-coinbase            False    missing: fetchOrder, cancelOrder, createOrder, fetchOHLCV
-coinbaseprime       True     missing opt: fetchTickers
-coinbasepro         True     missing opt: fetchTickers
-coincheck           False    missing: fetchOrder, fetchOHLCV
-coinegg             False    missing: fetchOHLCV
-coinex              True
-coinfalcon          False    missing: fetchOHLCV
-coinfloor           False    missing: fetchOrder, fetchOHLCV
-coingi              False    missing: fetchOrder, fetchOHLCV
-coinmarketcap       False    missing: fetchOrder, cancelOrder, createOrder, fetchBalance, fetchOHLCV
-coinmate            False    missing: fetchOHLCV
-coinone             False    missing: fetchOHLCV
-coinspot            False    missing: fetchOrder, cancelOrder, fetchOHLCV
-crex24              True
-currencycom         False    missing: fetchOrder
-delta               False    missing: fetchOrder
-deribit             True
-digifinex           True
-equos               True     missing opt: fetchTicker, fetchTickers
-eterbase            True
-exmo                False    missing: fetchOrder
-exx                 False    missing: fetchOHLCV
-fcoin               True     missing opt: fetchMyTrades, fetchTickers
-fcoinjp             True     missing opt: fetchMyTrades, fetchTickers
-flowbtc             False    missing: fetchOrder, fetchOHLCV
-foxbit              False    missing: fetchOrder, fetchOHLCV
-gateio              True
-gemini              True
-gopax               True
-hbtc                True
-hitbtc              True
-hollaex             False    missing: fetchOrder
-huobijp             True
-huobipro            True
-idex                True
-independentreserve  False    missing: fetchOHLCV
-indodax             False    missing: fetchOHLCV
-itbit               False    missing: fetchOHLCV
-kraken              True
-kucoin              True
-kuna                False    missing: fetchOHLCV
-lakebtc             False    missing: fetchOrder, fetchOHLCV
-latoken             False    missing: fetchOrder, fetchOHLCV
-lbank               True     missing opt: fetchMyTrades
-liquid              False    missing: fetchOHLCV
-luno                False    missing: fetchOHLCV
-lykke               False    missing: fetchOHLCV
-mercado             True     missing opt: fetchTickers
-mixcoins            False    missing: fetchOrder, fetchOHLCV
-ndax                True     missing opt: fetchTickers
-novadax             True
-oceanex             False    missing: fetchOHLCV
-okcoin              True
-okex                True
-paymium             False    missing: fetchOrder, fetchOHLCV
-phemex              False    Does not provide history.
-poloniex            False    missing: fetchOrder
-probit              True
-qtrade              True
-rightbtc            False    missing: fetchOrder
-ripio               False    missing: fetchOHLCV
-southxchange        False    missing: fetchOrder, fetchOHLCV
-stex                True
-surbitcoin          False    missing: fetchOrder, fetchOHLCV
-therock             False    missing: fetchOHLCV
-tidebit             False    missing: fetchOrder
-tidex               False    missing: fetchOHLCV
-timex               True
-upbit               True     missing opt: fetchMyTrades
-vbtc                False    missing: fetchOrder, fetchOHLCV
-vcc                 True
-wavesexchange       False    missing: fetchOrder
-whitebit            False    missing: fetchOrder, cancelOrder, createOrder, fetchBalance
-xbtce               False    missing: fetchOrder, fetchOHLCV
-xena                False    missing: fetchOrder
-yobit               False    missing: fetchOHLCV
-zaif                False    missing: fetchOrder, fetchOHLCV
-zb                  True     missing opt: fetchMyTrades
+Exchange name       Valid    Supported    Markets                 Reason
+------------------  -------  -----------  ----------------------  ---------------------------------------------------------------------------------
+binance             True     Official     spot, isolated futures
+bitflyer            False                 spot                    missing: fetchOrder. missing opt: fetchTickers.
+bitmart             True     Official     spot
+bybit               True                  spot, isolated futures
+gate                True     Official     spot, isolated futures
+htx                 True     Official     spot
+kraken              True     Official     spot
+okx                 True     Official     spot, isolated futures
 ```
 
+!!! info ""
+    Reduced output - supported and available exchanges may change over time.
+
 ## List Timeframes
 
 Use the `list-timeframes` subcommand to see the list of timeframes available for the exchange.
@@ -990,11 +832,7 @@ options:
   -h, --help            show this help message and exit
   --strategy-list STRATEGY_LIST [STRATEGY_LIST ...]
                         Provide a space-separated list of strategies to
-                        backtest. Please note that timeframe needs to be set
-                        either in config or via command line. When using this
-                        together with `--export trades`, the strategy-name is
-                        injected into the filename (so `backtest-data.json`
-                        becomes `backtest-data-SampleStrategy.json`
+                        be converted.
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).

docs/webhook-config.md

@@ -65,7 +65,7 @@ You can set the POST body format to Form-Encoded (default), JSON-Encoded, or raw
 
 The result would be a POST request with e.g. `{"text":"Status: running"}` body and `Content-Type: application/json` header which results `Status: running` message in the Mattermost channel.
 
-When using the Form-Encoded or JSON-Encoded configuration you can configure any number of payload values, and both the key and value will be ouput in the POST request. However, when using the raw data format you can only configure one value and it **must** be named `"data"`. In this instance the data key will not be output in the POST request, only the value. For example:
+When using the Form-Encoded or JSON-Encoded configuration you can configure any number of payload values, and both the key and value will be output in the POST request. However, when using the raw data format you can only configure one value and it **must** be named `"data"`. In this instance the data key will not be output in the POST request, only the value. For example:
 
 ```json
   "webhook": {

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ Freqtrade bot """
-__version__ = '2024.2-dev'
+__version__ = '2024.3-dev'
 
 if 'dev' in __version__:
     from pathlib import Path

freqtrade/commands/arguments.py

@@ -69,7 +69,8 @@ ARGS_CONVERT_DATA_TRADES = ["pairs", "format_from_trades", "format_to", "erase",
 ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase", "exchange"]
 ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes", "trading_mode", "candle_types"]
 
-ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"]
+ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades",
+                       "trading_mode"]
 
 ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode", "show_timerange"]
 

freqtrade/commands/data_commands.py

@@ -8,9 +8,10 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, DL_DATA_TIMEFRAMES, Confi
 from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_format,
                                       convert_trades_to_ohlcv)
 from freqtrade.data.history import download_data_main
-from freqtrade.enums import RunMode, TradingMode
+from freqtrade.enums import CandleType, RunMode, TradingMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes
+from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
 from freqtrade.util.migrations import migrate_data
 
@@ -62,13 +63,21 @@ def start_convert_trades(args: Dict[str, Any]) -> None:
 
     for timeframe in config['timeframes']:
         exchange.validate_timeframes(timeframe)
+    available_pairs = [
+        p for p in exchange.get_markets(
+            tradable_only=True, active_only=not config.get('include_inactive')
+            ).keys()
+    ]
+
+    expanded_pairs = dynamic_expand_pairlist(config, available_pairs)
 
     # Convert downloaded trade data to different timeframes
     convert_trades_to_ohlcv(
-        pairs=config.get('pairs', []), timeframes=config['timeframes'],
+        pairs=expanded_pairs, timeframes=config['timeframes'],
         datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
         data_format_ohlcv=config['dataformat_ohlcv'],
         data_format_trades=config['dataformat_trades'],
+        candle_type=config.get('candle_type_def', CandleType.SPOT)
     )
 
 

freqtrade/data/converter/trade_converter.py

@@ -11,7 +11,7 @@ from pandas import DataFrame, to_datetime
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TRADES_DTYPES,
                                  Config, TradeList)
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 from freqtrade.exceptions import OperationalException
 
 
@@ -88,10 +88,10 @@ def convert_trades_to_ohlcv(
     timeframes: List[str],
     datadir: Path,
     timerange: TimeRange,
-    erase: bool = False,
-    data_format_ohlcv: str = 'feather',
-    data_format_trades: str = 'feather',
-    candle_type: CandleType = CandleType.SPOT
+    erase: bool,
+    data_format_ohlcv: str,
+    data_format_trades: str,
+    candle_type: CandleType,
 ) -> None:
     """
     Convert stored trades data to ohlcv data
@@ -99,14 +99,12 @@ def convert_trades_to_ohlcv(
     from freqtrade.data.history.idatahandler import get_datahandler
     data_handler_trades = get_datahandler(datadir, data_format=data_format_trades)
     data_handler_ohlcv = get_datahandler(datadir, data_format=data_format_ohlcv)
-    if not pairs:
-        pairs = data_handler_trades.trades_get_pairs(datadir)
 
     logger.info(f"About to convert pairs: '{', '.join(pairs)}', "
                 f"intervals: '{', '.join(timeframes)}' to {datadir}")
-
+    trading_mode = TradingMode.FUTURES if candle_type != CandleType.SPOT else TradingMode.SPOT
     for pair in pairs:
-        trades = data_handler_trades.trades_load(pair)
+        trades = data_handler_trades.trades_load(pair, trading_mode)
         for timeframe in timeframes:
             if erase:
                 if data_handler_ohlcv.ohlcv_purge(pair, timeframe, candle_type=candle_type):
@@ -116,7 +114,7 @@ def convert_trades_to_ohlcv(
                 # Store ohlcv
                 data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv, candle_type=candle_type)
             except ValueError:
-                logger.exception(f'Could not convert {pair} to OHLCV.')
+                logger.warning(f'Could not convert {pair} to OHLCV.')
 
 
 def convert_trades_format(config: Config, convert_from: str, convert_to: str, erase: bool):
@@ -144,11 +142,12 @@ def convert_trades_format(config: Config, convert_from: str, convert_to: str, er
     if 'pairs' not in config:
         config['pairs'] = src.trades_get_pairs(config['datadir'])
     logger.info(f"Converting trades for {config['pairs']}")
-
+    trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT)
     for pair in config['pairs']:
-        data = src.trades_load(pair=pair)
+        data = src.trades_load(pair, trading_mode)
         logger.info(f"Converting {len(data)} trades for {pair}")
-        trg.trades_store(pair, data)
+        trg.trades_store(pair, data, trading_mode)
+
         if erase and convert_from != convert_to:
             logger.info(f"Deleting source Trade data for {pair}.")
-            src.trades_purge(pair=pair)
+            src.trades_purge(pair, trading_mode)

freqtrade/data/converter/trade_converter_kraken.py

@@ -7,7 +7,9 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, DEFAULT_TRADES_COLUMNS, C
 from freqtrade.data.converter.trade_converter import (trades_convert_types,
                                                       trades_df_remove_duplicates)
 from freqtrade.data.history.idatahandler import get_datahandler
+from freqtrade.enums import TradingMode
 from freqtrade.exceptions import OperationalException
+from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
 
 
@@ -38,12 +40,22 @@ def import_kraken_trades_from_csv(config: Config, convert_to: str):
     }
     logger.info(f"Found csv files for {', '.join(data_symbols)}.")
 
+    if pairs_raw := config.get('pairs'):
+        pairs = expand_pairlist(pairs_raw, [m[0] for m in markets])
+        markets = {m for m in markets if m[0] in pairs}
+        if not markets:
+            logger.info(f"No data found for pairs {', '.join(pairs_raw)}.")
+            return
+    logger.info(f"Converting pairs: {', '.join(m[0] for m in markets)}.")
+
     for pair, name in markets:
+        logger.debug(f"Converting pair {pair}, files */{name}.csv")
         dfs = []
         # Load and combine all csv files for this pair
         for f in tradesdir.rglob(f"{name}.csv"):
             df = pd.read_csv(f, names=KRAKEN_CSV_TRADE_COLUMNS)
-            dfs.append(df)
+            if not df.empty:
+                dfs.append(df)
 
         # Load existing trades data
         if not dfs:
@@ -52,19 +64,20 @@ def import_kraken_trades_from_csv(config: Config, convert_to: str):
             continue
 
         trades = pd.concat(dfs, ignore_index=True)
+        del dfs
 
         trades.loc[:, 'timestamp'] = trades['timestamp'] * 1e3
         trades.loc[:, 'cost'] = trades['price'] * trades['amount']
         for col in DEFAULT_TRADES_COLUMNS:
             if col not in trades.columns:
-                trades[col] = ''
-
+                trades.loc[:, col] = ''
         trades = trades[DEFAULT_TRADES_COLUMNS]
         trades = trades_convert_types(trades)
 
         trades_df = trades_df_remove_duplicates(trades)
+        del trades
         logger.info(f"{pair}: {len(trades_df)} trades, from "
                     f"{trades_df['date'].min():{DATETIME_PRINT_FORMAT}} to "
                     f"{trades_df['date'].max():{DATETIME_PRINT_FORMAT}}")
 
-        data_handler.trades_store(pair, trades_df)
+        data_handler.trades_store(pair, trades_df, TradingMode.SPOT)

freqtrade/data/history/featherdatahandler.py

@@ -5,7 +5,7 @@ from pandas import DataFrame, read_feather, to_datetime
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 
 from .idatahandler import IDataHandler
 
@@ -82,14 +82,15 @@ class FeatherDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_store(self, pair: str, data: DataFrame) -> None:
+    def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
         :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
+        :param trading_mode: Trading mode to use (used to determine the filename)
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         self.create_dir_if_needed(filename)
         data.reset_index(drop=True).to_feather(filename, compression_level=9, compression='lz4')
 
@@ -102,15 +103,18 @@ class FeatherDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
+    def _trades_load(
+            self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
+    ) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         # TODO: respect timerange ...
         :param pair: Load trades for this pair
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :param timerange: Timerange to load trades for - currently not implemented
         :return: Dataframe containing trades
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         if not filename.exists():
             return DataFrame(columns=DEFAULT_TRADES_COLUMNS)
 

freqtrade/data/history/hdf5datahandler.py

@@ -6,7 +6,7 @@ import pandas as pd
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 
 from .idatahandler import IDataHandler
 
@@ -35,7 +35,7 @@ class HDF5DataHandler(IDataHandler):
         self.create_dir_if_needed(filename)
 
         _data.loc[:, self._columns].to_hdf(
-            filename, key, mode='a', complevel=9, complib='blosc',
+            filename, key=key, mode='a', complevel=9, complib='blosc',
             format='table', data_columns=['date']
         )
 
@@ -100,17 +100,18 @@ class HDF5DataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_store(self, pair: str, data: pd.DataFrame) -> None:
+    def _trades_store(self, pair: str, data: pd.DataFrame, trading_mode: TradingMode) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
         :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
+        :param trading_mode: Trading mode to use (used to determine the filename)
         """
         key = self._pair_trades_key(pair)
 
         data.to_hdf(
-            self._pair_trades_filename(self._datadir, pair), key,
+            self._pair_trades_filename(self._datadir, pair, trading_mode), key=key,
             mode='a', complevel=9, complib='blosc',
             format='table', data_columns=['timestamp']
         )
@@ -124,15 +125,18 @@ class HDF5DataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> pd.DataFrame:
+    def _trades_load(
+        self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
+    ) -> pd.DataFrame:
         """
         Load a pair from h5 file.
         :param pair: Load trades for this pair
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :param timerange: Timerange to load trades for - currently not implemented
         :return: Dataframe containing trades
         """
         key = self._pair_trades_key(pair)
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
 
         if not filename.exists():
             return pd.DataFrame(columns=DEFAULT_TRADES_COLUMNS)

freqtrade/data/history/history_utils.py

@@ -13,7 +13,7 @@ from freqtrade.data.converter import (clean_ohlcv_dataframe, convert_trades_to_o
                                       ohlcv_to_dataframe, trades_df_remove_duplicates,
                                       trades_list_to_df)
 from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import Exchange
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
@@ -333,7 +333,8 @@ def _download_trades_history(exchange: Exchange,
                              pair: str, *,
                              new_pairs_days: int = 30,
                              timerange: Optional[TimeRange] = None,
-                             data_handler: IDataHandler
+                             data_handler: IDataHandler,
+                             trading_mode: TradingMode,
                              ) -> bool:
     """
     Download trade history from the exchange.
@@ -349,7 +350,7 @@ def _download_trades_history(exchange: Exchange,
             if timerange.stoptype == 'date':
                 until = timerange.stopts * 1000
 
-        trades = data_handler.trades_load(pair)
+        trades = data_handler.trades_load(pair, trading_mode)
 
         # TradesList columns are defined in constants.DEFAULT_TRADES_COLUMNS
         # DEFAULT_TRADES_COLUMNS: 0 -> timestamp
@@ -388,7 +389,7 @@ def _download_trades_history(exchange: Exchange,
         trades = concat([trades, new_trades_df], axis=0)
         # Remove duplicates to make sure we're not storing data we don't need
         trades = trades_df_remove_duplicates(trades)
-        data_handler.trades_store(pair, data=trades)
+        data_handler.trades_store(pair, trades, trading_mode)
 
         logger.debug("New Start: %s", 'None' if trades.empty else
                      f"{trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}")
@@ -405,8 +406,10 @@ def _download_trades_history(exchange: Exchange,
 
 
 def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
-                                 timerange: TimeRange, new_pairs_days: int = 30,
-                                 erase: bool = False, data_format: str = 'feather') -> List[str]:
+                                 timerange: TimeRange, trading_mode: TradingMode,
+                                 new_pairs_days: int = 30,
+                                 erase: bool = False, data_format: str = 'feather',
+                                 ) -> List[str]:
     """
     Refresh stored trades data for backtesting and hyperopt operations.
     Used by freqtrade download-data subcommand.
@@ -421,7 +424,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
             continue
 
         if erase:
-            if data_handler.trades_purge(pair):
+            if data_handler.trades_purge(pair, trading_mode):
                 logger.info(f'Deleting existing data for pair {pair}.')
 
         logger.info(f'Downloading trades for pair {pair}.')
@@ -429,7 +432,8 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
                                  pair=pair,
                                  new_pairs_days=new_pairs_days,
                                  timerange=timerange,
-                                 data_handler=data_handler)
+                                 data_handler=data_handler,
+                                 trading_mode=trading_mode)
     return pairs_not_available
 
 
@@ -516,12 +520,12 @@ def download_data_main(config: Config) -> None:
     # Start downloading
     try:
         if config.get('download_trades'):
-            if config.get('trading_mode') == 'futures':
-                raise OperationalException("Trade download not supported for futures.")
             pairs_not_available = refresh_backtest_trades_data(
                 exchange, pairs=expanded_pairs, datadir=config['datadir'],
                 timerange=timerange, new_pairs_days=config['new_pairs_days'],
-                erase=bool(config.get('erase')), data_format=config['dataformat_trades'])
+                erase=bool(config.get('erase')), data_format=config['dataformat_trades'],
+                trading_mode=config.get('trading_mode', TradingMode.SPOT),
+                )
 
             # Convert downloaded trade data to different timeframes
             convert_trades_to_ohlcv(
@@ -529,6 +533,7 @@ def download_data_main(config: Config) -> None:
                 datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
                 data_format_ohlcv=config['dataformat_ohlcv'],
                 data_format_trades=config['dataformat_trades'],
+                candle_type=config.get('candle_type_def', CandleType.SPOT),
             )
         else:
             if not exchange.get_option('ohlcv_has_history', True):

freqtrade/data/history/idatahandler.py

@@ -172,12 +172,13 @@ class IDataHandler(ABC):
         return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match]
 
     @abstractmethod
-    def _trades_store(self, pair: str, data: DataFrame) -> None:
+    def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
         :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
+        :param trading_mode: Trading mode to use (used to determine the filename)
         """
 
     @abstractmethod
@@ -190,45 +191,55 @@ class IDataHandler(ABC):
         """
 
     @abstractmethod
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
+    def _trades_load(
+        self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
+    ) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         :param pair: Load trades for this pair
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :param timerange: Timerange to load trades for - currently not implemented
         :return: Dataframe containing trades
         """
 
-    def trades_store(self, pair: str, data: DataFrame) -> None:
+    def trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
         :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
+        :param trading_mode: Trading mode to use (used to determine the filename)
         """
         # Filter on expected columns (will remove the actual date column).
-        self._trades_store(pair, data[DEFAULT_TRADES_COLUMNS])
+        self._trades_store(pair, data[DEFAULT_TRADES_COLUMNS], trading_mode)
 
-    def trades_purge(self, pair: str) -> bool:
+    def trades_purge(self, pair: str, trading_mode: TradingMode) -> bool:
         """
         Remove data for this pair
         :param pair: Delete data for this pair.
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :return: True when deleted, false if file did not exist.
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         if filename.exists():
             filename.unlink()
             return True
         return False
 
-    def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
+    def trades_load(
+            self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
+    ) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         Removes duplicates in the process.
         :param pair: Load trades for this pair
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :param timerange: Timerange to load trades for - currently not implemented
         :return: List of trades
         """
-        trades = trades_df_remove_duplicates(self._trades_load(pair, timerange=timerange))
+        trades = trades_df_remove_duplicates(
+            self._trades_load(pair, trading_mode, timerange=timerange)
+        )
 
         trades = trades_convert_types(trades)
         return trades
@@ -264,8 +275,12 @@ class IDataHandler(ABC):
         return filename
 
     @classmethod
-    def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path:
+    def _pair_trades_filename(cls, datadir: Path, pair: str, trading_mode: TradingMode) -> Path:
         pair_s = misc.pair_to_filename(pair)
+        if trading_mode == TradingMode.FUTURES:
+            # Futures pair ...
+            datadir = datadir.joinpath('futures')
+
         filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}')
         return filename
 

freqtrade/data/history/jsondatahandler.py

@@ -8,7 +8,7 @@ from freqtrade import misc
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
 from freqtrade.data.converter import trades_dict_to_list, trades_list_to_df
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 
 from .idatahandler import IDataHandler
 
@@ -37,7 +37,7 @@ class JsonDataHandler(IDataHandler):
         self.create_dir_if_needed(filename)
         _data = data.copy()
         # Convert date to int
-        _data['date'] = _data['date'].view(np.int64) // 1000 // 1000
+        _data['date'] = _data['date'].astype(np.int64) // 1000 // 1000
 
         # Reset index, select only appropriate columns and save as json
         _data.reset_index(drop=True).loc[:, self._columns].to_json(
@@ -94,14 +94,15 @@ class JsonDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_store(self, pair: str, data: DataFrame) -> None:
+    def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
         :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
+        :param trading_mode: Trading mode to use (used to determine the filename)
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         trades = data.values.tolist()
         misc.file_dump_json(filename, trades, is_zip=self._use_zip)
 
@@ -114,15 +115,18 @@ class JsonDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
+    def _trades_load(
+        self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
+    ) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         # TODO: respect timerange ...
         :param pair: Load trades for this pair
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :param timerange: Timerange to load trades for - currently not implemented
         :return: Dataframe containing trades
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         tradesdata = misc.file_load_json(filename)
 
         if not tradesdata:

freqtrade/data/history/parquetdatahandler.py

@@ -4,8 +4,8 @@ from typing import Optional
 from pandas import DataFrame, read_parquet, to_datetime
 
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList
-from freqtrade.enums import CandleType
+from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
+from freqtrade.enums import CandleType, TradingMode
 
 from .idatahandler import IDataHandler
 
@@ -81,14 +81,15 @@ class ParquetDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_store(self, pair: str, data: DataFrame) -> None:
+    def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
         :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
+        :param trading_mode: Trading mode to use (used to determine the filename)
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         self.create_dir_if_needed(filename)
         data.reset_index(drop=True).to_parquet(filename)
 
@@ -101,15 +102,18 @@ class ParquetDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+    def _trades_load(
+        self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
+    ) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         # TODO: respect timerange ...
         :param pair: Load trades for this pair
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :param timerange: Timerange to load trades for - currently not implemented
         :return: List of trades
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         if not filename.exists():
             return DataFrame(columns=DEFAULT_TRADES_COLUMNS)
 

freqtrade/data/metrics.py

@@ -143,8 +143,10 @@ def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date'
         starting_balance=starting_balance
     )
 
-    idxmin = max_drawdown_df['drawdown_relative'].idxmax() if relative \
-        else max_drawdown_df['drawdown'].idxmin()
+    idxmin = (
+        max_drawdown_df['drawdown_relative'].idxmax()
+        if relative else max_drawdown_df['drawdown'].idxmin()
+    )
     if idxmin == 0:
         raise ValueError("No losing trade, therefore no drawdown.")
     high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col]
@@ -191,6 +193,9 @@ def calculate_cagr(days_passed: int, starting_balance: float, final_balance: flo
     :param final_balance: Final balance to calculate CAGR against
     :return: CAGR
     """
+    if final_balance < 0:
+        # With leveraged trades, final_balance can become negative.
+        return 0
     return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1
 
 

freqtrade/exchange/bybit.py

@@ -25,6 +25,7 @@ class Bybit(Exchange):
     officially supported by the Freqtrade development team. So some features
     may still not work as expected.
     """
+    unified_account = False
 
     _ft_has: Dict = {
         "ohlcv_candle_limit": 1000,
@@ -82,9 +83,20 @@ class Bybit(Exchange):
         Must be overridden in child methods if required.
         """
         try:
-            if self.trading_mode == TradingMode.FUTURES and not self._config['dry_run']:
-                position_mode = self._api.set_position_mode(False)
-                self._log_exchange_response('set_position_mode', position_mode)
+            if not self._config['dry_run']:
+                if self.trading_mode == TradingMode.FUTURES:
+                    position_mode = self._api.set_position_mode(False)
+                    self._log_exchange_response('set_position_mode', position_mode)
+                is_unified = self._api.is_unified_enabled()
+                # Returns a tuple of bools, first for margin, second for Account
+                if is_unified and len(is_unified) > 1 and is_unified[1]:
+                    self.unified_account = True
+                    logger.info("Bybit: Unified account.")
+                    raise OperationalException("Bybit: Unified account is not supported. "
+                                               "Please use a standard (sub)account.")
+                else:
+                    self.unified_account = False
+                    logger.info("Bybit: Standard account.")
         except ccxt.DDoSProtection as e:
             raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:

freqtrade/exchange/common.py

@@ -2,7 +2,7 @@ import asyncio
 import logging
 import time
 from functools import wraps
-from typing import Any, Callable, Optional, TypeVar, cast, overload
+from typing import Any, Callable, Dict, List, Optional, TypeVar, cast, overload
 
 from freqtrade.constants import ExchangeConfig
 from freqtrade.exceptions import DDosProtection, RetryableOrderError, TemporaryError
@@ -60,16 +60,17 @@ SUPPORTED_EXCHANGES = [
     'okx',
 ]
 
-EXCHANGE_HAS_REQUIRED = [
+# either the main, or replacement methods (array) is required
+EXCHANGE_HAS_REQUIRED: Dict[str, List[str]] = {
     # Required / private
-    'fetchOrder',
-    'cancelOrder',
-    'createOrder',
-    'fetchBalance',
+    'fetchOrder': ['fetchOpenOrder', 'fetchClosedOrder'],
+    'cancelOrder': [],
+    'createOrder': [],
+    'fetchBalance': [],
 
     # Public endpoints
-    'fetchOHLCV',
-]
+    'fetchOHLCV': [],
+}
 
 EXCHANGE_HAS_OPTIONAL = [
     # Private
@@ -86,6 +87,7 @@ EXCHANGE_HAS_OPTIONAL = [
     # 'fetchPositions',  # Futures trading
     # 'fetchLeverageTiers',  # Futures initialization
     # 'fetchMarketLeverageTiers',  # Futures initialization
+    # 'fetchOpenOrder', 'fetchClosedOrder',  # replacement for fetchOrder
     # 'fetchOpenOrders', 'fetchClosedOrders',  # 'fetchOrders',  # Refinding balance...
 ]
 

freqtrade/exchange/exchange.py

@@ -8,7 +8,7 @@ import logging
 import signal
 from copy import deepcopy
 from datetime import datetime, timedelta, timezone
-from math import floor
+from math import floor, isnan
 from threading import Lock
 from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union
 
@@ -23,7 +23,7 @@ from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHAN
                                  BuySell, Config, EntryExit, ExchangeConfig,
                                  ListPairsWithTimeframes, MakerTaker, OBLiteral, PairWithTimeframe)
 from freqtrade.data.converter import clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_dict_to_list
-from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, PriceType, TradingMode
+from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, PriceType, RunMode, TradingMode
 from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError,
                                   InvalidOrderException, OperationalException, PricingError,
                                   RetryableOrderError, TemporaryError)
@@ -43,6 +43,7 @@ from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.util import dt_from_ts, dt_now
 from freqtrade.util.datetime_helpers import dt_humanize, dt_ts
+from freqtrade.util.periodic_cache import PeriodicCache
 
 
 logger = logging.getLogger(__name__)
@@ -131,6 +132,7 @@ class Exchange:
 
         # Holds candles
         self._klines: Dict[PairWithTimeframe, DataFrame] = {}
+        self._expiring_candle_cache: Dict[Tuple[str, int], PeriodicCache] = {}
 
         # Holds all open sell orders for dry_run
         self._dry_run_open_orders: Dict[str, Any] = {}
@@ -595,7 +597,11 @@ class Exchange:
             raise OperationalException(
                 f"Invalid timeframe '{timeframe}'. This exchange supports: {self.timeframes}")
 
-        if timeframe and timeframe_to_minutes(timeframe) < 1:
+        if (
+            timeframe
+            and self._config['runmode'] != RunMode.UTIL_EXCHANGE
+            and timeframe_to_minutes(timeframe) < 1
+        ):
             raise OperationalException("Timeframes < 1m are currently not supported by Freqtrade.")
 
     def validate_ordertypes(self, order_types: Dict) -> None:
@@ -653,7 +659,7 @@ class Exchange:
 
         candle_limit = self.ohlcv_candle_limit(
             timeframe, self._config['candle_type_def'],
-            int(date_minus_candles(timeframe, startup_candles).timestamp() * 1000)
+            dt_ts(date_minus_candles(timeframe, startup_candles))
             if timeframe else None)
         # Require one more candle - to account for the still open candle.
         candle_count = startup_candles + 1
@@ -1238,7 +1244,7 @@ class Exchange:
                 f'Insufficient funds to create {ordertype} {side} order on market {pair}. '
                 f'Tried to {side} amount {amount} at rate {limit_rate} with '
                 f'stop-price {stop_price_norm}. Message: {e}') from e
-        except (ccxt.InvalidOrder, ccxt.BadRequest) as e:
+        except (ccxt.InvalidOrder, ccxt.BadRequest, ccxt.OperationRejected) as e:
             # Errors:
             # `Order would trigger immediately.`
             raise InvalidOrderException(
@@ -1254,11 +1260,43 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
+    def fetch_order_emulated(self, order_id: str, pair: str, params: Dict) -> Dict:
+        """
+        Emulated fetch_order if the exchange doesn't support fetch_order, but requires separate
+        calls for open and closed orders.
+        """
+        try:
+            order = self._api.fetch_open_order(order_id, pair, params=params)
+            self._log_exchange_response('fetch_open_order', order)
+            order = self._order_contracts_to_amount(order)
+            return order
+        except ccxt.OrderNotFound:
+            try:
+                order = self._api.fetch_closed_order(order_id, pair, params=params)
+                self._log_exchange_response('fetch_closed_order', order)
+                order = self._order_contracts_to_amount(order)
+                return order
+            except ccxt.OrderNotFound as e:
+                raise RetryableOrderError(
+                    f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e
+        except ccxt.InvalidOrder as e:
+            raise InvalidOrderException(
+                f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
+        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
+            raise TemporaryError(
+                f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e
+        except ccxt.BaseError as e:
+            raise OperationalException(e) from e
+
     @retrier(retries=API_FETCH_ORDER_RETRY_COUNT)
     def fetch_order(self, order_id: str, pair: str, params: Dict = {}) -> Dict:
         if self._config['dry_run']:
             return self.fetch_dry_run_order(order_id)
         try:
+            if not self.exchange_has('fetchOrder'):
+                return self.fetch_order_emulated(order_id, pair, params)
             order = self._api.fetch_order(order_id, pair, params=params)
             self._log_exchange_response('fetch_order', order)
             order = self._order_contracts_to_amount(order)
@@ -2005,7 +2043,7 @@ class Exchange:
                 timeframe, candle_type, since_ms)
             move_to = one_call * self.required_candle_call_count
             now = timeframe_to_next_date(timeframe)
-            since_ms = int((now - timedelta(seconds=move_to // 1000)).timestamp() * 1000)
+            since_ms = dt_ts(now - timedelta(seconds=move_to // 1000))
 
         if since_ms:
             return self._async_get_historic_ohlcv(
@@ -2120,6 +2158,39 @@ class Exchange:
 
         return results_df
 
+    def refresh_ohlcv_with_cache(
+        self,
+        pairs: List[PairWithTimeframe],
+        since_ms: int
+    ) -> Dict[PairWithTimeframe, DataFrame]:
+        """
+        Refresh ohlcv data for all pairs in needed_pairs if necessary.
+        Caches data with expiring per timeframe.
+        Should only be used for pairlists which need "on time" expirarion, and no longer cache.
+        """
+
+        timeframes = {p[1] for p in pairs}
+        for timeframe in timeframes:
+            if (timeframe, since_ms) not in self._expiring_candle_cache:
+                timeframe_in_sec = timeframe_to_seconds(timeframe)
+                # Initialise cache
+                self._expiring_candle_cache[(timeframe, since_ms)] = PeriodicCache(
+                    ttl=timeframe_in_sec, maxsize=1000)
+
+        # Get candles from cache
+        candles = {
+            c: self._expiring_candle_cache[(c[1], since_ms)].get(c, None) for c in pairs
+            if c in self._expiring_candle_cache[(c[1], since_ms)]
+        }
+        pairs_to_download = [p for p in pairs if p not in candles]
+        if pairs_to_download:
+            candles = self.refresh_latest_ohlcv(
+                pairs_to_download, since_ms=since_ms, cache=False
+            )
+            for c, val in candles.items():
+                self._expiring_candle_cache[(c[1], since_ms)][c] = val
+        return candles
+
     def _now_is_time_to_refresh(self, pair: str, timeframe: str, candle_type: CandleType) -> bool:
         # Timeframe in seconds
         interval_in_sec = timeframe_to_seconds(timeframe)
@@ -2432,7 +2503,7 @@ class Exchange:
             )
 
         if type(since) is datetime:
-            since = int(since.timestamp()) * 1000   # * 1000 for ms
+            since = dt_ts(since)
 
         try:
             funding_history = self._api.fetch_funding_history(
@@ -2681,7 +2752,7 @@ class Exchange:
             self._log_exchange_response('set_leverage', res)
         except ccxt.DDoSProtection as e:
             raise DDosProtection(e) from e
-        except (ccxt.BadRequest, ccxt.InsufficientFunds) as e:
+        except (ccxt.BadRequest, ccxt.OperationRejected, ccxt.InsufficientFunds) as e:
             if not accept_fail:
                 raise TemporaryError(
                     f'Could not set leverage due to {e.__class__.__name__}. Message: {e}') from e
@@ -2723,7 +2794,7 @@ class Exchange:
             self._log_exchange_response('set_margin_mode', res)
         except ccxt.DDoSProtection as e:
             raise DDosProtection(e) from e
-        except ccxt.BadRequest as e:
+        except (ccxt.BadRequest, ccxt.OperationRejected) as e:
             if not accept_fail:
                 raise TemporaryError(
                     f'Could not set margin mode due to {e.__class__.__name__}. Message: {e}') from e
@@ -2762,7 +2833,7 @@ class Exchange:
 
         if not close_date:
             close_date = datetime.now(timezone.utc)
-        since_ms = int(timeframe_to_prev_date(timeframe, open_date).timestamp()) * 1000
+        since_ms = dt_ts(timeframe_to_prev_date(timeframe, open_date))
 
         mark_comb: PairWithTimeframe = (pair, timeframe, mark_price_type)
         funding_comb: PairWithTimeframe = (pair, timeframe_ff, CandleType.FUNDING_RATE)
@@ -2816,7 +2887,7 @@ class Exchange:
             else:
                 # Fill up missing funding_rate candles with fallback value
                 combined = mark_rates.merge(
-                    funding_rates, on='date', how="outer", suffixes=["_mark", "_fund"]
+                    funding_rates, on='date', how="left", suffixes=["_mark", "_fund"]
                     )
                 combined['open_fund'] = combined['open_fund'].fillna(futures_funding_rate)
                 return combined
@@ -2845,7 +2916,8 @@ class Exchange:
         if not df.empty:
             df1 = df[(df['date'] >= open_date) & (df['date'] <= close_date)]
             fees = sum(df1['open_fund'] * df1['open_mark'] * amount)
-
+        if isnan(fees):
+            fees = 0.0
         # Negate fees for longs as funding_fees expects it this way based on live endpoints.
         return fees if is_short else -fees
 

freqtrade/exchange/exchange_utils.py

@@ -40,21 +40,34 @@ def available_exchanges(ccxt_module: Optional[CcxtModuleType] = None) -> List[st
 
 
 def validate_exchange(exchange: str) -> Tuple[bool, str]:
+    """
+    returns: can_use, reason
+        with Reason including both missing and missing_opt
+    """
     ex_mod = getattr(ccxt, exchange.lower())()
+    result = True
+    reason = ''
     if not ex_mod or not ex_mod.has:
         return False, ''
-    missing = [k for k in EXCHANGE_HAS_REQUIRED if ex_mod.has.get(k) is not True]
+    missing = [
+        k for k, v in EXCHANGE_HAS_REQUIRED.items()
+        if ex_mod.has.get(k) is not True
+        and not (all(ex_mod.has.get(x) for x in v))
+    ]
     if missing:
-        return False, f"missing: {', '.join(missing)}"
+        result = False
+        reason += f"missing: {', '.join(missing)}"
 
     missing_opt = [k for k in EXCHANGE_HAS_OPTIONAL if not ex_mod.has.get(k)]
 
     if exchange.lower() in BAD_EXCHANGES:
-        return False, BAD_EXCHANGES.get(exchange.lower(), '')
-    if missing_opt:
-        return True, f"missing opt: {', '.join(missing_opt)}"
+        result = False
+        reason = BAD_EXCHANGES.get(exchange.lower(), '')
 
-    return True, ''
+    if missing_opt:
+        reason += f"{'. ' if reason else ''}missing opt: {', '.join(missing_opt)}. "
+
+    return result, reason
 
 
 def _build_exchange_list_entry(

freqtrade/freqai/prediction_models/XGBoostRegressor.py

@@ -36,8 +36,15 @@ class XGBoostRegressor(BaseRegressionModel):
             eval_set = None
             eval_weights = None
         else:
-            eval_set = [(data_dictionary["test_features"], data_dictionary["test_labels"])]
-            eval_weights = [data_dictionary['test_weights']]
+            eval_set = [
+                (data_dictionary["test_features"],
+                 data_dictionary["test_labels"]),
+                (X, y)
+            ]
+            eval_weights = [
+                data_dictionary['test_weights'],
+                data_dictionary['train_weights']
+            ]
 
         sample_weight = data_dictionary["train_weights"]
 

freqtrade/freqai/tensorboard/tensorboard.py

@@ -43,13 +43,11 @@ class TensorBoardCallback(BaseTensorBoardCallback):
         if not evals_log:
             return False
 
-        for data, metric in evals_log.items():
-            for metric_name, log in metric.items():
+        evals = ["validation", "train"]
+        for metric, eval in zip(evals_log.items(), evals):
+            for metric_name, log in metric[1].items():
                 score = log[-1][0] if isinstance(log[-1], tuple) else log[-1]
-                if data == "train":
-                    self.writer.add_scalar("train_loss", score, epoch)
-                else:
-                    self.writer.add_scalar("valid_loss", score, epoch)
+                self.writer.add_scalar(f"{eval}-{metric_name}", score, epoch)
 
         return False
 

freqtrade/freqai/utils.py

@@ -118,10 +118,12 @@ def plot_feature_importance(model: Any, pair: str, dk: FreqaiDataKitchen,
         mdl = models[label]
         if "catboost.core" in str(mdl.__class__):
             feature_importance = mdl.get_feature_importance()
-        elif "lightgbm.sklearn" or "xgb" in str(mdl.__class__):
+        elif "lightgbm.sklearn" in str(mdl.__class__):
+            feature_importance = mdl.feature_importances_
+        elif "xgb" in str(mdl.__class__):
             feature_importance = mdl.feature_importances_
         else:
-            logger.info('Model type not support for generating feature importances.')
+            logger.info('Model type does not support generating feature importances.')
             return
 
         # Data preparation

freqtrade/freqtradebot.py

@@ -82,7 +82,6 @@ class FreqtradeBot(LoggingMixin):
 
         PairLocks.timeframe = self.config['timeframe']
 
-        self.pairlists = PairListManager(self.exchange, self.config)
         self.trading_mode: TradingMode = self.config.get('trading_mode', TradingMode.SPOT)
         self.last_process: Optional[datetime] = None
 
@@ -129,8 +128,9 @@ class FreqtradeBot(LoggingMixin):
                 self.update_funding_fees()
                 self.wallets.update()
 
-            # TODO: This would be more efficient if scheduled in utc time, and performed at each
-            # TODO: funding interval, specified by funding_fee_times on the exchange classes
+            # This would be more efficient if scheduled in utc time, and performed at each
+            # funding interval, specified by funding_fee_times on the exchange classes
+            # However, this reduces the precision - and might therefore lead to problems.
             for time_slot in range(0, 24):
                 for minutes in [1, 31]:
                     t = str(time(time_slot, minutes, 2))
@@ -432,10 +432,6 @@ class FreqtradeBot(LoggingMixin):
             try:
                 fo = self.exchange.fetch_order_or_stoploss_order(order.order_id, order.ft_pair,
                                                                  order.ft_order_side == 'stoploss')
-                if order.ft_order_side == 'stoploss':
-                    if fo and fo['status'] == 'open':
-                        # Assume this as the open stoploss order
-                        trade.stoploss_order_id = order.order_id
                 if fo:
                     logger.info(f"Found {order} for trade {trade}.")
                     self.update_trade_state(trade, order.order_id, fo,
@@ -645,8 +641,7 @@ class FreqtradeBot(LoggingMixin):
         max_entry_stake = self.exchange.get_max_pair_stake_amount(trade.pair, current_entry_rate)
         stake_available = self.wallets.get_available_stake_amount()
         logger.debug(f"Calling adjust_trade_position for pair {trade.pair}")
-        stake_amount = strategy_safe_wrapper(self.strategy.adjust_trade_position,
-                                             default_retval=None, supress_error=True)(
+        stake_amount, order_tag = self.strategy._adjust_trade_position_internal(
             trade=trade,
             current_time=datetime.now(timezone.utc), current_rate=current_entry_rate,
             current_profit=current_entry_profit, min_stake=min_entry_stake,
@@ -665,7 +660,8 @@ class FreqtradeBot(LoggingMixin):
                 else:
                     logger.debug("Max adjustment entries is set to unlimited.")
             self.execute_entry(trade.pair, stake_amount, price=current_entry_rate,
-                               trade=trade, is_short=trade.is_short, mode='pos_adjust')
+                               trade=trade, is_short=trade.is_short, mode='pos_adjust',
+                               enter_tag=order_tag)
 
         if stake_amount is not None and stake_amount < 0.0:
             # We should decrease our position
@@ -684,7 +680,7 @@ class FreqtradeBot(LoggingMixin):
                 return
 
             self.execute_trade_exit(trade, current_exit_rate, exit_check=ExitCheckTuple(
-                exit_type=ExitType.PARTIAL_EXIT), sub_trade_amt=amount)
+                exit_type=ExitType.PARTIAL_EXIT), sub_trade_amt=amount, exit_tag=order_tag)
 
     def _check_depth_of_market(self, pair: str, conf: Dict, side: SignalDirection) -> bool:
         """
@@ -706,7 +702,7 @@ class FreqtradeBot(LoggingMixin):
         delta = f"Delta: {bids_ask_delta}"
 
         logger.info(
-            f"{bids}, {asks}, {delta}, Direction: {side.value}"
+            f"{bids}, {asks}, {delta}, Direction: {side.value} "
             f"Bid Price: {order_book['bids'][0][0]}, Ask Price: {order_book['asks'][0][0]}, "
             f"Immediate Bid Quantity: {order_book['bids'][0][1]}, "
             f"Immediate Ask Quantity: {order_book['asks'][0][1]}."
@@ -782,6 +778,7 @@ class FreqtradeBot(LoggingMixin):
             leverage=leverage
         )
         order_obj = Order.parse_from_ccxt_object(order, pair, side, amount, enter_limit_requested)
+        order_obj.ft_order_tag = enter_tag
         order_id = order['id']
         order_status = order.get('status')
         logger.info(f"Order {order_id} was created for {pair} and status is {order_status}.")
@@ -894,17 +891,15 @@ class FreqtradeBot(LoggingMixin):
 
     def cancel_stoploss_on_exchange(self, trade: Trade) -> Trade:
         # First cancelling stoploss on exchange ...
-        if trade.stoploss_order_id:
+        for oslo in trade.open_sl_orders:
             try:
-                logger.info(f"Cancelling stoploss on exchange for {trade}")
+                logger.info(f"Cancelling stoploss on exchange for {trade} "
+                            f"order: {oslo.order_id}")
                 co = self.exchange.cancel_stoploss_order_with_result(
-                    trade.stoploss_order_id, trade.pair, trade.amount)
-                self.update_trade_state(trade, trade.stoploss_order_id, co, stoploss_order=True)
-
-                # Reset stoploss order id.
-                trade.stoploss_order_id = None
+                    oslo.order_id, trade.pair, trade.amount)
+                self.update_trade_state(trade, oslo.order_id, co, stoploss_order=True)
             except InvalidOrderException:
-                logger.exception(f"Could not cancel stoploss order {trade.stoploss_order_id} "
+                logger.exception(f"Could not cancel stoploss order {oslo.order_id} "
                                  f"for pair {trade.pair}")
         return trade
 
@@ -967,7 +962,7 @@ class FreqtradeBot(LoggingMixin):
         # edge-case for now.
         min_stake_amount = self.exchange.get_min_pair_stake_amount(
             pair, enter_limit_requested,
-            self.strategy.stoploss if not mode != 'pos_adjust' else 0.0,
+            self.strategy.stoploss if not mode == 'pos_adjust' else 0.0,
             leverage)
         max_stake_amount = self.exchange.get_max_pair_stake_amount(
             pair, enter_limit_requested, leverage)
@@ -992,7 +987,7 @@ class FreqtradeBot(LoggingMixin):
 
         return enter_limit_requested, stake_amount, leverage
 
-    def _notify_enter(self, trade: Trade, order: Order, order_type: str,
+    def _notify_enter(self, trade: Trade, order: Order, order_type: Optional[str],
                       fill: bool = False, sub_trade: bool = False) -> None:
         """
         Sends rpc notification when a entry order occurred.
@@ -1016,7 +1011,7 @@ class FreqtradeBot(LoggingMixin):
             'direction': 'Short' if trade.is_short else 'Long',
             'limit': open_rate,  # Deprecated (?)
             'open_rate': open_rate,
-            'order_type': order_type,
+            'order_type': order_type or 'unknown',
             'stake_amount': trade.stake_amount,
             'stake_currency': self.config['stake_currency'],
             'base_currency': self.exchange.get_pair_base_currency(trade.pair),
@@ -1079,7 +1074,7 @@ class FreqtradeBot(LoggingMixin):
 
             if (
                 not trade.has_open_orders
-                and not trade.stoploss_order_id
+                and not trade.has_open_sl_orders
                 and not self.wallets.check_exit_amount(trade)
             ):
                 logger.warning(
@@ -1189,8 +1184,6 @@ class FreqtradeBot(LoggingMixin):
             order_obj = Order.parse_from_ccxt_object(stoploss_order, trade.pair, 'stoploss',
                                                      trade.amount, stop_price)
             trade.orders.append(order_obj)
-            trade.stoploss_order_id = str(stoploss_order['id'])
-            trade.stoploss_last_update = datetime.now(timezone.utc)
             return True
         except InsufficientFundsError as e:
             logger.warning(f"Unable to place stoploss order {e}.")
@@ -1198,13 +1191,11 @@ class FreqtradeBot(LoggingMixin):
             self.handle_insufficient_funds(trade)
 
         except InvalidOrderException as e:
-            trade.stoploss_order_id = None
             logger.error(f'Unable to place a stoploss order on exchange. {e}')
             logger.warning('Exiting the trade forcefully')
             self.emergency_exit(trade, stop_price)
 
         except ExchangeError:
-            trade.stoploss_order_id = None
             logger.exception('Unable to place a stoploss order on exchange.')
         return False
 
@@ -1218,27 +1209,28 @@ class FreqtradeBot(LoggingMixin):
         """
 
         logger.debug('Handling stoploss on exchange %s ...', trade)
-        stoploss_order = None
 
-        try:
-            # First we check if there is already a stoploss on exchange
-            stoploss_order = self.exchange.fetch_stoploss_order(
-                trade.stoploss_order_id, trade.pair) if trade.stoploss_order_id else None
-        except InvalidOrderException as exception:
-            logger.warning('Unable to fetch stoploss order: %s', exception)
+        stoploss_orders = []
+        for slo in trade.open_sl_orders:
+            stoploss_order = None
+            try:
+                # First we check if there is already a stoploss on exchange
+                stoploss_order = self.exchange.fetch_stoploss_order(
+                    slo.order_id, trade.pair) if slo.order_id else None
+            except InvalidOrderException as exception:
+                logger.warning('Unable to fetch stoploss order: %s', exception)
 
-        if stoploss_order:
-            self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order,
-                                    stoploss_order=True)
+            if stoploss_order:
+                stoploss_orders.append(stoploss_order)
+                self.update_trade_state(trade, slo.order_id, stoploss_order,
+                                        stoploss_order=True)
 
-        # We check if stoploss order is fulfilled
-        if stoploss_order and stoploss_order['status'] in ('closed', 'triggered'):
-            trade.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value
-            self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order,
-                                    stoploss_order=True)
-            self._notify_exit(trade, "stoploss", True)
-            self.handle_protections(trade.pair, trade.trade_direction)
-            return True
+            # We check if stoploss order is fulfilled
+            if stoploss_order and stoploss_order['status'] in ('closed', 'triggered'):
+                trade.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value
+                self._notify_exit(trade, "stoploss", True)
+                self.handle_protections(trade.pair, trade.trade_direction)
+                return True
 
         if trade.has_open_orders or not trade.is_open:
             # Trade has an open Buy or Sell order, Stoploss-handling can't happen in this case
@@ -1247,7 +1239,7 @@ class FreqtradeBot(LoggingMixin):
             return False
 
         # If enter order is fulfilled but there is no stoploss, we add a stoploss on exchange
-        if not stoploss_order:
+        if len(stoploss_orders) == 0:
             stop_price = trade.stoploss_or_liquidation
             if self.edge:
                 stoploss = self.edge.get_stoploss(pair=trade.pair)
@@ -1261,27 +1253,7 @@ class FreqtradeBot(LoggingMixin):
                 # in which case the trade will be closed - which we must check below.
                 return False
 
-        # If stoploss order is canceled for some reason we add it again
-        if (trade.is_open
-                and stoploss_order
-                and stoploss_order['status'] in ('canceled', 'cancelled')):
-            if self.create_stoploss_order(trade=trade, stop_price=trade.stoploss_or_liquidation):
-                return False
-            else:
-                logger.warning('Stoploss order was cancelled, but unable to recreate one.')
-
-        # Finally we check if stoploss on exchange should be moved up because of trailing.
-        # Triggered Orders are now real orders - so don't replace stoploss anymore
-        if (
-            trade.is_open and stoploss_order
-            and stoploss_order.get('status_stop') != 'triggered'
-            and (self.config.get('trailing_stop', False)
-                 or self.config.get('use_custom_stoploss', False))
-        ):
-            # if trailing stoploss is enabled we check if stoploss value has changed
-            # in which case we cancel stoploss order and put another one with new
-            # value immediately
-            self.handle_trailing_stoploss_on_exchange(trade, stoploss_order)
+        self.manage_trade_stoploss_orders(trade, stoploss_orders)
 
         return False
 
@@ -1317,6 +1289,42 @@ class FreqtradeBot(LoggingMixin):
                     logger.warning(f"Could not create trailing stoploss order "
                                    f"for pair {trade.pair}.")
 
+    def manage_trade_stoploss_orders(self, trade: Trade, stoploss_orders: List[Dict]):
+        """
+        Perform required actions acording to existing stoploss orders of trade
+        :param trade: Corresponding Trade
+        :param stoploss_orders: Current on exchange stoploss orders
+        :return: None
+        """
+        # If all stoploss orderd are canceled for some reason we add it again
+        canceled_sl_orders = [o for o in stoploss_orders
+                              if o['status'] in ('canceled', 'cancelled')]
+        if (
+            trade.is_open and
+            len(stoploss_orders) > 0 and
+            len(stoploss_orders) == len(canceled_sl_orders)
+        ):
+            if self.create_stoploss_order(trade=trade, stop_price=trade.stoploss_or_liquidation):
+                return False
+            else:
+                logger.warning('All Stoploss orders are cancelled, but unable to recreate one.')
+
+        active_sl_orders = [o for o in stoploss_orders if o not in canceled_sl_orders]
+        if len(active_sl_orders) > 0:
+            last_active_sl_order = active_sl_orders[-1]
+            # Finally we check if stoploss on exchange should be moved up because of trailing.
+            # Triggered Orders are now real orders - so don't replace stoploss anymore
+            if (trade.is_open and
+                    last_active_sl_order.get('status_stop') != 'triggered' and
+                    (self.config.get('trailing_stop', False) or
+                     self.config.get('use_custom_stoploss', False))):
+                # if trailing stoploss is enabled we check if stoploss value has changed
+                # in which case we cancel stoploss order and put another one with new
+                # value immediately
+                self.handle_trailing_stoploss_on_exchange(trade, last_active_sl_order)
+
+        return
+
     def manage_open_orders(self) -> None:
         """
         Management of open orders on exchange. Unfilled orders might be cancelled if timeout
@@ -1753,6 +1761,7 @@ class FreqtradeBot(LoggingMixin):
             return False
 
         order_obj = Order.parse_from_ccxt_object(order, trade.pair, trade.exit_side, amount, limit)
+        order_obj.ft_order_tag = exit_reason
         trade.orders.append(order_obj)
 
         trade.exit_order_status = ''
@@ -1767,7 +1776,7 @@ class FreqtradeBot(LoggingMixin):
 
         return True
 
-    def _notify_exit(self, trade: Trade, order_type: str, fill: bool = False,
+    def _notify_exit(self, trade: Trade, order_type: Optional[str], fill: bool = False,
                      sub_trade: bool = False, order: Optional[Order] = None) -> None:
         """
         Sends rpc notification when a sell occurred.
@@ -1799,7 +1808,7 @@ class FreqtradeBot(LoggingMixin):
             'gain': gain,
             'limit': order_rate,  # Deprecated
             'order_rate': order_rate,
-            'order_type': order_type,
+            'order_type': order_type or 'unknown',
             'amount': amount,
             'open_rate': trade.open_rate,
             'close_rate': order_rate,

freqtrade/optimize/analysis/lookahead_helpers.py

@@ -107,9 +107,9 @@ class LookaheadAnalysisSubFunctions:
                 csv_df = add_or_update_row(csv_df, new_row_data)
 
         # Fill NaN values with a default value (e.g., 0)
-        csv_df['total_signals'] = csv_df['total_signals'].fillna(0)
-        csv_df['biased_entry_signals'] = csv_df['biased_entry_signals'].fillna(0)
-        csv_df['biased_exit_signals'] = csv_df['biased_exit_signals'].fillna(0)
+        csv_df['total_signals'] = csv_df['total_signals'].astype(int).fillna(0)
+        csv_df['biased_entry_signals'] = csv_df['biased_entry_signals'].astype(int).fillna(0)
+        csv_df['biased_exit_signals'] = csv_df['biased_exit_signals'].astype(int).fillna(0)
 
         # Convert columns to integers
         csv_df['total_signals'] = csv_df['total_signals'].astype(int)

freqtrade/optimize/backtesting.py

@@ -23,7 +23,7 @@ from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType
                              TradingMode)
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.exchange import (amount_to_contract_precision, price_to_precision,
-                                timeframe_to_minutes, timeframe_to_seconds)
+                                timeframe_to_seconds)
 from freqtrade.exchange.exchange import Exchange
 from freqtrade.mixins import LoggingMixin
 from freqtrade.optimize.backtest_caching import get_strategy_run_id
@@ -33,8 +33,8 @@ from freqtrade.optimize.optimize_reports import (generate_backtest_stats, genera
                                                  show_backtest_results,
                                                  store_backtest_analysis_results,
                                                  store_backtest_stats)
-from freqtrade.persistence import (LocalTrade, Order, PairLocks, Trade, disable_database_use,
-                                   enable_database_use)
+from freqtrade.persistence import (CustomDataWrapper, LocalTrade, Order, PairLocks, Trade,
+                                   disable_database_use, enable_database_use)
 from freqtrade.plugins.pairlistmanager import PairListManager
 from freqtrade.plugins.protectionmanager import ProtectionManager
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -117,8 +117,9 @@ class Backtesting:
             raise OperationalException("Timeframe needs to be set in either "
                                        "configuration or as cli argument `--timeframe 5m`")
         self.timeframe = str(self.config.get('timeframe'))
-        self.timeframe_min = timeframe_to_minutes(self.timeframe)
-        self.timeframe_td = timedelta(minutes=self.timeframe_min)
+        self.timeframe_secs = timeframe_to_seconds(self.timeframe)
+        self.timeframe_min = self.timeframe_secs // 60
+        self.timeframe_td = timedelta(seconds=self.timeframe_secs)
         self.disable_database_use()
         self.init_backtest_detail()
         self.pairlists = PairListManager(self.exchange, self.config, self.dataprovider)
@@ -185,13 +186,14 @@ class Backtesting:
         # Load detail timeframe if specified
         self.timeframe_detail = str(self.config.get('timeframe_detail', ''))
         if self.timeframe_detail:
-            self.timeframe_detail_min = timeframe_to_minutes(self.timeframe_detail)
-            if self.timeframe_min <= self.timeframe_detail_min:
+            timeframe_detail_secs = timeframe_to_seconds(self.timeframe_detail)
+            self.timeframe_detail_td = timedelta(seconds=timeframe_detail_secs)
+            if self.timeframe_secs <= timeframe_detail_secs:
                 raise OperationalException(
                     "Detail timeframe must be smaller than strategy timeframe.")
 
         else:
-            self.timeframe_detail_min = 0
+            self.timeframe_detail_td = timedelta(seconds=0)
         self.detail_data: Dict[str, DataFrame] = {}
         self.futures_data: Dict[str, DataFrame] = {}
 
@@ -199,7 +201,7 @@ class Backtesting:
 
         self.prepare_backtest(False)
 
-        self.wallets = Wallets(self.config, self.exchange, log=False)
+        self.wallets = Wallets(self.config, self.exchange, is_backtest=True)
 
         self.progress = BTProgress()
         self.abort = False
@@ -335,6 +337,7 @@ class Backtesting:
         self.disable_database_use()
         PairLocks.reset_locks()
         Trade.reset_trades()
+        CustomDataWrapper.reset_custom_data()
         self.rejected_trades = 0
         self.timedout_entry_orders = 0
         self.timedout_exit_orders = 0
@@ -537,14 +540,14 @@ class Backtesting:
         min_stake = self.exchange.get_min_pair_stake_amount(trade.pair, current_rate, -0.1)
         max_stake = self.exchange.get_max_pair_stake_amount(trade.pair, current_rate)
         stake_available = self.wallets.get_available_stake_amount()
-        stake_amount = strategy_safe_wrapper(self.strategy.adjust_trade_position,
-                                             default_retval=None, supress_error=True)(
+        stake_amount, order_tag = self.strategy._adjust_trade_position_internal(
             trade=trade,  # type: ignore[arg-type]
             current_time=current_time, current_rate=current_rate,
             current_profit=current_profit, min_stake=min_stake,
             max_stake=min(max_stake, stake_available),
             current_entry_rate=current_rate, current_exit_rate=current_rate,
-            current_entry_profit=current_profit, current_exit_profit=current_profit)
+            current_entry_profit=current_profit, current_exit_profit=current_profit
+        )
 
         # Check if we should increase our position
         if stake_amount is not None and stake_amount > 0.0:
@@ -554,7 +557,8 @@ class Backtesting:
                 check_adjust_entry = (entry_count <= self.strategy.max_entry_position_adjustment)
             if check_adjust_entry:
                 pos_trade = self._enter_trade(
-                    trade.pair, row, 'short' if trade.is_short else 'long', stake_amount, trade)
+                    trade.pair, row, 'short' if trade.is_short else 'long', stake_amount, trade,
+                    entry_tag1=order_tag)
                 if pos_trade is not None:
                     self.wallets.update()
                     return pos_trade
@@ -569,7 +573,7 @@ class Backtesting:
             if min_stake and remaining != 0 and remaining < min_stake:
                 # Remaining stake is too low to be sold.
                 return trade
-            exit_ = ExitCheckTuple(ExitType.PARTIAL_EXIT)
+            exit_ = ExitCheckTuple(ExitType.PARTIAL_EXIT, order_tag)
             pos_trade = self._get_exit_for_signal(trade, row, exit_, current_time, amount)
             if pos_trade is not None:
                 order = pos_trade.orders[-1]
@@ -681,11 +685,11 @@ class Backtesting:
 
             trade.exit_reason = exit_reason
 
-            return self._exit_trade(trade, row, close_rate, amount_)
+            return self._exit_trade(trade, row, close_rate, amount_, exit_reason)
         return None
 
-    def _exit_trade(self, trade: LocalTrade, sell_row: Tuple,
-                    close_rate: float, amount: Optional[float] = None) -> Optional[LocalTrade]:
+    def _exit_trade(self, trade: LocalTrade, sell_row: Tuple, close_rate: float,
+                    amount: float, exit_reason: Optional[str]) -> Optional[LocalTrade]:
         self.order_id_counter += 1
         exit_candle_time = sell_row[DATE_IDX].to_pydatetime()
         order_type = self.strategy.order_types['exit']
@@ -712,6 +716,7 @@ class Backtesting:
             filled=0,
             remaining=amount,
             cost=amount * close_rate,
+            ft_order_tag=exit_reason,
         )
         order._trade_bt = trade
         trade.orders.append(order)
@@ -835,7 +840,9 @@ class Backtesting:
                      stake_amount: Optional[float] = None,
                      trade: Optional[LocalTrade] = None,
                      requested_rate: Optional[float] = None,
-                     requested_stake: Optional[float] = None) -> Optional[LocalTrade]:
+                     requested_stake: Optional[float] = None,
+                     entry_tag1: Optional[str] = None
+                     ) -> Optional[LocalTrade]:
         """
         :param trade: Trade to adjust - initial entry if None
         :param requested_rate: Adjusted entry rate
@@ -843,7 +850,7 @@ class Backtesting:
         """
 
         current_time = row[DATE_IDX].to_pydatetime()
-        entry_tag = row[ENTER_TAG_IDX] if len(row) >= ENTER_TAG_IDX + 1 else None
+        entry_tag = entry_tag1 or (row[ENTER_TAG_IDX] if len(row) >= ENTER_TAG_IDX + 1 else None)
         # let's call the custom entry price, using the open price as default price
         order_type = self.strategy.order_types['entry']
         pos_adjust = trade is not None and requested_rate is None
@@ -944,6 +951,7 @@ class Backtesting:
                 filled=0,
                 remaining=amount,
                 cost=amount * propose_rate + trade.fee_open,
+                ft_order_tag=entry_tag,
             )
             order._trade_bt = trade
             trade.orders.append(order)
@@ -963,7 +971,8 @@ class Backtesting:
                     # Ignore trade if entry-order did not fill yet
                     continue
                 exit_row = data[pair][-1]
-                self._exit_trade(trade, exit_row, exit_row[OPEN_IDX], trade.amount)
+                self._exit_trade(trade, exit_row, exit_row[OPEN_IDX], trade.amount,
+                                 ExitType.FORCE_EXIT.value)
                 trade.orders[-1].close_bt_order(exit_row[DATE_IDX].to_pydatetime(), trade)
 
                 trade.close_date = exit_row[DATE_IDX].to_pydatetime()
@@ -1262,7 +1271,7 @@ class Backtesting:
                         open_trade_count_start = self.backtest_loop(
                             det_row, pair, current_time_det, end_date,
                             open_trade_count_start, trade_dir, is_first)
-                        current_time_det += timedelta(minutes=self.timeframe_detail_min)
+                        current_time_det += self.timeframe_detail_td
                         is_first = False
                 else:
                     self.dataprovider._set_dataframe_max_date(current_time)

freqtrade/optimize/optimize_reports/optimize_reports.py

@@ -215,7 +215,7 @@ def _get_resample_from_period(period: str) -> str:
         # Weekly defaulting to Monday.
         return '1W-MON'
     if period == 'month':
-        return '1M'
+        return '1ME'
     raise ValueError(f"Period {period} is not supported.")
 
 

freqtrade/persistence/__init__.py

@@ -1,5 +1,6 @@
 # flake8: noqa: F401
 
+from freqtrade.persistence.custom_data import CustomDataWrapper
 from freqtrade.persistence.key_value_store import KeyStoreKeys, KeyValueStore
 from freqtrade.persistence.models import init_db
 from freqtrade.persistence.pairlock_middleware import PairLocks

freqtrade/persistence/custom_data.py

@@ -0,0 +1,174 @@
+import json
+import logging
+from datetime import datetime
+from typing import Any, ClassVar, List, Optional, Sequence
+
+from sqlalchemy import DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, select
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from freqtrade.constants import DATETIME_PRINT_FORMAT
+from freqtrade.persistence.base import ModelBase, SessionType
+from freqtrade.util import dt_now
+
+
+logger = logging.getLogger(__name__)
+
+
+class _CustomData(ModelBase):
+    """
+    CustomData database model
+    Keeps records of metadata as key/value store
+    for trades or global persistant values
+    One to many relationship with Trades:
+      - One trade can have many metadata entries
+      - One metadata entry can only be associated with one Trade
+    """
+    __tablename__ = 'trade_custom_data'
+    __allow_unmapped__ = True
+    session: ClassVar[SessionType]
+
+    # Uniqueness should be ensured over pair, order_id
+    # its likely that order_id is unique per Pair on some exchanges.
+    __table_args__ = (UniqueConstraint('ft_trade_id', 'cd_key', name="_trade_id_cd_key"),)
+
+    id = mapped_column(Integer, primary_key=True)
+    ft_trade_id = mapped_column(Integer, ForeignKey('trades.id'), index=True)
+
+    trade = relationship("Trade", back_populates="custom_data")
+
+    cd_key: Mapped[str] = mapped_column(String(255), nullable=False)
+    cd_type: Mapped[str] = mapped_column(String(25), nullable=False)
+    cd_value: Mapped[str] = mapped_column(Text, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, default=dt_now)
+    updated_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
+
+    # Empty container value - not persisted, but filled with cd_value on query
+    value: Any = None
+
+    def __repr__(self):
+        create_time = (self.created_at.strftime(DATETIME_PRINT_FORMAT)
+                       if self.created_at is not None else None)
+        update_time = (self.updated_at.strftime(DATETIME_PRINT_FORMAT)
+                       if self.updated_at is not None else None)
+        return (f'CustomData(id={self.id}, key={self.cd_key}, type={self.cd_type}, ' +
+                f'value={self.cd_value}, trade_id={self.ft_trade_id}, created={create_time}, ' +
+                f'updated={update_time})')
+
+    @classmethod
+    def query_cd(cls, key: Optional[str] = None,
+                 trade_id: Optional[int] = None) -> Sequence['_CustomData']:
+        """
+        Get all CustomData, if trade_id is not specified
+        return will be for generic values not tied to a trade
+        :param trade_id: id of the Trade
+        """
+        filters = []
+        if trade_id is not None:
+            filters.append(_CustomData.ft_trade_id == trade_id)
+        if key is not None:
+            filters.append(_CustomData.cd_key.ilike(key))
+
+        return _CustomData.session.scalars(select(_CustomData).filter(*filters)).all()
+
+
+class CustomDataWrapper:
+    """
+    CustomData middleware class
+    Abstracts the database layer away so it becomes optional - which will be necessary to support
+    backtesting and hyperopt in the future.
+    """
+
+    use_db = True
+    custom_data: List[_CustomData] = []
+    unserialized_types = ['bool', 'float', 'int', 'str']
+
+    @staticmethod
+    def _convert_custom_data(data: _CustomData) -> _CustomData:
+        if data.cd_type in CustomDataWrapper.unserialized_types:
+            data.value = data.cd_value
+            if data.cd_type == 'bool':
+                data.value = data.cd_value.lower() == 'true'
+            elif data.cd_type == 'int':
+                data.value = int(data.cd_value)
+            elif data.cd_type == 'float':
+                data.value = float(data.cd_value)
+        else:
+            data.value = json.loads(data.cd_value)
+        return data
+
+    @staticmethod
+    def reset_custom_data() -> None:
+        """
+        Resets all key-value pairs. Only active for backtesting mode.
+        """
+        if not CustomDataWrapper.use_db:
+            CustomDataWrapper.custom_data = []
+
+    @staticmethod
+    def delete_custom_data(trade_id: int) -> None:
+        _CustomData.session.query(_CustomData).filter(_CustomData.ft_trade_id == trade_id).delete()
+        _CustomData.session.commit()
+
+    @staticmethod
+    def get_custom_data(*, trade_id: int, key: Optional[str] = None) -> List[_CustomData]:
+
+        if CustomDataWrapper.use_db:
+            filters = [
+                _CustomData.ft_trade_id == trade_id,
+            ]
+            if key is not None:
+                filters.append(_CustomData.cd_key.ilike(key))
+            filtered_custom_data = _CustomData.session.scalars(select(_CustomData).filter(
+                *filters)).all()
+
+        else:
+            filtered_custom_data = [
+                data_entry for data_entry in CustomDataWrapper.custom_data
+                if (data_entry.ft_trade_id == trade_id)
+            ]
+            if key is not None:
+                filtered_custom_data = [
+                    data_entry for data_entry in filtered_custom_data
+                    if (data_entry.cd_key.casefold() == key.casefold())
+                ]
+        return [CustomDataWrapper._convert_custom_data(d) for d in filtered_custom_data]
+
+    @staticmethod
+    def set_custom_data(trade_id: int, key: str, value: Any) -> None:
+
+        value_type = type(value).__name__
+
+        if value_type not in CustomDataWrapper.unserialized_types:
+            try:
+                value_db = json.dumps(value)
+            except TypeError as e:
+                logger.warning(f"could not serialize {key} value due to {e}")
+                return
+        else:
+            value_db = str(value)
+
+        if trade_id is None:
+            trade_id = 0
+
+        custom_data = CustomDataWrapper.get_custom_data(trade_id=trade_id, key=key)
+        if custom_data:
+            data_entry = custom_data[0]
+            data_entry.cd_value = value_db
+            data_entry.updated_at = dt_now()
+        else:
+            data_entry = _CustomData(
+                ft_trade_id=trade_id,
+                cd_key=key,
+                cd_type=value_type,
+                cd_value=value_db,
+                created_at=dt_now(),
+            )
+        data_entry.value = value
+
+        if CustomDataWrapper.use_db and value_db is not None:
+            _CustomData.session.add(data_entry)
+            _CustomData.session.commit()
+        else:
+            if not custom_data:
+                CustomDataWrapper.custom_data.append(data_entry)
+            # Existing data will have updated interactively.

freqtrade/persistence/migrations.py

@@ -1,7 +1,7 @@
 import logging
 from typing import List, Optional
 
-from sqlalchemy import inspect, select, text, tuple_, update
+from sqlalchemy import inspect, select, text, update
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.persistence.trade_model import Order, Trade
@@ -91,8 +91,6 @@ def migrate_trades_and_orders_table(
     is_stop_loss_trailing = get_column_def(
         cols, 'is_stop_loss_trailing',
         f'coalesce({stop_loss_pct}, 0.0) <> coalesce({initial_stop_loss_pct}, 0.0)')
-    stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null')
-    stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null')
     max_rate = get_column_def(cols, 'max_rate', '0.0')
     min_rate = get_column_def(cols, 'min_rate', 'null')
     exit_reason = get_column_def(cols, 'sell_reason', get_column_def(cols, 'exit_reason', 'null'))
@@ -160,7 +158,7 @@ def migrate_trades_and_orders_table(
             open_rate_requested, close_rate, close_rate_requested, close_profit,
             stake_amount, amount, amount_requested, open_date, close_date,
             stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
-            is_stop_loss_trailing, stoploss_order_id, stoploss_last_update,
+            is_stop_loss_trailing,
             max_rate, min_rate, exit_reason, exit_order_status, strategy, enter_tag,
             timeframe, open_trade_value, close_profit_abs,
             trading_mode, leverage, liquidation_price, is_short,
@@ -180,7 +178,6 @@ def migrate_trades_and_orders_table(
             {initial_stop_loss} initial_stop_loss,
             {initial_stop_loss_pct} initial_stop_loss_pct,
             {is_stop_loss_trailing} is_stop_loss_trailing,
-            {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
             {max_rate} max_rate, {min_rate} min_rate,
             case when {exit_reason} = 'sell_signal' then 'exit_signal'
                  when {exit_reason} = 'custom_sell' then 'custom_exit'
@@ -223,6 +220,7 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List):
     ft_amount = get_column_def(cols_order, 'ft_amount', 'coalesce(amount, 0.0)')
     ft_price = get_column_def(cols_order, 'ft_price', 'coalesce(price, 0.0)')
     ft_cancel_reason = get_column_def(cols_order, 'ft_cancel_reason', 'null')
+    ft_order_tag = get_column_def(cols_order, 'ft_order_tag', 'null')
 
     # sqlite does not support literals for booleans
     with engine.begin() as connection:
@@ -230,13 +228,14 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List):
             insert into orders (id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
             status, symbol, order_type, side, price, amount, filled, average, remaining, cost,
             stop_price, order_date, order_filled_date, order_update_date, ft_fee_base, funding_fee,
-            ft_amount, ft_price, ft_cancel_reason
+            ft_amount, ft_price, ft_cancel_reason, ft_order_tag
             )
             select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
             status, symbol, order_type, side, price, amount, filled, {average} average, remaining,
             cost, {stop_price} stop_price, order_date, order_filled_date,
             order_update_date, {ft_fee_base} ft_fee_base, {funding_fee} funding_fee,
-            {ft_amount} ft_amount, {ft_price} ft_price, {ft_cancel_reason} ft_cancel_reason
+            {ft_amount} ft_amount, {ft_price} ft_price, {ft_cancel_reason} ft_cancel_reason,
+            {ft_order_tag} ft_order_tag
             from {table_back_name}
             """))
 
@@ -277,6 +276,8 @@ def fix_old_dry_orders(engine):
     with engine.begin() as connection:
 
         # Update current dry-run Orders where
+        # - stoploss order is Open (will be replaced eventually)
+        # 2nd query:
         # - current Order is open
         # - current Trade is closed
         # - current Order trade_id not equal to current Trade.id
@@ -284,11 +285,6 @@ def fix_old_dry_orders(engine):
 
         stmt = update(Order).where(
             Order.ft_is_open.is_(True),
-            tuple_(Order.ft_trade_id, Order.order_id).not_in(
-                select(
-                    Trade.id, Trade.stoploss_order_id
-                ).where(Trade.stoploss_order_id.is_not(None))
-                  ),
             Order.ft_order_side == 'stoploss',
             Order.order_id.like('dry%'),
 
@@ -331,8 +327,8 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
     # if ('orders' not in previous_tables
     # or not has_column(cols_orders, 'funding_fee')):
     migrating = False
-    # if not has_column(cols_orders, 'ft_cancel_reason'):
-    if not has_column(cols_trades, 'funding_fee_running'):
+    # if not has_column(cols_trades, 'funding_fee_running'):
+    if not has_column(cols_orders, 'ft_order_tag'):
         migrating = True
         logger.info(f"Running database migration for trades - "
                     f"backup: {table_back_name}, {order_table_bak_name}")

freqtrade/persistence/models.py

@@ -13,6 +13,7 @@ from sqlalchemy.pool import StaticPool
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.persistence.base import ModelBase
+from freqtrade.persistence.custom_data import _CustomData
 from freqtrade.persistence.key_value_store import _KeyValueStoreModel
 from freqtrade.persistence.migrations import check_migrate
 from freqtrade.persistence.pairlock import PairLock
@@ -78,6 +79,8 @@ def init_db(db_url: str) -> None:
     Order.session = Trade.session
     PairLock.session = Trade.session
     _KeyValueStoreModel.session = Trade.session
+    _CustomData.session = scoped_session(sessionmaker(bind=engine, autoflush=True),
+                                         scopefunc=get_request_or_thread_id)
 
     previous_tables = inspect(engine).get_table_names()
     ModelBase.metadata.create_all(engine)

freqtrade/persistence/trade_model.py

@@ -23,7 +23,8 @@ from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, amount_to_contract_precisi
 from freqtrade.leverage import interest
 from freqtrade.misc import safe_value_fallback
 from freqtrade.persistence.base import ModelBase, SessionType
-from freqtrade.util import FtPrecise, dt_from_ts, dt_now, dt_ts
+from freqtrade.persistence.custom_data import CustomDataWrapper, _CustomData
+from freqtrade.util import FtPrecise, dt_from_ts, dt_now, dt_ts, dt_ts_none
 
 
 logger = logging.getLogger(__name__)
@@ -73,8 +74,7 @@ class Order(ModelBase):
     order_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
     status: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
     symbol: Mapped[Optional[str]] = mapped_column(String(25), nullable=True)
-    # TODO: type: order_type type is Optional[str]
-    order_type: Mapped[str] = mapped_column(String(50), nullable=True)
+    order_type: Mapped[Optional[str]] = mapped_column(String(50), nullable=True)
     side: Mapped[str] = mapped_column(String(25), nullable=True)
     price: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
     average: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
@@ -89,6 +89,8 @@ class Order(ModelBase):
     funding_fee: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
 
     ft_fee_base: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
+    ft_order_tag: Mapped[Optional[str]] = mapped_column(String(CUSTOM_TAG_MAX_LENGTH),
+                                                        nullable=True)
 
     @property
     def order_date_utc(self) -> datetime:
@@ -175,6 +177,8 @@ class Order(ModelBase):
         order_date = safe_value_fallback(order, 'timestamp')
         if order_date:
             self.order_date = datetime.fromtimestamp(order_date / 1000, tz=timezone.utc)
+        elif not self.order_date:
+            self.order_date = dt_now()
 
         self.ft_is_open = True
         if self.status in NON_OPEN_EXCHANGE_STATES:
@@ -212,13 +216,17 @@ class Order(ModelBase):
         return order
 
     def to_json(self, entry_side: str, minified: bool = False) -> Dict[str, Any]:
+        """
+        :param minified: If True, only return a subset of the data is returned.
+                         Only used for backtesting.
+        """
         resp = {
             'amount': self.safe_amount,
             'safe_price': self.safe_price,
             'ft_order_side': self.ft_order_side,
-            'order_filled_timestamp': int(self.order_filled_date.replace(
-                tzinfo=timezone.utc).timestamp() * 1000) if self.order_filled_date else None,
+            'order_filled_timestamp': dt_ts_none(self.order_filled_utc),
             'ft_is_entry': self.ft_order_side == entry_side,
+            'ft_order_tag': self.ft_order_tag,
         }
         if not minified:
             resp.update({
@@ -369,10 +377,6 @@ class LocalTrade:
     # percentage value of the initial stop loss
     initial_stop_loss_pct: Optional[float] = None
     is_stop_loss_trailing: bool = False
-    # stoploss order id which is on exchange
-    stoploss_order_id: Optional[str] = None
-    # last update time of the stoploss order on exchange
-    stoploss_last_update: Optional[datetime] = None
     # absolute value of the highest reached price
     max_rate: Optional[float] = None
     # Lowest price reached
@@ -456,14 +460,25 @@ class LocalTrade:
             return self.open_date_utc
         return max([self.open_date_utc, dt_last_filled])
 
+    @property
+    def date_entry_fill_utc(self) -> Optional[datetime]:
+        """ Date of the first filled order"""
+        orders = self.select_filled_orders(self.entry_side)
+        if (
+            orders
+            and len(filled_date := [o.order_filled_utc for o in orders if o.order_filled_utc])
+        ):
+            return min(filled_date)
+        return None
+
     @property
     def open_date_utc(self):
         return self.open_date.replace(tzinfo=timezone.utc)
 
     @property
     def stoploss_last_update_utc(self):
-        if self.stoploss_last_update:
-            return self.stoploss_last_update.replace(tzinfo=timezone.utc)
+        if self.has_open_sl_orders:
+            return max(o.order_date_utc for o in self.open_sl_orders)
         return None
 
     @property
@@ -519,7 +534,7 @@ class LocalTrade:
         return [o for o in self.orders if o.ft_is_open and o.ft_order_side != 'stoploss']
 
     @property
-    def has_open_orders(self) -> int:
+    def has_open_orders(self) -> bool:
         """
         True if there are open orders for this trade excluding stoploss orders
         """
@@ -529,6 +544,37 @@ class LocalTrade:
         ]
         return len(open_orders_wo_sl) > 0
 
+    @property
+    def open_sl_orders(self) -> List[Order]:
+        """
+        All open stoploss orders for this trade
+        """
+        return [
+            o for o in self.orders
+            if o.ft_order_side in ['stoploss'] and o.ft_is_open
+        ]
+
+    @property
+    def has_open_sl_orders(self) -> bool:
+        """
+        True if there are open stoploss orders for this trade
+        """
+        open_sl_orders = [
+            o for o in self.orders
+            if o.ft_order_side in ['stoploss'] and o.ft_is_open
+        ]
+        return len(open_sl_orders) > 0
+
+    @property
+    def sl_orders(self) -> List[Order]:
+        """
+        All stoploss orders for this trade
+        """
+        return [
+            o for o in self.orders
+            if o.ft_order_side in ['stoploss']
+        ]
+
     @property
     def open_orders_ids(self) -> List[str]:
         open_orders_ids_wo_sl = [
@@ -589,15 +635,17 @@ class LocalTrade:
             'fee_close_currency': self.fee_close_currency,
 
             'open_date': self.open_date.strftime(DATETIME_PRINT_FORMAT),
-            'open_timestamp': int(self.open_date.replace(tzinfo=timezone.utc).timestamp() * 1000),
+            'open_timestamp': dt_ts_none(self.open_date_utc),
+            'open_fill_date': (self.date_entry_fill_utc.strftime(DATETIME_PRINT_FORMAT)
+                               if self.date_entry_fill_utc else None),
+            'open_fill_timestamp': dt_ts_none(self.date_entry_fill_utc),
             'open_rate': self.open_rate,
             'open_rate_requested': self.open_rate_requested,
             'open_trade_value': round(self.open_trade_value, 8),
 
             'close_date': (self.close_date.strftime(DATETIME_PRINT_FORMAT)
                            if self.close_date else None),
-            'close_timestamp': int(self.close_date.replace(
-                tzinfo=timezone.utc).timestamp() * 1000) if self.close_date else None,
+            'close_timestamp': dt_ts_none(self.close_date_utc),
             'realized_profit': self.realized_profit or 0.0,
             # Close-profit corresponds to relative realized_profit ratio
             'realized_profit_ratio': self.close_profit or None,
@@ -621,11 +669,9 @@ class LocalTrade:
             'stop_loss_abs': self.stop_loss,
             'stop_loss_ratio': self.stop_loss_pct if self.stop_loss_pct else None,
             'stop_loss_pct': (self.stop_loss_pct * 100) if self.stop_loss_pct else None,
-            'stoploss_order_id': self.stoploss_order_id,
-            'stoploss_last_update': (self.stoploss_last_update.strftime(DATETIME_PRINT_FORMAT)
-                                     if self.stoploss_last_update else None),
-            'stoploss_last_update_timestamp': int(self.stoploss_last_update.replace(
-                tzinfo=timezone.utc).timestamp() * 1000) if self.stoploss_last_update else None,
+            'stoploss_last_update': (self.stoploss_last_update_utc.strftime(DATETIME_PRINT_FORMAT)
+                                     if self.stoploss_last_update_utc else None),
+            'stoploss_last_update_timestamp': dt_ts_none(self.stoploss_last_update_utc),
             'initial_stop_loss_abs': self.initial_stop_loss,
             'initial_stop_loss_ratio': (self.initial_stop_loss_pct
                                         if self.initial_stop_loss_pct else None),
@@ -769,6 +815,7 @@ class LocalTrade:
             order.funding_fee = self.funding_fee_running
             # Reset running funding fees
             self.funding_fee_running = 0.0
+        order_type = order.order_type.upper() if order.order_type else None
 
         if order.ft_order_side == self.entry_side:
             # Update open rate and actual amount
@@ -776,21 +823,20 @@ class LocalTrade:
             self.amount = order.safe_amount_after_fee
             if self.is_open:
                 payment = "SELL" if self.is_short else "BUY"
-                logger.info(f'{order.order_type.upper()}_{payment} has been fulfilled for {self}.')
+                logger.info(f'{order_type}_{payment} has been fulfilled for {self}.')
 
             self.recalc_trade_from_orders()
         elif order.ft_order_side == self.exit_side:
             if self.is_open:
                 payment = "BUY" if self.is_short else "SELL"
                 # * On margin shorts, you buy a little bit more than the amount (amount + interest)
-                logger.info(f'{order.order_type.upper()}_{payment} has been fulfilled for {self}.')
+                logger.info(f'{order_type}_{payment} has been fulfilled for {self}.')
 
         elif order.ft_order_side == 'stoploss' and order.status not in ('open', ):
-            self.stoploss_order_id = None
             self.close_rate_requested = self.stop_loss
             self.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value
             if self.is_open and order.safe_filled > 0:
-                logger.info(f'{order.order_type.upper()} is hit for {self}.')
+                logger.info(f'{order_type} is hit for {self}.')
         else:
             raise ValueError(f'Unknown order type: {order.order_type}')
 
@@ -1169,6 +1215,40 @@ class LocalTrade:
                 or (o.ft_is_open is True and o.status is not None)
                 ]
 
+    def set_custom_data(self, key: str, value: Any) -> None:
+        """
+        Set custom data for this trade
+        :param key: key of the custom data
+        :param value: value of the custom data (must be JSON serializable)
+        """
+        CustomDataWrapper.set_custom_data(trade_id=self.id, key=key, value=value)
+
+    def get_custom_data(self, key: str, default: Any = None) -> Any:
+        """
+        Get custom data for this trade
+        :param key: key of the custom data
+        """
+        data = CustomDataWrapper.get_custom_data(trade_id=self.id, key=key)
+        if data:
+            return data[0].value
+        return default
+
+    def get_custom_data_entry(self, key: str) -> Optional[_CustomData]:
+        """
+        Get custom data for this trade
+        :param key: key of the custom data
+        """
+        data = CustomDataWrapper.get_custom_data(trade_id=self.id, key=key)
+        if data:
+            return data[0]
+        return None
+
+    def get_all_custom_data(self) -> List[_CustomData]:
+        """
+        Get all custom data for this trade
+        """
+        return CustomDataWrapper.get_custom_data(trade_id=self.id)
+
     @property
     def nr_of_successful_entries(self) -> int:
         """
@@ -1363,11 +1443,6 @@ class LocalTrade:
             exit_order_status=data["exit_order_status"],
             stop_loss=data["stop_loss_abs"],
             stop_loss_pct=data["stop_loss_ratio"],
-            stoploss_order_id=data["stoploss_order_id"],
-            stoploss_last_update=(
-                datetime.fromtimestamp(data["stoploss_last_update_timestamp"] // 1000,
-                                       tz=timezone.utc)
-                if data["stoploss_last_update_timestamp"] else None),
             initial_stop_loss=data["initial_stop_loss_abs"],
             initial_stop_loss_pct=data["initial_stop_loss_ratio"],
             min_rate=data["min_rate"],
@@ -1405,6 +1480,7 @@ class LocalTrade:
                 ft_price=order["price"],
                 remaining=order["remaining"],
                 funding_fee=order.get("funding_fee", None),
+                ft_order_tag=order.get("ft_order_tag", None),
             )
             trade.orders.append(order_obj)
 
@@ -1428,6 +1504,9 @@ class Trade(ModelBase, LocalTrade):
     orders: Mapped[List[Order]] = relationship(
         "Order", order_by="Order.id", cascade="all, delete-orphan", lazy="selectin",
         innerjoin=True)  # type: ignore
+    custom_data: Mapped[List[_CustomData]] = relationship(
+        "_CustomData", cascade="all, delete-orphan",
+        lazy="raise")
 
     exchange: Mapped[str] = mapped_column(String(25), nullable=False)  # type: ignore
     pair: Mapped[str] = mapped_column(String(25), nullable=False, index=True)  # type: ignore
@@ -1473,11 +1552,6 @@ class Trade(ModelBase, LocalTrade):
         Float(), nullable=True)  # type: ignore
     is_stop_loss_trailing: Mapped[bool] = mapped_column(
         nullable=False, default=False)  # type: ignore
-    # stoploss order id which is on exchange
-    stoploss_order_id: Mapped[Optional[str]] = mapped_column(
-        String(255), nullable=True, index=True)  # type: ignore
-    # last update time of the stoploss order on exchange
-    stoploss_last_update: Mapped[Optional[datetime]] = mapped_column(nullable=True)  # type: ignore
     # absolute value of the highest reached price
     max_rate: Mapped[Optional[float]] = mapped_column(
         Float(), nullable=True, default=0.0)  # type: ignore
@@ -1536,6 +1610,8 @@ class Trade(ModelBase, LocalTrade):
         for order in self.orders:
             Order.session.delete(order)
 
+        CustomDataWrapper.delete_custom_data(trade_id=self.id)
+
         Trade.session.delete(self)
         Trade.commit()
 

freqtrade/persistence/usedb_context.py

@@ -1,4 +1,5 @@
 
+from freqtrade.persistence.custom_data import CustomDataWrapper
 from freqtrade.persistence.pairlock_middleware import PairLocks
 from freqtrade.persistence.trade_model import Trade
 
@@ -11,6 +12,7 @@ def disable_database_use(timeframe: str) -> None:
     PairLocks.use_db = False
     PairLocks.timeframe = timeframe
     Trade.use_db = False
+    CustomDataWrapper.use_db = False
 
 
 def enable_database_use() -> None:
@@ -20,6 +22,7 @@ def enable_database_use() -> None:
     PairLocks.use_db = True
     PairLocks.timeframe = ''
     Trade.use_db = True
+    CustomDataWrapper.use_db = True
 
 
 class FtNoDBContext:

freqtrade/plugins/pairlist/VolatilityFilter.py

@@ -3,7 +3,6 @@ Volatility pairlist filter
 """
 import logging
 import sys
-from copy import deepcopy
 from datetime import timedelta
 from typing import Any, Dict, List, Optional
 
@@ -37,6 +36,7 @@ class VolatilityFilter(IPairList):
         self._max_volatility = pairlistconfig.get('max_volatility', sys.maxsize)
         self._refresh_period = pairlistconfig.get('refresh_period', 1440)
         self._def_candletype = self._config['candle_type_def']
+        self._sort_direction: Optional[str] = pairlistconfig.get('sort_direction', None)
 
         self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period)
 
@@ -46,6 +46,9 @@ class VolatilityFilter(IPairList):
         if self._days > candle_limit:
             raise OperationalException("VolatilityFilter requires lookback_days to not "
                                        f"exceed exchange max request size ({candle_limit})")
+        if self._sort_direction not in [None, 'asc', 'desc']:
+            raise OperationalException("VolatilityFilter requires sort_direction to be "
+                                       "either None (undefined), 'asc' or 'desc'")
 
     @property
     def needstickers(self) -> bool:
@@ -89,6 +92,13 @@ class VolatilityFilter(IPairList):
                 "description": "Maximum Volatility",
                 "help": "Maximum volatility a pair must have to be considered.",
             },
+            "sort_direction": {
+                "type": "option",
+                "default": None,
+                "options": ["", "asc", "desc"],
+                "description": "Sort pairlist",
+                "help": "Sort Pairlist ascending or descending by volatility.",
+            },
             **IPairList.refresh_period_parameter()
         }
 
@@ -103,50 +113,63 @@ class VolatilityFilter(IPairList):
             (p, '1d', self._def_candletype) for p in pairlist if p not in self._pair_cache]
 
         since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days))
-        # Get all candles
-        candles = {}
-        if needed_pairs:
-            candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms,
-                                                          cache=False)
+        candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms=since_ms)
 
-        if self._enabled:
-            for p in deepcopy(pairlist):
-                daily_candles = candles[(p, '1d', self._def_candletype)] if (
-                    p, '1d', self._def_candletype) in candles else None
-                if not self._validate_pair_loc(p, daily_candles):
-                    pairlist.remove(p)
-        return pairlist
+        resulting_pairlist: List[str] = []
+        volatilitys: Dict[str, float] = {}
+        for p in pairlist:
+            daily_candles = candles.get((p, '1d', self._def_candletype), None)
 
-    def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool:
-        """
-        Validate trading range
-        :param pair: Pair that's currently validated
-        :param daily_candles: Downloaded daily candles
-        :return: True if the pair can stay, false if it should be removed
-        """
+            volatility_avg = self._calculate_volatility(p, daily_candles)
+
+            if volatility_avg is not None:
+                if self._validate_pair_loc(p, volatility_avg):
+                    resulting_pairlist.append(p)
+                    volatilitys[p] = (
+                        volatility_avg if volatility_avg and not np.isnan(volatility_avg) else 0
+                    )
+            else:
+                self.log_once(f"Removed {p} from whitelist, no candles found.", logger.info)
+
+        if self._sort_direction:
+            resulting_pairlist = sorted(resulting_pairlist,
+                                        key=lambda p: volatilitys[p],
+                                        reverse=self._sort_direction == 'desc')
+        return resulting_pairlist
+
+    def _calculate_volatility(self, pair: str,  daily_candles: DataFrame) -> Optional[float]:
         # Check symbol in cache
-        cached_res = self._pair_cache.get(pair, None)
-        if cached_res is not None:
-            return cached_res
+        if (volatility_avg := self._pair_cache.get(pair, None)) is not None:
+            return volatility_avg
 
-        result = False
         if daily_candles is not None and not daily_candles.empty:
             returns = (np.log(daily_candles["close"].shift(1) / daily_candles["close"]))
             returns.fillna(0, inplace=True)
 
             volatility_series = returns.rolling(window=self._days).std() * np.sqrt(self._days)
             volatility_avg = volatility_series.mean()
+            self._pair_cache[pair] = volatility_avg
 
-            if self._min_volatility <= volatility_avg <= self._max_volatility:
-                result = True
-            else:
-                self.log_once(f"Removed {pair} from whitelist, because volatility "
-                              f"over {self._days} {plural(self._days, 'day')} "
-                              f"is: {volatility_avg:.3f} "
-                              f"which is not in the configured range of "
-                              f"{self._min_volatility}-{self._max_volatility}.",
-                              logger.info)
-                result = False
-            self._pair_cache[pair] = result
+            return volatility_avg
+        else:
+            return None
 
+    def _validate_pair_loc(self, pair: str, volatility_avg: float) -> bool:
+        """
+        Validate trading range
+        :param pair: Pair that's currently validated
+        :param volatility_avg: Average volatility
+        :return: True if the pair can stay, false if it should be removed
+        """
+
+        if self._min_volatility <= volatility_avg <= self._max_volatility:
+            result = True
+        else:
+            self.log_once(f"Removed {pair} from whitelist, because volatility "
+                          f"over {self._days} {plural(self._days, 'day')} "
+                          f"is: {volatility_avg:.3f} "
+                          f"which is not in the configured range of "
+                          f"{self._min_volatility}-{self._max_volatility}.",
+                          logger.info)
+            result = False
         return result

freqtrade/plugins/pairlist/VolumePairList.py

@@ -229,12 +229,8 @@ class VolumePairList(IPairList):
                 if p not in self._pair_cache
             ]
 
-            # Get all candles
-            candles = {}
-            if needed_pairs:
-                candles = self._exchange.refresh_latest_ohlcv(
-                    needed_pairs, since_ms=since_ms, cache=False
-                )
+            candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms)
+
             for i, p in enumerate(filtered_tickers):
                 contract_size = self._exchange.markets[p['symbol']].get('contractSize', 1.0) or 1.0
                 pair_candles = candles[

freqtrade/plugins/pairlist/rangestabilityfilter.py

@@ -2,7 +2,6 @@
 Rate of change pairlist filter
 """
 import logging
-from copy import deepcopy
 from datetime import timedelta
 from typing import Any, Dict, List, Optional
 
@@ -32,6 +31,7 @@ class RangeStabilityFilter(IPairList):
         self._max_rate_of_change = pairlistconfig.get('max_rate_of_change')
         self._refresh_period = pairlistconfig.get('refresh_period', 86400)
         self._def_candletype = self._config['candle_type_def']
+        self._sort_direction: Optional[str] = pairlistconfig.get('sort_direction', None)
 
         self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period)
 
@@ -41,6 +41,9 @@ class RangeStabilityFilter(IPairList):
         if self._days > candle_limit:
             raise OperationalException("RangeStabilityFilter requires lookback_days to not "
                                        f"exceed exchange max request size ({candle_limit})")
+        if self._sort_direction not in [None, 'asc', 'desc']:
+            raise OperationalException("RangeStabilityFilter requires sort_direction to be "
+                                       "either None (undefined), 'asc' or 'desc'")
 
     @property
     def needstickers(self) -> bool:
@@ -87,6 +90,13 @@ class RangeStabilityFilter(IPairList):
                 "description": "Maximum Rate of Change",
                 "help": "Maximum rate of change to filter pairs.",
             },
+            "sort_direction": {
+                "type": "option",
+                "default": None,
+                "options": ["", "asc", "desc"],
+                "description": "Sort pairlist",
+                "help": "Sort Pairlist ascending or descending by rate of change.",
+            },
             **IPairList.refresh_period_parameter()
         }
 
@@ -100,53 +110,65 @@ class RangeStabilityFilter(IPairList):
         needed_pairs: ListPairsWithTimeframes = [
             (p, '1d', self._def_candletype) for p in pairlist if p not in self._pair_cache]
 
-        since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days - 1))
-        # Get all candles
-        candles = {}
-        if needed_pairs:
-            candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms,
-                                                          cache=False)
+        since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days + 1))
+        candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms=since_ms)
 
-        if self._enabled:
-            for p in deepcopy(pairlist):
-                daily_candles = candles[(p, '1d', self._def_candletype)] if (
-                    p, '1d', self._def_candletype) in candles else None
-                if not self._validate_pair_loc(p, daily_candles):
-                    pairlist.remove(p)
-        return pairlist
+        resulting_pairlist: List[str] = []
+        pct_changes: Dict[str, float] = {}
 
-    def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool:
-        """
-        Validate trading range
-        :param pair: Pair that's currently validated
-        :param daily_candles: Downloaded daily candles
-        :return: True if the pair can stay, false if it should be removed
-        """
+        for p in pairlist:
+            daily_candles = candles.get((p, '1d', self._def_candletype), None)
+
+            pct_change = self._calculate_rate_of_change(p, daily_candles)
+
+            if pct_change is not None:
+                if self._validate_pair_loc(p, pct_change):
+                    resulting_pairlist.append(p)
+                    pct_changes[p] = pct_change
+            else:
+                self.log_once(f"Removed {p} from whitelist, no candles found.", logger.info)
+
+        if self._sort_direction:
+            resulting_pairlist = sorted(resulting_pairlist,
+                                        key=lambda p: pct_changes[p],
+                                        reverse=self._sort_direction == 'desc')
+        return resulting_pairlist
+
+    def _calculate_rate_of_change(self, pair: str, daily_candles: DataFrame) -> Optional[float]:
         # Check symbol in cache
-        cached_res = self._pair_cache.get(pair, None)
-        if cached_res is not None:
-            return cached_res
-
-        result = True
+        if (pct_change := self._pair_cache.get(pair, None)) is not None:
+            return pct_change
         if daily_candles is not None and not daily_candles.empty:
+
             highest_high = daily_candles['high'].max()
             lowest_low = daily_candles['low'].min()
             pct_change = ((highest_high - lowest_low) / lowest_low) if lowest_low > 0 else 0
-            if pct_change < self._min_rate_of_change:
-                self.log_once(f"Removed {pair} from whitelist, because rate of change "
-                              f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, "
-                              f"which is below the threshold of {self._min_rate_of_change}.",
-                              logger.info)
-                result = False
-            if self._max_rate_of_change:
-                if pct_change > self._max_rate_of_change:
-                    self.log_once(
-                        f"Removed {pair} from whitelist, because rate of change "
-                        f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, "
-                        f"which is above the threshold of {self._max_rate_of_change}.",
-                        logger.info)
-                    result = False
-            self._pair_cache[pair] = result
+            self._pair_cache[pair] = pct_change
+            return pct_change
         else:
-            self.log_once(f"Removed {pair} from whitelist, no candles found.", logger.info)
+            return None
+
+    def _validate_pair_loc(self, pair: str, pct_change: float) -> bool:
+        """
+        Validate trading range
+        :param pair: Pair that's currently validated
+        :param pct_change: Rate of change
+        :return: True if the pair can stay, false if it should be removed
+        """
+
+        result = True
+        if pct_change < self._min_rate_of_change:
+            self.log_once(f"Removed {pair} from whitelist, because rate of change "
+                          f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, "
+                          f"which is below the threshold of {self._min_rate_of_change}.",
+                          logger.info)
+            result = False
+        if self._max_rate_of_change:
+            if pct_change > self._max_rate_of_change:
+                self.log_once(
+                    f"Removed {pair} from whitelist, because rate of change "
+                    f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, "
+                    f"which is above the threshold of {self._max_rate_of_change}.",
+                    logger.info)
+                result = False
         return result

freqtrade/rpc/api_server/api_schemas.py

@@ -261,6 +261,7 @@ class OrderSchema(BaseModel):
     order_timestamp: Optional[int] = None
     order_filled_timestamp: Optional[int] = None
     ft_fee_base: Optional[float] = None
+    ft_order_tag: Optional[str] = None
 
 
 class TradeSchema(BaseModel):
@@ -287,6 +288,8 @@ class TradeSchema(BaseModel):
 
     open_date: str
     open_timestamp: int
+    open_fill_date: Optional[str]
+    open_fill_timestamp: Optional[int]
     open_rate: float
     open_rate_requested: Optional[float] = None
     open_trade_value: float
@@ -314,7 +317,6 @@ class TradeSchema(BaseModel):
     stop_loss_abs: Optional[float] = None
     stop_loss_ratio: Optional[float] = None
     stop_loss_pct: Optional[float] = None
-    stoploss_order_id: Optional[str] = None
     stoploss_last_update: Optional[str] = None
     stoploss_last_update_timestamp: Optional[int] = None
     initial_stop_loss_abs: Optional[float] = None
@@ -397,7 +399,7 @@ class ForceEnterPayload(BaseModel):
 
 
 class ForceExitPayload(BaseModel):
-    tradeid: str
+    tradeid: Union[str, int]
     ordertype: Optional[OrderTypeValues] = None
     amount: Optional[float] = None
 
@@ -557,3 +559,7 @@ class SysInfo(BaseModel):
 class Health(BaseModel):
     last_process: Optional[datetime] = None
     last_process_ts: Optional[int] = None
+    bot_start: Optional[datetime] = None
+    bot_start_ts: Optional[int] = None
+    bot_startup: Optional[datetime] = None
+    bot_startup_ts: Optional[int] = None

freqtrade/rpc/api_server/api_v1.py

@@ -215,7 +215,7 @@ def force_entry(payload: ForceEnterPayload, rpc: RPC = Depends(get_rpc)):
 @router.post('/forcesell', response_model=ResultMsg, tags=['trading'])
 def forceexit(payload: ForceExitPayload, rpc: RPC = Depends(get_rpc)):
     ordertype = payload.ordertype.value if payload.ordertype else None
-    return rpc._rpc_force_exit(payload.tradeid, ordertype, amount=payload.amount)
+    return rpc._rpc_force_exit(str(payload.tradeid), ordertype, amount=payload.amount)
 
 
 @router.get('/blacklist', response_model=BlacklistResponse, tags=['info', 'pairlist'])

freqtrade/rpc/rpc.py

@@ -291,6 +291,10 @@ class RPC:
                         profit_str += f" ({fiat_profit:.2f})"
                         fiat_profit_sum = fiat_profit if isnan(fiat_profit_sum) \
                             else fiat_profit_sum + fiat_profit
+                else:
+                    profit_str += f" ({trade_profit:.2f})"
+                    fiat_profit_sum = trade_profit if isnan(fiat_profit_sum) \
+                        else fiat_profit_sum + trade_profit
 
                 active_attempt_side_symbols = [
                     '*' if (oo and oo.ft_order_side == trade.entry_side) else '**'
@@ -317,6 +321,8 @@ class RPC:
             profitcol = "Profit"
             if self._fiat_converter:
                 profitcol += " (" + fiat_display_currency + ")"
+            else:
+                profitcol += " (" + stake_currency + ")"
 
             columns = [
                 'ID L/S' if nonspot else 'ID',
@@ -927,6 +933,7 @@ class RPC:
                                              is_short=is_short,
                                              enter_tag=enter_tag,
                                              leverage_=leverage,
+                                             mode='pos_adjust' if trade else 'initial'
                                              ):
                 Trade.commit()
                 trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first()
@@ -979,15 +986,16 @@ class RPC:
                 except (ExchangeError):
                     pass
 
-            # cancel stoploss on exchange ...
+            # cancel stoploss on exchange orders ...
             if (self._freqtrade.strategy.order_types.get('stoploss_on_exchange')
-                    and trade.stoploss_order_id):
-                try:
-                    self._freqtrade.exchange.cancel_stoploss_order(trade.stoploss_order_id,
-                                                                   trade.pair)
-                    c_count += 1
-                except (ExchangeError):
-                    pass
+                    and trade.has_open_sl_orders):
+
+                for oslo in trade.open_sl_orders:
+                    try:
+                        self._freqtrade.exchange.cancel_stoploss_order(oslo.order_id, trade.pair)
+                        c_count += 1
+                    except (ExchangeError):
+                        pass
 
             trade.delete()
             self._freqtrade.wallets.update()
@@ -998,6 +1006,32 @@ class RPC:
                 'cancel_order_count': c_count,
             }
 
+    def _rpc_list_custom_data(self, trade_id: int, key: Optional[str]) -> List[Dict[str, Any]]:
+        # Query for trade
+        trade = Trade.get_trades(trade_filter=[Trade.id == trade_id]).first()
+        if trade is None:
+            return []
+        # Query custom_data
+        custom_data = []
+        if key:
+            data = trade.get_custom_data(key=key)
+            if data:
+                custom_data = [data]
+        else:
+            custom_data = trade.get_all_custom_data()
+        return [
+            {
+                'id': data_entry.id,
+                'ft_trade_id': data_entry.ft_trade_id,
+                'cd_key': data_entry.cd_key,
+                'cd_type': data_entry.cd_type,
+                'cd_value': data_entry.cd_value,
+                'created_at': data_entry.created_at,
+                'updated_at': data_entry.updated_at
+            }
+            for data_entry in custom_data
+        ]
+
     def _rpc_performance(self) -> List[Dict[str, Any]]:
         """
         Handler for performance.
@@ -1154,7 +1188,7 @@ class RPC:
         }
         if has_content:
 
-            dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].view(int64) // 1000 // 1000
+            dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].astype(int64) // 1000 // 1000
             # Move signal close to separate column when signal for easy plotting
             for sig_type in signals.keys():
                 if sig_type in dataframe.columns:
@@ -1332,19 +1366,40 @@ class RPC:
 
     def health(self) -> Dict[str, Optional[Union[str, int]]]:
         last_p = self._freqtrade.last_process
-        if last_p is None:
-            return {
-                "last_process": None,
-                "last_process_loc": None,
-                "last_process_ts": None,
-            }
-
-        return {
-            "last_process": str(last_p),
-            "last_process_loc": format_date(last_p.astimezone(tzlocal())),
-            "last_process_ts": int(last_p.timestamp()),
+        res: Dict[str, Union[None, str, int]] = {
+            "last_process": None,
+            "last_process_loc": None,
+            "last_process_ts": None,
+            "bot_start": None,
+            "bot_start_loc": None,
+            "bot_start_ts": None,
+            "bot_startup": None,
+            "bot_startup_loc": None,
+            "bot_startup_ts": None,
         }
 
+        if last_p is not None:
+            res.update({
+                "last_process": str(last_p),
+                "last_process_loc": format_date(last_p.astimezone(tzlocal())),
+                "last_process_ts": int(last_p.timestamp()),
+            })
+
+        if (bot_start := KeyValueStore.get_datetime_value(KeyStoreKeys.BOT_START_TIME)):
+            res.update({
+                "bot_start": str(bot_start),
+                "bot_start_loc": format_date(bot_start.astimezone(tzlocal())),
+                "bot_start_ts": int(bot_start.timestamp()),
+            })
+        if (bot_startup := KeyValueStore.get_datetime_value(KeyStoreKeys.STARTUP_TIME)):
+            res.update({
+                "bot_startup": str(bot_startup),
+                "bot_startup_loc": format_date(bot_startup.astimezone(tzlocal())),
+                "bot_startup_ts": int(bot_startup.timestamp()),
+            })
+
+        return res
+
     def _update_market_direction(self, direction: MarketDirection) -> None:
         self._freqtrade.strategy.market_direction = direction
 

freqtrade/rpc/telegram.py

@@ -33,7 +33,7 @@ from freqtrade.misc import chunks, plural
 from freqtrade.persistence import Trade
 from freqtrade.rpc import RPC, RPCException, RPCHandler
 from freqtrade.rpc.rpc_types import RPCEntryMsg, RPCExitMsg, RPCOrderMsg, RPCSendMsg
-from freqtrade.util import dt_humanize, fmt_coin, round_value
+from freqtrade.util import dt_humanize, fmt_coin, format_date, round_value
 
 
 MAX_MESSAGE_LENGTH = MessageLimit.MAX_TEXT_LENGTH
@@ -243,6 +243,7 @@ class Telegram(RPCHandler):
             CommandHandler('version', self._version),
             CommandHandler('marketdir', self._changemarketdir),
             CommandHandler('order', self._order),
+            CommandHandler('list_custom_data', self._list_custom_data),
         ]
         callbacks = [
             CallbackQueryHandler(self._status_table, pattern='update_status_table'),
@@ -353,7 +354,7 @@ class Telegram(RPCHandler):
         message += f"*Amount:* `{round_value(msg['amount'], 8)}`\n"
         message += f"*Direction:* `{msg['direction']}"
         if msg.get('leverage') and msg.get('leverage', 1.0) != 1.0:
-            message += f" ({msg['leverage']:.1g}x)"
+            message += f" ({msg['leverage']:.3g}x)"
         message += "`\n"
         message += f"*Open Rate:* `{fmt_coin(msg['open_rate'], msg['quote_currency'])}`\n"
         if msg['type'] == RPCMessageType.ENTRY and msg['current_rate']:
@@ -371,7 +372,7 @@ class Telegram(RPCHandler):
             microsecond=0) - msg['open_date'].replace(microsecond=0)
         duration_min = duration.total_seconds() / 60
 
-        leverage_text = (f" ({msg['leverage']:.1g}x)"
+        leverage_text = (f" ({msg['leverage']:.3g}x)"
                          if msg.get('leverage') and msg.get('leverage', 1.0) != 1.0
                          else "")
 
@@ -1364,7 +1365,7 @@ class Telegram(RPCHandler):
     @authorized_only
     async def _enter_tag_performance(self, update: Update, context: CallbackContext) -> None:
         """
-        Handler for /buys PAIR .
+        Handler for /entries PAIR .
         Shows a performance statistic from finished trades
         :param bot: telegram bot
         :param update: message update
@@ -1375,28 +1376,28 @@ class Telegram(RPCHandler):
             pair = context.args[0]
 
         trades = self._rpc._rpc_enter_tag_performance(pair)
-        output = "<b>Entry Tag Performance:</b>\n"
+        output = "*Entry Tag Performance:*\n"
         for i, trade in enumerate(trades):
             stat_line = (
-                f"{i + 1}.\t <code>{trade['enter_tag']}\t"
+                f"{i + 1}.\t `{trade['enter_tag']}\t"
                 f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} "
                 f"({trade['profit_ratio']:.2%}) "
-                f"({trade['count']})</code>\n")
+                f"({trade['count']})`\n")
 
             if len(output + stat_line) >= MAX_MESSAGE_LENGTH:
-                await self._send_msg(output, parse_mode=ParseMode.HTML)
+                await self._send_msg(output, parse_mode=ParseMode.MARKDOWN)
                 output = stat_line
             else:
                 output += stat_line
 
-        await self._send_msg(output, parse_mode=ParseMode.HTML,
+        await self._send_msg(output, parse_mode=ParseMode.MARKDOWN,
                              reload_able=True, callback_path="update_enter_tag_performance",
                              query=update.callback_query)
 
     @authorized_only
     async def _exit_reason_performance(self, update: Update, context: CallbackContext) -> None:
         """
-        Handler for /sells.
+        Handler for /exits.
         Shows a performance statistic from finished trades
         :param bot: telegram bot
         :param update: message update
@@ -1407,21 +1408,21 @@ class Telegram(RPCHandler):
             pair = context.args[0]
 
         trades = self._rpc._rpc_exit_reason_performance(pair)
-        output = "<b>Exit Reason Performance:</b>\n"
+        output = "*Exit Reason Performance:*\n"
         for i, trade in enumerate(trades):
             stat_line = (
-                f"{i + 1}.\t <code>{trade['exit_reason']}\t"
+                f"{i + 1}.\t `{trade['exit_reason']}\t"
                 f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} "
                 f"({trade['profit_ratio']:.2%}) "
-                f"({trade['count']})</code>\n")
+                f"({trade['count']})`\n")
 
             if len(output + stat_line) >= MAX_MESSAGE_LENGTH:
-                await self._send_msg(output, parse_mode=ParseMode.HTML)
+                await self._send_msg(output, parse_mode=ParseMode.MARKDOWN)
                 output = stat_line
             else:
                 output += stat_line
 
-        await self._send_msg(output, parse_mode=ParseMode.HTML,
+        await self._send_msg(output, parse_mode=ParseMode.MARKDOWN,
                              reload_able=True, callback_path="update_exit_reason_performance",
                              query=update.callback_query)
 
@@ -1439,21 +1440,21 @@ class Telegram(RPCHandler):
             pair = context.args[0]
 
         trades = self._rpc._rpc_mix_tag_performance(pair)
-        output = "<b>Mix Tag Performance:</b>\n"
+        output = "*Mix Tag Performance:*\n"
         for i, trade in enumerate(trades):
             stat_line = (
-                f"{i + 1}.\t <code>{trade['mix_tag']}\t"
+                f"{i + 1}.\t `{trade['mix_tag']}\t"
                 f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} "
                 f"({trade['profit_ratio']:.2%}) "
-                f"({trade['count']})</code>\n")
+                f"({trade['count']})`\n")
 
             if len(output + stat_line) >= MAX_MESSAGE_LENGTH:
-                await self._send_msg(output, parse_mode=ParseMode.HTML)
+                await self._send_msg(output, parse_mode=ParseMode.MARKDOWN)
                 output = stat_line
             else:
                 output += stat_line
 
-        await self._send_msg(output, parse_mode=ParseMode.HTML,
+        await self._send_msg(output, parse_mode=ParseMode.MARKDOWN,
                              reload_able=True, callback_path="update_mix_tag_performance",
                              query=update.callback_query)
 
@@ -1667,6 +1668,8 @@ class Telegram(RPCHandler):
             "*/marketdir [long | short | even | none]:* `Updates the user managed variable "
             "that represents the current market direction. If no direction is provided `"
             "`the currently set market direction will be output.` \n"
+            "*/list_custom_data <trade_id> <key>:* `List custom_data for Trade ID & Key combo.`\n"
+            "`If no Key is supplied it will list all key-value pairs found for that Trade ID.`"
 
             "_Statistics_\n"
             "------------\n"
@@ -1676,8 +1679,8 @@ class Telegram(RPCHandler):
             "         *table :* `will display trades in a table`\n"
             "                `pending buy orders are marked with an asterisk (*)`\n"
             "                `pending sell orders are marked with a double asterisk (**)`\n"
-            "*/buys <pair|none>:* `Shows the enter_tag performance`\n"
-            "*/sells <pair|none>:* `Shows the exit reason performance`\n"
+            "*/entries <pair|none>:* `Shows the enter_tag performance`\n"
+            "*/exits <pair|none>:* `Shows the exit reason performance`\n"
             "*/mix_tags <pair|none>:* `Shows combined entry tag + exit reason performance`\n"
             "*/trades [limit]:* `Lists last closed trades (limited to 10 by default)`\n"
             "*/profit [<n>]:* `Lists cumulative profit from all finished trades, "
@@ -1689,7 +1692,7 @@ class Telegram(RPCHandler):
             "*/stats:* `Shows Wins / losses by Sell reason as well as "
             "Avg. holding durations for buys and sells.`\n"
             "*/help:* `This help message`\n"
-            "*/version:* `Show version`"
+            "*/version:* `Show version`\n"
             )
 
         await self._send_msg(message, parse_mode=ParseMode.MARKDOWN)
@@ -1701,7 +1704,9 @@ class Telegram(RPCHandler):
         Shows the last process timestamp
         """
         health = self._rpc.health()
-        message = f"Last process: `{health['last_process_loc']}`"
+        message = f"Last process: `{health['last_process_loc']}`\n"
+        message += f"Initial bot start: `{health['bot_start_loc']}`\n"
+        message += f"Last bot restart: `{health['bot_startup_loc']}`"
         await self._send_msg(message)
 
     @authorized_only
@@ -1766,6 +1771,53 @@ class Telegram(RPCHandler):
             f"*Current state:* `{val['state']}`"
         )
 
+    @authorized_only
+    async def _list_custom_data(self, update: Update, context: CallbackContext) -> None:
+        """
+        Handler for /list_custom_data <id> <key>.
+        List custom_data for specified trade (and key if supplied).
+        :param bot: telegram bot
+        :param update: message update
+        :return: None
+        """
+        try:
+            if not context.args or len(context.args) == 0:
+                raise RPCException("Trade-id not set.")
+            trade_id = int(context.args[0])
+            key = None if len(context.args) < 2 else str(context.args[1])
+
+            results = self._rpc._rpc_list_custom_data(trade_id, key)
+            messages = []
+            if len(results) > 0:
+                messages.append(
+                    'Found custom-data entr' + ('ies: ' if len(results) > 1 else 'y: ')
+                )
+                for result in results:
+                    lines = [
+                        f"*Key:* `{result['cd_key']}`",
+                        f"*ID:* `{result['id']}`",
+                        f"*Trade ID:* `{result['ft_trade_id']}`",
+                        f"*Type:* `{result['cd_type']}`",
+                        f"*Value:* `{result['cd_value']}`",
+                        f"*Create Date:* `{format_date(result['created_at'])}`",
+                        f"*Update Date:* `{format_date(result['updated_at'])}`"
+                    ]
+                    # Filter empty lines using list-comprehension
+                    messages.append("\n".join([line for line in lines if line]))
+                for msg in messages:
+                    if len(msg) > MAX_MESSAGE_LENGTH:
+                        msg = "Message dropped because length exceeds "
+                        msg += f"maximum allowed characters: {MAX_MESSAGE_LENGTH}"
+                        logger.warning(msg)
+                    await self._send_msg(msg)
+            else:
+                message = f"Didn't find any custom-data entries for Trade ID: `{trade_id}`"
+                message += f" and Key: `{key}`." if key is not None else ""
+                await self._send_msg(message)
+
+        except RPCException as e:
+            await self._send_msg(str(e))
+
     async def _update_msg(self, query: CallbackQuery, msg: str, callback_path: str = "",
                           reload_able: bool = False, parse_mode: str = ParseMode.MARKDOWN) -> None:
         if reload_able:
@@ -1777,13 +1829,9 @@ class Telegram(RPCHandler):
         msg += f"\nUpdated: {datetime.now().ctime()}"
         if not query.message:
             return
-        chat_id = query.message.chat_id
-        message_id = query.message.message_id
 
         try:
-            await self._app.bot.edit_message_text(
-                chat_id=chat_id,
-                message_id=message_id,
+            await query.edit_message_text(
                 text=msg,
                 parse_mode=parse_mode,
                 reply_markup=reply_markup

freqtrade/strategy/interface.py

@@ -511,7 +511,8 @@ class IStrategy(ABC, HyperStrategyMixin):
                               min_stake: Optional[float], max_stake: float,
                               current_entry_rate: float, current_exit_rate: float,
                               current_entry_profit: float, current_exit_profit: float,
-                              **kwargs) -> Optional[float]:
+                              **kwargs
+                              ) -> Union[Optional[float], Tuple[Optional[float], Optional[str]]]:
         """
         Custom trade adjustment logic, returning the stake amount that a trade should be
         increased or decreased.
@@ -537,6 +538,7 @@ class IStrategy(ABC, HyperStrategyMixin):
         :return float: Stake amount to adjust your trade,
                        Positive values to increase position, Negative values to decrease position.
                        Return None for no action.
+                       Optionally, return a tuple with a 2nd element with an order reason
         """
         return None
 
@@ -725,6 +727,36 @@ class IStrategy(ABC, HyperStrategyMixin):
 
     _ft_stop_uses_after_fill = False
 
+    def _adjust_trade_position_internal(
+            self, trade: Trade, current_time: datetime,
+            current_rate: float, current_profit: float,
+            min_stake: Optional[float], max_stake: float,
+            current_entry_rate: float, current_exit_rate: float,
+            current_entry_profit: float, current_exit_profit: float,
+            **kwargs
+    ) -> Tuple[Optional[float], str]:
+        """
+        wrapper around adjust_trade_position to handle the return value
+        """
+        resp = strategy_safe_wrapper(self.adjust_trade_position,
+                                     default_retval=(None, ''), supress_error=True)(
+            trade=trade, current_time=current_time,
+            current_rate=current_rate, current_profit=current_profit,
+            min_stake=min_stake, max_stake=max_stake,
+            current_entry_rate=current_entry_rate, current_exit_rate=current_exit_rate,
+            current_entry_profit=current_entry_profit, current_exit_profit=current_exit_profit,
+            **kwargs
+        )
+        order_tag = ''
+        if isinstance(resp, tuple):
+            if len(resp) >= 1:
+                stake_amount = resp[0]
+            if len(resp) > 1:
+                order_tag = resp[1] or ''
+        else:
+            stake_amount = resp
+        return stake_amount, order_tag
+
     def __informative_pairs_freqai(self) -> ListPairsWithTimeframes:
         """
         Create informative-pairs needed for FreqAI

freqtrade/templates/strategy_analysis_example.ipynb

@@ -35,7 +35,7 @@
     "project_root = \"somedir/freqtrade\"\n",
     "i=0\n",
     "try:\n",
-    "    os.chdirdir(project_root)\n",
+    "    os.chdir(project_root)\n",
     "    assert Path('LICENSE').is_file()\n",
     "except:\n",
     "    while i<4 and (not Path('LICENSE').is_file()):\n",
@@ -181,7 +181,7 @@
     "\n",
     "# if backtest_dir points to a directory, it'll automatically load the last backtest file.\n",
     "backtest_dir = config[\"user_data_dir\"] / \"backtest_results\"\n",
-    "# backtest_dir can also point to a specific file \n",
+    "# backtest_dir can also point to a specific file\n",
     "# backtest_dir = config[\"user_data_dir\"] / \"backtest_results/backtest-result-2020-07-01_20-04-22.json\""
    ]
   },

freqtrade/util/__init__.py

@@ -1,6 +1,6 @@
 from freqtrade.util.datetime_helpers import (dt_floor_day, dt_from_ts, dt_humanize, dt_now, dt_ts,
-                                             dt_ts_def, dt_utc, format_date, format_ms_time,
-                                             shorten_date)
+                                             dt_ts_def, dt_ts_none, dt_utc, format_date,
+                                             format_ms_time, shorten_date)
 from freqtrade.util.formatters import decimals_per_coin, fmt_coin, round_value
 from freqtrade.util.ft_precise import FtPrecise
 from freqtrade.util.periodic_cache import PeriodicCache
@@ -14,6 +14,7 @@ __all__ = [
     'dt_now',
     'dt_ts',
     'dt_ts_def',
+    'dt_ts_none',
     'dt_utc',
     'format_date',
     'format_ms_time',

freqtrade/util/datetime_helpers.py

@@ -31,13 +31,23 @@ def dt_ts(dt: Optional[datetime] = None) -> int:
 def dt_ts_def(dt: Optional[datetime], default: int = 0) -> int:
     """
     Return dt in ms as a timestamp in UTC.
-    If dt is None, return the current datetime in UTC.
+    If dt is None, return the given default.
     """
     if dt:
         return int(dt.timestamp() * 1000)
     return default
 
 
+def dt_ts_none(dt: Optional[datetime]) -> Optional[int]:
+    """
+    Return dt in ms as a timestamp in UTC.
+    If dt is None, return the given default.
+    """
+    if dt:
+        return int(dt.timestamp() * 1000)
+    return None
+
+
 def dt_floor_day(dt: datetime) -> datetime:
     """Return the floor of the day for the given datetime."""
     return dt.replace(hour=0, minute=0, second=0, microsecond=0)

freqtrade/wallets.py

@@ -36,9 +36,9 @@ class PositionWallet(NamedTuple):
 
 class Wallets:
 
-    def __init__(self, config: Config, exchange: Exchange, log: bool = True) -> None:
+    def __init__(self, config: Config, exchange: Exchange, is_backtest: bool = False) -> None:
         self._config = config
-        self._log = log
+        self._is_backtest = is_backtest
         self._exchange = exchange
         self._wallets: Dict[str, Wallet] = {}
         self._positions: Dict[str, PositionWallet] = {}
@@ -78,11 +78,11 @@ class Wallets:
         _wallets = {}
         _positions = {}
         open_trades = Trade.get_trades_proxy(is_open=True)
-        # If not backtesting...
-        # TODO: potentially remove the ._log workaround to determine backtest mode.
-        if self._log:
+        if not self._is_backtest:
+            # Live / Dry-run mode
             tot_profit = Trade.get_total_closed_profit()
         else:
+            # Backtest mode
             tot_profit = LocalTrade.total_profit
         tot_profit += sum(trade.realized_profit for trade in open_trades)
         tot_in_trades = sum(trade.stake_amount for trade in open_trades)
@@ -177,7 +177,7 @@ class Wallets:
                 self._update_live()
             else:
                 self._update_dry()
-            if self._log:
+            if not self._is_backtest:
                 logger.info('Wallets synced.')
             self._last_wallet_refresh = dt_now()
 
@@ -341,19 +341,19 @@ class Wallets:
             max_allowed_stake = min(max_allowed_stake, max_stake_amount - trade_amount)
 
         if min_stake_amount is not None and min_stake_amount > max_allowed_stake:
-            if self._log:
+            if not self._is_backtest:
                 logger.warning("Minimum stake amount > available balance. "
                                f"{min_stake_amount} > {max_allowed_stake}")
             return 0
         if min_stake_amount is not None and stake_amount < min_stake_amount:
-            if self._log:
+            if not self._is_backtest:
                 logger.info(
                     f"Stake amount for pair {pair} is too small "
                     f"({stake_amount} < {min_stake_amount}), adjusting to {min_stake_amount}."
                 )
             if stake_amount * 1.3 < min_stake_amount:
                 # Top-cap stake-amount adjustments to +30%.
-                if self._log:
+                if not self._is_backtest:
                     logger.info(
                         f"Adjusted stake amount for pair {pair} is more than 30% bigger than "
                         f"the desired stake amount of ({stake_amount:.8f} * 1.3 = "
@@ -363,7 +363,7 @@ class Wallets:
             stake_amount = min_stake_amount
 
         if stake_amount > max_allowed_stake:
-            if self._log:
+            if not self._is_backtest:
                 logger.info(
                     f"Stake amount for pair {pair} is too big "
                     f"({stake_amount} > {max_allowed_stake}), adjusting to {max_allowed_stake}."

pyproject.toml

@@ -115,6 +115,8 @@ ignore = ["freqtrade/vendor/**"]
 line-length = 100
 extend-exclude = [".env", ".venv"]
 target-version = "py38"
+
+[tool.ruff.lint]
 # Exclude UP036 as it's causing the "exit if < 3.9" to fail.
 extend-select = [
   "C90",    # mccabe
@@ -132,16 +134,17 @@ extend-select = [
   # "TCH",  # flake8-type-checking
   "PTH",    # flake8-use-pathlib
 ]
+
 extend-ignore = [
   "E241",  # Multiple spaces after comma
   "E272",  # Multiple spaces before keyword
   "E221",  # Multiple spaces before operator
 ]
 
-[tool.ruff.mccabe]
+[tool.ruff.lint.mccabe]
 max-complexity = 12
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "tests/*" = ["S"]
 
 [tool.flake8]

requirements-dev.txt

@@ -7,25 +7,25 @@
 -r docs/requirements-docs.txt
 
 coveralls==3.3.1
-ruff==0.1.15
-mypy==1.8.0
-pre-commit==3.6.0
-pytest==7.4.4
-pytest-asyncio==0.23.4
+ruff==0.3.0
+mypy==1.9.0
+pre-commit==3.6.2
+pytest==8.1.1
+pytest-asyncio==0.23.5.post1
 pytest-cov==4.1.0
 pytest-mock==3.12.0
 pytest-random-order==1.1.1
 pytest-xdist==3.5.0
 isort==5.13.2
 # For datetime mocking
-time-machine==2.13.0
+time-machine==2.14.0
 
 # Convert jupyter notebooks to markdown documents
-nbconvert==7.14.2
+nbconvert==7.16.2
 
 # mypy types
 types-cachetools==5.3.0.7
 types-filelock==3.2.7
-types-requests==2.31.0.20240125
+types-requests==2.31.0.20240311
 types-tabulate==0.9.0.20240106
-types-python-dateutil==2.8.19.20240106
+types-python-dateutil==2.8.19.20240311

requirements-freqai-rl.txt

@@ -8,4 +8,4 @@ gymnasium==0.29.1; python_version < '3.12'
 stable_baselines3==2.2.1; python_version < '3.12'
 sb3_contrib>=2.0.0a9; python_version < '3.12'
 # Progress bar for stable-baselines3 and sb3-contrib
-tqdm==4.66.1
+tqdm==4.66.2

requirements-freqai.txt

@@ -3,10 +3,10 @@
 -r requirements-plot.txt
 
 # Required for freqai
-scikit-learn==1.4.0
+scikit-learn==1.4.1.post1
 joblib==1.3.2
-catboost==1.2.2; 'arm' not in platform_machine and python_version < '3.12'
-lightgbm==4.2.0
+catboost==1.2.3; 'arm' not in platform_machine
+lightgbm==4.3.0
 xgboost==2.0.3
-tensorboard==2.15.1
+tensorboard==2.16.2
 datasieve==0.1.7

requirements-hyperopt.txt

@@ -3,6 +3,6 @@
 
 # Required for hyperopt
 scipy==1.12.0
-scikit-learn==1.4.0
+scikit-learn==1.4.1.post1
 ft-scikit-optimize==0.9.2
 filelock==3.13.1

requirements-plot.txt

@@ -1,4 +1,4 @@
 # Include all requirements to run the bot.
 -r requirements.txt
 
-plotly==5.18.0
+plotly==5.19.0

requirements.txt

@@ -1,44 +1,44 @@
-numpy==1.26.3
-pandas==2.1.4
+numpy==1.26.4
+pandas==2.2.1
 pandas-ta==0.3.14b
 
-ccxt==4.2.25
-cryptography==42.0.1
-aiohttp==3.9.2
-SQLAlchemy==2.0.25
-python-telegram-bot==20.7
+ccxt==4.2.66
+cryptography==42.0.5
+aiohttp==3.9.3
+SQLAlchemy==2.0.27
+python-telegram-bot==21.0.1
 # can't be hard-pinned due to telegram-bot pinning httpx with ~
 httpx>=0.24.1
 arrow==1.3.0
-cachetools==5.3.2
+cachetools==5.3.3
 requests==2.31.0
-urllib3==2.1.0
+urllib3==2.2.1
 jsonschema==4.21.1
 TA-Lib==0.4.28
-technical==1.4.2
+technical==1.4.3
 tabulate==0.9.0
 pycoingecko==3.1.0
 jinja2==3.1.3
 tables==3.9.1
 joblib==1.3.2
-rich==13.7.0
+rich==13.7.1
 pyarrow==15.0.0; platform_machine != 'armv7l'
 
 # find first, C search in arrays
 py_find_1st==1.1.6
 
 # Load ticker files 30% faster
-python-rapidjson==1.14
+python-rapidjson==1.16
 # Properly format api responses
-orjson==3.9.12
+orjson==3.9.15
 
 # Notify systemd
 sdnotify==0.3.2
 
 # API Server
-fastapi==0.109.0
-pydantic==2.5.3
-uvicorn==0.27.0
+fastapi==0.110.0
+pydantic==2.6.3
+uvicorn==0.28.0
 pyjwt==2.8.0
 aiofiles==23.2.1
 psutil==5.9.8
@@ -49,7 +49,8 @@ colorama==0.4.6
 questionary==2.0.1
 prompt-toolkit==3.0.36
 # Extensions to datetime library
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
+pytz==2024.1
 
 #Futures
 schedule==1.2.1
@@ -59,4 +60,4 @@ websockets==12.0
 janus==1.0.0
 
 ast-comments==1.2.1
-packaging==23.2
+packaging==24.0

setup.py

@@ -35,21 +35,21 @@ hdf5 = [
 
 develop = [
     'coveralls',
+    'isort',
     'mypy',
-    'ruff',
     'pre-commit',
-    'pytest',
     'pytest-asyncio',
     'pytest-cov',
     'pytest-mock',
     'pytest-random-order',
-    'isort',
+    'pytest',
+    'ruff',
     'time-machine',
     'types-cachetools',
     'types-filelock',
+    'types-python-dateutil'
     'types-requests',
     'types-tabulate',
-    'types-python-dateutil'
 ]
 
 jupyter = [
@@ -70,14 +70,17 @@ setup(
     ],
     install_requires=[
         # from requirements.txt
-        'ccxt>=4.2.15',
+        'ccxt>=4.2.47',
         'SQLAlchemy>=2.0.6',
         'python-telegram-bot>=20.1',
         'arrow>=1.0.0',
         'cachetools',
         'requests',
+        'httpx>=0.24.1',
         'urllib3',
         'jsonschema',
+        'numpy',
+        'pandas',
         'TA-Lib',
         'pandas-ta',
         'technical',
@@ -86,30 +89,28 @@ setup(
         'py_find_1st',
         'python-rapidjson',
         'orjson',
-        'sdnotify',
         'colorama',
         'jinja2',
         'questionary',
         'prompt-toolkit',
-        'numpy',
-        'pandas',
         'joblib>=1.2.0',
         'rich',
         'pyarrow; platform_machine != "armv7l"',
         'fastapi',
         'pydantic>=2.2.0',
+        'pyjwt',
+        'websockets',
         'uvicorn',
         'psutil',
-        'pyjwt',
-        'aiofiles',
         'schedule',
-        'websockets',
         'janus',
         'ast-comments',
+        'aiofiles',
         'aiohttp',
         'cryptography',
-        'httpx>=0.24.1',
+        'sdnotify',
         'python-dateutil',
+        'pytz',
         'packaging',
     ],
     extras_require={

setup.sh

@@ -161,7 +161,7 @@ function install_macos() {
         /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
     fi
 
-    brew install gettext
+    brew install gettext libomp
 
     #Gets number after decimal in python version
     version=$(egrep -o 3.\[0-9\]+ <<< $PYTHON | sed 's/3.//g')

tests/commands/test_commands.py

@@ -820,11 +820,6 @@ def test_download_data_trades(mocker):
         "--trading-mode", "futures",
         "--dl-trades"
     ]
-    with pytest.raises(OperationalException,
-                       match="Trade download not supported for futures."):
-        pargs = get_args(args)
-        pargs['config'] = None
-        start_download_data(pargs)
 
 
 def test_download_data_data_invalid(mocker):
@@ -842,10 +837,11 @@ def test_download_data_data_invalid(mocker):
         start_download_data(pargs)
 
 
-def test_start_convert_trades(mocker, caplog):
+def test_start_convert_trades(mocker):
     convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv',
                                 MagicMock(return_value=[]))
     patch_exchange(mocker)
+    mocker.patch(f'{EXMS}.get_markets')
     mocker.patch(f'{EXMS}.markets', PropertyMock(return_value={}))
     args = [
         "trades-to-ohlcv",

tests/conftest.py

@@ -142,8 +142,8 @@ def generate_trades_history(n_rows, start_date: Optional[datetime] = None, days=
     return df
 
 
-def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05'):
-    np.random.seed(42)
+def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05', random_seed=42):
+    np.random.seed(random_seed)
 
     base = np.random.normal(20, 2, size=size)
     if timeframe == '1y':
@@ -174,10 +174,10 @@ def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05'):
     return df
 
 
-def generate_test_data_raw(timeframe: str, size: int, start: str = '2020-07-05'):
+def generate_test_data_raw(timeframe: str, size: int, start: str = '2020-07-05', random_seed=42):
     """ Generates data in the ohlcv format used by ccxt """
-    df = generate_test_data(timeframe, size, start)
-    df['date'] = df.loc[:, 'date'].view(np.int64) // 1000 // 1000
+    df = generate_test_data(timeframe, size, start, random_seed)
+    df['date'] = df.loc[:, 'date'].astype(np.int64) // 1000 // 1000
     return list(list(x) for x in zip(*(df[x].values.tolist() for x in df.columns)))
 
 
@@ -3342,7 +3342,7 @@ def leverage_tiers():
                 'maintAmt': 386950.0
             },
         ],
-        "ADA/BUSD:BUSD": [
+        "ADA/USDT:USDT": [
             {
                 "minNotional": 0,
                 "maxNotional": 100000,
@@ -3386,7 +3386,7 @@ def leverage_tiers():
                 "maintAmt": 1527500.0
             },
         ],
-        'BNB/BUSD:BUSD': [
+        'XRP/USDT:USDT': [
             {
                 "minNotional": 0,       # stake(before leverage) = 0
                 "maxNotional": 100000,  # max stake(before leverage) = 5000

tests/conftest_trades.py

@@ -266,7 +266,6 @@ def mock_trade_5(fee, is_short: bool):
         exchange='binance',
         strategy='SampleStrategy',
         enter_tag='TEST1',
-        stoploss_order_id=f'prod_stoploss_{direc(is_short)}_3455',
         timeframe=5,
         is_short=is_short,
         stop_loss_pct=0.10,

tests/conftest_trades_usdt.py

@@ -282,7 +282,6 @@ def mock_trade_usdt_5(fee, is_short: bool):
         open_rate=2.0,
         exchange='binance',
         strategy='SampleStrategy',
-        stoploss_order_id=f'prod_stoploss_3455_{direc(is_short)}',
         timeframe=5,
         is_short=is_short,
     )

tests/data/test_btanalysis.py

@@ -455,6 +455,13 @@ def test_calculate_max_drawdown2():
     with pytest.raises(ValueError, match='No losing trade, therefore no drawdown.'):
         calculate_max_drawdown(df, date_col='open_date', value_col='profit')
 
+    df1 = DataFrame(zip(values[:5], dates[:5]), columns=['profit', 'open_date'])
+    df1.loc[:, 'profit'] = df1['profit'] * -1
+    # No winning trade ...
+    drawdown, hdate, ldate, hval, lval, drawdown_rel = calculate_max_drawdown(
+        df1, date_col='open_date', value_col='profit')
+    assert drawdown == 0.043965
+
 
 @pytest.mark.parametrize('profits,relative,highd,lowd,result,result_rel', [
     ([0.0, -500.0, 500.0, 10000.0, -1000.0], False, 3, 4, 1000.0, 0.090909),

tests/data/test_converter.py

@@ -542,7 +542,9 @@ def test_convert_trades_to_ohlcv(testdatadir, tmp_path, caplog):
 
     convert_trades_to_ohlcv([pair], timeframes=['1m', '5m'],
                             data_format_trades='jsongz',
-                            datadir=tmp_path, timerange=tr, erase=True)
+                            datadir=tmp_path, timerange=tr, erase=True,
+                            data_format_ohlcv='feather',
+                            candle_type=CandleType.SPOT)
 
     assert log_has("Deleting existing data for pair XRP/ETH, interval 1m.", caplog)
     # Load new data
@@ -556,5 +558,7 @@ def test_convert_trades_to_ohlcv(testdatadir, tmp_path, caplog):
 
     convert_trades_to_ohlcv(['NoDatapair'], timeframes=['1m', '5m'],
                             data_format_trades='jsongz',
-                            datadir=tmp_path, timerange=tr, erase=True)
+                            datadir=tmp_path, timerange=tr, erase=True,
+                            data_format_ohlcv='feather',
+                            candle_type=CandleType.SPOT)
     assert log_has(msg, caplog)

tests/data/test_datahandler.py

@@ -261,11 +261,11 @@ def test_datahandler_trades_not_supported(datahandler, testdatadir, ):
 def test_jsondatahandler_trades_load(testdatadir, caplog):
     dh = JsonGzDataHandler(testdatadir)
     logmsg = "Old trades format detected - converting"
-    dh.trades_load('XRP/ETH')
+    dh.trades_load('XRP/ETH', TradingMode.SPOT)
     assert not log_has(logmsg, caplog)
 
     # Test conversation is happening
-    dh.trades_load('XRP/OLD')
+    dh.trades_load('XRP/OLD', TradingMode.SPOT)
     assert log_has(logmsg, caplog)
 
 
@@ -300,16 +300,16 @@ def test_datahandler_trades_get_pairs(testdatadir, datahandler, expected):
 
 def test_hdf5datahandler_trades_load(testdatadir):
     dh = get_datahandler(testdatadir, 'hdf5')
-    trades = dh.trades_load('XRP/ETH')
+    trades = dh.trades_load('XRP/ETH', TradingMode.SPOT)
     assert isinstance(trades, DataFrame)
 
-    trades1 = dh.trades_load('UNITTEST/NONEXIST')
+    trades1 = dh.trades_load('UNITTEST/NONEXIST', TradingMode.SPOT)
     assert isinstance(trades1, DataFrame)
     assert trades1.empty
     # data goes from 2019-10-11 - 2019-10-13
     timerange = TimeRange.parse_timerange('20191011-20191012')
 
-    trades2 = dh._trades_load('XRP/ETH', timerange)
+    trades2 = dh._trades_load('XRP/ETH', TradingMode.SPOT, timerange)
     assert len(trades) > len(trades2)
     # Check that ID is None (If it's nan, it's wrong)
     assert trades2.iloc[0]['type'] is None
@@ -451,13 +451,13 @@ def test_hdf5datahandler_ohlcv_purge(mocker, testdatadir):
 @pytest.mark.parametrize('datahandler', ['jsongz', 'hdf5', 'feather', 'parquet'])
 def test_datahandler_trades_load(testdatadir, datahandler):
     dh = get_datahandler(testdatadir, datahandler)
-    trades = dh.trades_load('XRP/ETH')
+    trades = dh.trades_load('XRP/ETH', TradingMode.SPOT)
     assert isinstance(trades, DataFrame)
     assert trades.iloc[0]['timestamp'] == 1570752011620
     assert trades.iloc[0]['date'] == Timestamp('2019-10-11 00:00:11.620000+0000')
     assert trades.iloc[-1]['cost'] == 0.1986231
 
-    trades1 = dh.trades_load('UNITTEST/NONEXIST')
+    trades1 = dh.trades_load('UNITTEST/NONEXIST', TradingMode.SPOT)
     assert isinstance(trades, DataFrame)
     assert trades1.empty
 
@@ -465,15 +465,15 @@ def test_datahandler_trades_load(testdatadir, datahandler):
 @pytest.mark.parametrize('datahandler', ['jsongz', 'hdf5', 'feather', 'parquet'])
 def test_datahandler_trades_store(testdatadir, tmp_path, datahandler):
     dh = get_datahandler(testdatadir, datahandler)
-    trades = dh.trades_load('XRP/ETH')
+    trades = dh.trades_load('XRP/ETH', TradingMode.SPOT)
 
     dh1 = get_datahandler(tmp_path, datahandler)
-    dh1.trades_store('XRP/NEW', trades)
+    dh1.trades_store('XRP/NEW', trades, TradingMode.SPOT)
 
     file = tmp_path / f'XRP_NEW-trades.{dh1._get_file_extension()}'
     assert file.is_file()
     # Load trades back
-    trades_new = dh1.trades_load('XRP/NEW')
+    trades_new = dh1.trades_load('XRP/NEW', TradingMode.SPOT)
     assert_frame_equal(trades, trades_new, check_exact=True)
     assert len(trades_new) == len(trades)
 
@@ -483,11 +483,11 @@ def test_datahandler_trades_purge(mocker, testdatadir, datahandler):
     mocker.patch.object(Path, "exists", MagicMock(return_value=False))
     unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
     dh = get_datahandler(testdatadir, datahandler)
-    assert not dh.trades_purge('UNITTEST/NONEXIST')
+    assert not dh.trades_purge('UNITTEST/NONEXIST', TradingMode.SPOT)
     assert unlinkmock.call_count == 0
 
     mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    assert dh.trades_purge('UNITTEST/NONEXIST')
+    assert dh.trades_purge('UNITTEST/NONEXIST', TradingMode.SPOT)
     assert unlinkmock.call_count == 1
 
 

tests/data/test_download_data.py

@@ -78,10 +78,6 @@ def test_download_data_main_trades(mocker):
         "trading_mode": "futures",
     })
 
-    with pytest.raises(OperationalException,
-                       match="Trade download not supported for futures."):
-        download_data_main(config)
-
 
 def test_download_data_main_data_invalid(mocker):
     patch_exchange(mocker, id="kraken")

tests/data/test_history.py

@@ -23,7 +23,7 @@ from freqtrade.data.history.history_utils import (_download_pair_history, _downl
                                                   validate_backtest_data)
 from freqtrade.data.history.idatahandler import get_datahandler
 from freqtrade.data.history.jsondatahandler import JsonDataHandler, JsonGzDataHandler
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.misc import file_dump_json
 from freqtrade.resolvers import StrategyResolver
@@ -168,20 +168,21 @@ def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type):
     assert fn == Path(expected_result + '.gz')
 
 
-@pytest.mark.parametrize("pair,expected_result", [
-    ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-trades.json'),
-    ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-trades.json'),
-    ("ETHH20", 'freqtrade/hello/world/ETHH20-trades.json'),
-    (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-trades.json'),
-    ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-trades.json'),
-    ("ACC_OLD_BTC", 'freqtrade/hello/world/ACC_OLD_BTC-trades.json'),
+@pytest.mark.parametrize("pair,trading_mode,expected_result", [
+    ("ETH/BTC", '', 'freqtrade/hello/world/ETH_BTC-trades.json'),
+    ("ETH/USDT:USDT", 'futures', 'freqtrade/hello/world/futures/ETH_USDT_USDT-trades.json'),
+    ("Fabric Token/ETH", '', 'freqtrade/hello/world/Fabric_Token_ETH-trades.json'),
+    ("ETHH20", '', 'freqtrade/hello/world/ETHH20-trades.json'),
+    (".XBTBON2H", '', 'freqtrade/hello/world/_XBTBON2H-trades.json'),
+    ("ETHUSD.d", '', 'freqtrade/hello/world/ETHUSD_d-trades.json'),
+    ("ACC_OLD_BTC", '', 'freqtrade/hello/world/ACC_OLD_BTC-trades.json'),
 ])
-def test_json_pair_trades_filename(pair, expected_result):
-    fn = JsonDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair)
+def test_json_pair_trades_filename(pair, trading_mode, expected_result):
+    fn = JsonDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair, trading_mode)
     assert isinstance(fn, Path)
     assert fn == Path(expected_result)
 
-    fn = JsonGzDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair)
+    fn = JsonGzDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair, trading_mode)
     assert isinstance(fn, Path)
     assert fn == Path(expected_result + '.gz')
 
@@ -559,7 +560,8 @@ def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, tes
     unavailable_pairs = refresh_backtest_trades_data(exchange=ex,
                                                      pairs=["ETH/BTC", "XRP/BTC", "XRP/ETH"],
                                                      datadir=testdatadir,
-                                                     timerange=timerange, erase=True
+                                                     timerange=timerange, erase=True,
+                                                     trading_mode=TradingMode.SPOT,
                                                      )
 
     assert dl_mock.call_count == 2
@@ -584,7 +586,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
     assert not file1.is_file()
 
     assert _download_trades_history(data_handler=data_handler, exchange=exchange,
-                                    pair='ETH/BTC')
+                                    pair='ETH/BTC', trading_mode=TradingMode.SPOT)
     assert log_has("Current Amount of trades: 0", caplog)
     assert log_has("New Amount of trades: 6", caplog)
     assert ght_mock.call_count == 1
@@ -597,8 +599,9 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
     since_time = int(trades_history[-3][0] // 1000)
     since_time2 = int(trades_history[-1][0] // 1000)
     timerange = TimeRange('date', None, since_time, 0)
-    assert _download_trades_history(data_handler=data_handler, exchange=exchange,
-                                    pair='ETH/BTC', timerange=timerange)
+    assert _download_trades_history(
+        data_handler=data_handler, exchange=exchange, pair='ETH/BTC',
+        timerange=timerange, trading_mode=TradingMode.SPOT)
 
     assert ght_mock.call_count == 1
     # Check this in seconds - since we had to convert to seconds above too.
@@ -611,7 +614,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
     caplog.clear()
 
     assert not _download_trades_history(data_handler=data_handler, exchange=exchange,
-                                        pair='ETH/BTC')
+                                        pair='ETH/BTC', trading_mode=TradingMode.SPOT)
     assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog)
 
     file2 = tmp_path / 'XRP_ETH-trades.json.gz'
@@ -623,8 +626,9 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
     since_time = int(trades_history[0][0] // 1000) - 500
     timerange = TimeRange('date', None, since_time, 0)
 
-    assert _download_trades_history(data_handler=data_handler, exchange=exchange,
-                                    pair='XRP/ETH', timerange=timerange)
+    assert _download_trades_history(
+        data_handler=data_handler, exchange=exchange, pair='XRP/ETH',
+        timerange=timerange, trading_mode=TradingMode.SPOT)
 
     assert ght_mock.call_count == 1
 

tests/data/test_trade_converter_kraken.py

@@ -6,6 +6,7 @@ import pytest
 
 from freqtrade.data.converter.trade_converter_kraken import import_kraken_trades_from_csv
 from freqtrade.data.history.idatahandler import get_datahandler
+from freqtrade.enums import TradingMode
 from freqtrade.exceptions import OperationalException
 from tests.conftest import EXMS, log_has, log_has_re, patch_exchange
 
@@ -34,12 +35,13 @@ def test_import_kraken_trades_from_csv(testdatadir, tmp_path, caplog, default_co
 
     import_kraken_trades_from_csv(default_conf_usdt, 'feather')
     assert log_has("Found csv files for BCHEUR.", caplog)
+    assert log_has("Converting pairs: BCH/EUR.", caplog)
     assert log_has_re(r"BCH/EUR: 340 trades.* 2023-01-01.* 2023-01-02.*", caplog)
 
     assert dstfile.is_file()
 
     dh = get_datahandler(tmp_path, 'feather')
-    trades = dh.trades_load('BCH_EUR')
+    trades = dh.trades_load('BCH_EUR', TradingMode.SPOT)
     assert len(trades) == 340
 
     assert trades['date'].min().to_pydatetime() == datetime(2023, 1, 1, 0, 3, 56,
@@ -48,3 +50,10 @@ def test_import_kraken_trades_from_csv(testdatadir, tmp_path, caplog, default_co
                                                             tzinfo=timezone.utc)
     # ID is not filled
     assert len(trades.loc[trades['id'] != '']) == 0
+
+    caplog.clear()
+    default_conf_usdt['pairs'] = ['XRP/EUR']
+    # Filtered to non-existing pair
+    import_kraken_trades_from_csv(default_conf_usdt, 'feather')
+    assert log_has("Found csv files for BCHEUR.", caplog)
+    assert log_has("No data found for pairs XRP/EUR.", caplog)

tests/exchange/test_binance.py

@@ -596,10 +596,10 @@ async def test__async_get_historic_ohlcv_binance(default_conf, mocker, caplog, c
 
 
 @pytest.mark.parametrize('pair,nominal_value,mm_ratio,amt', [
-    ("BNB/BUSD:BUSD", 0.0, 0.025, 0),
+    ("XRP/USDT:USDT", 0.0, 0.025, 0),
     ("BNB/USDT:USDT", 100.0, 0.0065, 0),
     ("BTC/USDT:USDT", 170.30, 0.004, 0),
-    ("BNB/BUSD:BUSD", 999999.9, 0.1, 27500.0),
+    ("XRP/USDT:USDT", 999999.9, 0.1, 27500.0),
     ("BNB/USDT:USDT", 5000000.0, 0.15, 233035.0),
     ("BTC/USDT:USDT", 600000000, 0.5, 1.997038E8),
 ])

tests/exchange/test_bybit.py

@@ -1,20 +1,40 @@
 from datetime import datetime, timedelta, timezone
 from unittest.mock import MagicMock
 
+import pytest
+
 from freqtrade.enums.marginmode import MarginMode
 from freqtrade.enums.tradingmode import TradingMode
-from tests.conftest import EXMS, get_mock_coro, get_patched_exchange
+from freqtrade.exceptions import OperationalException
+from tests.conftest import EXMS, get_mock_coro, get_patched_exchange, log_has
 from tests.exchange.test_exchange import ccxt_exceptionhandlers
 
 
-def test_additional_exchange_init_bybit(default_conf, mocker):
+def test_additional_exchange_init_bybit(default_conf, mocker, caplog):
     default_conf['dry_run'] = False
     default_conf['trading_mode'] = TradingMode.FUTURES
     default_conf['margin_mode'] = MarginMode.ISOLATED
     api_mock = MagicMock()
     api_mock.set_position_mode = MagicMock(return_value={"dualSidePosition": False})
-    get_patched_exchange(mocker, default_conf, id="bybit", api_mock=api_mock)
+    api_mock.is_unified_enabled = MagicMock(return_value=[False, False])
+
+    exchange = get_patched_exchange(mocker, default_conf, id="bybit", api_mock=api_mock)
     assert api_mock.set_position_mode.call_count == 1
+    assert api_mock.is_unified_enabled.call_count == 1
+    assert exchange.unified_account is False
+
+    assert log_has("Bybit: Standard account.", caplog)
+
+    api_mock.set_position_mode.reset_mock()
+    api_mock.is_unified_enabled = MagicMock(return_value=[False, True])
+    with pytest.raises(OperationalException, match=r"Bybit: Unified account is not supported.*"):
+        get_patched_exchange(mocker, default_conf, id="bybit", api_mock=api_mock)
+    assert log_has("Bybit: Unified account.", caplog)
+    # exchange = get_patched_exchange(mocker, default_conf, id="bybit", api_mock=api_mock)
+    # assert api_mock.set_position_mode.call_count == 1
+    # assert api_mock.is_unified_enabled.call_count == 1
+    # assert exchange.unified_account is True
+
     ccxt_exceptionhandlers(mocker, default_conf, api_mock, 'bybit',
                            "additional_exchange_init", "set_position_mode")
 
@@ -111,6 +131,7 @@ def test_bybit_fetch_order_canceled_empty(default_conf_usdt, mocker):
         'amount': 20.0,
     })
 
+    mocker.patch(f"{EXMS}.exchange_has", return_value=True)
     exchange = get_patched_exchange(mocker, default_conf_usdt, api_mock, id='bybit')
 
     res = exchange.fetch_order('123', 'BTC/USDT')

tests/exchange/test_exchange.py

@@ -7,9 +7,10 @@ from unittest.mock import MagicMock, Mock, PropertyMock, patch
 
 import ccxt
 import pytest
+from numpy import NaN
 from pandas import DataFrame
 
-from freqtrade.enums import CandleType, MarginMode, TradingMode
+from freqtrade.enums import CandleType, MarginMode, RunMode, TradingMode
 from freqtrade.exceptions import (DDosProtection, DependencyException, ExchangeError,
                                   InsufficientFundsError, InvalidOrderException,
                                   OperationalException, PricingError, TemporaryError)
@@ -796,7 +797,9 @@ def test_validate_timeframes_failed(default_conf, mocker):
 
     mocker.patch(f'{EXMS}._init_ccxt', MagicMock(return_value=api_mock))
     mocker.patch(f'{EXMS}._load_markets', MagicMock(return_value={}))
-    mocker.patch(f'{EXMS}.validate_pairs', MagicMock())
+    mocker.patch(f'{EXMS}.validate_pairs')
+    mocker.patch(f'{EXMS}.validate_stakecurrency')
+    mocker.patch(f'{EXMS}.validate_pricing')
     with pytest.raises(OperationalException,
                        match=r"Invalid timeframe '3m'. This exchange supports.*"):
         Exchange(default_conf)
@@ -806,6 +809,10 @@ def test_validate_timeframes_failed(default_conf, mocker):
                        match=r"Timeframes < 1m are currently not supported by Freqtrade."):
         Exchange(default_conf)
 
+    # Will not raise an exception in util mode.
+    default_conf['runmode'] = RunMode.UTIL_EXCHANGE
+    Exchange(default_conf)
+
 
 def test_validate_timeframes_emulated_ohlcv_1(default_conf, mocker):
     default_conf["timeframe"] = "3m"
@@ -2297,6 +2304,66 @@ def test_refresh_latest_ohlcv_cache(mocker, default_conf, candle_type, time_mach
     assert res[pair2].at[0, 'open']
 
 
+def test_refresh_ohlcv_with_cache(mocker, default_conf, time_machine) -> None:
+    start = datetime(2021, 8, 1, 0, 0, 0, 0, tzinfo=timezone.utc)
+    ohlcv = generate_test_data_raw('1h', 100, start.strftime('%Y-%m-%d'))
+    time_machine.move_to(start, tick=False)
+    pairs = [
+        ('ETH/BTC', '1d', CandleType.SPOT),
+        ('TKN/BTC', '1d', CandleType.SPOT),
+        ('LTC/BTC', '1d', CandleType.SPOT),
+        ('LTC/BTC', '5m', CandleType.SPOT),
+        ('LTC/BTC', '1h', CandleType.SPOT),
+    ]
+
+    ohlcv_data = {
+        p: ohlcv for p in pairs
+    }
+    ohlcv_mock = mocker.patch(f"{EXMS}.refresh_latest_ohlcv", return_value=ohlcv_data)
+    mocker.patch(f"{EXMS}.ohlcv_candle_limit", return_value=100)
+    exchange = get_patched_exchange(mocker, default_conf)
+
+    assert len(exchange._expiring_candle_cache) == 0
+
+    res = exchange.refresh_ohlcv_with_cache(pairs, start.timestamp())
+    assert ohlcv_mock.call_count == 1
+    assert ohlcv_mock.call_args_list[0][0][0] == pairs
+    assert len(ohlcv_mock.call_args_list[0][0][0]) == 5
+
+    assert len(res) == 5
+    # length of 3 - as we have 3 different timeframes
+    assert len(exchange._expiring_candle_cache) == 3
+
+    ohlcv_mock.reset_mock()
+    res = exchange.refresh_ohlcv_with_cache(pairs, start.timestamp())
+    assert ohlcv_mock.call_count == 0
+
+    # Expire 5m cache
+    time_machine.move_to(start + timedelta(minutes=6), tick=False)
+
+    ohlcv_mock.reset_mock()
+    res = exchange.refresh_ohlcv_with_cache(pairs, start.timestamp())
+    assert ohlcv_mock.call_count == 1
+    assert len(ohlcv_mock.call_args_list[0][0][0]) == 1
+
+    # Expire 5m and 1h cache
+    time_machine.move_to(start + timedelta(hours=2), tick=False)
+
+    ohlcv_mock.reset_mock()
+    res = exchange.refresh_ohlcv_with_cache(pairs, start.timestamp())
+    assert ohlcv_mock.call_count == 1
+    assert len(ohlcv_mock.call_args_list[0][0][0]) == 2
+
+    # Expire all caches
+    time_machine.move_to(start + timedelta(days=1, hours=2), tick=False)
+
+    ohlcv_mock.reset_mock()
+    res = exchange.refresh_ohlcv_with_cache(pairs, start.timestamp())
+    assert ohlcv_mock.call_count == 1
+    assert len(ohlcv_mock.call_args_list[0][0][0]) == 5
+    assert ohlcv_mock.call_args_list[0][0][0] == pairs
+
+
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
 async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_name):
     ohlcv = [
@@ -3171,6 +3238,7 @@ def test_is_cancel_order_result_suitable(mocker, default_conf, exchange_name, or
 def test_cancel_order_with_result(default_conf, mocker, exchange_name, corder,
                                   call_corder, call_forder):
     default_conf['dry_run'] = False
+    mocker.patch(f"{EXMS}.exchange_has", return_value=True)
     api_mock = MagicMock()
     api_mock.cancel_order = MagicMock(return_value=corder)
     api_mock.fetch_order = MagicMock(return_value={})
@@ -3184,6 +3252,7 @@ def test_cancel_order_with_result(default_conf, mocker, exchange_name, corder,
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
 def test_cancel_order_with_result_error(default_conf, mocker, exchange_name, caplog):
     default_conf['dry_run'] = False
+    mocker.patch(f"{EXMS}.exchange_has", return_value=True)
     api_mock = MagicMock()
     api_mock.cancel_order = MagicMock(side_effect=ccxt.InvalidOrder("Did not find order"))
     api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder("Did not find order"))
@@ -3281,6 +3350,7 @@ def test_fetch_order(default_conf, mocker, exchange_name, caplog):
     order.myid = 123
     order.symbol = 'TKN/BTC'
 
+    mocker.patch(f"{EXMS}.exchange_has", return_value=True)
     exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
     exchange._dry_run_open_orders['X'] = order
     assert exchange.fetch_order('X', 'TKN/BTC').myid == 123
@@ -3325,10 +3395,80 @@ def test_fetch_order(default_conf, mocker, exchange_name, caplog):
                            order_id='_', pair='TKN/BTC')
 
 
+@pytest.mark.usefixtures("init_persistence")
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+def test_fetch_order_emulated(default_conf, mocker, exchange_name, caplog):
+    default_conf['dry_run'] = True
+    default_conf['exchange']['log_responses'] = True
+    order = MagicMock()
+    order.myid = 123
+    order.symbol = 'TKN/BTC'
+
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+    mocker.patch(f'{EXMS}.exchange_has', return_value=False)
+    exchange._dry_run_open_orders['X'] = order
+    # Dry run - regular fetch_order behavior
+    assert exchange.fetch_order('X', 'TKN/BTC').myid == 123
+
+    with pytest.raises(InvalidOrderException, match=r'Tried to get an invalid dry-run-order.*'):
+        exchange.fetch_order('Y', 'TKN/BTC')
+
+    default_conf['dry_run'] = False
+    mocker.patch(f'{EXMS}.exchange_has', return_value=False)
+    api_mock = MagicMock()
+    api_mock.fetch_open_order = MagicMock(
+        return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'})
+    api_mock.fetch_closed_order = MagicMock(
+        return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'})
+    exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+    assert exchange.fetch_order(
+        'X', 'TKN/BTC') == {'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}
+    assert log_has(
+        ("API fetch_open_order: {\'id\': \'123\', \'amount\': 2, \'symbol\': \'TKN/BTC\'}"
+         ),
+        caplog
+    )
+    assert api_mock.fetch_open_order.call_count == 1
+    assert api_mock.fetch_closed_order.call_count == 0
+    caplog.clear()
+
+    # open_order doesn't find order
+    api_mock.fetch_open_order = MagicMock(side_effect=ccxt.OrderNotFound("Order not found"))
+    api_mock.fetch_closed_order = MagicMock(
+        return_value={'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'})
+    exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+    assert exchange.fetch_order(
+        'X', 'TKN/BTC') == {'id': '123', 'amount': 2, 'symbol': 'TKN/BTC'}
+    assert log_has(
+        ("API fetch_closed_order: {\'id\': \'123\', \'amount\': 2, \'symbol\': \'TKN/BTC\'}"
+         ),
+        caplog
+    )
+    assert api_mock.fetch_open_order.call_count == 1
+    assert api_mock.fetch_closed_order.call_count == 1
+    caplog.clear()
+
+    with pytest.raises(InvalidOrderException):
+        api_mock.fetch_open_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found"))
+        api_mock.fetch_closed_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found"))
+        exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+        exchange.fetch_order(order_id='_', pair='TKN/BTC')
+    assert api_mock.fetch_open_order.call_count == 1
+
+    api_mock.fetch_open_order = MagicMock(side_effect=ccxt.OrderNotFound("Order not found"))
+    exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+
+    ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name,
+                           'fetch_order_emulated', 'fetch_open_order',
+                           retries=1,
+                           order_id='_', pair='TKN/BTC', params={})
+
+
 @pytest.mark.usefixtures("init_persistence")
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
 def test_fetch_stoploss_order(default_conf, mocker, exchange_name):
     default_conf['dry_run'] = True
+    mocker.patch(f"{EXMS}.exchange_has", return_value=True)
     order = MagicMock()
     order.myid = 123
     exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
@@ -4064,6 +4204,7 @@ def test_get_max_leverage_from_margin(default_conf, mocker, pair, nominal_value,
         (10, 0.0001, 2.0, 1.0, 0.002, 0.002),
         (10, 0.0002, 2.0, 0.01, 0.004, 0.00004),
         (10, 0.0002, 2.5, None, 0.005, None),
+        (10, 0.0002, NaN, None, 0.0, None),
     ])
 def test_calculate_funding_fees(
     default_conf,
@@ -4173,8 +4314,8 @@ def test_combine_funding_and_mark(
         assert len(df) == 1
 
     # Empty funding rates
-    funding_rates = DataFrame([], columns=['date', 'open'])
-    df = exchange.combine_funding_and_mark(funding_rates, mark_rates, futures_funding_rate)
+    funding_rates2 = DataFrame([], columns=['date', 'open'])
+    df = exchange.combine_funding_and_mark(funding_rates2, mark_rates, futures_funding_rate)
     if futures_funding_rate is not None:
         assert len(df) == 3
         assert df.iloc[0]['open_fund'] == futures_funding_rate
@@ -4183,6 +4324,12 @@ def test_combine_funding_and_mark(
     else:
         assert len(df) == 0
 
+    # Empty mark candles
+    mark_candles = DataFrame([], columns=['date', 'open'])
+    df = exchange.combine_funding_and_mark(funding_rates, mark_candles, futures_funding_rate)
+
+    assert len(df) == 0
+
 
 @pytest.mark.parametrize('exchange,rate_start,rate_end,d1,d2,amount,expected_fees', [
     ('binance', 0, 2, "2021-09-01 01:00:00", "2021-09-01 04:00:00",  30.0, 0.0),
@@ -4963,8 +5110,8 @@ def test_get_maintenance_ratio_and_amt_exceptions(mocker, default_conf, leverage
 
 
 @pytest.mark.parametrize('pair,value,mmr,maintAmt', [
-    ('ADA/BUSD:BUSD', 500, 0.025, 0.0),
-    ('ADA/BUSD:BUSD', 20000000, 0.5, 1527500.0),
+    ('ADA/USDT:USDT', 500, 0.025, 0.0),
+    ('ADA/USDT:USDT', 20000000, 0.5, 1527500.0),
     ('ZEC/USDT:USDT', 500, 0.01, 0.0),
     ('ZEC/USDT:USDT', 20000000, 0.5, 654500.0),
 ])
@@ -4999,10 +5146,10 @@ def test_get_max_leverage_futures(default_conf, mocker, leverage_tiers):
 
     exchange._leverage_tiers = leverage_tiers
 
-    assert exchange.get_max_leverage("BNB/BUSD:BUSD", 1.0) == 20.0
+    assert exchange.get_max_leverage("XRP/USDT:USDT", 1.0) == 20.0
     assert exchange.get_max_leverage("BNB/USDT:USDT", 100.0) == 75.0
     assert exchange.get_max_leverage("BTC/USDT:USDT", 170.30) == 125.0
-    assert pytest.approx(exchange.get_max_leverage("BNB/BUSD:BUSD", 99999.9)) == 5.000005
+    assert pytest.approx(exchange.get_max_leverage("XRP/USDT:USDT", 99999.9)) == 5.000005
     assert pytest.approx(exchange.get_max_leverage("BNB/USDT:USDT", 1500)) == 33.333333333333333
     assert exchange.get_max_leverage("BTC/USDT:USDT", 300000000) == 2.0
     assert exchange.get_max_leverage("BTC/USDT:USDT", 600000000) == 1.0  # Last tier

tests/exchange/test_okx.py

@@ -196,7 +196,7 @@ def test_get_max_pair_stake_amount_okx(default_conf, mocker, leverage_tiers):
     exchange = get_patched_exchange(mocker, default_conf, id="okx")
     exchange._leverage_tiers = leverage_tiers
 
-    assert exchange.get_max_pair_stake_amount('BNB/BUSD:BUSD', 1.0) == 30000000
+    assert exchange.get_max_pair_stake_amount('XRP/USDT:USDT', 1.0) == 30000000
     assert exchange.get_max_pair_stake_amount('BNB/USDT:USDT', 1.0) == 50000000
     assert exchange.get_max_pair_stake_amount('BTC/USDT:USDT', 1.0) == 1000000000
     assert exchange.get_max_pair_stake_amount('BTC/USDT:USDT', 1.0, 10.0) == 100000000

tests/exchange_online/conftest.py

@@ -324,7 +324,8 @@ def get_futures_exchange(exchange_name, exchange_conf, class_mocker):
 
 
 @pytest.fixture(params=EXCHANGES, scope="class")
-def exchange(request, exchange_conf):
+def exchange(request, exchange_conf, class_mocker):
+    class_mocker.patch('freqtrade.exchange.bybit.Bybit.additional_exchange_init')
     yield from get_exchange(request.param, exchange_conf)
 
 

tests/exchange_online/test_ccxt_compat.py

@@ -12,6 +12,7 @@ import pytest
 from freqtrade.enums import CandleType
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date
 from freqtrade.exchange.exchange import timeframe_to_msecs
+from freqtrade.util import dt_floor_day, dt_now, dt_ts
 from tests.exchange_online.conftest import EXCHANGE_FIXTURE_TYPE, EXCHANGES
 
 
@@ -187,6 +188,25 @@ class TestCCXTExchange:
         now = datetime.now(timezone.utc) - timedelta(minutes=(timeframe_to_minutes(timeframe) * 2))
         assert exch.klines(pair_tf).iloc[-1]['date'] >= timeframe_to_prev_date(timeframe, now)
 
+    def test_ccxt_fetch_ohlcv_startdate(self, exchange: EXCHANGE_FIXTURE_TYPE):
+        """
+        Test that pair data starts at the provided startdate
+        """
+        exch, exchangename = exchange
+        pair = EXCHANGES[exchangename]['pair']
+        timeframe = '1d'
+
+        pair_tf = (pair, timeframe, CandleType.SPOT)
+        # last 5 days ...
+        since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=6))
+        ohlcv = exch.refresh_latest_ohlcv([pair_tf], since_ms=since_ms)
+        assert isinstance(ohlcv, dict)
+        assert len(ohlcv[pair_tf]) == len(exch.klines(pair_tf))
+        # Check if last-timeframe is within the last 2 intervals
+        now = datetime.now(timezone.utc) - timedelta(minutes=(timeframe_to_minutes(timeframe) * 2))
+        assert exch.klines(pair_tf).iloc[-1]['date'] >= timeframe_to_prev_date(timeframe, now)
+        assert exch.klines(pair_tf)['date'].astype(int).iloc[0] // 1e6 == since_ms
+
     def ccxt__async_get_candle_history(
             self, exchange, exchangename, pair, timeframe, candle_type, factor=0.9):
 

tests/freqai/conftest.py

@@ -25,10 +25,15 @@ def is_mac() -> bool:
     return "Darwin" in machine
 
 
+def is_arm() -> bool:
+    machine = platform.machine()
+    return "arm" in machine or "aarch64" in machine
+
+
 @pytest.fixture(autouse=True)
 def patch_torch_initlogs(mocker) -> None:
 
-    if is_mac():
+    if is_mac() and not is_arm():
         # Mock torch import completely
         import sys
         import types

tests/freqai/test_freqai_interface.py

@@ -1,5 +1,4 @@
 import logging
-import platform
 import shutil
 from pathlib import Path
 from unittest.mock import MagicMock
@@ -15,19 +14,14 @@ from freqtrade.optimize.backtesting import Backtesting
 from freqtrade.persistence import Trade
 from freqtrade.plugins.pairlistmanager import PairListManager
 from tests.conftest import EXMS, create_mock_trades, get_patched_exchange, log_has_re
-from tests.freqai.conftest import (get_patched_freqai_strategy, is_mac, is_py12, make_rl_config,
-                                   mock_pytorch_mlp_model_training_parameters)
-
-
-def is_arm() -> bool:
-    machine = platform.machine()
-    return "arm" in machine or "aarch64" in machine
+from tests.freqai.conftest import (get_patched_freqai_strategy, is_arm, is_mac, is_py12,
+                                   make_rl_config, mock_pytorch_mlp_model_training_parameters)
 
 
 def can_run_model(model: str) -> None:
     is_pytorch_model = 'Reinforcement' in model or 'PyTorch' in model
 
-    if is_py12() and ("Catboost" in model or is_pytorch_model):
+    if is_py12() and is_pytorch_model:
         pytest.skip("Model not supported on python 3.12 yet.")
 
     if is_arm() and "Catboost" in model:
@@ -243,7 +237,7 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model):
 def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog):
     can_run_model(model)
     test_tb = True
-    if is_mac():
+    if is_mac() and not is_arm():
         test_tb = False
 
     freqai_conf.get("freqai", {}).update({"save_backtest_models": True})

tests/freqtradebot/test_integration.py

@@ -49,7 +49,7 @@ def test_may_execute_exit_stoploss_on_exchange_multi(default_conf, ticker, fee,
     stoploss_order_closed['filled'] = stoploss_order_closed['amount']
 
     # Sell first trade based on stoploss, keep 2nd and 3rd trade open
-    stop_orders = [stoploss_order_closed, stoploss_order_open, stoploss_order_open]
+    stop_orders = [stoploss_order_closed, stoploss_order_open.copy(), stoploss_order_open.copy()]
     stoploss_order_mock = MagicMock(
         side_effect=stop_orders)
     # Sell 3rd trade (not called for the first trade)
@@ -100,9 +100,10 @@ def test_may_execute_exit_stoploss_on_exchange_multi(default_conf, ticker, fee,
         stop_order = stop_orders[idx]
         stop_order['id'] = f"stop{idx}"
         oobj = Order.parse_from_ccxt_object(stop_order, trade.pair, 'stoploss')
+        oobj.ft_is_open = True
 
         trade.orders.append(oobj)
-        trade.stoploss_order_id = f"stop{idx}"
+        assert len(trade.open_sl_orders) == 1
 
     n = freqtrade.exit_positions(trades)
     assert n == 2
@@ -113,6 +114,7 @@ def test_may_execute_exit_stoploss_on_exchange_multi(default_conf, ticker, fee,
 
     # Only order for 3rd trade needs to be cancelled
     assert cancel_order_mock.call_count == 1
+    assert stoploss_order_mock.call_count == 3
     # Wallets must be updated between stoploss cancellation and selling, and will be updated again
     # during update_trade_state
     assert wallets_mock.call_count == 4
@@ -536,7 +538,7 @@ def test_dca_order_adjust_entry_replace_fails(
     # Create DCA order for 2nd trade (so we have 2 open orders on 2 trades)
     # this 2nd order won't fill.
 
-    freqtrade.strategy.adjust_trade_position = MagicMock(return_value=20)
+    freqtrade.strategy.adjust_trade_position = MagicMock(return_value=(20, 'PeNF'))
 
     freqtrade.process()
 
@@ -627,12 +629,13 @@ def test_dca_exiting(default_conf_usdt, ticker_usdt, fee, mocker, caplog, levera
     assert log_has_re(
         r"Remaining amount of \d\.\d+.* would be smaller than the minimum of 10.", caplog)
 
-    freqtrade.strategy.adjust_trade_position = MagicMock(return_value=-20)
+    freqtrade.strategy.adjust_trade_position = MagicMock(return_value=(-20, 'PES'))
 
     freqtrade.process()
     trade = Trade.get_trades().first()
     assert len(trade.orders) == 2
     assert trade.orders[-1].ft_order_side == 'sell'
+    assert trade.orders[-1].ft_order_tag == 'PES'
     assert pytest.approx(trade.stake_amount) == 40.198
     assert pytest.approx(trade.amount) == 20.099 * leverage
     assert trade.open_rate == 2.0

tests/optimize/conftest.py

@@ -30,7 +30,7 @@ def hyperopt_conf(default_conf):
 
 
 @pytest.fixture(autouse=True)
-def backtesting_cleanup() -> None:
+def backtesting_cleanup():
     yield None
 
     Backtesting.cleanup()

tests/optimize/test_backtest_detail.py

@@ -900,7 +900,7 @@ TESTS = [
 
 
 @pytest.mark.parametrize("data", TESTS)
-def test_backtest_results(default_conf, fee, mocker, caplog, data: BTContainer) -> None:
+def test_backtest_results(default_conf, mocker, caplog, data: BTContainer) -> None:
     """
     run functional tests
     """

tests/optimize/test_backtesting.py

@@ -742,14 +742,18 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None:
          'orders': [
             [
                 {'amount': 0.00957442, 'safe_price': 0.104445, 'ft_order_side': 'buy',
-                 'order_filled_timestamp': 1517251200000, 'ft_is_entry': True},
+                 'order_filled_timestamp': 1517251200000, 'ft_is_entry': True,
+                 'ft_order_tag': ''},
                 {'amount': 0.00957442, 'safe_price': 0.10496853383458644, 'ft_order_side': 'sell',
-                 'order_filled_timestamp': 1517265300000, 'ft_is_entry': False}
+                 'order_filled_timestamp': 1517265300000, 'ft_is_entry': False,
+                 'ft_order_tag': 'roi'}
             ], [
                 {'amount': 0.0097064, 'safe_price': 0.10302485, 'ft_order_side': 'buy',
-                 'order_filled_timestamp': 1517283000000, 'ft_is_entry': True},
+                 'order_filled_timestamp': 1517283000000, 'ft_is_entry': True,
+                 'ft_order_tag': ''},
                 {'amount': 0.0097064, 'safe_price': 0.10354126528822055, 'ft_order_side': 'sell',
-                 'order_filled_timestamp': 1517285400000, 'ft_is_entry': False}
+                 'order_filled_timestamp': 1517285400000, 'ft_is_entry': False,
+                 'ft_order_tag': 'roi'}
             ]
          ]
          })

tests/optimize/test_backtesting_adjust_position.py

@@ -57,28 +57,30 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) ->
                                      ),
          'close_date': pd.to_datetime([dt_utc(2018, 1, 29, 22, 00, 0),
                                        dt_utc(2018, 1, 30, 4, 10, 0)], utc=True),
-         'open_rate': [0.10401764894444211, 0.10302485],
-         'close_rate': [0.10453904066847439, 0.103541],
+         'open_rate': [0.10401764891917063, 0.10302485],
+         'close_rate': [0.10453904064307624, 0.10354126528822055],
          'fee_open': [0.0025, 0.0025],
          'fee_close': [0.0025, 0.0025],
          'trade_duration': [200, 40],
          'profit_ratio': [0.0, 0.0],
          'profit_abs': [0.0, 0.0],
          'exit_reason': [ExitType.ROI.value, ExitType.ROI.value],
-         'initial_stop_loss_abs': [0.0940005, 0.09272236],
+         'initial_stop_loss_abs': [0.0940005, 0.092722365],
          'initial_stop_loss_ratio': [-0.1, -0.1],
-         'stop_loss_abs': [0.0940005, 0.09272236],
+         'stop_loss_abs': [0.0940005, 0.092722365],
          'stop_loss_ratio': [-0.1, -0.1],
          'min_rate': [0.10370188, 0.10300000000000001],
-         'max_rate': [0.10481985, 0.1038888],
+         'max_rate': [0.10481985, 0.10388887000000001],
          'is_open': [False, False],
          'enter_tag': ['', ''],
          'leverage': [1.0, 1.0],
          'is_short': [False, False],
          'open_timestamp': [1517251200000, 1517283000000],
-         'close_timestamp': [1517265300000, 1517285400000],
+         'close_timestamp': [1517263200000, 1517285400000],
          })
-    pd.testing.assert_frame_equal(results.drop(columns=['orders']), expected)
+    results_no = results.drop(columns=['orders'])
+    pd.testing.assert_frame_equal(results_no, expected, check_exact=True)
+
     data_pair = processed[pair]
     assert len(results.iloc[0]['orders']) == 6
     assert len(results.iloc[1]['orders']) == 2
@@ -148,7 +150,7 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker, levera
     assert pytest.approx(trade.amount) == 47.61904762 * leverage
     assert len(trade.orders) == 1
     # Increase position by 100
-    backtesting.strategy.adjust_trade_position = MagicMock(return_value=100)
+    backtesting.strategy.adjust_trade_position = MagicMock(return_value=(100, 'PartIncrease'))
 
     trade = backtesting._get_adjust_trade_entry_for_candle(trade, row, current_time)
 
@@ -156,6 +158,7 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker, levera
     assert pytest.approx(trade.stake_amount) == 200.0
     assert pytest.approx(trade.amount) == 95.23809524 * leverage
     assert len(trade.orders) == 2
+    assert trade.orders[-1].ft_order_tag == 'PartIncrease'
     assert pytest.approx(trade.liquidation_price) == (0.1038916 if leverage == 1 else 1.2127791)
 
     # Reduce by more than amount - no change to trade.
@@ -171,13 +174,14 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker, levera
     assert pytest.approx(trade.liquidation_price) == (0.1038916 if leverage == 1 else 1.2127791)
 
     # Reduce position by 50
-    backtesting.strategy.adjust_trade_position = MagicMock(return_value=-100)
+    backtesting.strategy.adjust_trade_position = MagicMock(return_value=(-100, 'partDecrease'))
     trade = backtesting._get_adjust_trade_entry_for_candle(trade, row, current_time)
 
     assert trade
     assert pytest.approx(trade.stake_amount) == 100.0
     assert pytest.approx(trade.amount) == 47.61904762 * leverage
     assert len(trade.orders) == 3
+    assert trade.orders[-1].ft_order_tag == 'partDecrease'
     assert trade.nr_of_successful_entries == 2
     assert trade.nr_of_successful_exits == 1
     assert pytest.approx(trade.liquidation_price) == (0.1038916 if leverage == 1 else 1.2127791)

tests/optimize/test_optimize_reports.py

@@ -498,7 +498,7 @@ def test__get_resample_from_period():
 
     assert _get_resample_from_period('day') == '1d'
     assert _get_resample_from_period('week') == '1W-MON'
-    assert _get_resample_from_period('month') == '1M'
+    assert _get_resample_from_period('month') == '1ME'
     with pytest.raises(ValueError, match=r"Period noooo is not supported."):
         _get_resample_from_period('noooo')
 

tests/persistence/test_migrations.py

@@ -74,7 +74,7 @@ def test_init_dryrun_db(default_conf, tmpdir):
     assert Path(filename).is_file()
 
 
-def test_migrate_new(mocker, default_conf, fee, caplog):
+def test_migrate(mocker, default_conf, fee, caplog):
     """
     Test Database migration (starting with new pairformat)
     """
@@ -277,8 +277,6 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
     assert trade.exit_reason is None
     assert trade.strategy is None
     assert trade.timeframe == '5m'
-    assert trade.stoploss_order_id == 'dry_stop_order_id222'
-    assert trade.stoploss_last_update is None
     assert log_has("trying trades_bak1", caplog)
     assert log_has("trying trades_bak2", caplog)
     assert log_has("Running database migration for trades - backup: trades_bak2, orders_bak0",
@@ -294,9 +292,10 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
     assert orders[0].order_id == 'dry_buy_order'
     assert orders[0].ft_order_side == 'buy'
 
+    # All dry-run stoploss orders will be closed
     assert orders[-1].order_id == 'dry_stop_order_id222'
     assert orders[-1].ft_order_side == 'stoploss'
-    assert orders[-1].ft_is_open is True
+    assert orders[-1].ft_is_open is False
 
     assert orders[1].order_id == 'dry_buy_order22'
     assert orders[1].ft_order_side == 'buy'