Merge pull request #9749 from freqtrade/new_release

New release 2024.1

.github/workflows/ci.yml

@@ -25,7 +25,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-20.04, ubuntu-22.04 ]
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v4
@@ -36,14 +36,14 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Cache_dependencies
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: cache
       with:
         path: ~/dependencies/
         key: ${{ runner.os }}-dependencies
 
     - name: pip cache (linux)
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ~/.cache/pip
         key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip
@@ -125,7 +125,7 @@ jobs:
     strategy:
       matrix:
         os: [ "macos-latest", "macos-13" ]
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v4
@@ -137,14 +137,14 @@ jobs:
         check-latest: true
 
     - name: Cache_dependencies
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: cache
       with:
         path: ~/dependencies/
         key: ${{ matrix.os }}-dependencies
 
     - name: pip cache (macOS)
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ~/Library/Caches/pip
         key: ${{ matrix.os }}-${{ matrix.python-version }}-pip
@@ -238,7 +238,7 @@ jobs:
     strategy:
       matrix:
         os: [ windows-latest ]
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v4
@@ -249,7 +249,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Pip cache (Windows)
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ~\AppData\Local\pip\Cache
         key: ${{ matrix.os }}-${{ matrix.python-version }}-pip
@@ -368,14 +368,14 @@ jobs:
         python-version: "3.11"
 
     - name: Cache_dependencies
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: cache
       with:
         path: ~/dependencies/
         key: ${{ runner.os }}-dependencies
 
     - name: pip cache (linux)
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ~/.cache/pip
         key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip

.github/workflows/docker_update_readme.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Docker Hub Description
-      uses: peter-evans/dockerhub-description@v3
+      uses: peter-evans/dockerhub-description@v4
       env:
         DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USERNAME }}
         DOCKERHUB_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}

.pre-commit-config.yaml

@@ -2,7 +2,7 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
   - repo: https://github.com/pycqa/flake8
-    rev: "6.1.0"
+    rev: "7.0.0"
     hooks:
       - id: flake8
         additional_dependencies: [Flake8-pyproject]
@@ -16,10 +16,10 @@ repos:
         additional_dependencies:
           - types-cachetools==5.3.0.7
           - types-filelock==3.2.7
-          - types-requests==2.31.0.10
+          - types-requests==2.31.0.20240125
-          - types-tabulate==0.9.0.3
+          - types-tabulate==0.9.0.20240106
-          - types-python-dateutil==2.8.19.14
+          - types-python-dateutil==2.8.19.20240106
-          - SQLAlchemy==2.0.23
+          - SQLAlchemy==2.0.25
         # stages: [push]
 
   - repo: https://github.com/pycqa/isort
@@ -31,7 +31,7 @@ repos:
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.1.9'
+    rev: 'v0.1.14'
     hooks:
       - id: ruff
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11.6-slim-bookworm as base
+FROM python:3.11.7-slim-bookworm as base
 
 # Setup env
 ENV LANG C.UTF-8

README.md

@@ -30,7 +30,7 @@ Please read the [exchange specific notes](docs/exchanges.md) to learn about even
 - [X] [Binance](https://www.binance.com/)
 - [X] [Bitmart](https://bitmart.com/)
 - [X] [Gate.io](https://www.gate.io/ref/6266643)
-- [X] [Huobi](http://huobi.com/)
+- [X] [HTX](https://www.htx.com/) (Former Huobi)
 - [X] [Kraken](https://kraken.com/)
 - [X] [OKX](https://okx.com/) (Former OKEX)
 - [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_

config_examples/config_freqai.example.json

@@ -52,7 +52,7 @@
         "train_period_days": 15,
         "backtest_period_days": 7,
         "live_retrain_hours": 0,
-        "identifier": "uniqe-id",
+        "identifier": "unique-id",
         "feature_parameters": {
             "include_timeframes": [
                 "3m",

docker/Dockerfile.armhf

@@ -1,4 +1,4 @@
-FROM python:3.11.6-slim-bookworm as base
+FROM python:3.11.7-slim-bookworm as base
 
 # Setup env
 ENV LANG C.UTF-8

docker/Dockerfile.jupyter

@@ -1,8 +1,8 @@
 FROM freqtradeorg/freqtrade:develop_plot
 
 
-# Pin jupyter-client to avoid tornado version conflict
+# Pin prompt-toolkit to avoid questionary version conflict
-RUN pip install jupyterlab jupyter-client==7.3.4 --user --no-cache-dir
+RUN pip install jupyterlab "prompt-toolkit<=3.0.36" jupyter-client --user --no-cache-dir
 
 # Empty the ENTRYPOINT to allow all commands
 ENTRYPOINT []

docker/docker-compose-jupyter.yml

@@ -6,7 +6,7 @@ services:
       context: ..
       dockerfile: docker/Dockerfile.jupyter
     restart: unless-stopped
-    container_name: freqtrade
+    # container_name: freqtrade
     ports:
       - "127.0.0.1:8888:8888"
     volumes:

docs/configuration.md

@@ -572,9 +572,11 @@ In addition to fiat currencies, a range of crypto currencies is supported.
 The valid values are:
 
 ```json
-"BTC", "ETH", "XRP", "LTC", "BCH", "USDT"
+"BTC", "ETH", "XRP", "LTC", "BCH", "BNB"
 ```
 
+Removing `fiat_display_currency` completely from the configuration will skip initializing coingecko, and will not show any FIAT currency conversion. This has no importance for the correct functioning of the bot.
+
 ## Using Dry-run mode
 
 We recommend starting the bot in the Dry-run mode to see how your bot will

docs/exchanges.md

@@ -127,6 +127,8 @@ Freqtrade will not attempt to change these settings.
 
 ## Kraken
 
+Kraken supports [time_in_force](configuration.md#understand-order_time_in_force) with settings "GTC" (good till cancelled), "IOC" (immediate-or-cancel) and "PO" (Post only) settings.
+
 !!! Tip "Stoploss on Exchange"
     Kraken supports `stoploss_on_exchange` and can use both stop-loss-market and stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it.
     You can use either `"limit"` or `"market"` in the `order_types.stoploss` configuration setting to decide which type to use.
@@ -206,10 +208,10 @@ Kucoin supports [time_in_force](configuration.md#understand-order_time_in_force)
 For Kucoin, it is suggested to add `"KCS/<STAKE>"` to your blacklist to avoid issues, unless you are willing to maintain enough extra `KCS` on the account or unless you're willing to disable using `KCS` for fees. 
 Kucoin accounts may use `KCS` for fees, and if a trade happens to be on `KCS`, further trades may consume this position and make the initial `KCS` trade unsellable as the expected amount is not there anymore.
 
-## Huobi
+## HTX (formerly Huobi)
 
 !!! Tip "Stoploss on Exchange"
-    Huobi supports `stoploss_on_exchange` and uses `stop-limit` orders. It provides great advantages, so we recommend to benefit from it by enabling stoploss on exchange.
+    HTX supports `stoploss_on_exchange` and uses `stop-limit` orders. It provides great advantages, so we recommend to benefit from it by enabling stoploss on exchange.
 
 ## OKX (former OKEX)
 

docs/freqai-configuration.md

@@ -162,7 +162,8 @@ Below are the values you can expect to include/use inside a typical strategy dat
 | `df['&*_std/mean']` | Standard deviation and mean values of the defined labels during training (or live tracking with `fit_live_predictions_candles`). Commonly used to understand the rarity of a prediction (use the z-score as shown in `templates/FreqaiExampleStrategy.py` and explained [here](#creating-a-dynamic-target-threshold) to evaluate how often a particular prediction was observed during training or historically with `fit_live_predictions_candles`). <br> **Datatype:** Float.
 | `df['do_predict']` | Indication of an outlier data point. The return value is integer between -2 and 2, which lets you know if the prediction is trustworthy or not. `do_predict==1` means that the prediction is trustworthy. If the Dissimilarity Index (DI, see details [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di)) of the input data point is above the threshold defined in the config, FreqAI will subtract 1 from `do_predict`, resulting in `do_predict==0`. If `use_SVM_to_remove_outliers` is active, the Support Vector Machine (SVM, see details [here](freqai-feature-engineering.md#identifying-outliers-using-a-support-vector-machine-svm)) may also detect outliers in training and prediction data. In this case, the SVM will also subtract 1 from `do_predict`. If the input data point was considered an outlier by the SVM but not by the DI, or vice versa, the result will be `do_predict==0`. If both the DI and the SVM considers the input data point to be an outlier, the result will be `do_predict==-1`. As with the SVM, if `use_DBSCAN_to_remove_outliers` is active, DBSCAN (see details [here](freqai-feature-engineering.md#identifying-outliers-with-dbscan)) may also detect outliers and subtract 1 from `do_predict`. Hence, if both the SVM and DBSCAN are active and identify a datapoint that was above the DI threshold as an outlier, the result will be `do_predict==-2`. A particular case is when `do_predict == 2`, which means that the model has expired due to exceeding `expired_hours`. <br> **Datatype:** Integer between -2 and 2.
 | `df['DI_values']` | Dissimilarity Index (DI) values are proxies for the level of confidence FreqAI has in the prediction. A lower DI means the prediction is close to the training data, i.e., higher prediction confidence. See details about the DI [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di). <br> **Datatype:** Float.
-| `df['%*']` | Any dataframe column prepended with `%` in `feature_engineering_*()` is treated as a training feature. For example, you can include the RSI in the training feature set (similar to in `templates/FreqaiExampleStrategy.py`) by setting `df['%-rsi']`. See more details on how this is done [here](freqai-feature-engineering.md). <br> **Note:** Since the number of features prepended with `%` can multiply very quickly (10s of thousands of features are easily engineered using the multiplictative functionality of, e.g., `include_shifted_candles` and `include_timeframes` as described in the [parameter table](freqai-parameter-table.md)), these features are removed from the dataframe that is returned from FreqAI to the strategy. To keep a particular type of feature for plotting purposes, you would prepend it with `%%`. <br> **Datatype:** Depends on the output of the model.
+| `df['%*']` | Any dataframe column prepended with `%` in `feature_engineering_*()` is treated as a training feature. For example, you can include the RSI in the training feature set (similar to in `templates/FreqaiExampleStrategy.py`) by setting `df['%-rsi']`. See more details on how this is done [here](freqai-feature-engineering.md). <br> **Note:** Since the number of features prepended with `%` can multiply very quickly (10s of thousands of features are easily engineered using the multiplictative functionality of, e.g., `include_shifted_candles` and `include_timeframes` as described in the [parameter table](freqai-parameter-table.md)), these features are removed from the dataframe that is returned from FreqAI to the strategy. To keep a particular type of feature for plotting purposes, you would prepend it with `%%` (see details below). <br> **Datatype:** Depends on the feature created by the user.
+| `df['%%*']` | Any dataframe column prepended with `%%` in `feature_engineering_*()` is treated as a training feature, just the same as the above `%` prepend. However, in this case, the features are returned back to the strategy for FreqUI/plot-dataframe plotting and monitoring in Dry/Live/Backtesting <br> **Datatype:** Depends on the feature created by the user. Please note that features created in `feature_engineering_expand()` will have automatic FreqAI naming schemas depending on the expansions that you configured (i.e. `include_timeframes`, `include_corr_pairlist`, `indicators_periods_candles`, `include_shifted_candles`). So if you want to plot `%%-rsi` from `feature_engineering_expand_all()`, the final naming scheme for your plotting config would be: `%%-rsi-period_10_ETH/USDT:USDT_1h` for the `rsi` feature with `period=10`, `timeframe=1h`, and `pair=ETH/USDT:USDT` (the `:USDT` is added if you are using futures pairs). It is useful to simply add `print(dataframe.columns)` in your `populate_indicators()` after `self.freqai.start()` to see the full list of available features that are returned to the strategy for plotting purposes.
 
 ## Setting the `startup_candle_count`
 

docs/freqai-running.md

@@ -41,11 +41,11 @@ FreqAI stores new model files after each successful training. These files become
 
 ```json
     "freqai": {
-        "purge_old_models": true,
+        "purge_old_models": 4,
     }
 ```
 
-This will automatically purge all models older than the two most recently trained ones to save disk space.
+This will automatically purge all models older than the four most recently trained ones to save disk space. Inputing "0" will never purge any models.
 
 ## Backtesting
 
@@ -68,7 +68,7 @@ Backtesting mode requires [downloading the necessary data](#downloading-data-to-
     This way, you can return to using any model you wish by simply specifying the `identifier`.
 
 !!! Note
-    Backtesting calls `set_freqai_targets()` one time for each backtest window (where the number of windows is the full backtest timerange divided by the `backtest_period_days` parameter). Doing this means that the targets simulate dry/live behavior without look ahead bias. However, the definition of the features in `feature_engineering_*()` is performed once on the entire backtest timerange. This means that you should be sure that features do look-ahead into the future.
+    Backtesting calls `set_freqai_targets()` one time for each backtest window (where the number of windows is the full backtest timerange divided by the `backtest_period_days` parameter). Doing this means that the targets simulate dry/live behavior without look ahead bias. However, the definition of the features in `feature_engineering_*()` is performed once on the entire training timerange. This means that you should be sure that features do not look-ahead into the future.
     More details about look-ahead bias can be found in [Common Mistakes](strategy-customization.md#common-mistakes-when-developing-strategies).
 
 ---

docs/freqai.md

@@ -114,6 +114,11 @@ Here we compile some external materials that provide deeper looks into various c
 - [Real-time head-to-head: Adaptive modeling of financial market data using XGBoost and CatBoost](https://emergentmethods.medium.com/real-time-head-to-head-adaptive-modeling-of-financial-market-data-using-xgboost-and-catboost-995a115a7495)
 - [FreqAI - from price to prediction](https://emergentmethods.medium.com/freqai-from-price-to-prediction-6fadac18b665)
 
+
+## Support
+
+You can find support for FreqAI in a variety of places, including the [Freqtrade discord](https://discord.gg/Jd8JYeWHc4), the dedicated [FreqAI discord](https://discord.gg/7AMWACmbjT), and in [github issues](https://github.com/freqtrade/freqtrade/issues).
+
 ## Credits
 
 FreqAI is developed by a group of individuals who all contribute specific skillsets to the project.

docs/hyperopt.md

@@ -439,7 +439,7 @@ While this strategy is most likely too simple to provide consistent profit, it s
 ??? Hint "Performance tip"
     During normal hyperopting, indicators are calculated once and supplied to each epoch, linearly increasing RAM usage as a factor of increasing cores. As this also has performance implications, there are two alternatives to reduce RAM usage
 
-    * Move `ema_short` and `ema_long` calculations from `populate_indicators()` to `populate_entry_trend()`. Since `populate_entry_trend()` gonna be calculated every epochs, you don't need to use `.range` functionality.
+    * Move `ema_short` and `ema_long` calculations from `populate_indicators()` to `populate_entry_trend()`. Since `populate_entry_trend()` will be calculated every epoch, you don't need to use `.range` functionality.
     * hyperopt provides `--analyze-per-epoch` which will move the execution of `populate_indicators()` to the epoch process, calculating a single value per parameter per epoch instead of using the `.range` functionality. In this case, `.range` functionality will only return the actually used value.
 
     These alternatives will reduce RAM usage, but increase CPU usage. However, your hyperopting run will be less likely to fail due to Out Of Memory (OOM) issues.
@@ -926,6 +926,12 @@ Once the optimized strategy has been implemented into your strategy, you should
 
 To achieve same the results (number of trades, their durations, profit, etc.) as during Hyperopt, please use the same configuration and parameters (timerange, timeframe, ...) used for hyperopt `--dmmp`/`--disable-max-market-positions` and `--eps`/`--enable-position-stacking` for Backtesting.
 
-Should results not match, please double-check to make sure you transferred all conditions correctly.
+### Why do my backtest results not match my hyperopt results?
-Pay special care to the stoploss, max_open_trades and trailing stoploss parameters, as these are often set in configuration files, which override changes to the strategy.
+Should results not match, check the following factors:
-You should also carefully review the log of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss`, `max_open_trades` or `trailing_stop`).
+
+* You may have added parameters to hyperopt in `populate_indicators()` where they will be calculated only once **for all epochs**. If you are, for example, trying to optimise multiple SMA timeperiod values, the hyperoptable timeperiod parameter should be placed in `populate_entry_trend()` which is calculated every epoch. See [Optimizing an indicator parameter](https://www.freqtrade.io/en/stable/hyperopt/#optimizing-an-indicator-parameter).
+* If you have disabled the auto-export of hyperopt parameters into the JSON parameters file, double-check to make sure you transferred all hyperopted values into your strategy correctly.
+* Check the logs to verify what parameters are being set and what values are being used.
+* Pay special care to the stoploss, max_open_trades and trailing stoploss parameters, as these are often set in configuration files, which override changes to the strategy. Check the logs of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss`, `max_open_trades` or `trailing_stop`).
+* Verify that you do not have an unexpected parameters JSON file overriding the parameters or the default hyperopt settings in your strategy.
+* Verify that any protections that are enabled in backtesting are also enabled when hyperopting, and vice versa. When using `--space protection`, protections are auto-enabled for hyperopting.

docs/includes/pairlists.md

@@ -192,7 +192,8 @@ The RemotePairList is defined in the pairlists section of the configuration sett
         "refresh_period": 1800,
         "keep_pairlist_on_failure": true,
         "read_timeout": 60,
-        "bearer_token": "my-bearer-token"
+        "bearer_token": "my-bearer-token",
+        "save_to_file": "user_data/filename.json" 
     }
 ]
 ```
@@ -207,6 +208,42 @@ In "append" mode, the retrieved pairlist is added to the original pairlist. All
 
 The `pairlist_url` option specifies the URL of the remote server where the pairlist is located, or the path to a local file (if file:/// is prepended). This allows the user to use either a remote server or a local file as the source for the pairlist.
 
+The `save_to_file` option, when provided with a valid filename, saves the processed pairlist to that file in JSON format. This option is optional, and by default, the pairlist is not saved to a file.
+
+??? Example "Multi bot with shared pairlist example"
+    
+    `save_to_file` can be used to save the pairlist to a file with Bot1:
+
+    ```json
+    "pairlists": [
+        {
+            "method": "RemotePairList",
+            "mode": "whitelist",
+            "pairlist_url": "https://example.com/pairlist",
+            "number_assets": 10,
+            "refresh_period": 1800,
+            "keep_pairlist_on_failure": true,
+            "read_timeout": 60,
+            "save_to_file": "user_data/filename.json" 
+        }
+    ]
+    ```
+
+    This saved pairlist file can be loaded by Bot2, or any additional bot with this configuration:
+
+    ```json
+    "pairlists": [
+        {
+            "method": "RemotePairList",
+            "mode": "whitelist",
+            "pairlist_url": "file:///user_data/filename.json",
+            "number_assets": 10,
+            "refresh_period": 10,
+            "keep_pairlist_on_failure": true,
+        }
+    ]
+    ```    
+
 The user is responsible for providing a server or local file that returns a JSON object with the following structure:
 
 ```json

docs/includes/showcase.md

@@ -5,7 +5,7 @@ This section will highlight a few projects from members of the community.
 - [Example freqtrade strategies](https://github.com/freqtrade/freqtrade-strategies/)
 - [FrequentHippo - Grafana dashboard with dry/live runs and backtests](http://frequenthippo.ddns.net:3000/) (by hippocritical).
 - [Online pairlist generator](https://remotepairlist.com/) (by Blood4rc).
-- [Freqtrade Backtesting Project](https://bt.robot.co.network/) (by Blood4rc).
+- [Freqtrade Backtesting Project](https://strat.ninja/) (by Blood4rc).
 - [Freqtrade analysis notebook](https://github.com/froggleston/freqtrade_analysis_notebook) (by Froggleston).
 - [TUI for freqtrade](https://github.com/froggleston/freqtrade-frogtrade9000) (by Froggleston).
 - [Bot Academy](https://botacademy.ddns.net/) (by stash86) - Blog about crypto bot projects.

docs/index.md

@@ -42,7 +42,7 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual,
 - [X] [Binance](https://www.binance.com/)
 - [X] [Bitmart](https://bitmart.com/)
 - [X] [Gate.io](https://www.gate.io/ref/6266643)
-- [X] [Huobi](http://huobi.com/)
+- [X] [HTX](https://www.htx.com/) (Former Huobi)
 - [X] [Kraken](https://kraken.com/)
 - [X] [OKX](https://okx.com/) (Former OKEX)
 - [ ] [potentially many others through <img alt="ccxt" width="30px" src="assets/ccxt-logo.svg" />](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
-markdown==3.5.1
+markdown==3.5.2
 mkdocs==1.5.3
-mkdocs-material==9.5.3
+mkdocs-material==9.5.6
 mdx_truly_sane_lists==1.3
-pymdown-extensions==10.5
+pymdown-extensions==10.7
-jinja2==3.1.2
+jinja2==3.1.3

docs/stoploss.md

@@ -30,7 +30,7 @@ The Order-type will be ignored if only one mode is available.
 |----------|-------------|
 | Binance  | limit |
 | Binance Futures  | market, limit |
-| Huobi    | limit |
+| HTX (former Huobi)    | limit |
 | kraken   | market, limit |
 | Gate     | limit |
 | Okx      | limit |

docs/strategy-callbacks.md

@@ -760,22 +760,31 @@ The `position_adjustment_enable` strategy property enables the usage of `adjust_
 For performance reasons, it's disabled by default and freqtrade will show a warning message on startup if enabled.
 `adjust_trade_position()` can be used to perform additional orders, for example to manage risk with DCA (Dollar Cost Averaging) or to increase or decrease positions.
 
-`max_entry_position_adjustment` property is used to limit the number of additional entries per trade (on top of the first entry order) that the bot can execute. By default, the value is -1 which means the bot have no limit on number of adjustment entries.
-
-The strategy is expected to return a stake_amount (in stake currency) between `min_stake` and `max_stake` if and when an additional entry order should be made (position is increased -> buy order for long trades, sell order for short trades).
-If there are not enough funds in the wallet (the return value is above `max_stake`) then the signal will be ignored.
 Additional orders also result in additional fees and those orders don't count towards `max_open_trades`.
 
 This callback is **not** called when there is an open order (either buy or sell) waiting for execution.
 
 `adjust_trade_position()` is called very frequently for the duration of a trade, so you must keep your implementation as performant as possible.
 
-Additional entries are ignored once you have reached the maximum amount of extra entries that you have set on `max_entry_position_adjustment`, but the callback is called anyway looking for partial exits.
-
 Position adjustments will always be applied in the direction of the trade, so a positive value will always increase your position (negative values will decrease your position), no matter if it's a long or short trade.
 
 Modifications to leverage are not possible, and the stake-amount returned is assumed to be before applying leverage.
 
+### Increase position
+
+The strategy is expected to return a positive **stake_amount** (in stake currency) between `min_stake` and `max_stake` if and when an additional entry order should be made (position is increased -> buy order for long trades, sell order for short trades).
+
+If there are not enough funds in the wallet (the return value is above `max_stake`) then the signal will be ignored.
+`max_entry_position_adjustment` property is used to limit the number of additional entries per trade (on top of the first entry order) that the bot can execute. By default, the value is -1 which means the bot have no limit on number of adjustment entries.
+
+Additional entries are ignored once you have reached the maximum amount of extra entries that you have set on `max_entry_position_adjustment`, but the callback is called anyway looking for partial exits.
+
+### Decrease position
+
+The strategy is expected to return a negative stake_amount (in stake currency) for a partial exit.
+Returning the full owned stake at that point (based on the current price) (`-(trade.amount / trade.leverage) * current_exit_rate`) results in a full exit.  
+Returning a value more than the above (so remaining stake_amount would become negative) will result in the bot ignoring the signal.
+
 !!! Note "About stake size"
     Using fixed stake size means it will be the amount used for the first order, just like without position adjustment.
     If you wish to buy additional orders with DCA, then make sure to leave enough funds in the wallet for that.

docs/strategy-customization.md

@@ -156,9 +156,9 @@ def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame
 
 Out of the box, freqtrade installs the following technical libraries:
 
-* [ta-lib](http://mrjbq7.github.io/ta-lib/)
+- [ta-lib](https://ta-lib.github.io/ta-lib-python/)
-* [pandas-ta](https://twopirllc.github.io/pandas-ta/)
+- [pandas-ta](https://twopirllc.github.io/pandas-ta/)
-* [technical](https://github.com/freqtrade/technical/)
+- [technical](https://github.com/freqtrade/technical/)
 
 Additional technical libraries can be installed as necessary, or custom indicators may be written / invented by the strategy author.
 
@@ -367,6 +367,11 @@ class AwesomeStrategy(IStrategy):
     }
 ```
 
+??? info "Orders that don't fill immediately"
+    `minimal_roi` will take the `trade.open_date` as reference, which is the time the trade was initialized / the first order for this trade was placed.  
+    This will also hold true for limit orders that don't fill immediately  (usually in combination with "off-spot" prices through `custom_entry_price()`), as well as for cases where the initial order is replaced through `adjust_entry_price()`.
+    The time used will still be from the initial `trade.open_date` (when the initial order was first placed), not from the newly placed order date.
+
 ### Stoploss
 
 Setting a stoploss is highly recommended to protect your capital from strong moves against you.
@@ -1004,8 +1009,8 @@ This is a common pain-point, which can cause huge differences between backtestin
 
 The following lists some common patterns which should be avoided to prevent frustration:
 
-- don't use `shift(-1)`. This uses data from the future, which is not available.
+- don't use `shift(-1)` or other negative values. This uses data from the future in backtesting, which is not available in dry or live modes.
-- don't use `.iloc[-1]` or any other absolute position in the dataframe, this will be different between dry-run and backtesting.
+- don't use `.iloc[-1]` or any other absolute position in the dataframe within `populate_` functions, as this will be different between dry-run and backtesting. Absolute `iloc` indexing is safe to use in callbacks however - see [Strategy Callbacks](strategy-callbacks.md).
 - don't use `dataframe['volume'].mean()`. This uses the full DataFrame for backtesting, including data from the future. Use `dataframe['volume'].rolling(<window>).mean()` instead
 - don't use `.resample('1h')`. This uses the left border of the interval, so moves data from an hour to the start of the hour. Use `.resample('1h', label='right')` instead.
 

docs/webhook-config.md

@@ -134,6 +134,7 @@ Possible parameters are:
 * `stake_amount`
 * `stake_currency`
 * `base_currency`
+* `quote_currency`
 * `fiat_currency`
 * `order_type`
 * `current_rate`
@@ -155,6 +156,7 @@ Possible parameters are:
 * `stake_amount`
 * `stake_currency`
 * `base_currency`
+* `quote_currency`
 * `fiat_currency`
 * `order_type`
 * `current_rate`
@@ -176,6 +178,7 @@ Possible parameters are:
 * `stake_amount`
 * `stake_currency`
 * `base_currency`
+* `quote_currency`
 * `fiat_currency`
 * `order_type`
 * `current_rate`
@@ -199,6 +202,7 @@ Possible parameters are:
 * `profit_ratio`
 * `stake_currency`
 * `base_currency`
+* `quote_currency`
 * `fiat_currency`
 * `exit_reason`
 * `order_type`
@@ -224,6 +228,7 @@ Possible parameters are:
 * `profit_ratio`
 * `stake_currency`
 * `base_currency`
+* `quote_currency`
 * `fiat_currency`
 * `exit_reason`
 * `order_type`
@@ -249,6 +254,7 @@ Possible parameters are:
 * `profit_ratio`
 * `stake_currency`
 * `base_currency`
+* `quote_currency`
 * `fiat_currency`
 * `exit_reason`
 * `order_type`

docs/windows_installation.md

@@ -22,7 +22,7 @@ git clone https://github.com/freqtrade/freqtrade.git
 
 ### 2. Install ta-lib
 
-Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows).
+Install ta-lib according to the [ta-lib documentation](https://github.com/TA-Lib/ta-lib-python#windows).
 
 As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), Freqtrade provides these dependencies (in the binary wheel format) for the latest 3 Python versions (3.9, 3.10 and 3.11) and for 64bit Windows.
 These Wheels are also used by CI running on windows, and are therefore tested together with freqtrade.

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ Freqtrade bot """
-__version__ = '2023.12'
+__version__ = '2024.1'
 
 if 'dev' in __version__:
     from pathlib import Path

freqtrade/commands/arguments.py

@@ -219,27 +219,35 @@ class Arguments:
                                                 )
 
         # Add trade subcommand
-        trade_cmd = subparsers.add_parser('trade', help='Trade module.',
+        trade_cmd = subparsers.add_parser(
-                                          parents=[_common_parser, _strategy_parser])
+            'trade',
+            help='Trade module.',
+            parents=[_common_parser, _strategy_parser]
+        )
         trade_cmd.set_defaults(func=start_trading)
         self._build_args(optionlist=ARGS_TRADE, parser=trade_cmd)
 
         # add create-userdir subcommand
-        create_userdir_cmd = subparsers.add_parser('create-userdir',
+        create_userdir_cmd = subparsers.add_parser(
+            'create-userdir',
             help="Create user-data directory.",
         )
         create_userdir_cmd.set_defaults(func=start_create_userdir)
         self._build_args(optionlist=ARGS_CREATE_USERDIR, parser=create_userdir_cmd)
 
         # add new-config subcommand
-        build_config_cmd = subparsers.add_parser('new-config',
+        build_config_cmd = subparsers.add_parser(
-                                                 help="Create new config")
+            'new-config',
+            help="Create new config",
+        )
         build_config_cmd.set_defaults(func=start_new_config)
         self._build_args(optionlist=ARGS_BUILD_CONFIG, parser=build_config_cmd)
 
         # add new-strategy subcommand
-        build_strategy_cmd = subparsers.add_parser('new-strategy',
+        build_strategy_cmd = subparsers.add_parser(
-                                                   help="Create new strategy")
+            'new-strategy',
+            help="Create new strategy",
+        )
         build_strategy_cmd.set_defaults(func=start_new_strategy)
         self._build_args(optionlist=ARGS_BUILD_STRATEGY, parser=build_strategy_cmd)
 
@@ -289,8 +297,11 @@ class Arguments:
         self._build_args(optionlist=ARGS_LIST_DATA, parser=list_data_cmd)
 
         # Add backtesting subcommand
-        backtesting_cmd = subparsers.add_parser('backtesting', help='Backtesting module.',
+        backtesting_cmd = subparsers.add_parser(
-                                                parents=[_common_parser, _strategy_parser])
+            'backtesting',
+            help='Backtesting module.',
+            parents=[_common_parser, _strategy_parser]
+        )
         backtesting_cmd.set_defaults(func=start_backtesting)
         self._build_args(optionlist=ARGS_BACKTEST, parser=backtesting_cmd)
 
@@ -304,20 +315,27 @@ class Arguments:
         self._build_args(optionlist=ARGS_BACKTEST_SHOW, parser=backtesting_show_cmd)
 
         # Add backtesting analysis subcommand
-        analysis_cmd = subparsers.add_parser('backtesting-analysis',
+        analysis_cmd = subparsers.add_parser(
+            'backtesting-analysis',
             help='Backtest Analysis module.',
-                                             parents=[_common_parser])
+            parents=[_common_parser]
+        )
         analysis_cmd.set_defaults(func=start_analysis_entries_exits)
         self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd)
 
         # Add edge subcommand
-        edge_cmd = subparsers.add_parser('edge', help='Edge module.',
+        edge_cmd = subparsers.add_parser(
-                                         parents=[_common_parser, _strategy_parser])
+            'edge',
+            help='Edge module.',
+            parents=[_common_parser, _strategy_parser]
+        )
         edge_cmd.set_defaults(func=start_edge)
         self._build_args(optionlist=ARGS_EDGE, parser=edge_cmd)
 
         # Add hyperopt subcommand
-        hyperopt_cmd = subparsers.add_parser('hyperopt', help='Hyperopt module.',
+        hyperopt_cmd = subparsers.add_parser(
+            'hyperopt',
+            help='Hyperopt module.',
             parents=[_common_parser, _strategy_parser],
         )
         hyperopt_cmd.set_defaults(func=start_hyperopt)
@@ -447,16 +465,20 @@ class Arguments:
         self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd)
 
         # Add webserver subcommand
-        webserver_cmd = subparsers.add_parser('webserver', help='Webserver module.',
+        webserver_cmd = subparsers.add_parser(
-                                              parents=[_common_parser])
+            'webserver',
+            help='Webserver module.',
+            parents=[_common_parser]
+        )
         webserver_cmd.set_defaults(func=start_webserver)
         self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd)
 
         # Add strategy_updater subcommand
-        strategy_updater_cmd = subparsers.add_parser('strategy-updater',
+        strategy_updater_cmd = subparsers.add_parser(
-                                                     help='updates outdated strategy'
+            'strategy-updater',
-                                                          'files to the current version',
+            help='updates outdated strategy files to the current version',
-                                                     parents=[_common_parser])
+            parents=[_common_parser]
+        )
         strategy_updater_cmd.set_defaults(func=start_strategy_update)
         self._build_args(optionlist=ARGS_STRATEGY_UPDATER, parser=strategy_updater_cmd)
 
@@ -464,8 +486,8 @@ class Arguments:
         lookahead_analayis_cmd = subparsers.add_parser(
             'lookahead-analysis',
             help="Check for potential look ahead bias.",
-            parents=[_common_parser, _strategy_parser])
+            parents=[_common_parser, _strategy_parser]
-
+        )
         lookahead_analayis_cmd.set_defaults(func=start_lookahead_analysis)
 
         self._build_args(optionlist=ARGS_LOOKAHEAD_ANALYSIS,
@@ -475,8 +497,8 @@ class Arguments:
         recursive_analayis_cmd = subparsers.add_parser(
             'recursive-analysis',
             help="Check for potential recursive formula issue.",
-            parents=[_common_parser, _strategy_parser])
+            parents=[_common_parser, _strategy_parser]
-
+        )
         recursive_analayis_cmd.set_defaults(func=start_recursive_analysis)
 
         self._build_args(optionlist=ARGS_RECURSIVE_ANALYSIS,

freqtrade/commands/build_config_commands.py

@@ -109,7 +109,7 @@ def ask_user_config() -> Dict[str, Any]:
                 "binance",
                 "binanceus",
                 "gate",
-                "huobi",
+                "htx",
                 "kraken",
                 "kucoin",
                 "okx",

freqtrade/commands/data_commands.py

@@ -12,7 +12,7 @@ from freqtrade.enums import RunMode, TradingMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.resolvers import ExchangeResolver
-from freqtrade.util.binance_mig import migrate_binance_futures_data
+from freqtrade.util.migrations import migrate_data
 
 
 logger = logging.getLogger(__name__)
@@ -78,7 +78,7 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
     """
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
     if ohlcv:
-        migrate_binance_futures_data(config)
+        migrate_data(config)
         convert_ohlcv_format(config,
                              convert_from=args['format_from'],
                              convert_to=args['format_to'],
@@ -134,10 +134,10 @@ def start_list_data(args: Dict[str, Any]) -> None:
         print(tabulate([
             (pair, timeframe, candle_type,
                 start.strftime(DATETIME_PRINT_FORMAT),
-                end.strftime(DATETIME_PRINT_FORMAT))
+                end.strftime(DATETIME_PRINT_FORMAT), length)
-            for pair, timeframe, candle_type, start, end in sorted(
+            for pair, timeframe, candle_type, start, end, length in sorted(
                 paircombs1,
                 key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2]))
             ],
-            headers=("Pair", "Timeframe", "Type", 'From', 'To'),
+            headers=("Pair", "Timeframe", "Type", 'From', 'To', 'Candles'),
             tablefmt='psql', stralign='right'))

freqtrade/commands/optimize_commands.py

@@ -5,7 +5,7 @@ from freqtrade import constants
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
-from freqtrade.misc import round_coin_value
+from freqtrade.util import fmt_coin
 
 
 logger = logging.getLogger(__name__)
@@ -29,8 +29,8 @@ def setup_optimize_configuration(args: Dict[str, Any], method: RunMode) -> Dict[
         # tradable_balance_ratio
         if (config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT
                 and config['stake_amount'] > wallet_size):
-            wallet = round_coin_value(wallet_size, config['stake_currency'])
+            wallet = fmt_coin(wallet_size, config['stake_currency'])
-            stake = round_coin_value(config['stake_amount'], config['stake_currency'])
+            stake = fmt_coin(config['stake_amount'], config['stake_currency'])
             raise OperationalException(
                 f"Starting balance ({wallet}) is smaller than stake_amount {stake}. "
                 f"Wallet is calculated as `dry_run_wallet * tradable_balance_ratio`."

freqtrade/commands/pairlist_commands.py

@@ -15,6 +15,7 @@ def start_test_pairlist(args: Dict[str, Any]) -> None:
     """
     Test Pairlist configuration
     """
+    from freqtrade.persistence import FtNoDBContext
     from freqtrade.plugins.pairlistmanager import PairListManager
     config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
 
@@ -24,6 +25,7 @@ def start_test_pairlist(args: Dict[str, Any]) -> None:
     if not quote_currencies:
         quote_currencies = [config.get('stake_currency')]
     results = {}
+    with FtNoDBContext():
         for curr in quote_currencies:
             config['stake_currency'] = curr
             pairlists = PairListManager(exchange, config)

freqtrade/configuration/configuration.py

@@ -5,7 +5,7 @@ import logging
 import warnings
 from copy import deepcopy
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
 from freqtrade import constants
 from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings
@@ -68,6 +68,8 @@ class Configuration:
         config: Config = load_from_files(self.args.get("config", []))
 
         # Load environment variables
+        from freqtrade.commands.arguments import NO_CONF_ALLOWED
+        if self.args.get('command') not in NO_CONF_ALLOWED:
             env_data = enironment_vars_to_dict()
             config = deep_merge_dicts(env_data, config)
 
@@ -233,54 +235,37 @@ class Configuration:
             except ValueError:
                 pass
 
-        self._args_to_config(config, argname='timeframe_detail',
+        configurations = [
-                             logstring='Parameter --timeframe-detail detected, '
+            ('timeframe_detail',
-                             'using {} for intra-candle backtesting ...')
+             'Parameter --timeframe-detail detected, using {} for intra-candle backtesting ...'),
+            ('backtest_show_pair_list', 'Parameter --show-pair-list detected.'),
+            ('stake_amount',
+             'Parameter --stake-amount detected, overriding stake_amount to: {} ...'),
+            ('dry_run_wallet',
+             'Parameter --dry-run-wallet detected, overriding dry_run_wallet to: {} ...'),
+            ('fee', 'Parameter --fee detected, setting fee to: {} ...'),
+            ('timerange', 'Parameter --timerange detected: {} ...'),
+            ]
 
-        self._args_to_config(config, argname='backtest_show_pair_list',
+        self._args_to_config_loop(config, configurations)
-                             logstring='Parameter --show-pair-list detected.')
-
-        self._args_to_config(config, argname='stake_amount',
-                             logstring='Parameter --stake-amount detected, '
-                             'overriding stake_amount to: {} ...')
-        self._args_to_config(config, argname='dry_run_wallet',
-                             logstring='Parameter --dry-run-wallet detected, '
-                             'overriding dry_run_wallet to: {} ...')
-        self._args_to_config(config, argname='fee',
-                             logstring='Parameter --fee detected, '
-                             'setting fee to: {} ...')
-
-        self._args_to_config(config, argname='timerange',
-                             logstring='Parameter --timerange detected: {} ...')
 
         self._process_datadir_options(config)
 
         self._args_to_config(config, argname='strategy_list',
                              logstring='Using strategy list of {} strategies', logfun=len)
 
-        self._args_to_config(
+        configurations = [
-            config,
+            ('recursive_strategy_search',
-            argname='recursive_strategy_search',
+             'Recursively searching for a strategy in the strategies folder.'),
-            logstring='Recursively searching for a strategy in the strategies folder.',
+            ('timeframe', 'Overriding timeframe with Command line argument'),
-        )
+            ('export', 'Parameter --export detected: {} ...'),
-
+            ('backtest_breakdown', 'Parameter --breakdown detected ...'),
-        self._args_to_config(config, argname='timeframe',
+            ('backtest_cache', 'Parameter --cache={} detected ...'),
-                             logstring='Overriding timeframe with Command line argument')
+            ('disableparamexport', 'Parameter --disableparamexport detected: {} ...'),
-
+            ('freqai_backtest_live_models',
-        self._args_to_config(config, argname='export',
+             'Parameter --freqai-backtest-live-models detected ...'),
-                             logstring='Parameter --export detected: {} ...')
+        ]
-
+        self._args_to_config_loop(config, configurations)
-        self._args_to_config(config, argname='backtest_breakdown',
-                             logstring='Parameter --breakdown detected ...')
-
-        self._args_to_config(config, argname='backtest_cache',
-                             logstring='Parameter --cache={} detected ...')
-
-        self._args_to_config(config, argname='disableparamexport',
-                             logstring='Parameter --disableparamexport detected: {} ...')
-
-        self._args_to_config(config, argname='freqai_backtest_live_models',
-                             logstring='Parameter --freqai-backtest-live-models detected ...')
 
         # Edge section:
         if 'stoploss_range' in self.args and self.args["stoploss_range"]:
@@ -291,31 +276,18 @@ class Configuration:
             logger.info('Parameter --stoplosses detected: %s ...', self.args["stoploss_range"])
 
         # Hyperopt section
-        self._args_to_config(config, argname='hyperopt',
-                             logstring='Using Hyperopt class name: {}')
 
-        self._args_to_config(config, argname='hyperopt_path',
+        configurations = [
-                             logstring='Using additional Hyperopt lookup path: {}')
+            ('hyperopt', 'Using Hyperopt class name: {}'),
-
+            ('hyperopt_path', 'Using additional Hyperopt lookup path: {}'),
-        self._args_to_config(config, argname='hyperoptexportfilename',
+            ('hyperoptexportfilename', 'Using hyperopt file: {}'),
-                             logstring='Using hyperopt file: {}')
+            ('lookahead_analysis_exportfilename', 'Saving lookahead analysis results into {} ...'),
-
+            ('epochs', 'Parameter --epochs detected ... Will run Hyperopt with for {} epochs ...'),
-        self._args_to_config(config, argname='lookahead_analysis_exportfilename',
+            ('spaces', 'Parameter -s/--spaces detected: {}'),
-                             logstring='Saving lookahead analysis results into {} ...')
+            ('analyze_per_epoch', 'Parameter --analyze-per-epoch detected.'),
-
+            ('print_all', 'Parameter --print-all detected ...'),
-        self._args_to_config(config, argname='epochs',
+        ]
-                             logstring='Parameter --epochs detected ... '
+        self._args_to_config_loop(config, configurations)
-                             'Will run Hyperopt with for {} epochs ...'
-                             )
-
-        self._args_to_config(config, argname='spaces',
-                             logstring='Parameter -s/--spaces detected: {}')
-
-        self._args_to_config(config, argname='analyze_per_epoch',
-                             logstring='Parameter --analyze-per-epoch detected.')
-
-        self._args_to_config(config, argname='print_all',
-                             logstring='Parameter --print-all detected ...')
 
         if 'print_colorized' in self.args and not self.args["print_colorized"]:
             logger.info('Parameter --no-color detected ...')
@@ -323,123 +295,55 @@ class Configuration:
         else:
             config.update({'print_colorized': True})
 
-        self._args_to_config(config, argname='print_json',
+        configurations = [
-                             logstring='Parameter --print-json detected ...')
+            ('print_json', 'Parameter --print-json detected ...'),
+            ('export_csv', 'Parameter --export-csv detected: {}'),
+            ('hyperopt_jobs', 'Parameter -j/--job-workers detected: {}'),
+            ('hyperopt_random_state', 'Parameter --random-state detected: {}'),
+            ('hyperopt_min_trades', 'Parameter --min-trades detected: {}'),
+            ('hyperopt_loss', 'Using Hyperopt loss class name: {}'),
+            ('hyperopt_show_index', 'Parameter -n/--index detected: {}'),
+            ('hyperopt_list_best', 'Parameter --best detected: {}'),
+            ('hyperopt_list_profitable', 'Parameter --profitable detected: {}'),
+            ('hyperopt_list_min_trades', 'Parameter --min-trades detected: {}'),
+            ('hyperopt_list_max_trades', 'Parameter --max-trades detected: {}'),
+            ('hyperopt_list_min_avg_time', 'Parameter --min-avg-time detected: {}'),
+            ('hyperopt_list_max_avg_time', 'Parameter --max-avg-time detected: {}'),
+            ('hyperopt_list_min_avg_profit', 'Parameter --min-avg-profit detected: {}'),
+            ('hyperopt_list_max_avg_profit', 'Parameter --max-avg-profit detected: {}'),
+            ('hyperopt_list_min_total_profit', 'Parameter --min-total-profit detected: {}'),
+            ('hyperopt_list_max_total_profit', 'Parameter --max-total-profit detected: {}'),
+            ('hyperopt_list_min_objective', 'Parameter --min-objective detected: {}'),
+            ('hyperopt_list_max_objective', 'Parameter --max-objective detected: {}'),
+            ('hyperopt_list_no_details', 'Parameter --no-details detected: {}'),
+            ('hyperopt_show_no_header', 'Parameter --no-header detected: {}'),
+            ('hyperopt_ignore_missing_space', 'Paramter --ignore-missing-space detected: {}'),
+        ]
 
-        self._args_to_config(config, argname='export_csv',
+        self._args_to_config_loop(config, configurations)
-                             logstring='Parameter --export-csv detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_jobs',
-                             logstring='Parameter -j/--job-workers detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_random_state',
-                             logstring='Parameter --random-state detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_min_trades',
-                             logstring='Parameter --min-trades detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_loss',
-                             logstring='Using Hyperopt loss class name: {}')
-
-        self._args_to_config(config, argname='hyperopt_show_index',
-                             logstring='Parameter -n/--index detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_best',
-                             logstring='Parameter --best detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_profitable',
-                             logstring='Parameter --profitable detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_min_trades',
-                             logstring='Parameter --min-trades detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_max_trades',
-                             logstring='Parameter --max-trades detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_min_avg_time',
-                             logstring='Parameter --min-avg-time detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_max_avg_time',
-                             logstring='Parameter --max-avg-time detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_min_avg_profit',
-                             logstring='Parameter --min-avg-profit detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_max_avg_profit',
-                             logstring='Parameter --max-avg-profit detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_min_total_profit',
-                             logstring='Parameter --min-total-profit detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_max_total_profit',
-                             logstring='Parameter --max-total-profit detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_min_objective',
-                             logstring='Parameter --min-objective detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_max_objective',
-                             logstring='Parameter --max-objective detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_no_details',
-                             logstring='Parameter --no-details detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_show_no_header',
-                             logstring='Parameter --no-header detected: {}')
-
-        self._args_to_config(config, argname="hyperopt_ignore_missing_space",
-                             logstring="Paramter --ignore-missing-space detected: {}")
 
     def _process_plot_options(self, config: Config) -> None:
 
-        self._args_to_config(config, argname='pairs',
+        configurations = [
-                             logstring='Using pairs {}')
+            ('pairs', 'Using pairs {}'),
-
+            ('indicators1', 'Using indicators1: {}'),
-        self._args_to_config(config, argname='indicators1',
+            ('indicators2', 'Using indicators2: {}'),
-                             logstring='Using indicators1: {}')
+            ('trade_ids', 'Filtering on trade_ids: {}'),
-
+            ('plot_limit', 'Limiting plot to: {}'),
-        self._args_to_config(config, argname='indicators2',
+            ('plot_auto_open', 'Parameter --auto-open detected.'),
-                             logstring='Using indicators2: {}')
+            ('trade_source', 'Using trades from: {}'),
-
+            ('prepend_data', 'Prepend detected. Allowing data prepending.'),
-        self._args_to_config(config, argname='trade_ids',
+            ('erase', 'Erase detected. Deleting existing data.'),
-                             logstring='Filtering on trade_ids: {}')
+            ('no_trades', 'Parameter --no-trades detected.'),
-
+            ('timeframes', 'timeframes --timeframes: {}'),
-        self._args_to_config(config, argname='plot_limit',
+            ('days', 'Detected --days: {}'),
-                             logstring='Limiting plot to: {}')
+            ('include_inactive', 'Detected --include-inactive-pairs: {}'),
-
+            ('download_trades', 'Detected --dl-trades: {}'),
-        self._args_to_config(config, argname='plot_auto_open',
+            ('dataformat_ohlcv', 'Using "{}" to store OHLCV data.'),
-                             logstring='Parameter --auto-open detected.')
+            ('dataformat_trades', 'Using "{}" to store trades data.'),
-
+            ('show_timerange', 'Detected --show-timerange'),
-        self._args_to_config(config, argname='trade_source',
+        ]
-                             logstring='Using trades from: {}')
+        self._args_to_config_loop(config, configurations)
-
-        self._args_to_config(config, argname='prepend_data',
-                             logstring='Prepend detected. Allowing data prepending.')
-        self._args_to_config(config, argname='erase',
-                             logstring='Erase detected. Deleting existing data.')
-
-        self._args_to_config(config, argname='no_trades',
-                             logstring='Parameter --no-trades detected.')
-
-        self._args_to_config(config, argname='timeframes',
-                             logstring='timeframes --timeframes: {}')
-
-        self._args_to_config(config, argname='days',
-                             logstring='Detected --days: {}')
-
-        self._args_to_config(config, argname='include_inactive',
-                             logstring='Detected --include-inactive-pairs: {}')
-
-        self._args_to_config(config, argname='download_trades',
-                             logstring='Detected --dl-trades: {}')
-
-        self._args_to_config(config, argname='dataformat_ohlcv',
-                             logstring='Using "{}" to store OHLCV data.')
-
-        self._args_to_config(config, argname='dataformat_trades',
-                             logstring='Using "{}" to store trades data.')
-
-        self._args_to_config(config, argname='show_timerange',
-                             logstring='Detected --show-timerange')
 
     def _process_data_options(self, config: Config) -> None:
         self._args_to_config(config, argname='new_pairs_days',
@@ -453,45 +357,27 @@ class Configuration:
                              logstring='Detected --candle-types: {}')
 
     def _process_analyze_options(self, config: Config) -> None:
-        self._args_to_config(config, argname='analysis_groups',
+        configurations = [
-                             logstring='Analysis reason groups: {}')
+            ('analysis_groups', 'Analysis reason groups: {}'),
-
+            ('enter_reason_list', 'Analysis enter tag list: {}'),
-        self._args_to_config(config, argname='enter_reason_list',
+            ('exit_reason_list', 'Analysis exit tag list: {}'),
-                             logstring='Analysis enter tag list: {}')
+            ('indicator_list', 'Analysis indicator list: {}'),
-
+            ('timerange', 'Filter trades by timerange: {}'),
-        self._args_to_config(config, argname='exit_reason_list',
+            ('analysis_rejected', 'Analyse rejected signals: {}'),
-                             logstring='Analysis exit tag list: {}')
+            ('analysis_to_csv', 'Store analysis tables to CSV: {}'),
-
+            ('analysis_csv_path', 'Path to store analysis CSVs: {}'),
-        self._args_to_config(config, argname='indicator_list',
-                             logstring='Analysis indicator list: {}')
-
-        self._args_to_config(config, argname='timerange',
-                             logstring='Filter trades by timerange: {}')
-
-        self._args_to_config(config, argname='analysis_rejected',
-                             logstring='Analyse rejected signals: {}')
-
-        self._args_to_config(config, argname='analysis_to_csv',
-                             logstring='Store analysis tables to CSV: {}')
-
-        self._args_to_config(config, argname='analysis_csv_path',
-                             logstring='Path to store analysis CSVs: {}')
-
-        self._args_to_config(config, argname='analysis_csv_path',
-                             logstring='Path to store analysis CSVs: {}')
-
             # Lookahead analysis results
-        self._args_to_config(config, argname='targeted_trade_amount',
+            ('targeted_trade_amount', 'Targeted Trade amount: {}'),
-                             logstring='Targeted Trade amount: {}')
+            ('minimum_trade_amount', 'Minimum Trade amount: {}'),
+            ('lookahead_analysis_exportfilename', 'Path to store lookahead-analysis-results: {}'),
+            ('startup_candle', 'Startup candle to be used on recursive analysis: {}'),
+        ]
+        self._args_to_config_loop(config, configurations)
 
-        self._args_to_config(config, argname='minimum_trade_amount',
+    def _args_to_config_loop(self, config, configurations: List[Tuple[str, str]]) -> None:
-                             logstring='Minimum Trade amount: {}')
 
-        self._args_to_config(config, argname='lookahead_analysis_exportfilename',
+        for argname, logstring in configurations:
-                             logstring='Path to store lookahead-analysis-results: {}')
+            self._args_to_config(config, argname=argname, logstring=logstring)
-
-        self._args_to_config(config, argname='startup_candle',
-                             logstring='Startup candle to be used on recursive analysis: {}')
 
     def _process_runmode(self, config: Config) -> None:
 

freqtrade/configuration/environment_vars.py

@@ -9,7 +9,7 @@ from freqtrade.misc import deep_merge_dicts
 logger = logging.getLogger(__name__)
 
 
-def get_var_typed(val):
+def _get_var_typed(val):
     try:
         return int(val)
     except ValueError:
@@ -24,7 +24,7 @@ def get_var_typed(val):
     return val
 
 
-def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, Any]:
+def _flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, Any]:
     """
     Environment variables must be prefixed with FREQTRADE.
     FREQTRADE__{section}__{key}
@@ -40,7 +40,7 @@ def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str,
             logger.info(f"Loading variable '{env_var}'")
             key = env_var.replace(prefix, '')
             for k in reversed(key.split('__')):
-                val = {k.lower(): get_var_typed(val)
+                val = {k.lower(): _get_var_typed(val)
                        if not isinstance(val, dict) and k not in no_convert else val}
             relevant_vars = deep_merge_dicts(val, relevant_vars)
     return relevant_vars
@@ -52,4 +52,4 @@ def enironment_vars_to_dict() -> Dict[str, Any]:
     Relevant variables must follow the FREQTRADE__{section}__{key} pattern
     :return: Nested dict based on available and relevant variables.
     """
-    return flat_vars_to_nested_dict(os.environ.copy(), ENV_VAR_PREFIX)
+    return _flat_vars_to_nested_dict(os.environ.copy(), ENV_VAR_PREFIX)

freqtrade/constants.py

@@ -105,7 +105,7 @@ SUPPORTED_FIAT = [
     "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY",
     "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN",
     "RUB", "UAH", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR",
-    "USD", "BTC", "ETH", "XRP", "LTC", "BCH"
+    "USD", "BTC", "ETH", "XRP", "LTC", "BCH", "BNB"
 ]
 
 MINIMAL_CONFIG = {

freqtrade/data/btanalysis.py

@@ -175,36 +175,40 @@ def _get_backtest_files(dirname: Path) -> List[Path]:
     return list(reversed(sorted(dirname.glob('backtest-result-*-[0-9][0-9].json'))))
 
 
-def get_backtest_result(filename: Path) -> List[BacktestHistoryEntryType]:
+def _extract_backtest_result(filename: Path) -> List[BacktestHistoryEntryType]:
-    """
+    metadata = load_backtest_metadata(filename)
-    Get backtest result read from metadata file
-    """
     return [
         {
             'filename': filename.stem,
             'strategy': s,
-            'notes': v.get('notes', ''),
             'run_id': v['run_id'],
+            'notes': v.get('notes', ''),
+            # Backtest "run" time
             'backtest_start_time': v['backtest_start_time'],
-        } for s, v in load_backtest_metadata(filename).items()
+            # Backtest timerange
+            'backtest_start_ts': v.get('backtest_start_ts', None),
+            'backtest_end_ts': v.get('backtest_end_ts', None),
+            'timeframe': v.get('timeframe', None),
+            'timeframe_detail': v.get('timeframe_detail', None),
+        } for s, v in metadata.items()
     ]
 
 
+def get_backtest_result(filename: Path) -> List[BacktestHistoryEntryType]:
+    """
+    Get backtest result read from metadata file
+    """
+    return _extract_backtest_result(filename)
+
+
 def get_backtest_resultlist(dirname: Path) -> List[BacktestHistoryEntryType]:
     """
     Get list of backtest results read from metadata files
     """
     return [
-        {
+        result
-            'filename': filename.stem,
-            'strategy': s,
-            'run_id': v['run_id'],
-            'notes': v.get('notes', ''),
-            'backtest_start_time': v['backtest_start_time'],
-        }
         for filename in _get_backtest_files(dirname)
-        for s, v in load_backtest_metadata(filename).items()
+        for result in _extract_backtest_result(filename)
-        if v
     ]
 
 
@@ -353,10 +357,10 @@ def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataF
     :param timeframe: Timeframe used for backtest
     :return: dataframe with open-counts per time-period in timeframe
     """
-    from freqtrade.exchange import timeframe_to_minutes
+    from freqtrade.exchange import timeframe_to_resample_freq
-    timeframe_min = timeframe_to_minutes(timeframe)
+    timeframe_freq = timeframe_to_resample_freq(timeframe)
     dates = [pd.Series(pd.date_range(row[1]['open_date'], row[1]['close_date'],
-                                     freq=f"{timeframe_min}min"))
+                                     freq=timeframe_freq))
              for row in results[['open_date', 'close_date']].iterrows()]
     deltas = [len(x) for x in dates]
     dates = pd.Series(pd.concat(dates).values, name='date')
@@ -364,7 +368,7 @@ def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataF
 
     df2 = pd.concat([dates, df2], axis=1)
     df2 = df2.set_index('date')
-    df_final = df2.resample(f"{timeframe_min}min")[['pair']].count()
+    df_final = df2.resample(timeframe_freq)[['pair']].count()
     df_final = df_final.rename({'pair': 'open_trades'}, axis=1)
     return df_final
 

freqtrade/data/converter/converter.py

@@ -84,7 +84,7 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str)
     using the previous close as price for "open", "high" "low" and "close", volume is set to 0
 
     """
-    from freqtrade.exchange import timeframe_to_minutes
+    from freqtrade.exchange import timeframe_to_resample_freq
 
     ohlcv_dict = {
         'open': 'first',
@@ -93,13 +93,7 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str)
         'close': 'last',
         'volume': 'sum'
     }
-    timeframe_minutes = timeframe_to_minutes(timeframe)
+    resample_interval = timeframe_to_resample_freq(timeframe)
-    resample_interval = f'{timeframe_minutes}min'
-    if timeframe_minutes >= 43200 and timeframe_minutes < 525600:
-        # Monthly candles need special treatment to stick to the 1st of the month
-        resample_interval = f'{timeframe}S'
-    elif timeframe_minutes > 43200:
-        resample_interval = timeframe
     # Resample to create "NAN" values
     df = dataframe.resample(resample_interval, on='date').agg(ohlcv_dict)
 

freqtrade/data/converter/trade_converter.py

@@ -70,14 +70,13 @@ def trades_to_ohlcv(trades: DataFrame, timeframe: str) -> DataFrame:
     :return: OHLCV Dataframe.
     :raises: ValueError if no trades are provided
     """
-    from freqtrade.exchange import timeframe_to_minutes
+    from freqtrade.exchange import timeframe_to_resample_freq
-    timeframe_minutes = timeframe_to_minutes(timeframe)
     if trades.empty:
         raise ValueError('Trade-list empty.')
     df = trades.set_index('date', drop=True)
-
+    resample_interval = timeframe_to_resample_freq(timeframe)
-    df_new = df['price'].resample(f'{timeframe_minutes}min').ohlc()
+    df_new = df['price'].resample(resample_interval).ohlc()
-    df_new['volume'] = df['amount'].resample(f'{timeframe_minutes}min').sum()
+    df_new['volume'] = df['amount'].resample(resample_interval).sum()
     df_new['date'] = df_new.index
     # Drop 0 volume rows
     df_new = df_new.dropna()

freqtrade/data/dataprovider.py

@@ -311,11 +311,13 @@ class DataProvider:
             timerange = TimeRange.parse_timerange(None if self._config.get(
                 'timerange') is None else str(self._config.get('timerange')))
 
-            # It is not necessary to add the training candles, as they
+            startup_candles = self.get_required_startup(str(timeframe))
-            # were already added at the beginning of the backtest.
-            startup_candles = self.get_required_startup(str(timeframe), False)
             tf_seconds = timeframe_to_seconds(str(timeframe))
             timerange.subtract_start(tf_seconds * startup_candles)
+
+            logger.info(f"Loading data for {pair} {timeframe} "
+                        f"from {timerange.start_fmt} to {timerange.stop_fmt}")
+
             self.__cached_pairs_backtesting[saved_pair] = load_pair_history(
                 pair=pair,
                 timeframe=timeframe,
@@ -327,7 +329,7 @@ class DataProvider:
             )
         return self.__cached_pairs_backtesting[saved_pair].copy()
 
-    def get_required_startup(self, timeframe: str, add_train_candles: bool = True) -> int:
+    def get_required_startup(self, timeframe: str) -> int:
         freqai_config = self._config.get('freqai', {})
         if not freqai_config.get('enabled', False):
             return self._config.get('startup_candle_count', 0)
@@ -337,11 +339,10 @@ class DataProvider:
             # make sure the startupcandles is at least the set maximum indicator periods
             self._config['startup_candle_count'] = max(startup_candles, max(indicator_periods))
             tf_seconds = timeframe_to_seconds(timeframe)
-            train_candles = 0
-            if add_train_candles:
             train_candles = freqai_config['train_period_days'] * 86400 / tf_seconds
             total_candles = int(self._config['startup_candle_count'] + train_candles)
-            logger.info(f'Increasing startup_candle_count for freqai to {total_candles}')
+            logger.info(
+                f'Increasing startup_candle_count for freqai on {timeframe} to {total_candles}')
         return total_candles
 
     def get_pair_dataframe(

freqtrade/data/history/history_utils.py

@@ -8,7 +8,7 @@ from pandas import DataFrame, concat
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import (DATETIME_PRINT_FORMAT, DEFAULT_DATAFRAME_COLUMNS,
-                                 DL_DATA_TIMEFRAMES, Config)
+                                 DL_DATA_TIMEFRAMES, DOCS_LINK, Config)
 from freqtrade.data.converter import (clean_ohlcv_dataframe, convert_trades_to_ohlcv,
                                       ohlcv_to_dataframe, trades_df_remove_duplicates,
                                       trades_list_to_df)
@@ -18,8 +18,8 @@ from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import Exchange
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
 from freqtrade.util import dt_ts, format_ms_time
-from freqtrade.util.binance_mig import migrate_binance_futures_data
 from freqtrade.util.datetime_helpers import dt_now
+from freqtrade.util.migrations import migrate_data
 
 
 logger = logging.getLogger(__name__)
@@ -311,15 +311,19 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
             # Predefined candletype (and timeframe) depending on exchange
             # Downloads what is necessary to backtest based on futures data.
             tf_mark = exchange.get_option('mark_ohlcv_timeframe')
+            tf_funding_rate = exchange.get_option('funding_fee_timeframe')
+
             fr_candle_type = CandleType.from_string(exchange.get_option('mark_ohlcv_price'))
             # All exchanges need FundingRate for futures trading.
             # The timeframe is aligned to the mark-price timeframe.
-            for funding_candle_type in (CandleType.FUNDING_RATE, fr_candle_type):
+            combs = ((CandleType.FUNDING_RATE, tf_funding_rate), (fr_candle_type, tf_mark))
+            for candle_type_f, tf in combs:
+                logger.debug(f'Downloading pair {pair}, {candle_type_f}, interval {tf}.')
                 _download_pair_history(pair=pair, process=process,
                                        datadir=datadir, exchange=exchange,
                                        timerange=timerange, data_handler=data_handler,
-                                       timeframe=str(tf_mark), new_pairs_days=new_pairs_days,
+                                       timeframe=str(tf), new_pairs_days=new_pairs_days,
-                                       candle_type=funding_candle_type,
+                                       candle_type=candle_type_f,
                                        erase=erase, prepend=prepend)
 
     return pairs_not_available
@@ -500,6 +504,12 @@ def download_data_main(config: Config) -> None:
     logger.info(f"About to download pairs: {expanded_pairs}, "
                 f"intervals: {config['timeframes']} to {config['datadir']}")
 
+    if len(expanded_pairs) == 0:
+        logger.warning(
+            "No pairs available for download. "
+            "Please make sure you're using the correct Pair naming for your selected trade mode. \n"
+            f"More info: {DOCS_LINK}/bot-basics/#pair-naming")
+
     for timeframe in config['timeframes']:
         exchange.validate_timeframes(timeframe)
 
@@ -527,7 +537,7 @@ def download_data_main(config: Config) -> None:
                     "Please use `--dl-trades` instead for this exchange "
                     "(will unfortunately take a long time)."
                     )
-            migrate_binance_futures_data(config)
+            migrate_data(config, exchange)
             pairs_not_available = refresh_backtest_ohlcv_data(
                 exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
                 datadir=config['datadir'], timerange=timerange,

freqtrade/data/history/idatahandler.py

@@ -94,21 +94,22 @@ class IDataHandler(ABC):
         """
 
     def ohlcv_data_min_max(self, pair: str, timeframe: str,
-                           candle_type: CandleType) -> Tuple[datetime, datetime]:
+                           candle_type: CandleType) -> Tuple[datetime, datetime, int]:
         """
         Returns the min and max timestamp for the given pair and timeframe.
         :param pair: Pair to get min/max for
         :param timeframe: Timeframe to get min/max for
         :param candle_type: Any of the enum CandleType (must match trading mode!)
-        :return: (min, max)
+        :return: (min, max, len)
         """
-        data = self._ohlcv_load(pair, timeframe, None, candle_type)
+        df = self._ohlcv_load(pair, timeframe, None, candle_type)
-        if data.empty:
+        if df.empty:
             return (
                 datetime.fromtimestamp(0, tz=timezone.utc),
-                datetime.fromtimestamp(0, tz=timezone.utc)
+                datetime.fromtimestamp(0, tz=timezone.utc),
+                0,
             )
-        return data.iloc[0]['date'].to_pydatetime(), data.iloc[-1]['date'].to_pydatetime()
+        return df.iloc[0]['date'].to_pydatetime(), df.iloc[-1]['date'].to_pydatetime(), len(df)
 
     @abstractmethod
     def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange],
@@ -403,6 +404,34 @@ class IDataHandler(ABC):
             return
         file_old.rename(file_new)
 
+    def fix_funding_fee_timeframe(self, ff_timeframe: str):
+        """
+        Temporary method to migrate data from old funding fee timeframe to the correct timeframe
+        Applies to bybit and okx, where funding-fee and mark candles have different timeframes.
+        """
+        paircombs = self.ohlcv_get_available_data(self._datadir, TradingMode.FUTURES)
+        funding_rate_combs = [
+            f for f in paircombs if f[2] == CandleType.FUNDING_RATE and f[1] != ff_timeframe
+        ]
+
+        if funding_rate_combs:
+            logger.warning(
+                f'Migrating {len(funding_rate_combs)} funding fees to correct timeframe.')
+
+        for pair, timeframe, candletype in funding_rate_combs:
+            old_name = self._pair_data_filename(self._datadir, pair, timeframe, candletype)
+            new_name = self._pair_data_filename(self._datadir, pair, ff_timeframe, candletype)
+
+            if not Path(old_name).exists():
+                logger.warning(f'{old_name} does not exist, skipping.')
+                continue
+
+            if Path(new_name).exists():
+                logger.warning(f'{new_name} already exists, Removing.')
+                Path(new_name).unlink()
+
+            Path(old_name).rename(new_name)
+
 
 def get_datahandlerclass(datatype: str) -> Type[IDataHandler]:
     """

freqtrade/data/metrics.py

@@ -61,10 +61,10 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
     """
     if len(trades) == 0:
         raise ValueError("Trade dataframe empty.")
-    from freqtrade.exchange import timeframe_to_minutes
+    from freqtrade.exchange import timeframe_to_resample_freq
-    timeframe_minutes = timeframe_to_minutes(timeframe)
+    timeframe_freq = timeframe_to_resample_freq(timeframe)
     # Resample to timeframe to make sure trades match candles
-    _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date'
+    _trades_sum = trades.resample(timeframe_freq, on='close_date'
                                   )[['profit_abs']].sum()
     df.loc[:, col_name] = _trades_sum['profit_abs'].cumsum()
     # Set first value to 0

freqtrade/exchange/__init__.py

@@ -17,10 +17,11 @@ from freqtrade.exchange.exchange_utils import (ROUND_DOWN, ROUND_UP, amount_to_c
                                                market_is_active, price_to_precision,
                                                timeframe_to_minutes, timeframe_to_msecs,
                                                timeframe_to_next_date, timeframe_to_prev_date,
-                                               timeframe_to_seconds, validate_exchange)
+                                               timeframe_to_resample_freq, timeframe_to_seconds,
+                                               validate_exchange)
 from freqtrade.exchange.gate import Gate
 from freqtrade.exchange.hitbtc import Hitbtc
-from freqtrade.exchange.huobi import Huobi
+from freqtrade.exchange.htx import Htx
 from freqtrade.exchange.kraken import Kraken
 from freqtrade.exchange.kucoin import Kucoin
 from freqtrade.exchange.okx import Okx

freqtrade/exchange/common.py

@@ -48,13 +48,14 @@ MAP_EXCHANGE_CHILDCLASS = {
     'binanceusdm': 'binance',
     'okex': 'okx',
     'gateio': 'gate',
+    'huboi': 'htx',
 }
 
 SUPPORTED_EXCHANGES = [
     'binance',
     'bitmart',
     'gate',
-    'huobi',
+    'htx',
     'kraken',
     'okx',
 ]

freqtrade/exchange/exchange.py

@@ -80,6 +80,7 @@ class Exchange:
         "l2_limit_range_required": True,  # Allow Empty L2 limit (kucoin)
         "mark_ohlcv_price": "mark",
         "mark_ohlcv_timeframe": "8h",
+        "funding_fee_timeframe": "8h",
         "ccxt_futures_name": "swap",
         "needs_trading_fees": False,  # use fetch_trading_fees to cache fees
         "order_props_in_contracts": ['amount', 'filled', 'remaining'],
@@ -121,11 +122,12 @@ class Exchange:
         # Cache for 10 minutes ...
         self._cache_lock = Lock()
         self._fetch_tickers_cache: TTLCache = TTLCache(maxsize=2, ttl=60 * 10)
-        # Cache values for 1800 to avoid frequent polling of the exchange for prices
+        # Cache values for 300 to avoid frequent polling of the exchange for prices
         # Caching only applies to RPC methods, so prices for open trades are still
         # refreshed once every iteration.
-        self._exit_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800)
+        # Shouldn't be too high either, as it'll freeze UI updates in case of open orders.
-        self._entry_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800)
+        self._exit_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=300)
+        self._entry_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=300)
 
         # Holds candles
         self._klines: Dict[PairWithTimeframe, DataFrame] = {}
@@ -319,10 +321,11 @@ class Exchange:
         """
         pass
 
-    def _log_exchange_response(self, endpoint, response) -> None:
+    def _log_exchange_response(self, endpoint: str, response, *, add_info=None) -> None:
         """ Log exchange responses """
         if self.log_responses:
-            logger.info(f"API {endpoint}: {response}")
+            add_info_str = "" if add_info is None else f" {add_info}: "
+            logger.info(f"API {endpoint}: {add_info_str}{response}")
 
     def ohlcv_candle_limit(
             self, timeframe: str, candle_type: CandleType, since_ms: Optional[int] = None) -> int:
@@ -1384,7 +1387,7 @@ class Exchange:
             order = self.fetch_stoploss_order(order_id, pair)
         except InvalidOrderException:
             logger.warning(f"Could not fetch cancelled stoploss order {order_id}.")
-            order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
+            order = {'id': order_id, 'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
 
         return order
 
@@ -2213,13 +2216,13 @@ class Exchange:
     @retrier_async
     async def _async_fetch_trades(self, pair: str,
                                   since: Optional[int] = None,
-                                  params: Optional[dict] = None) -> List[List]:
+                                  params: Optional[dict] = None) -> Tuple[List[List], Any]:
         """
         Asyncronously gets trade history using fetch_trades.
         Handles exchange errors, does one call to the exchange.
         :param pair: Pair to fetch trade data for
         :param since: Since as integer timestamp in milliseconds
-        returns: List of dicts containing trades
+        returns: List of dicts containing trades, the next iteration value (new "since" or trade_id)
         """
         try:
             # fetch trades asynchronously
@@ -2234,7 +2237,8 @@ class Exchange:
                 )
                 trades = await self._api_async.fetch_trades(pair, since=since, limit=1000)
             trades = self._trades_contracts_to_amount(trades)
-            return trades_dict_to_list(trades)
+            pagination_value = self._get_trade_pagination_next_value(trades)
+            return trades_dict_to_list(trades), pagination_value
         except ccxt.NotSupported as e:
             raise OperationalException(
                 f'Exchange {self._api.name} does not support fetching historical trade data.'
@@ -2247,6 +2251,25 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(f'Could not fetch trade data. Msg: {e}') from e
 
+    def _valid_trade_pagination_id(self, pair: str, from_id: str) -> bool:
+        """
+        Verify trade-pagination id is valid.
+        Workaround for odd Kraken issue where ID is sometimes wrong.
+        """
+        return True
+
+    def _get_trade_pagination_next_value(self, trades: List[Dict]):
+        """
+        Extract pagination id for the next "from_id" value
+        Applies only to fetch_trade_history by id.
+        """
+        if not trades:
+            return None
+        if self._trades_pagination == 'id':
+            return trades[-1].get('id')
+        else:
+            return trades[-1].get('timestamp')
+
     async def _async_get_trade_history_id(self, pair: str,
                                           until: int,
                                           since: Optional[int] = None,
@@ -2262,33 +2285,35 @@ class Exchange:
         """
 
         trades: List[List] = []
+        # DEFAULT_TRADES_COLUMNS: 0 -> timestamp
+        # DEFAULT_TRADES_COLUMNS: 1 -> id
+        has_overlap = self._ft_has.get('trades_pagination_overlap', True)
+        # Skip last trade by default since its the key for the next call
+        x = slice(None, -1) if has_overlap else slice(None)
 
-        if not from_id:
+        if not from_id or not self._valid_trade_pagination_id(pair, from_id):
             # Fetch first elements using timebased method to get an ID to paginate on
             # Depending on the Exchange, this can introduce a drift at the start of the interval
             # of up to an hour.
             # e.g. Binance returns the "last 1000" candles within a 1h time interval
             # - so we will miss the first trades.
-            t = await self._async_fetch_trades(pair, since=since)
+            t, from_id = await self._async_fetch_trades(pair, since=since)
-            # DEFAULT_TRADES_COLUMNS: 0 -> timestamp
+            trades.extend(t[x])
-            # DEFAULT_TRADES_COLUMNS: 1 -> id
-            from_id = t[-1][1]
-            trades.extend(t[:-1])
         while True:
             try:
-                t = await self._async_fetch_trades(pair,
+                t, from_id_next = await self._async_fetch_trades(
-                                                   params={self._trades_pagination_arg: from_id})
+                    pair, params={self._trades_pagination_arg: from_id})
                 if t:
-                    # Skip last id since its the key for the next call
+                    trades.extend(t[x])
-                    trades.extend(t[:-1])
+                    if from_id == from_id_next or t[-1][0] > until:
-                    if from_id == t[-1][1] or t[-1][0] > until:
                         logger.debug(f"Stopping because from_id did not change. "
                                      f"Reached {t[-1][0]} > {until}")
                         # Reached the end of the defined-download period - add last trade as well.
+                        if has_overlap:
                             trades.extend(t[-1:])
                         break
 
-                    from_id = t[-1][1]
+                    from_id = from_id_next
                 else:
                     logger.debug("Stopping as no more trades were returned.")
                     break
@@ -2314,19 +2339,19 @@ class Exchange:
         # DEFAULT_TRADES_COLUMNS: 1 -> id
         while True:
             try:
-                t = await self._async_fetch_trades(pair, since=since)
+                t, since_next = await self._async_fetch_trades(pair, since=since)
                 if t:
                     # No more trades to download available at the exchange,
                     # So we repeatedly get the same trade over and over again.
-                    if since == t[-1][0] and len(t) == 1:
+                    if since == since_next and len(t) == 1:
                         logger.debug("Stopping because no more trades are available.")
                         break
-                    since = t[-1][0]
+                    since = since_next
                     trades.extend(t)
                     # Reached the end of the defined-download period
-                    if until and t[-1][0] > until:
+                    if until and since_next > until:
                         logger.debug(
-                            f"Stopping because until was reached. {t[-1][0]} > {until}")
+                            f"Stopping because until was reached. {since_next} > {until}")
                         break
                 else:
                     logger.debug("Stopping as no more trades were returned.")
@@ -2414,6 +2439,8 @@ class Exchange:
                 symbol=pair,
                 since=since
             )
+            self._log_exchange_response('funding_history', funding_history,
+                                        add_info=f"pair: {pair}, since: {since}")
             return sum(fee['amount'] for fee in funding_history)
         except ccxt.DDoSProtection as e:
             raise DDosProtection(e) from e
@@ -2730,17 +2757,16 @@ class Exchange:
             # Only really relevant for trades very close to the full hour
             open_date = timeframe_to_prev_date('1h', open_date)
         timeframe = self._ft_has['mark_ohlcv_timeframe']
-        timeframe_ff = self._ft_has.get('funding_fee_timeframe',
+        timeframe_ff = self._ft_has['funding_fee_timeframe']
-                                        self._ft_has['mark_ohlcv_timeframe'])
+        mark_price_type = CandleType.from_string(self._ft_has["mark_ohlcv_price"])
 
         if not close_date:
             close_date = datetime.now(timezone.utc)
         since_ms = int(timeframe_to_prev_date(timeframe, open_date).timestamp()) * 1000
 
-        mark_comb: PairWithTimeframe = (
+        mark_comb: PairWithTimeframe = (pair, timeframe, mark_price_type)
-            pair, timeframe, CandleType.from_string(self._ft_has["mark_ohlcv_price"]))
-
         funding_comb: PairWithTimeframe = (pair, timeframe_ff, CandleType.FUNDING_RATE)
+
         candle_histories = self.refresh_latest_ohlcv(
             [mark_comb, funding_comb],
             since_ms=since_ms,

freqtrade/exchange/exchange_utils.py

@@ -118,6 +118,27 @@ def timeframe_to_msecs(timeframe: str) -> int:
     return ccxt.Exchange.parse_timeframe(timeframe) * 1000
 
 
+def timeframe_to_resample_freq(timeframe: str) -> str:
+    """
+    Translates the timeframe interval value written in the human readable
+    form ('1m', '5m', '1h', '1d', '1w', etc.) to the resample frequency
+    used by pandas ('1T', '5T', '1H', '1D', '1W', etc.)
+    """
+    if timeframe == '1y':
+        return '1YS'
+    timeframe_seconds = timeframe_to_seconds(timeframe)
+    timeframe_minutes = timeframe_seconds // 60
+    resample_interval = f'{timeframe_seconds}s'
+    if 10000 < timeframe_minutes < 43200:
+        resample_interval = '1W-MON'
+    elif timeframe_minutes >= 43200 and timeframe_minutes < 525600:
+        # Monthly candles need special treatment to stick to the 1st of the month
+        resample_interval = f'{timeframe}S'
+    elif timeframe_minutes > 43200:
+        resample_interval = timeframe
+    return resample_interval
+
+
 def timeframe_to_prev_date(timeframe: str, date: Optional[datetime] = None) -> datetime:
     """
     Use Timeframe and determine the candle start date for this date.

freqtrade/exchange/htx.py

@@ -1,4 +1,4 @@
-""" Huobi exchange subclass """
+""" HTX exchange subclass """
 import logging
 from typing import Dict
 
@@ -9,9 +9,9 @@ from freqtrade.exchange import Exchange
 logger = logging.getLogger(__name__)
 
 
-class Huobi(Exchange):
+class Htx(Exchange):
     """
-    Huobi exchange class. Contains adjustments needed for Freqtrade to work
+    HTX exchange class. Contains adjustments needed for Freqtrade to work
     with this exchange.
     """
 

freqtrade/exchange/kraken.py

@@ -8,11 +8,9 @@ from pandas import DataFrame
 
 from freqtrade.constants import BuySell
 from freqtrade.enums import MarginMode, TradingMode
-from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException,
+from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError
-                                  OperationalException, TemporaryError)
 from freqtrade.exchange import Exchange
 from freqtrade.exchange.common import retrier
-from freqtrade.exchange.exchange_utils import ROUND_DOWN, ROUND_UP
 from freqtrade.exchange.types import Tickers
 
 
@@ -24,12 +22,15 @@ class Kraken(Exchange):
     _params: Dict = {"trading_agreement": "agree"}
     _ft_has: Dict = {
         "stoploss_on_exchange": True,
-        "stop_price_param": "stopPrice",
+        "stop_price_param": "stopLossPrice",
-        "stop_price_prop": "stopPrice",
+        "stop_price_prop": "stopLossPrice",
+        "stoploss_order_types": {"limit": "limit", "market": "market"},
+        "order_time_in_force": ["GTC", "IOC", "PO"],
         "ohlcv_candle_limit": 720,
         "ohlcv_has_history": False,
         "trades_pagination": "id",
         "trades_pagination_arg": "since",
+        "trades_pagination_overlap": False,
         "mark_ohlcv_timeframe": "4h",
     }
 
@@ -89,75 +90,6 @@ class Kraken(Exchange):
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
-    def stoploss_adjust(self, stop_loss: float, order: Dict, side: str) -> bool:
-        """
-        Verify stop_loss against stoploss-order value (limit or price)
-        Returns True if adjustment is necessary.
-        """
-        return (order['type'] in ('stop-loss', 'stop-loss-limit') and (
-                (side == "sell" and stop_loss > float(order['price'])) or
-                (side == "buy" and stop_loss < float(order['price']))
-                ))
-
-    @retrier(retries=0)
-    def create_stoploss(self, pair: str, amount: float, stop_price: float,
-                        order_types: Dict, side: BuySell, leverage: float) -> Dict:
-        """
-        Creates a stoploss market order.
-        Stoploss market orders is the only stoploss type supported by kraken.
-        TODO: investigate if this can be combined with generic implementation
-              (careful, prices are reversed)
-        """
-        params = self._params.copy()
-        if self.trading_mode == TradingMode.FUTURES:
-            params.update({'reduceOnly': True})
-
-        round_mode = ROUND_DOWN if side == 'buy' else ROUND_UP
-        if order_types.get('stoploss', 'market') == 'limit':
-            ordertype = "stop-loss-limit"
-            limit_price_pct = order_types.get('stoploss_on_exchange_limit_ratio', 0.99)
-            if side == "sell":
-                limit_rate = stop_price * limit_price_pct
-            else:
-                limit_rate = stop_price * (2 - limit_price_pct)
-            params['price2'] = self.price_to_precision(pair, limit_rate, rounding_mode=round_mode)
-        else:
-            ordertype = "stop-loss"
-
-        stop_price = self.price_to_precision(pair, stop_price, rounding_mode=round_mode)
-
-        if self._config['dry_run']:
-            dry_order = self.create_dry_run_order(
-                pair, ordertype, side, amount, stop_price, leverage, stop_loss=True)
-            return dry_order
-
-        try:
-            amount = self.amount_to_precision(pair, amount)
-
-            order = self._api.create_order(symbol=pair, type=ordertype, side=side,
-                                           amount=amount, price=stop_price, params=params)
-            self._log_exchange_response('create_stoploss_order', order)
-            logger.info('stoploss order added for %s. '
-                        'stop price: %s.', pair, stop_price)
-            return order
-        except ccxt.InsufficientFunds as e:
-            raise InsufficientFundsError(
-                f'Insufficient funds to create {ordertype} {side} order on market {pair}. '
-                f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
-                f'Message: {e}') from e
-        except ccxt.InvalidOrder as e:
-            raise InvalidOrderException(
-                f'Could not create {ordertype} {side} order on market {pair}. '
-                f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
-                f'Message: {e}') from e
-        except ccxt.DDoSProtection as e:
-            raise DDosProtection(e) from e
-        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
-            raise TemporaryError(
-                f'Could not place {side} order due to {e.__class__.__name__}. Message: {e}') from e
-        except ccxt.BaseError as e:
-            raise OperationalException(e) from e
-
     def _set_leverage(
         self,
         leverage: float,
@@ -187,6 +119,9 @@ class Kraken(Exchange):
         )
         if leverage > 1.0:
             params['leverage'] = round(leverage)
+        if time_in_force == 'PO':
+            params.pop('timeInForce', None)
+            params['postOnly'] = True
         return params
 
     def calculate_funding_fees(
@@ -223,18 +158,30 @@ class Kraken(Exchange):
 
         return fees if is_short else -fees
 
-    def _trades_contracts_to_amount(self, trades: List) -> List:
+    def _get_trade_pagination_next_value(self, trades: List[Dict]):
         """
-        Fix "last" id issue for kraken data downloads
+        Extract pagination id for the next "from_id" value
-        This whole override can probably be removed once the following
+        Applies only to fetch_trade_history by id.
-        issue is closed in ccxt: https://github.com/ccxt/ccxt/issues/15827
         """
-        super()._trades_contracts_to_amount(trades)
+        if len(trades) > 0:
             if (
-            len(trades) > 0
+                isinstance(trades[-1].get('info'), list)
-            and isinstance(trades[-1].get('info'), list)
                 and len(trades[-1].get('info', [])) > 7
             ):
+                # Trade response's "last" value.
+                return trades[-1].get('info', [])[-1]
+            # Fall back to timestamp if info is somehow empty.
+            return trades[-1].get('timestamp')
+        return None
 
-            trades[-1]['id'] = trades[-1].get('info', [])[-1]
+    def _valid_trade_pagination_id(self, pair: str, from_id: str) -> bool:
-        return trades
+        """
+        Verify trade-pagination id is valid.
+        Workaround for odd Kraken issue where ID is sometimes wrong.
+        """
+        # Regular id's are in timestamp format 1705443695120072285
+        # If the id is smaller than 19 characters, it's not a valid timestamp.
+        if len(from_id) >= 19:
+            return True
+        logger.debug(f"{pair} - trade-pagination id is not valid. Fallback to timestamp.")
+        return False

freqtrade/exchange/okx.py

@@ -228,7 +228,7 @@ class Okx(Exchange):
                 f'StoplossOrder not found (pair: {pair} id: {order_id}).')
 
     def get_order_id_conditional(self, order: Dict[str, Any]) -> str:
-        if order['type'] == 'stop':
+        if order.get('type', '') == 'stop':
             return safe_value_fallback2(order, order, 'id_stop', 'id')
         return order['id']
 

freqtrade/freqai/base_models/FreqaiMultiOutputClassifier.py

@@ -1,9 +1,8 @@
 import numpy as np
-from joblib import Parallel
 from sklearn.base import is_classifier
 from sklearn.multioutput import MultiOutputClassifier, _fit_estimator
-from sklearn.utils.fixes import delayed
 from sklearn.utils.multiclass import check_classification_targets
+from sklearn.utils.parallel import Parallel, delayed
 from sklearn.utils.validation import has_fit_parameter
 
 from freqtrade.exceptions import OperationalException

freqtrade/freqai/base_models/FreqaiMultiOutputRegressor.py

@@ -1,6 +1,5 @@
-from joblib import Parallel
 from sklearn.multioutput import MultiOutputRegressor, _fit_estimator
-from sklearn.utils.fixes import delayed
+from sklearn.utils.parallel import Parallel, delayed
 from sklearn.utils.validation import has_fit_parameter
 
 

freqtrade/freqai/data_kitchen.py

@@ -432,8 +432,12 @@ class FreqaiDataKitchen:
         if self.freqai_config["feature_parameters"].get("DI_threshold", 0) > 0:
             append_df["DI_values"] = self.DI_values
 
+        user_cols = [col for col in dataframe_backtest.columns if col.startswith("%%")]
+        cols = ["date"]
+        cols.extend(user_cols)
+
         dataframe_backtest.reset_index(drop=True, inplace=True)
-        merged_df = pd.concat([dataframe_backtest["date"], append_df], axis=1)
+        merged_df = pd.concat([dataframe_backtest[cols], append_df], axis=1)
         return merged_df
 
     def append_predictions(self, append_df: DataFrame) -> None:
@@ -451,7 +455,8 @@ class FreqaiDataKitchen:
         Back fill values to before the backtesting range so that the dataframe matches size
         when it goes back to the strategy. These rows are not included in the backtest.
         """
-        to_keep = [col for col in dataframe.columns if not col.startswith("&")]
+        to_keep = [col for col in dataframe.columns if
+                   not col.startswith("&") and not col.startswith("%%")]
         self.return_dataframe = pd.merge(dataframe[to_keep],
                                          self.full_df, how='left', on='date')
         self.return_dataframe[self.full_df.columns] = (
@@ -709,6 +714,8 @@ class FreqaiDataKitchen:
                 pair, tf, strategy, corr_dataframes, base_dataframes, is_corr_pairs)
             informative_copy = informative_df.copy()
 
+            logger.debug(f"Populating features for {pair} {tf}")
+
             for t in self.freqai_config["feature_parameters"]["indicator_periods_candles"]:
                 df_features = strategy.feature_engineering_expand_all(
                     informative_copy.copy(), t, metadata=metadata)
@@ -788,6 +795,7 @@ class FreqaiDataKitchen:
 
         if not prediction_dataframe.empty:
             dataframe = prediction_dataframe.copy()
+            base_dataframes[self.config["timeframe"]] = dataframe.copy()
         else:
             dataframe = base_dataframes[self.config["timeframe"]].copy()
 

freqtrade/freqai/utils.py

@@ -13,7 +13,6 @@ from freqtrade.data.dataprovider import DataProvider
 from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_seconds
-from freqtrade.exchange.exchange import market_is_active
 from freqtrade.freqai.data_drawer import FreqaiDataDrawer
 from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
@@ -33,8 +32,11 @@ def download_all_data_for_training(dp: DataProvider, config: Config) -> None:
 
     if dp._exchange is None:
         raise OperationalException('No exchange object found.')
-    markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m)
+    markets = [
-               or config.get('include_inactive')]
+        p for p in dp._exchange.get_markets(
+            tradable_only=True, active_only=not config.get('include_inactive')
+            ).keys()
+    ]
 
     all_pairs = dynamic_expand_pairlist(config, markets)
 

freqtrade/freqtradebot.py

@@ -18,8 +18,8 @@ from freqtrade.constants import BuySell, Config, EntryExecuteMode, ExchangeConfi
 from freqtrade.data.converter import order_book_to_dataframe
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.edge import Edge
-from freqtrade.enums import (ExitCheckTuple, ExitType, RPCMessageType, RunMode, SignalDirection,
+from freqtrade.enums import (ExitCheckTuple, ExitType, RPCMessageType, SignalDirection, State,
-                             State, TradingMode)
+                             TradingMode)
 from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError,
                                   InvalidOrderException, PricingError)
 from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, remove_exchange_credentials,
@@ -38,7 +38,7 @@ from freqtrade.rpc.rpc_types import (ProfitLossStr, RPCCancelMsg, RPCEntryMsg, R
 from freqtrade.strategy.interface import IStrategy
 from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
 from freqtrade.util import FtPrecise
-from freqtrade.util.binance_mig import migrate_binance_futures_names
+from freqtrade.util.migrations import migrate_binance_futures_names
 from freqtrade.wallets import Wallets
 
 
@@ -83,6 +83,8 @@ class FreqtradeBot(LoggingMixin):
         PairLocks.timeframe = self.config['timeframe']
 
         self.pairlists = PairListManager(self.exchange, self.config)
+        self.trading_mode: TradingMode = self.config.get('trading_mode', TradingMode.SPOT)
+        self.last_process: Optional[datetime] = None
 
         # RPC runs in separate threads, can start handling external commands just after
         # initialization, even before Freqtradebot has a chance to start its throttling,
@@ -119,8 +121,6 @@ class FreqtradeBot(LoggingMixin):
         self._exit_lock = Lock()
         LoggingMixin.__init__(self, logger, timeframe_to_seconds(self.strategy.timeframe))
 
-        self.trading_mode: TradingMode = self.config.get('trading_mode', TradingMode.SPOT)
-
         self._schedule = Scheduler()
 
         if self.trading_mode == TradingMode.FUTURES:
@@ -135,7 +135,6 @@ class FreqtradeBot(LoggingMixin):
                 for minutes in [1, 31]:
                     t = str(time(time_slot, minutes, 2))
                     self._schedule.every().day.at(t).do(update)
-        self.last_process: Optional[datetime] = None
 
         self.strategy.ft_bot_start()
         # Initialize protections AFTER bot start - otherwise parameters are not loaded.
@@ -673,20 +672,13 @@ class FreqtradeBot(LoggingMixin):
             amount = self.exchange.amount_to_contract_precision(
                 trade.pair,
                 abs(float(FtPrecise(stake_amount * trade.leverage) / FtPrecise(current_exit_rate))))
-            if amount > trade.amount:
-                # This is currently ineffective as remaining would become < min tradable
-                # Fixing this would require checking for 0.0 there -
-                # if we decide that this callback is allowed to "fully exit"
-                logger.info(
-                    f"Adjusting amount to trade.amount as it is higher. {amount} > {trade.amount}")
-                amount = trade.amount
 
             if amount == 0.0:
                 logger.info("Amount to exit is 0.0 due to exchange limits - not exiting.")
                 return
 
             remaining = (trade.amount - amount) * current_exit_rate
-            if min_exit_stake and remaining < min_exit_stake:
+            if min_exit_stake and remaining != 0 and remaining < min_exit_stake:
                 logger.info(f"Remaining amount of {remaining} would be smaller "
                             f"than the minimum of {min_exit_stake}.")
                 return
@@ -1010,8 +1002,6 @@ class FreqtradeBot(LoggingMixin):
         if open_rate is None:
             open_rate = trade.open_rate
 
-        current_rate = trade.open_rate_requested
-        if self.dataprovider.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
         current_rate = self.exchange.get_rate(
             trade.pair, side='entry', is_short=trade.is_short, refresh=False)
 
@@ -1030,6 +1020,7 @@ class FreqtradeBot(LoggingMixin):
             'stake_amount': trade.stake_amount,
             'stake_currency': self.config['stake_currency'],
             'base_currency': self.exchange.get_pair_base_currency(trade.pair),
+            'quote_currency': self.exchange.get_pair_quote_currency(trade.pair),
             'fiat_currency': self.config.get('fiat_display_currency', None),
             'amount': order.safe_amount_after_fee if fill else (order.amount or trade.amount),
             'open_date': trade.open_date_utc or datetime.now(timezone.utc),
@@ -1063,6 +1054,7 @@ class FreqtradeBot(LoggingMixin):
             'open_rate': trade.open_rate,
             'stake_currency': self.config['stake_currency'],
             'base_currency': self.exchange.get_pair_base_currency(trade.pair),
+            'quote_currency': self.exchange.get_pair_quote_currency(trade.pair),
             'fiat_currency': self.config.get('fiat_display_currency', None),
             'amount': trade.amount,
             'open_date': trade.open_date,
@@ -1348,10 +1340,12 @@ class FreqtradeBot(LoggingMixin):
                 not_closed = order['status'] == 'open' or fully_cancelled
 
                 if not_closed:
-                    if fully_cancelled or (
+                    if (
+                        fully_cancelled or (
                             open_order and self.strategy.ft_check_timed_out(
                                 trade, open_order, datetime.now(timezone.utc)
                             )
+                        )
                     ):
                         self.handle_cancel_order(
                             order, open_order, trade, constants.CANCEL_REASON['TIMEOUT']
@@ -1430,11 +1424,11 @@ class FreqtradeBot(LoggingMixin):
             # New candle
             proposed_rate = self.exchange.get_rate(
                 trade.pair, side='entry', is_short=trade.is_short, refresh=True)
-            adjusted_entry_price = strategy_safe_wrapper(self.strategy.adjust_entry_price,
+            adjusted_entry_price = strategy_safe_wrapper(
-                                                         default_retval=order_obj.price)(
+                self.strategy.adjust_entry_price, default_retval=order_obj.safe_placement_price)(
                 trade=trade, order=order_obj, pair=trade.pair,
                 current_time=datetime.now(timezone.utc), proposed_rate=proposed_rate,
-                current_order_rate=order_obj.safe_price, entry_tag=trade.enter_tag,
+                current_order_rate=order_obj.safe_placement_price, entry_tag=trade.enter_tag,
                 side=trade.trade_direction)
 
             replacing = True
@@ -1442,7 +1436,7 @@ class FreqtradeBot(LoggingMixin):
             if not adjusted_entry_price:
                 replacing = False
                 cancel_reason = constants.CANCEL_REASON['USER_CANCEL']
-            if order_obj.price != adjusted_entry_price:
+            if order_obj.safe_placement_price != adjusted_entry_price:
                 # cancel existing order if new price is supplied or None
                 res = self.handle_cancel_enter(trade, order, order_obj, cancel_reason,
                                                replacing=replacing)
@@ -1810,20 +1804,22 @@ class FreqtradeBot(LoggingMixin):
             'open_rate': trade.open_rate,
             'close_rate': order_rate,
             'current_rate': current_rate,
-            'profit_amount': profit.profit_abs if fill else profit.total_profit,
+            'profit_amount': profit.profit_abs,
             'profit_ratio': profit.profit_ratio,
             'buy_tag': trade.enter_tag,
             'enter_tag': trade.enter_tag,
-            'sell_reason': trade.exit_reason,  # Deprecated
             'exit_reason': trade.exit_reason,
             'open_date': trade.open_date_utc,
             'close_date': trade.close_date_utc or datetime.now(timezone.utc),
             'stake_amount': trade.stake_amount,
             'stake_currency': self.config['stake_currency'],
             'base_currency': self.exchange.get_pair_base_currency(trade.pair),
+            'quote_currency': self.exchange.get_pair_quote_currency(trade.pair),
             'fiat_currency': self.config.get('fiat_display_currency'),
             'sub_trade': sub_trade,
             'cumulative_profit': trade.realized_profit,
+            'final_profit_ratio': trade.close_profit if not trade.is_open else None,
+            'is_final_exit': trade.is_open is False,
         }
 
         # Send the message
@@ -1865,12 +1861,12 @@ class FreqtradeBot(LoggingMixin):
             'profit_ratio': profit.profit_ratio,
             'buy_tag': trade.enter_tag,
             'enter_tag': trade.enter_tag,
-            'sell_reason': trade.exit_reason,  # Deprecated
             'exit_reason': trade.exit_reason,
             'open_date': trade.open_date,
             'close_date': trade.close_date or datetime.now(timezone.utc),
             'stake_currency': self.config['stake_currency'],
             'base_currency': self.exchange.get_pair_base_currency(trade.pair),
+            'quote_currency': self.exchange.get_pair_quote_currency(trade.pair),
             'fiat_currency': self.config.get('fiat_display_currency', None),
             'reason': reason,
             'sub_trade': sub_trade,
@@ -1978,15 +1974,16 @@ class FreqtradeBot(LoggingMixin):
             self, trade: Trade, order: Order, stoploss_order: bool, send_msg: bool):
         """send "fill" notifications"""
 
-        sub_trade = not isclose(order.safe_amount_after_fee,
-                                trade.amount, abs_tol=constants.MATH_CLOSE_PREC)
         if order.ft_order_side == trade.exit_side:
             # Exit notification
             if send_msg and not stoploss_order and order.order_id not in trade.open_orders_ids:
-                self._notify_exit(trade, '', fill=True, sub_trade=sub_trade, order=order)
+                self._notify_exit(trade, order.order_type, fill=True,
+                                  sub_trade=trade.is_open, order=order)
             if not trade.is_open:
                 self.handle_protections(trade.pair, trade.trade_direction)
         elif send_msg and order.order_id not in trade.open_orders_ids and not stoploss_order:
+            sub_trade = not isclose(order.safe_amount_after_fee,
+                                    trade.amount, abs_tol=constants.MATH_CLOSE_PREC)
             # Enter fill
             self._notify_enter(trade, order, order.order_type, fill=True, sub_trade=sub_trade)
 

freqtrade/misc.py

@@ -11,41 +11,12 @@ from urllib.parse import urlparse
 import pandas as pd
 import rapidjson
 
-from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN
 from freqtrade.enums import SignalTagType, SignalType
 
 
 logger = logging.getLogger(__name__)
 
 
-def decimals_per_coin(coin: str):
-    """
-    Helper method getting decimal amount for this coin
-    example usage: f".{decimals_per_coin('USD')}f"
-    :param coin: Which coin are we printing the price / value for
-    """
-    return DECIMALS_PER_COIN.get(coin, DECIMAL_PER_COIN_FALLBACK)
-
-
-def round_coin_value(
-        value: float, coin: str, show_coin_name=True, keep_trailing_zeros=False) -> str:
-    """
-    Get price value for this coin
-    :param value: Value to be printed
-    :param coin: Which coin are we printing the price / value for
-    :param show_coin_name: Return string in format: "222.22 USDT" or "222.22"
-    :param keep_trailing_zeros: Keep trailing zeros "222.200" vs. "222.2"
-    :return: Formatted / rounded value (with or without coin name)
-    """
-    val = f"{value:.{decimals_per_coin(coin)}f}"
-    if not keep_trailing_zeros:
-        val = val.rstrip('0').rstrip('.')
-    if show_coin_name:
-        val = f"{val} {coin}"
-
-    return val
-
-
 def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool = True) -> None:
     """
     Dump JSON data into a file

freqtrade/optimize/backtesting.py

@@ -33,14 +33,15 @@ from freqtrade.optimize.optimize_reports import (generate_backtest_stats, genera
                                                  show_backtest_results,
                                                  store_backtest_analysis_results,
                                                  store_backtest_stats)
-from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade
+from freqtrade.persistence import (LocalTrade, Order, PairLocks, Trade, disable_database_use,
+                                   enable_database_use)
 from freqtrade.plugins.pairlistmanager import PairListManager
 from freqtrade.plugins.protectionmanager import ProtectionManager
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 from freqtrade.strategy.interface import IStrategy
 from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
 from freqtrade.types import BacktestResultType, get_BacktestResultType_default
-from freqtrade.util.binance_mig import migrate_binance_futures_data
+from freqtrade.util.migrations import migrate_data
 from freqtrade.wallets import Wallets
 
 
@@ -116,8 +117,9 @@ class Backtesting:
             raise OperationalException("Timeframe needs to be set in either "
                                        "configuration or as cli argument `--timeframe 5m`")
         self.timeframe = str(self.config.get('timeframe'))
-        self.disable_database_use()
         self.timeframe_min = timeframe_to_minutes(self.timeframe)
+        self.timeframe_td = timedelta(minutes=self.timeframe_min)
+        self.disable_database_use()
         self.init_backtest_detail()
         self.pairlists = PairListManager(self.exchange, self.config, self.dataprovider)
         self._validate_pairlists_for_backtesting()
@@ -145,19 +147,20 @@ class Backtesting:
         self.required_startup = max([strat.startup_candle_count for strat in self.strategylist])
         self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe)
 
-        if self.config.get('freqai', {}).get('enabled', False):
-            # For FreqAI, increase the required_startup to includes the training data
-            self.required_startup = self.dataprovider.get_required_startup(self.timeframe)
-
         # Add maximum startup candle count to configuration for informative pairs support
         self.config['startup_candle_count'] = self.required_startup
 
+        if self.config.get('freqai', {}).get('enabled', False):
+            # For FreqAI, increase the required_startup to includes the training data
+            # This value should NOT be written to startup_candle_count
+            self.required_startup = self.dataprovider.get_required_startup(self.timeframe)
+
         self.trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT)
         # strategies which define "can_short=True" will fail to load in Spot mode.
         self._can_short = self.trading_mode != TradingMode.SPOT
         self._position_stacking: bool = self.config.get('position_stacking', False)
         self.enable_protections: bool = self.config.get('enable_protections', False)
-        migrate_binance_futures_data(config)
+        migrate_data(config, self.exchange)
 
         self.init_backtest()
 
@@ -176,8 +179,7 @@ class Backtesting:
     @staticmethod
     def cleanup():
         LoggingMixin.show_output = True
-        PairLocks.use_db = True
+        enable_database_use()
-        Trade.use_db = True
 
     def init_backtest_detail(self) -> None:
         # Load detail timeframe if specified
@@ -239,7 +241,7 @@ class Backtesting:
             pairs=self.pairlists.whitelist,
             timeframe=self.timeframe,
             timerange=self.timerange,
-            startup_candles=self.config['startup_candle_count'],
+            startup_candles=self.required_startup,
             fail_without_data=True,
             data_format=self.config['dataformat_ohlcv'],
             candle_type=self.config.get('candle_type_def', CandleType.SPOT)
@@ -276,8 +278,10 @@ class Backtesting:
         else:
             self.detail_data = {}
         if self.trading_mode == TradingMode.FUTURES:
-            self.funding_fee_timeframe: str = self.exchange.get_option('mark_ohlcv_timeframe')
+            self.funding_fee_timeframe: str = self.exchange.get_option('funding_fee_timeframe')
             self.funding_fee_timeframe_secs: int = timeframe_to_seconds(self.funding_fee_timeframe)
+            mark_timeframe: str = self.exchange.get_option('mark_ohlcv_timeframe')
+
             # Load additional futures data.
             funding_rates_dict = history.load_data(
                 datadir=self.config['datadir'],
@@ -294,7 +298,7 @@ class Backtesting:
             mark_rates_dict = history.load_data(
                 datadir=self.config['datadir'],
                 pairs=self.pairlists.whitelist,
-                timeframe=self.funding_fee_timeframe,
+                timeframe=mark_timeframe,
                 timerange=self.timerange,
                 startup_candles=0,
                 fail_without_data=True,
@@ -322,9 +326,7 @@ class Backtesting:
             self.futures_data = {}
 
     def disable_database_use(self):
-        PairLocks.use_db = False
+        disable_database_use(self.timeframe)
-        PairLocks.timeframe = self.timeframe
-        Trade.use_db = False
 
     def prepare_backtest(self, enable_protections):
         """
@@ -530,7 +532,7 @@ class Backtesting:
     def _get_adjust_trade_entry_for_candle(
             self, trade: LocalTrade, row: Tuple, current_time: datetime
     ) -> LocalTrade:
-        current_rate = row[OPEN_IDX]
+        current_rate: float = row[OPEN_IDX]
         current_profit = trade.calc_profit_ratio(current_rate)
         min_stake = self.exchange.get_min_pair_stake_amount(trade.pair, current_rate, -0.1)
         max_stake = self.exchange.get_max_pair_stake_amount(trade.pair, current_rate)
@@ -563,11 +565,8 @@ class Backtesting:
                 self.precision_mode, trade.contract_size)
             if amount == 0.0:
                 return trade
-            if amount > trade.amount:
-                # This is currently ineffective as remaining would become < min tradable
-                amount = trade.amount
             remaining = (trade.amount - amount) * current_rate
-            if remaining < min_stake:
+            if min_stake and remaining != 0 and remaining < min_stake:
                 # Remaining stake is too low to be sold.
                 return trade
             exit_ = ExitCheckTuple(ExitType.PARTIAL_EXIT)
@@ -1207,10 +1206,10 @@ class Backtesting:
 
         # Indexes per pair, so some pairs are allowed to have a missing start.
         indexes: Dict = defaultdict(int)
-        current_time = start_date + timedelta(minutes=self.timeframe_min)
+        current_time = start_date + self.timeframe_td
 
         self.progress.init_step(BacktestState.BACKTEST, int(
-            (end_date - start_date) / timedelta(minutes=self.timeframe_min)))
+            (end_date - start_date) / self.timeframe_td))
         # Loop timerange and get candle for each pair at that point in time
         while current_time <= end_date:
             open_trade_count_start = LocalTrade.bt_open_open_trade_count
@@ -1237,7 +1236,7 @@ class Backtesting:
                     # Spread out into detail timeframe.
                     # Should only happen when we are either in a trade for this pair
                     # or when we got the signal for a new trade.
-                    exit_candle_end = current_detail_time + timedelta(minutes=self.timeframe_min)
+                    exit_candle_end = current_detail_time + self.timeframe_td
 
                     detail_data = self.detail_data[pair]
                     detail_data = detail_data.loc[
@@ -1273,7 +1272,7 @@ class Backtesting:
 
             # Move time one configured time_interval ahead.
             self.progress.increment()
-            current_time += timedelta(minutes=self.timeframe_min)
+            current_time += self.timeframe_td
 
         self.handle_left_open(LocalTrade.bt_trades_open_pp, data=data)
         self.wallets.update()

freqtrade/optimize/base_analysis.py

@@ -54,7 +54,7 @@ class BaseAnalysis:
             self.full_varHolder.from_dt = parsed_timerange.startdt
 
         if parsed_timerange.stopdt is None:
-            self.full_varHolder.to_dt = datetime.utcnow()
+            self.full_varHolder.to_dt = datetime.now(timezone.utc)
         else:
             self.full_varHolder.to_dt = parsed_timerange.stopdt
 

freqtrade/optimize/hyperopt_tools.py

@@ -14,9 +14,10 @@ from pandas import isna, json_normalize
 from freqtrade.constants import FTHYPT_FILEVERSION, Config
 from freqtrade.enums import HyperoptState
 from freqtrade.exceptions import OperationalException
-from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict, safe_value_fallback2
+from freqtrade.misc import deep_merge_dicts, round_dict, safe_value_fallback2
 from freqtrade.optimize.hyperopt_epoch_filters import hyperopt_filter_epochs
 from freqtrade.optimize.optimize_reports import generate_wins_draws_losses
+from freqtrade.util import fmt_coin
 
 
 logger = logging.getLogger(__name__)
@@ -405,7 +406,7 @@ class HyperoptTools:
 
         trials[f"Max Drawdown{' (Acct)' if has_account_drawdown else ''}"] = trials.apply(
             lambda x: "{} {}".format(
-                round_coin_value(x['max_drawdown_abs'], stake_currency, keep_trailing_zeros=True),
+                fmt_coin(x['max_drawdown_abs'], stake_currency, keep_trailing_zeros=True),
                 (f"({x['max_drawdown_account']:,.2%})"
                     if has_account_drawdown
                     else f"({x['max_drawdown']:,.2%})"
@@ -420,7 +421,7 @@ class HyperoptTools:
 
         trials['Profit'] = trials.apply(
             lambda x: '{} {}'.format(
-                round_coin_value(x['Total profit'], stake_currency, keep_trailing_zeros=True),
+                fmt_coin(x['Total profit'], stake_currency, keep_trailing_zeros=True),
                 f"({x['Profit']:,.2%})".rjust(10, ' ')
             ).rjust(25 + len(stake_currency))
             if x['Total profit'] != 0.0 else '--'.rjust(25 + len(stake_currency)),

freqtrade/optimize/optimize_reports/bt_output.py

@@ -4,9 +4,9 @@ from typing import Any, Dict, List
 from tabulate import tabulate
 
 from freqtrade.constants import UNLIMITED_STAKE_AMOUNT, Config
-from freqtrade.misc import decimals_per_coin, round_coin_value
 from freqtrade.optimize.optimize_reports.optimize_reports import generate_periodic_breakdown_stats
 from freqtrade.types import BacktestResultType
+from freqtrade.util import decimals_per_coin, fmt_coin
 
 
 logger = logging.getLogger(__name__)
@@ -63,7 +63,7 @@ def text_table_bt_results(pair_results: List[Dict[str, Any]], stake_currency: st
 def text_table_exit_reason(exit_reason_stats: List[Dict[str, Any]], stake_currency: str) -> str:
     """
     Generate small table outlining Backtest results
-    :param sell_reason_stats: Exit reason metrics
+    :param exit_reason_stats: Exit reason metrics
     :param stake_currency: Stakecurrency used
     :return: pretty printed table with tabulate as string
     """
@@ -81,7 +81,7 @@ def text_table_exit_reason(exit_reason_stats: List[Dict[str, Any]], stake_curren
         t.get('exit_reason', t.get('sell_reason')), t['trades'],
         generate_wins_draws_losses(t['wins'], t['draws'], t['losses']),
         t['profit_mean_pct'], t['profit_sum_pct'],
-        round_coin_value(t['profit_total_abs'], stake_currency, False),
+        fmt_coin(t['profit_total_abs'], stake_currency, False),
         t['profit_total_pct'],
     ] for t in exit_reason_stats]
     return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right")
@@ -134,7 +134,7 @@ def text_table_periodic_breakdown(days_breakdown_stats: List[Dict[str, Any]],
         'Losses',
     ]
     output = [[
-        d['date'], round_coin_value(d['profit_abs'], stake_currency, False),
+        d['date'], fmt_coin(d['profit_abs'], stake_currency, False),
         d['wins'], d['draws'], d['loses'],
     ] for d in days_breakdown_stats]
     return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right")
@@ -187,9 +187,9 @@ def text_table_add_metrics(strat_results: Dict) -> str:
              f"{strat_results.get('trade_count_short', 0)}"),
             ('Total profit Long %', f"{strat_results['profit_total_long']:.2%}"),
             ('Total profit Short %', f"{strat_results['profit_total_short']:.2%}"),
-            ('Absolute profit Long', round_coin_value(strat_results['profit_total_long_abs'],
+            ('Absolute profit Long', fmt_coin(strat_results['profit_total_long_abs'],
                                               strat_results['stake_currency'])),
-            ('Absolute profit Short', round_coin_value(strat_results['profit_total_short_abs'],
+            ('Absolute profit Short', fmt_coin(strat_results['profit_total_short_abs'],
                                                strat_results['stake_currency'])),
         ] if strat_results.get('trade_count_short', 0) > 0 else []
 
@@ -203,11 +203,11 @@ def text_table_add_metrics(strat_results: Dict) -> str:
             ('Absolute Drawdown (Account)', f"{strat_results['max_drawdown_account']:.2%}")
             if 'max_drawdown_account' in strat_results else (
                 'Drawdown', f"{strat_results['max_drawdown']:.2%}"),
-            ('Absolute Drawdown', round_coin_value(strat_results['max_drawdown_abs'],
+            ('Absolute Drawdown', fmt_coin(strat_results['max_drawdown_abs'],
                                            strat_results['stake_currency'])),
-            ('Drawdown high', round_coin_value(strat_results['max_drawdown_high'],
+            ('Drawdown high', fmt_coin(strat_results['max_drawdown_high'],
                                        strat_results['stake_currency'])),
-            ('Drawdown low', round_coin_value(strat_results['max_drawdown_low'],
+            ('Drawdown low', fmt_coin(strat_results['max_drawdown_low'],
                                       strat_results['stake_currency'])),
             ('Drawdown Start', strat_results['drawdown_start']),
             ('Drawdown End', strat_results['drawdown_end']),
@@ -230,11 +230,11 @@ def text_table_add_metrics(strat_results: Dict) -> str:
             ('Total/Daily Avg Trades',
                 f"{strat_results['total_trades']} / {strat_results['trades_per_day']}"),
 
-            ('Starting balance', round_coin_value(strat_results['starting_balance'],
+            ('Starting balance', fmt_coin(strat_results['starting_balance'],
                                           strat_results['stake_currency'])),
-            ('Final balance', round_coin_value(strat_results['final_balance'],
+            ('Final balance', fmt_coin(strat_results['final_balance'],
                                        strat_results['stake_currency'])),
-            ('Absolute profit ', round_coin_value(strat_results['profit_total_abs'],
+            ('Absolute profit ', fmt_coin(strat_results['profit_total_abs'],
                                           strat_results['stake_currency'])),
             ('Total profit %', f"{strat_results['profit_total']:.2%}"),
             ('CAGR %', f"{strat_results['cagr']:.2%}" if 'cagr' in strat_results else 'N/A'),
@@ -249,9 +249,9 @@ def text_table_add_metrics(strat_results: Dict) -> str:
             ('Trades per day', strat_results['trades_per_day']),
             ('Avg. daily profit %',
              f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"),
-            ('Avg. stake amount', round_coin_value(strat_results['avg_stake_amount'],
+            ('Avg. stake amount', fmt_coin(strat_results['avg_stake_amount'],
                                            strat_results['stake_currency'])),
-            ('Total trade volume', round_coin_value(strat_results['total_volume'],
+            ('Total trade volume', fmt_coin(strat_results['total_volume'],
                                             strat_results['stake_currency'])),
             *short_metrics,
             ('', ''),  # Empty line to improve readability
@@ -263,9 +263,9 @@ def text_table_add_metrics(strat_results: Dict) -> str:
             ('Worst trade', f"{worst_trade['pair']} "
                             f"{worst_trade['profit_ratio']:.2%}"),
 
-            ('Best day', round_coin_value(strat_results['backtest_best_day_abs'],
+            ('Best day', fmt_coin(strat_results['backtest_best_day_abs'],
                                   strat_results['stake_currency'])),
-            ('Worst day', round_coin_value(strat_results['backtest_worst_day_abs'],
+            ('Worst day', fmt_coin(strat_results['backtest_worst_day_abs'],
                                    strat_results['stake_currency'])),
             ('Days win/draw/lose', f"{strat_results['winning_days']} / "
                 f"{strat_results['draw_days']} / {strat_results['losing_days']}"),
@@ -281,10 +281,8 @@ def text_table_add_metrics(strat_results: Dict) -> str:
             *entry_adjustment_metrics,
             ('', ''),  # Empty line to improve readability
 
-            ('Min balance', round_coin_value(strat_results['csum_min'],
+            ('Min balance', fmt_coin(strat_results['csum_min'], strat_results['stake_currency'])),
-                                             strat_results['stake_currency'])),
+            ('Max balance', fmt_coin(strat_results['csum_max'], strat_results['stake_currency'])),
-            ('Max balance', round_coin_value(strat_results['csum_max'],
-                                             strat_results['stake_currency'])),
 
             *drawdown_metrics,
             ('Market change', f"{strat_results['market_change']:.2%}"),
@@ -292,9 +290,8 @@ def text_table_add_metrics(strat_results: Dict) -> str:
 
         return tabulate(metrics, headers=["Metric", "Value"], tablefmt="orgtbl")
     else:
-        start_balance = round_coin_value(strat_results['starting_balance'],
+        start_balance = fmt_coin(strat_results['starting_balance'], strat_results['stake_currency'])
-                                         strat_results['stake_currency'])
+        stake_amount = fmt_coin(
-        stake_amount = round_coin_value(
             strat_results['stake_amount'], strat_results['stake_currency']
         ) if strat_results['stake_amount'] != UNLIMITED_STAKE_AMOUNT else 'unlimited'
 
@@ -322,19 +319,15 @@ def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency:
         print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
     print(table)
 
-    if (results.get('results_per_enter_tag') is not None
+    if (results.get('results_per_enter_tag') is not None):
-            or results.get('results_per_buy_tag') is not None):
+        table = text_table_tags("enter_tag", results['results_per_enter_tag'], stake_currency)
-        # results_per_buy_tag is deprecated and should be removed 2 versions after short golive.
-        table = text_table_tags(
-            "enter_tag",
-            results.get('results_per_enter_tag', results.get('results_per_buy_tag')),
-            stake_currency=stake_currency)
 
         if isinstance(table, str) and len(table) > 0:
             print(' ENTER TAG STATS '.center(len(table.splitlines()[0]), '='))
         print(table)
 
-    exit_reasons = results.get('exit_reason_summary', results.get('sell_reason_summary'))
+    exit_reasons = results.get('exit_reason_summary')
+    if exit_reasons:
         table = text_table_exit_reason(exit_reason_stats=exit_reasons,
                                        stake_currency=stake_currency)
         if isinstance(table, str) and len(table) > 0:

freqtrade/optimize/optimize_reports/optimize_reports.py

@@ -10,8 +10,8 @@ from freqtrade.constants import BACKTEST_BREAKDOWNS, DATETIME_PRINT_FORMAT, IntO
 from freqtrade.data.metrics import (calculate_cagr, calculate_calmar, calculate_csum,
                                     calculate_expectancy, calculate_market_change,
                                     calculate_max_drawdown, calculate_sharpe, calculate_sortino)
-from freqtrade.misc import decimals_per_coin, round_coin_value
 from freqtrade.types import BacktestResultType
+from freqtrade.util import decimals_per_coin, fmt_coin
 
 
 logger = logging.getLogger(__name__)
@@ -203,7 +203,7 @@ def generate_strategy_comparison(bt_stats: Dict) -> List[Dict]:
         # Update "key" to strategy (results_per_pair has it as "Total").
         tabular_data[-1]['key'] = strategy
         tabular_data[-1]['max_drawdown_account'] = result['max_drawdown_account']
-        tabular_data[-1]['max_drawdown_abs'] = round_coin_value(
+        tabular_data[-1]['max_drawdown_abs'] = fmt_coin(
             result['max_drawdown_abs'], result['stake_currency'], False)
     return tabular_data
 
@@ -561,6 +561,10 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame],
         metadata[strategy] = {
             'run_id': content['run_id'],
             'backtest_start_time': content['backtest_start_time'],
+            'timeframe': content['config']['timeframe'],
+            'timeframe_detail': content['config'].get('timeframe_detail', None),
+            'backtest_start_ts': int(min_date.timestamp()),
+            'backtest_end_ts': int(max_date.timestamp()),
         }
         result['strategy'][strategy] = strat_stats
 

freqtrade/persistence/__init__.py

@@ -4,3 +4,5 @@ from freqtrade.persistence.key_value_store import KeyStoreKeys, KeyValueStore
 from freqtrade.persistence.models import init_db
 from freqtrade.persistence.pairlock_middleware import PairLocks
 from freqtrade.persistence.trade_model import LocalTrade, Order, Trade
+from freqtrade.persistence.usedb_context import (FtNoDBContext, disable_database_use,
+                                                 enable_database_use)

freqtrade/persistence/trade_model.py

@@ -106,6 +106,11 @@ class Order(ModelBase):
     def safe_amount(self) -> float:
         return self.amount or self.ft_amount
 
+    @property
+    def safe_placement_price(self) -> float:
+        """Price at which the order was placed"""
+        return self.price or self.stop_price or self.ft_price
+
     @property
     def safe_price(self) -> float:
         return self.average or self.price or self.stop_price or self.ft_price
@@ -146,7 +151,7 @@ class Order(ModelBase):
 
         return (f"Order(id={self.id}, trade={self.ft_trade_id}, order_id={self.order_id}, "
                 f"side={self.side}, filled={self.safe_filled}, price={self.safe_price}, "
-                f"status={self.status}, date={self.order_date:{DATETIME_PRINT_FORMAT}})")
+                f"status={self.status}, date={self.order_date_utc:{DATETIME_PRINT_FORMAT}})")
 
     def update_from_ccxt_object(self, order):
         """
@@ -542,7 +547,9 @@ class LocalTrade:
                 f"{self.trading_mode.value} trading requires param interest_rate on trades")
 
     def __repr__(self):
-        open_since = self.open_date.strftime(DATETIME_PRINT_FORMAT) if self.is_open else 'closed'
+        open_since = (
+            self.open_date_utc.strftime(DATETIME_PRINT_FORMAT) if self.is_open else 'closed'
+        )
 
         return (
             f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, '
@@ -1603,7 +1610,7 @@ class Trade(ModelBase, LocalTrade):
         :return: unsorted query object
         """
         query = Trade.get_trades_query(trade_filter, include_orders)
-        # this sholud remain split. if use_db is False, session is not available and the above will
+        # this should remain split. if use_db is False, session is not available and the above will
         # raise an exception.
         return Trade.session.scalars(query)
 
@@ -1635,7 +1642,7 @@ class Trade(ModelBase, LocalTrade):
         Retrieves total realized profit
         """
         if Trade.use_db:
-            total_profit: float = Trade.session.execute(
+            total_profit = Trade.session.execute(
                 select(func.sum(Trade.close_profit_abs)).filter(Trade.is_open.is_(False))
             ).scalar_one()
         else:
@@ -1843,4 +1850,4 @@ class Trade(ModelBase, LocalTrade):
                 Order.order_filled_date >= start_date,
                 Order.status == 'closed'
             )).scalar_one()
-        return trading_volume
+        return trading_volume or 0.0

freqtrade/persistence/usedb_context.py

@@ -0,0 +1,33 @@
+
+from freqtrade.persistence.pairlock_middleware import PairLocks
+from freqtrade.persistence.trade_model import Trade
+
+
+def disable_database_use(timeframe: str) -> None:
+    """
+    Disable database usage for PairLocks and Trade models.
+    Used for backtesting, and some other utility commands.
+    """
+    PairLocks.use_db = False
+    PairLocks.timeframe = timeframe
+    Trade.use_db = False
+
+
+def enable_database_use() -> None:
+    """
+    Cleanup function to restore database usage.
+    """
+    PairLocks.use_db = True
+    PairLocks.timeframe = ''
+    Trade.use_db = True
+
+
+class FtNoDBContext:
+    def __init__(self, timeframe: str = ''):
+        self.timeframe = timeframe
+
+    def __enter__(self):
+        disable_database_use(self.timeframe)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        enable_database_use()

freqtrade/plugins/pairlist/RemotePairList.py

@@ -52,6 +52,7 @@ class RemotePairList(IPairList):
         self._read_timeout = self._pairlistconfig.get('read_timeout', 60)
         self._bearer_token = self._pairlistconfig.get('bearer_token', '')
         self._init_done = False
+        self._save_to_file = self._pairlistconfig.get('save_to_file', None)
         self._last_pairlist: List[Any] = list()
 
         if self._mode not in ['whitelist', 'blacklist']:
@@ -136,6 +137,12 @@ class RemotePairList(IPairList):
                 "description": "Bearer token",
                 "help": "Bearer token - used for auth against the upstream service.",
             },
+            "save_to_file": {
+                "type": "string",
+                "default": "",
+                "description": "Filename to save processed pairlist to.",
+                "help": "Specify a filename to save the processed pairlist in JSON format.",
+            },
         }
 
     def process_json(self, jsonparse) -> List[str]:
@@ -184,31 +191,26 @@ class RemotePairList(IPairList):
                 try:
                     pairlist = self.process_json(jsonparse)
                 except Exception as e:
-
+                    pairlist = self._handle_error(f'Failed processing JSON data: {type(e)}')
-                    if self._init_done:
-                        pairlist = self.return_last_pairlist()
-                        logger.warning(f'Error while processing JSON data: {type(e)}')
             else:
-                        raise OperationalException(f'Error while processing JSON data: {type(e)}')
+                pairlist = self._handle_error(f'RemotePairList is not of type JSON.'
-
+                                              f' {self._pairlist_url}')
-            else:
-                if self._init_done:
-                    self.log_once(f'Error: RemotePairList is not of type JSON: '
-                                  f' {self._pairlist_url}', logger.info)
-                    pairlist = self.return_last_pairlist()
-                else:
-                    raise OperationalException('RemotePairList is not of type JSON, abort.')
 
         except requests.exceptions.RequestException:
-            self.log_once(f'Was not able to fetch pairlist from:'
+            pairlist = self._handle_error(f'Was not able to fetch pairlist from:'
-                          f' {self._pairlist_url}', logger.info)
+                                          f' {self._pairlist_url}')
-
-            pairlist = self.return_last_pairlist()
 
             time_elapsed = 0
 
         return pairlist, time_elapsed
 
+    def _handle_error(self, error: str) -> List[str]:
+        if self._init_done:
+            self.log_once("Error: " + error, logger.info)
+            return self.return_last_pairlist()
+        else:
+            raise OperationalException(error)
+
     def gen_pairlist(self, tickers: Tickers) -> List[str]:
         """
         Generate the pairlist
@@ -236,20 +238,15 @@ class RemotePairList(IPairList):
 
                 if file_path.exists():
                     with file_path.open() as json_file:
+                        try:
                             # Load the JSON data into a dictionary
                             jsonparse = rapidjson.load(json_file, parse_mode=CONFIG_PARSE_MODE)
-
-                        try:
                             pairlist = self.process_json(jsonparse)
                         except Exception as e:
-                            if self._init_done:
+                            pairlist = self._handle_error(f'processing JSON data: {type(e)}')
-                                pairlist = self.return_last_pairlist()
-                                logger.warning(f'Error while processing JSON data: {type(e)}')
                 else:
-                                raise OperationalException('Error while processing'
+                    pairlist = self._handle_error(f"{self._pairlist_url} does not exist.")
-                                                           f'JSON data: {type(e)}')
+
-                else:
-                    raise ValueError(f"{self._pairlist_url} does not exist.")
             else:
                 # Fetch Pairlist from Remote URL
                 pairlist, time_elapsed = self.fetch_pairlist()
@@ -273,8 +270,23 @@ class RemotePairList(IPairList):
 
         self._last_pairlist = list(pairlist)
 
+        if self._save_to_file:
+            self.save_pairlist(pairlist, self._save_to_file)
+
         return pairlist
 
+    def save_pairlist(self, pairlist: List[str], filename: str) -> None:
+        pairlist_data = {
+            "pairs": pairlist
+        }
+        try:
+            file_path = Path(filename)
+            with file_path.open('w') as json_file:
+                rapidjson.dump(pairlist_data, json_file)
+                logger.info(f"Processed pairlist saved to {filename}")
+        except Exception as e:
+            logger.error(f"Error saving processed pairlist to {filename}: {e}")
+
     def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
         Filters and sorts pairlist and returns the whitelist again.

freqtrade/plugins/pairlist/VolumePairList.py

@@ -62,16 +62,16 @@ class VolumePairList(IPairList):
 
         # get timeframe in minutes and seconds
         self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe)
-        self._tf_in_sec = self._tf_in_min * 60
+        _tf_in_sec = self._tf_in_min * 60
 
         # wether to use range lookback or not
         self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)
 
-        if self._use_range & (self._refresh_period < self._tf_in_sec):
+        if self._use_range & (self._refresh_period < _tf_in_sec):
             raise OperationalException(
                 f'Refresh period of {self._refresh_period} seconds is smaller than one '
                 f'timeframe of {self._lookback_timeframe}. Please adjust refresh_period '
-                f'to at least {self._tf_in_sec} and restart the bot.'
+                f'to at least {_tf_in_sec} and restart the bot.'
             )
 
         if (not self._use_range and not (

freqtrade/rpc/api_server/api_auth.py

@@ -1,6 +1,6 @@
 import logging
 import secrets
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, Union
 
 import jwt
@@ -88,14 +88,14 @@ async def validate_ws_token(
 def create_token(data: dict, secret_key: str, token_type: str = "access") -> str:
     to_encode = data.copy()
     if token_type == "access":
-        expire = datetime.utcnow() + timedelta(minutes=15)
+        expire = datetime.now(timezone.utc) + timedelta(minutes=15)
     elif token_type == "refresh":
-        expire = datetime.utcnow() + timedelta(days=30)
+        expire = datetime.now(timezone.utc) + timedelta(days=30)
     else:
         raise ValueError()
     to_encode.update({
         "exp": expire,
-        "iat": datetime.utcnow(),
+        "iat": datetime.now(timezone.utc),
         "type": token_type,
     })
     encoded_jwt = jwt.encode(to_encode, secret_key, algorithm=ALGORITHM)

freqtrade/rpc/api_server/api_background_tasks.py

@@ -7,6 +7,7 @@ from fastapi.exceptions import HTTPException
 from freqtrade.constants import Config
 from freqtrade.enums import CandleType
 from freqtrade.exceptions import OperationalException
+from freqtrade.persistence import FtNoDBContext
 from freqtrade.rpc.api_server.api_schemas import (BackgroundTaskStatus, BgJobStarted,
                                                   ExchangeModePayloadMixin, PairListsPayload,
                                                   PairListsResponse, WhitelistEvaluateResponse)
@@ -57,7 +58,7 @@ def __run_pairlist(job_id: str, config_loc: Config):
 
         ApiBG.jobs[job_id]['is_running'] = True
         from freqtrade.plugins.pairlistmanager import PairListManager
-
+        with FtNoDBContext():
             exchange = get_exchange(config_loc)
             pairlists = PairListManager(exchange, config_loc)
             pairlists.refresh_pairlist()

freqtrade/rpc/api_server/api_schemas.py

@@ -538,6 +538,10 @@ class BacktestHistoryEntry(BaseModel):
     run_id: str
     backtest_start_time: int
     notes: Optional[str] = ''
+    backtest_start_ts: Optional[int] = None
+    backtest_end_ts: Optional[int] = None
+    timeframe: Optional[str] = None
+    timeframe_detail: Optional[str] = None
 
 
 class BacktestMetadataUpdate(BaseModel):

freqtrade/rpc/api_server/webserver.py

@@ -107,7 +107,7 @@ class ApiServer(RPCHandler):
             ApiServer._message_stream.publish(msg)
 
     def handle_rpc_exception(self, request, exc):
-        logger.exception(f"API Error calling: {exc}")
+        logger.error(f"API Error calling: {exc}")
         return JSONResponse(
             status_code=502,
             content={'error': f"Error querying {request.url.path}: {exc.message}"}

freqtrade/rpc/rpc.py

@@ -25,13 +25,13 @@ from freqtrade.exceptions import ExchangeError, PricingError
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
 from freqtrade.exchange.types import Tickers
 from freqtrade.loggers import bufferHandler
-from freqtrade.misc import decimals_per_coin
 from freqtrade.persistence import KeyStoreKeys, KeyValueStore, PairLocks, Trade
 from freqtrade.persistence.models import PairLock
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
 from freqtrade.rpc.rpc_types import RPCSendMsg
-from freqtrade.util import dt_humanize, dt_now, dt_ts_def, format_date, shorten_date
+from freqtrade.util import (decimals_per_coin, dt_humanize, dt_now, dt_ts_def, format_date,
+                            shorten_date)
 from freqtrade.wallets import PositionWallet, Wallet
 
 

freqtrade/rpc/rpc_types.py

@@ -51,6 +51,7 @@ class __RPCEntryExitMsgBase(RPCSendMsgBase):
     exchange: str
     pair: str
     base_currency: str
+    quote_currency: str
     leverage: Optional[float]
     direction: str
     limit: float
@@ -81,11 +82,12 @@ class RPCExitMsg(__RPCEntryExitMsgBase):
     close_rate: float
     profit_amount: float
     profit_ratio: float
-    sell_reason: Optional[str]
     exit_reason: Optional[str]
     close_date: datetime
     # current_rate: Optional[float]
     order_rate: Optional[float]
+    final_profit_ratio: Optional[float]
+    is_final_exit: bool
 
 
 class RPCExitCancelMsg(__RPCEntryExitMsgBase):
@@ -94,7 +96,6 @@ class RPCExitCancelMsg(__RPCEntryExitMsgBase):
     gain: ProfitLossStr
     profit_amount: float
     profit_ratio: float
-    sell_reason: Optional[str]
     exit_reason: Optional[str]
     close_date: datetime
 
@@ -117,6 +118,9 @@ class RPCNewCandleMsg(RPCSendMsgBase):
     data: PairWithTimeframe
 
 
+RPCOrderMsg = Union[RPCEntryMsg, RPCExitMsg, RPCExitCancelMsg, RPCCancelMsg]
+
+
 RPCSendMsg = Union[
     RPCStatusMsg,
     RPCStrategyMsg,

freqtrade/rpc/telegram.py

@@ -10,12 +10,12 @@ import re
 from copy import deepcopy
 from dataclasses import dataclass
 from datetime import date, datetime, timedelta
-from functools import partial
+from functools import partial, wraps
 from html import escape
 from itertools import chain
 from math import isnan
 from threading import Thread
-from typing import Any, Callable, Coroutine, Dict, List, Optional, Union
+from typing import Any, Callable, Coroutine, Dict, List, Literal, Optional, Union
 
 from tabulate import tabulate
 from telegram import (CallbackQuery, InlineKeyboardButton, InlineKeyboardMarkup, KeyboardButton,
@@ -29,11 +29,11 @@ from freqtrade.__init__ import __version__
 from freqtrade.constants import DUST_PER_COIN, Config
 from freqtrade.enums import MarketDirection, RPCMessageType, SignalDirection, TradingMode
 from freqtrade.exceptions import OperationalException
-from freqtrade.misc import chunks, plural, round_coin_value
+from freqtrade.misc import chunks, plural
 from freqtrade.persistence import Trade
 from freqtrade.rpc import RPC, RPCException, RPCHandler
-from freqtrade.rpc.rpc_types import RPCSendMsg
+from freqtrade.rpc.rpc_types import RPCEntryMsg, RPCExitMsg, RPCOrderMsg, RPCSendMsg
-from freqtrade.util import dt_humanize
+from freqtrade.util import dt_humanize, fmt_coin, round_value
 
 
 MAX_MESSAGE_LENGTH = MessageLimit.MAX_TEXT_LENGTH
@@ -44,6 +44,23 @@ logger = logging.getLogger(__name__)
 logger.debug('Included module rpc.telegram ...')
 
 
+def safe_async_db(func: Callable[..., Any]):
+    """
+    Decorator to safely handle sessions when switching async context
+    :param func: function to decorate
+    :return: decorated function
+    """
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        """ Decorator logic """
+        try:
+            return func(*args, **kwargs)
+        finally:
+            Trade.session.remove()
+
+    return wrapper
+
+
 @dataclass
 class TimeunitMappings:
     header: str
@@ -61,6 +78,7 @@ def authorized_only(command_handler: Callable[..., Coroutine[Any, Any, None]]):
     :return: decorated function
     """
 
+    @wraps(command_handler)
     async def wrapper(self, *args, **kwargs):
         """ Decorator logic """
         update = kwargs.get('update') or args[0]
@@ -286,7 +304,7 @@ class Telegram(RPCHandler):
         asyncio.run_coroutine_threadsafe(self._cleanup_telegram(), self._loop)
         self._thread.join()
 
-    def _exchange_from_msg(self, msg: Dict[str, Any]) -> str:
+    def _exchange_from_msg(self, msg: RPCOrderMsg) -> str:
         """
         Extracts the exchange name from the given message.
         :param msg: The message to extract the exchange name from.
@@ -310,164 +328,172 @@ class Telegram(RPCHandler):
 
         return ''
 
-    def _format_entry_msg(self, msg: Dict[str, Any]) -> str:
+    def _format_entry_msg(self, msg: RPCEntryMsg) -> str:
-        if self._rpc._fiat_converter:
+
-            msg['stake_amount_fiat'] = self._rpc._fiat_converter.convert_amount(
-                msg['stake_amount'], msg['stake_currency'], msg['fiat_currency'])
-        else:
-            msg['stake_amount_fiat'] = 0
         is_fill = msg['type'] in [RPCMessageType.ENTRY_FILL]
         emoji = '\N{CHECK MARK}' if is_fill else '\N{LARGE BLUE CIRCLE}'
 
-        entry_side = ({'enter': 'Long', 'entered': 'Longed'} if msg['direction'] == 'Long'
+        terminology = {
-                      else {'enter': 'Short', 'entered': 'Shorted'})
+            '1_enter': 'New Trade',
+            '1_entered': 'New Trade filled',
+            'x_enter': 'Increasing position',
+            'x_entered': 'Position increase filled',
+        }
+
+        key = f"{'x' if msg['sub_trade'] else '1'}_{'entered' if is_fill else 'enter'}"
+        wording = terminology[key]
+
         message = (
             f"{emoji} *{self._exchange_from_msg(msg)}:*"
-            f" {entry_side['entered'] if is_fill else entry_side['enter']} {msg['pair']}"
+            f" {wording} (#{msg['trade_id']})\n"
-            f" (#{msg['trade_id']})\n"
+            f"*Pair:* `{msg['pair']}`\n"
         )
         message += self._add_analyzed_candle(msg['pair'])
         message += f"*Enter Tag:* `{msg['enter_tag']}`\n" if msg.get('enter_tag') else ""
-        message += f"*Amount:* `{msg['amount']:.8f}`\n"
+        message += f"*Amount:* `{round_value(msg['amount'], 8)}`\n"
+        message += f"*Direction:* `{msg['direction']}"
         if msg.get('leverage') and msg.get('leverage', 1.0) != 1.0:
-            message += f"*Leverage:* `{msg['leverage']}`\n"
+            message += f" ({msg['leverage']:.1g}x)"
+        message += "`\n"
+        message += f"*Open Rate:* `{fmt_coin(msg['open_rate'], msg['quote_currency'])}`\n"
+        if msg['type'] == RPCMessageType.ENTRY and msg['current_rate']:
+            message += f"*Current Rate:* `{fmt_coin(msg['current_rate'], msg['quote_currency'])}`\n"
 
-        if msg['type'] in [RPCMessageType.ENTRY_FILL]:
+        profit_fiat_extra = self.__format_profit_fiat(msg, 'stake_amount')  # type: ignore
-            message += f"*Open Rate:* `{msg['open_rate']:.8f}`\n"
+        total = fmt_coin(msg['stake_amount'], msg['quote_currency'])
-        elif msg['type'] in [RPCMessageType.ENTRY]:
-            message += f"*Open Rate:* `{msg['open_rate']:.8f}`\n"\
-                       f"*Current Rate:* `{msg['current_rate']:.8f}`\n"
 
-        message += f"*Total:* `({round_coin_value(msg['stake_amount'], msg['stake_currency'])}"
+        message += f"*{'New ' if msg['sub_trade'] else ''}Total:* `{total}{profit_fiat_extra}`"
 
-        if msg.get('fiat_currency'):
-            message += f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}"
-
-        message += ")`"
         return message
 
-    def _format_exit_msg(self, msg: Dict[str, Any]) -> str:
+    def _format_exit_msg(self, msg: RPCExitMsg) -> str:
-        msg['amount'] = round(msg['amount'], 8)
+        duration = msg['close_date'].replace(
-        msg['profit_percent'] = round(msg['profit_ratio'] * 100, 2)
-        msg['duration'] = msg['close_date'].replace(
             microsecond=0) - msg['open_date'].replace(microsecond=0)
-        msg['duration_min'] = msg['duration'].total_seconds() / 60
+        duration_min = duration.total_seconds() / 60
 
-        msg['enter_tag'] = msg['enter_tag'] if "enter_tag" in msg.keys() else None
+        leverage_text = (f" ({msg['leverage']:.1g}x)"
-        msg['emoji'] = self._get_sell_emoji(msg)
-        msg['leverage_text'] = (f"*Leverage:* `{msg['leverage']:.1f}`\n"
                          if msg.get('leverage') and msg.get('leverage', 1.0) != 1.0
                          else "")
 
-        # Check if all sell properties are available.
+        profit_fiat_extra = self.__format_profit_fiat(msg, 'profit_amount')
-        # This might not be the case if the message origin is triggered by /forceexit
+
-        if (all(prop in msg for prop in ['gain', 'fiat_currency', 'stake_currency'])
+        profit_extra = (
-                and self._rpc._fiat_converter):
+            f" ({msg['gain']}: {fmt_coin(msg['profit_amount'], msg['quote_currency'])}"
-            msg['profit_fiat'] = self._rpc._fiat_converter.convert_amount(
+            f"{profit_fiat_extra})")
-                msg['profit_amount'], msg['stake_currency'], msg['fiat_currency'])
-            msg['profit_extra'] = f" / {msg['profit_fiat']:.3f} {msg['fiat_currency']}"
-        else:
-            msg['profit_extra'] = ''
-        msg['profit_extra'] = (
-            f" ({msg['gain']}: {msg['profit_amount']:.8f} {msg['stake_currency']}"
-            f"{msg['profit_extra']})")
 
         is_fill = msg['type'] == RPCMessageType.EXIT_FILL
         is_sub_trade = msg.get('sub_trade')
         is_sub_profit = msg['profit_amount'] != msg.get('cumulative_profit')
-        profit_prefix = ('Sub ' if is_sub_profit else 'Cumulative ') if is_sub_trade else ''
+        is_final_exit = msg.get('is_final_exit', False) and is_sub_profit
+        profit_prefix = 'Sub ' if is_sub_trade else ''
         cp_extra = ''
         exit_wording = 'Exited' if is_fill else 'Exiting'
-        if is_sub_profit and is_sub_trade:
+        if is_sub_trade or is_final_exit:
-            if self._rpc._fiat_converter:
+            cp_fiat = self.__format_profit_fiat(msg, 'cumulative_profit')
-                cp_fiat = self._rpc._fiat_converter.convert_amount(
-                    msg['cumulative_profit'], msg['stake_currency'], msg['fiat_currency'])
-                cp_extra = f" / {cp_fiat:.3f} {msg['fiat_currency']}"
-            exit_wording = f"Partially {exit_wording.lower()}"
-            cp_extra = (
-                f"*Cumulative Profit:* (`{msg['cumulative_profit']:.8f} "
-                f"{msg['stake_currency']}{cp_extra}`)\n"
-            )
 
+            if is_final_exit:
+                profit_prefix = 'Sub '
+                cp_extra = (
+                    f"*Final Profit:* `{msg['final_profit_ratio']:.2%} "
+                    f"({msg['cumulative_profit']:.8f} {msg['quote_currency']}{cp_fiat})`\n"
+                )
+            else:
+                exit_wording = f"Partially {exit_wording.lower()}"
+                if msg['cumulative_profit']:
+                    cp_extra = (
+                        f"*Cumulative Profit:* `"
+                        f"{fmt_coin(msg['cumulative_profit'], msg['stake_currency'])}{cp_fiat}`\n"
+                    )
+        enter_tag = f"*Enter Tag:* `{msg['enter_tag']}`\n" if msg.get('enter_tag') else ""
         message = (
-            f"{msg['emoji']} *{self._exchange_from_msg(msg)}:* "
+            f"{self._get_exit_emoji(msg)} *{self._exchange_from_msg(msg)}:* "
             f"{exit_wording} {msg['pair']} (#{msg['trade_id']})\n"
             f"{self._add_analyzed_candle(msg['pair'])}"
             f"*{f'{profit_prefix}Profit' if is_fill else f'Unrealized {profit_prefix}Profit'}:* "
-            f"`{msg['profit_ratio']:.2%}{msg['profit_extra']}`\n"
+            f"`{msg['profit_ratio']:.2%}{profit_extra}`\n"
             f"{cp_extra}"
-            f"*Enter Tag:* `{msg['enter_tag']}`\n"
+            f"{enter_tag}"
             f"*Exit Reason:* `{msg['exit_reason']}`\n"
-            f"*Direction:* `{msg['direction']}`\n"
+            f"*Direction:* `{msg['direction']}"
-            f"{msg['leverage_text']}"
+            f"{leverage_text}`\n"
-            f"*Amount:* `{msg['amount']:.8f}`\n"
+            f"*Amount:* `{round_value(msg['amount'], 8)}`\n"
-            f"*Open Rate:* `{msg['open_rate']:.8f}`\n"
+            f"*Open Rate:* `{fmt_coin(msg['open_rate'], msg['quote_currency'])}`\n"
         )
-        if msg['type'] == RPCMessageType.EXIT:
+        if msg['type'] == RPCMessageType.EXIT and msg['current_rate']:
-            message += f"*Current Rate:* `{msg['current_rate']:.8f}`\n"
+            message += f"*Current Rate:* `{fmt_coin(msg['current_rate'], msg['quote_currency'])}`\n"
             if msg['order_rate']:
-                message += f"*Exit Rate:* `{msg['order_rate']:.8f}`"
+                message += f"*Exit Rate:* `{fmt_coin(msg['order_rate'], msg['quote_currency'])}`"
-
         elif msg['type'] == RPCMessageType.EXIT_FILL:
-            message += f"*Exit Rate:* `{msg['close_rate']:.8f}`"
+            message += f"*Exit Rate:* `{fmt_coin(msg['close_rate'], msg['quote_currency'])}`"
+
         if is_sub_trade:
-            if self._rpc._fiat_converter:
+            stake_amount_fiat = self.__format_profit_fiat(msg, 'stake_amount')
-                msg['stake_amount_fiat'] = self._rpc._fiat_converter.convert_amount(
-                    msg['stake_amount'], msg['stake_currency'], msg['fiat_currency'])
-            else:
-                msg['stake_amount_fiat'] = 0
-            rem = round_coin_value(msg['stake_amount'], msg['stake_currency'])
-            message += f"\n*Remaining:* `({rem}"
 
-            if msg.get('fiat_currency', None):
+            rem = fmt_coin(msg['stake_amount'], msg['quote_currency'])
-                message += f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}"
+            message += f"\n*Remaining:* `{rem}{stake_amount_fiat}`"
-
-            message += ")`"
         else:
-            message += f"\n*Duration:* `{msg['duration']} ({msg['duration_min']:.1f} min)`"
+            message += f"\n*Duration:* `{duration} ({duration_min:.1f} min)`"
         return message
 
-    def compose_message(self, msg: Dict[str, Any], msg_type: RPCMessageType) -> Optional[str]:
+    def __format_profit_fiat(
-        if msg_type in [RPCMessageType.ENTRY, RPCMessageType.ENTRY_FILL]:
+            self,
+            msg: RPCExitMsg,
+            key: Literal['stake_amount', 'profit_amount', 'cumulative_profit']
+    ) -> str:
+        """
+        Format Fiat currency to append to regular profit output
+        """
+        profit_fiat_extra = ''
+        if self._rpc._fiat_converter and (fiat_currency := msg.get('fiat_currency')):
+            profit_fiat = self._rpc._fiat_converter.convert_amount(
+                    msg[key], msg['stake_currency'], fiat_currency)
+            profit_fiat_extra = f" / {profit_fiat:.3f} {fiat_currency}"
+        return profit_fiat_extra
+
+    def compose_message(self, msg: RPCSendMsg) -> Optional[str]:
+        if msg['type'] == RPCMessageType.ENTRY or msg['type'] == RPCMessageType.ENTRY_FILL:
             message = self._format_entry_msg(msg)
 
-        elif msg_type in [RPCMessageType.EXIT, RPCMessageType.EXIT_FILL]:
+        elif msg['type'] == RPCMessageType.EXIT or msg['type'] == RPCMessageType.EXIT_FILL:
             message = self._format_exit_msg(msg)
 
-        elif msg_type in (RPCMessageType.ENTRY_CANCEL, RPCMessageType.EXIT_CANCEL):
+        elif (
-            msg['message_side'] = 'enter' if msg_type in [RPCMessageType.ENTRY_CANCEL] else 'exit'
+            msg['type'] == RPCMessageType.ENTRY_CANCEL
+            or msg['type'] == RPCMessageType.EXIT_CANCEL
+        ):
+            message_side = 'enter' if msg['type'] == RPCMessageType.ENTRY_CANCEL else 'exit'
             message = (f"\N{WARNING SIGN} *{self._exchange_from_msg(msg)}:* "
                        f"Cancelling {'partial ' if msg.get('sub_trade') else ''}"
-                       f"{msg['message_side']} Order for {msg['pair']} "
+                       f"{message_side} Order for {msg['pair']} "
                        f"(#{msg['trade_id']}). Reason: {msg['reason']}.")
 
-        elif msg_type == RPCMessageType.PROTECTION_TRIGGER:
+        elif msg['type'] == RPCMessageType.PROTECTION_TRIGGER:
             message = (
                 f"*Protection* triggered due to {msg['reason']}. "
                 f"`{msg['pair']}` will be locked until `{msg['lock_end_time']}`."
             )
 
-        elif msg_type == RPCMessageType.PROTECTION_TRIGGER_GLOBAL:
+        elif msg['type'] == RPCMessageType.PROTECTION_TRIGGER_GLOBAL:
             message = (
                 f"*Protection* triggered due to {msg['reason']}. "
                 f"*All pairs* will be locked until `{msg['lock_end_time']}`."
             )
 
-        elif msg_type == RPCMessageType.STATUS:
+        elif msg['type'] == RPCMessageType.STATUS:
             message = f"*Status:* `{msg['status']}`"
 
-        elif msg_type == RPCMessageType.WARNING:
+        elif msg['type'] == RPCMessageType.WARNING:
             message = f"\N{WARNING SIGN} *Warning:* `{msg['status']}`"
-        elif msg_type == RPCMessageType.EXCEPTION:
+        elif msg['type'] == RPCMessageType.EXCEPTION:
             # Errors will contain exceptions, which are wrapped in tripple ticks.
             message = f"\N{WARNING SIGN} *ERROR:* \n {msg['status']}"
 
-        elif msg_type == RPCMessageType.STARTUP:
+        elif msg['type'] == RPCMessageType.STARTUP:
             message = f"{msg['status']}"
-        elif msg_type == RPCMessageType.STRATEGY_MSG:
+        elif msg['type'] == RPCMessageType.STRATEGY_MSG:
             message = f"{msg['msg']}"
         else:
-            logger.debug("Unknown message type: %s", msg_type)
+            logger.debug("Unknown message type: %s", msg['type'])
             return None
         return message
 
@@ -495,20 +521,20 @@ class Telegram(RPCHandler):
             # Notification disabled
             return
 
-        message = self.compose_message(deepcopy(msg), msg_type)  # type: ignore
+        message = self.compose_message(deepcopy(msg))
         if message:
             asyncio.run_coroutine_threadsafe(
                 self._send_msg(message, disable_notification=(noti == 'silent')),
                 self._loop)
 
-    def _get_sell_emoji(self, msg):
+    def _get_exit_emoji(self, msg):
         """
-        Get emoji for sell-side
+        Get emoji for exit-messages
         """
 
-        if float(msg['profit_percent']) >= 5.0:
+        if float(msg['profit_ratio']) >= 0.05:
             return "\N{ROCKET}"
-        elif float(msg['profit_percent']) >= 0.0:
+        elif float(msg['profit_ratio']) >= 0.0:
             return "\N{EIGHT SPOKED ASTERISK}"
         elif msg['exit_reason'] == "stop_loss":
             return "\N{WARNING SIGN}"
@@ -537,7 +563,7 @@ class Telegram(RPCHandler):
             if order_nr == 1:
                 lines.append(
                     f"*Amount:* {cur_entry_amount:.8g} "
-                    f"({round_coin_value(order['cost'], quote_currency)})"
+                    f"({fmt_coin(order['cost'], quote_currency)})"
                 )
                 lines.append(f"*Average Price:* {cur_entry_average:.8g}")
             else:
@@ -547,7 +573,7 @@ class Telegram(RPCHandler):
                     lines.append("({})".format(dt_humanize(order["order_filled_date"],
                                                            granularity=["day", "hour", "minute"])))
                 lines.append(f"*Amount:* {cur_entry_amount:.8g} "
-                             f"({round_coin_value(order['cost'], quote_currency)})")
+                             f"({fmt_coin(order['cost'], quote_currency)})")
                 lines.append(f"*Average {wording} Price:* {cur_entry_average:.8g} "
                              f"({price_to_1st_entry:.2%} from 1st entry rate)")
                 lines.append(f"*Order Filled:* {order['order_filled_date']}")
@@ -633,12 +659,12 @@ class Telegram(RPCHandler):
             r['num_exits'] = len([o for o in r['orders'] if not o['ft_is_entry']
                                  and not o['ft_order_side'] == 'stoploss'])
             r['exit_reason'] = r.get('exit_reason', "")
-            r['stake_amount_r'] = round_coin_value(r['stake_amount'], r['quote_currency'])
+            r['stake_amount_r'] = fmt_coin(r['stake_amount'], r['quote_currency'])
-            r['max_stake_amount_r'] = round_coin_value(
+            r['max_stake_amount_r'] = fmt_coin(
                 r['max_stake_amount'] or r['stake_amount'], r['quote_currency'])
-            r['profit_abs_r'] = round_coin_value(r['profit_abs'], r['quote_currency'])
+            r['profit_abs_r'] = fmt_coin(r['profit_abs'], r['quote_currency'])
-            r['realized_profit_r'] = round_coin_value(r['realized_profit'], r['quote_currency'])
+            r['realized_profit_r'] = fmt_coin(r['realized_profit'], r['quote_currency'])
-            r['total_profit_abs_r'] = round_coin_value(
+            r['total_profit_abs_r'] = fmt_coin(
                 r['total_profit_abs'], r['quote_currency'])
             lines = [
                 "*Trade ID:* `{trade_id}`" +
@@ -781,7 +807,7 @@ class Telegram(RPCHandler):
         )
         stats_tab = tabulate(
             [[f"{period['date']:{val.dateformat}} ({period['trade_count']})",
-              f"{round_coin_value(period['abs_profit'], stats['stake_currency'])}",
+              f"{fmt_coin(period['abs_profit'], stats['stake_currency'])}",
               f"{period['fiat_value']:.2f} {stats['fiat_display_currency']}",
               f"{period['rel_profit']:.2%}",
               ] for period in stats['data']],
@@ -883,19 +909,19 @@ class Telegram(RPCHandler):
             # Message to display
             if stats['closed_trade_count'] > 0:
                 markdown_msg = ("*ROI:* Closed trades\n"
-                                f"∙ `{round_coin_value(profit_closed_coin, stake_cur)} "
+                                f"∙ `{fmt_coin(profit_closed_coin, stake_cur)} "
                                 f"({profit_closed_ratio_mean:.2%}) "
                                 f"({profit_closed_percent} \N{GREEK CAPITAL LETTER SIGMA}%)`\n"
-                                f"∙ `{round_coin_value(profit_closed_fiat, fiat_disp_cur)}`\n")
+                                f"∙ `{fmt_coin(profit_closed_fiat, fiat_disp_cur)}`\n")
             else:
                 markdown_msg = "`No closed trade` \n"
 
             markdown_msg += (
                 f"*ROI:* All trades\n"
-                f"∙ `{round_coin_value(profit_all_coin, stake_cur)} "
+                f"∙ `{fmt_coin(profit_all_coin, stake_cur)} "
                 f"({profit_all_ratio_mean:.2%}) "
                 f"({profit_all_percent} \N{GREEK CAPITAL LETTER SIGMA}%)`\n"
-                f"∙ `{round_coin_value(profit_all_fiat, fiat_disp_cur)}`\n"
+                f"∙ `{fmt_coin(profit_all_fiat, fiat_disp_cur)}`\n"
                 f"*Total Trade Count:* `{trade_count}`\n"
                 f"*Bot started:* `{stats['bot_start_date']}`\n"
                 f"*{'First Trade opened' if not timescale else 'Showing Profit since'}:* "
@@ -909,14 +935,14 @@ class Telegram(RPCHandler):
                 markdown_msg += (
                     f"\n*Avg. Duration:* `{avg_duration}`\n"
                     f"*Best Performing:* `{best_pair}: {best_pair_profit_ratio:.2%}`\n"
-                    f"*Trading volume:* `{round_coin_value(stats['trading_volume'], stake_cur)}`\n"
+                    f"*Trading volume:* `{fmt_coin(stats['trading_volume'], stake_cur)}`\n"
                     f"*Profit factor:* `{stats['profit_factor']:.2f}`\n"
                     f"*Max Drawdown:* `{stats['max_drawdown']:.2%} "
-                    f"({round_coin_value(stats['max_drawdown_abs'], stake_cur)})`\n"
+                    f"({fmt_coin(stats['max_drawdown_abs'], stake_cur)})`\n"
                     f"    from `{stats['max_drawdown_start']} "
-                    f"({round_coin_value(stats['drawdown_high'], stake_cur)})`\n"
+                    f"({fmt_coin(stats['drawdown_high'], stake_cur)})`\n"
                     f"    to `{stats['max_drawdown_end']} "
-                    f"({round_coin_value(stats['drawdown_low'], stake_cur)})`\n"
+                    f"({fmt_coin(stats['drawdown_low'], stake_cur)})`\n"
                 )
         await self._send_msg(markdown_msg, reload_able=True, callback_path="update_profit",
                              query=update.callback_query)
@@ -984,9 +1010,9 @@ class Telegram(RPCHandler):
         output = ''
         if self._config['dry_run']:
             output += "*Warning:* Simulated balances in Dry Mode.\n"
-        starting_cap = round_coin_value(result['starting_capital'], self._config['stake_currency'])
+        starting_cap = fmt_coin(result['starting_capital'], self._config['stake_currency'])
         output += f"Starting capital: `{starting_cap}`"
-        starting_cap_fiat = round_coin_value(
+        starting_cap_fiat = fmt_coin(
             result['starting_capital_fiat'], self._config['fiat_display_currency']
         ) if result['starting_capital_fiat'] > 0 else ''
         output += (f" `, {starting_cap_fiat}`.\n"
@@ -1006,9 +1032,9 @@ class Telegram(RPCHandler):
                         f"\t`{curr['side']}: {curr['position']:.8f}`\n"
                         f"\t`Leverage: {curr['leverage']:.1f}`\n"
                         f"\t`Est. {curr['stake']}: "
-                        f"{round_coin_value(curr['est_stake'], curr['stake'], False)}`\n")
+                        f"{fmt_coin(curr['est_stake'], curr['stake'], False)}`\n")
                 else:
-                    est_stake = round_coin_value(
+                    est_stake = fmt_coin(
                         curr['est_stake' if full_result else 'est_stake_bot'], curr['stake'], False)
 
                     curr_output = (
@@ -1036,13 +1062,13 @@ class Telegram(RPCHandler):
                 f"{plural(total_dust_currencies, 'Currency', 'Currencies')} "
                 f"(< {balance_dust_level} {result['stake']}):*\n"
                 f"\t`Est. {result['stake']}: "
-                f"{round_coin_value(total_dust_balance, result['stake'], False)}`\n")
+                f"{fmt_coin(total_dust_balance, result['stake'], False)}`\n")
         tc = result['trade_count'] > 0
         stake_improve = f" `({result['starting_capital_ratio']:.2%})`" if tc else ''
         fiat_val = f" `({result['starting_capital_fiat_ratio']:.2%})`" if tc else ''
-        value = round_coin_value(
+        value = fmt_coin(
             result['value' if full_result else 'value_bot'], result['symbol'], False)
-        total_stake = round_coin_value(
+        total_stake = fmt_coin(
             result['total' if full_result else 'total_bot'], result['stake'], False)
         output += (
             f"\n*Estimated Value{' (Bot managed assets only)' if not full_result else ''}*:\n"
@@ -1150,7 +1176,7 @@ class Telegram(RPCHandler):
             try:
                 loop = asyncio.get_running_loop()
                 # Workaround to avoid nested loops
-                await loop.run_in_executor(None, self._rpc._rpc_force_exit, trade_id)
+                await loop.run_in_executor(None, safe_async_db(self._rpc._rpc_force_exit), trade_id)
             except RPCException as e:
                 await self._send_msg(str(e))
 
@@ -1176,6 +1202,7 @@ class Telegram(RPCHandler):
     async def _force_enter_action(self, pair, price: Optional[float], order_side: SignalDirection):
         if pair != 'cancel':
             try:
+                @safe_async_db
                 def _force_enter():
                     self._rpc._rpc_force_entry(pair, price, order_side=order_side)
                 loop = asyncio.get_running_loop()
@@ -1320,7 +1347,7 @@ class Telegram(RPCHandler):
         for i, trade in enumerate(trades):
             stat_line = (
                 f"{i + 1}.\t <code>{trade['pair']}\t"
-                f"{round_coin_value(trade['profit_abs'], self._config['stake_currency'])} "
+                f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} "
                 f"({trade['profit_ratio']:.2%}) "
                 f"({trade['count']})</code>\n")
 
@@ -1352,7 +1379,7 @@ class Telegram(RPCHandler):
         for i, trade in enumerate(trades):
             stat_line = (
                 f"{i + 1}.\t <code>{trade['enter_tag']}\t"
-                f"{round_coin_value(trade['profit_abs'], self._config['stake_currency'])} "
+                f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} "
                 f"({trade['profit_ratio']:.2%}) "
                 f"({trade['count']})</code>\n")
 
@@ -1384,7 +1411,7 @@ class Telegram(RPCHandler):
         for i, trade in enumerate(trades):
             stat_line = (
                 f"{i + 1}.\t <code>{trade['exit_reason']}\t"
-                f"{round_coin_value(trade['profit_abs'], self._config['stake_currency'])} "
+                f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} "
                 f"({trade['profit_ratio']:.2%}) "
                 f"({trade['count']})</code>\n")
 
@@ -1416,7 +1443,7 @@ class Telegram(RPCHandler):
         for i, trade in enumerate(trades):
             stat_line = (
                 f"{i + 1}.\t <code>{trade['mix_tag']}\t"
-                f"{round_coin_value(trade['profit_abs'], self._config['stake_currency'])} "
+                f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} "
                 f"({trade['profit_ratio']:.2%}) "
                 f"({trade['count']})</code>\n")
 

freqtrade/strategy/interface.py

@@ -1004,7 +1004,7 @@ class IStrategy(ABC, HyperStrategyMixin):
         :param is_short: Indicating existing trade direction.
         :return: (enter, exit) A bool-tuple with enter / exit values.
         """
-        latest, latest_date = self.get_latest_candle(pair, timeframe, dataframe)
+        latest, _latest_date = self.get_latest_candle(pair, timeframe, dataframe)
         if latest is None:
             return False, False, None
 
@@ -1388,7 +1388,8 @@ class IStrategy(ABC, HyperStrategyMixin):
         """
 
         logger.debug(f"Populating enter signals for pair {metadata.get('pair')}.")
-
+        # Initialize column to work around Pandas bug #56503.
+        dataframe.loc[:, 'enter_tag'] = ''
         df = self.populate_entry_trend(dataframe, metadata)
         if 'enter_long' not in df.columns:
             df = df.rename({'buy': 'enter_long', 'buy_tag': 'enter_tag'}, axis='columns')
@@ -1404,6 +1405,8 @@ class IStrategy(ABC, HyperStrategyMixin):
             currently traded pair
         :return: DataFrame with exit column
         """
+        # Initialize column to work around Pandas bug #56503.
+        dataframe.loc[:, 'exit_tag'] = ''
         logger.debug(f"Populating exit signals for pair {metadata.get('pair')}.")
         df = self.populate_exit_trend(dataframe, metadata)
         if 'exit_long' not in df.columns:

freqtrade/templates/FreqaiExampleHybridStrategy.py

@@ -29,7 +29,7 @@ class FreqaiExampleHybridStrategy(IStrategy):
         "enabled": true,
         "purge_old_models": 2,
         "train_period_days": 15,
-        "identifier": "uniqe-id",
+        "identifier": "unique-id",
         "feature_parameters": {
             "include_timeframes": [
                 "3m",

freqtrade/types/backtest_result_type.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 from typing_extensions import TypedDict
 
@@ -26,3 +26,7 @@ class BacktestHistoryEntryType(BacktestMetadataType):
     filename: str
     strategy: str
     notes: str
+    backtest_start_ts: Optional[int]
+    backtest_end_ts: Optional[int]
+    timeframe: Optional[str]
+    timeframe_detail: Optional[str]

freqtrade/util/__init__.py

@@ -1,6 +1,7 @@
 from freqtrade.util.datetime_helpers import (dt_floor_day, dt_from_ts, dt_humanize, dt_now, dt_ts,
                                              dt_ts_def, dt_utc, format_date, format_ms_time,
                                              shorten_date)
+from freqtrade.util.formatters import decimals_per_coin, fmt_coin, round_value
 from freqtrade.util.ft_precise import FtPrecise
 from freqtrade.util.periodic_cache import PeriodicCache
 from freqtrade.util.template_renderer import render_template, render_template_with_fallback  # noqa
@@ -19,4 +20,7 @@ __all__ = [
     'FtPrecise',
     'PeriodicCache',
     'shorten_date',
+    'decimals_per_coin',
+    'round_value',
+    'fmt_coin',
 ]

freqtrade/util/formatters.py

@@ -0,0 +1,42 @@
+from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN
+
+
+def decimals_per_coin(coin: str):
+    """
+    Helper method getting decimal amount for this coin
+    example usage: f".{decimals_per_coin('USD')}f"
+    :param coin: Which coin are we printing the price / value for
+    """
+    return DECIMALS_PER_COIN.get(coin, DECIMAL_PER_COIN_FALLBACK)
+
+
+def round_value(value: float, decimals: int, keep_trailing_zeros=False) -> str:
+    """
+    Round value to given decimals
+    :param value: Value to be rounded
+    :param decimals: Number of decimals to round to
+    :param keep_trailing_zeros: Keep trailing zeros "222.200" vs. "222.2"
+    :return: Rounded value as string
+    """
+    val = f"{value:.{decimals}f}"
+    if not keep_trailing_zeros:
+        val = val.rstrip('0').rstrip('.')
+    return val
+
+
+def fmt_coin(
+        value: float, coin: str, show_coin_name=True, keep_trailing_zeros=False) -> str:
+    """
+    Format price value for this coin
+    :param value: Value to be printed
+    :param coin: Which coin are we printing the price / value for
+    :param show_coin_name: Return string in format: "222.22 USDT" or "222.22"
+    :param keep_trailing_zeros: Keep trailing zeros "222.200" vs. "222.2"
+    :return: Formatted / rounded value (with or without coin name)
+    """
+    val = f"{value:.{decimals_per_coin(coin)}f}"
+    val = round_value(value, decimals_per_coin(coin), keep_trailing_zeros)
+    if show_coin_name:
+        val = f"{val} {coin}"
+
+    return val

freqtrade/util/migrations/__init__.py

@@ -0,0 +1,12 @@
+from typing import Optional
+
+from freqtrade.exchange import Exchange
+from freqtrade.util.migrations.binance_mig import migrate_binance_futures_names  # noqa F401
+from freqtrade.util.migrations.binance_mig import migrate_binance_futures_data
+from freqtrade.util.migrations.funding_rate_mig import migrate_funding_fee_timeframe
+
+
+def migrate_data(config, exchange: Optional[Exchange] = None):
+    migrate_binance_futures_data(config)
+
+    migrate_funding_fee_timeframe(config, exchange)

freqtrade/util/migrations/funding_rate_mig.py

@@ -0,0 +1,27 @@
+import logging
+from typing import Optional
+
+from freqtrade.constants import Config
+from freqtrade.data.history.idatahandler import get_datahandler
+from freqtrade.enums import TradingMode
+from freqtrade.exchange import Exchange
+
+
+logger = logging.getLogger(__name__)
+
+
+def migrate_funding_fee_timeframe(config: Config, exchange: Optional[Exchange]):
+    if (
+        config.get('trading_mode', TradingMode.SPOT) != TradingMode.FUTURES
+    ):
+        # only act on futures
+        return
+
+    if not exchange:
+        from freqtrade.resolvers import ExchangeResolver
+        exchange = ExchangeResolver.load_exchange(config, validate=False)
+
+    ff_timeframe = exchange.get_option('funding_fee_timeframe')
+
+    dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv'])
+    dhc.fix_funding_fee_timeframe(ff_timeframe)

requirements-dev.txt

@@ -7,25 +7,25 @@
 -r docs/requirements-docs.txt
 
 coveralls==3.3.1
-ruff==0.1.9
+ruff==0.1.14
 mypy==1.8.0
 pre-commit==3.6.0
-pytest==7.4.3
+pytest==7.4.4
-pytest-asyncio==0.21.1
+pytest-asyncio==0.23.4
 pytest-cov==4.1.0
 pytest-mock==3.12.0
-pytest-random-order==1.1.0
+pytest-random-order==1.1.1
 pytest-xdist==3.5.0
 isort==5.13.2
 # For datetime mocking
 time-machine==2.13.0
 
 # Convert jupyter notebooks to markdown documents
-nbconvert==7.13.1
+nbconvert==7.14.2
 
 # mypy types
 types-cachetools==5.3.0.7
 types-filelock==3.2.7
-types-requests==2.31.0.10
+types-requests==2.31.0.20240125
-types-tabulate==0.9.0.3
+types-tabulate==0.9.0.20240106
-types-python-dateutil==2.8.19.14
+types-python-dateutil==2.8.19.20240106

requirements-freqai-rl.txt

@@ -2,10 +2,10 @@
 -r requirements-freqai.txt
 
 # Required for freqai-rl
-torch==2.1.2
+torch==2.1.2; python_version < '3.12'
 #until these branches will be released we can use this
-gymnasium==0.29.1
+gymnasium==0.29.1; python_version < '3.12'
-stable_baselines3==2.2.1
+stable_baselines3==2.2.1; python_version < '3.12'
-sb3_contrib>=2.0.0a9
+sb3_contrib>=2.0.0a9; python_version < '3.12'
 # Progress bar for stable-baselines3 and sb3-contrib
 tqdm==4.66.1

requirements-freqai.txt

@@ -3,9 +3,9 @@
 -r requirements-plot.txt
 
 # Required for freqai
-scikit-learn==1.3.2
+scikit-learn==1.4.0
 joblib==1.3.2
-catboost==1.2.2; 'arm' not in platform_machine
+catboost==1.2.2; 'arm' not in platform_machine and python_version < '3.12'
 lightgbm==4.2.0
 xgboost==2.0.3
 tensorboard==2.15.1

requirements-hyperopt.txt

@@ -2,7 +2,7 @@
 -r requirements.txt
 
 # Required for hyperopt
-scipy==1.11.4
+scipy==1.12.0
-scikit-learn==1.3.2
+scikit-learn==1.4.0
 ft-scikit-optimize==0.9.2
 filelock==3.13.1

requirements.txt

@@ -1,11 +1,11 @@
-numpy==1.26.2
+numpy==1.26.3
 pandas==2.1.4
 pandas-ta==0.3.14b
 
-ccxt==4.1.98
+ccxt==4.2.25
-cryptography==41.0.7
+cryptography==42.0.1
-aiohttp==3.9.1
+aiohttp==3.9.2
-SQLAlchemy==2.0.23
+SQLAlchemy==2.0.25
 python-telegram-bot==20.7
 # can't be hard-pinned due to telegram-bot pinning httpx with ~
 httpx>=0.24.1
@@ -13,16 +13,16 @@ arrow==1.3.0
 cachetools==5.3.2
 requests==2.31.0
 urllib3==2.1.0
-jsonschema==4.20.0
+jsonschema==4.21.1
 TA-Lib==0.4.28
 technical==1.4.2
 tabulate==0.9.0
 pycoingecko==3.1.0
-jinja2==3.1.2
+jinja2==3.1.3
 tables==3.9.1
 joblib==1.3.2
 rich==13.7.0
-pyarrow==14.0.2; platform_machine != 'armv7l'
+pyarrow==15.0.0; platform_machine != 'armv7l'
 
 # find first, C search in arrays
 py_find_1st==1.1.6
@@ -30,18 +30,18 @@ py_find_1st==1.1.6
 # Load ticker files 30% faster
 python-rapidjson==1.14
 # Properly format api responses
-orjson==3.9.10
+orjson==3.9.12
 
 # Notify systemd
 sdnotify==0.3.2
 
 # API Server
-fastapi==0.105.0
+fastapi==0.109.0
 pydantic==2.5.3
-uvicorn==0.25.0
+uvicorn==0.27.0
 pyjwt==2.8.0
 aiofiles==23.2.1
-psutil==5.9.7
+psutil==5.9.8
 
 # Support for colorized terminal output
 colorama==0.4.6
@@ -58,5 +58,5 @@ schedule==1.2.1
 websockets==12.0
 janus==1.0.0
 
-ast-comments==1.2.0
+ast-comments==1.2.1
 packaging==23.2

setup.py

@@ -70,7 +70,7 @@ setup(
     ],
     install_requires=[
         # from requirements.txt
-        'ccxt>=4.0.0',
+        'ccxt>=4.2.15',
         'SQLAlchemy>=2.0.6',
         'python-telegram-bot>=20.1',
         'arrow>=1.0.0',

tests/commands/test_commands.py

@@ -772,7 +772,7 @@ def test_download_data_all_pairs(mocker, markets):
     pargs = get_args(args)
     pargs['config'] = None
     start_download_data(pargs)
-    expected = set(['ETH/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT'])
+    expected = set(['BTC/USDT', 'ETH/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT'])
     assert set(dl_mock.call_args_list[0][1]['pairs']) == expected
     assert dl_mock.call_count == 1
 
@@ -788,7 +788,7 @@ def test_download_data_all_pairs(mocker, markets):
     pargs = get_args(args)
     pargs['config'] = None
     start_download_data(pargs)
-    expected = set(['ETH/USDT', 'LTC/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT'])
+    expected = set(['BTC/USDT', 'ETH/USDT', 'LTC/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT'])
     assert set(dl_mock.call_args_list[0][1]['pairs']) == expected
 
 
@@ -1445,12 +1445,13 @@ def test_start_list_data(testdatadir, capsys):
     start_list_data(pargs)
     captured = capsys.readouterr()
     assert "Found 2 pair / timeframe combinations." in captured.out
-    assert ("\n|    Pair |   Timeframe |   Type |                From |                  To |\n"
+    assert (
-            in captured.out)
+        "\n|    Pair |   Timeframe |   Type "
+        "|                From |                  To |   Candles |\n") in captured.out
     assert "UNITTEST/BTC" not in captured.out
     assert (
-            "\n| XRP/ETH |          1m |   spot | 2019-10-11 00:00:00 | 2019-10-13 11:19:00 |\n"
+        "\n| XRP/ETH |          1m |   spot | "
-            in captured.out)
+        "2019-10-11 00:00:00 | 2019-10-13 11:19:00 |      2469 |\n") in captured.out
 
 
 @pytest.mark.usefixtures("init_persistence")
@@ -1508,7 +1509,7 @@ def test_backtesting_show(mocker, testdatadir, capsys):
     pargs['config'] = None
     start_backtesting_show(pargs)
     assert sbr.call_count == 1
-    out, err = capsys.readouterr()
+    out, _err = capsys.readouterr()
     assert "Pairs for Strategy" in out
 
 

tests/conftest.py

@@ -3,7 +3,7 @@ import json
 import logging
 import re
 from copy import deepcopy
-from datetime import timedelta
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Optional
 from unittest.mock import MagicMock, Mock, PropertyMock
@@ -18,13 +18,11 @@ from freqtrade.commands import Arguments
 from freqtrade.data.converter import ohlcv_to_dataframe, trades_list_to_df
 from freqtrade.edge import PairInfo
 from freqtrade.enums import CandleType, MarginMode, RunMode, SignalDirection, TradingMode
-from freqtrade.exchange import Exchange
+from freqtrade.exchange import Exchange, timeframe_to_minutes, timeframe_to_seconds
-from freqtrade.exchange.exchange import timeframe_to_minutes
 from freqtrade.freqtradebot import FreqtradeBot
 from freqtrade.persistence import LocalTrade, Order, Trade, init_db
 from freqtrade.resolvers import ExchangeResolver
-from freqtrade.util import dt_ts
+from freqtrade.util import dt_now, dt_ts
-from freqtrade.util.datetime_helpers import dt_now
 from freqtrade.worker import Worker
 from tests.conftest_trades import (leverage_trade, mock_trade_1, mock_trade_2, mock_trade_3,
                                    mock_trade_4, mock_trade_5, mock_trade_6, short_trade)
@@ -107,17 +105,62 @@ def get_args(args):
     return Arguments(args).get_parsed_arg()
 
 
+def generate_trades_history(n_rows, start_date: Optional[datetime] = None, days=5):
+    np.random.seed(42)
+    if not start_date:
+        start_date = datetime(2020, 1, 1, tzinfo=timezone.utc)
+
+        # Generate random data
+    end_date = start_date + timedelta(days=days)
+    _start_timestamp = start_date.timestamp()
+    _end_timestamp = pd.to_datetime(end_date).timestamp()
+
+    random_timestamps_in_seconds = np.random.uniform(_start_timestamp, _end_timestamp, n_rows)
+    timestamp = pd.to_datetime(random_timestamps_in_seconds, unit='s')
+
+    id = [
+        f'a{np.random.randint(1e6, 1e7 - 1)}cd{np.random.randint(100, 999)}'
+        for _ in range(n_rows)
+    ]
+
+    side = np.random.choice(['buy', 'sell'], n_rows)
+
+    # Initial price and subsequent changes
+    initial_price = 0.019626
+    price_changes = np.random.normal(0, initial_price * 0.05, n_rows)
+    price = np.cumsum(np.concatenate(([initial_price], price_changes)))[:n_rows]
+
+    amount = np.random.uniform(0.011, 20, n_rows)
+    cost = price * amount
+
+    # Create DataFrame
+    df = pd.DataFrame({'timestamp': timestamp, 'id': id, 'type': None, 'side': side,
+                       'price': price, 'amount': amount, 'cost': cost})
+    df['date'] = pd.to_datetime(df['timestamp'], unit='ms', utc=True)
+    df = df.sort_values('timestamp').reset_index(drop=True)
+    assert list(df.columns) == constants.DEFAULT_TRADES_COLUMNS + ['date']
+    return df
+
+
 def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05'):
     np.random.seed(42)
 
     base = np.random.normal(20, 2, size=size)
-    if timeframe == '1M':
+    if timeframe == '1y':
+        date = pd.date_range(start, periods=size, freq='1YS', tz='UTC')
+    elif timeframe == '1M':
         date = pd.date_range(start, periods=size, freq='1MS', tz='UTC')
-    elif timeframe == '1w':
+    elif timeframe == '3M':
+        date = pd.date_range(start, periods=size, freq='3MS', tz='UTC')
+    elif timeframe == '1w' or timeframe == '7d':
         date = pd.date_range(start, periods=size, freq='1W-MON', tz='UTC')
     else:
         tf_mins = timeframe_to_minutes(timeframe)
+        if tf_mins >= 1:
             date = pd.date_range(start, periods=size, freq=f'{tf_mins}min', tz='UTC')
+        else:
+            tf_secs = timeframe_to_seconds(timeframe)
+            date = pd.date_range(start, periods=size, freq=f'{tf_secs}s', tz='UTC')
     df = pd.DataFrame({
         'date': date,
         'open': base,
@@ -553,6 +596,7 @@ def get_default_conf(testdatadir):
         "internals": {},
         "export": "none",
         "dataformat_ohlcv": "feather",
+        "runmode": "dry_run",
         "candle_type_def": CandleType.SPOT,
     }
     return configuration
@@ -961,6 +1005,58 @@ def get_markets():
                 'maintenance_rate': '0.005',
             },
         },
+        'BTC/USDT': {
+            'id': 'USDT-BTC',
+            'symbol': 'BTC/USDT',
+            'base': 'BTC',
+            'quote': 'USDT',
+            'settle': None,
+            'baseId': 'BTC',
+            'quoteId': 'USDT',
+            'settleId': None,
+            'type': 'spot',
+            'spot': True,
+            'margin': True,
+            'swap': False,
+            'future': False,
+            'option': False,
+            'active': True,
+            'contract': None,
+            'linear': None,
+            'inverse': None,
+            'taker': 0.0006,
+            'maker': 0.0002,
+            'contractSize': None,
+            'expiry': None,
+            'expiryDateTime': None,
+            'strike': None,
+            'optionType': None,
+            'precision': {
+                'amount': 4,
+                'price': 4,
+            },
+            'limits': {
+                'leverage': {
+                    'min': 1,
+                    'max': 100,
+                },
+                'amount': {
+                    'min': 0.000221,
+                    'max': None,
+                },
+                'price': {
+                    'min': 1e-02,
+                    'max': None,
+                },
+                'cost': {
+                    'min': None,
+                    'max': None,
+                },
+            },
+            'info': {
+                'maintenance_rate': '0.005',
+            },
+        },
         'LTC/USDT': {
             'id': 'USDT-LTC',
             'symbol': 'LTC/USDT',
@@ -2385,14 +2481,7 @@ def trades_history_df(trades_history):
 
 @pytest.fixture(scope="function")
 def fetch_trades_result():
-    return [{'info': {'a': 126181329,
+    return [{'info': ['0.01962700', '0.04000000', '1565798399.4631551', 'b', 'm', '', '126181329'],
-                      'p': '0.01962700',
-                      'q': '0.04000000',
-                      'f': 138604155,
-                      'l': 138604155,
-                      'T': 1565798399463,
-                      'm': False,
-                      'M': True},
              'timestamp': 1565798399463,
              'datetime': '2019-08-14T15:59:59.463Z',
              'symbol': 'ETH/BTC',
@@ -2405,14 +2494,7 @@ def fetch_trades_result():
              'amount': 0.04,
              'cost': 0.00078508,
              'fee': None},
-            {'info': {'a': 126181330,
+            {'info': ['0.01962700', '0.24400000', '1565798399.6291551', 'b', 'm', '', '126181330'],
-                      'p': '0.01962700',
-                      'q': '0.24400000',
-                      'f': 138604156,
-                      'l': 138604156,
-                      'T': 1565798399629,
-                      'm': False,
-                      'M': True},
              'timestamp': 1565798399629,
              'datetime': '2019-08-14T15:59:59.629Z',
              'symbol': 'ETH/BTC',
@@ -2425,14 +2507,7 @@ def fetch_trades_result():
              'amount': 0.244,
              'cost': 0.004788987999999999,
              'fee': None},
-            {'info': {'a': 126181331,
+            {'info': ['0.01962600', '0.01100000', '1565798399.7521551', 's', 'm', '', '126181331'],
-                      'p': '0.01962600',
-                      'q': '0.01100000',
-                      'f': 138604157,
-                      'l': 138604157,
-                      'T': 1565798399752,
-                      'm': True,
-                      'M': True},
              'timestamp': 1565798399752,
              'datetime': '2019-08-14T15:59:59.752Z',
              'symbol': 'ETH/BTC',
@@ -2445,14 +2520,7 @@ def fetch_trades_result():
              'amount': 0.011,
              'cost': 0.00021588599999999999,
              'fee': None},
-            {'info': {'a': 126181332,
+            {'info': ['0.01962600', '0.01100000', '1565798399.8621551', 's', 'm', '', '126181332'],
-                      'p': '0.01962600',
-                      'q': '0.01100000',
-                      'f': 138604158,
-                      'l': 138604158,
-                      'T': 1565798399862,
-                      'm': True,
-                      'M': True},
              'timestamp': 1565798399862,
              'datetime': '2019-08-14T15:59:59.862Z',
              'symbol': 'ETH/BTC',
@@ -2465,14 +2533,8 @@ def fetch_trades_result():
              'amount': 0.011,
              'cost': 0.00021588599999999999,
              'fee': None},
-            {'info': {'a': 126181333,
+            {'info': ['0.01952600', '0.01200000', '1565798399.8721551', 's', 'm', '', '126181333',
-                      'p': '0.01952600',
+                      1565798399872512133],
-                      'q': '0.01200000',
-                      'f': 138604158,
-                      'l': 138604158,
-                      'T': 1565798399872,
-                      'm': True,
-                      'M': True},
              'timestamp': 1565798399872,
              'datetime': '2019-08-14T15:59:59.872Z',
              'symbol': 'ETH/BTC',

tests/data/test_converter.py

@@ -17,7 +17,8 @@ from freqtrade.data.history import (get_timerange, load_data, load_pair_history,
                                     validate_backtest_data)
 from freqtrade.data.history.idatahandler import IDataHandler
 from freqtrade.enums import CandleType
-from tests.conftest import generate_test_data, log_has, log_has_re
+from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
+from tests.conftest import generate_test_data, generate_trades_history, log_has, log_has_re
 from tests.data.test_history import _clean_test_file
 
 
@@ -51,6 +52,49 @@ def test_trades_to_ohlcv(trades_history_df, caplog):
     assert 'close' in df.columns
     assert df.iloc[0, :]['high'] == 0.019627
     assert df.iloc[0, :]['low'] == 0.019626
+    assert df.iloc[0, :]['date'] == pd.Timestamp('2019-08-14 15:59:00+0000')
+
+    df_1h = trades_to_ohlcv(trades_history_df, '1h')
+    assert len(df_1h) == 1
+    assert df_1h.iloc[0, :]['high'] == 0.019627
+    assert df_1h.iloc[0, :]['low'] == 0.019626
+    assert df_1h.iloc[0, :]['date'] == pd.Timestamp('2019-08-14 15:00:00+0000')
+
+    df_1s = trades_to_ohlcv(trades_history_df, '1s')
+    assert len(df_1s) == 2
+    assert df_1s.iloc[0, :]['high'] == 0.019627
+    assert df_1s.iloc[0, :]['low'] == 0.019627
+    assert df_1s.iloc[0, :]['date'] == pd.Timestamp('2019-08-14 15:59:49+0000')
+    assert df_1s.iloc[-1, :]['date'] == pd.Timestamp('2019-08-14 15:59:59+0000')
+
+
+@pytest.mark.parametrize('timeframe,rows,days,candles,start,end,weekday', [
+    ('1s', 20_000, 5, 19522, '2020-01-01 00:00:05', '2020-01-05 23:59:27', None),
+    ('1m', 20_000, 5, 6745, '2020-01-01 00:00:00', '2020-01-05 23:59:00', None),
+    ('5m', 20_000, 5, 1440, '2020-01-01 00:00:00', '2020-01-05 23:55:00', None),
+    ('15m', 20_000, 5, 480, '2020-01-01 00:00:00', '2020-01-05 23:45:00', None),
+    ('1h', 20_000, 5, 120, '2020-01-01 00:00:00', '2020-01-05 23:00:00', None),
+    ('2h', 20_000, 5, 60, '2020-01-01 00:00:00', '2020-01-05 22:00:00', None),
+    ('4h', 20_000, 5, 30, '2020-01-01 00:00:00', '2020-01-05 20:00:00', None),
+    ('8h', 20_000, 5, 15, '2020-01-01 00:00:00', '2020-01-05 16:00:00', None),
+    ('12h', 20_000, 5, 10, '2020-01-01 00:00:00', '2020-01-05 12:00:00', None),
+    ('1d', 20_000, 5, 5, '2020-01-01 00:00:00', '2020-01-05 00:00:00', 'Sunday'),
+    ('7d', 20_000, 37, 6, '2020-01-06 00:00:00', '2020-02-10 00:00:00', 'Monday'),
+    ('1w', 20_000, 37, 6, '2020-01-06 00:00:00', '2020-02-10 00:00:00', 'Monday'),
+    ('1M', 20_000, 74, 3, '2020-01-01 00:00:00', '2020-03-01 00:00:00', None),
+    ('3M', 20_000, 100, 2, '2020-01-01 00:00:00', '2020-04-01 00:00:00', None),
+    ('1y', 20_000, 1000, 3, '2020-01-01 00:00:00', '2022-01-01 00:00:00', None),
+])
+def test_trades_to_ohlcv_multi(timeframe, rows, days, candles, start, end, weekday):
+    trades_history = generate_trades_history(n_rows=rows, days=days)
+    df = trades_to_ohlcv(trades_history, timeframe)
+    assert not df.empty
+    assert len(df) == candles
+    assert df.iloc[0, :]['date'] == pd.Timestamp(f'{start}+0000')
+    assert df.iloc[-1, :]['date'] == pd.Timestamp(f'{end}+0000')
+    if weekday:
+        # Weekday is only relevant for daily and weekly candles.
+        assert df.iloc[-1, :]['date'].day_name() == weekday
 
 
 def test_ohlcv_fill_up_missing_data(testdatadir, caplog):
@@ -132,6 +176,45 @@ def test_ohlcv_fill_up_missing_data2(caplog):
                       f"{len(data)} - after: {len(data2)}.*", caplog)
 
 
+@pytest.mark.parametrize('timeframe', [
+    '1s', '1m', '5m', '15m', '1h', '2h', '4h', '8h', '12h', '1d', '7d', '1w', '1M', '3M', '1y'
+])
+def test_ohlcv_to_dataframe_multi(timeframe):
+    data = generate_test_data(timeframe, 180)
+    assert len(data) == 180
+    df = ohlcv_to_dataframe(data, timeframe, 'UNITTEST/USDT')
+    assert len(df) == len(data) - 1
+    df1 = ohlcv_to_dataframe(data, timeframe, 'UNITTEST/USDT', drop_incomplete=False)
+    assert len(df1) == len(data)
+    assert data.equals(df1)
+
+    data1 = data.copy()
+    if timeframe in ('1M', '3M', '1y'):
+        data1.loc[:, 'date'] = data1.loc[:, 'date'] + pd.to_timedelta('1w')
+    else:
+        # Shift by half a timeframe
+        data1.loc[:, 'date'] = data1.loc[:, 'date'] + (pd.to_timedelta(timeframe) / 2)
+    df2 = ohlcv_to_dataframe(data1, timeframe, 'UNITTEST/USDT')
+
+    assert len(df2) == len(data) - 1
+    tfs = timeframe_to_seconds(timeframe)
+    tfm = timeframe_to_minutes(timeframe)
+    if 1 <= tfm < 10000:
+        # minute based resampling does not work on timeframes >= 1 week
+        ohlcv_dict = {
+            'open': 'first',
+            'high': 'max',
+            'low': 'min',
+            'close': 'last',
+            'volume': 'sum'
+        }
+        dfs = data1.resample(f"{tfs}s", on='date').agg(ohlcv_dict).reset_index(drop=False)
+        dfm = data1.resample(f"{tfm}min", on='date').agg(ohlcv_dict).reset_index(drop=False)
+
+        assert dfs.equals(dfm)
+        assert dfs.equals(df1)
+
+
 def test_ohlcv_to_dataframe_1M():
 
     # Monthly ticks from 2019-09-01 to 2023-07-01

tests/data/test_datahandler.py

@@ -148,19 +148,25 @@ def test_jsondatahandler_ohlcv_load(testdatadir, caplog):
 def test_datahandler_ohlcv_data_min_max(testdatadir):
     dh = JsonDataHandler(testdatadir)
     min_max = dh.ohlcv_data_min_max('UNITTEST/BTC', '5m', 'spot')
-    assert len(min_max) == 2
+    assert len(min_max) == 3
 
     # Empty pair
     min_max = dh.ohlcv_data_min_max('UNITTEST/BTC', '8m', 'spot')
-    assert len(min_max) == 2
+    assert len(min_max) == 3
     assert min_max[0] == datetime.fromtimestamp(0, tz=timezone.utc)
     assert min_max[0] == min_max[1]
     # Empty pair2
-    min_max = dh.ohlcv_data_min_max('NOPAIR/XXX', '4m', 'spot')
+    min_max = dh.ohlcv_data_min_max('NOPAIR/XXX', '41m', 'spot')
-    assert len(min_max) == 2
+    assert len(min_max) == 3
     assert min_max[0] == datetime.fromtimestamp(0, tz=timezone.utc)
     assert min_max[0] == min_max[1]
 
+    # Existing pair ...
+    min_max = dh.ohlcv_data_min_max('UNITTEST/BTC', '1m', 'spot')
+    assert len(min_max) == 3
+    assert min_max[0] == datetime(2017, 11, 4, 23, 2, tzinfo=timezone.utc)
+    assert min_max[1] == datetime(2017, 11, 14, 22, 59, tzinfo=timezone.utc)
+
 
 def test_datahandler__check_empty_df(testdatadir, caplog):
     dh = JsonDataHandler(testdatadir)

tests/data/test_dataprovider.py

@@ -194,7 +194,7 @@ def test_get_producer_df(default_conf):
     assert la == empty_la
 
     # non existent timeframe, empty dataframe
-    datframe, la = dataprovider.get_producer_df(pair, timeframe='1h')
+    _dataframe, la = dataprovider.get_producer_df(pair, timeframe='1h')
     assert dataframe.empty
     assert la == empty_la
 
@@ -508,16 +508,13 @@ def test_dp_get_required_startup(default_conf_usdt):
     dp = DataProvider(default_conf_usdt, None)
 
     # No FreqAI config
-    assert dp.get_required_startup('5m', False) == 0
+    assert dp.get_required_startup('5m') == 0
-    assert dp.get_required_startup('1h', False) == 0
+    assert dp.get_required_startup('1h') == 0
-    assert dp.get_required_startup('1d', False) == 0
-    assert dp.get_required_startup('1d', True) == 0
     assert dp.get_required_startup('1d') == 0
 
     dp._config['startup_candle_count'] = 20
-    assert dp.get_required_startup('5m', False) == 20
+    assert dp.get_required_startup('5m') == 20
-    assert dp.get_required_startup('5m', True) == 20
+    assert dp.get_required_startup('1h') == 20
-    assert dp.get_required_startup('1h', False) == 20
     assert dp.get_required_startup('1h') == 20
 
     # With freqAI config
@@ -532,37 +529,19 @@ def test_dp_get_required_startup(default_conf_usdt):
             ]
         }
     }
-    assert dp.get_required_startup('5m', False) == 20
+    assert dp.get_required_startup('5m') == 5780
-    assert dp.get_required_startup('5m', True) == 5780
+    assert dp.get_required_startup('1h') == 500
-
-    assert dp.get_required_startup('1h', False) == 20
-    assert dp.get_required_startup('1h', True) == 500
-
-    assert dp.get_required_startup('1d', False) == 20
-    assert dp.get_required_startup('1d', True) == 40
     assert dp.get_required_startup('1d') == 40
 
     # FreqAI kindof ignores startup_candle_count if it's below indicator_periods_candles
     dp._config['startup_candle_count'] = 0
-    assert dp.get_required_startup('5m', False) == 20
+    assert dp.get_required_startup('5m') == 5780
-    assert dp.get_required_startup('5m', True) == 5780
+    assert dp.get_required_startup('1h') == 500
-
-    assert dp.get_required_startup('1h', False) == 20
-    assert dp.get_required_startup('1h', True) == 500
-
-    assert dp.get_required_startup('1d', False) == 20
-    assert dp.get_required_startup('1d', True) == 40
     assert dp.get_required_startup('1d') == 40
 
     dp._config['freqai']['feature_parameters']['indicator_periods_candles'][1] = 50
-    assert dp.get_required_startup('5m', False) == 50
+    assert dp.get_required_startup('5m') == 5810
-    assert dp.get_required_startup('5m', True) == 5810
+    assert dp.get_required_startup('1h') == 530
-
-    assert dp.get_required_startup('1h', False) == 50
-    assert dp.get_required_startup('1h', True) == 530
-
-    assert dp.get_required_startup('1d', False) == 50
-    assert dp.get_required_startup('1d', True) == 70
     assert dp.get_required_startup('1d') == 70
 
     # scenario from issue https://github.com/freqtrade/freqtrade/issues/9432
@@ -577,12 +556,6 @@ def test_dp_get_required_startup(default_conf_usdt):
         }
     }
     dp._config['startup_candle_count'] = 40
-    assert dp.get_required_startup('5m', False) == 40
+    assert dp.get_required_startup('5m') == 51880
-    assert dp.get_required_startup('5m', True) == 51880
+    assert dp.get_required_startup('1h') == 4360
-
-    assert dp.get_required_startup('1h', False) == 40
-    assert dp.get_required_startup('1h', True) == 4360
-
-    assert dp.get_required_startup('1d', False) == 40
-    assert dp.get_required_startup('1d', True) == 220
     assert dp.get_required_startup('1d') == 220

tests/data/test_download_data.py

@@ -38,7 +38,7 @@ def test_download_data_main_all_pairs(mocker, markets):
         "timeframes": ["5m", "1h"]
     })
     download_data_main(config)
-    expected = set(['ETH/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT'])
+    expected = set(['BTC/USDT', 'ETH/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT'])
     assert set(dl_mock.call_args_list[0][1]['pairs']) == expected
     assert dl_mock.call_count == 1
 
@@ -50,7 +50,7 @@ def test_download_data_main_all_pairs(mocker, markets):
         "include_inactive": True
     })
     download_data_main(config)
-    expected = set(['ETH/USDT', 'LTC/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT'])
+    expected = set(['BTC/USDT', 'ETH/USDT', 'LTC/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT'])
     assert set(dl_mock.call_args_list[0][1]['pairs']) == expected
 
 

tests/data/test_history.py

@@ -508,8 +508,9 @@ def test_refresh_backtest_ohlcv_data(
 
     mocker.patch.object(Path, "exists", MagicMock(return_value=True))
     mocker.patch.object(Path, "unlink", MagicMock())
+    default_conf['trading_mode'] = trademode
 
-    ex = get_patched_exchange(mocker, default_conf)
+    ex = get_patched_exchange(mocker, default_conf, id='bybit')
     timerange = TimeRange.parse_timerange("20190101-20190102")
     refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"],
                                 timeframes=["1m", "5m"], datadir=testdatadir,
@@ -521,6 +522,9 @@ def test_refresh_backtest_ohlcv_data(
     assert dl_mock.call_args[1]['timerange'].starttype == 'date'
 
     assert log_has_re(r"Downloading pair ETH/BTC, .* interval 1m\.", caplog)
+    if trademode == 'futures':
+        assert log_has_re(r"Downloading pair ETH/BTC, funding_rate, interval 8h\.", caplog)
+        assert log_has_re(r"Downloading pair ETH/BTC, mark, interval 4h\.", caplog)
 
 
 def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):

tests/exchange/test_exchange.py

@@ -55,7 +55,7 @@ get_entry_rate_data = [
     ('bid', 6, 5, None, 0, 5),  # last not available - uses bid
 ]
 
-get_sell_rate_data = [
+get_exit_rate_data = [
     ('bid', 12.0, 11.0, 11.5, 0.0, 11.0),  # full bid side
     ('bid', 12.0, 11.0, 11.5, 1.0, 11.5),  # full last side
     ('bid', 12.0, 11.0, 11.5, 0.5, 11.25),  # between bid and lat
@@ -2512,8 +2512,10 @@ def test_fetch_l2_order_book_exception(default_conf, mocker, exchange_name):
 
 @pytest.mark.parametrize("side,ask,bid,last,last_ab,expected", get_entry_rate_data)
 def test_get_entry_rate(mocker, default_conf, caplog, side, ask, bid,
-                        last, last_ab, expected) -> None:
+                        last, last_ab, expected, time_machine) -> None:
     caplog.set_level(logging.DEBUG)
+    start_dt = datetime(2023, 12, 1, 0, 10, 0, tzinfo=timezone.utc)
+    time_machine.move_to(start_dt, tick=False)
     if last_ab is None:
         del default_conf['entry_pricing']['price_last_balance']
     else:
@@ -2521,39 +2523,65 @@ def test_get_entry_rate(mocker, default_conf, caplog, side, ask, bid,
     default_conf['entry_pricing']['price_side'] = side
     exchange = get_patched_exchange(mocker, default_conf)
     mocker.patch(f'{EXMS}.fetch_ticker', return_value={'ask': ask, 'last': last, 'bid': bid})
+    log_msg = "Using cached entry rate for ETH/BTC."
 
     assert exchange.get_rate('ETH/BTC', side="entry", is_short=False, refresh=True) == expected
-    assert not log_has("Using cached entry rate for ETH/BTC.", caplog)
+    assert not log_has(log_msg, caplog)
 
+    time_machine.move_to(start_dt + timedelta(minutes=4), tick=False)
+    # Running a 2nd time without Refresh!
+    caplog.clear()
     assert exchange.get_rate('ETH/BTC', side="entry", is_short=False, refresh=False) == expected
-    assert log_has("Using cached entry rate for ETH/BTC.", caplog)
+    assert log_has(log_msg, caplog)
+
+    time_machine.move_to(start_dt + timedelta(minutes=6), tick=False)
+    # Running a 2nd time - forces refresh due to ttl timeout
+    caplog.clear()
+    assert exchange.get_rate('ETH/BTC', side="entry", is_short=False, refresh=False) == expected
+    assert not log_has(log_msg, caplog)
+
     # Running a 2nd time with Refresh on!
     caplog.clear()
     assert exchange.get_rate('ETH/BTC', side="entry", is_short=False, refresh=True) == expected
-    assert not log_has("Using cached entry rate for ETH/BTC.", caplog)
+    assert not log_has(log_msg, caplog)
 
 
-@pytest.mark.parametrize('side,ask,bid,last,last_ab,expected', get_sell_rate_data)
+@pytest.mark.parametrize('side,ask,bid,last,last_ab,expected', get_exit_rate_data)
 def test_get_exit_rate(default_conf, mocker, caplog, side, bid, ask,
-                       last, last_ab, expected) -> None:
+                       last, last_ab, expected, time_machine) -> None:
     caplog.set_level(logging.DEBUG)
+    start_dt = datetime(2023, 12, 1, 0, 10, 0, tzinfo=timezone.utc)
+    time_machine.move_to(start_dt, tick=False)
 
     default_conf['exit_pricing']['price_side'] = side
     if last_ab is not None:
         default_conf['exit_pricing']['price_last_balance'] = last_ab
     mocker.patch(f'{EXMS}.fetch_ticker', return_value={'ask': ask, 'bid': bid, 'last': last})
     pair = "ETH/BTC"
+    log_msg = "Using cached exit rate for ETH/BTC."
 
     # Test regular mode
     exchange = get_patched_exchange(mocker, default_conf)
     rate = exchange.get_rate(pair, side="exit", is_short=False, refresh=True)
-    assert not log_has("Using cached exit rate for ETH/BTC.", caplog)
+    assert not log_has(log_msg, caplog)
     assert isinstance(rate, float)
     assert rate == expected
     # Use caching
-    rate = exchange.get_rate(pair, side="exit", is_short=False, refresh=False)
+    caplog.clear()
-    assert rate == expected
+    assert exchange.get_rate(pair, side="exit", is_short=False, refresh=False) == expected
-    assert log_has("Using cached exit rate for ETH/BTC.", caplog)
+    assert log_has(log_msg, caplog)
+
+    time_machine.move_to(start_dt + timedelta(minutes=4), tick=False)
+    # Caching still active - TTL didn't expire
+    caplog.clear()
+    assert exchange.get_rate(pair, side="exit", is_short=False, refresh=False) == expected
+    assert log_has(log_msg, caplog)
+
+    time_machine.move_to(start_dt + timedelta(minutes=6), tick=False)
+    # Caching expired - refresh forced
+    caplog.clear()
+    assert exchange.get_rate(pair, side="exit", is_short=False, refresh=False) == expected
+    assert not log_has(log_msg, caplog)
 
 
 @pytest.mark.parametrize("entry,is_short,side,ask,bid,last,last_ab,expected", [
@@ -2649,7 +2677,7 @@ def test_get_exit_rate_exception(default_conf, mocker, is_short):
 @pytest.mark.parametrize("side,ask,bid,last,last_ab,expected", get_entry_rate_data)
 @pytest.mark.parametrize("side2", ['bid', 'ask'])
 @pytest.mark.parametrize("use_order_book", [True, False])
-def test_get_rates_testing_buy(mocker, default_conf, caplog, side, ask, bid,
+def test_get_rates_testing_entry(mocker, default_conf, caplog, side, ask, bid,
                                  last, last_ab, expected,
                                  side2, use_order_book, order_book_l2) -> None:
     caplog.set_level(logging.DEBUG)
@@ -2685,10 +2713,10 @@ def test_get_rates_testing_buy(mocker, default_conf, caplog, side, ask, bid,
     assert api_mock.fetch_ticker.call_count == 1
 
 
-@pytest.mark.parametrize('side,ask,bid,last,last_ab,expected', get_sell_rate_data)
+@pytest.mark.parametrize('side,ask,bid,last,last_ab,expected', get_exit_rate_data)
 @pytest.mark.parametrize("side2", ['bid', 'ask'])
 @pytest.mark.parametrize("use_order_book", [True, False])
-def test_get_rates_testing_sell(default_conf, mocker, caplog, side, bid, ask,
+def test_get_rates_testing_exit(default_conf, mocker, caplog, side, bid, ask,
                                 last, last_ab, expected,
                                 side2, use_order_book, order_book_l2) -> None:
     caplog.set_level(logging.DEBUG)
@@ -2816,10 +2844,17 @@ async def test__async_fetch_trades(default_conf, mocker, caplog, exchange_name,
     exchange._api_async.fetch_trades = get_mock_coro(fetch_trades_result)
 
     pair = 'ETH/BTC'
-    res = await exchange._async_fetch_trades(pair, since=None, params=None)
+    res, pagid = await exchange._async_fetch_trades(pair, since=None, params=None)
     assert isinstance(res, list)
     assert isinstance(res[0], list)
     assert isinstance(res[1], list)
+    if exchange._trades_pagination == 'id':
+        if exchange_name == 'kraken':
+            assert pagid == 1565798399872512133
+        else:
+            assert pagid == '126181333'
+    else:
+        assert pagid == 1565798399872
 
     assert exchange._api_async.fetch_trades.call_count == 1
     assert exchange._api_async.fetch_trades.call_args[0][0] == pair
@@ -2828,11 +2863,20 @@ async def test__async_fetch_trades(default_conf, mocker, caplog, exchange_name,
     assert log_has_re(f"Fetching trades for pair {pair}, since .*", caplog)
     caplog.clear()
     exchange._api_async.fetch_trades.reset_mock()
-    res = await exchange._async_fetch_trades(pair, since=None, params={'from': '123'})
+    res, pagid = await exchange._async_fetch_trades(pair, since=None, params={'from': '123'})
     assert exchange._api_async.fetch_trades.call_count == 1
     assert exchange._api_async.fetch_trades.call_args[0][0] == pair
     assert exchange._api_async.fetch_trades.call_args[1]['limit'] == 1000
     assert exchange._api_async.fetch_trades.call_args[1]['params'] == {'from': '123'}
+
+    if exchange._trades_pagination == 'id':
+        if exchange_name == 'kraken':
+            assert pagid == 1565798399872512133
+        else:
+            assert pagid == '126181333'
+    else:
+        assert pagid == 1565798399872
+
     assert log_has_re(f"Fetching trades for pair {pair}, params: .*", caplog)
     exchange.close()
 
@@ -2887,8 +2931,9 @@ async def test__async_fetch_trades_contract_size(default_conf, mocker, caplog, e
     )
 
     pair = 'ETH/USDT:USDT'
-    res = await exchange._async_fetch_trades(pair, since=None, params=None)
+    res, pagid = await exchange._async_fetch_trades(pair, since=None, params=None)
     assert res[0][5] == 300
+    assert pagid is not None
     exchange.close()
 
 
@@ -2898,13 +2943,17 @@ async def test__async_get_trade_history_id(default_conf, mocker, exchange_name,
                                            fetch_trades_result):
 
     exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+    if exchange._trades_pagination != 'id':
+        exchange.close()
+        pytest.skip("Exchange does not support pagination by trade id")
     pagination_arg = exchange._trades_pagination_arg
 
     async def mock_get_trade_hist(pair, *args, **kwargs):
         if 'since' in kwargs:
             # Return first 3
             return fetch_trades_result[:-2]
-        elif kwargs.get('params', {}).get(pagination_arg) == fetch_trades_result[-3]['id']:
+        elif kwargs.get('params', {}).get(pagination_arg) in (
+                fetch_trades_result[-3]['id'], 1565798399752):
             # Return 2
             return fetch_trades_result[-3:-1]
         else:
@@ -2920,6 +2969,7 @@ async def test__async_get_trade_history_id(default_conf, mocker, exchange_name,
     assert isinstance(ret, tuple)
     assert ret[0] == pair
     assert isinstance(ret[1], list)
+    if exchange_name != 'kraken':
         assert len(ret[1]) == len(fetch_trades_result)
     assert exchange._api_async.fetch_trades.call_count == 3
     fetch_trades_cal = exchange._api_async.fetch_trades.call_args_list
@@ -2933,6 +2983,21 @@ async def test__async_get_trade_history_id(default_conf, mocker, exchange_name,
     assert exchange._ft_has['trades_pagination_arg'] in fetch_trades_cal[1][1]['params']
 
 
+@pytest.mark.parametrize('trade_id, expected', [
+    ('1234', True),
+    ('170544369512007228', True),
+    ('1705443695120072285', True),
+    ('170544369512007228555', True),
+])
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+def test__valid_trade_pagination_id(mocker, default_conf_usdt, exchange_name, trade_id, expected):
+    if exchange_name == 'kraken':
+        pytest.skip("Kraken has a different pagination id format, and an explicit test.")
+    exchange = get_patched_exchange(mocker, default_conf_usdt, id=exchange_name)
+
+    assert exchange._valid_trade_pagination_id('XRP/USDT', trade_id) == expected
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
 async def test__async_get_trade_history_time(default_conf, mocker, caplog, exchange_name,
@@ -2948,6 +3013,9 @@ async def test__async_get_trade_history_time(default_conf, mocker, caplog, excha
 
     caplog.set_level(logging.DEBUG)
     exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+    if exchange._trades_pagination != 'time':
+        exchange.close()
+        pytest.skip("Exchange does not support pagination by timestamp")
     # Monkey-patch async function
     exchange._api_async.fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
     pair = 'ETH/BTC'
@@ -2980,9 +3048,9 @@ async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog,
 
     async def mock_get_trade_hist(pair, *args, **kwargs):
         if kwargs['since'] == trades_history[0][0]:
-            return trades_history[:-1]
+            return trades_history[:-1], trades_history[:-1][-1][0]
         else:
-            return []
+            return [], None
 
     caplog.set_level(logging.DEBUG)
     exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
@@ -3194,7 +3262,7 @@ def test_cancel_stoploss_order_with_result(default_conf, mocker, exchange_name):
     mocker.patch(f'{mock_prefix}.fetch_stoploss_order', side_effect=exc)
     co = exchange.cancel_stoploss_order_with_result(order_id='_', pair='TKN/BTC', amount=555)
     assert co['amount'] == 555
-    assert co == {'fee': {}, 'status': 'canceled', 'amount': 555, 'info': {}}
+    assert co == {'id': '_', 'fee': {}, 'status': 'canceled', 'amount': 555, 'info': {}}
 
     with pytest.raises(InvalidOrderException):
         exc = InvalidOrderException("Did not find order")
@@ -5284,3 +5352,4 @@ def test_price_to_precision_with_default_conf(default_conf, mocker):
     patched_ex = get_patched_exchange(mocker, conf)
     prec_price = patched_ex.price_to_precision("XRP/USDT", 1.0000000101)
     assert prec_price == 1.00000001
+    assert prec_price == 1.00000001

tests/exchange/test_exchange_utils.py

@@ -10,7 +10,7 @@ from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import (amount_to_contract_precision, amount_to_precision,
                                 date_minus_candles, price_to_precision, timeframe_to_minutes,
                                 timeframe_to_msecs, timeframe_to_next_date, timeframe_to_prev_date,
-                                timeframe_to_seconds)
+                                timeframe_to_resample_freq, timeframe_to_seconds)
 from freqtrade.exchange.check_exchange import check_exchange
 from tests.conftest import log_has_re
 
@@ -124,6 +124,21 @@ def test_timeframe_to_msecs():
     assert timeframe_to_msecs("1d") == 86400000
 
 
+@pytest.mark.parametrize("timeframe,expected", [
+    ("1s", '1s'),
+    ("15s", '15s'),
+    ("5m", '300s'),
+    ("10m", '600s'),
+    ("1h", '3600s'),
+    ("1d", '86400s'),
+    ("1w", '1W-MON'),
+    ("1M", '1MS'),
+    ("1y", '1YS'),
+])
+def test_timeframe_to_resample_freq(timeframe, expected):
+    assert timeframe_to_resample_freq(timeframe) == expected
+
+
 def test_timeframe_to_prev_date():
     # 2019-08-12 13:22:08
     date = datetime.fromtimestamp(1565616128, tz=timezone.utc)

tests/exchange/test_htx.py

@@ -14,7 +14,7 @@ from tests.exchange.test_exchange import ccxt_exceptionhandlers
     (0.99, 220 * 0.99, "sell"),
     (0.98, 220 * 0.98, "sell"),
 ])
-def test_create_stoploss_order_huobi(default_conf, mocker, limitratio, expected, side):
+def test_create_stoploss_order_htx(default_conf, mocker, limitratio, expected, side):
     api_mock = MagicMock()
     order_id = f'test_prod_buy_{randint(0, 10 ** 6)}'
     order_type = 'stop-limit'
@@ -29,7 +29,7 @@ def test_create_stoploss_order_huobi(default_conf, mocker, limitratio, expected,
     mocker.patch(f'{EXMS}.amount_to_precision', lambda s, x, y: y)
     mocker.patch(f'{EXMS}.price_to_precision', lambda s, x, y, **kwargs: y)
 
-    exchange = get_patched_exchange(mocker, default_conf, api_mock, 'huobi')
+    exchange = get_patched_exchange(mocker, default_conf, api_mock, 'htx')
 
     with pytest.raises(InvalidOrderException):
         order = exchange.create_stoploss(pair='ETH/BTC', amount=1, stop_price=190,
@@ -58,7 +58,7 @@ def test_create_stoploss_order_huobi(default_conf, mocker, limitratio, expected,
     # test exception handling
     with pytest.raises(DependencyException):
         api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds("0 balance"))
-        exchange = get_patched_exchange(mocker, default_conf, api_mock, 'huobi')
+        exchange = get_patched_exchange(mocker, default_conf, api_mock, 'htx')
         exchange.create_stoploss(pair='ETH/BTC', amount=1, stop_price=220,
                                  order_types={},  side=side, leverage=1.0)
 
@@ -69,20 +69,20 @@ def test_create_stoploss_order_huobi(default_conf, mocker, limitratio, expected,
         exchange.create_stoploss(pair='ETH/BTC', amount=1, stop_price=220,
                                  order_types={},  side=side, leverage=1.0)
 
-    ccxt_exceptionhandlers(mocker, default_conf, api_mock, "huobi",
+    ccxt_exceptionhandlers(mocker, default_conf, api_mock, "htx",
                            "create_stoploss", "create_order", retries=1,
                            pair='ETH/BTC', amount=1, stop_price=220, order_types={},
                            side=side, leverage=1.0)
 
 
-def test_create_stoploss_order_dry_run_huobi(default_conf, mocker):
+def test_create_stoploss_order_dry_run_htx(default_conf, mocker):
     api_mock = MagicMock()
     order_type = 'stop-limit'
     default_conf['dry_run'] = True
     mocker.patch(f'{EXMS}.amount_to_precision', lambda s, x, y: y)
     mocker.patch(f'{EXMS}.price_to_precision', lambda s, x, y, **kwargs: y)
 
-    exchange = get_patched_exchange(mocker, default_conf, api_mock, 'huobi')
+    exchange = get_patched_exchange(mocker, default_conf, api_mock, 'htx')
 
     with pytest.raises(InvalidOrderException):
         order = exchange.create_stoploss(pair='ETH/BTC', amount=1, stop_price=190,
@@ -103,8 +103,8 @@ def test_create_stoploss_order_dry_run_huobi(default_conf, mocker):
     assert order['amount'] == 1
 
 
-def test_stoploss_adjust_huobi(mocker, default_conf):
+def test_stoploss_adjust_htx(mocker, default_conf):
-    exchange = get_patched_exchange(mocker, default_conf, id='huobi')
+    exchange = get_patched_exchange(mocker, default_conf, id='htx')
     order = {
         'type': 'stop',
         'price': 1500,

tests/exchange/test_kraken.py

@@ -13,11 +13,14 @@ STOPLOSS_ORDERTYPE = 'stop-loss'
 STOPLOSS_LIMIT_ORDERTYPE = 'stop-loss-limit'
 
 
-def test_buy_kraken_trading_agreement(default_conf, mocker):
+@pytest.mark.parametrize("order_type,time_in_force,expected_params", [
+    ('limit', 'ioc', {'timeInForce': 'IOC', 'trading_agreement': 'agree'}),
+    ('limit', 'PO', {'postOnly': True, 'trading_agreement': 'agree'}),
+    ('market', None, {'trading_agreement': 'agree'})
+])
+def test_kraken_trading_agreement(default_conf, mocker, order_type, time_in_force, expected_params):
     api_mock = MagicMock()
-    order_id = f'test_prod_buy_{randint(0, 10 ** 6)}'
+    order_id = f'test_prod_{order_type}_{randint(0, 10 ** 6)}'
-    order_type = 'limit'
-    time_in_force = 'ioc'
     api_mock.options = {}
     api_mock.create_order = MagicMock(return_value={
         'id': order_id,
@@ -49,41 +52,9 @@ def test_buy_kraken_trading_agreement(default_conf, mocker):
     assert api_mock.create_order.call_args[0][1] == order_type
     assert api_mock.create_order.call_args[0][2] == 'buy'
     assert api_mock.create_order.call_args[0][3] == 1
-    assert api_mock.create_order.call_args[0][4] == 200
+    assert api_mock.create_order.call_args[0][4] == (200 if order_type == 'limit' else None)
-    assert api_mock.create_order.call_args[0][5] == {'timeInForce': 'IOC',
-                                                     'trading_agreement': 'agree'}
 
-
+    assert api_mock.create_order.call_args[0][5] == expected_params
-def test_sell_kraken_trading_agreement(default_conf, mocker):
-    api_mock = MagicMock()
-    order_id = f'test_prod_sell_{randint(0, 10 ** 6)}'
-    order_type = 'market'
-    api_mock.options = {}
-    api_mock.create_order = MagicMock(return_value={
-        'id': order_id,
-        'symbol': 'ETH/BTC',
-        'info': {
-            'foo': 'bar'
-        }
-    })
-    default_conf['dry_run'] = False
-
-    mocker.patch(f'{EXMS}.amount_to_precision', lambda s, x, y: y)
-    mocker.patch(f'{EXMS}.price_to_precision', lambda s, x, y: y)
-    exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kraken")
-
-    order = exchange.create_order(pair='ETH/BTC', ordertype=order_type,
-                                  side="sell", amount=1, rate=200, leverage=1.0)
-
-    assert 'id' in order
-    assert 'info' in order
-    assert order['id'] == order_id
-    assert api_mock.create_order.call_args[0][0] == 'ETH/BTC'
-    assert api_mock.create_order.call_args[0][1] == order_type
-    assert api_mock.create_order.call_args[0][2] == 'sell'
-    assert api_mock.create_order.call_args[0][3] == 1
-    assert api_mock.create_order.call_args[0][4] is None
-    assert api_mock.create_order.call_args[0][5] == {'trading_agreement': 'agree'}
 
 
 def test_get_balances_prod(default_conf, mocker):
@@ -212,19 +183,17 @@ def test_create_stoploss_order_kraken(default_conf, mocker, ordertype, side, adj
     assert 'info' in order
     assert order['id'] == order_id
     assert api_mock.create_order.call_args_list[0][1]['symbol'] == 'ETH/BTC'
-    if ordertype == 'limit':
+    assert api_mock.create_order.call_args_list[0][1]['type'] == ordertype
-        assert api_mock.create_order.call_args_list[0][1]['type'] == STOPLOSS_LIMIT_ORDERTYPE
     assert api_mock.create_order.call_args_list[0][1]['params'] == {
         'trading_agreement': 'agree',
-            'price2': adjustedprice
+        'stopLossPrice': 220
     }
-    else:
-        assert api_mock.create_order.call_args_list[0][1]['type'] == STOPLOSS_ORDERTYPE
-        assert api_mock.create_order.call_args_list[0][1]['params'] == {
-            'trading_agreement': 'agree'}
     assert api_mock.create_order.call_args_list[0][1]['side'] == side
     assert api_mock.create_order.call_args_list[0][1]['amount'] == 1
-    assert api_mock.create_order.call_args_list[0][1]['price'] == 220
+    if ordertype == 'limit':
+        assert api_mock.create_order.call_args_list[0][1]['price'] == adjustedprice
+    else:
+        assert api_mock.create_order.call_args_list[0][1]['price'] is None
 
     # test exception handling
     with pytest.raises(DependencyException):
@@ -282,7 +251,7 @@ def test_create_stoploss_order_dry_run_kraken(default_conf, mocker, side):
     assert 'info' in order
     assert 'type' in order
 
-    assert order['type'] == STOPLOSS_ORDERTYPE
+    assert order['type'] == 'market'
     assert order['price'] == 220
     assert order['amount'] == 1
 
@@ -294,11 +263,22 @@ def test_create_stoploss_order_dry_run_kraken(default_conf, mocker, side):
 def test_stoploss_adjust_kraken(mocker, default_conf, sl1, sl2, sl3, side):
     exchange = get_patched_exchange(mocker, default_conf, id='kraken')
     order = {
-        'type': STOPLOSS_ORDERTYPE,
+        'type': 'market',
-        'price': 1500,
+        'stopLossPrice': 1500,
     }
     assert exchange.stoploss_adjust(sl1, order, side=side)
     assert not exchange.stoploss_adjust(sl2, order, side=side)
-    # Test with invalid order case ...
+    # diff. order type ...
-    order['type'] = 'stop_loss_limit'
+    order['type'] = 'limit'
-    assert not exchange.stoploss_adjust(sl3, order, side=side)
+    assert exchange.stoploss_adjust(sl3, order, side=side)
+
+
+@pytest.mark.parametrize('trade_id, expected', [
+    ('1234', False),
+    ('170544369512007228', False),
+    ('1705443695120072285', True),
+    ('170544369512007228555', True),
+])
+def test__valid_trade_pagination_id_kraken(mocker, default_conf_usdt, trade_id, expected):
+    exchange = get_patched_exchange(mocker, default_conf_usdt, id='kraken')
+    assert exchange._valid_trade_pagination_id('XRP/USDT', trade_id) == expected

tests/exchange_online/conftest.py

@@ -247,7 +247,7 @@ EXCHANGES = {
         'timeframe': '1h',
         'orderbook_max_entries': 50,
     },
-    'huobi': {
+    'htx': {
         'pair': 'ETH/BTC',
         'stake_currency': 'BTC',
         'hasQuoteVolume': True,

tests/freqai/conftest.py

@@ -1,4 +1,5 @@
 import platform
+import sys
 from copy import deepcopy
 from pathlib import Path
 from typing import Any, Dict
@@ -15,6 +16,10 @@ from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver
 from tests.conftest import get_patched_exchange
 
 
+def is_py12() -> bool:
+    return sys.version_info >= (3, 12)
+
+
 def is_mac() -> bool:
     machine = platform.system()
     return "Darwin" in machine
@@ -31,7 +36,7 @@ def patch_torch_initlogs(mocker) -> None:
         module_name = 'torch'
         mocked_module = types.ModuleType(module_name)
         sys.modules[module_name] = mocked_module
-    else:
+    elif not is_py12():
         mocker.patch("torch._logging._init_logs")
 
 
@@ -54,7 +59,7 @@ def freqai_conf(default_conf, tmp_path):
                 "backtest_period_days": 10,
                 "live_retrain_hours": 0,
                 "expiration_hours": 1,
-                "identifier": "uniqe-id100",
+                "identifier": "unique-id100",
                 "live_trained_timestamp": 0,
                 "data_kitchen_thread_count": 2,
                 "activate_tensorboard": False,

tests/freqai/test_freqai_backtesting.py

@@ -6,11 +6,17 @@ from unittest.mock import PropertyMock
 import pytest
 
 from freqtrade.commands.optimize_commands import setup_optimize_configuration
+from freqtrade.configuration.timerange import TimeRange
+from freqtrade.data import history
+from freqtrade.data.dataprovider import DataProvider
 from freqtrade.enums import RunMode
+from freqtrade.enums.candletype import CandleType
 from freqtrade.exceptions import OperationalException
+from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
 from freqtrade.optimize.backtesting import Backtesting
-from tests.conftest import (CURRENT_TEST_STRATEGY, get_args, log_has_re, patch_exchange,
+from tests.conftest import (CURRENT_TEST_STRATEGY, get_args, get_patched_exchange, log_has_re,
-                            patched_configuration_load_config_file)
+                            patch_exchange, patched_configuration_load_config_file)
+from tests.freqai.conftest import get_patched_freqai_strategy
 
 
 def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir, caplog):
@@ -40,7 +46,16 @@ def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir, c
     Backtesting.cleanup()
 
 
-def test_freqai_backtest_load_data(freqai_conf, mocker, caplog):
+@pytest.mark.parametrize(
+    "timeframe, expected_startup_candle_count",
+    [
+        ("5m", 876),
+        ("15m", 492),
+        ("1d", 302),
+    ],
+)
+def test_freqai_backtest_load_data(freqai_conf, mocker, caplog,
+                                   timeframe, expected_startup_candle_count):
     patch_exchange(mocker)
 
     now = datetime.now(timezone.utc)
@@ -48,10 +63,14 @@ def test_freqai_backtest_load_data(freqai_conf, mocker, caplog):
                  PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT']))
     mocker.patch('freqtrade.optimize.backtesting.history.load_data')
     mocker.patch('freqtrade.optimize.backtesting.history.get_timerange', return_value=(now, now))
+    freqai_conf['timeframe'] = timeframe
+    freqai_conf.get('freqai', {}).get('feature_parameters', {}).update({'include_timeframes': []})
     backtesting = Backtesting(deepcopy(freqai_conf))
     backtesting.load_bt_data()
 
-    assert log_has_re('Increasing startup_candle_count for freqai to.*', caplog)
+    assert log_has_re(f'Increasing startup_candle_count for freqai on {timeframe} '
+                      f'to {expected_startup_candle_count}', caplog)
+    assert history.load_data.call_args[1]['startup_candles'] == expected_startup_candle_count
 
     Backtesting.cleanup()
 
@@ -85,3 +104,35 @@ def test_freqai_backtest_live_models_model_not_found(freqai_conf, mocker, testda
         Backtesting(bt_config)
 
     Backtesting.cleanup()
+
+
+def test_freqai_backtest_consistent_timerange(mocker, freqai_conf):
+    freqai_conf['runmode'] = 'backtest'
+    mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist',
+                 PropertyMock(return_value=['XRP/USDT:USDT']))
+
+    gbs = mocker.patch('freqtrade.optimize.backtesting.generate_backtest_stats')
+
+    freqai_conf['candle_type_def'] = CandleType.FUTURES
+    freqai_conf.get('exchange', {}).update({'pair_whitelist': ['XRP/USDT:USDT']})
+    freqai_conf.get('freqai', {}).get('feature_parameters', {}).update(
+        {'include_timeframes': ['5m', '1h'], 'include_corr_pairlist': []})
+    freqai_conf['timerange'] = '20211120-20211121'
+
+    strategy = get_patched_freqai_strategy(mocker, freqai_conf)
+    exchange = get_patched_exchange(mocker, freqai_conf)
+
+    strategy.dp = DataProvider(freqai_conf, exchange)
+    strategy.freqai_info = freqai_conf.get("freqai", {})
+    freqai = strategy.freqai
+    freqai.dk = FreqaiDataKitchen(freqai_conf)
+
+    timerange = TimeRange.parse_timerange("20211115-20211122")
+    freqai.dd.load_all_pair_histories(timerange, freqai.dk)
+
+    backtesting = Backtesting(deepcopy(freqai_conf))
+    backtesting.start()
+
+    gbs.call_args[1]['min_date'] == datetime(2021, 11, 20, 0, 0, tzinfo=timezone.utc)
+    gbs.call_args[1]['max_date'] == datetime(2021, 11, 21, 0, 0, tzinfo=timezone.utc)
+    Backtesting.cleanup()