Merge branch 'develop' into add-custom-roi-strategy-callback

.github/ISSUE_TEMPLATE/bug_report.md

@@ -12,7 +12,7 @@ Have you searched for similar issues before posting it?
 If you have discovered a bug in the bot, please [search the issue tracker](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue). 
 If it hasn't been reported, please create a new issue.
 
-Please do not use bug reports to request new features.
+Please do not use the bug report template to request new features.
 -->
 
 ## Describe your environment

.github/ISSUE_TEMPLATE/question.md

@@ -8,9 +8,12 @@ assignees: ''
 ---
 <!-- 
 Have you searched for similar issues before posting it?
-Did you have a VERY good look at the [documentation](https://www.freqtrade.io/en/latest/) and are sure that the question is not explained there
+Did you have a VERY good look at the [documentation](https://www.freqtrade.io/) and are sure that the question is not explained there
 
 Please do not use the question template to report bugs or to request new features.
+
+Has your strategy or configuration been generated by an AI model, and is now not working?
+Please consult the documentation. We'll close such issues and point to the documentation.
 -->
 
 ## Describe your environment
@@ -22,4 +25,4 @@ Please do not use the question template to report bugs or to request new feature
   
 ## Your question
 
-*Ask the question you have not been able to find an answer in the [Documentation](https://www.freqtrade.io/en/latest/)*
+*Ask the question you have not been able to find an answer in the [Documentation](https://www.freqtrade.io/)*

.github/workflows/binance-lev-tier-update.yml

@@ -34,7 +34,7 @@ jobs:
       run: python build_helpers/binance_update_lev_tiers.py
 
 
-    - uses: peter-evans/create-pull-request@v7
+    - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
       with:
         token: ${{ secrets.REPO_SCOPED_TOKEN }}
         add-paths: freqtrade/exchange/binance_leverage_tiers.json

.github/workflows/ci.yml

@@ -38,8 +38,9 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v6.0.1
       with:
+        activate-environment: true
         enable-cache: true
         python-version: ${{ matrix.python-version }}
         cache-dependency-glob: "requirements**.txt"
@@ -144,7 +145,7 @@ jobs:
         mypy freqtrade scripts tests
 
     - name: Discord notification
-      uses: rjstone/discord-webhook-notify@v1
+      uses: rjstone/discord-webhook-notify@1399c1b2d57cc05894d506d2cfdc33c5f012b993 #v1.1.1
       if: failure() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)
       with:
           severity: error
@@ -170,8 +171,9 @@ jobs:
         check-latest: true
 
     - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v6.0.1
       with:
+        activate-environment: true
         enable-cache: true
         python-version: ${{ matrix.python-version }}
         cache-dependency-glob: "requirements**.txt"
@@ -270,7 +272,7 @@ jobs:
         mypy freqtrade scripts
 
     - name: Discord notification
-      uses: rjstone/discord-webhook-notify@v1
+      uses: rjstone/discord-webhook-notify@1399c1b2d57cc05894d506d2cfdc33c5f012b993 #v1.1.1
       if: failure() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)
       with:
           severity: info
@@ -296,8 +298,9 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v6.0.1
       with:
+        activate-environment: true
         enable-cache: true
         python-version: ${{ matrix.python-version }}
         cache-dependency-glob: "requirements**.txt"
@@ -363,7 +366,7 @@ jobs:
       shell: powershell
 
     - name: Discord notification
-      uses: rjstone/discord-webhook-notify@v1
+      uses: rjstone/discord-webhook-notify@1399c1b2d57cc05894d506d2cfdc33c5f012b993 #v1.1.1
       if: failure() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)
       with:
           severity: error
@@ -397,7 +400,7 @@ jobs:
     - uses: actions/setup-python@v5
       with:
         python-version: "3.12"
-    - uses: pre-commit/action@v3.0.1
+    - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
 
   docs-check:
     runs-on: ubuntu-22.04
@@ -421,7 +424,7 @@ jobs:
         mkdocs build
 
     - name: Discord notification
-      uses: rjstone/discord-webhook-notify@v1
+      uses: rjstone/discord-webhook-notify@1399c1b2d57cc05894d506d2cfdc33c5f012b993 #v1.1.1
       if: failure() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)
       with:
           severity: error
@@ -431,7 +434,7 @@ jobs:
 
   build-linux-online:
     # Run pytest with "live" checks
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@v4
       with:
@@ -443,8 +446,9 @@ jobs:
         python-version: "3.12"
 
     - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v6.0.1
       with:
+        activate-environment: true
         enable-cache: true
         python-version: "3.12"
         cache-dependency-glob: "requirements**.txt"
@@ -501,14 +505,14 @@ jobs:
 
     - name: Check user permission
       id: check
-      uses: scherermichael-oss/action-has-permission@1.0.6
+      uses: scherermichael-oss/action-has-permission@136e061bfe093832d87f090dd768e14e27a740d3 # 1.0.6
       with:
         required-permission: write
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Discord notification
-      uses: rjstone/discord-webhook-notify@v1
+      uses: rjstone/discord-webhook-notify@1399c1b2d57cc05894d506d2cfdc33c5f012b993 #v1.1.1
       if: always() && steps.check.outputs.has-permission && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)
       with:
           severity: info
@@ -580,7 +584,7 @@ jobs:
         merge-multiple: true
 
     - name: Publish to PyPI (Test)
-      uses: pypa/gh-action-pypi-publish@v1.12.4
+      uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4
       with:
         repository-url: https://test.pypi.org/legacy/
 
@@ -609,7 +613,7 @@ jobs:
         merge-multiple: true
 
     - name: Publish to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.12.4
+      uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4
 
 
   deploy-docker:
@@ -650,11 +654,11 @@ jobs:
           docker version -f '{{.Server.Experimental}}'
 
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
+      uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
 
     - name: Set up Docker Buildx
       id: buildx
-      uses: docker/setup-buildx-action@v3
+      uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 #v3.10.0
 
     - name: Available platforms
       run: echo ${PLATFORMS}
@@ -703,7 +707,7 @@ jobs:
         build_helpers/publish_docker_arm64.sh
 
     - name: Discord notification
-      uses: rjstone/discord-webhook-notify@v1
+      uses: rjstone/discord-webhook-notify@1399c1b2d57cc05894d506d2cfdc33c5f012b993 #v1.1.1
       if: always() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) && (github.event_name != 'schedule')
       with:
           severity: info

.github/workflows/devcontainer-build.yml

@@ -28,13 +28,13 @@ jobs:
       with:
         persist-credentials: false
     - name: Login to GitHub Container Registry
-      uses: docker/login-action@v3
+      uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
       with:
         registry: ghcr.io
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
     - name: Pre-build dev container image
-      uses: devcontainers/ci@v0.3
+      uses: devcontainers/ci@8bf61b26e9c3a98f69cb6ce2f88d24ff59b785c6 # v0.3.19
       with:
         subFolder: .github
         imageName: ghcr.io/${{ github.repository }}-devcontainer

.github/workflows/docker-update-readme.yml

@@ -16,7 +16,7 @@ jobs:
         persist-credentials: false
 
     - name: Docker Hub Description
-      uses: peter-evans/dockerhub-description@v4
+      uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4.0.2
       with:
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/pre-commit-update.yml

@@ -28,7 +28,7 @@ jobs:
     - name: Run auto-update
       run: pre-commit autoupdate
 
-    - uses: peter-evans/create-pull-request@v7
+    - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
       with:
         token: ${{ secrets.REPO_SCOPED_TOKEN }}
         add-paths: .pre-commit-config.yaml

.pre-commit-config.yaml

@@ -43,7 +43,7 @@ repos:
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.11.5'
+    rev: 'v0.11.9'
     hooks:
       - id: ruff
       - id: ruff-format
@@ -82,6 +82,6 @@ repos:
 
   # Ensure github actions remain safe
   - repo: https://github.com/woodruffw/zizmor-pre-commit
-    rev: v1.5.2
+    rev: v1.7.0
     hooks:
     - id: zizmor

build_helpers/binance_update_lev_tiers.py

@@ -12,7 +12,12 @@ secret = os.environ.get("FREQTRADE__EXCHANGE__SECRET")
 proxy = os.environ.get("CI_WEB_PROXY")
 
 exchange = ccxt.binance(
-    {"apiKey": key, "secret": secret, "httpsProxy": proxy, "options": {"defaultType": "swap"}}
+    {
+        "apiKey": key,
+        "secret": secret,
+        "httpsProxy": proxy,
+        "options": {"defaultType": "swap"},
+    }
 )
 _ = exchange.load_markets()
 

docs/advanced-hyperopt.md

@@ -161,56 +161,53 @@ class MyAwesomeStrategy(IStrategy):
 
 ### Overriding Base estimator
 
-You can define your own estimator for Hyperopt by implementing `generate_estimator()` in the Hyperopt subclass.
+You can define your own optuna sampler for Hyperopt by implementing `generate_estimator()` in the Hyperopt subclass.
 
 ```python
 class MyAwesomeStrategy(IStrategy):
     class HyperOpt:
         def generate_estimator(dimensions: List['Dimension'], **kwargs):
-            return "RF"
+            return "NSGAIIISampler"
 
 ```
 
-Possible values are either one of "GP", "RF", "ET", "GBRT" (Details can be found in the [scikit-optimize documentation](https://scikit-optimize.github.io/)), or "an instance of a class that inherits from `RegressorMixin` (from sklearn) and where the `predict` method has an optional `return_std` argument, which returns `std(Y | x)` along with `E[Y | x]`".
+Possible values are either one of "NSGAIISampler", "TPESampler", "GPSampler", "CmaEsSampler", "NSGAIIISampler", "QMCSampler" (Details can be found in the [optuna-samplers documentation](https://optuna.readthedocs.io/en/stable/reference/samplers/index.html)), or "an instance of a class that inherits from `optuna.samplers.BaseSampler`".
 
-Some research will be necessary to find additional Regressors.
+Some research will be necessary to find additional Samplers (from optunahub) for example.
-
-Example for `ExtraTreesRegressor` ("ET") with additional parameters:
-
-```python
-class MyAwesomeStrategy(IStrategy):
-    class HyperOpt:
-        def generate_estimator(dimensions: List['Dimension'], **kwargs):
-            from skopt.learning import ExtraTreesRegressor
-            # Corresponds to "ET" - but allows additional parameters.
-            return ExtraTreesRegressor(n_estimators=100)
-
-```
-
-The `dimensions` parameter is the list of `skopt.space.Dimension` objects corresponding to the parameters to be optimized. It can be used to create isotropic kernels for the `skopt.learning.GaussianProcessRegressor` estimator. Here's an example:
-
-```python
-class MyAwesomeStrategy(IStrategy):
-    class HyperOpt:
-        def generate_estimator(dimensions: List['Dimension'], **kwargs):
-            from skopt.utils import cook_estimator
-            from skopt.learning.gaussian_process.kernels import (Matern, ConstantKernel)
-            kernel_bounds = (0.0001, 10000)
-            kernel = (
-                ConstantKernel(1.0, kernel_bounds) * 
-                Matern(length_scale=np.ones(len(dimensions)), length_scale_bounds=[kernel_bounds for d in dimensions], nu=2.5)
-            )
-            kernel += (
-                ConstantKernel(1.0, kernel_bounds) * 
-                Matern(length_scale=np.ones(len(dimensions)), length_scale_bounds=[kernel_bounds for d in dimensions], nu=1.5)
-            )
-
-            return cook_estimator("GP", space=dimensions, kernel=kernel, n_restarts_optimizer=2)
-```
 
 !!! Note
     While custom estimators can be provided, it's up to you as User to do research on possible parameters and analyze / understand which ones should be used.
-    If you're unsure about this, best use one of the Defaults (`"ET"` has proven to be the most versatile) without further parameters.
+    If you're unsure about this, best use one of the Defaults (`"NSGAIIISampler"` has proven to be the most versatile) without further parameters.
+
+??? Example "Using `AutoSampler` from Optunahub"
+
+    [AutoSampler docs](https://hub.optuna.org/samplers/auto_sampler/)
+    
+    Install the necessary dependencies 
+    ``` bash
+    pip install optunahub cmaes torch scipy
+    ```
+    Implement `generate_estimator()`  in your strategy
+
+    ``` python
+    # ...
+    from freqtrade.strategy.interface import IStrategy
+    from typing import List
+    import optunahub
+    # ... 
+
+    class my_strategy(IStrategy):
+        class HyperOpt:
+            def generate_estimator(dimensions: List["Dimension"], **kwargs):
+                if "random_state" in kwargs.keys():
+                    return optunahub.load_module("samplers/auto_sampler").AutoSampler(seed=kwargs["random_state"])
+                else:
+                    return optunahub.load_module("samplers/auto_sampler").AutoSampler()
+
+    ```
+
+    Obviously the same approach will work for all other Samplers optuna supports.
+
 
 ## Space options
 

docs/advanced-setup.md

@@ -189,13 +189,15 @@ as the watchdog.
 ## Advanced Logging
 
 Freqtrade uses the default logging module provided by python.
-Python allows for extensive [logging configuration](https://docs.python.org/3/library/logging.config.html#logging.config.dictConfig) in this regards - way more than what can be covered here.
+Python allows for extensive [logging configuration](https://docs.python.org/3/library/logging.config.html#logging.config.dictConfig) in this regard - way more than what can be covered here.
 
-Default logging (Colored terminal output) is setup by default if no `log_config` is provided.
+Default logging format (coloured terminal output) is set up by default if no `log_config` is provided in your freqtrade configuration.
 Using `--logfile logfile.log` will enable the RotatingFileHandler.
-If you're not content with the log format - or with the default settings provided for the RotatingFileHandler, you can customize logging to your liking.
 
-The default configuration looks roughly like the below - with the file handler being provided - but not enabled.
+If you're not content with the log format, or with the default settings provided for the RotatingFileHandler, you can customize logging to your liking by adding the `log_config` configuration to your freqtrade configuration file(s).
+
+The default configuration looks roughly like the below, with the file handler being provided but not enabled as the `filename` is commented out.
+Uncomment this line and supply a valid path/filename to enable it.
 
 ``` json hl_lines="5-7 13-16 27"
 {
@@ -237,12 +239,12 @@ The default configuration looks roughly like the below - with the file handler b
     Highlighted lines in the above code-block define the Rich handler and belong together.
     The formatter "standard" and "file" will belong to the FileHandler.
 
-Each handler must use one of the defined formatters (by name) - and it's class must be available and a valid logging class.
+Each handler must use one of the defined formatters (by name), its class must be available, and must be a valid logging class.
-To actually use a handler - it must be in the "handlers" section inside the "root" segment.
+To actually use a handler, it must be in the "handlers" section inside the "root" segment.
 If this section is left out, freqtrade will provide no output (in the non-configured handler, anyway).
 
 !!! Tip "Explicit log configuration"
-    We recommend to extract the logging configuration from your main configuration, and provide it to your bot via [multiple configuration files](configuration.md#multiple-configuration-files) functionality. This will avoid unnecessary code duplication.
+    We recommend to extract the logging configuration from your main freqtrade configuration file, and provide it to your bot via [multiple configuration files](configuration.md#multiple-configuration-files) functionality. This will avoid unnecessary code duplication.
 
 ---
 

docs/commands/backtesting-analysis.md

@@ -59,7 +59,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/backtesting-show.md

@@ -30,7 +30,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/backtesting.md

@@ -89,7 +89,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/convert-data.md

@@ -45,7 +45,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/convert-trade-data.md

@@ -34,7 +34,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/download-data.md

@@ -63,7 +63,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/edge.md

@@ -50,7 +50,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/hyperopt-list.md

@@ -55,7 +55,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/hyperopt-show.md

@@ -37,7 +37,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/hyperopt.md

@@ -103,7 +103,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/list-data.md

@@ -41,7 +41,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/list-exchanges.md

@@ -22,7 +22,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/list-freqaimodels.md

@@ -24,7 +24,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/list-hyperoptloss.md

@@ -24,7 +24,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/list-markets.md

@@ -39,7 +39,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/list-pairs.md

@@ -39,7 +39,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/list-strategies.md

@@ -27,7 +27,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/list-timeframes.md

@@ -23,7 +23,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/lookahead-analysis.md

@@ -89,7 +89,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/plot-dataframe.md

@@ -63,7 +63,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/plot-profit.md

@@ -49,7 +49,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/recursive-analysis.md

@@ -41,7 +41,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/show-trades.md

@@ -30,7 +30,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/strategy-updater.md

@@ -34,7 +34,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/trade.md

@@ -36,7 +36,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/trades-to-ohlcv.md

@@ -41,7 +41,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/commands/webserver.md

@@ -20,7 +20,9 @@ Common arguments:
                         exists). Multiple --config options may be used. Can be
                         set to `-` to read config from stdin.
   -d PATH, --datadir PATH, --data-dir PATH
-                        Path to directory with historical backtesting data.
+                        Path to the base directory of the exchange with
+                        historical backtesting data. To see futures data, use
+                        trading-mode additionally.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 

docs/faq.md

@@ -219,10 +219,7 @@ On Windows, the `--logfile` option is also supported by Freqtrade and you can us
 First of all, most indicator libraries don't have GPU support - as such, there would be little benefit for indicator calculations.
 The GPU improvements would only apply to pandas-native calculations - or ones written by yourself.
 
-For hyperopt, freqtrade is using scikit-optimize, which is built on top of scikit-learn.
+GPU's are only good at crunching numbers (floating point operations).
-Their statement about GPU support is [pretty clear](https://scikit-learn.org/stable/faq.html#will-you-add-gpu-support).
-
-GPU's also are only good at crunching numbers (floating point operations).
 For hyperopt, we need both number-crunching (find next parameters) and running python code (running backtesting).
 As such, GPU's are not too well suited for most parts of hyperopt.
 

docs/freqai-feature-engineering.md

@@ -181,7 +181,7 @@ You can ask for each of the defined features to be included also for informative
 In total, the number of features the user of the presented example strategy has created is: length of `include_timeframes` * no. features in `feature_engineering_expand_*()` * length of `include_corr_pairlist` * no. `include_shifted_candles` * length of `indicator_periods_candles`
  $= 3 * 3 * 3 * 2 * 2 = 108$.
  
- !!! note "Learn more about creative feature engineering"
+!!! note "Learn more about creative feature engineering"
     Check out our [medium article](https://emergentmethods.medium.com/freqai-from-price-to-prediction-6fadac18b665) geared toward helping users learn how to creatively engineer features.
 
 ### Gain finer control over `feature_engineering_*` functions with `metadata`

docs/hyperopt.md

@@ -1,10 +1,10 @@
 # Hyperopt
 
 This page explains how to tune your strategy by finding the optimal
-parameters, a process called hyperparameter optimization. The bot uses algorithms included in the `scikit-optimize` package to accomplish this.
+parameters, a process called hyperparameter optimization. The bot uses algorithms included in the `optuna` package to accomplish this.
 The search will burn all your CPU cores, make your laptop sound like a fighter jet and still take a long time.
 
-In general, the search for best parameters starts with a few random combinations (see [below](#reproducible-results) for more details) and then uses Bayesian search with a ML regressor algorithm (currently ExtraTreesRegressor) to quickly find a combination of parameters in the search hyperspace that minimizes the value of the [loss function](#loss-functions).
+In general, the search for best parameters starts with a few random combinations (see [below](#reproducible-results) for more details) and then uses one of optuna's sampler algorithms (currently NSGAIIISampler) to quickly find a combination of parameters in the search hyperspace that minimizes the value of the [loss function](#loss-functions).
 
 Hyperopt requires historic data to be available, just as backtesting does (hyperopt runs backtesting many times with different parameters).
 To learn how to get data for the pairs and exchange you're interested in, head over to the [Data Downloading](data-download.md) section of the documentation.

docs/installation.md

@@ -87,7 +87,7 @@ OS Specific steps are listed first, the common section below is necessary for al
 
 
     ```bash
-    sudo apt-get install python3-venv libatlas-base-dev cmake curl
+    sudo apt-get install python3-venv libatlas-base-dev cmake curl libffi-dev
     # Use piwheels.org to speed up installation
     sudo echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > tee /etc/pip.conf
 

docs/lookahead-analysis.md

@@ -1,23 +1,21 @@
 # Lookahead analysis
 
-This page explains how to validate your strategy in terms of look ahead bias.
+This page explains how to validate your strategy in terms of lookahead bias.
 
-Checking look ahead bias is the bane of any strategy since it is sometimes very easy to introduce backtest bias -
+Lookahead bias is the bane of any strategy since it is sometimes very easy to introduce this bias, but can be very hard to detect.
-but very hard to detect.
 
-Backtesting initializes all timestamps at once and calculates all indicators in the beginning.
+Backtesting initializes all timestamps (loads the whole dataframe into memory) and calculates all indicators at once.
-This means that if your indicators or entry/exit signals could look into future candles and falsify your backtest.
+This means that if your indicators or entry/exit signals look into future candles, this will falsify your backtest.
 
-Lookahead-analysis requires historic data to be available.
+The `lookahead-analysis` command requires historic data to be available.
 To learn how to get data for the pairs and exchange you're interested in,
 head over to the [Data Downloading](data-download.md) section of the documentation.
+`lookahead-analysis` also supports freqai strategies.
 
-This command is built upon backtesting since it internally chains backtests and pokes at the strategy to provoke it to show look ahead bias.
+This command internally chains backtests and pokes at the strategy to provoke it to show lookahead bias.
-This is done by not looking at the strategy itself - but at the results it returned.
+This is done by not looking at the strategy code itself, but at changed indicator values and moved entries/exits compared to the full backtest.
-The results are things like changed indicator-values and moved entries/exits compared to the full backtest.
 
-You can use commands of [Backtesting](backtesting.md).
+`lookahead-analysis` can use the typical options of [Backtesting](backtesting.md), but forces the following options:
-It also supports the lookahead-analysis of freqai strategies.
 
 - `--cache` is forced to "none".
 - `--max-open-trades` is forced to be at least equal to the number of pairs.
@@ -25,48 +23,83 @@ It also supports the lookahead-analysis of freqai strategies.
 - `--stake-amount` is forced to be a static 10000 (10k).
 - `--enable-protections` is forced to be off.
 
-Those are set to avoid users accidentally generating false positives.
+These are set to avoid users accidentally generating false positives.
 
 ## Lookahead-analysis command reference
 
 --8<-- "commands/lookahead-analysis.md"
 
-!!! Note ""
+!!! Note
-    The above Output was reduced to options `lookahead-analysis` adds on top of regular backtesting commands.
+    The above output was reduced to options that `lookahead-analysis` adds on top of regular backtesting commands.
-
-### Summary
-
-Checks a given strategy for look ahead bias via lookahead-analysis
-Look ahead bias means that the backtest uses data from future candles thereby not making it viable beyond backtesting
-and producing false hopes for the one backtesting.
 
 ### Introduction
 
-Many strategies - without the programmer knowing - have fallen prey to look ahead bias.
+Many strategies, without the programmer knowing, have fallen prey to lookahead bias.
+This typically makes the strategy backtest look profitable, sometimes to extremes,  but this is not realistic as the strategy is "cheating" by looking at data it would not have in dry or live modes.
 
-Any backtest will populate the full dataframe including all time stamps at the beginning.
+The reason why strategies can "cheat" is because the freqtrade backtesting process populates the full dataframe including all candle timestamps at the outset.
-If the programmer is not careful or oblivious how things work internally
+If the programmer is not careful or oblivious how things work internally 
-(which sometimes can be really hard to find out) then it will just look into the future making the strategy amazing
+(which sometimes can be really hard to find out) then the strategy will look into the future.
-but not realistic.
 
-This command is made to try to verify the validity in the form of the aforementioned look ahead bias.
+This command is made to try to verify the validity in the form of the aforementioned lookahead bias.
 
 ### How does the command work?
 
 It will start with a backtest of all pairs to generate a baseline for indicators and entries/exits.
-After the backtest ran, it will look if the `minimum-trade-amount` is met
+After this initial backtest runs, it will look if the `minimum-trade-amount` is met and if not cancel the lookahead-analysis for this strategy.  
-and if not cancel the lookahead-analysis for this strategy.
+If this happens, use a wider timerange to get more trades for the analysis, or use a timerange where more trades occur.
 
-After setting the baseline it will then do additional runs for every entry and exit separately.
+After setting the baseline it will then do additional backtest runs for every entry and exit separately.  
-When a verification-backtest is done, it will compare the indicators as the signal (either entry or exit) and report the bias.
+When these verification backtests complete, it will compare the indicators at the signal candles (both entry or exit)
-After all signals have been verified or falsified a result-table will be generated for the user to see.
+and report the bias.  
+After all signals have been verified or falsified a result table will be generated for the user to see.
+
+### How to find and remove bias? How can I salvage a biased strategy?
+
+If you found a biased strategy online and want to have the same results, just without bias,
+then you will be out of luck most of the time.
+Usually the bias in the strategy is THE driving factor for "too good to be true" profits.
+Removing conditions or indicators that push the profits up from bias will usually make the strategy significantly worse.
+You might be able to salvage it partially if the biased indicators or conditions are not the core of the strategy, or there
+are other entry and exit signals that are not biased.
+
+### Examples of lookahead-bias
+
+- `shift(-10)` looks 10 candles into the future.
+- Using `iloc[]` in populate_* functions to access a specific row in the dataframe.
+- For-loops are prone to introduce lookahead bias if you don't tightly control which numbers are looped through.
+- Aggregation functions like `.mean()`, `.min()` and `.max()`, without a rolling window,
+  will calculate the value over the **whole** dataframe, so the signal candle will "see" a value including future candles.
+  A non-biased example would be to look back candles using `rolling()` instead:
+  e.g. `dataframe['volume_mean_12'] = dataframe['volume'].rolling(12).mean()`
+- `ta.MACD(dataframe, 12, 26, 1)` will introduce bias with a signalperiod of 1.
+
+### What do the columns in the results table mean?
+
+- `filename`: name of the checked strategy file
+- `strategy`: checked strategy class name
+- `has_bias`: result of the lookahead-analysis. `No` would be good, `Yes` would be bad.
+- `total_signals`: number of checked signals (default is 20)
+- `biased_entry_signals`: found bias in that many entries
+- `biased_exit_signals`: found bias in that many exits
+- `biased_indicators`: shows you the indicators themselves that are defined in populate_indicators
+
+You might get false positives in the `biased_exit_signals` if you have biased entry signals paired with those exits.
+However, a biased entry will usually result in a biased exit too,
+even if the exit itself does not produce the bias -
+especially if your entry and exit conditions use the same biased indicator.
+
+**Address the bias in the entries first, then address the exits.**
 
 ### Caveats
 
 - `lookahead-analysis` can only verify / falsify the trades it calculated and verified.
-If the strategy has many different signals / signal types, it's up to you to select appropriate parameters to ensure that all signals have triggered at least once. Not triggered signals will not have been verified.
+If the strategy has many different signals / signal types, it's up to you to select appropriate parameters to ensure that all signals have triggered at least once. Signals that are not triggered will not have been verified.  
-This could lead to a false-negative (the strategy will then be reported as non-biased).
+This would lead to a false-negative, i.e. the strategy will be reported as non-biased.
-- `lookahead-analysis` has access to everything that backtesting has too.
+- `lookahead-analysis` has access to the same backtesting options and this can introduce problems.
-Please don't provoke any configs like enabling position stacking.
+Please don't use any options like enabling position stacking as this will distort the number of checked signals.
-If you decide to do so, then make doubly sure that you won't ever run out of `max_open_trades` amount and neither leftover money in your wallet.
+If you decide to do so, then make doubly sure that you won't ever run out of `max_open_trades` slots, 
-- In the results table, the `biased_indicators` column will falsely flag FreqAI target indicators defined in `set_freqai_targets()` as biased. These are not biased and can safely be ignored.
+and that you have enough capital in the backtest wallet configuration.
+- In the results table, the `biased_indicators` column 
+will falsely flag FreqAI target indicators defined in `set_freqai_targets()` as biased.  
+**These are not biased and can safely be ignored.**

docs/producer-consumer.md

@@ -50,6 +50,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect
 | `ping_timeout` | Ping timeout <br>*Defaults to `10`.*<br> **Datatype:** Integer - in seconds.
 | `sleep_time` | Sleep time before retrying to connect.<br>*Defaults to `10`.*<br> **Datatype:** Integer - in seconds.
 | `remove_entry_exit_signals` | Remove signal columns from the dataframe (set them to 0) on dataframe receipt.<br>*Defaults to `false`.*<br> **Datatype:** Boolean.
+| `initial_candle_limit` | Initial candles to expect from the Producer.<br>*Defaults to `1500`.*<br> **Datatype:** Integer - Number of candles.
 | `message_size_limit` | Size limit per message<br>*Defaults to `8`.*<br> **Datatype:** Integer - Megabytes.
 
 Instead of (or as well as) calculating indicators in `populate_indicators()` the follower instance listens on the connection to a producer instance's messages (or multiple producer instances in advanced configurations) and requests the producer's most recently analyzed dataframes for each pair in the active whitelist.

docs/requirements-docs.txt

@@ -1,7 +1,7 @@
 markdown==3.8
 mkdocs==1.6.1
-mkdocs-material==9.6.11
+mkdocs-material==9.6.13
 mdx_truly_sane_lists==1.3
-pymdown-extensions==10.14.3
+pymdown-extensions==10.15
 jinja2==3.1.6
 mike==2.1.3

docs/strategy-101.md

@@ -165,18 +165,23 @@ If there is any significant difference, verify that your entry and exit signals
 
 ## Controlling or monitoring a running bot
 
-Once your bot is running in dry or live mode, Freqtrade has five mechanisms to control or monitor a running bot:
+Once your bot is running in dry or live mode, Freqtrade has six mechanisms to control or monitor a running bot:
 
 - **[FreqUI](freq-ui.md)**: The easiest to get started with, FreqUI is a web interface to see and control current activity of your bot.
 - **[Telegram](telegram-usage.md)**: On mobile devices, Telegram integration is available to get alerts about your bot activity and to control certain aspects.
 - **[FTUI](https://github.com/freqtrade/ftui)**: FTUI is a terminal (command line) interface to Freqtrade, and allows monitoring of a running bot only.
-- **[REST API](rest-api.md)**: The REST API allows programmers to develop their own tools to interact with a Freqtrade bot.
+- **[freqtrade-client](rest-api.md#consuming-the-api)**: A python implementation of the REST API, making it easy to make requests and consume bot responses from your python apps or the command line.
+- **[REST API endpoints](rest-api.md#available-endpoints)**: The REST API allows programmers to develop their own tools to interact with a Freqtrade bot.
 - **[Webhooks](webhook-config.md)**: Freqtrade can send information to other services, e.g. discord, by webhooks.
 
 ### Logs
 
 Freqtrade generates extensive debugging logs to help you understand what's happening. Please familiarise yourself with the information and error messages you might see in your bot logs.
 
+Logging by default occurs on standard out (the command line). If you want to write out to a file instead, many freqtrade commands, including the `trade` command, accept the `--logfile` option to write to a file.
+
+Check the [FAQ](faq.md#how-do-i-search-the-bot-logs-for-something) for examples.
+
 ## Final Thoughts
 
 Algo trading is difficult, and most public strategies are not good performers due to the time and effort to make a strategy work profitably in multiple scenarios.

docs/strategy-callbacks.md

@@ -179,6 +179,8 @@ Returning `None` will be interpreted as "no desire to change", and is the only s
 
 Stoploss on exchange works similar to `trailing_stop`, and the stoploss on exchange is updated as configured in `stoploss_on_exchange_interval` ([More details about stoploss on exchange](stoploss.md#stop-loss-on-exchangefreqtrade)).
 
+If you're on futures markets, please take note of the [stoploss and leverage](stoploss.md#stoploss-and-leverage) section, as the stoploss value returned from `custom_stoploss` is the risk for this trade - not the relative price movement.
+
 !!! Note "Use of dates"
     All time-based calculations should be done based on `current_time` - using `datetime.now()` or `datetime.utcnow()` is discouraged, as this will break backtesting support.
 
@@ -234,7 +236,7 @@ class AwesomeStrategy(IStrategy):
         :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
         :return float: New stoploss value, relative to the current_rate
         """
-        return -0.04
+        return -0.04 * trade.leverage
 ```
 
 #### Time based trailing stop
@@ -256,9 +258,9 @@ class AwesomeStrategy(IStrategy):
 
         # Make sure you have the longest interval first - these conditions are evaluated from top to bottom.
         if current_time - timedelta(minutes=120) > trade.open_date_utc:
-            return -0.05
+            return -0.05 * trade.leverage
         elif current_time - timedelta(minutes=60) > trade.open_date_utc:
-            return -0.10
+            return -0.10 * trade.leverage
         return None
 ```
 
@@ -285,9 +287,9 @@ class AwesomeStrategy(IStrategy):
             return stoploss_from_open(0.10, current_profit, is_short=trade.is_short, leverage=trade.leverage)
         # Make sure you have the longest interval first - these conditions are evaluated from top to bottom.
         if current_time - timedelta(minutes=120) > trade.open_date_utc:
-            return -0.05
+            return -0.05 * trade.leverage
         elif current_time - timedelta(minutes=60) > trade.open_date_utc:
-            return -0.10
+            return -0.10 * trade.leverage
         return None
 ```
 
@@ -310,10 +312,10 @@ class AwesomeStrategy(IStrategy):
                         **kwargs) -> float | None:
 
         if pair in ("ETH/BTC", "XRP/BTC"):
-            return -0.10
+            return -0.10 * trade.leverage
         elif pair in ("LTC/BTC"):
-            return -0.05
+            return -0.05 * trade.leverage
-        return -0.15
+        return -0.15 * trade.leverage
 ```
 
 #### Trailing stoploss with positive offset
@@ -342,7 +344,7 @@ class AwesomeStrategy(IStrategy):
         desired_stoploss = current_profit / 2
 
         # Use a minimum of 2.5% and a maximum of 5%
-        return max(min(desired_stoploss, 0.05), 0.025)
+        return max(min(desired_stoploss, 0.05), 0.025) * trade.leverage
 ```
 
 #### Stepped stoploss
@@ -1236,3 +1238,119 @@ class AwesomeStrategy(IStrategy):
         return None
 
 ```
+
+## Plot annotations callback
+
+The plot annotations callback is called whenever freqUI requests data to display a chart.
+This callback has no meaning in the trade cycle context and is only used for charting purposes.
+
+The strategy can then return a list of `AnnotationType` objects to be displayed on the chart.
+Depending on the content returned - the chart can display horizontal areas, vertical areas, or boxes.
+
+The full object looks like this:
+
+``` json
+{
+    "type": "area", // Type of the annotation, currently only "area" is supported
+    "start": "2024-01-01 15:00:00", // Start date of the area
+    "end": "2024-01-01 16:00:00",  // End date of the area
+    "y_start": 94000.2,  // Price / y axis value
+    "y_end": 98000, // Price / y axis value
+    "color": "",
+    "label": "some label"
+}
+```
+
+The below example will mark the chart with areas for the hours 8 and 15, with a grey color, highlighting the market open and close hours.
+This is obviously a very basic example.
+
+``` python
+# Default imports
+
+class AwesomeStrategy(IStrategy):
+    def plot_annotations(
+        self, pair: str, start_date: datetime, end_date: datetime, dataframe: DataFrame, **kwargs
+    ) -> list[AnnotationType]:
+        """
+        Retrieve area annotations for a chart.
+        Must be returned as array, with type, label, color, start, end, y_start, y_end.
+        All settings except for type are optional - though it usually makes sense to include either
+        "start and end" or "y_start and y_end" for either horizontal or vertical plots
+        (or all 4 for boxes).
+        :param pair: Pair that's currently analyzed
+        :param start_date: Start date of the chart data being requested
+        :param end_date: End date of the chart data being requested
+        :param dataframe: DataFrame with the analyzed data for the chart
+        :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
+        :return: List of AnnotationType objects
+        """
+        annotations = []
+        while start_dt < end_date:
+            start_dt += timedelta(hours=1)
+            if start_dt.hour in (8, 15):
+                annotations.append(
+                    {
+                        "type": "area",
+                        "label": "Trade open and close hours",
+                        "start": start_dt,
+                        "end": start_dt + timedelta(hours=1),
+                        # Omitting y_start and y_end will result in a vertical area spanning the whole height of the main Chart
+                        "color": "rgba(133, 133, 133, 0.4)",
+                    }
+                )
+
+        return annotations
+
+```
+
+Entries will be validated, and won't be passed to the UI if they don't correspond to the expected schema and will log an error if they don't.
+
+!!! Warning "Many annotations"
+    Using too many annotations can cause the UI to hang, especially when plotting large amounts of historic data.
+    Use the annotation feature with care.
+
+### Plot annotations example
+
+![FreqUI - plot Annotations](assets/freqUI-chart-annotations-dark.png#only-dark)
+![FreqUI - plot Annotations](assets/freqUI-chart-annotations-light.png#only-light)
+
+??? Info "Code used for the plot above"
+    This is an example code and should be treated as such.
+
+    ``` python
+    # Default imports
+
+    class AwesomeStrategy(IStrategy):
+        def plot_annotations(
+            self, pair: str, start_date: datetime, end_date: datetime, dataframe: DataFrame, **kwargs
+        ) -> list[AnnotationType]:
+            annotations = []
+            while start_dt < end_date:
+                start_dt += timedelta(hours=1)
+                if (start_dt.hour % 4) == 0:
+                    mark_areas.append(
+                        {
+                            "type": "area",
+                            "label": "4h",
+                            "start": start_dt,
+                            "end": start_dt + timedelta(hours=1),
+                            "color": "rgba(133, 133, 133, 0.4)",
+                        }
+                    )
+                elif (start_dt.hour % 2) == 0:
+                price = dataframe.loc[dataframe["date"] == start_dt, ["close"]].mean()
+                    mark_areas.append(
+                        {
+                            "type": "area",
+                            "label": "2h",
+                            "start": start_dt,
+                            "end": start_dt + timedelta(hours=1),
+                            "y_end": price * 1.01,
+                            "y_start": price * 0.99,
+                            "color": "rgba(0, 255, 0, 0.4)",
+                        }
+                    )
+
+            return annotations
+
+    ```

docs/trade-object.md

@@ -25,6 +25,7 @@ The following attributes / properties are available for each individual trade -
 | `close_date_utc` | datetime | Timestamp when trade was closed - in UTC. |
 | `close_profit` | float | Relative profit at the time of trade closure. `0.01` == 1% |
 | `close_profit_abs` | float | Absolute profit (in stake currency) at the time of trade closure. |
+| `realized_profit` | float | Absolute already realized profit (in stake currency) while the trade is still open. |
 | `leverage` | float | Leverage used for this trade - defaults to 1.0 in spot markets. |
 | `enter_tag` | string | Tag provided on entry via the `enter_tag` column in the dataframe. |
 | `is_short` | boolean | True for short trades, False otherwise. |
@@ -133,15 +134,17 @@ Most properties here can be None as they are dependent on the exchange response.
 |------------|-------------|-------------|
 | `trade` | Trade | Trade object this order is attached to |
 | `ft_pair` | string | Pair this order is for |
-| `ft_is_open` | boolean | is the order filled? |
+| `ft_is_open` | boolean | is the order still open? |
 | `order_type` | string | Order type as defined on the exchange - usually market, limit or stoploss |
-| `status` | string | Status as defined by ccxt. Usually open, closed, expired or canceled |
+| `status` | string | Status as defined by [ccxt's order structure](https://docs.ccxt.com/#/README?id=order-structure). Usually open, closed, expired, canceled or rejected |
-| `side` | string | Buy or Sell |
+| `side` | string | buy or sell |
 | `price` | float | Price the order was placed at |
 | `average` | float | Average price the order filled at |
 | `amount` | float | Amount in base currency |
-| `filled` | float | Filled amount (in base currency) |
+| `filled` | float | Filled amount (in base currency) (use `safe_filled` instead) |
-| `remaining` | float | Remaining amount |
+| `safe_filled` | float | Filled amount (in base currency) - guaranteed to not be None |
+| `remaining` | float | Remaining amount (use `safe_remaining` instead) |
+| `safe_remaining` | float | Remaining amount - either taken from the exchange or calculated. |
 | `cost` | float | Cost of the order - usually average * filled (*Exchange dependent on futures, may contain the cost with or without leverage and may be in contracts.*) |
 | `stake_amount` | float | Stake amount used for this order. *Added in 2023.7.* |
 | `stake_amount_filled` | float | Filled Stake amount used for this order. *Added in 2024.11.* |

freqtrade/__init__.py

@@ -1,6 +1,6 @@
 """Freqtrade bot"""
 
-__version__ = "2025.4-dev"
+__version__ = "2025.5-dev"
 
 if "dev" in __version__:
     from pathlib import Path

freqtrade/commands/cli_options.py

@@ -83,7 +83,8 @@ AVAILABLE_CLI_OPTIONS = {
         "-d",
         "--datadir",
         "--data-dir",
-        help="Path to directory with historical backtesting data.",
+        help="Path to the base directory of the exchange with historical backtesting data. "
+        "To see futures data, use trading-mode additionally.",
         metavar="PATH",
     ),
     "user_data_dir": Arg(

freqtrade/configuration/config_validation.py

@@ -104,7 +104,7 @@ def _validate_unlimited_amount(conf: dict[str, Any]) -> None:
     """
     if (
         not conf.get("edge", {}).get("enabled")
-        and conf.get("max_open_trades") == float("inf")
+        and (conf.get("max_open_trades") == float("inf") or conf.get("max_open_trades") == -1)
         and conf.get("stake_amount") == UNLIMITED_STAKE_AMOUNT
     ):
         raise ConfigurationError("`max_open_trades` and `stake_amount` cannot both be unlimited.")

freqtrade/constants.py

@@ -71,6 +71,19 @@ DEFAULT_DATAFRAME_COLUMNS = ["date", "open", "high", "low", "close", "volume"]
 # it has wide consequences for stored trades files
 DEFAULT_TRADES_COLUMNS = ["timestamp", "id", "type", "side", "price", "amount", "cost"]
 DEFAULT_ORDERFLOW_COLUMNS = ["level", "bid", "ask", "delta"]
+ORDERFLOW_ADDED_COLUMNS = [
+    "trades",
+    "orderflow",
+    "imbalances",
+    "stacked_imbalances_bid",
+    "stacked_imbalances_ask",
+    "max_delta",
+    "min_delta",
+    "bid",
+    "ask",
+    "delta",
+    "total_trades",
+]
 TRADES_DTYPES = {
     "timestamp": "int64",
     "id": "str",

freqtrade/data/btanalysis/__init__.py

@@ -0,0 +1,32 @@
+# flake8: noqa: F401
+from .bt_fileutils import (
+    BT_DATA_COLUMNS,
+    delete_backtest_result,
+    extract_trades_of_period,
+    find_existing_backtest_stats,
+    get_backtest_market_change,
+    get_backtest_result,
+    get_backtest_resultlist,
+    get_latest_backtest_filename,
+    get_latest_hyperopt_file,
+    get_latest_hyperopt_filename,
+    get_latest_optimize_filename,
+    load_and_merge_backtest_result,
+    load_backtest_analysis_data,
+    load_backtest_data,
+    load_backtest_metadata,
+    load_backtest_stats,
+    load_exit_signal_candles,
+    load_file_from_zip,
+    load_rejected_signals,
+    load_signal_candles,
+    load_trades,
+    load_trades_from_db,
+    trade_list_to_dataframe,
+    update_backtest_metadata,
+)
+from .historic_precision import get_tick_size_over_time
+from .trade_parallelism import (
+    analyze_trade_parallelism,
+    evaluate_result_multi,
+)

freqtrade/data/btanalysis/bt_fileutils.py

@@ -13,7 +13,7 @@ from typing import Any, Literal
 import numpy as np
 import pandas as pd
 
-from freqtrade.constants import LAST_BT_RESULT_FN, IntOrInf
+from freqtrade.constants import LAST_BT_RESULT_FN
 from freqtrade.exceptions import ConfigurationError, OperationalException
 from freqtrade.ft_types import BacktestHistoryEntryType, BacktestResultType
 from freqtrade.misc import file_dump_json, json_load
@@ -52,6 +52,7 @@ BT_DATA_COLUMNS = [
     "open_timestamp",
     "close_timestamp",
     "orders",
+    "funding_fees",
 ]
 
 
@@ -356,6 +357,8 @@ def _load_backtest_data_df_compatibility(df: pd.DataFrame) -> pd.DataFrame:
         df["max_stake_amount"] = df["stake_amount"]
     if "orders" not in df.columns:
         df["orders"] = None
+    if "funding_fees" not in df.columns:
+        df["funding_fees"] = 0.0
     return df
 
 
@@ -491,55 +494,6 @@ def load_exit_signal_candles(backtest_dir: Path) -> dict[str, dict[str, pd.DataF
     return load_backtest_analysis_data(backtest_dir, "exited")
 
 
-def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataFrame:
-    """
-    Find overlapping trades by expanding each trade once per period it was open
-    and then counting overlaps.
-    :param results: Results Dataframe - can be loaded
-    :param timeframe: Timeframe used for backtest
-    :return: dataframe with open-counts per time-period in timeframe
-    """
-    from freqtrade.exchange import timeframe_to_resample_freq
-
-    timeframe_freq = timeframe_to_resample_freq(timeframe)
-    dates = [
-        pd.Series(
-            pd.date_range(
-                row[1]["open_date"],
-                row[1]["close_date"],
-                freq=timeframe_freq,
-                # Exclude right boundary - the date is the candle open date.
-                inclusive="left",
-            )
-        )
-        for row in results[["open_date", "close_date"]].iterrows()
-    ]
-    deltas = [len(x) for x in dates]
-    dates = pd.Series(pd.concat(dates).values, name="date")
-    df2 = pd.DataFrame(np.repeat(results.values, deltas, axis=0), columns=results.columns)
-
-    df2 = pd.concat([dates, df2], axis=1)
-    df2 = df2.set_index("date")
-    df_final = df2.resample(timeframe_freq)[["pair"]].count()
-    df_final = df_final.rename({"pair": "open_trades"}, axis=1)
-    return df_final
-
-
-def evaluate_result_multi(
-    results: pd.DataFrame, timeframe: str, max_open_trades: IntOrInf
-) -> pd.DataFrame:
-    """
-    Find overlapping trades by expanding each trade once per period it was open
-    and then counting overlaps
-    :param results: Results Dataframe - can be loaded
-    :param timeframe: Frequency used for the backtest
-    :param max_open_trades: parameter max_open_trades used during backtest run
-    :return: dataframe with open-counts per time-period in freq
-    """
-    df_final = analyze_trade_parallelism(results, timeframe)
-    return df_final[df_final["open_trades"] > max_open_trades]
-
-
 def trade_list_to_dataframe(trades: list[Trade] | list[LocalTrade]) -> pd.DataFrame:
     """
     Convert list of Trade objects to pandas Dataframe

freqtrade/data/btanalysis/historic_precision.py

@@ -0,0 +1,27 @@
+from pandas import DataFrame, Series
+
+
+def get_tick_size_over_time(candles: DataFrame) -> Series:
+    """
+    Calculate the number of significant digits for candles over time.
+    It's using the Monthly maximum of the number of significant digits for each month.
+    :param candles: DataFrame with OHLCV data
+    :return: Series with the average number of significant digits for each month
+    """
+    # count the number of significant digits for the open and close prices
+    for col in ["open", "high", "low", "close"]:
+        candles[f"{col}_count"] = (
+            candles[col].round(14).astype(str).str.extract(r"\.(\d*[1-9])")[0].str.len()
+        )
+    candles["max_count"] = candles[["open_count", "close_count", "high_count", "low_count"]].max(
+        axis=1
+    )
+
+    candles1 = candles.set_index("date", drop=True)
+    # Group by month and calculate the average number of significant digits
+    monthly_count_avg1 = candles1["max_count"].resample("MS").max()
+    # monthly_open_count_avg
+    # convert monthly_open_count_avg from 5.0 to 0.00001, 4.0 to 0.0001, ...
+    monthly_open_count_avg = 1 / 10**monthly_count_avg1
+
+    return monthly_open_count_avg

freqtrade/data/btanalysis/trade_parallelism.py

@@ -0,0 +1,60 @@
+import logging
+
+import numpy as np
+import pandas as pd
+
+from freqtrade.constants import IntOrInf
+
+
+logger = logging.getLogger(__name__)
+
+
+def analyze_trade_parallelism(trades: pd.DataFrame, timeframe: str) -> pd.DataFrame:
+    """
+    Find overlapping trades by expanding each trade once per period it was open
+    and then counting overlaps.
+    :param trades: Trades Dataframe - can be loaded from backtest, or created
+        via trade_list_to_dataframe
+    :param timeframe: Timeframe used for backtest
+    :return: dataframe with open-counts per time-period in timeframe
+    """
+    from freqtrade.exchange import timeframe_to_resample_freq
+
+    timeframe_freq = timeframe_to_resample_freq(timeframe)
+    dates = [
+        pd.Series(
+            pd.date_range(
+                row[1]["open_date"],
+                row[1]["close_date"],
+                freq=timeframe_freq,
+                # Exclude right boundary - the date is the candle open date.
+                inclusive="left",
+            )
+        )
+        for row in trades[["open_date", "close_date"]].iterrows()
+    ]
+    deltas = [len(x) for x in dates]
+    dates = pd.Series(pd.concat(dates).values, name="date")
+    df2 = pd.DataFrame(np.repeat(trades.values, deltas, axis=0), columns=trades.columns)
+
+    df2 = pd.concat([dates, df2], axis=1)
+    df2 = df2.set_index("date")
+    df_final = df2.resample(timeframe_freq)[["pair"]].count()
+    df_final = df_final.rename({"pair": "open_trades"}, axis=1)
+    return df_final
+
+
+def evaluate_result_multi(
+    trades: pd.DataFrame, timeframe: str, max_open_trades: IntOrInf
+) -> pd.DataFrame:
+    """
+    Find overlapping trades by expanding each trade once per period it was open
+    and then counting overlaps
+    :param trades: Trades Dataframe - can be loaded from backtest, or created
+        via trade_list_to_dataframe
+    :param timeframe: Frequency used for the backtest
+    :param max_open_trades: parameter max_open_trades used during backtest run
+    :return: dataframe with open-counts per time-period in freq
+    """
+    df_final = analyze_trade_parallelism(trades, timeframe)
+    return df_final[df_final["open_trades"] > max_open_trades]

freqtrade/data/converter/orderflow.py

@@ -9,26 +9,12 @@ from datetime import datetime
 import numpy as np
 import pandas as pd
 
-from freqtrade.constants import DEFAULT_ORDERFLOW_COLUMNS, Config
+from freqtrade.constants import DEFAULT_ORDERFLOW_COLUMNS, ORDERFLOW_ADDED_COLUMNS, Config
 from freqtrade.exceptions import DependencyException
 
 
 logger = logging.getLogger(__name__)
 
-ORDERFLOW_ADDED_COLUMNS = [
-    "trades",
-    "orderflow",
-    "imbalances",
-    "stacked_imbalances_bid",
-    "stacked_imbalances_ask",
-    "max_delta",
-    "min_delta",
-    "bid",
-    "ask",
-    "delta",
-    "total_trades",
-]
-
 
 def _init_dataframe_with_trades_columns(dataframe: pd.DataFrame):
     """

freqtrade/data/metrics.py

@@ -10,7 +10,9 @@ import pandas as pd
 logger = logging.getLogger(__name__)
 
 
-def calculate_market_change(data: dict[str, pd.DataFrame], column: str = "close") -> float:
+def calculate_market_change(
+    data: dict[str, pd.DataFrame], column: str = "close", min_date: datetime | None = None
+) -> float:
     """
     Calculate market change based on "column".
     Calculation is done by taking the first non-null and the last non-null element of each column
@@ -19,14 +21,24 @@ def calculate_market_change(data: dict[str, pd.DataFrame], column: str = "close"
 
     :param data: Dict of Dataframes, dict key should be pair.
     :param column: Column in the original dataframes to use
+    :param min_date: Minimum date to consider for calculations. Market change should only be
+        calculated for data actually backtested, excluding startup periods.
     :return:
     """
     tmp_means = []
     for pair, df in data.items():
-        start = df[column].dropna().iloc[0]
+        df1 = df
-        end = df[column].dropna().iloc[-1]
+        if min_date is not None:
+            df1 = df1[df1["date"] >= min_date]
+        if df1.empty:
+            logger.warning(f"Pair {pair} has no data after {min_date}.")
+            continue
+        start = df1[column].dropna().iloc[0]
+        end = df1[column].dropna().iloc[-1]
         tmp_means.append((end - start) / start)
 
+    if not tmp_means:
+        return 0.0
     return float(np.mean(tmp_means))
 
 

freqtrade/exchange/binance.py

@@ -143,7 +143,7 @@ class Binance(Exchange):
         Does not work for other exchanges, which don't return the earliest data when called with "0"
         :param candle_type: Any of the enum CandleType (must match trading mode!)
         """
-        if is_new_pair:
+        if is_new_pair and candle_type in (CandleType.SPOT, CandleType.FUTURES, CandleType.MARK):
             with self._loop_lock:
                 x = self.loop.run_until_complete(
                     self._async_get_candle_history(pair, timeframe, candle_type, 0)

freqtrade/exchange/exchange.py

@@ -267,11 +267,11 @@ class Exchange:
             exchange_conf.get("ccxt_async_config", {}), ccxt_async_config
         )
         self._api_async = self._init_ccxt(exchange_conf, False, ccxt_async_config)
-        self._has_watch_ohlcv = self.exchange_has("watchOHLCV") and self._ft_has["ws_enabled"]
+        _has_watch_ohlcv = self.exchange_has("watchOHLCV") and self._ft_has["ws_enabled"]
         if (
             self._config["runmode"] in TRADE_MODES
             and exchange_conf.get("enable_ws", True)
-            and self._has_watch_ohlcv
+            and _has_watch_ohlcv
         ):
             self._ws_async = self._init_ccxt(exchange_conf, False, ccxt_async_config)
             self._exchange_ws = ExchangeWS(self._config, self._ws_async)
@@ -961,7 +961,7 @@ class Exchange:
             return 1 / pow(10, precision)
 
     def get_min_pair_stake_amount(
-        self, pair: str, price: float, stoploss: float, leverage: float | None = 1.0
+        self, pair: str, price: float, stoploss: float, leverage: float = 1.0
     ) -> float | None:
         return self._get_stake_amount_limit(pair, price, stoploss, "min", leverage)
 
@@ -980,7 +980,7 @@ class Exchange:
         price: float,
         stoploss: float,
         limit: Literal["min", "max"],
-        leverage: float | None = 1.0,
+        leverage: float = 1.0,
     ) -> float | None:
         isMin = limit == "min"
 
@@ -989,6 +989,8 @@ class Exchange:
         except KeyError:
             raise ValueError(f"Can't get market information for symbol {pair}")
 
+        stake_limits = []
+        limits = market["limits"]
         if isMin:
             # reserve some percent defined in config (5% default) + stoploss
             margin_reserve: float = 1.0 + self._config.get(
@@ -998,11 +1000,12 @@ class Exchange:
             # it should not be more than 50%
             stoploss_reserve = max(min(stoploss_reserve, 1.5), 1)
         else:
+            # is_max
             margin_reserve = 1.0
             stoploss_reserve = 1.0
+            if max_from_tiers := self._get_max_notional_from_tiers(pair, leverage=leverage):
+                stake_limits.append(max_from_tiers)
 
-        stake_limits = []
-        limits = market["limits"]
         if limits["cost"][limit] is not None:
             stake_limits.append(
                 self._contracts_to_amount(pair, limits["cost"][limit]) * stoploss_reserve
@@ -2411,6 +2414,45 @@ class Exchange:
         data = sorted(data, key=lambda x: x[0])
         return pair, timeframe, candle_type, data, self._ohlcv_partial_candle
 
+    def _try_build_from_websocket(
+        self, pair: str, timeframe: str, candle_type: CandleType
+    ) -> Coroutine[Any, Any, OHLCVResponse] | None:
+        """
+        Try to build a coroutine to get data from websocket.
+        """
+        if self._can_use_websocket(self._exchange_ws, pair, timeframe, candle_type):
+            candle_ts = dt_ts(timeframe_to_prev_date(timeframe))
+            prev_candle_ts = dt_ts(date_minus_candles(timeframe, 1))
+            candles = self._exchange_ws.ohlcvs(pair, timeframe)
+            half_candle = int(candle_ts - (candle_ts - prev_candle_ts) * 0.5)
+            last_refresh_time = int(
+                self._exchange_ws.klines_last_refresh.get((pair, timeframe, candle_type), 0)
+            )
+
+            if candles and candles[-1][0] >= prev_candle_ts and last_refresh_time >= half_candle:
+                # Usable result, candle contains the previous candle.
+                # Also, we check if the last refresh time is no more than half the candle ago.
+                logger.debug(f"reuse watch result for {pair}, {timeframe}, {last_refresh_time}")
+
+                return self._exchange_ws.get_ohlcv(pair, timeframe, candle_type, candle_ts)
+            logger.info(
+                f"Couldn't reuse watch for {pair}, {timeframe}, falling back to REST api. "
+                f"{candle_ts < last_refresh_time}, {candle_ts}, {last_refresh_time}, "
+                f"{format_ms_time(candle_ts)}, {format_ms_time(last_refresh_time)} "
+            )
+        return None
+
+    def _can_use_websocket(
+        self, exchange_ws: ExchangeWS | None, pair: str, timeframe: str, candle_type: CandleType
+    ) -> TypeGuard[ExchangeWS]:
+        """
+        Check if we can use websocket for this pair.
+        Acts as typeguard for exchangeWs
+        """
+        if exchange_ws and candle_type in (CandleType.SPOT, CandleType.FUTURES):
+            return True
+        return False
+
     def _build_coroutine(
         self,
         pair: str,
@@ -2420,8 +2462,8 @@ class Exchange:
         cache: bool,
     ) -> Coroutine[Any, Any, OHLCVResponse]:
         not_all_data = cache and self.required_candle_call_count > 1
-        if cache and candle_type in (CandleType.SPOT, CandleType.FUTURES):
+        if cache:
-            if self._has_watch_ohlcv and self._exchange_ws:
+            if self._can_use_websocket(self._exchange_ws, pair, timeframe, candle_type):
                 # Subscribe to websocket
                 self._exchange_ws.schedule_ohlcv(pair, timeframe, candle_type)
 
@@ -2429,30 +2471,9 @@ class Exchange:
             candle_limit = self.ohlcv_candle_limit(timeframe, candle_type)
             min_ts = dt_ts(date_minus_candles(timeframe, candle_limit - 5))
 
-            if self._exchange_ws:
+            if ws_resp := self._try_build_from_websocket(pair, timeframe, candle_type):
-                candle_ts = dt_ts(timeframe_to_prev_date(timeframe))
+                # We have a usable websocket response
-                prev_candle_ts = dt_ts(date_minus_candles(timeframe, 1))
+                return ws_resp
-                candles = self._exchange_ws.ohlcvs(pair, timeframe)
-                half_candle = int(candle_ts - (candle_ts - prev_candle_ts) * 0.5)
-                last_refresh_time = int(
-                    self._exchange_ws.klines_last_refresh.get((pair, timeframe, candle_type), 0)
-                )
-
-                if (
-                    candles
-                    and candles[-1][0] >= prev_candle_ts
-                    and last_refresh_time >= half_candle
-                ):
-                    # Usable result, candle contains the previous candle.
-                    # Also, we check if the last refresh time is no more than half the candle ago.
-                    logger.debug(f"reuse watch result for {pair}, {timeframe}, {last_refresh_time}")
-
-                    return self._exchange_ws.get_ohlcv(pair, timeframe, candle_type, candle_ts)
-                logger.info(
-                    f"Couldn't reuse watch for {pair}, {timeframe}, falling back to REST api. "
-                    f"{candle_ts < last_refresh_time}, {candle_ts}, {last_refresh_time}, "
-                    f"{format_ms_time(candle_ts)}, {format_ms_time(last_refresh_time)} "
-                )
 
             # Check if 1 call can get us updated candles without hole in the data.
             if min_ts < self._pairs_last_refresh_time.get((pair, timeframe, candle_type), 0):
@@ -3361,42 +3382,22 @@ class Exchange:
             pair_tiers = self._leverage_tiers[pair]
 
             if stake_amount == 0:
-                return self._leverage_tiers[pair][0]["maxLeverage"]  # Max lev for lowest amount
+                return pair_tiers[0]["maxLeverage"]  # Max lev for lowest amount
 
-            for tier_index in range(len(pair_tiers)):
+            # Find the appropriate tier based on stake_amount
-                tier = pair_tiers[tier_index]
+            prior_max_lev = None
-                lev = tier["maxLeverage"]
+            for tier in pair_tiers:
+                min_stake = tier["minNotional"] / (prior_max_lev or tier["maxLeverage"])
+                max_stake = tier["maxNotional"] / tier["maxLeverage"]
+                prior_max_lev = tier["maxLeverage"]
+                # Adjust notional by leverage to do a proper comparison
+                if min_stake <= stake_amount <= max_stake:
+                    return tier["maxLeverage"]
 
-                if tier_index < len(pair_tiers) - 1:
+            #     else:  # if on the last tier
-                    next_tier = pair_tiers[tier_index + 1]
+            if stake_amount > max_stake:
-                    next_floor = next_tier["minNotional"] / next_tier["maxLeverage"]
+                # If stake is > than max tradeable amount
-                    if next_floor > stake_amount:  # Next tier min too high for stake amount
+                raise InvalidOrderException(f"Amount {stake_amount} too high for {pair}")
-                        return min((tier["maxNotional"] / stake_amount), lev)
-                        #
-                        # With the two leverage tiers below,
-                        # - a stake amount of 150 would mean a max leverage of (10000 / 150) = 66.66
-                        # - stakes below 133.33 = max_lev of 75
-                        # - stakes between 133.33-200 = max_lev of 10000/stake = 50.01-74.99
-                        # - stakes from 200 + 1000 = max_lev of 50
-                        #
-                        # {
-                        #     "min": 0,      # stake = 0.0
-                        #     "max": 10000,  # max_stake@75 = 10000/75 = 133.33333333333334
-                        #     "lev": 75,
-                        # },
-                        # {
-                        #     "min": 10000,  # stake = 200.0
-                        #     "max": 50000,  # max_stake@50 = 50000/50 = 1000.0
-                        #     "lev": 50,
-                        # }
-                        #
-
-                else:  # if on the last tier
-                    if stake_amount > tier["maxNotional"]:
-                        # If stake is > than max tradeable amount
-                        raise InvalidOrderException(f"Amount {stake_amount} too high for {pair}")
-                    else:
-                        return tier["maxLeverage"]
 
             raise OperationalException(
                 "Looped through all tiers without finding a max leverage. Should never be reached"
@@ -3411,6 +3412,23 @@ class Exchange:
         else:
             return 1.0
 
+    def _get_max_notional_from_tiers(self, pair: str, leverage: float) -> float | None:
+        """
+        get max_notional from leverage_tiers
+        :param pair: The base/quote currency pair being traded
+        :param leverage: The leverage to be used
+        :return: The maximum notional value for the given leverage or None if not found
+        """
+        if self.trading_mode != TradingMode.FUTURES:
+            return None
+        if pair not in self._leverage_tiers:
+            return None
+        pair_tiers = self._leverage_tiers[pair]
+        for tier in reversed(pair_tiers):
+            if leverage <= tier["maxLeverage"]:
+                return tier["maxNotional"]
+        return None
+
     @retrier
     def _set_leverage(
         self,

freqtrade/freqai/data_kitchen.py

@@ -16,7 +16,7 @@ from pandas import DataFrame
 from sklearn.model_selection import train_test_split
 
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import DOCS_LINK, Config
+from freqtrade.constants import DOCS_LINK, ORDERFLOW_ADDED_COLUMNS, Config
 from freqtrade.data.converter import reduce_dataframe_footprint
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_seconds
@@ -709,6 +709,11 @@ class FreqaiDataKitchen:
         skip_columns = [
             (f"{s}_{suffix}") for s in ["date", "open", "high", "low", "close", "volume"]
         ]
+
+        for s in ORDERFLOW_ADDED_COLUMNS:
+            if s in dataframe.columns and f"{s}_{suffix}" in dataframe.columns:
+                skip_columns.append(f"{s}_{suffix}")
+
         dataframe = dataframe.drop(columns=skip_columns)
         return dataframe
 

freqtrade/freqtradebot.py

@@ -760,12 +760,14 @@ class FreqtradeBot(LoggingMixin):
         current_exit_profit = trade.calc_profit_ratio(current_exit_rate)
 
         min_entry_stake = self.exchange.get_min_pair_stake_amount(
-            trade.pair, current_entry_rate, 0.0
+            trade.pair, current_entry_rate, 0.0, trade.leverage
         )
         min_exit_stake = self.exchange.get_min_pair_stake_amount(
-            trade.pair, current_exit_rate, self.strategy.stoploss
+            trade.pair, current_exit_rate, self.strategy.stoploss, trade.leverage
+        )
+        max_entry_stake = self.exchange.get_max_pair_stake_amount(
+            trade.pair, current_entry_rate, trade.leverage
         )
-        max_entry_stake = self.exchange.get_max_pair_stake_amount(trade.pair, current_entry_rate)
         stake_available = self.wallets.get_available_stake_amount()
         logger.debug(f"Calling adjust_trade_position for pair {trade.pair}")
         stake_amount, order_tag = self.strategy._adjust_trade_position_internal(

freqtrade/ft_types/__init__.py

@@ -7,4 +7,5 @@ from freqtrade.ft_types.backtest_result_type import (
     BacktestResultType,
     get_BacktestResultType_default,
 )
+from freqtrade.ft_types.plot_annotation_type import AnnotationType
 from freqtrade.ft_types.valid_exchanges_type import ValidExchangesType

freqtrade/ft_types/plot_annotation_type.py

@@ -0,0 +1,18 @@
+from datetime import datetime
+from typing import Literal
+
+from pydantic import TypeAdapter
+from typing_extensions import Required, TypedDict
+
+
+class AnnotationType(TypedDict, total=False):
+    type: Required[Literal["area"]]
+    start: str | datetime
+    end: str | datetime
+    y_start: float
+    y_end: float
+    color: str
+    label: str
+
+
+AnnotationTypeTA = TypeAdapter(AnnotationType)

freqtrade/loggers/__init__.py

@@ -123,7 +123,7 @@ def _add_formatter(log_config: dict[str, Any], format_name: str, format_: str):
 
 def _create_log_config(config: Config) -> dict[str, Any]:
     # Get log_config from user config or use default
-    log_config = config.get("log_config", deepcopy(FT_LOGGING_CONFIG))
+    log_config = deepcopy(config.get("log_config", FT_LOGGING_CONFIG))
 
     if logfile := config.get("logfile"):
         s = logfile.split(":")

freqtrade/optimize/analysis/lookahead.py

@@ -129,7 +129,6 @@ class LookaheadAnalysis(BaseAnalysis):
         backtesting._set_strategy(backtesting.strategylist[0])
 
         varholder.data, varholder.timerange = backtesting.load_bt_data()
-        backtesting.load_bt_data_detail()
         varholder.timeframe = backtesting.timeframe
 
         varholder.indicators = backtesting.strategy.advise_all_indicators(varholder.data)

freqtrade/optimize/analysis/recursive.py

@@ -149,7 +149,6 @@ class RecursiveAnalysis(BaseAnalysis):
         backtesting._set_strategy(backtesting.strategylist[0])
 
         varholder.data, varholder.timerange = backtesting.load_bt_data()
-        backtesting.load_bt_data_detail()
         varholder.timeframe = backtesting.timeframe
 
         varholder.indicators = backtesting.strategy.advise_all_indicators(varholder.data)

freqtrade/optimize/backtesting.py

@@ -9,14 +9,18 @@ from collections import defaultdict
 from copy import deepcopy
 from datetime import datetime, timedelta
 
-from numpy import nan
+from numpy import isnan, nan
-from pandas import DataFrame
+from pandas import DataFrame, Series
 
 from freqtrade import constants
 from freqtrade.configuration import TimeRange, validate_config_consistency
 from freqtrade.constants import DATETIME_PRINT_FORMAT, Config, IntOrInf, LongShort
 from freqtrade.data import history
-from freqtrade.data.btanalysis import find_existing_backtest_stats, trade_list_to_dataframe
+from freqtrade.data.btanalysis import (
+    find_existing_backtest_stats,
+    get_tick_size_over_time,
+    trade_list_to_dataframe,
+)
 from freqtrade.data.converter import trim_dataframe, trim_dataframes
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.data.metrics import combined_dataframes_with_rel_mean
@@ -35,7 +39,7 @@ from freqtrade.exchange import (
     price_to_precision,
     timeframe_to_seconds,
 )
-from freqtrade.exchange.exchange import Exchange
+from freqtrade.exchange.exchange import TICK_SIZE, Exchange
 from freqtrade.ft_types import (
     BacktestContentType,
     BacktestContentTypeIcomplete,
@@ -121,9 +125,10 @@ class Backtesting:
         self.order_id_counter: int = 0
 
         config["dry_run"] = True
+        self.price_pair_prec: dict[str, Series] = {}
         self.run_ids: dict[str, str] = {}
         self.strategylist: list[IStrategy] = []
-        self.all_results: dict[str, BacktestContentType] = {}
+        self.all_bt_content: dict[str, BacktestContentType] = {}
         self.analysis_results: dict[str, dict[str, DataFrame]] = {
             "signals": {},
             "rejected": {},
@@ -315,9 +320,15 @@ class Backtesting:
         )
 
         self.progress.set_new_value(1)
+        self._load_bt_data_detail()
+        self.price_pair_prec = {}
+        for pair in self.pairlists.whitelist:
+            if pair in data:
+                # Load price precision logic
+                self.price_pair_prec[pair] = get_tick_size_over_time(data[pair])
         return data, self.timerange
 
-    def load_bt_data_detail(self) -> None:
+    def _load_bt_data_detail(self) -> None:
         """
         Loads backtest detail data (smaller timeframe) if necessary.
         """
@@ -384,6 +395,22 @@ class Backtesting:
         else:
             self.futures_data = {}
 
+    def get_pair_precision(self, pair: str, current_time: datetime) -> tuple[float | None, int]:
+        """
+        Get pair precision at that moment in time
+        :param pair: Pair to get precision for
+        :param current_time: Time to get precision for
+        :return: tuple of price precision, precision_mode_price for the pair at that given time.
+        """
+        precision_series = self.price_pair_prec.get(pair)
+        if precision_series is not None:
+            precision = precision_series.asof(current_time)
+
+            if not isnan(precision):
+                # Force tick size if we define the precision
+                return precision, TICK_SIZE
+        return self.exchange.get_precision_price(pair), self.precision_mode_price
+
     def disable_database_use(self):
         disable_database_use(self.timeframe)
 
@@ -806,7 +833,7 @@ class Backtesting:
                     )
                     if rate is not None and rate != close_rate:
                         close_rate = price_to_precision(
-                            rate, trade.price_precision, self.precision_mode_price
+                            rate, trade.price_precision, trade.precision_mode_price
                         )
                     # We can't place orders lower than current low.
                     # freqtrade does not support this in live, and the order would fill immediately
@@ -939,6 +966,7 @@ class Backtesting:
         trade: LocalTrade | None,
         order_type: str,
         price_precision: float | None,
+        precision_mode_price: int,
     ) -> tuple[float, float, float, float]:
         if order_type == "limit":
             new_rate = strategy_safe_wrapper(
@@ -954,9 +982,7 @@ class Backtesting:
             # We can't place orders higher than current high (otherwise it'd be a stop limit entry)
             # which freqtrade does not support in live.
             if new_rate is not None and new_rate != propose_rate:
-                propose_rate = price_to_precision(
+                propose_rate = price_to_precision(new_rate, price_precision, precision_mode_price)
-                    new_rate, price_precision, self.precision_mode_price
-                )
             if direction == "short":
                 propose_rate = max(propose_rate, row[LOW_IDX])
             else:
@@ -1049,7 +1075,7 @@ class Backtesting:
         pos_adjust = trade is not None and requested_rate is None
 
         stake_amount_ = stake_amount or (trade.stake_amount if trade else 0.0)
-        precision_price = self.exchange.get_precision_price(pair)
+        precision_price, precision_mode_price = self.get_pair_precision(pair, current_time)
 
         propose_rate, stake_amount, leverage, min_stake_amount = self.get_valid_price_and_stake(
             pair,
@@ -1062,6 +1088,7 @@ class Backtesting:
             trade,
             order_type,
             precision_price,
+            precision_mode_price,
         )
 
         # replace proposed rate if another rate was requested
@@ -1137,7 +1164,7 @@ class Backtesting:
                     amount_precision=precision_amount,
                     price_precision=precision_price,
                     precision_mode=self.precision_mode,
-                    precision_mode_price=self.precision_mode_price,
+                    precision_mode_price=precision_mode_price,
                     contract_size=contract_size,
                     orders=[],
                 )
@@ -1731,7 +1758,7 @@ class Backtesting:
                 "backtest_end_time": int(backtest_end_time.timestamp()),
             }
         )
-        self.all_results[strategy_name] = results
+        self.all_bt_content[strategy_name] = results
 
         if (
             self.config.get("export", "none") == "signals"
@@ -1781,7 +1808,6 @@ class Backtesting:
         data: dict[str, DataFrame] = {}
 
         data, timerange = self.load_bt_data()
-        self.load_bt_data_detail()
         logger.info("Dataload complete. Calculating indicators")
 
         self.load_prior_backtest()
@@ -1794,9 +1820,9 @@ class Backtesting:
             min_date, max_date = self.backtest_one_strategy(strat, data, timerange)
 
         # Update old results with new ones.
-        if len(self.all_results) > 0:
+        if len(self.all_bt_content) > 0:
             results = generate_backtest_stats(
-                data, self.all_results, min_date=min_date, max_date=max_date
+                data, self.all_bt_content, min_date=min_date, max_date=max_date
             )
             if self.results:
                 self.results["metadata"].update(results["metadata"])

freqtrade/optimize/hyperopt/hyperopt.py

@@ -4,6 +4,7 @@
 This module contains the hyperopt logic
 """
 
+import gc
 import logging
 import random
 from datetime import datetime
@@ -13,7 +14,7 @@ from pathlib import Path
 from typing import Any
 
 import rapidjson
-from joblib import Parallel, cpu_count, delayed, wrap_non_picklable_objects
+from joblib import Parallel, cpu_count
 
 from freqtrade.constants import FTHYPT_FILEVERSION, LAST_BT_RESULT_FN, Config
 from freqtrade.enums import HyperoptState
@@ -35,9 +36,6 @@ logger = logging.getLogger(__name__)
 
 INITIAL_POINTS = 30
 
-# Keep no more than SKOPT_MODEL_QUEUE_SIZE models
-# in the skopt model queue, to optimize memory consumption
-SKOPT_MODEL_QUEUE_SIZE = 10
 
 log_queue: Any
 
@@ -92,7 +90,7 @@ class Hyperopt:
         self.hyperopt_table_header = 0
         self.print_json = self.config.get("print_json", False)
 
-        self.hyperopter = HyperOptimizer(self.config)
+        self.hyperopter = HyperOptimizer(self.config, self.data_pickle_file)
 
     @staticmethod
     def get_lock_filename(config: Config) -> str:
@@ -158,14 +156,20 @@ class Hyperopt:
                 log_queue, logging.INFO if self.config["verbosity"] < 1 else logging.DEBUG
             )
 
-            return self.hyperopter.generate_optimizer(*args, **kwargs)
+            return self.hyperopter.generate_optimizer_wrapped(*args, **kwargs)
 
-        return parallel(delayed(wrap_non_picklable_objects(optimizer_wrapper))(v) for v in asked)
+        return parallel(optimizer_wrapper(v) for v in asked)
 
     def _set_random_state(self, random_state: int | None) -> int:
         return random_state or random.randint(1, 2**16 - 1)  # noqa: S311
 
-    def get_asked_points(self, n_points: int) -> tuple[list[list[Any]], list[bool]]:
+    def get_optuna_asked_points(self, n_points: int, dimensions: dict) -> list[Any]:
+        asked: list[list[Any]] = []
+        for i in range(n_points):
+            asked.append(self.opt.ask(dimensions))
+        return asked
+
+    def get_asked_points(self, n_points: int, dimensions: dict) -> tuple[list[Any], list[bool]]:
         """
         Enforce points returned from `self.opt.ask` have not been already evaluated
 
@@ -191,19 +195,19 @@ class Hyperopt:
         while i < 5 and len(asked_non_tried) < n_points:
             if i < 3:
                 self.opt.cache_ = {}
-                asked = unique_list(self.opt.ask(n_points=n_points * 5 if i > 0 else n_points))
+                asked = unique_list(
+                    self.get_optuna_asked_points(
+                        n_points=n_points * 5 if i > 0 else n_points, dimensions=dimensions
+                    )
+                )
                 is_random = [False for _ in range(len(asked))]
             else:
                 asked = unique_list(self.opt.space.rvs(n_samples=n_points * 5))
                 is_random = [True for _ in range(len(asked))]
             is_random_non_tried += [
-                rand
+                rand for x, rand in zip(asked, is_random, strict=False) if x not in asked_non_tried
-                for x, rand in zip(asked, is_random, strict=False)
-                if x not in self.opt.Xi and x not in asked_non_tried
-            ]
-            asked_non_tried += [
-                x for x in asked if x not in self.opt.Xi and x not in asked_non_tried
             ]
+            asked_non_tried += [x for x in asked if x not in asked_non_tried]
             i += 1
 
         if asked_non_tried:
@@ -212,7 +216,9 @@ class Hyperopt:
                 is_random_non_tried[: min(len(asked_non_tried), n_points)],
             )
         else:
-            return self.opt.ask(n_points=n_points), [False for _ in range(n_points)]
+            return self.get_optuna_asked_points(n_points=n_points, dimensions=dimensions), [
+                False for _ in range(n_points)
+            ]
 
     def evaluate_result(self, val: dict[str, Any], current: int, is_random: bool):
         """
@@ -258,9 +264,7 @@ class Hyperopt:
         config_jobs = self.config.get("hyperopt_jobs", -1)
         logger.info(f"Number of parallel jobs set as: {config_jobs}")
 
-        self.opt = self.hyperopter.get_optimizer(
+        self.opt = self.hyperopter.get_optimizer(self.random_state)
-            config_jobs, self.random_state, INITIAL_POINTS, SKOPT_MODEL_QUEUE_SIZE
-        )
         self._setup_logging_mp_workaround()
         try:
             with Parallel(n_jobs=config_jobs) as parallel:
@@ -276,9 +280,11 @@ class Hyperopt:
                     if self.analyze_per_epoch:
                         # First analysis not in parallel mode when using --analyze-per-epoch.
                         # This allows dataprovider to load it's informative cache.
-                        asked, is_random = self.get_asked_points(n_points=1)
+                        asked, is_random = self.get_asked_points(
-                        f_val0 = self.hyperopter.generate_optimizer(asked[0])
+                            n_points=1, dimensions=self.hyperopter.o_dimensions
-                        self.opt.tell(asked, [f_val0["loss"]])
+                        )
+                        f_val0 = self.hyperopter.generate_optimizer(asked[0].params)
+                        self.opt.tell(asked[0], [f_val0["loss"]])
                         self.evaluate_result(f_val0, 1, is_random[0])
                         pbar.update(task, advance=1)
                         start += 1
@@ -290,9 +296,17 @@ class Hyperopt:
                         n_rest = (i + 1) * jobs - (self.total_epochs - start)
                         current_jobs = jobs - n_rest if n_rest > 0 else jobs
 
-                        asked, is_random = self.get_asked_points(n_points=current_jobs)
+                        asked, is_random = self.get_asked_points(
-                        f_val = self.run_optimizer_parallel(parallel, asked)
+                            n_points=current_jobs, dimensions=self.hyperopter.o_dimensions
-                        self.opt.tell(asked, [v["loss"] for v in f_val])
+                        )
+
+                        f_val = self.run_optimizer_parallel(
+                            parallel,
+                            [asked1.params for asked1 in asked],
+                        )
+                        f_val_loss = [v["loss"] for v in f_val]
+                        for o_ask, v in zip(asked, f_val_loss, strict=False):
+                            self.opt.tell(o_ask, v)
 
                         for j, val in enumerate(f_val):
                             # Use human-friendly indexes here (starting from 1)
@@ -301,6 +315,7 @@ class Hyperopt:
                             self.evaluate_result(val, current, is_random[j])
                             pbar.update(task, advance=1)
                         logging_mp_handle(log_queue)
+                        gc.collect()
 
         except KeyboardInterrupt:
             print("User interrupted..")

freqtrade/optimize/hyperopt/hyperopt_auto.py

@@ -12,7 +12,7 @@ from freqtrade.exceptions import OperationalException
 
 
 with suppress(ImportError):
-    from skopt.space import Dimension
+    from freqtrade.optimize.space import Dimension
 
 from freqtrade.optimize.hyperopt.hyperopt_interface import EstimatorType, IHyperOpt
 

freqtrade/optimize/hyperopt/hyperopt_interface.py

@@ -8,19 +8,18 @@ import math
 from abc import ABC
 from typing import TypeAlias
 
-from sklearn.base import RegressorMixin
+from optuna.samplers import BaseSampler
-from skopt.space import Categorical, Dimension, Integer
 
 from freqtrade.constants import Config
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.misc import round_dict
-from freqtrade.optimize.space import SKDecimal
+from freqtrade.optimize.space import Categorical, Dimension, Integer, SKDecimal
 from freqtrade.strategy import IStrategy
 
 
 logger = logging.getLogger(__name__)
 
-EstimatorType: TypeAlias = RegressorMixin | str
+EstimatorType: TypeAlias = BaseSampler | str
 
 
 class IHyperOpt(ABC):
@@ -44,10 +43,11 @@ class IHyperOpt(ABC):
     def generate_estimator(self, dimensions: list[Dimension], **kwargs) -> EstimatorType:
         """
         Return base_estimator.
-        Can be any of "GP", "RF", "ET", "GBRT" or an instance of a class
+        Can be any of "TPESampler", "GPSampler", "CmaEsSampler", "NSGAIISampler"
-        inheriting from RegressorMixin (from sklearn).
+        "NSGAIIISampler", "QMCSampler" or an instance of a class
+        inheriting from BaseSampler (from optuna.samplers).
         """
-        return "ET"
+        return "NSGAIIISampler"
 
     def generate_roi_table(self, params: dict) -> dict[int, float]:
         """

freqtrade/optimize/hyperopt/hyperopt_optimizer.py

@@ -7,10 +7,13 @@ import logging
 import sys
 import warnings
 from datetime import datetime, timezone
+from pathlib import Path
 from typing import Any
 
-from joblib import dump, load
+import optuna
+from joblib import delayed, dump, load, wrap_non_picklable_objects
 from joblib.externals import cloudpickle
+from optuna.exceptions import ExperimentalWarning
 from pandas import DataFrame
 
 from freqtrade.constants import DATETIME_PRINT_FORMAT, Config
@@ -20,7 +23,7 @@ from freqtrade.data.metrics import calculate_market_change
 from freqtrade.enums import HyperoptState
 from freqtrade.exceptions import OperationalException
 from freqtrade.ft_types import BacktestContentType
-from freqtrade.misc import deep_merge_dicts
+from freqtrade.misc import deep_merge_dicts, round_dict
 from freqtrade.optimize.backtesting import Backtesting
 
 # Import IHyperOptLoss to allow unpickling classes from these modules
@@ -28,21 +31,31 @@ from freqtrade.optimize.hyperopt.hyperopt_auto import HyperOptAuto
 from freqtrade.optimize.hyperopt_loss.hyperopt_loss_interface import IHyperOptLoss
 from freqtrade.optimize.hyperopt_tools import HyperoptStateContainer, HyperoptTools
 from freqtrade.optimize.optimize_reports import generate_strategy_stats
+from freqtrade.optimize.space import (
+    DimensionProtocol,
+    SKDecimal,
+    ft_CategoricalDistribution,
+    ft_FloatDistribution,
+    ft_IntDistribution,
+)
 from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver
 from freqtrade.util.dry_run_wallet import get_dry_run_wallet
 
 
-# Suppress scikit-learn FutureWarnings from skopt
-with warnings.catch_warnings():
-    warnings.filterwarnings("ignore", category=FutureWarning)
-    from skopt import Optimizer
-    from skopt.space import Dimension
-
 logger = logging.getLogger(__name__)
 
 
 MAX_LOSS = 100000  # just a big enough number to be bad result in loss optimization
 
+optuna_samplers_dict = {
+    "TPESampler": optuna.samplers.TPESampler,
+    "GPSampler": optuna.samplers.GPSampler,
+    "CmaEsSampler": optuna.samplers.CmaEsSampler,
+    "NSGAIISampler": optuna.samplers.NSGAIISampler,
+    "NSGAIIISampler": optuna.samplers.NSGAIIISampler,
+    "QMCSampler": optuna.samplers.QMCSampler,
+}
+
 
 class HyperOptimizer:
     """
@@ -50,15 +63,16 @@ class HyperOptimizer:
     This class is sent to the hyperopt worker processes.
     """
 
-    def __init__(self, config: Config) -> None:
+    def __init__(self, config: Config, data_pickle_file: Path) -> None:
-        self.buy_space: list[Dimension] = []
+        self.buy_space: list[DimensionProtocol] = []
-        self.sell_space: list[Dimension] = []
+        self.sell_space: list[DimensionProtocol] = []
-        self.protection_space: list[Dimension] = []
+        self.protection_space: list[DimensionProtocol] = []
-        self.roi_space: list[Dimension] = []
+        self.roi_space: list[DimensionProtocol] = []
-        self.stoploss_space: list[Dimension] = []
+        self.stoploss_space: list[DimensionProtocol] = []
-        self.trailing_space: list[Dimension] = []
+        self.trailing_space: list[DimensionProtocol] = []
-        self.max_open_trades_space: list[Dimension] = []
+        self.max_open_trades_space: list[DimensionProtocol] = []
-        self.dimensions: list[Dimension] = []
+        self.dimensions: list[DimensionProtocol] = []
+        self.o_dimensions: dict = {}
 
         self.config = config
         self.min_date: datetime
@@ -86,9 +100,7 @@ class HyperOptimizer:
         )
         self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function
 
-        self.data_pickle_file = (
+        self.data_pickle_file = data_pickle_file
-            self.config["user_data_dir"] / "hyperopt_results" / "hyperopt_tickerdata.pkl"
-        )
 
         self.market_change = 0.0
 
@@ -127,18 +139,6 @@ class HyperOptimizer:
                     cloudpickle.register_pickle_by_value(mod)
                 self.hyperopt_pickle_magic(modules.__bases__)
 
-    def _get_params_dict(
-        self, dimensions: list[Dimension], raw_params: list[Any]
-    ) -> dict[str, Any]:
-        # Ensure the number of dimensions match
-        # the number of parameters in the list.
-        if len(raw_params) != len(dimensions):
-            raise ValueError("Mismatch in number of search-space dimensions.")
-
-        # Return a dict where the keys are the names of the dimensions
-        # and the values are taken from the list of parameters.
-        return {d.name: v for d, v in zip(dimensions, raw_params, strict=False)}
-
     def _get_params_details(self, params: dict) -> dict:
         """
         Return the params for each space
@@ -146,27 +146,36 @@ class HyperOptimizer:
         result: dict = {}
 
         if HyperoptTools.has_space(self.config, "buy"):
-            result["buy"] = {p.name: params.get(p.name) for p in self.buy_space}
+            result["buy"] = round_dict({p.name: params.get(p.name) for p in self.buy_space}, 13)
         if HyperoptTools.has_space(self.config, "sell"):
-            result["sell"] = {p.name: params.get(p.name) for p in self.sell_space}
+            result["sell"] = round_dict({p.name: params.get(p.name) for p in self.sell_space}, 13)
         if HyperoptTools.has_space(self.config, "protection"):
-            result["protection"] = {p.name: params.get(p.name) for p in self.protection_space}
+            result["protection"] = round_dict(
+                {p.name: params.get(p.name) for p in self.protection_space}, 13
+            )
         if HyperoptTools.has_space(self.config, "roi"):
-            result["roi"] = {
+            result["roi"] = round_dict(
-                str(k): v for k, v in self.custom_hyperopt.generate_roi_table(params).items()
+                {str(k): v for k, v in self.custom_hyperopt.generate_roi_table(params).items()}, 13
-            }
+            )
         if HyperoptTools.has_space(self.config, "stoploss"):
-            result["stoploss"] = {p.name: params.get(p.name) for p in self.stoploss_space}
+            result["stoploss"] = round_dict(
+                {p.name: params.get(p.name) for p in self.stoploss_space}, 13
+            )
         if HyperoptTools.has_space(self.config, "trailing"):
-            result["trailing"] = self.custom_hyperopt.generate_trailing_params(params)
+            result["trailing"] = round_dict(
+                self.custom_hyperopt.generate_trailing_params(params), 13
+            )
         if HyperoptTools.has_space(self.config, "trades"):
-            result["max_open_trades"] = {
+            result["max_open_trades"] = round_dict(
-                "max_open_trades": (
+                {
-                    self.backtesting.strategy.max_open_trades
+                    "max_open_trades": (
-                    if self.backtesting.strategy.max_open_trades != float("inf")
+                        self.backtesting.strategy.max_open_trades
-                    else -1
+                        if self.backtesting.strategy.max_open_trades != float("inf")
-                )
+                        else -1
-            }
+                    )
+                },
+                13,
+            )
 
         return result
 
@@ -246,7 +255,12 @@ class HyperOptimizer:
                 # noinspection PyProtectedMember
                 attr.value = params_dict[attr_name]
 
-    def generate_optimizer(self, raw_params: list[Any]) -> dict[str, Any]:
+    @delayed
+    @wrap_non_picklable_objects
+    def generate_optimizer_wrapped(self, params_dict: dict[str, Any]) -> dict[str, Any]:
+        return self.generate_optimizer(params_dict)
+
+    def generate_optimizer(self, params_dict: dict[str, Any]) -> dict[str, Any]:
         """
         Used Optimize function.
         Called once per epoch to optimize whatever is configured.
@@ -254,7 +268,6 @@ class HyperOptimizer:
         """
         HyperoptStateContainer.set_state(HyperoptState.OPTIMIZE)
         backtest_start_time = datetime.now(timezone.utc)
-        params_dict = self._get_params_dict(self.dimensions, raw_params)
 
         # Apply parameters
         if HyperoptTools.has_space(self.config, "buy"):
@@ -304,9 +317,9 @@ class HyperOptimizer:
 
         with self.data_pickle_file.open("rb") as f:
             processed = load(f, mmap_mode="r")
-            if self.analyze_per_epoch:
+        if self.analyze_per_epoch:
-                # Data is not yet analyzed, rerun populate_indicators.
+            # Data is not yet analyzed, rerun populate_indicators.
-                processed = self.advise_and_trim(processed)
+            processed = self.advise_and_trim(processed)
 
         bt_results = self.backtesting.backtest(
             processed=processed, start_date=self.min_date, end_date=self.max_date
@@ -318,10 +331,10 @@ class HyperOptimizer:
                 "backtest_end_time": int(backtest_end_time.timestamp()),
             }
         )
-
+        result = self._get_results_dict(
-        return self._get_results_dict(
             bt_results, self.min_date, self.max_date, params_dict, processed=processed
         )
+        return result
 
     def _get_results_dict(
         self,
@@ -378,33 +391,41 @@ class HyperOptimizer:
             "total_profit": total_profit,
         }
 
+    def convert_dimensions_to_optuna_space(self, s_dimensions: list[DimensionProtocol]) -> dict:
+        o_dimensions: dict[str, optuna.distributions.BaseDistribution] = {}
+        for original_dim in s_dimensions:
+            if isinstance(
+                original_dim,
+                ft_CategoricalDistribution | ft_IntDistribution | ft_FloatDistribution | SKDecimal,
+            ):
+                o_dimensions[original_dim.name] = original_dim
+            else:
+                raise OperationalException(
+                    f"Unknown search space {original_dim.name} - {original_dim} / \
+                        {type(original_dim)}"
+                )
+        return o_dimensions
+
     def get_optimizer(
         self,
-        cpu_count: int,
         random_state: int,
-        initial_points: int,
+    ):
-        model_queue_size: int,
+        o_sampler = self.custom_hyperopt.generate_estimator(
-    ) -> Optimizer:
+            dimensions=self.dimensions, random_state=random_state
-        dimensions = self.dimensions
-        estimator = self.custom_hyperopt.generate_estimator(dimensions=dimensions)
-
-        acq_optimizer = "sampling"
-        if isinstance(estimator, str):
-            if estimator not in ("GP", "RF", "ET", "GBRT"):
-                raise OperationalException(f"Estimator {estimator} not supported.")
-            else:
-                acq_optimizer = "auto"
-
-        logger.info(f"Using estimator {estimator}.")
-        return Optimizer(
-            dimensions,
-            base_estimator=estimator,
-            acq_optimizer=acq_optimizer,
-            n_initial_points=initial_points,
-            acq_optimizer_kwargs={"n_jobs": cpu_count},
-            random_state=random_state,
-            model_queue_size=model_queue_size,
         )
+        self.o_dimensions = self.convert_dimensions_to_optuna_space(self.dimensions)
+
+        if isinstance(o_sampler, str):
+            if o_sampler not in optuna_samplers_dict.keys():
+                raise OperationalException(f"Optuna Sampler {o_sampler} not supported.")
+            with warnings.catch_warnings():
+                warnings.filterwarnings(action="ignore", category=ExperimentalWarning)
+                sampler = optuna_samplers_dict[o_sampler](seed=random_state)
+        else:
+            sampler = o_sampler
+
+        logger.info(f"Using optuna sampler {o_sampler}.")
+        return optuna.create_study(sampler=sampler, direction="minimize")
 
     def advise_and_trim(self, data: dict[str, DataFrame]) -> dict[str, DataFrame]:
         preprocessed = self.backtesting.strategy.advise_all_indicators(data)
@@ -423,7 +444,6 @@ class HyperOptimizer:
     def prepare_hyperopt_data(self) -> None:
         HyperoptStateContainer.set_state(HyperoptState.DATALOAD)
         data, self.timerange = self.backtesting.load_bt_data()
-        self.backtesting.load_bt_data_detail()
         logger.info("Dataload complete. Calculating indicators")
 
         if not self.analyze_per_epoch:

freqtrade/optimize/optimize_reports/bt_output.py

@@ -102,7 +102,9 @@ def text_table_tags(
         [
             *(
                 (
-                    (t["key"] if isinstance(t["key"], list) else [t["key"], ""])
+                    list(t["key"])
+                    if isinstance(t["key"], list | tuple)
+                    else [t["key"], ""]
                     if is_list
                     else [t["key"]]
                 )

freqtrade/optimize/optimize_reports/optimize_reports.py

@@ -649,7 +649,7 @@ def generate_backtest_stats(
     :return: Dictionary containing results per strategy and a strategy summary.
     """
     result: BacktestResultType = get_BacktestResultType_default()
-    market_change = calculate_market_change(btdata, "close")
+    market_change = calculate_market_change(btdata, "close", min_date=min_date)
     metadata = {}
     pairlist = list(btdata.keys())
     for strategy, content in all_results.items():

freqtrade/optimize/space/__init__.py

@@ -1,3 +1,16 @@
-from skopt.space import Categorical, Dimension, Integer, Real  # noqa: F401
+from .decimalspace import SKDecimal
+from .optunaspaces import (
+    DimensionProtocol,
+    ft_CategoricalDistribution,
+    ft_FloatDistribution,
+    ft_IntDistribution,
+)
 
-from .decimalspace import SKDecimal  # noqa: F401
+
+# Alias for the distribution classes
+Dimension = DimensionProtocol
+Categorical = ft_CategoricalDistribution
+Integer = ft_IntDistribution
+Real = ft_FloatDistribution
+
+__all__ = ["Categorical", "Dimension", "Integer", "Real", "SKDecimal"]

freqtrade/optimize/space/decimalspace.py

@@ -1,47 +1,35 @@
-import numpy as np
+from optuna.distributions import FloatDistribution
-from skopt.space import Integer
 
 
-class SKDecimal(Integer):
+class SKDecimal(FloatDistribution):
     def __init__(
         self,
-        low,
+        low: float,
-        high,
+        high: float,
-        decimals=3,
+        *,
-        prior="uniform",
+        step: float | None = None,
-        base=10,
+        decimals: int | None = None,
-        transform=None,
         name=None,
-        dtype=np.int64,
     ):
-        self.decimals = decimals
+        """
+        FloatDistribution with a fixed step size.
+        Only one of step or decimals can be set.
+        :param low: lower bound
+        :param high: upper bound
+        :param step: step size (e.g. 0.001)
+        :param decimals: number of decimal places to round to (e.g. 3)
+        :param name: name of the distribution
+        """
+        if decimals is not None and step is not None:
+            raise ValueError("You can only set one of decimals or step")
+        if decimals is None and step is None:
+            raise ValueError("You must set one of decimals or step")
+        # Convert decimals to step
+        self.step = step or (1 / 10**decimals if decimals else 1)
+        self.name = name
 
-        self.pow_dot_one = pow(0.1, self.decimals)
+        super().__init__(
-        self.pow_ten = pow(10, self.decimals)
+            low=round(low, decimals) if decimals else low,
-
+            high=round(high, decimals) if decimals else high,
-        _low = int(low * self.pow_ten)
+            step=self.step,
-        _high = int(high * self.pow_ten)
-        # trunc to precision to avoid points out of space
-        self.low_orig = round(_low * self.pow_dot_one, self.decimals)
-        self.high_orig = round(_high * self.pow_dot_one, self.decimals)
-
-        super().__init__(_low, _high, prior, base, transform, name, dtype)
-
-    def __repr__(self):
-        return (
-            f"Decimal(low={self.low_orig}, high={self.high_orig}, decimals={self.decimals}, "
-            f"prior='{self.prior}', transform='{self.transform_}')"
         )
-
-    def __contains__(self, point):
-        if isinstance(point, list):
-            point = np.array(point)
-        return self.low_orig <= point <= self.high_orig
-
-    def transform(self, Xt):
-        return super().transform([int(v * self.pow_ten) for v in Xt])
-
-    def inverse_transform(self, Xt):
-        res = super().inverse_transform(Xt)
-        # equivalent to [round(x * pow(0.1, self.decimals), self.decimals) for x in res]
-        return [int(v) / self.pow_ten for v in res]

freqtrade/optimize/space/optunaspaces.py

@@ -0,0 +1,59 @@
+from collections.abc import Sequence
+from typing import Any, Protocol
+
+from optuna.distributions import CategoricalDistribution, FloatDistribution, IntDistribution
+
+
+class DimensionProtocol(Protocol):
+    name: str
+
+
+class ft_CategoricalDistribution(CategoricalDistribution):
+    def __init__(
+        self,
+        categories: Sequence[Any],
+        name: str,
+        **kwargs,
+    ):
+        self.name = name
+        self.categories = categories
+        # if len(categories) <= 1:
+        #     raise Exception(f"need at least 2 categories for {name}")
+        return super().__init__(categories)
+
+    def __repr__(self):
+        return f"CategoricalDistribution({self.categories})"
+
+
+class ft_IntDistribution(IntDistribution):
+    def __init__(
+        self,
+        low: int | float,
+        high: int | float,
+        name: str,
+        **kwargs,
+    ):
+        self.name = name
+        self.low = int(low)
+        self.high = int(high)
+        return super().__init__(self.low, self.high, **kwargs)
+
+    def __repr__(self):
+        return f"IntDistribution(low={self.low}, high={self.high})"
+
+
+class ft_FloatDistribution(FloatDistribution):
+    def __init__(
+        self,
+        low: float,
+        high: float,
+        name: str,
+        **kwargs,
+    ):
+        self.name = name
+        self.low = low
+        self.high = high
+        return super().__init__(low, high, **kwargs)
+
+    def __repr__(self):
+        return f"FloatDistribution(low={self.low}, high={self.high}, step={self.step})"

freqtrade/persistence/key_value_store.py

@@ -1,6 +1,6 @@
 from datetime import datetime, timezone
 from enum import Enum
-from typing import ClassVar
+from typing import ClassVar, Literal
 
 from sqlalchemy import String
 from sqlalchemy.orm import Mapped, mapped_column
@@ -18,9 +18,11 @@ class ValueTypesEnum(str, Enum):
     INT = "int"
 
 
-class KeyStoreKeys(str, Enum):
+KeyStoreKeys = Literal[
-    BOT_START_TIME = "bot_start_time"
+    "bot_start_time",
-    STARTUP_TIME = "startup_time"
+    "startup_time",
+    "binance_migration",
+]
 
 
 class _KeyValueStoreModel(ModelBase):
@@ -192,7 +194,7 @@ class KeyValueStore:
         return kv.int_value
 
 
-def set_startup_time():
+def set_startup_time() -> None:
     """
     sets bot_start_time to the first trade open date - or "now" on new databases.
     sets startup_time to "now"

freqtrade/persistence/migrations.py

@@ -1,6 +1,6 @@
 import logging
 
-from sqlalchemy import inspect, select, text, update
+from sqlalchemy import Engine, inspect, select, text, update
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.persistence.trade_model import Order, Trade
@@ -9,7 +9,7 @@ from freqtrade.persistence.trade_model import Order, Trade
 logger = logging.getLogger(__name__)
 
 
-def get_table_names_for_table(inspector, tabletype) -> list[str]:
+def get_table_names_for_table(inspector, tabletype: str) -> list[str]:
     return [t for t in inspector.get_table_names() if t.startswith(tabletype)]
 
 
@@ -350,7 +350,7 @@ def fix_wrong_max_stake_amount(engine):
         connection.execute(stmt)
 
 
-def check_migrate(engine, decl_base, previous_tables) -> None:
+def check_migrate(engine: Engine, decl_base, previous_tables: list[str]) -> None:
     """
     Checks if migration is necessary and migrates if necessary
     """

freqtrade/persistence/trade_model.py

@@ -1027,12 +1027,12 @@ class LocalTrade:
         Calculate the open_rate including open_fee.
         :return: Price in of the open trade incl. Fees
         """
-        open_trade = FtPrecise(amount) * FtPrecise(open_rate)
+        open_value = FtPrecise(amount) * FtPrecise(open_rate)
-        fees = open_trade * FtPrecise(self.fee_open)
+        fees = open_value * FtPrecise(self.fee_open)
         if self.is_short:
-            return float(open_trade - fees)
+            return float(open_value - fees)
         else:
-            return float(open_trade + fees)
+            return float(open_value + fees)
 
     def recalc_open_trade_value(self) -> None:
         """
@@ -1062,13 +1062,13 @@ class LocalTrade:
         return interest(exchange_name=self.exchange, borrowed=borrowed, rate=rate, hours=hours)
 
     def _calc_base_close(self, amount: FtPrecise, rate: float, fee: float | None) -> FtPrecise:
-        close_trade = amount * FtPrecise(rate)
+        close_value = amount * FtPrecise(rate)
-        fees = close_trade * FtPrecise(fee or 0.0)
+        fees = close_value * FtPrecise(fee or 0.0)
 
         if self.is_short:
-            return close_trade + fees
+            return close_value + fees
         else:
-            return close_trade - fees
+            return close_value - fees
 
     def calc_close_trade_value(self, rate: float, amount: float | None = None) -> float:
         """
@@ -1802,11 +1802,15 @@ class Trade(ModelBase, LocalTrade):
         close_date: datetime | None = None,
     ) -> list["LocalTrade"]:
         """
-        Helper function to query Trades.j
+        Helper function to query Trades.
         Returns a List of trades, filtered on the parameters given.
         In live mode, converts the filter to a database query and returns all rows
         In Backtest mode, uses filters on Trade.bt_trades to get the result.
-
+        :param pair: Filter by pair
+        :param is_open: Filter by open/closed status
+        :param open_date: Filter by open_date (filters via trade.open_date > input)
+        :param close_date: Filter by close_date (filters via trade.close_date > input)
+                           and will implicitly only return closed trades.
         :return: unsorted List[Trade]
         """
         if Trade.use_db:

freqtrade/resolvers/strategy_resolver.py

@@ -79,7 +79,7 @@ class StrategyResolver(IResolver):
             ("ignore_buying_expired_candle_after", 0),
             ("position_adjustment_enable", False),
             ("max_entry_position_adjustment", -1),
-            ("max_open_trades", -1),
+            ("max_open_trades", float("inf")),
         ]
         for attribute, default in attributes:
             StrategyResolver._override_attribute_helper(strategy, config, attribute, default)

freqtrade/rpc/api_server/api_backtest.py

@@ -62,7 +62,6 @@ def __run_backtest_bg(btconfig: Config):
             from freqtrade.optimize.backtesting import Backtesting
 
             ApiBG.bt["bt"] = Backtesting(btconfig)
-            ApiBG.bt["bt"].load_bt_data_detail()
         else:
             ApiBG.bt["bt"].config = btconfig
             ApiBG.bt["bt"].init_backtest()
@@ -96,7 +95,10 @@ def __run_backtest_bg(btconfig: Config):
             )
 
             ApiBG.bt["bt"].results = generate_backtest_stats(
-                ApiBG.bt["data"], ApiBG.bt["bt"].all_results, min_date=min_date, max_date=max_date
+                ApiBG.bt["data"],
+                ApiBG.bt["bt"].all_bt_content,
+                min_date=min_date,
+                max_date=max_date,
             )
 
             if btconfig.get("export", "none") == "trades":

freqtrade/rpc/api_server/api_schemas.py

@@ -5,7 +5,7 @@ from pydantic import AwareDatetime, BaseModel, RootModel, SerializeAsAny, model_
 
 from freqtrade.constants import DL_DATA_TIMEFRAMES, IntOrInf
 from freqtrade.enums import MarginMode, OrderTypeValues, SignalDirection, TradingMode
-from freqtrade.ft_types import ValidExchangesType
+from freqtrade.ft_types import AnnotationType, ValidExchangesType
 from freqtrade.rpc.api_server.webserver_bgwork import ProgressTask
 
 
@@ -539,6 +539,7 @@ class PairHistory(BaseModel):
     columns: list[str]
     all_columns: list[str] = []
     data: SerializeAsAny[list[Any]]
+    annotations: list[AnnotationType] | None = None
     length: int
     buy_signals: int
     sell_signals: int

freqtrade/rpc/rpc.py

@@ -32,8 +32,9 @@ from freqtrade.enums import (
 from freqtrade.exceptions import ExchangeError, PricingError
 from freqtrade.exchange import Exchange, timeframe_to_minutes, timeframe_to_msecs
 from freqtrade.exchange.exchange_utils import price_to_precision
+from freqtrade.ft_types import AnnotationType
 from freqtrade.loggers import bufferHandler
-from freqtrade.persistence import CustomDataWrapper, KeyStoreKeys, KeyValueStore, PairLocks, Trade
+from freqtrade.persistence import CustomDataWrapper, KeyValueStore, PairLocks, Trade
 from freqtrade.persistence.models import PairLock
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
@@ -635,7 +636,7 @@ class RPC:
         first_date = trades[0].open_date_utc if trades else None
         last_date = trades[-1].open_date_utc if trades else None
         num = float(len(durations) or 1)
-        bot_start = KeyValueStore.get_datetime_value(KeyStoreKeys.BOT_START_TIME)
+        bot_start = KeyValueStore.get_datetime_value("bot_start_time")
         return {
             "profit_closed_coin": profit_closed_coin_sum,
             "profit_closed_percent_mean": round(profit_closed_ratio_mean * 100, 2),
@@ -1356,6 +1357,7 @@ class RPC:
         dataframe: DataFrame,
         last_analyzed: datetime,
         selected_cols: list[str] | None,
+        annotations: list[AnnotationType],
     ) -> dict[str, Any]:
         has_content = len(dataframe) != 0
         dataframe_columns = list(dataframe.columns)
@@ -1411,6 +1413,7 @@ class RPC:
             "data_start_ts": 0,
             "data_stop": "",
             "data_stop_ts": 0,
+            "annotations": annotations,
         }
         if has_content:
             res.update(
@@ -1429,8 +1432,16 @@ class RPC:
         """Analyzed dataframe in Dict form"""
 
         _data, last_analyzed = self.__rpc_analysed_dataframe_raw(pair, timeframe, limit)
+        annotations = self._freqtrade.strategy.ft_plot_annotations(pair=pair, dataframe=_data)
+
         return RPC._convert_dataframe_to_dict(
-            self._freqtrade.config["strategy"], pair, timeframe, _data, last_analyzed, selected_cols
+            self._freqtrade.config["strategy"],
+            pair,
+            timeframe,
+            _data,
+            last_analyzed,
+            selected_cols,
+            annotations,
         )
 
     def __rpc_analysed_dataframe_raw(
@@ -1531,6 +1542,7 @@ class RPC:
                 )
             data = _data[pair]
 
+        annotations = []
         if config.get("strategy"):
             strategy.dp = DataProvider(config, exchange=exchange, pairlists=None)
             strategy.ft_bot_start()
@@ -1539,6 +1551,8 @@ class RPC:
             df_analyzed = trim_dataframe(
                 df_analyzed, timerange_parsed, startup_candles=startup_candles
             )
+            annotations = strategy.ft_plot_annotations(pair=pair, dataframe=df_analyzed)
+
         else:
             df_analyzed = data
 
@@ -1549,6 +1563,7 @@ class RPC:
             df_analyzed.copy(),
             dt_now(),
             selected_cols,
+            annotations,
         )
 
     def _rpc_plot_config(self) -> dict[str, Any]:
@@ -1601,7 +1616,7 @@ class RPC:
                 }
             )
 
-        if bot_start := KeyValueStore.get_datetime_value(KeyStoreKeys.BOT_START_TIME):
+        if bot_start := KeyValueStore.get_datetime_value("bot_start_time"):
             res.update(
                 {
                     "bot_start": str(bot_start),
@@ -1609,7 +1624,7 @@ class RPC:
                     "bot_start_ts": int(bot_start.timestamp()),
                 }
             )
-        if bot_startup := KeyValueStore.get_datetime_value(KeyStoreKeys.STARTUP_TIME):
+        if bot_startup := KeyValueStore.get_datetime_value("startup_time"):
             res.update(
                 {
                     "bot_startup": str(bot_startup),

freqtrade/strategy/__init__.py

@@ -6,6 +6,7 @@ from freqtrade.exchange import (
     timeframe_to_prev_date,
     timeframe_to_seconds,
 )
+from freqtrade.ft_types import AnnotationType
 from freqtrade.persistence import Order, PairLocks, Trade
 from freqtrade.strategy.informative_decorator import informative
 from freqtrade.strategy.interface import IStrategy
@@ -44,4 +45,5 @@ __all__ = [
     "merge_informative_pair",
     "stoploss_from_absolute",
     "stoploss_from_open",
+    "AnnotationType",
 ]

freqtrade/strategy/interface.py

@@ -9,6 +9,7 @@ from datetime import datetime, timedelta, timezone
 from math import isinf, isnan
 
 from pandas import DataFrame
+from pydantic import ValidationError
 
 from freqtrade.constants import CUSTOM_TAG_MAX_LENGTH, Config, IntOrInf, ListPairsWithTimeframes
 from freqtrade.data.converter import populate_dataframe_with_trades
@@ -27,6 +28,7 @@ from freqtrade.enums import (
 )
 from freqtrade.exceptions import OperationalException, StrategyError
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date, timeframe_to_seconds
+from freqtrade.ft_types import AnnotationType
 from freqtrade.misc import remove_entry_exit_signals
 from freqtrade.persistence import Order, PairLocks, Trade
 from freqtrade.strategy.hyper import HyperStrategyMixin
@@ -862,6 +864,24 @@ class IStrategy(ABC, HyperStrategyMixin):
         """
         return None
 
+    def plot_annotations(
+        self, pair: str, start_date: datetime, end_date: datetime, dataframe: DataFrame, **kwargs
+    ) -> list[AnnotationType]:
+        """
+        Retrieve area annotations for a chart.
+        Must be returned as array, with type, label, color, start, end, y_start, y_end.
+        All settings except for type are optional - though it usually makes sense to include either
+        "start and end" or "y_start and y_end" for either horizontal or vertical plots
+        (or all 4 for boxes).
+        :param pair: Pair that's currently analyzed
+        :param start_date: Start date of the chart data being requested
+        :param end_date: End date of the chart data being requested
+        :param dataframe: DataFrame with the analyzed data for the chart
+        :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
+        :return: List of AnnotationType objects
+        """
+        return []
+
     def populate_any_indicators(
         self,
         pair: str,
@@ -1839,3 +1859,33 @@ class IStrategy(ABC, HyperStrategyMixin):
         if "exit_long" not in df.columns:
             df = df.rename({"sell": "exit_long"}, axis="columns")
         return df
+
+    def ft_plot_annotations(self, pair: str, dataframe: DataFrame) -> list[AnnotationType]:
+        """
+        Internal wrapper around plot_dataframe
+        """
+        if len(dataframe) > 0:
+            annotations = strategy_safe_wrapper(self.plot_annotations)(
+                pair=pair,
+                dataframe=dataframe,
+                start_date=dataframe.iloc[0]["date"].to_pydatetime(),
+                end_date=dataframe.iloc[-1]["date"].to_pydatetime(),
+            )
+
+            from freqtrade.ft_types.plot_annotation_type import AnnotationTypeTA
+
+            annotations_new: list[AnnotationType] = []
+            for annotation in annotations:
+                if isinstance(annotation, dict):
+                    # Convert to AnnotationType
+                    try:
+                        AnnotationTypeTA.validate_python(annotation)
+                        annotations_new.append(annotation)
+                    except ValidationError as e:
+                        logger.error(f"Invalid annotation data: {annotation}. Error: {e}")
+                else:
+                    # Already an AnnotationType
+                    annotations_new.append(annotation)
+
+            return annotations_new
+        return []

freqtrade/strategy/parameters.py

@@ -14,9 +14,12 @@ from freqtrade.optimize.hyperopt_tools import HyperoptStateContainer
 
 
 with suppress(ImportError):
-    from skopt.space import Categorical, Integer, Real
+    from freqtrade.optimize.space import (
-
+        Categorical,
-    from freqtrade.optimize.space import SKDecimal
+        Integer,
+        Real,
+        SKDecimal,
+    )
 
 from freqtrade.exceptions import OperationalException
 
@@ -51,7 +54,8 @@ class BaseParameter(ABC):
          name is prefixed with 'buy_' or 'sell_'.
         :param optimize: Include parameter in hyperopt optimizations.
         :param load: Load parameter value from {space}_params.
-        :param kwargs: Extra parameters to skopt.space.(Integer|Real|Categorical).
+        :param kwargs: Extra parameters to optuna.distributions.
+                (IntDistribution|FloatDistribution|CategoricalDistribution).
         """
         if "name" in kwargs:
             raise OperationalException(
@@ -109,7 +113,7 @@ class NumericParameter(BaseParameter):
                       parameter fieldname is prefixed with 'buy_' or 'sell_'.
         :param optimize: Include parameter in hyperopt optimizations.
         :param load: Load parameter value from {space}_params.
-        :param kwargs: Extra parameters to skopt.space.*.
+        :param kwargs: Extra parameters to optuna.distributions.*.
         """
         if high is not None and isinstance(low, Sequence):
             raise OperationalException(f"{self.__class__.__name__} space invalid.")
@@ -151,7 +155,7 @@ class IntParameter(NumericParameter):
                       parameter fieldname is prefixed with 'buy_' or 'sell_'.
         :param optimize: Include parameter in hyperopt optimizations.
         :param load: Load parameter value from {space}_params.
-        :param kwargs: Extra parameters to skopt.space.Integer.
+        :param kwargs: Extra parameters to optuna.distributions.IntDistribution.
         """
 
         super().__init__(
@@ -160,7 +164,7 @@ class IntParameter(NumericParameter):
 
     def get_space(self, name: str) -> "Integer":
         """
-        Create skopt optimization space.
+        Create optuna distribution space.
         :param name: A name of parameter field.
         """
         return Integer(low=self.low, high=self.high, name=name, **self._space_params)
@@ -174,7 +178,7 @@ class IntParameter(NumericParameter):
         calculating 100ds of indicators.
         """
         if self.can_optimize():
-            # Scikit-optimize ranges are "inclusive", while python's "range" is exclusive
+            # optuna distributions ranges are "inclusive", while python's "range" is exclusive
             return range(self.low, self.high + 1)
         else:
             return range(self.value, self.value + 1)
@@ -205,7 +209,7 @@ class RealParameter(NumericParameter):
                       parameter fieldname is prefixed with 'buy_' or 'sell_'.
         :param optimize: Include parameter in hyperopt optimizations.
         :param load: Load parameter value from {space}_params.
-        :param kwargs: Extra parameters to skopt.space.Real.
+        :param kwargs: Extra parameters to optuna.distributions.FloatDistribution.
         """
         super().__init__(
             low=low, high=high, default=default, space=space, optimize=optimize, load=load, **kwargs
@@ -213,7 +217,7 @@ class RealParameter(NumericParameter):
 
     def get_space(self, name: str) -> "Real":
         """
-        Create skopt optimization space.
+        Create optimization space.
         :param name: A name of parameter field.
         """
         return Real(low=self.low, high=self.high, name=name, **self._space_params)
@@ -246,7 +250,7 @@ class DecimalParameter(NumericParameter):
                       parameter fieldname is prefixed with 'buy_' or 'sell_'.
         :param optimize: Include parameter in hyperopt optimizations.
         :param load: Load parameter value from {space}_params.
-        :param kwargs: Extra parameters to skopt.space.Integer.
+        :param kwargs: Extra parameters to optuna's NumericParameter.
         """
         self._decimals = decimals
         default = round(default, self._decimals)
@@ -257,7 +261,7 @@ class DecimalParameter(NumericParameter):
 
     def get_space(self, name: str) -> "SKDecimal":
         """
-        Create skopt optimization space.
+        Create optimization space.
         :param name: A name of parameter field.
         """
         return SKDecimal(
@@ -305,7 +309,8 @@ class CategoricalParameter(BaseParameter):
          name is prefixed with 'buy_' or 'sell_'.
         :param optimize: Include parameter in hyperopt optimizations.
         :param load: Load parameter value from {space}_params.
-        :param kwargs: Extra parameters to skopt.space.Categorical.
+        :param kwargs: Compatibility. Optuna's CategoricalDistribution does not
+                       accept extra parameters
         """
         if len(categories) < 2:
             raise OperationalException(
@@ -316,10 +321,10 @@ class CategoricalParameter(BaseParameter):
 
     def get_space(self, name: str) -> "Categorical":
         """
-        Create skopt optimization space.
+        Create optuna distribution space.
         :param name: A name of parameter field.
         """
-        return Categorical(self.opt_range, name=name, **self._space_params)
+        return Categorical(self.opt_range, name=name)
 
     @property
     def range(self):
@@ -355,7 +360,7 @@ class BooleanParameter(CategoricalParameter):
          name is prefixed with 'buy_' or 'sell_'.
         :param optimize: Include parameter in hyperopt optimizations.
         :param load: Load parameter value from {space}_params.
-        :param kwargs: Extra parameters to skopt.space.Categorical.
+        :param kwargs: Extra parameters to optuna.distributions.CategoricalDistribution.
         """
 
         categories = [True, False]

freqtrade/templates/base_strategy.py.j2

@@ -28,12 +28,12 @@ from freqtrade.strategy import (
     merge_informative_pair,
     stoploss_from_absolute,
     stoploss_from_open,
+    AnnotationType,
 )
 
 # --------------------------------
 # Add your lib to import here
 import talib.abstract as ta
-import pandas_ta as pta
 from technical import qtpylib
 
 

freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2

@@ -399,3 +399,21 @@ def order_filled(
     :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
     """
     pass
+
+def plot_annotations(
+    self, pair: str, start_date: datetime, end_date: datetime, dataframe: DataFrame, **kwargs
+) -> list[AnnotationType]:
+    """
+    Retrieve area annotations for a chart.
+    Must be returned as array, with type, label, color, start, end, y_start, y_end.
+    All settings except for type are optional - though it usually makes sense to include either
+    "start and end" or "y_start and y_end" for either horizontal or vertical plots
+    (or all 4 for boxes).
+    :param pair: Pair that's currently analyzed
+    :param start_date: Start date of the chart data being requested
+    :param end_date: End date of the chart data being requested
+    :param dataframe: DataFrame with the analyzed data for the chart
+    :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
+    :return: List of AnnotationType objects
+    """
+    return []

freqtrade/util/migrations/__init__.py

@@ -4,6 +4,9 @@ from freqtrade.util.migrations.funding_rate_mig import migrate_funding_fee_timef
 
 
 def migrate_data(config, exchange: Exchange | None = None):
+    """
+    Migrate persisted data from old formats to new formats
+    """
     migrate_binance_futures_data(config)
 
     migrate_funding_fee_timeframe(config, exchange)

freqtrade/util/migrations/binance_mig.py

@@ -6,8 +6,8 @@ from sqlalchemy import select
 from freqtrade.constants import DOCS_LINK, Config
 from freqtrade.enums import TradingMode
 from freqtrade.exceptions import OperationalException
+from freqtrade.persistence import KeyValueStore, Trade
 from freqtrade.persistence.pairlock import PairLock
-from freqtrade.persistence.trade_model import Trade
 
 
 logger = logging.getLogger(__name__)
@@ -20,6 +20,9 @@ def migrate_binance_futures_names(config: Config):
     ):
         # only act on new futures
         return
+    if KeyValueStore.get_int_value("binance_migration"):
+        # already migrated
+        return
     import ccxt
 
     if version.parse("2.6.26") > version.parse(ccxt.__version__):
@@ -29,10 +32,11 @@ def migrate_binance_futures_names(config: Config):
         )
     _migrate_binance_futures_db(config)
     migrate_binance_futures_data(config)
+    KeyValueStore.store_value("binance_migration", 1)
 
 
 def _migrate_binance_futures_db(config: Config):
-    logger.warning("Migrating binance futures pairs in database.")
+    logger.info("Migrating binance futures pairs in database.")
     trades = Trade.get_trades([Trade.exchange == "binance", Trade.trading_mode == "FUTURES"]).all()
     for trade in trades:
         if ":" in trade.pair:
@@ -52,7 +56,7 @@ def _migrate_binance_futures_db(config: Config):
     # print(pls)
     # pls.update({'pair': concat(PairLock.pair,':USDT')})
     Trade.commit()
-    logger.warning("Done migrating binance futures pairs in database.")
+    logger.info("Done migrating binance futures pairs in database.")
 
 
 def migrate_binance_futures_data(config: Config):

ft_client/freqtrade_client/__init__.py

@@ -1,7 +1,7 @@
 from freqtrade_client.ft_rest_client import FtRestClient
 
 
-__version__ = "2025.4-dev"
+__version__ = "2025.5-dev"
 
 if "dev" in __version__:
     from pathlib import Path

pyproject.toml

@@ -79,7 +79,8 @@ plot = ["plotly>=4.0"]
 hyperopt = [
   "scipy",
   "scikit-learn",
-  "ft-scikit-optimize>=0.9.2",
+  "optuna > 4.0.0",
+  "cmaes",
   "filelock",
 ]
 freqai = [

requirements-dev.txt

@@ -7,7 +7,7 @@
 -r docs/requirements-docs.txt
 
 coveralls==4.0.1
-ruff==0.11.5
+ruff==0.11.9
 mypy==1.15.0
 pre-commit==4.2.0
 pytest==8.3.5
@@ -15,7 +15,7 @@ pytest-asyncio==0.26.0
 pytest-cov==6.1.1
 pytest-mock==3.14.0
 pytest-random-order==1.1.1
-pytest-timeout==2.3.1
+pytest-timeout==2.4.0
 pytest-xdist==3.6.1
 isort==6.0.1
 # For datetime mocking

requirements-freqai-rl.txt

@@ -2,7 +2,7 @@
 -r requirements-freqai.txt
 
 # Required for freqai-rl
-torch==2.6.0; sys_platform != 'darwin' or platform_machine != 'x86_64'
+torch==2.7.0; sys_platform != 'darwin' or platform_machine != 'x86_64'
 gymnasium==0.29.1
 # SB3 >=2.5.0 depends on torch 2.3.0 - which implies it dropped support x86 macos
 stable_baselines3==2.4.1; sys_platform == 'darwin' and platform_machine == 'x86_64'

requirements-freqai.txt

@@ -4,9 +4,9 @@
 
 # Required for freqai
 scikit-learn==1.6.1
-joblib==1.4.2
+joblib==1.5.0
 catboost==1.2.8; 'arm' not in platform_machine
 lightgbm==4.6.0
 xgboost==2.1.4
 tensorboard==2.19.0
-datasieve==0.1.7
+datasieve==0.1.9