Update download-artifact to pattern.

New release

.github/dependabot.yml

@@ -10,8 +10,17 @@ updates:
   directory: "/"
   schedule:
     interval: weekly
+    time: "03:00"
+    timezone: "Etc/UTC"
   open-pull-requests-limit: 15
   target-branch: develop
+  groups:
+    types:
+      patterns:
+        - "types-*"
+    pytest:
+      patterns:
+        - "pytest*"
 
 - package-ecosystem: "github-actions"
   directory: "/"

.github/workflows/binance-lev-tier-update.yml

@@ -0,0 +1,47 @@
+name: Binance Leverage tiers update
+
+on:
+  schedule:
+    - cron: "0 3 * * 4"
+  # on demand
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  auto-update:
+    runs-on: ubuntu-latest
+    environment:
+      name: develop
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: "3.11"
+
+    - name: Install ccxt
+      run: pip install ccxt
+
+    - name: Run leverage tier update
+      env:
+        CI_WEB_PROXY: ${{ secrets.CI_WEB_PROXY }}
+        FREQTRADE__EXCHANGE__KEY: ${{ secrets.BINANCE_EXCHANGE_KEY }}
+        FREQTRADE__EXCHANGE__SECRET: ${{ secrets.BINANCE_EXCHANGE_SECRET }}
+      run: python build_helpers/binance_update_lev_tiers.py
+
+
+    - uses: peter-evans/create-pull-request@v6
+      with:
+        token: ${{ secrets.REPO_SCOPED_TOKEN }}
+        add-paths: freqtrade/exchange/binance_leverage_tiers.json
+        labels: |
+          Tech maintenance
+          Dependencies
+        branch: update/binance-leverage-tiers
+        title: Update Binance Leverage Tiers
+        commit-message: "chore: update pre-commit hooks"
+        committer: Freqtrade Bot <noreply@github.com>
+        body: Update binance leverage tiers.
+        delete-branch: true

.github/workflows/ci.yml

@@ -11,7 +11,7 @@ on:
     types: [published]
   pull_request:
   schedule:
-    - cron:  '0 5 * * 4'
+    - cron:  '0 3 * * 4'
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}"
@@ -19,32 +19,31 @@ concurrency:
 permissions:
   repository-projects: read
 jobs:
-  build_linux:
+  build-linux:
 
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ ubuntu-20.04, ubuntu-22.04 ]
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v4
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Cache_dependencies
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: cache
       with:
         path: ~/dependencies/
         key: ${{ runner.os }}-dependencies
 
     - name: pip cache (linux)
-      uses: actions/cache@v3
+      uses: actions/cache@v4
-      if: runner.os == 'Linux'
       with:
         path: ~/.cache/pip
         key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip
@@ -55,18 +54,22 @@ jobs:
         cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd ..
 
     - name: Installation - *nix
-      if: runner.os == 'Linux'
       run: |
         python -m pip install --upgrade pip wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
         export TA_LIBRARY_PATH=${HOME}/dependencies/lib
         export TA_INCLUDE_PATH=${HOME}/dependencies/include
         pip install -r requirements-dev.txt
+        pip install -e ft_client/
         pip install -e .
 
+    - name: Check for version alignment
+      run: |
+        python build_helpers/freqtrade_client_version_align.py
+
     - name: Tests
       run: |
-        pytest --random-order --cov=freqtrade --cov-config=.coveragerc
+        pytest --random-order --cov=freqtrade --cov=freqtrade_client --cov-config=.coveragerc
 
     - name: Coveralls
       if: (runner.os == 'Linux' && matrix.python-version == '3.10' && matrix.os == 'ubuntu-22.04')
@@ -122,43 +125,44 @@ jobs:
           details: Freqtrade CI failed on ${{ matrix.os }}
           webhookUrl: ${{ secrets.DISCORD_WEBHOOK }}
 
-  build_macos:
+  build-macos:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ macos-latest ]
+        os: [ "macos-latest", "macos-13", "macos-14" ]
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        exclude:
+          - os: "macos-14"
+            python-version: "3.9"
 
     steps:
     - uses: actions/checkout@v4
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
         check-latest: true
 
     - name: Cache_dependencies
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: cache
       with:
         path: ~/dependencies/
-        key: ${{ runner.os }}-dependencies
+        key: ${{ matrix.os }}-dependencies
 
     - name: pip cache (macOS)
-      uses: actions/cache@v3
+      uses: actions/cache@v4
-      if: runner.os == 'macOS'
       with:
         path: ~/Library/Caches/pip
-        key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip
+        key: ${{ matrix.os }}-${{ matrix.python-version }}-pip
 
     - name: TA binary *nix
       if: steps.cache.outputs.cache-hit != 'true'
       run: |
         cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd ..
 
-    - name: Installation - macOS
+    - name: Installation - macOS (Brew)
-      if: runner.os == 'macOS'
       run: |
         # brew update
         # TODO: Should be the brew upgrade
@@ -166,21 +170,30 @@ jobs:
         # https://github.com/actions/runner-images/issues/6817
         rm /usr/local/bin/2to3 || true
         rm /usr/local/bin/2to3-3.11 || true
+        rm /usr/local/bin/2to3-3.12 || true
         rm /usr/local/bin/idle3 || true
         rm /usr/local/bin/idle3.11 || true
+        rm /usr/local/bin/idle3.12 || true
         rm /usr/local/bin/pydoc3 || true
         rm /usr/local/bin/pydoc3.11 || true
+        rm /usr/local/bin/pydoc3.12 || true
         rm /usr/local/bin/python3 || true
         rm /usr/local/bin/python3.11 || true
+        rm /usr/local/bin/python3.12 || true
         rm /usr/local/bin/python3-config || true
         rm /usr/local/bin/python3.11-config || true
+        rm /usr/local/bin/python3.12-config || true
 
-        brew install hdf5 c-blosc
+        brew install hdf5 c-blosc libomp
+
+    - name: Installation (python)
+      run: |
         python -m pip install --upgrade pip wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
         export TA_LIBRARY_PATH=${HOME}/dependencies/lib
         export TA_INCLUDE_PATH=${HOME}/dependencies/include
         pip install -r requirements-dev.txt
+        pip install -e ft_client/
         pip install -e .
 
     - name: Tests
@@ -231,24 +244,24 @@ jobs:
           details: Test Succeeded!
           webhookUrl: ${{ secrets.DISCORD_WEBHOOK }}
 
-  build_windows:
+  build-windows:
 
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ windows-latest ]
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v4
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Pip cache (Windows)
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ~\AppData\Local\pip\Cache
         key: ${{ matrix.os }}-${{ matrix.python-version }}-pip
@@ -301,13 +314,13 @@ jobs:
           details: Test Failed
           webhookUrl: ${{ secrets.DISCORD_WEBHOOK }}
 
-  mypy_version_check:
+  mypy-version-check:
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v4
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: "3.10"
 
@@ -321,12 +334,12 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: "3.10"
-    - uses: pre-commit/action@v3.0.0
+    - uses: pre-commit/action@v3.0.1
 
-  docs_check:
+  docs-check:
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v4
@@ -336,7 +349,7 @@ jobs:
         ./tests/test_docs.sh
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: "3.11"
 
@@ -355,27 +368,26 @@ jobs:
           webhookUrl: ${{ secrets.DISCORD_WEBHOOK }}
 
 
-  build_linux_online:
+  build-linux-online:
     # Run pytest with "live" checks
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v4
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
-        python-version: "3.9"
+        python-version: "3.11"
 
     - name: Cache_dependencies
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: cache
       with:
         path: ~/dependencies/
         key: ${{ runner.os }}-dependencies
 
     - name: pip cache (linux)
-      uses: actions/cache@v3
+      uses: actions/cache@v4
-      if: runner.os == 'Linux'
       with:
         path: ~/.cache/pip
         key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip
@@ -386,36 +398,36 @@ jobs:
         cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd ..
 
     - name: Installation - *nix
-      if: runner.os == 'Linux'
       run: |
         python -m pip install --upgrade pip wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
         export TA_LIBRARY_PATH=${HOME}/dependencies/lib
         export TA_INCLUDE_PATH=${HOME}/dependencies/include
         pip install -r requirements-dev.txt
+        pip install -e ft_client/
         pip install -e .
 
     - name: Tests incl. ccxt compatibility tests
       env:
         CI_WEB_PROXY: http://152.67.78.211:13128
       run: |
-        pytest --random-order --cov=freqtrade --cov-config=.coveragerc --longrun
+        pytest --random-order --longrun --durations 20 -n auto
 
 
   # Notify only once - when CI completes (and after deploy) in case it's successfull
   notify-complete:
     needs: [
-      build_linux,
+      build-linux,
-      build_macos,
+      build-macos,
-      build_windows,
+      build-windows,
-      docs_check,
+      docs-check,
-      mypy_version_check,
+      mypy-version-check,
       pre-commit,
-      build_linux_online
+      build-linux-online
     ]
     runs-on: ubuntu-22.04
     # Discord notification can't handle schedule events
-    if: (github.event_name != 'schedule')
+    if: github.event_name != 'schedule' && github.repository == 'freqtrade/freqtrade'
     permissions:
       repository-projects: read
     steps:
@@ -436,8 +448,78 @@ jobs:
           details: Test Completed!
           webhookUrl: ${{ secrets.DISCORD_WEBHOOK }}
 
-  deploy:
+  build:
-    needs: [ build_linux, build_macos, build_windows, docs_check, mypy_version_check, pre-commit ]
+    name: "Build"
+    needs: [ build-linux, build-macos, build-windows, docs-check, mypy-version-check, pre-commit ]
+    runs-on: ubuntu-22.04
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.11"
+
+    - name: Build distribution
+      run: |
+        pip install -U build
+        python -m build --sdist --wheel
+
+    - name: Upload artifacts 📦
+      uses: actions/upload-artifact@v4
+      with:
+        name: freqtrade-build
+        path: |
+          dist
+        retention-days: 10
+
+    - name: Build Client distribution
+      run: |
+        pip install -U build
+        python -m build --sdist --wheel ft_client
+
+    - name: Upload artifacts 📦
+      uses: actions/upload-artifact@v4
+      with:
+        name: freqtrade-client-build
+        path: |
+          ft_client/dist
+        retention-days: 10
+
+  deploy-pypi:
+    name: "Deploy to PyPI"
+    needs: [ build ]
+    runs-on: ubuntu-22.04
+    if: (github.event_name == 'release')
+    environment:
+      name: release
+      url: https://pypi.org/p/freqtrade
+    permissions:
+      id-token: write
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Download artifact  📦
+      uses: actions/download-artifact@v4
+      with:
+        pattern: freqtrade*-build
+        path: dist
+        merge-multiple: true
+
+
+    - name: Publish to PyPI (Test)
+      uses: pypa/gh-action-pypi-publish@v1.8.14
+      with:
+        repository-url: https://test.pypi.org/legacy/
+
+    - name: Publish to PyPI
+      uses: pypa/gh-action-pypi-publish@v1.8.14
+
+
+  deploy-docker:
+    needs: [ build-linux, build-macos, build-windows, docs-check, mypy-version-check, pre-commit ]
     runs-on: ubuntu-22.04
 
     if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade'
@@ -446,34 +528,15 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: "3.11"
 
     - name: Extract branch name
-      shell: bash
+      id: extract-branch
-      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF##*/})"
-      id: extract_branch
-
-    - name: Build distribution
       run: |
-        pip install -U setuptools wheel
+        echo "GITHUB_REF='${GITHUB_REF}'"
-        python setup.py sdist bdist_wheel
+        echo "branch=${GITHUB_REF##*/}" >> "$GITHUB_OUTPUT"
-
-    - name: Publish to PyPI (Test)
-      uses: pypa/gh-action-pypi-publish@v1.8.10
-      if: (github.event_name == 'release')
-      with:
-        user: __token__
-        password: ${{ secrets.pypi_test_password }}
-        repository_url: https://test.pypi.org/legacy/
-
-    - name: Publish to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.8.10
-      if: (github.event_name == 'release')
-      with:
-        user: __token__
-        password: ${{ secrets.pypi_password }}
 
     - name: Dockerhub login
       env:
@@ -502,14 +565,15 @@ jobs:
 
     - name: Build and test and push docker images
       env:
-        BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}
+        BRANCH_NAME: ${{ steps.extract-branch.outputs.branch }}
       run: |
         build_helpers/publish_docker_multi.sh
 
-  deploy_arm:
+  deploy-arm:
+    name: "Deploy Docker"
     permissions:
       packages: write
-    needs: [ deploy ]
+    needs: [ deploy-docker ]
     # Only run on 64bit machines
     runs-on: [self-hosted, linux, ARM64]
     if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade'
@@ -518,9 +582,10 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Extract branch name
-      shell: bash
+      id: extract-branch
-      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF##*/})"
+      run: |
-      id: extract_branch
+        echo "GITHUB_REF='${GITHUB_REF}'"
+        echo "branch=${GITHUB_REF##*/}" >> "$GITHUB_OUTPUT"
 
     - name: Dockerhub login
       env:
@@ -531,7 +596,7 @@ jobs:
 
     - name: Build and test and push docker images
       env:
-        BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}
+        BRANCH_NAME: ${{ steps.extract-branch.outputs.branch }}
         GHCR_USERNAME: ${{ github.actor }}
         GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |

.github/workflows/docker-update-readme.yml

@@ -0,0 +1,18 @@
+name: Update Docker Hub Description
+on:
+  push:
+    branches:
+    - stable
+
+jobs:
+  dockerHubDescription:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Docker Hub Description
+      uses: peter-evans/dockerhub-description@v4
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+        repository: freqtradeorg/freqtrade

.github/workflows/docker_update_readme.yml

@@ -1,17 +0,0 @@
-name: Update Docker Hub Description
-on:
-  push:
-    branches:
-    - stable
-
-jobs:
-  dockerHubDescription:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: Docker Hub Description
-      uses: peter-evans/dockerhub-description@v3
-      env:
-        DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-        DOCKERHUB_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-        DOCKERHUB_REPOSITORY: freqtradeorg/freqtrade

.github/workflows/pre-commit-update.yml

@@ -0,0 +1,44 @@
+name: Pre-commit auto-update
+
+on:
+  schedule:
+    - cron: "0 3 * * 2"
+  # on demand
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  auto-update:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: "3.11"
+
+
+    - name: Install pre-commit
+      run: pip install pre-commit
+
+    - name: Run auto-update
+      run: pre-commit autoupdate
+
+    - name: Run pre-commit
+      run: pre-commit run --all-files
+
+    - uses: peter-evans/create-pull-request@v6
+      with:
+        token: ${{ secrets.REPO_SCOPED_TOKEN }}
+        add-paths: .pre-commit-config.yaml
+        labels: |
+          Tech maintenance
+          Dependencies
+        branch: update/pre-commit-hooks
+        title: Update pre-commit hooks
+        commit-message: "chore: update pre-commit hooks"
+        committer: Freqtrade Bot <noreply@github.com>
+        body: Update versions of pre-commit hooks to latest version.
+        delete-branch: true

.gitignore

@@ -111,7 +111,6 @@ target/
 #exceptions
 !*.gitkeep
 !config_examples/config_binance.example.json
-!config_examples/config_bittrex.example.json
 !config_examples/config_full.example.json
 !config_examples/config_kraken.example.json
 !config_examples/config_freqai.example.json

.pre-commit-config.yaml

@@ -2,27 +2,28 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
   - repo: https://github.com/pycqa/flake8
-    rev: "6.0.0"
+    rev: "7.0.0"
     hooks:
       - id: flake8
+        additional_dependencies: [Flake8-pyproject]
         # stages: [push]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.7.0"
+    rev: "v1.9.0"
     hooks:
       - id: mypy
         exclude: build_helpers
         additional_dependencies:
           - types-cachetools==5.3.0.7
           - types-filelock==3.2.7
-          - types-requests==2.31.0.10
+          - types-requests==2.31.0.20240311
-          - types-tabulate==0.9.0.3
+          - types-tabulate==0.9.0.20240106
-          - types-python-dateutil==2.8.19.14
+          - types-python-dateutil==2.9.0.20240316
-          - SQLAlchemy==2.0.23
+          - SQLAlchemy==2.0.29
         # stages: [push]
 
   - repo: https://github.com/pycqa/isort
-    rev: "5.12.0"
+    rev: "5.13.2"
     hooks:
       - id: isort
         name: isort (python)
@@ -30,12 +31,12 @@ repos:
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.1.1'
+    rev: 'v0.3.4'
     hooks:
       - id: ruff
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: end-of-file-fixer
         exclude: |

CONTRIBUTING.md

@@ -48,7 +48,7 @@ pytest tests/test_<file_name>.py::test_<method_name>
 #### Run Ruff
 
 ```bash
-ruff .
+ruff check .
 ```
 
 We receive a lot of code that fails the `ruff` checks.

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11.6-slim-bookworm as base
+FROM python:3.12.2-slim-bookworm as base
 
 # Setup env
 ENV LANG C.UTF-8

MANIFEST.in

@@ -5,3 +5,5 @@ recursive-include freqtrade/templates/ *.j2 *.ipynb
 include freqtrade/exchange/binance_leverage_tiers.json
 include freqtrade/rpc/api_server/ui/fallback_file.html
 include freqtrade/rpc/api_server/ui/favicon.ico
+
+prune tests

README.md

@@ -30,7 +30,7 @@ Please read the [exchange specific notes](docs/exchanges.md) to learn about even
 - [X] [Binance](https://www.binance.com/)
 - [X] [Bitmart](https://bitmart.com/)
 - [X] [Gate.io](https://www.gate.io/ref/6266643)
-- [X] [Huobi](http://huobi.com/)
+- [X] [HTX](https://www.htx.com/) (Former Huobi)
 - [X] [Kraken](https://kraken.com/)
 - [X] [OKX](https://okx.com/) (Former OKEX)
 - [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_

build_helpers/binance_update_lev_tiers.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+import json
+import os
+from pathlib import Path
+
+import ccxt
+
+
+key = os.environ.get('FREQTRADE__EXCHANGE__KEY')
+secret = os.environ.get('FREQTRADE__EXCHANGE__SECRET')
+
+proxy = os.environ.get('CI_WEB_PROXY')
+
+exchange = ccxt.binance({
+    'apiKey': key,
+    'secret': secret,
+    'httpsProxy': proxy,
+    'options': {'defaultType': 'swap'}
+    })
+_ = exchange.load_markets()
+
+lev_tiers = exchange.fetch_leverage_tiers()
+
+# Assumes this is running in the root of the repository.
+file = Path('freqtrade/exchange/binance_leverage_tiers.json')
+json.dump(dict(sorted(lev_tiers.items())), file.open('w'), indent=2)

build_helpers/freqtrade_client_version_align.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+from freqtrade_client import __version__ as client_version
+
+from freqtrade import __version__ as ft_version
+
+
+def main():
+    if ft_version != client_version:
+        print(f"Versions do not match: \n"
+              f"ft: {ft_version} \n"
+              f"client: {client_version}")
+        exit(1)
+    print(f"Versions match: ft: {ft_version}, client: {client_version}")
+    exit(0)
+
+
+if __name__ == '__main__':
+    main()

config_examples/config_freqai.example.json

@@ -52,7 +52,7 @@
         "train_period_days": 15,
         "backtest_period_days": 7,
         "live_retrain_hours": 0,
-        "identifier": "uniqe-id",
+        "identifier": "unique-id",
         "feature_parameters": {
             "include_timeframes": [
                 "3m",

docker/Dockerfile.armhf

@@ -1,4 +1,4 @@
-FROM python:3.11.6-slim-bookworm as base
+FROM python:3.11.8-slim-bookworm as base
 
 # Setup env
 ENV LANG C.UTF-8

docker/Dockerfile.jupyter

@@ -1,8 +1,8 @@
 FROM freqtradeorg/freqtrade:develop_plot
 
 
-# Pin jupyter-client to avoid tornado version conflict
+# Pin prompt-toolkit to avoid questionary version conflict
-RUN pip install jupyterlab jupyter-client==7.3.4 --user --no-cache-dir
+RUN pip install jupyterlab "prompt-toolkit<=3.0.36" jupyter-client --user --no-cache-dir
 
 # Empty the ENTRYPOINT to allow all commands
 ENTRYPOINT []

docker/docker-compose-jupyter.yml

@@ -6,7 +6,7 @@ services:
       context: ..
       dockerfile: docker/Dockerfile.jupyter
     restart: unless-stopped
-    container_name: freqtrade
+    # container_name: freqtrade
     ports:
       - "127.0.0.1:8888:8888"
     volumes:

docs/advanced-backtesting.md

@@ -109,12 +109,12 @@ automatically accessible by including them on the indicator-list, and these incl
 - **open_date     :** trade open datetime
 - **close_date    :** trade close datetime
 - **min_rate      :** minimum price seen throughout the position
-- **max_rate      :** maxiumum price seen throughout the position
+- **max_rate      :** maximum price seen throughout the position
 - **open          :** signal candle open price
 - **close         :** signal candle close price
 - **high          :** signal candle high price
 - **low           :** signal candle low price
-- **volume        :** signal candle volumne
+- **volume        :** signal candle volume
 - **profit_ratio  :** trade profit ratio
 - **profit_abs    :** absolute profit return of the trade 
 

docs/backtesting.md

@@ -252,34 +252,34 @@ The most important in the backtesting is to understand the result.
 A backtesting result will look like that:
 
 ```
-========================================================= BACKTESTING REPORT =========================================================
+================================================ BACKTESTING REPORT =================================================
-| Pair     | Entries |   Avg Profit % |   Cum Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration |  Wins Draws Loss   Win%  |
+| Pair     | Entries |   Avg Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration |  Wins Draws Loss   Win%  |
-|:---------|--------:|---------------:|---------------:|-----------------:|---------------:|:-------------|-------------------------:|
+|:---------|--------:|---------------:|-----------------:|---------------:|:-------------|-------------------------:|
-| ADA/BTC  |      35 |          -0.11 |          -3.88 |      -0.00019428 |          -1.94 | 4:35:00      |    14     0    21   40.0 |
+| ADA/BTC  |      35 |          -0.11 |      -0.00019428 |          -1.94 | 4:35:00      |    14     0    21   40.0 |
-| ARK/BTC  |      11 |          -0.41 |          -4.52 |      -0.00022647 |          -2.26 | 2:03:00      |     3     0     8   27.3 |
+| ARK/BTC  |      11 |          -0.41 |      -0.00022647 |          -2.26 | 2:03:00      |     3     0     8   27.3 |
-| BTS/BTC  |      32 |           0.31 |           9.78 |       0.00048938 |           4.89 | 5:05:00      |    18     0    14   56.2 |
+| BTS/BTC  |      32 |           0.31 |       0.00048938 |           4.89 | 5:05:00      |    18     0    14   56.2 |
-| DASH/BTC |      13 |          -0.08 |          -1.07 |      -0.00005343 |          -0.53 | 4:39:00      |     6     0     7   46.2 |
+| DASH/BTC |      13 |          -0.08 |      -0.00005343 |          -0.53 | 4:39:00      |     6     0     7   46.2 |
-| ENG/BTC  |      18 |           1.36 |          24.54 |       0.00122807 |          12.27 | 2:50:00      |     8     0    10   44.4 |
+| ENG/BTC  |      18 |           1.36 |       0.00122807 |          12.27 | 2:50:00      |     8     0    10   44.4 |
-| EOS/BTC  |      36 |           0.08 |           3.06 |       0.00015304 |           1.53 | 3:34:00      |    16     0    20   44.4 |
+| EOS/BTC  |      36 |           0.08 |       0.00015304 |           1.53 | 3:34:00      |    16     0    20   44.4 |
-| ETC/BTC  |      26 |           0.37 |           9.51 |       0.00047576 |           4.75 | 6:14:00      |    11     0    15   42.3 |
+| ETC/BTC  |      26 |           0.37 |       0.00047576 |           4.75 | 6:14:00      |    11     0    15   42.3 |
-| ETH/BTC  |      33 |           0.30 |           9.96 |       0.00049856 |           4.98 | 7:31:00      |    16     0    17   48.5 |
+| ETH/BTC  |      33 |           0.30 |       0.00049856 |           4.98 | 7:31:00      |    16     0    17   48.5 |
-| IOTA/BTC |      32 |           0.03 |           1.09 |       0.00005444 |           0.54 | 3:12:00      |    14     0    18   43.8 |
+| IOTA/BTC |      32 |           0.03 |       0.00005444 |           0.54 | 3:12:00      |    14     0    18   43.8 |
-| LSK/BTC  |      15 |           1.75 |          26.26 |       0.00131413 |          13.13 | 2:58:00      |     6     0     9   40.0 |
+| LSK/BTC  |      15 |           1.75 |       0.00131413 |          13.13 | 2:58:00      |     6     0     9   40.0 |
-| LTC/BTC  |      32 |          -0.04 |          -1.38 |      -0.00006886 |          -0.69 | 4:49:00      |    11     0    21   34.4 |
+| LTC/BTC  |      32 |          -0.04 |      -0.00006886 |          -0.69 | 4:49:00      |    11     0    21   34.4 |
-| NANO/BTC |      17 |           1.26 |          21.39 |       0.00107058 |          10.70 | 1:55:00      |    10     0     7   58.5 |
+| NANO/BTC |      17 |           1.26 |       0.00107058 |          10.70 | 1:55:00      |    10     0     7   58.5 |
-| NEO/BTC  |      23 |           0.82 |          18.97 |       0.00094936 |           9.48 | 2:59:00      |    10     0    13   43.5 |
+| NEO/BTC  |      23 |           0.82 |       0.00094936 |           9.48 | 2:59:00      |    10     0    13   43.5 |
-| REQ/BTC  |       9 |           1.17 |          10.54 |       0.00052734 |           5.27 | 3:47:00      |     4     0     5   44.4 |
+| REQ/BTC  |       9 |           1.17 |       0.00052734 |           5.27 | 3:47:00      |     4     0     5   44.4 |
-| XLM/BTC  |      16 |           1.22 |          19.54 |       0.00097800 |           9.77 | 3:15:00      |     7     0     9   43.8 |
+| XLM/BTC  |      16 |           1.22 |       0.00097800 |           9.77 | 3:15:00      |     7     0     9   43.8 |
-| XMR/BTC  |      23 |          -0.18 |          -4.13 |      -0.00020696 |          -2.07 | 5:30:00      |    12     0    11   52.2 |
+| XMR/BTC  |      23 |          -0.18 |      -0.00020696 |          -2.07 | 5:30:00      |    12     0    11   52.2 |
-| XRP/BTC  |      35 |           0.66 |          22.96 |       0.00114897 |          11.48 | 3:49:00      |    12     0    23   34.3 |
+| XRP/BTC  |      35 |           0.66 |       0.00114897 |          11.48 | 3:49:00      |    12     0    23   34.3 |
-| ZEC/BTC  |      22 |          -0.46 |         -10.18 |      -0.00050971 |          -5.09 | 2:22:00      |     7     0    15   31.8 |
+| ZEC/BTC  |      22 |          -0.46 |      -0.00050971 |          -5.09 | 2:22:00      |     7     0    15   31.8 |
-| TOTAL    |     429 |           0.36 |         152.41 |       0.00762792 |          76.20 | 4:12:00      |   186     0   243   43.4 |
+| TOTAL    |     429 |           0.36 |       0.00762792 |          76.20 | 4:12:00      |   186     0   243   43.4 |
-====================================================== LEFT OPEN TRADES REPORT ======================================================
+============================================= LEFT OPEN TRADES REPORT =============================================
-| Pair     |  Entries |   Avg Profit % |   Cum Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Win Draw Loss Win% |
+| Pair     |  Entries |   Avg Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Win Draw Loss Win% |
-|:---------|---------:|---------------:|---------------:|-----------------:|---------------:|:---------------|--------------------:|
+|:---------|---------:|---------------:|-----------------:|---------------:|:---------------|--------------------:|
-| ADA/BTC  |        1 |           0.89 |           0.89 |       0.00004434 |           0.44 | 6:00:00        |    1    0    0  100 |
+| ADA/BTC  |        1 |           0.89 |       0.00004434 |           0.44 | 6:00:00        |    1    0    0  100 |
-| LTC/BTC  |        1 |           0.68 |           0.68 |       0.00003421 |           0.34 | 2:00:00        |    1    0    0  100 |
+| LTC/BTC  |        1 |           0.68 |       0.00003421 |           0.34 | 2:00:00        |    1    0    0  100 |
-| TOTAL    |        2 |           0.78 |           1.57 |       0.00007855 |           0.78 | 4:00:00        |    2    0    0  100 |
+| TOTAL    |        2 |           0.78 |       0.00007855 |           0.78 | 4:00:00        |    2    0    0  100 |
 ==================== EXIT REASON STATS ====================
 | Exit Reason        |   Exits |  Wins |  Draws |  Losses |
 |:-------------------|--------:|------:|-------:|--------:|
@@ -358,7 +358,7 @@ here:
 The bot has made `429` trades for an average duration of `4:12:00`, with a performance of `76.20%` (profit), that means it has
 earned a total of `0.00762792 BTC` starting with a capital of 0.01 BTC.
 
-The column `Avg Profit %` shows the average profit for all trades made while the column `Cum Profit %` sums up all the profits/losses.
+The column `Avg Profit %` shows the average profit for all trades made.
 The column `Tot Profit %` shows instead the total profit % in relation to the starting balance.
 In the above results, we have a starting balance of 0.01 BTC and an absolute profit of 0.00762792 BTC - so the `Tot Profit %` will be `(0.00762792 / 0.01) * 100 ~= 76.2%`.
 
@@ -464,7 +464,7 @@ It contains some useful key metrics about performance of your strategy on backte
 - `Profit factor`: profit / loss.
 - `Avg. stake amount`: Average stake amount, either `stake_amount` or the average when using dynamic stake amount.
 - `Total trade volume`: Volume generated on the exchange to reach the above profit.
-- `Best Pair` / `Worst Pair`: Best and worst performing pair, and it's corresponding `Cum Profit %`.
+- `Best Pair` / `Worst Pair`: Best and worst performing pair, and it's corresponding `Tot Profit %`.
 - `Best Trade` / `Worst Trade`: Biggest single winning trade and biggest single losing trade.
 - `Best day` / `Worst day`: Best and worst day based on daily profit.
 - `Days win/draw/lose`: Winning / Losing days (draws are usually days without closed trade).
@@ -618,22 +618,22 @@ To compare multiple strategies, a list of Strategies can be provided to backtest
 This is limited to 1 timeframe value per run. However, data is only loaded once from disk so if you have multiple
 strategies you'd like to compare, this will give a nice runtime boost.
 
-All listed Strategies need to be in the same directory.
+All listed Strategies need to be in the same directory, unless also `--recursive-strategy-search` is specified, where sub-directories within the strategy directory are also considered.
 
 ``` bash
 freqtrade backtesting --timerange 20180401-20180410 --timeframe 5m --strategy-list Strategy001 Strategy002 --export trades
 ```
 
-This will save the results to `user_data/backtest_results/backtest-result-<strategy>.json`, injecting the strategy-name into the target filename.
+This will save the results to `user_data/backtest_results/backtest-result-<datetime>.json`, including results for both `Strategy001` and `Strategy002`.
 There will be an additional table comparing win/losses of the different strategies (identical to the "Total" row in the first table).
 Detailed output for all strategies one after the other will be available, so make sure to scroll up to see the details per strategy.
 
 ```
-=========================================================== STRATEGY SUMMARY ===========================================================================
+================================================== STRATEGY SUMMARY ===================================================================
-| Strategy    |  Entries |   Avg Profit % |   Cum Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Wins |  Draws | Losses | Drawdown % |
+| Strategy    |  Entries |   Avg Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Wins |  Draws | Losses | Drawdown % |
-|:------------|---------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|-------:|-----------:|
+|:------------|---------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|-------:|-----------:|
-| Strategy1   |      429 |           0.36 |         152.41 |       0.00762792 |          76.20 | 4:12:00        |   186 |      0 |    243 |       45.2 |
+| Strategy1   |      429 |           0.36 |       0.00762792 |          76.20 | 4:12:00        |   186 |      0 |    243 |       45.2 |
-| Strategy2   |     1487 |          -0.13 |        -197.58 |      -0.00988917 |         -98.79 | 4:43:00        |   662 |      0 |    825 |     241.68 |
+| Strategy2   |     1487 |          -0.13 |      -0.00988917 |         -98.79 | 4:43:00        |   662 |      0 |    825 |     241.68 |
 ```
 
 ## Next step

docs/bot-basics.md

@@ -33,7 +33,6 @@ For spot pairs, naming will be `base/quote` (e.g. `ETH/USDT`).
 
 For futures pairs, naming will be `base/quote:settle` (e.g. `ETH/USDT:USDT`).
 
-
 ## Bot execution logic
 
 Starting freqtrade in dry-run or live mode (using `freqtrade trade`) will start the bot and start the bot iteration loop.
@@ -50,10 +49,12 @@ By default, the bot loop runs every few seconds (`internals.process_throttle_sec
   * Call `populate_indicators()`
   * Call `populate_entry_trend()`
   * Call `populate_exit_trend()`
-* Check timeouts for open orders.
+* Update trades open order state from exchange.
-  * Calls `check_entry_timeout()` strategy callback for open entry orders.
+  * Call `order_filled()` strategy callback for filled orders.
-  * Calls `check_exit_timeout()` strategy callback for open exit orders.
+  * Check timeouts for open orders.
-  * Calls `adjust_entry_price()` strategy callback for open entry orders.
+    * Calls `check_entry_timeout()` strategy callback for open entry orders.
+    * Calls `check_exit_timeout()` strategy callback for open exit orders.
+    * Calls `adjust_entry_price()` strategy callback for open entry orders.
 * Verifies existing positions and eventually places exit orders.
   * Considers stoploss, ROI and exit-signal, `custom_exit()` and `custom_stoploss()`.
   * Determine exit-price based on `exit_pricing` configuration setting or by using the `custom_exit_price()` callback.
@@ -86,8 +87,10 @@ This loop will be repeated again and again until the bot is stopped.
   * In Margin and Futures mode, `leverage()` strategy callback is called to determine the desired leverage.
   * Determine stake size by calling the `custom_stake_amount()` callback.
   * Check position adjustments for open trades if enabled and call `adjust_trade_position()` to determine if an additional order is requested.
+  * Call `order_filled()` strategy callback for filled entry orders.
   * Call `custom_stoploss()` and `custom_exit()` to find custom exit points.
   * For exits based on exit-signal, custom-exit and partial exits: Call `custom_exit_price()` to determine exit price (Prices are moved to be within the closing candle).
+  * Call `order_filled()` strategy callback for filled exit orders.
 * Generate backtest report output
 
 !!! Note

docs/configuration.md

@@ -14,7 +14,7 @@ You can specify a different configuration file used by the bot with the `-c/--co
 If you used the [Quick start](docker_quickstart.md#docker-quick-start) method for installing
 the bot, the installation script should have already created the default configuration file (`config.json`) for you.
 
-If the default configuration file is not created we recommend to use `freqtrade new-config --config config.json` to generate a basic configuration file.
+If the default configuration file is not created we recommend to use `freqtrade new-config --config user_data/config.json` to generate a basic configuration file.
 
 The Freqtrade configuration file is to be written in JSON format.
 
@@ -49,6 +49,13 @@ FREQTRADE__EXCHANGE__SECRET=<yourExchangeSecret>
 !!! Note
     Environment variables detected are logged at startup - so if you can't find why a value is not what you think it should be based on the configuration, make sure it's not loaded from an environment variable.
 
+!!! Tip "Validate combined result"
+    You can use the [show-config subcommand](utils.md#show-config) to see the final, combined configuration.
+
+??? Warning "Loading sequence"
+    Environment variables are loaded after the initial configuration. As such, you cannot provide the path to the configuration through environment variables. Please use `--config path/to/config.json` for that.
+    This also applies to user_dir to some degree. while the user directory can be set through environment variables - the configuration will **not** be loaded from that location.
+
 ### Multiple configuration files
 
 Multiple configuration files can be specified and used by the bot or the bot can read its configuration parameters from the process standard input stream.
@@ -56,6 +63,9 @@ Multiple configuration files can be specified and used by the bot or the bot can
 You can specify additional configuration files in `add_config_files`. Files specified in this parameter will be loaded and merged with the initial config file. The files are resolved relative to the initial configuration file.
 This is similar to using multiple `--config` parameters, but simpler in usage as you don't have to specify all files for all commands.
 
+!!! Tip "Validate combined result"
+    You can use the [show-config subcommand](utils.md#show-config) to see the final, combined configuration.
+
 !!! Tip "Use multiple configuration files to keep secrets secret"
     You can use a 2nd configuration file containing your secrets. That way you can share your "primary" configuration file, while still keeping your API keys for yourself.
     The 2nd file should only specify what you intend to override.
@@ -321,11 +331,13 @@ For example, if you have 10 ETH available in your wallet on the exchange and `tr
 To fully utilize compounding profits when using multiple bots on the same exchange account, you'll want to limit each bot to a certain starting balance.
 This can be accomplished by setting `available_capital` to the desired starting balance.
 
-Assuming your account has 10.000 USDT and you want to run 2 different strategies on this exchange.
+Assuming your account has 10000 USDT and you want to run 2 different strategies on this exchange.
 You'd set `available_capital=5000` - granting each bot an initial capital of 5000 USDT.
 The bot will then split this starting balance equally into `max_open_trades` buckets.
 Profitable trades will result in increased stake-sizes for this bot - without affecting the stake-sizes of the other bot.
 
+Adjusting `available_capital` requires reloading the configuration to take effect. Adjusting the `available_capital` adds the difference between the previous `available_capital` and the new `available_capital`. Decreasing the available capital when trades are open doesn't exit the trades. The difference is returned to the wallet when the trades conclude. The outcome of this differs depending on the price movement between the adjustment and exiting the trades.
+
 !!! Warning "Incompatible with `tradable_balance_ratio`"
     Setting this option will replace any configuration of `tradable_balance_ratio`.
 
@@ -503,13 +515,13 @@ Configuration:
     Please carefully read the section [Market order pricing](#market-order-pricing) section when using market orders.
 
 !!! Note "Stoploss on exchange"
-    `stoploss_on_exchange_interval` is not mandatory. Do not change its value if you are
+    `order_types.stoploss_on_exchange_interval` is not mandatory. Do not change its value if you are
     unsure of what you are doing. For more information about how stoploss works please
     refer to [the stoploss documentation](stoploss.md).
 
-    If `stoploss_on_exchange` is enabled and the stoploss is cancelled manually on the exchange, then the bot will create a new stoploss order.
+    If `order_types.stoploss_on_exchange` is enabled and the stoploss is cancelled manually on the exchange, then the bot will create a new stoploss order.
 
-!!! Warning "Warning: stoploss_on_exchange failures"
+!!! Warning "Warning: order_types.stoploss_on_exchange failures"
     If stoploss on exchange creation fails for some reason, then an "emergency exit" is initiated. By default, this will exit the trade using a market order. The order-type for the emergency-exit can be changed by setting the `emergency_exit` value in the `order_types` dictionary - however, this is not advised.
 
 ### Understand order_time_in_force
@@ -572,9 +584,11 @@ In addition to fiat currencies, a range of crypto currencies is supported.
 The valid values are:
 
 ```json
-"BTC", "ETH", "XRP", "LTC", "BCH", "USDT"
+"BTC", "ETH", "XRP", "LTC", "BCH", "BNB"
 ```
 
+Removing `fiat_display_currency` completely from the configuration will skip initializing coingecko, and will not show any FIAT currency conversion. This has no importance for the correct functioning of the bot.
+
 ## Using Dry-run mode
 
 We recommend starting the bot in the Dry-run mode to see how your bot will

docs/developer.md

@@ -129,6 +129,8 @@ Below is an outline of exception inheritance hierarchy:
 + FreqtradeException
 |
 +---+ OperationalException
+|   |
+|   +---+ ConfigurationError
 |
 +---+ DependencyException
 |   |
@@ -376,7 +378,7 @@ from pathlib import Path
 
 exchange = ccxt.binance({
     'apiKey': '<apikey>',
-    'secret': '<secret>'
+    'secret': '<secret>',
     'options': {'defaultType': 'swap'}
     })
 _ = exchange.load_markets()
@@ -419,6 +421,9 @@ This part of the documentation is aimed at maintainers, and shows how to create
 
 ### Create release branch
 
+!!! Note
+    Make sure that the `stable` branch is up-to-date!
+
 First, pick a commit that's about one week old (to not include latest additions to releases).
 
 ``` bash
@@ -431,14 +436,11 @@ Determine if crucial bugfixes have been made between this commit and the current
 * Merge the release branch (stable) into this branch.
 * Edit `freqtrade/__init__.py` and add the version matching the current date (for example `2019.7` for July 2019). Minor versions can be `2019.7.1` should we need to do a second release that month. Version numbers must follow allowed versions from PEP0440 to avoid failures pushing to pypi.
 * Commit this part.
-* push that branch to the remote and create a PR against the stable branch.
+* Push that branch to the remote and create a PR against the **stable branch**.
 * Update develop version to next version following the pattern `2019.8-dev`.
 
 ### Create changelog from git commits
 
-!!! Note
-    Make sure that the `stable` branch is up-to-date!
-
 ``` bash
 # Needs to be done before merging / pulling that branch.
 git log --oneline --no-decorate --no-merges stable..new_release

docs/exchanges.md

@@ -68,6 +68,8 @@ Binance supports [time_in_force](configuration.md#understand-order_time_in_force
 For Binance, it is suggested to add `"BNB/<STAKE>"` to your blacklist to avoid issues, unless you are willing to maintain enough extra `BNB` on the account or unless you're willing to disable using `BNB` for fees.
 Binance accounts may use `BNB` for fees, and if a trade happens to be on `BNB`, further trades may consume this position and make the initial BNB trade unsellable as the expected amount is not there anymore.
 
+If not enough `BNB` is available to cover transaction fees, then fees will not be covered by `BNB` and no fee reduction will occur. Freqtrade will never buy BNB to cover for fees. BNB needs to be bought and monitored manually to this end.
+
 ### Binance sites
 
 Binance has been split into 2, and users must use the correct ccxt exchange ID for their exchange, otherwise API keys are not recognized.
@@ -127,6 +129,8 @@ Freqtrade will not attempt to change these settings.
 
 ## Kraken
 
+Kraken supports [time_in_force](configuration.md#understand-order_time_in_force) with settings "GTC" (good till cancelled), "IOC" (immediate-or-cancel) and "PO" (Post only) settings.
+
 !!! Tip "Stoploss on Exchange"
     Kraken supports `stoploss_on_exchange` and can use both stop-loss-market and stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it.
     You can use either `"limit"` or `"market"` in the `order_types.stoploss` configuration setting to decide which type to use.
@@ -181,48 +185,6 @@ freqtrade download-data --exchange kraken --dl-trades -p BTC/EUR BCH/EUR
     Please pay attention that rateLimit configuration entry holds delay in milliseconds between requests, NOT requests\sec rate.
     So, in order to mitigate Kraken API "Rate limit exceeded" exception, this configuration should be increased, NOT decreased.
 
-## Bittrex
-
-### Order types
-
-Bittrex does not support market orders. If you have a message at the bot startup about this, you should change order type values set in your configuration and/or in the strategy from `"market"` to `"limit"`. See some more details on this [here in the FAQ](faq.md#im-getting-the-exchange-bittrex-does-not-support-market-orders-message-and-cannot-run-my-strategy).
-
-Bittrex also does not support `VolumePairlist` due to limited / split API constellation at the moment.
-Please use `StaticPairlist`. Other pairlists (other than `VolumePairlist`) should not be affected.
-
-### Volume pairlist
-
-Bittrex does not support the direct usage of VolumePairList. This can however be worked around by using the advanced mode with `lookback_days: 1` (or more), which will emulate 24h volume.
-
-Read more in the [pairlist documentation](plugins.md#volumepairlist-advanced-mode).
-
-### Restricted markets
-
-Bittrex split its exchange into US and International versions.
-The International version has more pairs available, however the API always returns all pairs, so there is currently no automated way to detect if you're affected by the restriction.
-
-If you have restricted pairs in your whitelist, you'll get a warning message in the log on Freqtrade startup for each restricted pair.
-
-The warning message will look similar to the following:
-
-``` output
-[...] Message: bittrex {"success":false,"message":"RESTRICTED_MARKET","result":null,"explanation":null}"
-```
-
-If you're an "International" customer on the Bittrex exchange, then this warning will probably not impact you.
-If you're a US customer, the bot will fail to create orders for these pairs, and you should remove them from your whitelist.
-
-You can get a list of restricted markets by using the following snippet:
-
-``` python
-import ccxt
-ct = ccxt.bittrex()
-lm = ct.load_markets()
-
-res = [p for p, x in lm.items() if 'US' in x['info']['prohibitedIn']]
-print(res)
-```
-
 ## Kucoin
 
 Kucoin requires a passphrase for each api key, you will therefore need to add this key into the configuration so your exchange section looks as follows:
@@ -248,10 +210,10 @@ Kucoin supports [time_in_force](configuration.md#understand-order_time_in_force)
 For Kucoin, it is suggested to add `"KCS/<STAKE>"` to your blacklist to avoid issues, unless you are willing to maintain enough extra `KCS` on the account or unless you're willing to disable using `KCS` for fees. 
 Kucoin accounts may use `KCS` for fees, and if a trade happens to be on `KCS`, further trades may consume this position and make the initial `KCS` trade unsellable as the expected amount is not there anymore.
 
-## Huobi
+## HTX (formerly Huobi)
 
 !!! Tip "Stoploss on Exchange"
-    Huobi supports `stoploss_on_exchange` and uses `stop-limit` orders. It provides great advantages, so we recommend to benefit from it by enabling stoploss on exchange.
+    HTX supports `stoploss_on_exchange` and uses `stop-limit` orders. It provides great advantages, so we recommend to benefit from it by enabling stoploss on exchange.
 
 ## OKX (former OKEX)
 

docs/faq.md

@@ -128,15 +128,9 @@ This warning can point to one of the below problems:
 * Barely traded pair -> Check the pair on the exchange webpage, look at the timeframe your strategy uses. If the pair does not have any volume in some candles (usually visualized with a "volume 0" bar, and a "_" as candle), this pair did not have any trades in this timeframe. These pairs should ideally be avoided, as they can cause problems with order-filling.
 * API problem -> API returns wrong data (this only here for completeness, and should not happen with supported exchanges).
 
-### I'm getting the "RESTRICTED_MARKET" message in the log
-
-Currently known to happen for US Bittrex users.
-
-Read [the Bittrex section about restricted markets](exchanges.md#restricted-markets) for more information.
-
 ### I'm getting the "Exchange XXX does not support market orders." message and cannot run my strategy
 
-As the message says, your exchange does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Your strategy was probably written with other exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of the exchanges supporting market orders (but not for Bittrex and Gate.io).
+As the message says, your exchange does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Your strategy was probably written with other exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of the exchanges supporting market orders (but not for Gate.io).
 
 To fix this, redefine order types in the strategy to use "limit" instead of "market":
 

docs/freqai-configuration.md

@@ -32,6 +32,9 @@ FreqAI is configured through the typical [Freqtrade config file](configuration.m
 
 A full example config is available in `config_examples/config_freqai.example.json`.
 
+!!! Note
+    The `identifier` is commonly overlooked by newcomers, however, this value plays an important role in your configuration. This value is a unique ID that you choose to describe one of your runs. Keeping it the same allows you to maintain crash resilience as well as faster backtesting. As soon as you want to try a new run (new features, new model, etc.), you should change this value (or delete the `user_data/models/unique-id` folder. More details available in the [parameter table](freqai-parameter-table.md#feature-parameters).
+
 ## Building a FreqAI strategy
 
 The FreqAI strategy requires including the following lines of code in the standard [Freqtrade strategy](strategy-customization.md):
@@ -162,7 +165,8 @@ Below are the values you can expect to include/use inside a typical strategy dat
 | `df['&*_std/mean']` | Standard deviation and mean values of the defined labels during training (or live tracking with `fit_live_predictions_candles`). Commonly used to understand the rarity of a prediction (use the z-score as shown in `templates/FreqaiExampleStrategy.py` and explained [here](#creating-a-dynamic-target-threshold) to evaluate how often a particular prediction was observed during training or historically with `fit_live_predictions_candles`). <br> **Datatype:** Float.
 | `df['do_predict']` | Indication of an outlier data point. The return value is integer between -2 and 2, which lets you know if the prediction is trustworthy or not. `do_predict==1` means that the prediction is trustworthy. If the Dissimilarity Index (DI, see details [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di)) of the input data point is above the threshold defined in the config, FreqAI will subtract 1 from `do_predict`, resulting in `do_predict==0`. If `use_SVM_to_remove_outliers` is active, the Support Vector Machine (SVM, see details [here](freqai-feature-engineering.md#identifying-outliers-using-a-support-vector-machine-svm)) may also detect outliers in training and prediction data. In this case, the SVM will also subtract 1 from `do_predict`. If the input data point was considered an outlier by the SVM but not by the DI, or vice versa, the result will be `do_predict==0`. If both the DI and the SVM considers the input data point to be an outlier, the result will be `do_predict==-1`. As with the SVM, if `use_DBSCAN_to_remove_outliers` is active, DBSCAN (see details [here](freqai-feature-engineering.md#identifying-outliers-with-dbscan)) may also detect outliers and subtract 1 from `do_predict`. Hence, if both the SVM and DBSCAN are active and identify a datapoint that was above the DI threshold as an outlier, the result will be `do_predict==-2`. A particular case is when `do_predict == 2`, which means that the model has expired due to exceeding `expired_hours`. <br> **Datatype:** Integer between -2 and 2.
 | `df['DI_values']` | Dissimilarity Index (DI) values are proxies for the level of confidence FreqAI has in the prediction. A lower DI means the prediction is close to the training data, i.e., higher prediction confidence. See details about the DI [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di). <br> **Datatype:** Float.
-| `df['%*']` | Any dataframe column prepended with `%` in `feature_engineering_*()` is treated as a training feature. For example, you can include the RSI in the training feature set (similar to in `templates/FreqaiExampleStrategy.py`) by setting `df['%-rsi']`. See more details on how this is done [here](freqai-feature-engineering.md). <br> **Note:** Since the number of features prepended with `%` can multiply very quickly (10s of thousands of features are easily engineered using the multiplictative functionality of, e.g., `include_shifted_candles` and `include_timeframes` as described in the [parameter table](freqai-parameter-table.md)), these features are removed from the dataframe that is returned from FreqAI to the strategy. To keep a particular type of feature for plotting purposes, you would prepend it with `%%`. <br> **Datatype:** Depends on the output of the model.
+| `df['%*']` | Any dataframe column prepended with `%` in `feature_engineering_*()` is treated as a training feature. For example, you can include the RSI in the training feature set (similar to in `templates/FreqaiExampleStrategy.py`) by setting `df['%-rsi']`. See more details on how this is done [here](freqai-feature-engineering.md). <br> **Note:** Since the number of features prepended with `%` can multiply very quickly (10s of thousands of features are easily engineered using the multiplictative functionality of, e.g., `include_shifted_candles` and `include_timeframes` as described in the [parameter table](freqai-parameter-table.md)), these features are removed from the dataframe that is returned from FreqAI to the strategy. To keep a particular type of feature for plotting purposes, you would prepend it with `%%` (see details below). <br> **Datatype:** Depends on the feature created by the user.
+| `df['%%*']` | Any dataframe column prepended with `%%` in `feature_engineering_*()` is treated as a training feature, just the same as the above `%` prepend. However, in this case, the features are returned back to the strategy for FreqUI/plot-dataframe plotting and monitoring in Dry/Live/Backtesting <br> **Datatype:** Depends on the feature created by the user. Please note that features created in `feature_engineering_expand()` will have automatic FreqAI naming schemas depending on the expansions that you configured (i.e. `include_timeframes`, `include_corr_pairlist`, `indicators_periods_candles`, `include_shifted_candles`). So if you want to plot `%%-rsi` from `feature_engineering_expand_all()`, the final naming scheme for your plotting config would be: `%%-rsi-period_10_ETH/USDT:USDT_1h` for the `rsi` feature with `period=10`, `timeframe=1h`, and `pair=ETH/USDT:USDT` (the `:USDT` is added if you are using futures pairs). It is useful to simply add `print(dataframe.columns)` in your `populate_indicators()` after `self.freqai.start()` to see the full list of available features that are returned to the strategy for plotting purposes.
 
 ## Setting the `startup_candle_count`
 

docs/freqai-parameter-table.md

@@ -75,7 +75,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the
 | `rl_config` | A dictionary containing the control parameters for a Reinforcement Learning model. <br> **Datatype:** Dictionary.
 | `train_cycles` | Training time steps will be set based on the `train_cycles * number of training data points. <br> **Datatype:** Integer.
 | `max_trade_duration_candles`| Guides the agent training to keep trades below desired length. Example usage shown in `prediction_models/ReinforcementLearner.py` within the customizable `calculate_reward()` function. <br> **Datatype:** int.
-| `model_type` | Model string from stable_baselines3 or SBcontrib. Available strings include: `'TRPO', 'ARS', 'RecurrentPPO', 'MaskablePPO', 'PPO', 'A2C', 'DQN'`. User should ensure that `model_training_parameters` match those available to the corresponding stable_baselines3 model by visiting their documentaiton. [PPO doc](https://stable-baselines3.readthedocs.io/en/master/modules/ppo.html) (external website) <br> **Datatype:** string.
+| `model_type` | Model string from stable_baselines3 or SBcontrib. Available strings include: `'TRPO', 'ARS', 'RecurrentPPO', 'MaskablePPO', 'PPO', 'A2C', 'DQN'`. User should ensure that `model_training_parameters` match those available to the corresponding stable_baselines3 model by visiting their documentation. [PPO doc](https://stable-baselines3.readthedocs.io/en/master/modules/ppo.html) (external website) <br> **Datatype:** string.
 | `policy_type` | One of the available policy types from stable_baselines3 <br> **Datatype:** string.
 | `max_training_drawdown_pct` | The maximum drawdown that the agent is allowed to experience during training. <br> **Datatype:** float. <br> Default: 0.8
 | `cpu_count` | Number of threads/cpus to dedicate to the Reinforcement Learning training process (depending on if `ReinforcementLearning_multiproc` is selected or not). Recommended to leave this untouched, by default, this value is set to the total number of physical cores minus 1. <br> **Datatype:** int. 

docs/freqai-reinforcement-learning.md

@@ -142,7 +142,7 @@ Parameter details can be found [here](freqai-parameter-table.md), but in general
 As you begin to modify the strategy and the prediction model, you will quickly realize some important differences between the Reinforcement Learner and the Regressors/Classifiers. Firstly, the strategy does not set a target value (no labels!). Instead, you set the `calculate_reward()` function inside the `MyRLEnv` class (see below). A default `calculate_reward()` is provided inside `prediction_models/ReinforcementLearner.py` to demonstrate the necessary building blocks for creating rewards, but this is *not* designed for production. Users *must* create their own custom reinforcement learning model class or use a pre-built one from outside the Freqtrade source code and save it to `user_data/freqaimodels`. It is inside the `calculate_reward()` where creative theories about the market can be expressed. For example, you can reward your agent when it makes a winning trade, and penalize the agent when it makes a losing trade. Or perhaps, you wish to reward the agent for entering trades, and penalize the agent for sitting in trades too long. Below we show examples of how these rewards are all calculated:
 
 !!! note "Hint"
-    The best reward functions are ones that are continuously differentiable, and well scaled. In other words, adding a single large negative penalty to a rare event is not a good idea, and the neural net will not be able to learn that function. Instead, it is better to add a small negative penalty to a common event. This will help the agent learn faster. Not only this, but you can help improve the continuity of your rewards/penalties by having them scale with severity according to some linear/exponential functions. In other words, you'd slowly scale the penalty as the duration of the trade increases. This is better than a single large penalty occuring at a single point in time.
+    The best reward functions are ones that are continuously differentiable, and well scaled. In other words, adding a single large negative penalty to a rare event is not a good idea, and the neural net will not be able to learn that function. Instead, it is better to add a small negative penalty to a common event. This will help the agent learn faster. Not only this, but you can help improve the continuity of your rewards/penalties by having them scale with severity according to some linear/exponential functions. In other words, you'd slowly scale the penalty as the duration of the trade increases. This is better than a single large penalty occurring at a single point in time.
 
 ```python
 from freqtrade.freqai.prediction_models.ReinforcementLearner import ReinforcementLearner

docs/freqai-running.md

@@ -41,11 +41,11 @@ FreqAI stores new model files after each successful training. These files become
 
 ```json
     "freqai": {
-        "purge_old_models": true,
+        "purge_old_models": 4,
     }
 ```
 
-This will automatically purge all models older than the two most recently trained ones to save disk space.
+This will automatically purge all models older than the four most recently trained ones to save disk space. Inputing "0" will never purge any models.
 
 ## Backtesting
 
@@ -68,7 +68,7 @@ Backtesting mode requires [downloading the necessary data](#downloading-data-to-
     This way, you can return to using any model you wish by simply specifying the `identifier`.
 
 !!! Note
-    Backtesting calls `set_freqai_targets()` one time for each backtest window (where the number of windows is the full backtest timerange divided by the `backtest_period_days` parameter). Doing this means that the targets simulate dry/live behavior without look ahead bias. However, the definition of the features in `feature_engineering_*()` is performed once on the entire backtest timerange. This means that you should be sure that features do look-ahead into the future.
+    Backtesting calls `set_freqai_targets()` one time for each backtest window (where the number of windows is the full backtest timerange divided by the `backtest_period_days` parameter). Doing this means that the targets simulate dry/live behavior without look ahead bias. However, the definition of the features in `feature_engineering_*()` is performed once on the entire training timerange. This means that you should be sure that features do not look-ahead into the future.
     More details about look-ahead bias can be found in [Common Mistakes](strategy-customization.md#common-mistakes-when-developing-strategies).
 
 ---

docs/freqai.md

@@ -114,6 +114,11 @@ Here we compile some external materials that provide deeper looks into various c
 - [Real-time head-to-head: Adaptive modeling of financial market data using XGBoost and CatBoost](https://emergentmethods.medium.com/real-time-head-to-head-adaptive-modeling-of-financial-market-data-using-xgboost-and-catboost-995a115a7495)
 - [FreqAI - from price to prediction](https://emergentmethods.medium.com/freqai-from-price-to-prediction-6fadac18b665)
 
+
+## Support
+
+You can find support for FreqAI in a variety of places, including the [Freqtrade discord](https://discord.gg/Jd8JYeWHc4), the dedicated [FreqAI discord](https://discord.gg/7AMWACmbjT), and in [github issues](https://github.com/freqtrade/freqtrade/issues).
+
 ## Credits
 
 FreqAI is developed by a group of individuals who all contribute specific skillsets to the project.

docs/hyperopt.md

@@ -439,7 +439,7 @@ While this strategy is most likely too simple to provide consistent profit, it s
 ??? Hint "Performance tip"
     During normal hyperopting, indicators are calculated once and supplied to each epoch, linearly increasing RAM usage as a factor of increasing cores. As this also has performance implications, there are two alternatives to reduce RAM usage
 
-    * Move `ema_short` and `ema_long` calculations from `populate_indicators()` to `populate_entry_trend()`. Since `populate_entry_trend()` gonna be calculated every epochs, you don't need to use `.range` functionality.
+    * Move `ema_short` and `ema_long` calculations from `populate_indicators()` to `populate_entry_trend()`. Since `populate_entry_trend()` will be calculated every epoch, you don't need to use `.range` functionality.
     * hyperopt provides `--analyze-per-epoch` which will move the execution of `populate_indicators()` to the epoch process, calculating a single value per parameter per epoch instead of using the `.range` functionality. In this case, `.range` functionality will only return the actually used value.
 
     These alternatives will reduce RAM usage, but increase CPU usage. However, your hyperopting run will be less likely to fail due to Out Of Memory (OOM) issues.
@@ -926,6 +926,12 @@ Once the optimized strategy has been implemented into your strategy, you should
 
 To achieve same the results (number of trades, their durations, profit, etc.) as during Hyperopt, please use the same configuration and parameters (timerange, timeframe, ...) used for hyperopt `--dmmp`/`--disable-max-market-positions` and `--eps`/`--enable-position-stacking` for Backtesting.
 
-Should results not match, please double-check to make sure you transferred all conditions correctly.
+### Why do my backtest results not match my hyperopt results?
-Pay special care to the stoploss, max_open_trades and trailing stoploss parameters, as these are often set in configuration files, which override changes to the strategy.
+Should results not match, check the following factors:
-You should also carefully review the log of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss`, `max_open_trades` or `trailing_stop`).
+
+* You may have added parameters to hyperopt in `populate_indicators()` where they will be calculated only once **for all epochs**. If you are, for example, trying to optimise multiple SMA timeperiod values, the hyperoptable timeperiod parameter should be placed in `populate_entry_trend()` which is calculated every epoch. See [Optimizing an indicator parameter](https://www.freqtrade.io/en/stable/hyperopt/#optimizing-an-indicator-parameter).
+* If you have disabled the auto-export of hyperopt parameters into the JSON parameters file, double-check to make sure you transferred all hyperopted values into your strategy correctly.
+* Check the logs to verify what parameters are being set and what values are being used.
+* Pay special care to the stoploss, max_open_trades and trailing stoploss parameters, as these are often set in configuration files, which override changes to the strategy. Check the logs of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss`, `max_open_trades` or `trailing_stop`).
+* Verify that you do not have an unexpected parameters JSON file overriding the parameters or the default hyperopt settings in your strategy.
+* Verify that any protections that are enabled in backtesting are also enabled when hyperopting, and vice versa. When using `--space protection`, protections are auto-enabled for hyperopting.

docs/includes/pairlists.md

@@ -6,7 +6,7 @@ In your configuration, you can use Static Pairlist (defined by the [`StaticPairL
 
 Additionally, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter), [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist.
 
-If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You should always configure either `StaticPairList` or `VolumePairList` as the starting Pairlist Handler.
+If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You can define either `StaticPairList`, `VolumePairList`, `ProducerPairList`, `RemotePairList` or `MarketCapPairList` as the starting Pairlist Handler.
 
 Inactive markets are always removed from the resulting pairlist. Explicitly blacklisted pairs (those in the `pair_blacklist` configuration setting) are also always removed from the resulting pairlist.
 
@@ -24,6 +24,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged
 * [`VolumePairList`](#volume-pair-list)
 * [`ProducerPairList`](#producerpairlist)
 * [`RemotePairList`](#remotepairlist)
+* [`MarketCapPairList`](#marketcappairlist)
 * [`AgeFilter`](#agefilter)
 * [`FullTradesFilter`](#fulltradesfilter)
 * [`OffsetFilter`](#offsetfilter)
@@ -67,7 +68,7 @@ When used in the leading position of the chain of Pairlist Handlers, the `pair_w
 
 The `refresh_period` setting allows to define the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes).
 The pairlist cache (`refresh_period`) on `VolumePairList` is only applicable to generating pairlists.
-Filtering instances (not the first position in the list) will not apply any cache and will always use up-to-date data.
+Filtering instances (not the first position in the list) will not apply any cache (beyond caching candles for the duration of the candle in advanced mode) and will always use up-to-date data.
 
 `VolumePairList` is per default based on the ticker data from exchange, as reported by the ccxt library:
 
@@ -80,12 +81,14 @@ Filtering instances (not the first position in the list) will not apply any cach
         "number_assets": 20,
         "sort_key": "quoteVolume",
         "min_value": 0,
+        "max_value": 8000000,
         "refresh_period": 1800
     }
 ],
 ```
 
 You can define a minimum volume with `min_value` - which will filter out pairs with a volume lower than the specified value in the specified timerange.
+In addition to that, you can also define a maximum volume with `max_value` - which will filter out pairs with a volume higher than the specified value in the specified timerange.
 
 ##### VolumePairList Advanced mode
 
@@ -112,8 +115,8 @@ For convenience `lookback_days` can be specified, which will imply that 1d candl
 !!! Warning "Performance implications when using lookback range"
     If used in first position in combination with lookback, the computation of the range based volume can be time and resource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `VolumeFilter` to narrow the pairlist down for further range volume calculation.
 
-??? Tip "Unsupported exchanges (Bittrex, Gemini)"
+??? Tip "Unsupported exchanges"
-    On some exchanges (like Bittrex and Gemini), regular VolumePairList does not work as the api does not natively provide 24h volume. This can be worked around by using candle data to build the volume.
+    On some exchanges (like Gemini), regular VolumePairList does not work as the api does not natively provide 24h volume. This can be worked around by using candle data to build the volume.
     To roughly simulate 24h volume, you can use the following configuration.
     Please note that These pairlists will only refresh once per day.
 
@@ -192,14 +195,15 @@ The RemotePairList is defined in the pairlists section of the configuration sett
         "refresh_period": 1800,
         "keep_pairlist_on_failure": true,
         "read_timeout": 60,
-        "bearer_token": "my-bearer-token"
+        "bearer_token": "my-bearer-token",
+        "save_to_file": "user_data/filename.json" 
     }
 ]
 ```
 
 The optional `mode` option specifies if the pairlist should be used as a `blacklist` or as a `whitelist`. The default value is "whitelist".
 
-The optional `processing_mode` option in the RemotePairList configuration determines how the retrieved pairlist is processed. It can have two values: "filter" or "append".
+The optional `processing_mode` option in the RemotePairList configuration determines how the retrieved pairlist is processed. It can have two values: "filter" or "append". The default value is "filter".
 
 In "filter" mode, the retrieved pairlist is used as a filter. Only the pairs present in both the original pairlist and the retrieved pairlist are included in the final pairlist. Other pairs are filtered out.
 
@@ -207,6 +211,42 @@ In "append" mode, the retrieved pairlist is added to the original pairlist. All
 
 The `pairlist_url` option specifies the URL of the remote server where the pairlist is located, or the path to a local file (if file:/// is prepended). This allows the user to use either a remote server or a local file as the source for the pairlist.
 
+The `save_to_file` option, when provided with a valid filename, saves the processed pairlist to that file in JSON format. This option is optional, and by default, the pairlist is not saved to a file.
+
+??? Example "Multi bot with shared pairlist example"
+    
+    `save_to_file` can be used to save the pairlist to a file with Bot1:
+
+    ```json
+    "pairlists": [
+        {
+            "method": "RemotePairList",
+            "mode": "whitelist",
+            "pairlist_url": "https://example.com/pairlist",
+            "number_assets": 10,
+            "refresh_period": 1800,
+            "keep_pairlist_on_failure": true,
+            "read_timeout": 60,
+            "save_to_file": "user_data/filename.json" 
+        }
+    ]
+    ```
+
+    This saved pairlist file can be loaded by Bot2, or any additional bot with this configuration:
+
+    ```json
+    "pairlists": [
+        {
+            "method": "RemotePairList",
+            "mode": "whitelist",
+            "pairlist_url": "file:///user_data/filename.json",
+            "number_assets": 10,
+            "refresh_period": 10,
+            "keep_pairlist_on_failure": true,
+        }
+    ]
+    ```    
+
 The user is responsible for providing a server or local file that returns a JSON object with the following structure:
 
 ```json
@@ -227,6 +267,25 @@ The optional `bearer_token` will be included in the requests Authorization Heade
 !!! Note
     In case of a server error the last received pairlist will be kept if `keep_pairlist_on_failure` is set to true, when set to false a empty pairlist is returned.
 
+#### MarketCapPairList
+
+`MarketCapPairList` employs sorting/filtering of pairs by their marketcap rank based of CoinGecko. It will only recognize coins up to the coin placed at rank 250. The returned pairlist will be sorted based of their marketcap ranks.
+
+```json
+"pairlists": [
+    {
+        "method": "MarketCapPairList",
+        "number_assets": 20,
+        "max_rank": 50,
+        "refresh_period": 86400
+    }
+]
+```
+
+`number_assets` defines the maximum number of pairs returned by the pairlist. `max_rank` will determine the maximum rank used in creating/filtering the pairlist. It's expected that some coins within the top `max_rank` marketcap will not be included in the resulting pairlist since not all pairs will have active trading pairs in your preferred market/stake/exchange combination.
+
+`refresh_period` setting defines the period (in seconds) at which the marketcap rank data will be refreshed. Defaults to 86,400s (1 day). The pairlist cache (`refresh_period`) is applicable on both generating pairlists (first position in the list) and filtering instances (not the first position in the list).
+
 #### AgeFilter
 
 Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`) or more than `max_days_listed` days (defaults `None` mean infinity).
@@ -312,6 +371,11 @@ As this Filter uses past performance of the bot, it'll have some startup-period
 
 Filters low-value coins which would not allow setting stoplosses.
 
+Namely, pairs are blacklisted if a variance of one percent or more in the stop price would be caused by precision rounding on the exchange, i.e. `rounded(stop_price) <= rounded(stop_price * 0.99)`. The idea is to avoid coins with a value VERY close to their lower trading boundary, not allowing setting of proper stoploss.
+
+!!! Tip "PerformanceFilter is pointless for futures trading"
+    The above does not apply to shorts. And for longs, in theory the trade will be liquidated first.
+
 !!! Warning "Backtesting"
     `PrecisionFilter` does not support backtesting mode using multiple strategies.
 
@@ -393,6 +457,8 @@ If the trading range over the last 10 days is <1% or >99%, remove the pair from
 ]
 ```
 
+Adding `"sort_direction": "asc"` or `"sort_direction": "desc"` enables sorting for this pairlist.
+
 !!! Tip
     This Filter can be used to automatically remove stable coin pairs, which have a very low trading range, and are therefore extremely difficult to trade with profit.
     Additionally, it can also be used to automatically remove pairs with extreme high/low variance over a given amount of time.
@@ -403,7 +469,7 @@ Volatility is the degree of historical variation of a pairs over time, it is mea
 
 This filter removes pairs if the average volatility over a `lookback_days` days is below `min_volatility` or above `max_volatility`. Since this is a filter that requires additional data, the results are cached for `refresh_period`.
 
-This filter can be used to narrow down your pairs to a certain volatility or avoid very volatile pairs. 
+This filter can be used to narrow down your pairs to a certain volatility or avoid very volatile pairs.
 
 In the below example:
 If the volatility over the last 10 days is not in the range of 0.05-0.50, remove the pair from the whitelist. The filter is applied every 24h.
@@ -420,6 +486,8 @@ If the volatility over the last 10 days is not in the range of 0.05-0.50, remove
 ]
 ```
 
+Adding `"sort_direction": "asc"` or `"sort_direction": "desc"` enables sorting mode for this pairlist.
+
 ### Full example of Pairlist Handlers
 
 The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#pricefilter), filtering all assets where 1 price unit is > 1%. Then the [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) is applied and pairs are finally shuffled with the random seed set to some predefined value.

docs/includes/showcase.md

@@ -5,7 +5,7 @@ This section will highlight a few projects from members of the community.
 - [Example freqtrade strategies](https://github.com/freqtrade/freqtrade-strategies/)
 - [FrequentHippo - Grafana dashboard with dry/live runs and backtests](http://frequenthippo.ddns.net:3000/) (by hippocritical).
 - [Online pairlist generator](https://remotepairlist.com/) (by Blood4rc).
-- [Freqtrade Backtesting Project](https://bt.robot.co.network/) (by Blood4rc).
+- [Freqtrade Backtesting Project](https://strat.ninja/) (by Blood4rc).
 - [Freqtrade analysis notebook](https://github.com/froggleston/freqtrade_analysis_notebook) (by Froggleston).
 - [TUI for freqtrade](https://github.com/froggleston/freqtrade-frogtrade9000) (by Froggleston).
 - [Bot Academy](https://botacademy.ddns.net/) (by stash86) - Blog about crypto bot projects.

docs/index.md

@@ -42,7 +42,7 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual,
 - [X] [Binance](https://www.binance.com/)
 - [X] [Bitmart](https://bitmart.com/)
 - [X] [Gate.io](https://www.gate.io/ref/6266643)
-- [X] [Huobi](http://huobi.com/)
+- [X] [HTX](https://www.htx.com/) (Former Huobi)
 - [X] [Kraken](https://kraken.com/)
 - [X] [OKX](https://okx.com/) (Former OKEX)
 - [ ] [potentially many others through <img alt="ccxt" width="30px" src="assets/ccxt-logo.svg" />](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_

docs/lookahead-analysis.md

@@ -23,6 +23,7 @@ It also supports the lookahead-analysis of freqai strategies.
 - `--max-open-trades` is forced to be at least equal to the number of pairs.
 - `--dry-run-wallet` is forced to be basically infinite (1 billion).
 - `--stake-amount` is forced to be a static 10000 (10k).
+- `--enable-protections` is forced to be off.
 
 Those are set to avoid users accidentally generating false positives.
 
@@ -40,7 +41,6 @@ usage: freqtrade lookahead-analysis [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                     [--max-open-trades INT]
                                     [--stake-amount STAKE_AMOUNT]
                                     [--fee FLOAT] [-p PAIRS [PAIRS ...]]
-                                    [--enable-protections]
                                     [--dry-run-wallet DRY_RUN_WALLET]
                                     [--timeframe-detail TIMEFRAME_DETAIL]
                                     [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]]

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
-markdown==3.5.1
+markdown==3.6
 mkdocs==1.5.3
-mkdocs-material==9.4.14
+mkdocs-material==9.5.15
 mdx_truly_sane_lists==1.3
-pymdown-extensions==10.5
+pymdown-extensions==10.7.1
-jinja2==3.1.2
+jinja2==3.1.3

docs/rest-api.md

@@ -95,11 +95,13 @@ Make sure that the following 2 lines are available in your docker-compose file:
 
 ### Consuming the API
 
-You can consume the API by using the script `scripts/rest_client.py`.
+You can consume the API by using `freqtrade-client` (also available as `scripts/rest_client.py`).
-The client script only requires the `requests` module, so Freqtrade does not need to be installed on the system.
+This command can be installed independent of the bot by using `pip install freqtrade-client`.
+
+This module is designed to be lightweight, and only depends on the `requests` and `python-rapidjson` modules, skipping all heavy dependencies freqtrade otherwise needs.
 
 ``` bash
-python3 scripts/rest_client.py <command> [optional parameters]
+freqtrade-client <command> [optional parameters]
 ```
 
 By default, the script assumes `127.0.0.1` (localhost) and port `8080` to be used, however you can specify a configuration file to override this behaviour.
@@ -120,9 +122,27 @@ By default, the script assumes `127.0.0.1` (localhost) and port `8080` to be use
 ```
 
 ``` bash
-python3 scripts/rest_client.py --config rest_config.json <command> [optional parameters]
+freqtrade-client --config rest_config.json <command> [optional parameters]
 ```
 
+??? Note "Programmatic use"
+    The `freqtrade-client` package (installable independent of freqtrade) can be used in your own scripts to interact with the freqtrade API.
+    to do so, please use the following:
+
+    ``` python
+    from freqtrade_client import FtRestClient
+    
+
+    client = FtRestClient(server_url, username, password)
+
+    # Get the status of the bot
+    ping = client.ping()
+    print(ping)
+    # ... 
+    ```
+
+    For a full list of available commands, please refer to the list below.
+
 ### Available endpoints
 
 |  Command | Description |
@@ -176,7 +196,7 @@ python3 scripts/rest_client.py --config rest_config.json <command> [optional par
 Possible commands can be listed from the rest-client script using the `help` command.
 
 ``` bash
-python3 scripts/rest_client.py help
+freqtrade-client help
 ```
 
 ``` output

docs/sql_cheatsheet.md

@@ -109,7 +109,7 @@ Freqtrade does not depend or install any additional database driver. Please refe
 The following systems have been tested and are known to work with freqtrade:
 
 * sqlite (default)
-* PostgreSQL)
+* PostgreSQL
 * MariaDB
 
 !!! Warning

docs/stoploss.md

@@ -30,7 +30,7 @@ The Order-type will be ignored if only one mode is available.
 |----------|-------------|
 | Binance  | limit |
 | Binance Futures  | market, limit |
-| Huobi    | limit |
+| HTX (former Huobi)    | limit |
 | kraken   | market, limit |
 | Gate     | limit |
 | Okx      | limit |

docs/strategy-advanced.md

@@ -11,34 +11,129 @@ The call sequence of the methods described here is covered under [bot execution
 !!! Tip
     Start off with a strategy template containing all available callback methods by running `freqtrade new-strategy --strategy MyAwesomeStrategy --template advanced`
 
-## Storing information
+## Storing information (Persistent)
 
-Storing information can be accomplished by creating a new dictionary within the strategy class.
+Freqtrade allows storing/retrieving user custom information associated with a specific trade in the database.
 
-The name of the variable can be chosen at will, but should be prefixed with `custom_` to avoid naming collisions with predefined strategy variables.
+Using a trade object, information can be stored using `trade.set_custom_data(key='my_key', value=my_value)` and retrieved using `trade.get_custom_data(key='my_key')`. Each data entry is associated with a trade and a user supplied key (of type `string`). This means that this can only be used in callbacks that also provide a trade object.
+
+For the data to be able to be stored within the database, freqtrade must serialized the data. This is done by converting the data to a JSON formatted string.
+Freqtrade will attempt to reverse this action on retrieval, so from a strategy perspective, this should not be relevant.
 
 ```python
+from freqtrade.persistence import Trade
+from datetime import timedelta
+
 class AwesomeStrategy(IStrategy):
-    # Create custom dictionary
-    custom_info = {}
 
-    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+    def bot_loop_start(self, **kwargs) -> None:
-        # Check if the entry already exists
+        for trade in Trade.get_open_order_trades():
-        if not metadata["pair"] in self.custom_info:
+            fills = trade.select_filled_orders(trade.entry_side)
-            # Create empty entry for this pair
+            if trade.pair == 'ETH/USDT':
-            self.custom_info[metadata["pair"]] = {}
+                trade_entry_type = trade.get_custom_data(key='entry_type')
+                if trade_entry_type is None:
+                    trade_entry_type = 'breakout' if 'entry_1' in trade.enter_tag else 'dip'
+                elif fills > 1:
+                    trade_entry_type = 'buy_up'
+                trade.set_custom_data(key='entry_type', value=trade_entry_type)
+        return super().bot_loop_start(**kwargs)
 
-        if "crosstime" in self.custom_info[metadata["pair"]]:
+    def adjust_entry_price(self, trade: Trade, order: Optional[Order], pair: str,
-            self.custom_info[metadata["pair"]]["crosstime"] += 1
+                           current_time: datetime, proposed_rate: float, current_order_rate: float,
-        else:
+                           entry_tag: Optional[str], side: str, **kwargs) -> float:
-            self.custom_info[metadata["pair"]]["crosstime"] = 1
+        # Limit orders to use and follow SMA200 as price target for the first 10 minutes since entry trigger for BTC/USDT pair.
+        if (
+            pair == 'BTC/USDT' 
+            and entry_tag == 'long_sma200' 
+            and side == 'long' 
+            and (current_time - timedelta(minutes=10)) > trade.open_date_utc 
+            and order.filled == 0.0
+        ):
+            dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe)
+            current_candle = dataframe.iloc[-1].squeeze()
+            # store information about entry adjustment
+            existing_count = trade.get_custom_data('num_entry_adjustments', default=0)
+            if not existing_count:
+                existing_count = 1
+            else:
+                existing_count += 1
+            trade.set_custom_data(key='num_entry_adjustments', value=existing_count)
+
+            # adjust order price
+            return current_candle['sma_200']
+
+        # default: maintain existing order
+        return current_order_rate
+
+    def custom_exit(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs):
+
+        entry_adjustment_count = trade.get_custom_data(key='num_entry_adjustments')
+        trade_entry_type = trade.get_custom_data(key='entry_type')
+        if entry_adjustment_count is None:
+            if current_profit > 0.01 and (current_time - timedelta(minutes=100) > trade.open_date_utc):
+                return True, 'exit_1'
+        else
+            if entry_adjustment_count > 0 and if current_profit > 0.05:
+                return True, 'exit_2'
+            if trade_entry_type == 'breakout' and current_profit > 0.1:
+                return True, 'exit_3
+
+        return False, None
 ```
 
-!!! Warning
+The above is a simple example - there are simpler ways to retrieve trade data like entry-adjustments.
-    The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash.
 
 !!! Note
-    If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
+    It is recommended that simple data types are used `[bool, int, float, str]` to ensure no issues when serializing the data that needs to be stored.
+    Storing big junks of data may lead to unintended side-effects, like a database becoming big (and as a consequence, also slow).
+
+!!! Warning "Non-serializable data"
+    If supplied data cannot be serialized a warning is logged and the entry for the specified `key` will contain `None` as data.
+
+??? Note "All attributes"
+    custom-data has the following accessors through the Trade object (assumed as `trade` below):
+
+    * `trade.get_custom_data(key='something', default=0)` - Returns the actual value given in the type provided.
+    * `trade.get_custom_data_entry(key='something')` - Returns the entry - including metadata. The value is accessible via `.value` property.
+    * `trade.set_custom_data(key='something', value={'some': 'value'})` - set or update the corresponding key for this trade. Value must be serializable - and we recommend to keep the stored data relatively small.
+
+    "value" can be any type (both in setting and receiving) - but must be json serializable.
+
+## Storing information (Non-Persistent)
+
+!!! Warning "Deprecated"
+    This method of storing information is deprecated and we do advise against using non-persistent storage.  
+    Please use [Persistent Storage](#storing-information-persistent) instead.
+
+    It's content has therefore been collapsed.
+
+??? Abstract "Storing information"
+    Storing information can be accomplished by creating a new dictionary within the strategy class.
+
+    The name of the variable can be chosen at will, but should be prefixed with `custom_` to avoid naming collisions with predefined strategy variables.
+
+    ```python
+    class AwesomeStrategy(IStrategy):
+        # Create custom dictionary
+        custom_info = {}
+
+        def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+            # Check if the entry already exists
+            if not metadata["pair"] in self.custom_info:
+                # Create empty entry for this pair
+                self.custom_info[metadata["pair"]] = {}
+
+            if "crosstime" in self.custom_info[metadata["pair"]]:
+                self.custom_info[metadata["pair"]]["crosstime"] += 1
+            else:
+                self.custom_info[metadata["pair"]]["crosstime"] = 1
+    ```
+
+    !!! Warning
+        The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash.
+
+    !!! Note
+        If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
 
 ## Dataframe access
 

docs/strategy-callbacks.md

@@ -19,6 +19,7 @@ Currently available callbacks:
 * [`adjust_trade_position()`](#adjust-trade-position)
 * [`adjust_entry_price()`](#adjust-entry-price)
 * [`leverage()`](#leverage-callback)
+* [`order_filled()`](#order-filled-callback)
 
 !!! Tip "Callback calling sequence"
     You can find the callback calling sequence in [bot-basics](bot-basics.md#bot-execution-logic)
@@ -489,7 +490,7 @@ The helper function `stoploss_from_absolute()` can be used to convert from an ab
             dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe)
             trade_date = timeframe_to_prev_date(self.timeframe, trade.open_date_utc)
             candle = dataframe.iloc[-1].squeeze()
-            sign = 1 if trade.is_short else -1
+            side = 1 if trade.is_short else -1
             return stoploss_from_absolute(current_rate + (side * candle['atr'] * 2), 
                                           current_rate, is_short=trade.is_short,
                                           leverage=trade.leverage)
@@ -760,28 +761,38 @@ The `position_adjustment_enable` strategy property enables the usage of `adjust_
 For performance reasons, it's disabled by default and freqtrade will show a warning message on startup if enabled.
 `adjust_trade_position()` can be used to perform additional orders, for example to manage risk with DCA (Dollar Cost Averaging) or to increase or decrease positions.
 
-`max_entry_position_adjustment` property is used to limit the number of additional entries per trade (on top of the first entry order) that the bot can execute. By default, the value is -1 which means the bot have no limit on number of adjustment entries.
-
-The strategy is expected to return a stake_amount (in stake currency) between `min_stake` and `max_stake` if and when an additional entry order should be made (position is increased -> buy order for long trades, sell order for short trades).
-If there are not enough funds in the wallet (the return value is above `max_stake`) then the signal will be ignored.
 Additional orders also result in additional fees and those orders don't count towards `max_open_trades`.
 
 This callback is **not** called when there is an open order (either buy or sell) waiting for execution.
 
 `adjust_trade_position()` is called very frequently for the duration of a trade, so you must keep your implementation as performant as possible.
 
-Additional entries are ignored once you have reached the maximum amount of extra entries that you have set on `max_entry_position_adjustment`, but the callback is called anyway looking for partial exits.
-
 Position adjustments will always be applied in the direction of the trade, so a positive value will always increase your position (negative values will decrease your position), no matter if it's a long or short trade.
+Adjustment orders can be assigned with a tag by returning a 2 element Tuple, with the first element being the adjustment amount, and the 2nd element the tag (e.g. `return 250, 'increase_favorable_conditions'`).
 
 Modifications to leverage are not possible, and the stake-amount returned is assumed to be before applying leverage.
 
+### Increase position
+
+The strategy is expected to return a positive **stake_amount** (in stake currency) between `min_stake` and `max_stake` if and when an additional entry order should be made (position is increased -> buy order for long trades, sell order for short trades).
+
+If there are not enough funds in the wallet (the return value is above `max_stake`) then the signal will be ignored.
+`max_entry_position_adjustment` property is used to limit the number of additional entries per trade (on top of the first entry order) that the bot can execute. By default, the value is -1 which means the bot have no limit on number of adjustment entries.
+
+Additional entries are ignored once you have reached the maximum amount of extra entries that you have set on `max_entry_position_adjustment`, but the callback is called anyway looking for partial exits.
+
+### Decrease position
+
+The strategy is expected to return a negative stake_amount (in stake currency) for a partial exit.
+Returning the full owned stake at that point (`-trade.stake_amount`) results in a full exit.  
+Returning a value more than the above (so remaining stake_amount would become negative) will result in the bot ignoring the signal.
+
 !!! Note "About stake size"
     Using fixed stake size means it will be the amount used for the first order, just like without position adjustment.
     If you wish to buy additional orders with DCA, then make sure to leave enough funds in the wallet for that.
     Using 'unlimited' stake amount with DCA orders requires you to also implement the `custom_stake_amount()` callback to avoid allocating all funds to the initial order.
 
-!!! Warning
+!!! Warning "Stoploss calculation"
     Stoploss is still calculated from the initial opening price, not averaged price.
     Regular stoploss rules still apply (cannot move down).
 
@@ -791,6 +802,11 @@ Modifications to leverage are not possible, and the stake-amount returned is ass
     During backtesting this callback is called for each candle in `timeframe` or `timeframe_detail`, so run-time performance will be affected.
     This can also cause deviating results between live and backtesting, since backtesting can adjust the trade only once per candle, whereas live could adjust the trade multiple times per candle.
 
+!!! Warning "Performance with many position adjustments"
+    Position adjustments can be a good approach to increase a strategy's output - but it can also have drawbacks if using this feature extensively.  
+    Each of the orders will be attached to the trade object for the duration of the trade - hence increasing memory usage.
+    Trades with long duration and 10s or even 100ds of position adjustments are therefore not recommended, and should be closed at regular intervals to not affect performance.
+
 ``` python
 from freqtrade.persistence import Trade
 
@@ -824,7 +840,8 @@ class DigDeeperStrategy(IStrategy):
                               min_stake: Optional[float], max_stake: float,
                               current_entry_rate: float, current_exit_rate: float,
                               current_entry_profit: float, current_exit_profit: float,
-                              **kwargs) -> Optional[float]:
+                              **kwargs
+                              ) -> Union[Optional[float], Tuple[Optional[float], Optional[str]]]:
         """
         Custom trade adjustment logic, returning the stake amount that a trade should be
         increased or decreased.
@@ -850,11 +867,12 @@ class DigDeeperStrategy(IStrategy):
         :return float: Stake amount to adjust your trade,
                        Positive values to increase position, Negative values to decrease position.
                        Return None for no action.
+                       Optionally, return a tuple with a 2nd element with an order reason
         """
 
         if current_profit > 0.05 and trade.nr_of_successful_exits == 0:
             # Take half of the profit at +5%
-            return -(trade.stake_amount / 2)
+            return -(trade.stake_amount / 2), 'half_profit_5%'
 
         if current_profit > -0.05:
             return None
@@ -882,7 +900,7 @@ class DigDeeperStrategy(IStrategy):
             stake_amount = filled_entries[0].stake_amount
             # This then calculates current safety order size
             stake_amount = stake_amount * (1 + (count_of_entries * 0.25))
-            return stake_amount
+            return stake_amount, '1/3rd_increase'
         except Exception as exception:
             return None
 
@@ -1005,3 +1023,33 @@ class AwesomeStrategy(IStrategy):
 
 All profit calculations include leverage. Stoploss / ROI also include leverage in their calculation.
 Defining a stoploss of 10% at 10x leverage would trigger the stoploss with a 1% move to the downside.
+
+## Order filled Callback
+
+The `order_filled()` callback may be used to perform specific actions based on the current trade state after an order is filled.
+It will be called independent of the order type (entry, exit, stoploss or position adjustment).
+
+Assuming that your strategy needs to store the high value of the candle at trade entry, this is possible with this callback as the following example show.
+
+``` python
+class AwesomeStrategy(IStrategy):
+    def order_filled(self, pair: str, trade: Trade, order: Order, current_time: datetime, **kwargs) -> None:
+        """
+        Called right after an order fills. 
+        Will be called for all order types (entry, exit, stoploss, position adjustment).
+        :param pair: Pair for trade
+        :param trade: trade object.
+        :param order: Order object.
+        :param current_time: datetime object, containing the current datetime
+        :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
+        """
+        # Obtain pair dataframe (just to show how to access it)
+        dataframe, _ = self.dp.get_analyzed_dataframe(trade.pair, self.timeframe)
+        last_candle = dataframe.iloc[-1].squeeze()
+        
+        if (trade.nr_of_successful_entries == 1) and (order.ft_order_side == trade.entry_side):
+            trade.set_custom_data(key='entry_candle_high', value=last_candle['high'])
+
+        return None
+
+```

docs/strategy-customization.md

@@ -156,9 +156,9 @@ def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame
 
 Out of the box, freqtrade installs the following technical libraries:
 
-* [ta-lib](http://mrjbq7.github.io/ta-lib/)
+- [ta-lib](https://ta-lib.github.io/ta-lib-python/)
-* [pandas-ta](https://twopirllc.github.io/pandas-ta/)
+- [pandas-ta](https://twopirllc.github.io/pandas-ta/)
-* [technical](https://github.com/freqtrade/technical/)
+- [technical](https://github.com/freqtrade/technical/)
 
 Additional technical libraries can be installed as necessary, or custom indicators may be written / invented by the strategy author.
 
@@ -367,6 +367,11 @@ class AwesomeStrategy(IStrategy):
     }
 ```
 
+??? info "Orders that don't fill immediately"
+    `minimal_roi` will take the `trade.open_date` as reference, which is the time the trade was initialized / the first order for this trade was placed.  
+    This will also hold true for limit orders that don't fill immediately  (usually in combination with "off-spot" prices through `custom_entry_price()`), as well as for cases where the initial order is replaced through `adjust_entry_price()`.
+    The time used will still be from the initial `trade.open_date` (when the initial order was first placed), not from the newly placed order date.
+
 ### Stoploss
 
 Setting a stoploss is highly recommended to protect your capital from strong moves against you.
@@ -1004,8 +1009,8 @@ This is a common pain-point, which can cause huge differences between backtestin
 
 The following lists some common patterns which should be avoided to prevent frustration:
 
-- don't use `shift(-1)`. This uses data from the future, which is not available.
+- don't use `shift(-1)` or other negative values. This uses data from the future in backtesting, which is not available in dry or live modes.
-- don't use `.iloc[-1]` or any other absolute position in the dataframe, this will be different between dry-run and backtesting.
+- don't use `.iloc[-1]` or any other absolute position in the dataframe within `populate_` functions, as this will be different between dry-run and backtesting. Absolute `iloc` indexing is safe to use in callbacks however - see [Strategy Callbacks](strategy-callbacks.md).
 - don't use `dataframe['volume'].mean()`. This uses the full DataFrame for backtesting, including data from the future. Use `dataframe['volume'].rolling(<window>).mean()` instead
 - don't use `.resample('1h')`. This uses the left border of the interval, so moves data from an hour to the start of the hour. Use `.resample('1h', label='right')` instead.
 

docs/strategy_analysis_example.md

@@ -19,7 +19,7 @@ from pathlib import Path
 project_root = "somedir/freqtrade"
 i=0
 try:
-    os.chdirdir(project_root)
+    os.chdir(project_root)
     assert Path('LICENSE').is_file()
 except:
     while i<4 and (not Path('LICENSE').is_file()):

docs/telegram-usage.md

@@ -59,7 +59,7 @@ For the Freqtrade configuration, you can then use the the full value (including
    "chat_id": "-1001332619709"
 ```
 !!! Warning "Using telegram groups"
-    When using telegram groups, you're giving every member of the telegram group access to your freqtrade bot and to all commands possible via telegram. Please make sure that you can trust everyone in the telegram group to avoid unpleasent surprises.
+    When using telegram groups, you're giving every member of the telegram group access to your freqtrade bot and to all commands possible via telegram. Please make sure that you can trust everyone in the telegram group to avoid unpleasant surprises.
 
 ## Control telegram noise
 
@@ -181,6 +181,7 @@ official commands. You can ask at any moment for help with `/help`.
 | `/locks` | Show currently locked pairs.
 | `/unlock <pair or lock_id>` | Remove the lock for this pair (or for this lock id).
 | `/marketdir [long | short | even | none]` | Updates the user managed variable that represents the current market direction. If no direction is provided, the currently set direction will be displayed.
+| `/list_custom_data <trade_id> [key]` | List custom_data for Trade ID & Key combination. If no Key is supplied it will list all key-value pairs found for that Trade ID.
 | **Modify Trade states** |
 | `/forceexit <trade_id> | /fx <tradeid>` | Instantly exits the given trade  (Ignoring `minimum_roi`).
 | `/forceexit all | /fx all` | Instantly exits all open trades (Ignoring `minimum_roi`).

docs/updating.md

@@ -6,7 +6,7 @@ To update your freqtrade installation, please use one of the below methods, corr
     Breaking changes / changed behavior will be documented in the changelog that is posted alongside every release.
     For the develop branch, please follow PR's to avoid being surprised by changes.
 
-## docker
+## Docker
 
 !!! Note "Legacy installations using the `master` image"
     We're switching from master to stable for the release Images - please adjust your docker-file and replace `freqtradeorg/freqtrade:master` with `freqtradeorg/freqtrade:stable`

docs/utils.md

@@ -54,7 +54,7 @@ optional arguments:
 ### Create config examples
 
 ```
-$ freqtrade new-config --config config_binance.json
+$ freqtrade new-config --config user_data/config_binance.json
 
 ? Do you want to enable Dry-run (simulated trades)?  Yes
 ? Please insert your stake currency: BTC
@@ -66,6 +66,53 @@ $ freqtrade new-config --config config_binance.json
 ? Do you want to enable Telegram?  No
 ```
 
+## Show config
+
+Show configuration file (with sensitive values redacted by default).
+Especially useful with [split configuration files](configuration.md#multiple-configuration-files) or [environment variables](configuration.md#environment-variables), where this command will show the merged configuration.
+
+![Show config output](assets/show-config-output.png)
+
+```
+usage: freqtrade show-config [-h] [--userdir PATH] [-c PATH]
+                             [--show-sensitive]
+
+options:
+  -h, --help            show this help message and exit
+  --userdir PATH, --user-data-dir PATH
+                        Path to userdata directory.
+  -c PATH, --config PATH
+                        Specify configuration file (default:
+                        `userdir/config.json` or `config.json` whichever
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
+  --show-sensitive      Show secrets in the output.
+```
+
+``` output
+Your combined configuration is:
+{
+  "exit_pricing": {
+    "price_side": "other",
+    "use_order_book": true,
+    "order_book_top": 1
+  },
+  "stake_currency": "USDT",
+  "exchange": {
+    "name": "binance",
+    "key": "REDACTED",
+    "secret": "REDACTED",
+    "ccxt_config": {},
+    "ccxt_async_config": {},
+  }
+  // ...
+}
+```
+
+!!! Warning "Sharing information provided by this command"
+    We try to remove all known sensitive information from the default output (without `--show-sensitive`). 
+    Yet, please do double-check for sensitive values in your output to make sure you're not accidentally exposing some private info.
+
 ## Create new strategy
 
 Creates a new strategy from a template similar to SampleStrategy.
@@ -219,209 +266,49 @@ optional arguments:
   -a, --all         Print all exchanges known to the ccxt library.
 ```
 
-* Example: see exchanges available for the bot:
+Example: see exchanges available for the bot:
+
 ```
 $ freqtrade list-exchanges
 Exchanges available for Freqtrade:
-Exchange name    Valid    reason
+Exchange name       Supported    Markets                 Reason
----------------  -------  --------------------------------------------
+------------------  -----------  ----------------------  ------------------------------------------------------------------------
-aax              True
+binance             Official     spot, isolated futures
-ascendex         True     missing opt: fetchMyTrades
+bitmart             Official     spot
-bequant          True
+bybit                            spot, isolated futures
-bibox            True
+gate                Official     spot, isolated futures
-bigone           True
+htx                 Official     spot
-binance          True
+huobi                            spot
-binanceus        True
+kraken              Official     spot
-bitbank          True     missing opt: fetchTickers
+okx                 Official     spot, isolated futures
-bitcoincom       True
-bitfinex         True
-bitforex         True     missing opt: fetchMyTrades, fetchTickers
-bitget           True
-bithumb          True     missing opt: fetchMyTrades
-bitkk            True     missing opt: fetchMyTrades
-bitmart          True
-bitmax           True     missing opt: fetchMyTrades
-bitpanda         True
-bittrex          True
-bitvavo          True
-bitz             True     missing opt: fetchMyTrades
-btcalpha         True     missing opt: fetchTicker, fetchTickers
-btcmarkets       True     missing opt: fetchTickers
-buda             True     missing opt: fetchMyTrades, fetchTickers
-bw               True     missing opt: fetchMyTrades, fetchL2OrderBook
-bybit            True
-bytetrade        True
-cdax             True
-cex              True     missing opt: fetchMyTrades
-coinbaseprime    True     missing opt: fetchTickers
-coinbasepro      True     missing opt: fetchTickers
-coinex           True
-crex24           True
-deribit          True
-digifinex        True
-equos            True     missing opt: fetchTicker, fetchTickers
-eterbase         True
-fcoin            True     missing opt: fetchMyTrades, fetchTickers
-fcoinjp          True     missing opt: fetchMyTrades, fetchTickers
-gateio           True
-gemini           True
-gopax            True
-hbtc             True
-hitbtc           True
-huobijp          True
-huobipro         True
-idex             True
-kraken           True
-kucoin           True
-lbank            True     missing opt: fetchMyTrades
-mercado          True     missing opt: fetchTickers
-ndax             True     missing opt: fetchTickers
-novadax          True
-okcoin           True
-okex             True
-probit           True
-qtrade           True
-stex             True
-timex            True
-upbit            True     missing opt: fetchMyTrades
-vcc              True
-zb               True     missing opt: fetchMyTrades
-
 ```
 
+!!! info ""
+    Output reduced for clarity - supported and available exchanges may change over time.
+
 !!! Note "missing opt exchanges"
     Values with "missing opt:" might need special configuration (e.g. using orderbook if `fetchTickers` is missing) - but should in theory work (although we cannot guarantee they will).
 
-* Example: see all exchanges supported by the ccxt library (including 'bad' ones, i.e. those that are known to not work with Freqtrade):
+Example: see all exchanges supported by the ccxt library (including 'bad' ones, i.e. those that are known to not work with Freqtrade)
+
 ```
 $ freqtrade list-exchanges -a
 All exchanges supported by the ccxt library:
-Exchange name       Valid    reason
+Exchange name       Valid    Supported    Markets                 Reason
-------------------  -------  ---------------------------------------------------------------------------------------
+------------------  -------  -----------  ----------------------  ---------------------------------------------------------------------------------
-aax                 True
+binance             True     Official     spot, isolated futures
-aofex               False    missing: fetchOrder
+bitflyer            False                 spot                    missing: fetchOrder. missing opt: fetchTickers.
-ascendex            True     missing opt: fetchMyTrades
+bitmart             True     Official     spot
-bequant             True
+bybit               True                  spot, isolated futures
-bibox               True
+gate                True     Official     spot, isolated futures
-bigone              True
+htx                 True     Official     spot
-binance             True
+kraken              True     Official     spot
-binanceus           True
+okx                 True     Official     spot, isolated futures
-bit2c               False    missing: fetchOrder, fetchOHLCV
-bitbank             True     missing opt: fetchTickers
-bitbay              False    missing: fetchOrder
-bitcoincom          True
-bitfinex            True
-bitfinex2           False    missing: fetchOrder
-bitflyer            False    missing: fetchOrder, fetchOHLCV
-bitforex            True     missing opt: fetchMyTrades, fetchTickers
-bitget              True
-bithumb             True     missing opt: fetchMyTrades
-bitkk               True     missing opt: fetchMyTrades
-bitmart             True
-bitmax              True     missing opt: fetchMyTrades
-bitmex              False    Various reasons.
-bitpanda            True
-bitso               False    missing: fetchOHLCV
-bitstamp            True     missing opt: fetchTickers
-bitstamp1           False    missing: fetchOrder, fetchOHLCV
-bittrex             True
-bitvavo             True
-bitz                True     missing opt: fetchMyTrades
-bl3p                False    missing: fetchOrder, fetchOHLCV
-bleutrade           False    missing: fetchOrder
-braziliex           False    missing: fetchOHLCV
-btcalpha            True     missing opt: fetchTicker, fetchTickers
-btcbox              False    missing: fetchOHLCV
-btcmarkets          True     missing opt: fetchTickers
-btctradeua          False    missing: fetchOrder, fetchOHLCV
-btcturk             False    missing: fetchOrder
-buda                True     missing opt: fetchMyTrades, fetchTickers
-bw                  True     missing opt: fetchMyTrades, fetchL2OrderBook
-bybit               True
-bytetrade           True
-cdax                True
-cex                 True     missing opt: fetchMyTrades
-chilebit            False    missing: fetchOrder, fetchOHLCV
-coinbase            False    missing: fetchOrder, cancelOrder, createOrder, fetchOHLCV
-coinbaseprime       True     missing opt: fetchTickers
-coinbasepro         True     missing opt: fetchTickers
-coincheck           False    missing: fetchOrder, fetchOHLCV
-coinegg             False    missing: fetchOHLCV
-coinex              True
-coinfalcon          False    missing: fetchOHLCV
-coinfloor           False    missing: fetchOrder, fetchOHLCV
-coingi              False    missing: fetchOrder, fetchOHLCV
-coinmarketcap       False    missing: fetchOrder, cancelOrder, createOrder, fetchBalance, fetchOHLCV
-coinmate            False    missing: fetchOHLCV
-coinone             False    missing: fetchOHLCV
-coinspot            False    missing: fetchOrder, cancelOrder, fetchOHLCV
-crex24              True
-currencycom         False    missing: fetchOrder
-delta               False    missing: fetchOrder
-deribit             True
-digifinex           True
-equos               True     missing opt: fetchTicker, fetchTickers
-eterbase            True
-exmo                False    missing: fetchOrder
-exx                 False    missing: fetchOHLCV
-fcoin               True     missing opt: fetchMyTrades, fetchTickers
-fcoinjp             True     missing opt: fetchMyTrades, fetchTickers
-flowbtc             False    missing: fetchOrder, fetchOHLCV
-foxbit              False    missing: fetchOrder, fetchOHLCV
-gateio              True
-gemini              True
-gopax               True
-hbtc                True
-hitbtc              True
-hollaex             False    missing: fetchOrder
-huobijp             True
-huobipro            True
-idex                True
-independentreserve  False    missing: fetchOHLCV
-indodax             False    missing: fetchOHLCV
-itbit               False    missing: fetchOHLCV
-kraken              True
-kucoin              True
-kuna                False    missing: fetchOHLCV
-lakebtc             False    missing: fetchOrder, fetchOHLCV
-latoken             False    missing: fetchOrder, fetchOHLCV
-lbank               True     missing opt: fetchMyTrades
-liquid              False    missing: fetchOHLCV
-luno                False    missing: fetchOHLCV
-lykke               False    missing: fetchOHLCV
-mercado             True     missing opt: fetchTickers
-mixcoins            False    missing: fetchOrder, fetchOHLCV
-ndax                True     missing opt: fetchTickers
-novadax             True
-oceanex             False    missing: fetchOHLCV
-okcoin              True
-okex                True
-paymium             False    missing: fetchOrder, fetchOHLCV
-phemex              False    Does not provide history.
-poloniex            False    missing: fetchOrder
-probit              True
-qtrade              True
-rightbtc            False    missing: fetchOrder
-ripio               False    missing: fetchOHLCV
-southxchange        False    missing: fetchOrder, fetchOHLCV
-stex                True
-surbitcoin          False    missing: fetchOrder, fetchOHLCV
-therock             False    missing: fetchOHLCV
-tidebit             False    missing: fetchOrder
-tidex               False    missing: fetchOHLCV
-timex               True
-upbit               True     missing opt: fetchMyTrades
-vbtc                False    missing: fetchOrder, fetchOHLCV
-vcc                 True
-wavesexchange       False    missing: fetchOrder
-whitebit            False    missing: fetchOrder, cancelOrder, createOrder, fetchBalance
-xbtce               False    missing: fetchOrder, fetchOHLCV
-xena                False    missing: fetchOrder
-yobit               False    missing: fetchOHLCV
-zaif                False    missing: fetchOrder, fetchOHLCV
-zb                  True     missing opt: fetchMyTrades
 ```
 
+!!! info ""
+    Reduced output - supported and available exchanges may change over time.
+
 ## List Timeframes
 
 Use the `list-timeframes` subcommand to see the list of timeframes available for the exchange.
@@ -992,11 +879,7 @@ options:
   -h, --help            show this help message and exit
   --strategy-list STRATEGY_LIST [STRATEGY_LIST ...]
                         Provide a space-separated list of strategies to
-                        backtest. Please note that timeframe needs to be set
+                        be converted.
-                        either in config or via command line. When using this
-                        together with `--export trades`, the strategy-name is
-                        injected into the filename (so `backtest-data.json`
-                        becomes `backtest-data-SampleStrategy.json`
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).

docs/webhook-config.md

@@ -65,7 +65,7 @@ You can set the POST body format to Form-Encoded (default), JSON-Encoded, or raw
 
 The result would be a POST request with e.g. `{"text":"Status: running"}` body and `Content-Type: application/json` header which results `Status: running` message in the Mattermost channel.
 
-When using the Form-Encoded or JSON-Encoded configuration you can configure any number of payload values, and both the key and value will be ouput in the POST request. However, when using the raw data format you can only configure one value and it **must** be named `"data"`. In this instance the data key will not be output in the POST request, only the value. For example:
+When using the Form-Encoded or JSON-Encoded configuration you can configure any number of payload values, and both the key and value will be output in the POST request. However, when using the raw data format you can only configure one value and it **must** be named `"data"`. In this instance the data key will not be output in the POST request, only the value. For example:
 
 ```json
   "webhook": {
@@ -134,6 +134,7 @@ Possible parameters are:
 * `stake_amount`
 * `stake_currency`
 * `base_currency`
+* `quote_currency`
 * `fiat_currency`
 * `order_type`
 * `current_rate`
@@ -155,6 +156,7 @@ Possible parameters are:
 * `stake_amount`
 * `stake_currency`
 * `base_currency`
+* `quote_currency`
 * `fiat_currency`
 * `order_type`
 * `current_rate`
@@ -176,6 +178,7 @@ Possible parameters are:
 * `stake_amount`
 * `stake_currency`
 * `base_currency`
+* `quote_currency`
 * `fiat_currency`
 * `order_type`
 * `current_rate`
@@ -199,6 +202,7 @@ Possible parameters are:
 * `profit_ratio`
 * `stake_currency`
 * `base_currency`
+* `quote_currency`
 * `fiat_currency`
 * `exit_reason`
 * `order_type`
@@ -224,6 +228,7 @@ Possible parameters are:
 * `profit_ratio`
 * `stake_currency`
 * `base_currency`
+* `quote_currency`
 * `fiat_currency`
 * `exit_reason`
 * `order_type`
@@ -249,6 +254,7 @@ Possible parameters are:
 * `profit_ratio`
 * `stake_currency`
 * `base_currency`
+* `quote_currency`
 * `fiat_currency`
 * `exit_reason`
 * `order_type`

docs/windows_installation.md

@@ -22,7 +22,7 @@ git clone https://github.com/freqtrade/freqtrade.git
 
 ### 2. Install ta-lib
 
-Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows).
+Install ta-lib according to the [ta-lib documentation](https://github.com/TA-Lib/ta-lib-python#windows).
 
 As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), Freqtrade provides these dependencies (in the binary wheel format) for the latest 3 Python versions (3.9, 3.10 and 3.11) and for 64bit Windows.
 These Wheels are also used by CI running on windows, and are therefore tested together with freqtrade.

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ Freqtrade bot """
-__version__ = '2023.11'
+__version__ = '2024.3'
 
 if 'dev' in __version__:
     from pathlib import Path

freqtrade/commands/__init__.py

@@ -8,7 +8,7 @@ Note: Be careful with file-scoped imports in these subfiles.
 """
 from freqtrade.commands.analyze_commands import start_analysis_entries_exits
 from freqtrade.commands.arguments import Arguments
-from freqtrade.commands.build_config_commands import start_new_config
+from freqtrade.commands.build_config_commands import start_new_config, start_show_config
 from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades,
                                               start_download_data, start_list_data)
 from freqtrade.commands.db_commands import start_convert_db

freqtrade/commands/analyze_commands.py

@@ -4,7 +4,7 @@ from typing import Any, Dict
 
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.enums import RunMode
-from freqtrade.exceptions import OperationalException
+from freqtrade.exceptions import ConfigurationError, OperationalException
 
 
 logger = logging.getLogger(__name__)
@@ -34,9 +34,9 @@ def setup_analyze_configuration(args: Dict[str, Any], method: RunMode) -> Dict[s
                     btfile = Path(config['exportfilename'])
                     signals_file = f"{btfile.parent}/{btfile.stem}_signals.pkl"
                 else:
-                    raise OperationalException(f"{config['exportfilename']} does not exist.")
+                    raise ConfigurationError(f"{config['exportfilename']} does not exist.")
         else:
-            raise OperationalException('exportfilename not in config.')
+            raise ConfigurationError('exportfilename not in config.')
 
         if (not Path(signals_file).exists()):
             raise OperationalException(

freqtrade/commands/arguments.py

@@ -62,6 +62,7 @@ ARGS_TEST_PAIRLIST = ["user_data_dir", "verbosity", "config", "quote_currencies"
 ARGS_CREATE_USERDIR = ["user_data_dir", "reset"]
 
 ARGS_BUILD_CONFIG = ["config"]
+ARGS_SHOW_CONFIG = ["user_data_dir", "config", "show_sensitive"]
 
 ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"]
 
@@ -69,7 +70,8 @@ ARGS_CONVERT_DATA_TRADES = ["pairs", "format_from_trades", "format_to", "erase",
 ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase", "exchange"]
 ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes", "trading_mode", "candle_types"]
 
-ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"]
+ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades",
+                       "trading_mode"]
 
 ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode", "show_timerange"]
 
@@ -208,9 +210,9 @@ class Arguments:
                                         start_list_strategies, start_list_timeframes,
                                         start_lookahead_analysis, start_new_config,
                                         start_new_strategy, start_plot_dataframe, start_plot_profit,
-                                        start_recursive_analysis, start_show_trades,
+                                        start_recursive_analysis, start_show_config,
-                                        start_strategy_update, start_test_pairlist, start_trading,
+                                        start_show_trades, start_strategy_update,
-                                        start_webserver)
+                                        start_test_pairlist, start_trading, start_webserver)
 
         subparsers = self.parser.add_subparsers(dest='command',
                                                 # Use custom message when no subhandler is added
@@ -219,27 +221,43 @@ class Arguments:
                                                 )
 
         # Add trade subcommand
-        trade_cmd = subparsers.add_parser('trade', help='Trade module.',
+        trade_cmd = subparsers.add_parser(
-                                          parents=[_common_parser, _strategy_parser])
+            'trade',
+            help='Trade module.',
+            parents=[_common_parser, _strategy_parser]
+        )
         trade_cmd.set_defaults(func=start_trading)
         self._build_args(optionlist=ARGS_TRADE, parser=trade_cmd)
 
         # add create-userdir subcommand
-        create_userdir_cmd = subparsers.add_parser('create-userdir',
+        create_userdir_cmd = subparsers.add_parser(
-                                                   help="Create user-data directory.",
+            'create-userdir',
-                                                   )
+            help="Create user-data directory.",
+        )
         create_userdir_cmd.set_defaults(func=start_create_userdir)
         self._build_args(optionlist=ARGS_CREATE_USERDIR, parser=create_userdir_cmd)
 
         # add new-config subcommand
-        build_config_cmd = subparsers.add_parser('new-config',
+        build_config_cmd = subparsers.add_parser(
-                                                 help="Create new config")
+            'new-config',
+            help="Create new config",
+        )
         build_config_cmd.set_defaults(func=start_new_config)
         self._build_args(optionlist=ARGS_BUILD_CONFIG, parser=build_config_cmd)
 
+        # add show-config subcommand
+        show_config_cmd = subparsers.add_parser(
+            'show-config',
+            help="Show resolved config",
+        )
+        show_config_cmd.set_defaults(func=start_show_config)
+        self._build_args(optionlist=ARGS_SHOW_CONFIG, parser=show_config_cmd)
+
         # add new-strategy subcommand
-        build_strategy_cmd = subparsers.add_parser('new-strategy',
+        build_strategy_cmd = subparsers.add_parser(
-                                                   help="Create new strategy")
+            'new-strategy',
+            help="Create new strategy",
+        )
         build_strategy_cmd.set_defaults(func=start_new_strategy)
         self._build_args(optionlist=ARGS_BUILD_STRATEGY, parser=build_strategy_cmd)
 
@@ -289,8 +307,11 @@ class Arguments:
         self._build_args(optionlist=ARGS_LIST_DATA, parser=list_data_cmd)
 
         # Add backtesting subcommand
-        backtesting_cmd = subparsers.add_parser('backtesting', help='Backtesting module.',
+        backtesting_cmd = subparsers.add_parser(
-                                                parents=[_common_parser, _strategy_parser])
+            'backtesting',
+            help='Backtesting module.',
+            parents=[_common_parser, _strategy_parser]
+        )
         backtesting_cmd.set_defaults(func=start_backtesting)
         self._build_args(optionlist=ARGS_BACKTEST, parser=backtesting_cmd)
 
@@ -304,22 +325,29 @@ class Arguments:
         self._build_args(optionlist=ARGS_BACKTEST_SHOW, parser=backtesting_show_cmd)
 
         # Add backtesting analysis subcommand
-        analysis_cmd = subparsers.add_parser('backtesting-analysis',
+        analysis_cmd = subparsers.add_parser(
-                                             help='Backtest Analysis module.',
+            'backtesting-analysis',
-                                             parents=[_common_parser])
+            help='Backtest Analysis module.',
+            parents=[_common_parser]
+        )
         analysis_cmd.set_defaults(func=start_analysis_entries_exits)
         self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd)
 
         # Add edge subcommand
-        edge_cmd = subparsers.add_parser('edge', help='Edge module.',
+        edge_cmd = subparsers.add_parser(
-                                         parents=[_common_parser, _strategy_parser])
+            'edge',
+            help='Edge module.',
+            parents=[_common_parser, _strategy_parser]
+        )
         edge_cmd.set_defaults(func=start_edge)
         self._build_args(optionlist=ARGS_EDGE, parser=edge_cmd)
 
         # Add hyperopt subcommand
-        hyperopt_cmd = subparsers.add_parser('hyperopt', help='Hyperopt module.',
+        hyperopt_cmd = subparsers.add_parser(
-                                             parents=[_common_parser, _strategy_parser],
+            'hyperopt',
-                                             )
+            help='Hyperopt module.',
+            parents=[_common_parser, _strategy_parser],
+        )
         hyperopt_cmd.set_defaults(func=start_hyperopt)
         self._build_args(optionlist=ARGS_HYPEROPT, parser=hyperopt_cmd)
 
@@ -447,16 +475,20 @@ class Arguments:
         self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd)
 
         # Add webserver subcommand
-        webserver_cmd = subparsers.add_parser('webserver', help='Webserver module.',
+        webserver_cmd = subparsers.add_parser(
-                                              parents=[_common_parser])
+            'webserver',
+            help='Webserver module.',
+            parents=[_common_parser]
+        )
         webserver_cmd.set_defaults(func=start_webserver)
         self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd)
 
         # Add strategy_updater subcommand
-        strategy_updater_cmd = subparsers.add_parser('strategy-updater',
+        strategy_updater_cmd = subparsers.add_parser(
-                                                     help='updates outdated strategy'
+            'strategy-updater',
-                                                          'files to the current version',
+            help='updates outdated strategy files to the current version',
-                                                     parents=[_common_parser])
+            parents=[_common_parser]
+        )
         strategy_updater_cmd.set_defaults(func=start_strategy_update)
         self._build_args(optionlist=ARGS_STRATEGY_UPDATER, parser=strategy_updater_cmd)
 
@@ -464,8 +496,8 @@ class Arguments:
         lookahead_analayis_cmd = subparsers.add_parser(
             'lookahead-analysis',
             help="Check for potential look ahead bias.",
-            parents=[_common_parser, _strategy_parser])
+            parents=[_common_parser, _strategy_parser]
-
+        )
         lookahead_analayis_cmd.set_defaults(func=start_lookahead_analysis)
 
         self._build_args(optionlist=ARGS_LOOKAHEAD_ANALYSIS,
@@ -475,8 +507,8 @@ class Arguments:
         recursive_analayis_cmd = subparsers.add_parser(
             'recursive-analysis',
             help="Check for potential recursive formula issue.",
-            parents=[_common_parser, _strategy_parser])
+            parents=[_common_parser, _strategy_parser]
-
+        )
         recursive_analayis_cmd.set_defaults(func=start_recursive_analysis)
 
         self._build_args(optionlist=ARGS_RECURSIVE_ANALYSIS,

freqtrade/commands/build_config_commands.py

@@ -5,9 +5,12 @@ from typing import Any, Dict, List
 
 from questionary import Separator, prompt
 
+from freqtrade.configuration import sanitize_config
+from freqtrade.configuration.config_setup import setup_utils_configuration
 from freqtrade.configuration.detect_environment import running_in_docker
 from freqtrade.configuration.directory_operations import chown_user_directory
 from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
+from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS, available_exchanges
 from freqtrade.util import render_template
@@ -109,7 +112,7 @@ def ask_user_config() -> Dict[str, Any]:
                 "binance",
                 "binanceus",
                 "gate",
-                "huobi",
+                "htx",
                 "kraken",
                 "kucoin",
                 "okx",
@@ -264,3 +267,19 @@ def start_new_config(args: Dict[str, Any]) -> None:
                 "Please delete it or use a different configuration file name.")
     selections = ask_user_config()
     deploy_new_config(config_path, selections)
+
+
+def start_show_config(args: Dict[str, Any]) -> None:
+
+    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE, set_dry=False)
+
+    # TODO: Sanitize from sensitive info before printing
+
+    print("Your combined configuration is:")
+    config_sanitized = sanitize_config(
+        config['original_config'],
+        show_sensitive=args.get('show_sensitive', False)
+    )
+
+    from rich import print_json
+    print_json(data=config_sanitized)

freqtrade/commands/cli_options.py

@@ -716,4 +716,10 @@ AVAILABLE_CLI_OPTIONS = {
         help='Specify startup candles to be checked (`199`, `499`, `999`, `1999`).',
         nargs='+',
     ),
+    "show_sensitive": Arg(
+        '--show-sensitive',
+        help='Show secrets in the output.',
+        action='store_true',
+        default=False,
+    ),
 }

freqtrade/commands/data_commands.py

@@ -8,11 +8,12 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, DL_DATA_TIMEFRAMES, Confi
 from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_format,
                                       convert_trades_to_ohlcv)
 from freqtrade.data.history import download_data_main
-from freqtrade.enums import RunMode, TradingMode
+from freqtrade.enums import CandleType, RunMode, TradingMode
-from freqtrade.exceptions import OperationalException
+from freqtrade.exceptions import ConfigurationError
 from freqtrade.exchange import timeframe_to_minutes
+from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
-from freqtrade.util.binance_mig import migrate_binance_futures_data
+from freqtrade.util.migrations import migrate_data
 
 
 logger = logging.getLogger(__name__)
@@ -20,11 +21,11 @@ logger = logging.getLogger(__name__)
 
 def _check_data_config_download_sanity(config: Config) -> None:
     if 'days' in config and 'timerange' in config:
-        raise OperationalException("--days and --timerange are mutually exclusive. "
+        raise ConfigurationError("--days and --timerange are mutually exclusive. "
-                                   "You can only specify one or the other.")
+                                 "You can only specify one or the other.")
 
     if 'pairs' not in config:
-        raise OperationalException(
+        raise ConfigurationError(
             "Downloading data requires a list of pairs. "
             "Please check the documentation on how to configure this.")
 
@@ -62,13 +63,21 @@ def start_convert_trades(args: Dict[str, Any]) -> None:
 
     for timeframe in config['timeframes']:
         exchange.validate_timeframes(timeframe)
+    available_pairs = [
+        p for p in exchange.get_markets(
+            tradable_only=True, active_only=not config.get('include_inactive')
+            ).keys()
+    ]
+
+    expanded_pairs = dynamic_expand_pairlist(config, available_pairs)
 
     # Convert downloaded trade data to different timeframes
     convert_trades_to_ohlcv(
-        pairs=config.get('pairs', []), timeframes=config['timeframes'],
+        pairs=expanded_pairs, timeframes=config['timeframes'],
         datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
         data_format_ohlcv=config['dataformat_ohlcv'],
         data_format_trades=config['dataformat_trades'],
+        candle_type=config.get('candle_type_def', CandleType.SPOT)
     )
 
 
@@ -78,7 +87,7 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
     """
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
     if ohlcv:
-        migrate_binance_futures_data(config)
+        migrate_data(config)
         convert_ohlcv_format(config,
                              convert_from=args['format_from'],
                              convert_to=args['format_to'],
@@ -98,7 +107,7 @@ def start_list_data(args: Dict[str, Any]) -> None:
 
     from tabulate import tabulate
 
-    from freqtrade.data.history.idatahandler import get_datahandler
+    from freqtrade.data.history import get_datahandler
     dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv'])
 
     paircombs = dhc.ohlcv_get_available_data(
@@ -134,10 +143,10 @@ def start_list_data(args: Dict[str, Any]) -> None:
         print(tabulate([
             (pair, timeframe, candle_type,
                 start.strftime(DATETIME_PRINT_FORMAT),
-                end.strftime(DATETIME_PRINT_FORMAT))
+                end.strftime(DATETIME_PRINT_FORMAT), length)
-            for pair, timeframe, candle_type, start, end in sorted(
+            for pair, timeframe, candle_type, start, end, length in sorted(
                 paircombs1,
                 key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2]))
             ],
-            headers=("Pair", "Timeframe", "Type", 'From', 'To'),
+            headers=("Pair", "Timeframe", "Type", 'From', 'To', 'Candles'),
             tablefmt='psql', stralign='right'))

freqtrade/commands/deploy_commands.py

@@ -9,7 +9,7 @@ from freqtrade.configuration import setup_utils_configuration
 from freqtrade.configuration.directory_operations import copy_sample_files, create_userdata_dir
 from freqtrade.constants import USERPATH_STRATEGIES
 from freqtrade.enums import RunMode
-from freqtrade.exceptions import OperationalException
+from freqtrade.exceptions import ConfigurationError, OperationalException
 from freqtrade.util import render_template, render_template_with_fallback
 
 
@@ -89,7 +89,7 @@ def start_new_strategy(args: Dict[str, Any]) -> None:
         deploy_new_strategy(args['strategy'], new_path, args['template'])
 
     else:
-        raise OperationalException("`new-strategy` requires --strategy to be set.")
+        raise ConfigurationError("`new-strategy` requires --strategy to be set.")
 
 
 def clean_ui_subdir(directory: Path):

freqtrade/commands/list_commands.py

@@ -10,7 +10,7 @@ from tabulate import tabulate
 
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.enums import RunMode
-from freqtrade.exceptions import OperationalException
+from freqtrade.exceptions import ConfigurationError, OperationalException
 from freqtrade.exchange import list_available_exchanges, market_is_active
 from freqtrade.misc import parse_db_uri_for_logging, plural
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -246,7 +246,7 @@ def start_show_trades(args: Dict[str, Any]) -> None:
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
     if 'db_url' not in config:
-        raise OperationalException("--db-url is required for this command.")
+        raise ConfigurationError("--db-url is required for this command.")
 
     logger.info(f'Using DB: "{parse_db_uri_for_logging(config["db_url"])}"')
     init_db(config['db_url'])

freqtrade/commands/optimize_commands.py

@@ -4,8 +4,8 @@ from typing import Any, Dict
 from freqtrade import constants
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.enums import RunMode
-from freqtrade.exceptions import OperationalException
+from freqtrade.exceptions import ConfigurationError, OperationalException
-from freqtrade.misc import round_coin_value
+from freqtrade.util import fmt_coin
 
 
 logger = logging.getLogger(__name__)
@@ -29,9 +29,9 @@ def setup_optimize_configuration(args: Dict[str, Any], method: RunMode) -> Dict[
         # tradable_balance_ratio
         if (config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT
                 and config['stake_amount'] > wallet_size):
-            wallet = round_coin_value(wallet_size, config['stake_currency'])
+            wallet = fmt_coin(wallet_size, config['stake_currency'])
-            stake = round_coin_value(config['stake_amount'], config['stake_currency'])
+            stake = fmt_coin(config['stake_amount'], config['stake_currency'])
-            raise OperationalException(
+            raise ConfigurationError(
                 f"Starting balance ({wallet}) is smaller than stake_amount {stake}. "
                 f"Wallet is calculated as `dry_run_wallet * tradable_balance_ratio`."
                 )

freqtrade/commands/pairlist_commands.py

@@ -15,6 +15,7 @@ def start_test_pairlist(args: Dict[str, Any]) -> None:
     """
     Test Pairlist configuration
     """
+    from freqtrade.persistence import FtNoDBContext
     from freqtrade.plugins.pairlistmanager import PairListManager
     config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
 
@@ -24,11 +25,12 @@ def start_test_pairlist(args: Dict[str, Any]) -> None:
     if not quote_currencies:
         quote_currencies = [config.get('stake_currency')]
     results = {}
-    for curr in quote_currencies:
+    with FtNoDBContext():
-        config['stake_currency'] = curr
+        for curr in quote_currencies:
-        pairlists = PairListManager(exchange, config)
+            config['stake_currency'] = curr
-        pairlists.refresh_pairlist()
+            pairlists = PairListManager(exchange, config)
-        results[curr] = pairlists.whitelist
+            pairlists.refresh_pairlist()
+            results[curr] = pairlists.whitelist
 
     for curr, pairlist in results.items():
         if not args.get('print_one_column', False) and not args.get('list_pairs_print_json', False):

freqtrade/commands/plot_commands.py

@@ -2,12 +2,12 @@ from typing import Any, Dict
 
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.enums import RunMode
-from freqtrade.exceptions import OperationalException
+from freqtrade.exceptions import ConfigurationError
 
 
 def validate_plot_args(args: Dict[str, Any]) -> None:
     if not args.get('datadir') and not args.get('config'):
-        raise OperationalException(
+        raise ConfigurationError(
             "You need to specify either `--datadir` or `--config` "
             "for plot-profit and plot-dataframe.")
 

freqtrade/commands/trade_commands.py

@@ -23,11 +23,6 @@ def start_trading(args: Dict[str, Any]) -> int:
         signal.signal(signal.SIGTERM, term_handler)
         worker = Worker(args)
         worker.run()
-    except Exception as e:
-        logger.error(str(e))
-        logger.exception("Fatal exception!")
-    except (KeyboardInterrupt):
-        logger.info('SIGINT received, aborting ...')
     finally:
         if worker:
             logger.info("worker found ... calling exit")

freqtrade/configuration/__init__.py

@@ -1,5 +1,6 @@
 # flake8: noqa: F401
 
+from freqtrade.configuration.config_secrets import sanitize_config
 from freqtrade.configuration.config_setup import setup_utils_configuration
 from freqtrade.configuration.config_validation import validate_config_consistency
 from freqtrade.configuration.configuration import Configuration

freqtrade/configuration/config_secrets.py

@@ -0,0 +1,36 @@
+from copy import deepcopy
+
+from freqtrade.constants import Config
+
+
+def sanitize_config(config: Config, *, show_sensitive: bool = False) -> Config:
+    """
+    Remove sensitive information from the config.
+    :param config: Configuration
+    :param show_sensitive: Show sensitive information
+    :return: Configuration
+    """
+    if show_sensitive:
+        return config
+    keys_to_remove = [
+        "exchange.key",
+        "exchange.secret",
+        "exchange.password",
+        "exchange.uid",
+        "telegram.token",
+        "telegram.chat_id",
+        "discord.webhook_url",
+        "api_server.password",
+    ]
+    config = deepcopy(config)
+    for key in keys_to_remove:
+        if '.' in key:
+            nested_keys = key.split('.')
+            nested_config = config
+            for nested_key in nested_keys[:-1]:
+                nested_config = nested_config.get(nested_key, {})
+            nested_config[nested_keys[-1]] = 'REDACTED'
+        else:
+            config[key] = 'REDACTED'
+
+    return config

freqtrade/configuration/config_setup.py

@@ -10,7 +10,8 @@ from .configuration import Configuration
 logger = logging.getLogger(__name__)
 
 
-def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]:
+def setup_utils_configuration(
+        args: Dict[str, Any], method: RunMode, *, set_dry: bool = True) -> Dict[str, Any]:
     """
     Prepare the configuration for utils subcommands
     :param args: Cli args from Arguments()
@@ -21,7 +22,8 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str
     config = configuration.get_config()
 
     # Ensure these modes are using Dry-run
-    config['dry_run'] = True
+    if set_dry:
+        config['dry_run'] = True
     validate_config_consistency(config, preliminary=True)
 
     return config

freqtrade/configuration/config_validation.py

@@ -9,7 +9,7 @@ from jsonschema.exceptions import ValidationError, best_match
 from freqtrade import constants
 from freqtrade.configuration.deprecated_settings import process_deprecated_setting
 from freqtrade.enums import RunMode, TradingMode
-from freqtrade.exceptions import OperationalException
+from freqtrade.exceptions import ConfigurationError
 
 
 logger = logging.getLogger(__name__)
@@ -67,13 +67,13 @@ def validate_config_schema(conf: Dict[str, Any], preliminary: bool = False) -> D
         )
 
 
-def validate_config_consistency(conf: Dict[str, Any], preliminary: bool = False) -> None:
+def validate_config_consistency(conf: Dict[str, Any], *, preliminary: bool = False) -> None:
     """
     Validate the configuration consistency.
     Should be ran after loading both configuration and strategy,
     since strategies can set certain configuration settings too.
     :param conf: Config in JSON format
-    :return: Returns None if everything is ok, otherwise throw an OperationalException
+    :return: Returns None if everything is ok, otherwise throw an ConfigurationError
     """
 
     # validating trailing stoploss
@@ -86,7 +86,7 @@ def validate_config_consistency(conf: Dict[str, Any], preliminary: bool = False)
     _validate_ask_orderbook(conf)
     _validate_freqai_hyperopt(conf)
     _validate_freqai_backtest(conf)
-    _validate_freqai_include_timeframes(conf)
+    _validate_freqai_include_timeframes(conf, preliminary=preliminary)
     _validate_consumers(conf)
     validate_migrated_strategy_settings(conf)
 
@@ -98,12 +98,12 @@ def validate_config_consistency(conf: Dict[str, Any], preliminary: bool = False)
 def _validate_unlimited_amount(conf: Dict[str, Any]) -> None:
     """
     If edge is disabled, either max_open_trades or stake_amount need to be set.
-    :raise: OperationalException if config validation failed
+    :raise: ConfigurationError if config validation failed
     """
     if (not conf.get('edge', {}).get('enabled')
         and conf.get('max_open_trades') == float('inf')
             and conf.get('stake_amount') == constants.UNLIMITED_STAKE_AMOUNT):
-        raise OperationalException("`max_open_trades` and `stake_amount` cannot both be unlimited.")
+        raise ConfigurationError("`max_open_trades` and `stake_amount` cannot both be unlimited.")
 
 
 def _validate_price_config(conf: Dict[str, Any]) -> None:
@@ -113,18 +113,18 @@ def _validate_price_config(conf: Dict[str, Any]) -> None:
     # TODO: The below could be an enforced setting when using market orders
     if (conf.get('order_types', {}).get('entry') == 'market'
             and conf.get('entry_pricing', {}).get('price_side') not in ('ask', 'other')):
-        raise OperationalException(
+        raise ConfigurationError(
             'Market entry orders require entry_pricing.price_side = "other".')
 
     if (conf.get('order_types', {}).get('exit') == 'market'
             and conf.get('exit_pricing', {}).get('price_side') not in ('bid', 'other')):
-        raise OperationalException('Market exit orders require exit_pricing.price_side = "other".')
+        raise ConfigurationError('Market exit orders require exit_pricing.price_side = "other".')
 
 
 def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None:
 
     if conf.get('stoploss') == 0.0:
-        raise OperationalException(
+        raise ConfigurationError(
             'The config stoploss needs to be different from 0 to avoid problems with sell orders.'
         )
     # Skip if trailing stoploss is not activated
@@ -137,17 +137,17 @@ def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None:
 
     if tsl_only_offset:
         if tsl_positive == 0.0:
-            raise OperationalException(
+            raise ConfigurationError(
                 'The config trailing_only_offset_is_reached needs '
                 'trailing_stop_positive_offset to be more than 0 in your config.')
     if tsl_positive > 0 and 0 < tsl_offset <= tsl_positive:
-        raise OperationalException(
+        raise ConfigurationError(
             'The config trailing_stop_positive_offset needs '
             'to be greater than trailing_stop_positive in your config.')
 
     # Fetch again without default
     if 'trailing_stop_positive' in conf and float(conf['trailing_stop_positive']) == 0.0:
-        raise OperationalException(
+        raise ConfigurationError(
             'The config trailing_stop_positive needs to be different from 0 '
             'to avoid problems with sell orders.'
         )
@@ -162,7 +162,7 @@ def _validate_edge(conf: Dict[str, Any]) -> None:
         return
 
     if not conf.get('use_exit_signal', True):
-        raise OperationalException(
+        raise ConfigurationError(
             "Edge requires `use_exit_signal` to be True, otherwise no sells will happen."
         )
 
@@ -178,7 +178,7 @@ def _validate_whitelist(conf: Dict[str, Any]) -> None:
     for pl in conf.get('pairlists', [{'method': 'StaticPairList'}]):
         if (isinstance(pl, dict) and pl.get('method') == 'StaticPairList'
                 and not conf.get('exchange', {}).get('pair_whitelist')):
-            raise OperationalException("StaticPairList requires pair_whitelist to be set.")
+            raise ConfigurationError("StaticPairList requires pair_whitelist to be set.")
 
 
 def _validate_protections(conf: Dict[str, Any]) -> None:
@@ -188,13 +188,13 @@ def _validate_protections(conf: Dict[str, Any]) -> None:
 
     for prot in conf.get('protections', []):
         if ('stop_duration' in prot and 'stop_duration_candles' in prot):
-            raise OperationalException(
+            raise ConfigurationError(
                 "Protections must specify either `stop_duration` or `stop_duration_candles`.\n"
                 f"Please fix the protection {prot.get('method')}"
             )
 
         if ('lookback_period' in prot and 'lookback_period_candles' in prot):
-            raise OperationalException(
+            raise ConfigurationError(
                 "Protections must specify either `lookback_period` or `lookback_period_candles`.\n"
                 f"Please fix the protection {prot.get('method')}"
             )
@@ -206,7 +206,7 @@ def _validate_ask_orderbook(conf: Dict[str, Any]) -> None:
     ob_max = ask_strategy.get('order_book_max')
     if ob_min is not None and ob_max is not None and ask_strategy.get('use_order_book'):
         if ob_min != ob_max:
-            raise OperationalException(
+            raise ConfigurationError(
                 "Using order_book_max != order_book_min in exit_pricing is no longer supported."
                 "Please pick one value and use `order_book_top` in the future."
             )
@@ -234,7 +234,7 @@ def _validate_time_in_force(conf: Dict[str, Any]) -> None:
     time_in_force = conf.get('order_time_in_force', {})
     if 'buy' in time_in_force or 'sell' in time_in_force:
         if conf.get('trading_mode', TradingMode.SPOT) != TradingMode.SPOT:
-            raise OperationalException(
+            raise ConfigurationError(
                 "Please migrate your time_in_force settings to use 'entry' and 'exit'.")
         else:
             logger.warning(
@@ -255,7 +255,7 @@ def _validate_order_types(conf: Dict[str, Any]) -> None:
                        'forcesell', 'emergencyexit', 'forceexit', 'forceentry']
     if any(x in order_types for x in old_order_types):
         if conf.get('trading_mode', TradingMode.SPOT) != TradingMode.SPOT:
-            raise OperationalException(
+            raise ConfigurationError(
                 "Please migrate your order_types settings to use the new wording.")
         else:
             logger.warning(
@@ -280,7 +280,7 @@ def _validate_unfilledtimeout(conf: Dict[str, Any]) -> None:
     unfilledtimeout = conf.get('unfilledtimeout', {})
     if any(x in unfilledtimeout for x in ['buy', 'sell']):
         if conf.get('trading_mode', TradingMode.SPOT) != TradingMode.SPOT:
-            raise OperationalException(
+            raise ConfigurationError(
                 "Please migrate your unfilledtimeout settings to use the new wording.")
         else:
 
@@ -300,7 +300,7 @@ def _validate_pricing_rules(conf: Dict[str, Any]) -> None:
 
     if conf.get('ask_strategy') or conf.get('bid_strategy'):
         if conf.get('trading_mode', TradingMode.SPOT) != TradingMode.SPOT:
-            raise OperationalException(
+            raise ConfigurationError(
                 "Please migrate your pricing settings to use the new wording.")
         else:
 
@@ -331,11 +331,11 @@ def _validate_freqai_hyperopt(conf: Dict[str, Any]) -> None:
     freqai_enabled = conf.get('freqai', {}).get('enabled', False)
     analyze_per_epoch = conf.get('analyze_per_epoch', False)
     if analyze_per_epoch and freqai_enabled:
-        raise OperationalException(
+        raise ConfigurationError(
             'Using analyze-per-epoch parameter is not supported with a FreqAI strategy.')
 
 
-def _validate_freqai_include_timeframes(conf: Dict[str, Any]) -> None:
+def _validate_freqai_include_timeframes(conf: Dict[str, Any], preliminary: bool) -> None:
     freqai_enabled = conf.get('freqai', {}).get('enabled', False)
     if freqai_enabled:
         main_tf = conf.get('timeframe', '5m')
@@ -350,12 +350,12 @@ def _validate_freqai_include_timeframes(conf: Dict[str, Any]) -> None:
             if tf_s < main_tf_s:
                 offending_lines.append(tf)
         if offending_lines:
-            raise OperationalException(
+            raise ConfigurationError(
                 f"Main timeframe of {main_tf} must be smaller or equal to FreqAI "
                 f"`include_timeframes`.Offending include-timeframes: {', '.join(offending_lines)}")
 
         # Ensure that the base timeframe is included in the include_timeframes list
-        if main_tf not in freqai_include_timeframes:
+        if not preliminary and main_tf not in freqai_include_timeframes:
             feature_parameters = conf.get('freqai', {}).get('feature_parameters', {})
             include_timeframes = [main_tf] + freqai_include_timeframes
             conf.get('freqai', {}).get('feature_parameters', {}) \
@@ -368,17 +368,17 @@ def _validate_freqai_backtest(conf: Dict[str, Any]) -> None:
         timerange = conf.get('timerange')
         freqai_backtest_live_models = conf.get('freqai_backtest_live_models', False)
         if freqai_backtest_live_models and freqai_enabled and timerange:
-            raise OperationalException(
+            raise ConfigurationError(
                 'Using timerange parameter is not supported with '
                 '--freqai-backtest-live-models parameter.')
 
         if freqai_backtest_live_models and not freqai_enabled:
-            raise OperationalException(
+            raise ConfigurationError(
                 'Using --freqai-backtest-live-models parameter is only '
                 'supported with a FreqAI strategy.')
 
         if freqai_enabled and not freqai_backtest_live_models and not timerange:
-            raise OperationalException(
+            raise ConfigurationError(
                 'Please pass --timerange if you intend to use FreqAI for backtesting.')
 
 
@@ -386,12 +386,12 @@ def _validate_consumers(conf: Dict[str, Any]) -> None:
     emc_conf = conf.get('external_message_consumer', {})
     if emc_conf.get('enabled', False):
         if len(emc_conf.get('producers', [])) < 1:
-            raise OperationalException("You must specify at least 1 Producer to connect to.")
+            raise ConfigurationError("You must specify at least 1 Producer to connect to.")
 
         producer_names = [p['name'] for p in emc_conf.get('producers', [])]
         duplicates = [item for item, count in Counter(producer_names).items() if count > 1]
         if duplicates:
-            raise OperationalException(
+            raise ConfigurationError(
                 f"Producer names must be unique. Duplicate: {', '.join(duplicates)}")
         if conf.get('process_only_new_candles', True):
             # Warning here or require it?

freqtrade/configuration/configuration.py

@@ -5,7 +5,7 @@ import logging
 import warnings
 from copy import deepcopy
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
 from freqtrade import constants
 from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings
@@ -68,8 +68,10 @@ class Configuration:
         config: Config = load_from_files(self.args.get("config", []))
 
         # Load environment variables
-        env_data = enironment_vars_to_dict()
+        from freqtrade.commands.arguments import NO_CONF_ALLOWED
-        config = deep_merge_dicts(env_data, config)
+        if self.args.get('command') not in NO_CONF_ALLOWED:
+            env_data = enironment_vars_to_dict()
+            config = deep_merge_dicts(env_data, config)
 
         # Normalize config
         if 'internals' not in config:
@@ -198,6 +200,12 @@ class Configuration:
             config['exportfilename'] = (config['user_data_dir']
                                         / 'backtest_results')
 
+        if self.args.get('show_sensitive'):
+            logger.warning(
+                "Sensitive information will be shown in the upcomming output. "
+                "Please make sure to never share this output without redacting "
+                "the information yourself.")
+
     def _process_optimize_options(self, config: Config) -> None:
 
         # This will override the strategy configuration
@@ -233,54 +241,37 @@ class Configuration:
             except ValueError:
                 pass
 
-        self._args_to_config(config, argname='timeframe_detail',
+        configurations = [
-                             logstring='Parameter --timeframe-detail detected, '
+            ('timeframe_detail',
-                             'using {} for intra-candle backtesting ...')
+             'Parameter --timeframe-detail detected, using {} for intra-candle backtesting ...'),
+            ('backtest_show_pair_list', 'Parameter --show-pair-list detected.'),
+            ('stake_amount',
+             'Parameter --stake-amount detected, overriding stake_amount to: {} ...'),
+            ('dry_run_wallet',
+             'Parameter --dry-run-wallet detected, overriding dry_run_wallet to: {} ...'),
+            ('fee', 'Parameter --fee detected, setting fee to: {} ...'),
+            ('timerange', 'Parameter --timerange detected: {} ...'),
+            ]
 
-        self._args_to_config(config, argname='backtest_show_pair_list',
+        self._args_to_config_loop(config, configurations)
-                             logstring='Parameter --show-pair-list detected.')
-
-        self._args_to_config(config, argname='stake_amount',
-                             logstring='Parameter --stake-amount detected, '
-                             'overriding stake_amount to: {} ...')
-        self._args_to_config(config, argname='dry_run_wallet',
-                             logstring='Parameter --dry-run-wallet detected, '
-                             'overriding dry_run_wallet to: {} ...')
-        self._args_to_config(config, argname='fee',
-                             logstring='Parameter --fee detected, '
-                             'setting fee to: {} ...')
-
-        self._args_to_config(config, argname='timerange',
-                             logstring='Parameter --timerange detected: {} ...')
 
         self._process_datadir_options(config)
 
         self._args_to_config(config, argname='strategy_list',
                              logstring='Using strategy list of {} strategies', logfun=len)
 
-        self._args_to_config(
+        configurations = [
-            config,
+            ('recursive_strategy_search',
-            argname='recursive_strategy_search',
+             'Recursively searching for a strategy in the strategies folder.'),
-            logstring='Recursively searching for a strategy in the strategies folder.',
+            ('timeframe', 'Overriding timeframe with Command line argument'),
-        )
+            ('export', 'Parameter --export detected: {} ...'),
-
+            ('backtest_breakdown', 'Parameter --breakdown detected ...'),
-        self._args_to_config(config, argname='timeframe',
+            ('backtest_cache', 'Parameter --cache={} detected ...'),
-                             logstring='Overriding timeframe with Command line argument')
+            ('disableparamexport', 'Parameter --disableparamexport detected: {} ...'),
-
+            ('freqai_backtest_live_models',
-        self._args_to_config(config, argname='export',
+             'Parameter --freqai-backtest-live-models detected ...'),
-                             logstring='Parameter --export detected: {} ...')
+        ]
-
+        self._args_to_config_loop(config, configurations)
-        self._args_to_config(config, argname='backtest_breakdown',
-                             logstring='Parameter --breakdown detected ...')
-
-        self._args_to_config(config, argname='backtest_cache',
-                             logstring='Parameter --cache={} detected ...')
-
-        self._args_to_config(config, argname='disableparamexport',
-                             logstring='Parameter --disableparamexport detected: {} ...')
-
-        self._args_to_config(config, argname='freqai_backtest_live_models',
-                             logstring='Parameter --freqai-backtest-live-models detected ...')
 
         # Edge section:
         if 'stoploss_range' in self.args and self.args["stoploss_range"]:
@@ -291,31 +282,18 @@ class Configuration:
             logger.info('Parameter --stoplosses detected: %s ...', self.args["stoploss_range"])
 
         # Hyperopt section
-        self._args_to_config(config, argname='hyperopt',
-                             logstring='Using Hyperopt class name: {}')
 
-        self._args_to_config(config, argname='hyperopt_path',
+        configurations = [
-                             logstring='Using additional Hyperopt lookup path: {}')
+            ('hyperopt', 'Using Hyperopt class name: {}'),
-
+            ('hyperopt_path', 'Using additional Hyperopt lookup path: {}'),
-        self._args_to_config(config, argname='hyperoptexportfilename',
+            ('hyperoptexportfilename', 'Using hyperopt file: {}'),
-                             logstring='Using hyperopt file: {}')
+            ('lookahead_analysis_exportfilename', 'Saving lookahead analysis results into {} ...'),
-
+            ('epochs', 'Parameter --epochs detected ... Will run Hyperopt with for {} epochs ...'),
-        self._args_to_config(config, argname='lookahead_analysis_exportfilename',
+            ('spaces', 'Parameter -s/--spaces detected: {}'),
-                             logstring='Saving lookahead analysis results into {} ...')
+            ('analyze_per_epoch', 'Parameter --analyze-per-epoch detected.'),
-
+            ('print_all', 'Parameter --print-all detected ...'),
-        self._args_to_config(config, argname='epochs',
+        ]
-                             logstring='Parameter --epochs detected ... '
+        self._args_to_config_loop(config, configurations)
-                             'Will run Hyperopt with for {} epochs ...'
-                             )
-
-        self._args_to_config(config, argname='spaces',
-                             logstring='Parameter -s/--spaces detected: {}')
-
-        self._args_to_config(config, argname='analyze_per_epoch',
-                             logstring='Parameter --analyze-per-epoch detected.')
-
-        self._args_to_config(config, argname='print_all',
-                             logstring='Parameter --print-all detected ...')
 
         if 'print_colorized' in self.args and not self.args["print_colorized"]:
             logger.info('Parameter --no-color detected ...')
@@ -323,123 +301,55 @@ class Configuration:
         else:
             config.update({'print_colorized': True})
 
-        self._args_to_config(config, argname='print_json',
+        configurations = [
-                             logstring='Parameter --print-json detected ...')
+            ('print_json', 'Parameter --print-json detected ...'),
+            ('export_csv', 'Parameter --export-csv detected: {}'),
+            ('hyperopt_jobs', 'Parameter -j/--job-workers detected: {}'),
+            ('hyperopt_random_state', 'Parameter --random-state detected: {}'),
+            ('hyperopt_min_trades', 'Parameter --min-trades detected: {}'),
+            ('hyperopt_loss', 'Using Hyperopt loss class name: {}'),
+            ('hyperopt_show_index', 'Parameter -n/--index detected: {}'),
+            ('hyperopt_list_best', 'Parameter --best detected: {}'),
+            ('hyperopt_list_profitable', 'Parameter --profitable detected: {}'),
+            ('hyperopt_list_min_trades', 'Parameter --min-trades detected: {}'),
+            ('hyperopt_list_max_trades', 'Parameter --max-trades detected: {}'),
+            ('hyperopt_list_min_avg_time', 'Parameter --min-avg-time detected: {}'),
+            ('hyperopt_list_max_avg_time', 'Parameter --max-avg-time detected: {}'),
+            ('hyperopt_list_min_avg_profit', 'Parameter --min-avg-profit detected: {}'),
+            ('hyperopt_list_max_avg_profit', 'Parameter --max-avg-profit detected: {}'),
+            ('hyperopt_list_min_total_profit', 'Parameter --min-total-profit detected: {}'),
+            ('hyperopt_list_max_total_profit', 'Parameter --max-total-profit detected: {}'),
+            ('hyperopt_list_min_objective', 'Parameter --min-objective detected: {}'),
+            ('hyperopt_list_max_objective', 'Parameter --max-objective detected: {}'),
+            ('hyperopt_list_no_details', 'Parameter --no-details detected: {}'),
+            ('hyperopt_show_no_header', 'Parameter --no-header detected: {}'),
+            ('hyperopt_ignore_missing_space', 'Paramter --ignore-missing-space detected: {}'),
+        ]
 
-        self._args_to_config(config, argname='export_csv',
+        self._args_to_config_loop(config, configurations)
-                             logstring='Parameter --export-csv detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_jobs',
-                             logstring='Parameter -j/--job-workers detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_random_state',
-                             logstring='Parameter --random-state detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_min_trades',
-                             logstring='Parameter --min-trades detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_loss',
-                             logstring='Using Hyperopt loss class name: {}')
-
-        self._args_to_config(config, argname='hyperopt_show_index',
-                             logstring='Parameter -n/--index detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_best',
-                             logstring='Parameter --best detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_profitable',
-                             logstring='Parameter --profitable detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_min_trades',
-                             logstring='Parameter --min-trades detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_max_trades',
-                             logstring='Parameter --max-trades detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_min_avg_time',
-                             logstring='Parameter --min-avg-time detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_max_avg_time',
-                             logstring='Parameter --max-avg-time detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_min_avg_profit',
-                             logstring='Parameter --min-avg-profit detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_max_avg_profit',
-                             logstring='Parameter --max-avg-profit detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_min_total_profit',
-                             logstring='Parameter --min-total-profit detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_max_total_profit',
-                             logstring='Parameter --max-total-profit detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_min_objective',
-                             logstring='Parameter --min-objective detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_max_objective',
-                             logstring='Parameter --max-objective detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_no_details',
-                             logstring='Parameter --no-details detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_show_no_header',
-                             logstring='Parameter --no-header detected: {}')
-
-        self._args_to_config(config, argname="hyperopt_ignore_missing_space",
-                             logstring="Paramter --ignore-missing-space detected: {}")
 
     def _process_plot_options(self, config: Config) -> None:
 
-        self._args_to_config(config, argname='pairs',
+        configurations = [
-                             logstring='Using pairs {}')
+            ('pairs', 'Using pairs {}'),
-
+            ('indicators1', 'Using indicators1: {}'),
-        self._args_to_config(config, argname='indicators1',
+            ('indicators2', 'Using indicators2: {}'),
-                             logstring='Using indicators1: {}')
+            ('trade_ids', 'Filtering on trade_ids: {}'),
-
+            ('plot_limit', 'Limiting plot to: {}'),
-        self._args_to_config(config, argname='indicators2',
+            ('plot_auto_open', 'Parameter --auto-open detected.'),
-                             logstring='Using indicators2: {}')
+            ('trade_source', 'Using trades from: {}'),
-
+            ('prepend_data', 'Prepend detected. Allowing data prepending.'),
-        self._args_to_config(config, argname='trade_ids',
+            ('erase', 'Erase detected. Deleting existing data.'),
-                             logstring='Filtering on trade_ids: {}')
+            ('no_trades', 'Parameter --no-trades detected.'),
-
+            ('timeframes', 'timeframes --timeframes: {}'),
-        self._args_to_config(config, argname='plot_limit',
+            ('days', 'Detected --days: {}'),
-                             logstring='Limiting plot to: {}')
+            ('include_inactive', 'Detected --include-inactive-pairs: {}'),
-
+            ('download_trades', 'Detected --dl-trades: {}'),
-        self._args_to_config(config, argname='plot_auto_open',
+            ('dataformat_ohlcv', 'Using "{}" to store OHLCV data.'),
-                             logstring='Parameter --auto-open detected.')
+            ('dataformat_trades', 'Using "{}" to store trades data.'),
-
+            ('show_timerange', 'Detected --show-timerange'),
-        self._args_to_config(config, argname='trade_source',
+        ]
-                             logstring='Using trades from: {}')
+        self._args_to_config_loop(config, configurations)
-
-        self._args_to_config(config, argname='prepend_data',
-                             logstring='Prepend detected. Allowing data prepending.')
-        self._args_to_config(config, argname='erase',
-                             logstring='Erase detected. Deleting existing data.')
-
-        self._args_to_config(config, argname='no_trades',
-                             logstring='Parameter --no-trades detected.')
-
-        self._args_to_config(config, argname='timeframes',
-                             logstring='timeframes --timeframes: {}')
-
-        self._args_to_config(config, argname='days',
-                             logstring='Detected --days: {}')
-
-        self._args_to_config(config, argname='include_inactive',
-                             logstring='Detected --include-inactive-pairs: {}')
-
-        self._args_to_config(config, argname='download_trades',
-                             logstring='Detected --dl-trades: {}')
-
-        self._args_to_config(config, argname='dataformat_ohlcv',
-                             logstring='Using "{}" to store OHLCV data.')
-
-        self._args_to_config(config, argname='dataformat_trades',
-                             logstring='Using "{}" to store trades data.')
-
-        self._args_to_config(config, argname='show_timerange',
-                             logstring='Detected --show-timerange')
 
     def _process_data_options(self, config: Config) -> None:
         self._args_to_config(config, argname='new_pairs_days',
@@ -453,45 +363,27 @@ class Configuration:
                              logstring='Detected --candle-types: {}')
 
     def _process_analyze_options(self, config: Config) -> None:
-        self._args_to_config(config, argname='analysis_groups',
+        configurations = [
-                             logstring='Analysis reason groups: {}')
+            ('analysis_groups', 'Analysis reason groups: {}'),
+            ('enter_reason_list', 'Analysis enter tag list: {}'),
+            ('exit_reason_list', 'Analysis exit tag list: {}'),
+            ('indicator_list', 'Analysis indicator list: {}'),
+            ('timerange', 'Filter trades by timerange: {}'),
+            ('analysis_rejected', 'Analyse rejected signals: {}'),
+            ('analysis_to_csv', 'Store analysis tables to CSV: {}'),
+            ('analysis_csv_path', 'Path to store analysis CSVs: {}'),
+            # Lookahead analysis results
+            ('targeted_trade_amount', 'Targeted Trade amount: {}'),
+            ('minimum_trade_amount', 'Minimum Trade amount: {}'),
+            ('lookahead_analysis_exportfilename', 'Path to store lookahead-analysis-results: {}'),
+            ('startup_candle', 'Startup candle to be used on recursive analysis: {}'),
+        ]
+        self._args_to_config_loop(config, configurations)
 
-        self._args_to_config(config, argname='enter_reason_list',
+    def _args_to_config_loop(self, config, configurations: List[Tuple[str, str]]) -> None:
-                             logstring='Analysis enter tag list: {}')
 
-        self._args_to_config(config, argname='exit_reason_list',
+        for argname, logstring in configurations:
-                             logstring='Analysis exit tag list: {}')
+            self._args_to_config(config, argname=argname, logstring=logstring)
-
-        self._args_to_config(config, argname='indicator_list',
-                             logstring='Analysis indicator list: {}')
-
-        self._args_to_config(config, argname='timerange',
-                             logstring='Filter trades by timerange: {}')
-
-        self._args_to_config(config, argname='analysis_rejected',
-                             logstring='Analyse rejected signals: {}')
-
-        self._args_to_config(config, argname='analysis_to_csv',
-                             logstring='Store analysis tables to CSV: {}')
-
-        self._args_to_config(config, argname='analysis_csv_path',
-                             logstring='Path to store analysis CSVs: {}')
-
-        self._args_to_config(config, argname='analysis_csv_path',
-                             logstring='Path to store analysis CSVs: {}')
-
-        # Lookahead analysis results
-        self._args_to_config(config, argname='targeted_trade_amount',
-                             logstring='Targeted Trade amount: {}')
-
-        self._args_to_config(config, argname='minimum_trade_amount',
-                             logstring='Minimum Trade amount: {}')
-
-        self._args_to_config(config, argname='lookahead_analysis_exportfilename',
-                             logstring='Path to store lookahead-analysis-results: {}')
-
-        self._args_to_config(config, argname='startup_candle',
-                             logstring='Startup candle to be used on recursive analysis: {}')
 
     def _process_runmode(self, config: Config) -> None:
 

freqtrade/configuration/deprecated_settings.py

@@ -6,7 +6,7 @@ import logging
 from typing import Optional
 
 from freqtrade.constants import Config
-from freqtrade.exceptions import OperationalException
+from freqtrade.exceptions import ConfigurationError, OperationalException
 
 
 logger = logging.getLogger(__name__)
@@ -41,7 +41,7 @@ def process_removed_setting(config: Config,
     section1_config = config.get(section1, {})
     if name1 in section1_config:
         section_2 = f"{section2}.{name2}" if section2 else f"{name2}"
-        raise OperationalException(
+        raise ConfigurationError(
             f"Setting `{section1}.{name1}` has been moved to `{section_2}. "
             f"Please delete it from your configuration and use the `{section_2}` "
             "setting instead."
@@ -122,7 +122,7 @@ def process_temporary_deprecated_settings(config: Config) -> None:
                             None, 'ignore_roi_if_entry_signal')
     if (config.get('edge', {}).get('enabled', False)
        and 'capital_available_percentage' in config.get('edge', {})):
-        raise OperationalException(
+        raise ConfigurationError(
             "DEPRECATED: "
             "Using 'edge.capital_available_percentage' has been deprecated in favor of "
             "'tradable_balance_ratio'. Please migrate your configuration to "
@@ -131,7 +131,7 @@ def process_temporary_deprecated_settings(config: Config) -> None:
         )
     if 'ticker_interval' in config:
 
-        raise OperationalException(
+        raise ConfigurationError(
             "DEPRECATED: 'ticker_interval' detected. "
             "Please use 'timeframe' instead of 'ticker_interval."
         )

freqtrade/configuration/environment_vars.py

@@ -9,7 +9,7 @@ from freqtrade.misc import deep_merge_dicts
 logger = logging.getLogger(__name__)
 
 
-def get_var_typed(val):
+def _get_var_typed(val):
     try:
         return int(val)
     except ValueError:
@@ -24,7 +24,7 @@ def get_var_typed(val):
     return val
 
 
-def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, Any]:
+def _flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, Any]:
     """
     Environment variables must be prefixed with FREQTRADE.
     FREQTRADE__{section}__{key}
@@ -40,7 +40,7 @@ def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str,
             logger.info(f"Loading variable '{env_var}'")
             key = env_var.replace(prefix, '')
             for k in reversed(key.split('__')):
-                val = {k.lower(): get_var_typed(val)
+                val = {k.lower(): _get_var_typed(val)
                        if not isinstance(val, dict) and k not in no_convert else val}
             relevant_vars = deep_merge_dicts(val, relevant_vars)
     return relevant_vars
@@ -52,4 +52,4 @@ def enironment_vars_to_dict() -> Dict[str, Any]:
     Relevant variables must follow the FREQTRADE__{section}__{key} pattern
     :return: Nested dict based on available and relevant variables.
     """
-    return flat_vars_to_nested_dict(os.environ.copy(), ENV_VAR_PREFIX)
+    return _flat_vars_to_nested_dict(os.environ.copy(), ENV_VAR_PREFIX)

freqtrade/configuration/load_config.py

@@ -11,7 +11,7 @@ from typing import Any, Dict, List, Optional
 import rapidjson
 
 from freqtrade.constants import MINIMAL_CONFIG, Config
-from freqtrade.exceptions import OperationalException
+from freqtrade.exceptions import ConfigurationError, OperationalException
 from freqtrade.misc import deep_merge_dicts
 
 
@@ -46,7 +46,7 @@ def load_file(path: Path) -> Dict[str, Any]:
         with path.open('r') as file:
             config = rapidjson.load(file, parse_mode=CONFIG_PARSE_MODE)
     except FileNotFoundError:
-        raise OperationalException(f'File "{path}" not found!')
+        raise OperationalException(f'File "{path}" not found!') from None
     return config
 
 
@@ -63,10 +63,10 @@ def load_config_file(path: str) -> Dict[str, Any]:
     except FileNotFoundError:
         raise OperationalException(
             f'Config file "{path}" not found!'
-            ' Please create a config file or check whether it exists.')
+            ' Please create a config file or check whether it exists.') from None
     except rapidjson.JSONDecodeError as e:
         err_range = log_config_error_range(path, str(e))
-        raise OperationalException(
+        raise ConfigurationError(
             f'{e}\n'
             f'Please verify the following segment of your configuration:\n{err_range}'
             if err_range else 'Please verify your configuration file for syntax errors.'
@@ -83,7 +83,7 @@ def load_from_files(
     """
     config: Config = {}
     if level > 5:
-        raise OperationalException("Config loop detected.")
+        raise ConfigurationError("Config loop detected.")
 
     if not files:
         return deepcopy(MINIMAL_CONFIG)

freqtrade/configuration/timerange.py

@@ -9,7 +9,7 @@ from typing import Optional
 from typing_extensions import Self
 
 from freqtrade.constants import DATETIME_PRINT_FORMAT
-from freqtrade.exceptions import OperationalException
+from freqtrade.exceptions import ConfigurationError
 
 
 logger = logging.getLogger(__name__)
@@ -156,7 +156,7 @@ class TimeRange:
                     else:
                         stop = int(stops)
                 if start > stop > 0:
-                    raise OperationalException(
+                    raise ConfigurationError(
                         f'Start date is after stop date for timerange "{text}"')
                 return cls(stype[0], stype[1], start, stop)
-        raise OperationalException(f'Incorrect syntax for timerange "{text}"')
+        raise ConfigurationError(f'Incorrect syntax for timerange "{text}"')

freqtrade/constants.py

@@ -33,9 +33,10 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
                          'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss',
                          'ProfitDrawDownHyperOptLoss']
 AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', 'RemotePairList',
-                       'AgeFilter', "FullTradesFilter", 'OffsetFilter', 'PerformanceFilter',
+                       'MarketCapPairList', 'AgeFilter', "FullTradesFilter", 'OffsetFilter',
-                       'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
+                       'PerformanceFilter', 'PrecisionFilter', 'PriceFilter',
-                       'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
+                       'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter',
+                       'VolatilityFilter']
 AVAILABLE_PROTECTIONS = ['CooldownPeriod',
                          'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard']
 AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5', 'feather', 'parquet']
@@ -105,7 +106,7 @@ SUPPORTED_FIAT = [
     "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY",
     "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN",
     "RUB", "UAH", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR",
-    "USD", "BTC", "ETH", "XRP", "LTC", "BCH"
+    "USD", "BTC", "ETH", "XRP", "LTC", "BCH", "BNB"
 ]
 
 MINIMAL_CONFIG = {

freqtrade/data/btanalysis.py

@@ -11,7 +11,7 @@ import numpy as np
 import pandas as pd
 
 from freqtrade.constants import LAST_BT_RESULT_FN, IntOrInf
-from freqtrade.exceptions import OperationalException
+from freqtrade.exceptions import ConfigurationError, OperationalException
 from freqtrade.misc import file_dump_json, json_load
 from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename
 from freqtrade.persistence import LocalTrade, Trade, init_db
@@ -106,7 +106,7 @@ def get_latest_hyperopt_file(
         directory = Path(directory)
     if predef_filename:
         if Path(predef_filename).is_absolute():
-            raise OperationalException(
+            raise ConfigurationError(
                 "--hyperopt-filename expects only the filename, not an absolute path.")
         return directory / predef_filename
     return directory / get_latest_hyperopt_filename(directory)
@@ -175,36 +175,40 @@ def _get_backtest_files(dirname: Path) -> List[Path]:
     return list(reversed(sorted(dirname.glob('backtest-result-*-[0-9][0-9].json'))))
 
 
-def get_backtest_result(filename: Path) -> List[BacktestHistoryEntryType]:
+def _extract_backtest_result(filename: Path) -> List[BacktestHistoryEntryType]:
-    """
+    metadata = load_backtest_metadata(filename)
-    Get backtest result read from metadata file
-    """
     return [
         {
             'filename': filename.stem,
             'strategy': s,
-            'notes': v.get('notes', ''),
             'run_id': v['run_id'],
+            'notes': v.get('notes', ''),
+            # Backtest "run" time
             'backtest_start_time': v['backtest_start_time'],
-        } for s, v in load_backtest_metadata(filename).items()
+            # Backtest timerange
+            'backtest_start_ts': v.get('backtest_start_ts', None),
+            'backtest_end_ts': v.get('backtest_end_ts', None),
+            'timeframe': v.get('timeframe', None),
+            'timeframe_detail': v.get('timeframe_detail', None),
+        } for s, v in metadata.items()
     ]
 
 
+def get_backtest_result(filename: Path) -> List[BacktestHistoryEntryType]:
+    """
+    Get backtest result read from metadata file
+    """
+    return _extract_backtest_result(filename)
+
+
 def get_backtest_resultlist(dirname: Path) -> List[BacktestHistoryEntryType]:
     """
     Get list of backtest results read from metadata files
     """
     return [
-        {
+        result
-            'filename': filename.stem,
-            'strategy': s,
-            'run_id': v['run_id'],
-            'notes': v.get('notes', ''),
-            'backtest_start_time': v['backtest_start_time'],
-        }
         for filename in _get_backtest_files(dirname)
-        for s, v in load_backtest_metadata(filename).items()
+        for result in _extract_backtest_result(filename)
-        if v
     ]
 
 
@@ -326,7 +330,10 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
                                  "Please specify a strategy.")
 
         if strategy not in data['strategy']:
-            raise ValueError(f"Strategy {strategy} not available in the backtest result.")
+            raise ValueError(
+                f"Strategy {strategy} not available in the backtest result. "
+                f"Available strategies are '{','.join(data['strategy'].keys())}'"
+                )
 
         data = data['strategy'][strategy]['trades']
         df = pd.DataFrame(data)
@@ -350,10 +357,10 @@ def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataF
     :param timeframe: Timeframe used for backtest
     :return: dataframe with open-counts per time-period in timeframe
     """
-    from freqtrade.exchange import timeframe_to_minutes
+    from freqtrade.exchange import timeframe_to_resample_freq
-    timeframe_min = timeframe_to_minutes(timeframe)
+    timeframe_freq = timeframe_to_resample_freq(timeframe)
     dates = [pd.Series(pd.date_range(row[1]['open_date'], row[1]['close_date'],
-                                     freq=f"{timeframe_min}min"))
+                                     freq=timeframe_freq))
              for row in results[['open_date', 'close_date']].iterrows()]
     deltas = [len(x) for x in dates]
     dates = pd.Series(pd.concat(dates).values, name='date')
@@ -361,7 +368,7 @@ def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataF
 
     df2 = pd.concat([dates, df2], axis=1)
     df2 = df2.set_index('date')
-    df_final = df2.resample(f"{timeframe_min}min")[['pair']].count()
+    df_final = df2.resample(timeframe_freq)[['pair']].count()
     df_final = df_final.rename({'pair': 'open_trades'}, axis=1)
     return df_final
 

freqtrade/data/converter/converter.py

@@ -84,7 +84,7 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str)
     using the previous close as price for "open", "high" "low" and "close", volume is set to 0
 
     """
-    from freqtrade.exchange import timeframe_to_minutes
+    from freqtrade.exchange import timeframe_to_resample_freq
 
     ohlcv_dict = {
         'open': 'first',
@@ -93,13 +93,7 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str)
         'close': 'last',
         'volume': 'sum'
     }
-    timeframe_minutes = timeframe_to_minutes(timeframe)
+    resample_interval = timeframe_to_resample_freq(timeframe)
-    resample_interval = f'{timeframe_minutes}min'
-    if timeframe_minutes >= 43200 and timeframe_minutes < 525600:
-        # Monthly candles need special treatment to stick to the 1st of the month
-        resample_interval = f'{timeframe}S'
-    elif timeframe_minutes > 43200:
-        resample_interval = timeframe
     # Resample to create "NAN" values
     df = dataframe.resample(resample_interval, on='date').agg(ohlcv_dict)
 
@@ -116,8 +110,8 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str)
     len_after = len(df)
     pct_missing = (len_after - len_before) / len_before if len_before > 0 else 0
     if len_before != len_after:
-        message = (f"Missing data fillup for {pair}: before: {len_before} - after: {len_after}"
+        message = (f"Missing data fillup for {pair}, {timeframe}: "
-                   f" - {pct_missing:.2%}")
+                   f"before: {len_before} - after: {len_after} - {pct_missing:.2%}")
         if pct_missing > 0.01:
             logger.info(message)
         else:
@@ -206,7 +200,7 @@ def convert_ohlcv_format(
     :param convert_to: Target format
     :param erase: Erase source data (does not apply if source and target format are identical)
     """
-    from freqtrade.data.history.idatahandler import get_datahandler
+    from freqtrade.data.history import get_datahandler
     src = get_datahandler(config['datadir'], convert_from)
     trg = get_datahandler(config['datadir'], convert_to)
     timeframes = config.get('timeframes', [config.get('timeframe')])

freqtrade/data/converter/trade_converter.py

@@ -11,7 +11,7 @@ from pandas import DataFrame, to_datetime
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TRADES_DTYPES,
                                  Config, TradeList)
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 from freqtrade.exceptions import OperationalException
 
 
@@ -70,14 +70,13 @@ def trades_to_ohlcv(trades: DataFrame, timeframe: str) -> DataFrame:
     :return: OHLCV Dataframe.
     :raises: ValueError if no trades are provided
     """
-    from freqtrade.exchange import timeframe_to_minutes
+    from freqtrade.exchange import timeframe_to_resample_freq
-    timeframe_minutes = timeframe_to_minutes(timeframe)
     if trades.empty:
         raise ValueError('Trade-list empty.')
     df = trades.set_index('date', drop=True)
-
+    resample_interval = timeframe_to_resample_freq(timeframe)
-    df_new = df['price'].resample(f'{timeframe_minutes}min').ohlc()
+    df_new = df['price'].resample(resample_interval).ohlc()
-    df_new['volume'] = df['amount'].resample(f'{timeframe_minutes}min').sum()
+    df_new['volume'] = df['amount'].resample(resample_interval).sum()
     df_new['date'] = df_new.index
     # Drop 0 volume rows
     df_new = df_new.dropna()
@@ -89,25 +88,23 @@ def convert_trades_to_ohlcv(
     timeframes: List[str],
     datadir: Path,
     timerange: TimeRange,
-    erase: bool = False,
+    erase: bool,
-    data_format_ohlcv: str = 'feather',
+    data_format_ohlcv: str,
-    data_format_trades: str = 'feather',
+    data_format_trades: str,
-    candle_type: CandleType = CandleType.SPOT
+    candle_type: CandleType,
 ) -> None:
     """
     Convert stored trades data to ohlcv data
     """
-    from freqtrade.data.history.idatahandler import get_datahandler
+    from freqtrade.data.history import get_datahandler
     data_handler_trades = get_datahandler(datadir, data_format=data_format_trades)
     data_handler_ohlcv = get_datahandler(datadir, data_format=data_format_ohlcv)
-    if not pairs:
-        pairs = data_handler_trades.trades_get_pairs(datadir)
 
     logger.info(f"About to convert pairs: '{', '.join(pairs)}', "
                 f"intervals: '{', '.join(timeframes)}' to {datadir}")
-
+    trading_mode = TradingMode.FUTURES if candle_type != CandleType.SPOT else TradingMode.SPOT
     for pair in pairs:
-        trades = data_handler_trades.trades_load(pair)
+        trades = data_handler_trades.trades_load(pair, trading_mode)
         for timeframe in timeframes:
             if erase:
                 if data_handler_ohlcv.ohlcv_purge(pair, timeframe, candle_type=candle_type):
@@ -117,7 +114,7 @@ def convert_trades_to_ohlcv(
                 # Store ohlcv
                 data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv, candle_type=candle_type)
             except ValueError:
-                logger.exception(f'Could not convert {pair} to OHLCV.')
+                logger.warning(f'Could not convert {pair} to OHLCV.')
 
 
 def convert_trades_format(config: Config, convert_from: str, convert_to: str, erase: bool):
@@ -138,18 +135,19 @@ def convert_trades_format(config: Config, convert_from: str, convert_to: str, er
         import_kraken_trades_from_csv(config, convert_to)
         return
 
-    from freqtrade.data.history.idatahandler import get_datahandler
+    from freqtrade.data.history import get_datahandler
     src = get_datahandler(config['datadir'], convert_from)
     trg = get_datahandler(config['datadir'], convert_to)
 
     if 'pairs' not in config:
         config['pairs'] = src.trades_get_pairs(config['datadir'])
     logger.info(f"Converting trades for {config['pairs']}")
-
+    trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT)
     for pair in config['pairs']:
-        data = src.trades_load(pair=pair)
+        data = src.trades_load(pair, trading_mode)
         logger.info(f"Converting {len(data)} trades for {pair}")
-        trg.trades_store(pair, data)
+        trg.trades_store(pair, data, trading_mode)
+
         if erase and convert_from != convert_to:
             logger.info(f"Deleting source Trade data for {pair}.")
-            src.trades_purge(pair=pair)
+            src.trades_purge(pair, trading_mode)

freqtrade/data/converter/trade_converter_kraken.py

@@ -6,8 +6,10 @@ import pandas as pd
 from freqtrade.constants import DATETIME_PRINT_FORMAT, DEFAULT_TRADES_COLUMNS, Config
 from freqtrade.data.converter.trade_converter import (trades_convert_types,
                                                       trades_df_remove_duplicates)
-from freqtrade.data.history.idatahandler import get_datahandler
+from freqtrade.data.history import get_datahandler
+from freqtrade.enums import TradingMode
 from freqtrade.exceptions import OperationalException
+from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
 
 
@@ -38,12 +40,22 @@ def import_kraken_trades_from_csv(config: Config, convert_to: str):
     }
     logger.info(f"Found csv files for {', '.join(data_symbols)}.")
 
+    if pairs_raw := config.get('pairs'):
+        pairs = expand_pairlist(pairs_raw, [m[0] for m in markets])
+        markets = {m for m in markets if m[0] in pairs}
+        if not markets:
+            logger.info(f"No data found for pairs {', '.join(pairs_raw)}.")
+            return
+    logger.info(f"Converting pairs: {', '.join(m[0] for m in markets)}.")
+
     for pair, name in markets:
+        logger.debug(f"Converting pair {pair}, files */{name}.csv")
         dfs = []
         # Load and combine all csv files for this pair
         for f in tradesdir.rglob(f"{name}.csv"):
             df = pd.read_csv(f, names=KRAKEN_CSV_TRADE_COLUMNS)
-            dfs.append(df)
+            if not df.empty:
+                dfs.append(df)
 
         # Load existing trades data
         if not dfs:
@@ -52,19 +64,20 @@ def import_kraken_trades_from_csv(config: Config, convert_to: str):
             continue
 
         trades = pd.concat(dfs, ignore_index=True)
+        del dfs
 
         trades.loc[:, 'timestamp'] = trades['timestamp'] * 1e3
         trades.loc[:, 'cost'] = trades['price'] * trades['amount']
         for col in DEFAULT_TRADES_COLUMNS:
             if col not in trades.columns:
-                trades[col] = ''
+                trades.loc[:, col] = ''
-
         trades = trades[DEFAULT_TRADES_COLUMNS]
         trades = trades_convert_types(trades)
 
         trades_df = trades_df_remove_duplicates(trades)
+        del trades
         logger.info(f"{pair}: {len(trades_df)} trades, from "
                     f"{trades_df['date'].min():{DATETIME_PRINT_FORMAT}} to "
                     f"{trades_df['date'].max():{DATETIME_PRINT_FORMAT}}")
 
-        data_handler.trades_store(pair, trades_df)
+        data_handler.trades_store(pair, trades_df, TradingMode.SPOT)

freqtrade/data/dataprovider.py

@@ -311,11 +311,13 @@ class DataProvider:
             timerange = TimeRange.parse_timerange(None if self._config.get(
                 'timerange') is None else str(self._config.get('timerange')))
 
-            # It is not necessary to add the training candles, as they
+            startup_candles = self.get_required_startup(str(timeframe))
-            # were already added at the beginning of the backtest.
-            startup_candles = self.get_required_startup(str(timeframe), False)
             tf_seconds = timeframe_to_seconds(str(timeframe))
             timerange.subtract_start(tf_seconds * startup_candles)
+
+            logger.info(f"Loading data for {pair} {timeframe} "
+                        f"from {timerange.start_fmt} to {timerange.stop_fmt}")
+
             self.__cached_pairs_backtesting[saved_pair] = load_pair_history(
                 pair=pair,
                 timeframe=timeframe,
@@ -327,7 +329,7 @@ class DataProvider:
             )
         return self.__cached_pairs_backtesting[saved_pair].copy()
 
-    def get_required_startup(self, timeframe: str, add_train_candles: bool = True) -> int:
+    def get_required_startup(self, timeframe: str) -> int:
         freqai_config = self._config.get('freqai', {})
         if not freqai_config.get('enabled', False):
             return self._config.get('startup_candle_count', 0)
@@ -337,12 +339,11 @@ class DataProvider:
             # make sure the startupcandles is at least the set maximum indicator periods
             self._config['startup_candle_count'] = max(startup_candles, max(indicator_periods))
             tf_seconds = timeframe_to_seconds(timeframe)
-            train_candles = 0
+            train_candles = freqai_config['train_period_days'] * 86400 / tf_seconds
-            if add_train_candles:
-                train_candles = freqai_config['train_period_days'] * 86400 / tf_seconds
             total_candles = int(self._config['startup_candle_count'] + train_candles)
-            logger.info(f'Increasing startup_candle_count for freqai to {total_candles}')
+            logger.info(
-            return total_candles
+                f'Increasing startup_candle_count for freqai on {timeframe} to {total_candles}')
+        return total_candles
 
     def get_pair_dataframe(
         self,

freqtrade/data/history/__init__.py

@@ -6,7 +6,7 @@ Includes:
 * download data from exchange and store to disk
 """
 # flake8: noqa: F401
+from .datahandlers import get_datahandler
 from .history_utils import (convert_trades_to_ohlcv, download_data_main, get_timerange, load_data,
                             load_pair_history, refresh_backtest_ohlcv_data,
                             refresh_backtest_trades_data, refresh_data, validate_backtest_data)
-from .idatahandler import get_datahandler

freqtrade/data/history/datahandlers/__init__.py

@@ -0,0 +1,2 @@
+# flake8: noqa: F401
+from .idatahandler import IDataHandler, get_datahandler

freqtrade/data/history/datahandlers/featherdatahandler.py

@@ -5,7 +5,7 @@ from pandas import DataFrame, read_feather, to_datetime
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 
 from .idatahandler import IDataHandler
 
@@ -82,14 +82,15 @@ class FeatherDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_store(self, pair: str, data: DataFrame) -> None:
+    def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
         :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
+        :param trading_mode: Trading mode to use (used to determine the filename)
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         self.create_dir_if_needed(filename)
         data.reset_index(drop=True).to_feather(filename, compression_level=9, compression='lz4')
 
@@ -102,15 +103,18 @@ class FeatherDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
+    def _trades_load(
+            self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
+    ) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         # TODO: respect timerange ...
         :param pair: Load trades for this pair
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :param timerange: Timerange to load trades for - currently not implemented
         :return: Dataframe containing trades
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         if not filename.exists():
             return DataFrame(columns=DEFAULT_TRADES_COLUMNS)
 

freqtrade/data/history/datahandlers/hdf5datahandler.py

@@ -6,7 +6,7 @@ import pandas as pd
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 
 from .idatahandler import IDataHandler
 
@@ -35,7 +35,7 @@ class HDF5DataHandler(IDataHandler):
         self.create_dir_if_needed(filename)
 
         _data.loc[:, self._columns].to_hdf(
-            filename, key, mode='a', complevel=9, complib='blosc',
+            filename, key=key, mode='a', complevel=9, complib='blosc',
             format='table', data_columns=['date']
         )
 
@@ -100,17 +100,18 @@ class HDF5DataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_store(self, pair: str, data: pd.DataFrame) -> None:
+    def _trades_store(self, pair: str, data: pd.DataFrame, trading_mode: TradingMode) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
         :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
+        :param trading_mode: Trading mode to use (used to determine the filename)
         """
         key = self._pair_trades_key(pair)
 
         data.to_hdf(
-            self._pair_trades_filename(self._datadir, pair), key,
+            self._pair_trades_filename(self._datadir, pair, trading_mode), key=key,
             mode='a', complevel=9, complib='blosc',
             format='table', data_columns=['timestamp']
         )
@@ -124,15 +125,18 @@ class HDF5DataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> pd.DataFrame:
+    def _trades_load(
+        self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
+    ) -> pd.DataFrame:
         """
         Load a pair from h5 file.
         :param pair: Load trades for this pair
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :param timerange: Timerange to load trades for - currently not implemented
         :return: Dataframe containing trades
         """
         key = self._pair_trades_key(pair)
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
 
         if not filename.exists():
             return pd.DataFrame(columns=DEFAULT_TRADES_COLUMNS)

freqtrade/data/history/datahandlers/idatahandler.py

@@ -94,21 +94,22 @@ class IDataHandler(ABC):
         """
 
     def ohlcv_data_min_max(self, pair: str, timeframe: str,
-                           candle_type: CandleType) -> Tuple[datetime, datetime]:
+                           candle_type: CandleType) -> Tuple[datetime, datetime, int]:
         """
         Returns the min and max timestamp for the given pair and timeframe.
         :param pair: Pair to get min/max for
         :param timeframe: Timeframe to get min/max for
         :param candle_type: Any of the enum CandleType (must match trading mode!)
-        :return: (min, max)
+        :return: (min, max, len)
         """
-        data = self._ohlcv_load(pair, timeframe, None, candle_type)
+        df = self._ohlcv_load(pair, timeframe, None, candle_type)
-        if data.empty:
+        if df.empty:
             return (
                 datetime.fromtimestamp(0, tz=timezone.utc),
-                datetime.fromtimestamp(0, tz=timezone.utc)
+                datetime.fromtimestamp(0, tz=timezone.utc),
+                0,
             )
-        return data.iloc[0]['date'].to_pydatetime(), data.iloc[-1]['date'].to_pydatetime()
+        return df.iloc[0]['date'].to_pydatetime(), df.iloc[-1]['date'].to_pydatetime(), len(df)
 
     @abstractmethod
     def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange],
@@ -171,12 +172,13 @@ class IDataHandler(ABC):
         return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match]
 
     @abstractmethod
-    def _trades_store(self, pair: str, data: DataFrame) -> None:
+    def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
         :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
+        :param trading_mode: Trading mode to use (used to determine the filename)
         """
 
     @abstractmethod
@@ -189,45 +191,55 @@ class IDataHandler(ABC):
         """
 
     @abstractmethod
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
+    def _trades_load(
+        self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
+    ) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         :param pair: Load trades for this pair
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :param timerange: Timerange to load trades for - currently not implemented
         :return: Dataframe containing trades
         """
 
-    def trades_store(self, pair: str, data: DataFrame) -> None:
+    def trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
         :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
+        :param trading_mode: Trading mode to use (used to determine the filename)
         """
         # Filter on expected columns (will remove the actual date column).
-        self._trades_store(pair, data[DEFAULT_TRADES_COLUMNS])
+        self._trades_store(pair, data[DEFAULT_TRADES_COLUMNS], trading_mode)
 
-    def trades_purge(self, pair: str) -> bool:
+    def trades_purge(self, pair: str, trading_mode: TradingMode) -> bool:
         """
         Remove data for this pair
         :param pair: Delete data for this pair.
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :return: True when deleted, false if file did not exist.
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         if filename.exists():
             filename.unlink()
             return True
         return False
 
-    def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
+    def trades_load(
+            self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
+    ) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         Removes duplicates in the process.
         :param pair: Load trades for this pair
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :param timerange: Timerange to load trades for - currently not implemented
         :return: List of trades
         """
-        trades = trades_df_remove_duplicates(self._trades_load(pair, timerange=timerange))
+        trades = trades_df_remove_duplicates(
+            self._trades_load(pair, trading_mode, timerange=timerange)
+        )
 
         trades = trades_convert_types(trades)
         return trades
@@ -263,8 +275,12 @@ class IDataHandler(ABC):
         return filename
 
     @classmethod
-    def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path:
+    def _pair_trades_filename(cls, datadir: Path, pair: str, trading_mode: TradingMode) -> Path:
         pair_s = misc.pair_to_filename(pair)
+        if trading_mode == TradingMode.FUTURES:
+            # Futures pair ...
+            datadir = datadir.joinpath('futures')
+
         filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}')
         return filename
 
@@ -403,6 +419,34 @@ class IDataHandler(ABC):
             return
         file_old.rename(file_new)
 
+    def fix_funding_fee_timeframe(self, ff_timeframe: str):
+        """
+        Temporary method to migrate data from old funding fee timeframe to the correct timeframe
+        Applies to bybit and okx, where funding-fee and mark candles have different timeframes.
+        """
+        paircombs = self.ohlcv_get_available_data(self._datadir, TradingMode.FUTURES)
+        funding_rate_combs = [
+            f for f in paircombs if f[2] == CandleType.FUNDING_RATE and f[1] != ff_timeframe
+        ]
+
+        if funding_rate_combs:
+            logger.warning(
+                f'Migrating {len(funding_rate_combs)} funding fees to correct timeframe.')
+
+        for pair, timeframe, candletype in funding_rate_combs:
+            old_name = self._pair_data_filename(self._datadir, pair, timeframe, candletype)
+            new_name = self._pair_data_filename(self._datadir, pair, ff_timeframe, candletype)
+
+            if not Path(old_name).exists():
+                logger.warning(f'{old_name} does not exist, skipping.')
+                continue
+
+            if Path(new_name).exists():
+                logger.warning(f'{new_name} already exists, Removing.')
+                Path(new_name).unlink()
+
+            Path(old_name).rename(new_name)
+
 
 def get_datahandlerclass(datatype: str) -> Type[IDataHandler]:
     """

freqtrade/data/history/datahandlers/jsondatahandler.py

@@ -8,7 +8,7 @@ from freqtrade import misc
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
 from freqtrade.data.converter import trades_dict_to_list, trades_list_to_df
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 
 from .idatahandler import IDataHandler
 
@@ -37,7 +37,7 @@ class JsonDataHandler(IDataHandler):
         self.create_dir_if_needed(filename)
         _data = data.copy()
         # Convert date to int
-        _data['date'] = _data['date'].view(np.int64) // 1000 // 1000
+        _data['date'] = _data['date'].astype(np.int64) // 1000 // 1000
 
         # Reset index, select only appropriate columns and save as json
         _data.reset_index(drop=True).loc[:, self._columns].to_json(
@@ -94,14 +94,15 @@ class JsonDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_store(self, pair: str, data: DataFrame) -> None:
+    def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
         :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
+        :param trading_mode: Trading mode to use (used to determine the filename)
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         trades = data.values.tolist()
         misc.file_dump_json(filename, trades, is_zip=self._use_zip)
 
@@ -114,15 +115,18 @@ class JsonDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
+    def _trades_load(
+        self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
+    ) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         # TODO: respect timerange ...
         :param pair: Load trades for this pair
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :param timerange: Timerange to load trades for - currently not implemented
         :return: Dataframe containing trades
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         tradesdata = misc.file_load_json(filename)
 
         if not tradesdata:

freqtrade/data/history/datahandlers/parquetdatahandler.py

@@ -4,8 +4,8 @@ from typing import Optional
 from pandas import DataFrame, read_parquet, to_datetime
 
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList
+from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 
 from .idatahandler import IDataHandler
 
@@ -81,14 +81,15 @@ class ParquetDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_store(self, pair: str, data: DataFrame) -> None:
+    def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
         :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
+        :param trading_mode: Trading mode to use (used to determine the filename)
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         self.create_dir_if_needed(filename)
         data.reset_index(drop=True).to_parquet(filename)
 
@@ -101,15 +102,18 @@ class ParquetDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+    def _trades_load(
+        self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
+    ) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         # TODO: respect timerange ...
         :param pair: Load trades for this pair
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :param timerange: Timerange to load trades for - currently not implemented
         :return: List of trades
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         if not filename.exists():
             return DataFrame(columns=DEFAULT_TRADES_COLUMNS)
 

freqtrade/data/history/history_utils.py

@@ -8,18 +8,18 @@ from pandas import DataFrame, concat
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import (DATETIME_PRINT_FORMAT, DEFAULT_DATAFRAME_COLUMNS,
-                                 DL_DATA_TIMEFRAMES, Config)
+                                 DL_DATA_TIMEFRAMES, DOCS_LINK, Config)
 from freqtrade.data.converter import (clean_ohlcv_dataframe, convert_trades_to_ohlcv,
                                       ohlcv_to_dataframe, trades_df_remove_duplicates,
                                       trades_list_to_df)
-from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
+from freqtrade.data.history.datahandlers import IDataHandler, get_datahandler
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import Exchange
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
 from freqtrade.util import dt_ts, format_ms_time
-from freqtrade.util.binance_mig import migrate_binance_futures_data
 from freqtrade.util.datetime_helpers import dt_now
+from freqtrade.util.migrations import migrate_data
 
 
 logger = logging.getLogger(__name__)
@@ -311,15 +311,19 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
             # Predefined candletype (and timeframe) depending on exchange
             # Downloads what is necessary to backtest based on futures data.
             tf_mark = exchange.get_option('mark_ohlcv_timeframe')
+            tf_funding_rate = exchange.get_option('funding_fee_timeframe')
+
             fr_candle_type = CandleType.from_string(exchange.get_option('mark_ohlcv_price'))
             # All exchanges need FundingRate for futures trading.
             # The timeframe is aligned to the mark-price timeframe.
-            for funding_candle_type in (CandleType.FUNDING_RATE, fr_candle_type):
+            combs = ((CandleType.FUNDING_RATE, tf_funding_rate), (fr_candle_type, tf_mark))
+            for candle_type_f, tf in combs:
+                logger.debug(f'Downloading pair {pair}, {candle_type_f}, interval {tf}.')
                 _download_pair_history(pair=pair, process=process,
                                        datadir=datadir, exchange=exchange,
                                        timerange=timerange, data_handler=data_handler,
-                                       timeframe=str(tf_mark), new_pairs_days=new_pairs_days,
+                                       timeframe=str(tf), new_pairs_days=new_pairs_days,
-                                       candle_type=funding_candle_type,
+                                       candle_type=candle_type_f,
                                        erase=erase, prepend=prepend)
 
     return pairs_not_available
@@ -329,7 +333,8 @@ def _download_trades_history(exchange: Exchange,
                              pair: str, *,
                              new_pairs_days: int = 30,
                              timerange: Optional[TimeRange] = None,
-                             data_handler: IDataHandler
+                             data_handler: IDataHandler,
+                             trading_mode: TradingMode,
                              ) -> bool:
     """
     Download trade history from the exchange.
@@ -345,7 +350,7 @@ def _download_trades_history(exchange: Exchange,
             if timerange.stoptype == 'date':
                 until = timerange.stopts * 1000
 
-        trades = data_handler.trades_load(pair)
+        trades = data_handler.trades_load(pair, trading_mode)
 
         # TradesList columns are defined in constants.DEFAULT_TRADES_COLUMNS
         # DEFAULT_TRADES_COLUMNS: 0 -> timestamp
@@ -384,7 +389,7 @@ def _download_trades_history(exchange: Exchange,
         trades = concat([trades, new_trades_df], axis=0)
         # Remove duplicates to make sure we're not storing data we don't need
         trades = trades_df_remove_duplicates(trades)
-        data_handler.trades_store(pair, data=trades)
+        data_handler.trades_store(pair, trades, trading_mode)
 
         logger.debug("New Start: %s", 'None' if trades.empty else
                      f"{trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}")
@@ -401,8 +406,10 @@ def _download_trades_history(exchange: Exchange,
 
 
 def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
-                                 timerange: TimeRange, new_pairs_days: int = 30,
+                                 timerange: TimeRange, trading_mode: TradingMode,
-                                 erase: bool = False, data_format: str = 'feather') -> List[str]:
+                                 new_pairs_days: int = 30,
+                                 erase: bool = False, data_format: str = 'feather',
+                                 ) -> List[str]:
     """
     Refresh stored trades data for backtesting and hyperopt operations.
     Used by freqtrade download-data subcommand.
@@ -417,7 +424,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
             continue
 
         if erase:
-            if data_handler.trades_purge(pair):
+            if data_handler.trades_purge(pair, trading_mode):
                 logger.info(f'Deleting existing data for pair {pair}.')
 
         logger.info(f'Downloading trades for pair {pair}.')
@@ -425,7 +432,8 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
                                  pair=pair,
                                  new_pairs_days=new_pairs_days,
                                  timerange=timerange,
-                                 data_handler=data_handler)
+                                 data_handler=data_handler,
+                                 trading_mode=trading_mode)
     return pairs_not_available
 
 
@@ -500,18 +508,24 @@ def download_data_main(config: Config) -> None:
     logger.info(f"About to download pairs: {expanded_pairs}, "
                 f"intervals: {config['timeframes']} to {config['datadir']}")
 
+    if len(expanded_pairs) == 0:
+        logger.warning(
+            "No pairs available for download. "
+            "Please make sure you're using the correct Pair naming for your selected trade mode. \n"
+            f"More info: {DOCS_LINK}/bot-basics/#pair-naming")
+
     for timeframe in config['timeframes']:
         exchange.validate_timeframes(timeframe)
 
     # Start downloading
     try:
         if config.get('download_trades'):
-            if config.get('trading_mode') == 'futures':
-                raise OperationalException("Trade download not supported for futures.")
             pairs_not_available = refresh_backtest_trades_data(
                 exchange, pairs=expanded_pairs, datadir=config['datadir'],
                 timerange=timerange, new_pairs_days=config['new_pairs_days'],
-                erase=bool(config.get('erase')), data_format=config['dataformat_trades'])
+                erase=bool(config.get('erase')), data_format=config['dataformat_trades'],
+                trading_mode=config.get('trading_mode', TradingMode.SPOT),
+                )
 
             # Convert downloaded trade data to different timeframes
             convert_trades_to_ohlcv(
@@ -519,6 +533,7 @@ def download_data_main(config: Config) -> None:
                 datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
                 data_format_ohlcv=config['dataformat_ohlcv'],
                 data_format_trades=config['dataformat_trades'],
+                candle_type=config.get('candle_type_def', CandleType.SPOT),
             )
         else:
             if not exchange.get_option('ohlcv_has_history', True):
@@ -527,7 +542,7 @@ def download_data_main(config: Config) -> None:
                     "Please use `--dl-trades` instead for this exchange "
                     "(will unfortunately take a long time)."
                     )
-            migrate_binance_futures_data(config)
+            migrate_data(config, exchange)
             pairs_not_available = refresh_backtest_ohlcv_data(
                 exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
                 datadir=config['datadir'], timerange=timerange,

freqtrade/data/metrics.py

@@ -61,10 +61,10 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
     """
     if len(trades) == 0:
         raise ValueError("Trade dataframe empty.")
-    from freqtrade.exchange import timeframe_to_minutes
+    from freqtrade.exchange import timeframe_to_resample_freq
-    timeframe_minutes = timeframe_to_minutes(timeframe)
+    timeframe_freq = timeframe_to_resample_freq(timeframe)
     # Resample to timeframe to make sure trades match candles
-    _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date'
+    _trades_sum = trades.resample(timeframe_freq, on='close_date'
                                   )[['profit_abs']].sum()
     df.loc[:, col_name] = _trades_sum['profit_abs'].cumsum()
     # Set first value to 0
@@ -143,8 +143,10 @@ def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date'
         starting_balance=starting_balance
     )
 
-    idxmin = max_drawdown_df['drawdown_relative'].idxmax() if relative \
+    idxmin = (
-        else max_drawdown_df['drawdown'].idxmin()
+        max_drawdown_df['drawdown_relative'].idxmax()
+        if relative else max_drawdown_df['drawdown'].idxmin()
+    )
     if idxmin == 0:
         raise ValueError("No losing trade, therefore no drawdown.")
     high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col]
@@ -191,6 +193,9 @@ def calculate_cagr(days_passed: int, starting_balance: float, final_balance: flo
     :param final_balance: Final balance to calculate CAGR against
     :return: CAGR
     """
+    if final_balance < 0:
+        # With leveraged trades, final_balance can become negative.
+        return 0
     return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1
 
 

freqtrade/exceptions.py

@@ -12,6 +12,12 @@ class OperationalException(FreqtradeException):
     """
 
 
+class ConfigurationError(OperationalException):
+    """
+    Configuration error. Usually caused by invalid configuration.
+    """
+
+
 class DependencyException(FreqtradeException):
     """
     Indicates that an assumed dependency is not met.

freqtrade/exchange/__init__.py

@@ -6,7 +6,6 @@ from freqtrade.exchange.exchange import Exchange
 from freqtrade.exchange.binance import Binance
 from freqtrade.exchange.bitmart import Bitmart
 from freqtrade.exchange.bitpanda import Bitpanda
-from freqtrade.exchange.bittrex import Bittrex
 from freqtrade.exchange.bitvavo import Bitvavo
 from freqtrade.exchange.bybit import Bybit
 from freqtrade.exchange.coinbasepro import Coinbasepro
@@ -16,12 +15,15 @@ from freqtrade.exchange.exchange_utils import (ROUND_DOWN, ROUND_UP, amount_to_c
                                                contracts_to_amount, date_minus_candles,
                                                is_exchange_known_ccxt, list_available_exchanges,
                                                market_is_active, price_to_precision,
-                                               timeframe_to_minutes, timeframe_to_msecs,
+                                               validate_exchange)
-                                               timeframe_to_next_date, timeframe_to_prev_date,
+from freqtrade.exchange.exchange_utils_timeframe import (timeframe_to_minutes, timeframe_to_msecs,
-                                               timeframe_to_seconds, validate_exchange)
+                                                         timeframe_to_next_date,
+                                                         timeframe_to_prev_date,
+                                                         timeframe_to_resample_freq,
+                                                         timeframe_to_seconds)
 from freqtrade.exchange.gate import Gate
 from freqtrade.exchange.hitbtc import Hitbtc
-from freqtrade.exchange.huobi import Huobi
+from freqtrade.exchange.htx import Htx
 from freqtrade.exchange.kraken import Kraken
 from freqtrade.exchange.kucoin import Kucoin
 from freqtrade.exchange.okx import Okx

freqtrade/exchange/bittrex.py

@@ -1,25 +0,0 @@
-""" Bittrex exchange subclass """
-import logging
-from typing import Dict
-
-from freqtrade.exchange import Exchange
-
-
-logger = logging.getLogger(__name__)
-
-
-class Bittrex(Exchange):
-    """
-    Bittrex exchange class. Contains adjustments needed for Freqtrade to work
-    with this exchange.
-    """
-
-    _ft_has: Dict = {
-        "ohlcv_candle_limit_per_timeframe": {
-            '1m': 1440,
-            '5m': 288,
-            '1h': 744,
-            '1d': 365,
-        },
-        "l2_limit_range": [1, 25, 500],
-    }

freqtrade/exchange/bybit.py

@@ -25,10 +25,12 @@ class Bybit(Exchange):
     officially supported by the Freqtrade development team. So some features
     may still not work as expected.
     """
+    unified_account = False
 
     _ft_has: Dict = {
         "ohlcv_candle_limit": 1000,
         "ohlcv_has_history": True,
+        "order_time_in_force": ["GTC", "FOK", "IOC", "PO"],
     }
     _ft_has_futures: Dict = {
         "ohlcv_has_history": True,
@@ -81,9 +83,20 @@ class Bybit(Exchange):
         Must be overridden in child methods if required.
         """
         try:
-            if self.trading_mode == TradingMode.FUTURES and not self._config['dry_run']:
+            if not self._config['dry_run']:
-                position_mode = self._api.set_position_mode(False)
+                if self.trading_mode == TradingMode.FUTURES:
-                self._log_exchange_response('set_position_mode', position_mode)
+                    position_mode = self._api.set_position_mode(False)
+                    self._log_exchange_response('set_position_mode', position_mode)
+                is_unified = self._api.is_unified_enabled()
+                # Returns a tuple of bools, first for margin, second for Account
+                if is_unified and len(is_unified) > 1 and is_unified[1]:
+                    self.unified_account = True
+                    logger.info("Bybit: Unified account.")
+                    raise OperationalException("Bybit: Unified account is not supported. "
+                                               "Please use a standard (sub)account.")
+                else:
+                    self.unified_account = False
+                    logger.info("Bybit: Standard account.")
         except ccxt.DDoSProtection as e:
             raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:

freqtrade/exchange/common.py

@@ -2,7 +2,7 @@ import asyncio
 import logging
 import time
 from functools import wraps
-from typing import Any, Callable, Optional, TypeVar, cast, overload
+from typing import Any, Callable, Dict, List, Optional, TypeVar, cast, overload
 
 from freqtrade.constants import ExchangeConfig
 from freqtrade.exceptions import DDosProtection, RetryableOrderError, TemporaryError
@@ -48,27 +48,29 @@ MAP_EXCHANGE_CHILDCLASS = {
     'binanceusdm': 'binance',
     'okex': 'okx',
     'gateio': 'gate',
+    'huboi': 'htx',
 }
 
 SUPPORTED_EXCHANGES = [
     'binance',
     'bitmart',
     'gate',
-    'huobi',
+    'htx',
     'kraken',
     'okx',
 ]
 
-EXCHANGE_HAS_REQUIRED = [
+# either the main, or replacement methods (array) is required
+EXCHANGE_HAS_REQUIRED: Dict[str, List[str]] = {
     # Required / private
-    'fetchOrder',
+    'fetchOrder': ['fetchOpenOrder', 'fetchClosedOrder'],
-    'cancelOrder',
+    'cancelOrder': [],
-    'createOrder',
+    'createOrder': [],
-    'fetchBalance',
+    'fetchBalance': [],
 
     # Public endpoints
-    'fetchOHLCV',
+    'fetchOHLCV': [],
-]
+}
 
 EXCHANGE_HAS_OPTIONAL = [
     # Private
@@ -85,6 +87,7 @@ EXCHANGE_HAS_OPTIONAL = [
     # 'fetchPositions',  # Futures trading
     # 'fetchLeverageTiers',  # Futures initialization
     # 'fetchMarketLeverageTiers',  # Futures initialization
+    # 'fetchOpenOrder', 'fetchClosedOrder',  # replacement for fetchOrder
     # 'fetchOpenOrders', 'fetchClosedOrders',  # 'fetchOrders',  # Refinding balance...
 ]
 

freqtrade/exchange/exchange.py

@@ -8,7 +8,7 @@ import logging
 import signal
 from copy import deepcopy
 from datetime import datetime, timedelta, timezone
-from math import floor
+from math import floor, isnan
 from threading import Lock
 from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union
 
@@ -23,26 +23,29 @@ from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHAN
                                  BuySell, Config, EntryExit, ExchangeConfig,
                                  ListPairsWithTimeframes, MakerTaker, OBLiteral, PairWithTimeframe)
 from freqtrade.data.converter import clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_dict_to_list
-from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, PriceType, TradingMode
+from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, PriceType, RunMode, TradingMode
-from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError,
+from freqtrade.exceptions import (ConfigurationError, DDosProtection, ExchangeError,
-                                  InvalidOrderException, OperationalException, PricingError,
+                                  InsufficientFundsError, InvalidOrderException,
-                                  RetryableOrderError, TemporaryError)
+                                  OperationalException, PricingError, RetryableOrderError,
+                                  TemporaryError)
 from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, remove_exchange_credentials,
                                        retrier, retrier_async)
 from freqtrade.exchange.exchange_utils import (ROUND, ROUND_DOWN, ROUND_UP, CcxtModuleType,
                                                amount_to_contract_precision, amount_to_contracts,
                                                amount_to_precision, contracts_to_amount,
                                                date_minus_candles, is_exchange_known_ccxt,
-                                               market_is_active, price_to_precision,
+                                               market_is_active, price_to_precision)
-                                               timeframe_to_minutes, timeframe_to_msecs,
+from freqtrade.exchange.exchange_utils_timeframe import (timeframe_to_minutes, timeframe_to_msecs,
-                                               timeframe_to_next_date, timeframe_to_prev_date,
+                                                         timeframe_to_next_date,
-                                               timeframe_to_seconds)
+                                                         timeframe_to_prev_date,
+                                                         timeframe_to_seconds)
 from freqtrade.exchange.types import OHLCVResponse, OrderBook, Ticker, Tickers
 from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_json,
                             safe_value_fallback2)
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.util import dt_from_ts, dt_now
 from freqtrade.util.datetime_helpers import dt_humanize, dt_ts
+from freqtrade.util.periodic_cache import PeriodicCache
 
 
 logger = logging.getLogger(__name__)
@@ -80,11 +83,14 @@ class Exchange:
         "l2_limit_range_required": True,  # Allow Empty L2 limit (kucoin)
         "mark_ohlcv_price": "mark",
         "mark_ohlcv_timeframe": "8h",
+        "funding_fee_timeframe": "8h",
         "ccxt_futures_name": "swap",
         "needs_trading_fees": False,  # use fetch_trading_fees to cache fees
         "order_props_in_contracts": ['amount', 'filled', 'remaining'],
         # Override createMarketBuyOrderRequiresPrice where ccxt has it wrong
         "marketOrderRequiresPrice": False,
+        "exchange_has_overrides": {},  # Dictionary overriding ccxt's "has".
+        # Expected to be in the format {"fetchOHLCV": True} or {"fetchOHLCV": False}
     }
     _ft_has: Dict = {}
     _ft_has_futures: Dict = {}
@@ -121,14 +127,16 @@ class Exchange:
         # Cache for 10 minutes ...
         self._cache_lock = Lock()
         self._fetch_tickers_cache: TTLCache = TTLCache(maxsize=2, ttl=60 * 10)
-        # Cache values for 1800 to avoid frequent polling of the exchange for prices
+        # Cache values for 300 to avoid frequent polling of the exchange for prices
         # Caching only applies to RPC methods, so prices for open trades are still
         # refreshed once every iteration.
-        self._exit_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800)
+        # Shouldn't be too high either, as it'll freeze UI updates in case of open orders.
-        self._entry_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800)
+        self._exit_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=300)
+        self._entry_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=300)
 
         # Holds candles
         self._klines: Dict[PairWithTimeframe, DataFrame] = {}
+        self._expiring_candle_cache: Dict[Tuple[str, int], PeriodicCache] = {}
 
         # Holds all open sell orders for dry_run
         self._dry_run_open_orders: Dict[str, Any] = {}
@@ -319,10 +327,11 @@ class Exchange:
         """
         pass
 
-    def _log_exchange_response(self, endpoint, response) -> None:
+    def _log_exchange_response(self, endpoint: str, response, *, add_info=None) -> None:
         """ Log exchange responses """
         if self.log_responses:
-            logger.info(f"API {endpoint}: {response}")
+            add_info_str = "" if add_info is None else f" {add_info}: "
+            logger.info(f"API {endpoint}: {add_info_str}{response}")
 
     def ohlcv_candle_limit(
             self, timeframe: str, candle_type: CandleType, since_ms: Optional[int] = None) -> int:
@@ -330,6 +339,7 @@ class Exchange:
         Exchange ohlcv candle limit
         Uses ohlcv_candle_limit_per_timeframe if the exchange has different limits
         per timeframe (e.g. bittrex), otherwise falls back to ohlcv_candle_limit
+        TODO: this is most likely no longer needed since only bittrex needed this.
         :param timeframe: Timeframe to check
         :param candle_type: Candle-type
         :param since_ms: Starting timestamp
@@ -520,7 +530,7 @@ class Exchange:
             )
         quote_currencies = self.get_quote_currencies()
         if stake_currency not in quote_currencies:
-            raise OperationalException(
+            raise ConfigurationError(
                 f"{stake_currency} is not available as stake on {self.name}. "
                 f"Available currencies are: {', '.join(quote_currencies)}")
 
@@ -588,11 +598,15 @@ class Exchange:
                 f"is therefore not supported. ccxt fetchOHLCV: {self.exchange_has('fetchOHLCV')}")
 
         if timeframe and (timeframe not in self.timeframes):
-            raise OperationalException(
+            raise ConfigurationError(
                 f"Invalid timeframe '{timeframe}'. This exchange supports: {self.timeframes}")
 
-        if timeframe and timeframe_to_minutes(timeframe) < 1:
+        if (
-            raise OperationalException("Timeframes < 1m are currently not supported by Freqtrade.")
+            timeframe
+            and self._config['runmode'] != RunMode.UTIL_EXCHANGE
+            and timeframe_to_minutes(timeframe) < 1
+        ):
+            raise ConfigurationError("Timeframes < 1m are currently not supported by Freqtrade.")
 
     def validate_ordertypes(self, order_types: Dict) -> None:
         """
@@ -600,7 +614,7 @@ class Exchange:
         """
         if any(v == 'market' for k, v in order_types.items()):
             if not self.exchange_has('createMarketOrder'):
-                raise OperationalException(
+                raise ConfigurationError(
                     f'Exchange {self.name} does not support market orders.')
         self.validate_stop_ordertypes(order_types)
 
@@ -610,7 +624,7 @@ class Exchange:
         """
         if (order_types.get("stoploss_on_exchange")
                 and not self._ft_has.get("stoploss_on_exchange", False)):
-            raise OperationalException(
+            raise ConfigurationError(
                 f'On exchange stoploss is not supported for {self.name}.'
             )
         if self.trading_mode == TradingMode.FUTURES:
@@ -620,17 +634,17 @@ class Exchange:
                 and 'stoploss_price_type' in order_types
                 and order_types['stoploss_price_type'] not in price_mapping
             ):
-                raise OperationalException(
+                raise ConfigurationError(
                     f'On exchange stoploss price type is not supported for {self.name}.'
                 )
 
     def validate_pricing(self, pricing: Dict) -> None:
         if pricing.get('use_order_book', False) and not self.exchange_has('fetchL2OrderBook'):
-            raise OperationalException(f'Orderbook not available for {self.name}.')
+            raise ConfigurationError(f'Orderbook not available for {self.name}.')
         if (not pricing.get('use_order_book', False) and (
                 not self.exchange_has('fetchTicker')
                 or not self._ft_has['tickers_have_price'])):
-            raise OperationalException(f'Ticker pricing not available for {self.name}.')
+            raise ConfigurationError(f'Ticker pricing not available for {self.name}.')
 
     def validate_order_time_in_force(self, order_time_in_force: Dict) -> None:
         """
@@ -638,7 +652,7 @@ class Exchange:
         """
         if any(v.upper() not in self._ft_has["order_time_in_force"]
                for k, v in order_time_in_force.items()):
-            raise OperationalException(
+            raise ConfigurationError(
                 f'Time in force policies are not supported for {self.name} yet.')
 
     def validate_required_startup_candles(self, startup_candles: int, timeframe: str) -> int:
@@ -649,7 +663,7 @@ class Exchange:
 
         candle_limit = self.ohlcv_candle_limit(
             timeframe, self._config['candle_type_def'],
-            int(date_minus_candles(timeframe, startup_candles).timestamp() * 1000)
+            dt_ts(date_minus_candles(timeframe, startup_candles))
             if timeframe else None)
         # Require one more candle - to account for the still open candle.
         candle_count = startup_candles + 1
@@ -660,12 +674,12 @@ class Exchange:
 
             if required_candle_call_count > 5:
                 # Only allow 5 calls per pair to somewhat limit the impact
-                raise OperationalException(
+                raise ConfigurationError(
                     f"This strategy requires {startup_candles} candles to start, "
                     "which is more than 5x "
                     f"the amount of candles {self.name} provides for {timeframe}.")
         elif required_candle_call_count > 1:
-            raise OperationalException(
+            raise ConfigurationError(
                 f"This strategy requires {startup_candles} candles to start, which is more than "
                 f"the amount of candles {self.name} provides for {timeframe}.")
         if required_candle_call_count > 1:
@@ -706,6 +720,8 @@ class Exchange:
         :param endpoint: Name of endpoint (e.g. 'fetchOHLCV', 'fetchTickers')
         :return: bool
         """
+        if endpoint in self._ft_has.get('exchange_has_overrides', {}):
+            return self._ft_has['exchange_has_overrides'][endpoint]
         return endpoint in self._api.has and self._api.has[endpoint]
 
     def get_precision_amount(self, pair: str) -> Optional[float]:
@@ -1234,7 +1250,7 @@ class Exchange:
                 f'Insufficient funds to create {ordertype} {side} order on market {pair}. '
                 f'Tried to {side} amount {amount} at rate {limit_rate} with '
                 f'stop-price {stop_price_norm}. Message: {e}') from e
-        except (ccxt.InvalidOrder, ccxt.BadRequest) as e:
+        except (ccxt.InvalidOrder, ccxt.BadRequest, ccxt.OperationRejected) as e:
             # Errors:
             # `Order would trigger immediately.`
             raise InvalidOrderException(
@@ -1250,11 +1266,43 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
+    def fetch_order_emulated(self, order_id: str, pair: str, params: Dict) -> Dict:
+        """
+        Emulated fetch_order if the exchange doesn't support fetch_order, but requires separate
+        calls for open and closed orders.
+        """
+        try:
+            order = self._api.fetch_open_order(order_id, pair, params=params)
+            self._log_exchange_response('fetch_open_order', order)
+            order = self._order_contracts_to_amount(order)
+            return order
+        except ccxt.OrderNotFound:
+            try:
+                order = self._api.fetch_closed_order(order_id, pair, params=params)
+                self._log_exchange_response('fetch_closed_order', order)
+                order = self._order_contracts_to_amount(order)
+                return order
+            except ccxt.OrderNotFound as e:
+                raise RetryableOrderError(
+                    f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e
+        except ccxt.InvalidOrder as e:
+            raise InvalidOrderException(
+                f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
+        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
+            raise TemporaryError(
+                f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e
+        except ccxt.BaseError as e:
+            raise OperationalException(e) from e
+
     @retrier(retries=API_FETCH_ORDER_RETRY_COUNT)
     def fetch_order(self, order_id: str, pair: str, params: Dict = {}) -> Dict:
         if self._config['dry_run']:
             return self.fetch_dry_run_order(order_id)
         try:
+            if not self.exchange_has('fetchOrder'):
+                return self.fetch_order_emulated(order_id, pair, params)
             order = self._api.fetch_order(order_id, pair, params=params)
             self._log_exchange_response('fetch_order', order)
             order = self._order_contracts_to_amount(order)
@@ -1383,7 +1431,7 @@ class Exchange:
             order = self.fetch_stoploss_order(order_id, pair)
         except InvalidOrderException:
             logger.warning(f"Could not fetch cancelled stoploss order {order_id}.")
-            order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
+            order = {'id': order_id, 'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
 
         return order
 
@@ -2001,7 +2049,7 @@ class Exchange:
                 timeframe, candle_type, since_ms)
             move_to = one_call * self.required_candle_call_count
             now = timeframe_to_next_date(timeframe)
-            since_ms = int((now - timedelta(seconds=move_to // 1000)).timestamp() * 1000)
+            since_ms = dt_ts(now - timedelta(seconds=move_to // 1000))
 
         if since_ms:
             return self._async_get_historic_ohlcv(
@@ -2116,6 +2164,39 @@ class Exchange:
 
         return results_df
 
+    def refresh_ohlcv_with_cache(
+        self,
+        pairs: List[PairWithTimeframe],
+        since_ms: int
+    ) -> Dict[PairWithTimeframe, DataFrame]:
+        """
+        Refresh ohlcv data for all pairs in needed_pairs if necessary.
+        Caches data with expiring per timeframe.
+        Should only be used for pairlists which need "on time" expirarion, and no longer cache.
+        """
+
+        timeframes = {p[1] for p in pairs}
+        for timeframe in timeframes:
+            if (timeframe, since_ms) not in self._expiring_candle_cache:
+                timeframe_in_sec = timeframe_to_seconds(timeframe)
+                # Initialise cache
+                self._expiring_candle_cache[(timeframe, since_ms)] = PeriodicCache(
+                    ttl=timeframe_in_sec, maxsize=1000)
+
+        # Get candles from cache
+        candles = {
+            c: self._expiring_candle_cache[(c[1], since_ms)].get(c, None) for c in pairs
+            if c in self._expiring_candle_cache[(c[1], since_ms)]
+        }
+        pairs_to_download = [p for p in pairs if p not in candles]
+        if pairs_to_download:
+            candles = self.refresh_latest_ohlcv(
+                pairs_to_download, since_ms=since_ms, cache=False
+            )
+            for c, val in candles.items():
+                self._expiring_candle_cache[(c[1], since_ms)][c] = val
+        return candles
+
     def _now_is_time_to_refresh(self, pair: str, timeframe: str, candle_type: CandleType) -> bool:
         # Timeframe in seconds
         interval_in_sec = timeframe_to_seconds(timeframe)
@@ -2212,13 +2293,13 @@ class Exchange:
     @retrier_async
     async def _async_fetch_trades(self, pair: str,
                                   since: Optional[int] = None,
-                                  params: Optional[dict] = None) -> List[List]:
+                                  params: Optional[dict] = None) -> Tuple[List[List], Any]:
         """
         Asyncronously gets trade history using fetch_trades.
         Handles exchange errors, does one call to the exchange.
         :param pair: Pair to fetch trade data for
         :param since: Since as integer timestamp in milliseconds
-        returns: List of dicts containing trades
+        returns: List of dicts containing trades, the next iteration value (new "since" or trade_id)
         """
         try:
             # fetch trades asynchronously
@@ -2233,7 +2314,8 @@ class Exchange:
                 )
                 trades = await self._api_async.fetch_trades(pair, since=since, limit=1000)
             trades = self._trades_contracts_to_amount(trades)
-            return trades_dict_to_list(trades)
+            pagination_value = self._get_trade_pagination_next_value(trades)
+            return trades_dict_to_list(trades), pagination_value
         except ccxt.NotSupported as e:
             raise OperationalException(
                 f'Exchange {self._api.name} does not support fetching historical trade data.'
@@ -2246,6 +2328,25 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(f'Could not fetch trade data. Msg: {e}') from e
 
+    def _valid_trade_pagination_id(self, pair: str, from_id: str) -> bool:
+        """
+        Verify trade-pagination id is valid.
+        Workaround for odd Kraken issue where ID is sometimes wrong.
+        """
+        return True
+
+    def _get_trade_pagination_next_value(self, trades: List[Dict]):
+        """
+        Extract pagination id for the next "from_id" value
+        Applies only to fetch_trade_history by id.
+        """
+        if not trades:
+            return None
+        if self._trades_pagination == 'id':
+            return trades[-1].get('id')
+        else:
+            return trades[-1].get('timestamp')
+
     async def _async_get_trade_history_id(self, pair: str,
                                           until: int,
                                           since: Optional[int] = None,
@@ -2261,33 +2362,35 @@ class Exchange:
         """
 
         trades: List[List] = []
+        # DEFAULT_TRADES_COLUMNS: 0 -> timestamp
+        # DEFAULT_TRADES_COLUMNS: 1 -> id
+        has_overlap = self._ft_has.get('trades_pagination_overlap', True)
+        # Skip last trade by default since its the key for the next call
+        x = slice(None, -1) if has_overlap else slice(None)
 
-        if not from_id:
+        if not from_id or not self._valid_trade_pagination_id(pair, from_id):
             # Fetch first elements using timebased method to get an ID to paginate on
             # Depending on the Exchange, this can introduce a drift at the start of the interval
             # of up to an hour.
             # e.g. Binance returns the "last 1000" candles within a 1h time interval
             # - so we will miss the first trades.
-            t = await self._async_fetch_trades(pair, since=since)
+            t, from_id = await self._async_fetch_trades(pair, since=since)
-            # DEFAULT_TRADES_COLUMNS: 0 -> timestamp
+            trades.extend(t[x])
-            # DEFAULT_TRADES_COLUMNS: 1 -> id
-            from_id = t[-1][1]
-            trades.extend(t[:-1])
         while True:
             try:
-                t = await self._async_fetch_trades(pair,
+                t, from_id_next = await self._async_fetch_trades(
-                                                   params={self._trades_pagination_arg: from_id})
+                    pair, params={self._trades_pagination_arg: from_id})
                 if t:
-                    # Skip last id since its the key for the next call
+                    trades.extend(t[x])
-                    trades.extend(t[:-1])
+                    if from_id == from_id_next or t[-1][0] > until:
-                    if from_id == t[-1][1] or t[-1][0] > until:
                         logger.debug(f"Stopping because from_id did not change. "
                                      f"Reached {t[-1][0]} > {until}")
                         # Reached the end of the defined-download period - add last trade as well.
-                        trades.extend(t[-1:])
+                        if has_overlap:
+                            trades.extend(t[-1:])
                         break
 
-                    from_id = t[-1][1]
+                    from_id = from_id_next
                 else:
                     logger.debug("Stopping as no more trades were returned.")
                     break
@@ -2313,19 +2416,19 @@ class Exchange:
         # DEFAULT_TRADES_COLUMNS: 1 -> id
         while True:
             try:
-                t = await self._async_fetch_trades(pair, since=since)
+                t, since_next = await self._async_fetch_trades(pair, since=since)
                 if t:
                     # No more trades to download available at the exchange,
                     # So we repeatedly get the same trade over and over again.
-                    if since == t[-1][0] and len(t) == 1:
+                    if since == since_next and len(t) == 1:
                         logger.debug("Stopping because no more trades are available.")
                         break
-                    since = t[-1][0]
+                    since = since_next
                     trades.extend(t)
                     # Reached the end of the defined-download period
-                    if until and t[-1][0] > until:
+                    if until and since_next > until:
                         logger.debug(
-                            f"Stopping because until was reached. {t[-1][0]} > {until}")
+                            f"Stopping because until was reached. {since_next} > {until}")
                         break
                 else:
                     logger.debug("Stopping as no more trades were returned.")
@@ -2406,13 +2509,15 @@ class Exchange:
             )
 
         if type(since) is datetime:
-            since = int(since.timestamp()) * 1000   # * 1000 for ms
+            since = dt_ts(since)
 
         try:
             funding_history = self._api.fetch_funding_history(
                 symbol=pair,
                 since=since
             )
+            self._log_exchange_response('funding_history', funding_history,
+                                        add_info=f"pair: {pair}, since: {since}")
             return sum(fee['amount'] for fee in funding_history)
         except ccxt.DDoSProtection as e:
             raise DDosProtection(e) from e
@@ -2653,7 +2758,7 @@ class Exchange:
             self._log_exchange_response('set_leverage', res)
         except ccxt.DDoSProtection as e:
             raise DDosProtection(e) from e
-        except (ccxt.BadRequest, ccxt.InsufficientFunds) as e:
+        except (ccxt.BadRequest, ccxt.OperationRejected, ccxt.InsufficientFunds) as e:
             if not accept_fail:
                 raise TemporaryError(
                     f'Could not set leverage due to {e.__class__.__name__}. Message: {e}') from e
@@ -2695,7 +2800,7 @@ class Exchange:
             self._log_exchange_response('set_margin_mode', res)
         except ccxt.DDoSProtection as e:
             raise DDosProtection(e) from e
-        except ccxt.BadRequest as e:
+        except (ccxt.BadRequest, ccxt.OperationRejected) as e:
             if not accept_fail:
                 raise TemporaryError(
                     f'Could not set margin mode due to {e.__class__.__name__}. Message: {e}') from e
@@ -2729,17 +2834,16 @@ class Exchange:
             # Only really relevant for trades very close to the full hour
             open_date = timeframe_to_prev_date('1h', open_date)
         timeframe = self._ft_has['mark_ohlcv_timeframe']
-        timeframe_ff = self._ft_has.get('funding_fee_timeframe',
+        timeframe_ff = self._ft_has['funding_fee_timeframe']
-                                        self._ft_has['mark_ohlcv_timeframe'])
+        mark_price_type = CandleType.from_string(self._ft_has["mark_ohlcv_price"])
 
         if not close_date:
             close_date = datetime.now(timezone.utc)
-        since_ms = int(timeframe_to_prev_date(timeframe, open_date).timestamp()) * 1000
+        since_ms = dt_ts(timeframe_to_prev_date(timeframe, open_date))
-
-        mark_comb: PairWithTimeframe = (
-            pair, timeframe, CandleType.from_string(self._ft_has["mark_ohlcv_price"]))
 
+        mark_comb: PairWithTimeframe = (pair, timeframe, mark_price_type)
         funding_comb: PairWithTimeframe = (pair, timeframe_ff, CandleType.FUNDING_RATE)
+
         candle_histories = self.refresh_latest_ohlcv(
             [mark_comb, funding_comb],
             since_ms=since_ms,
@@ -2789,7 +2893,7 @@ class Exchange:
             else:
                 # Fill up missing funding_rate candles with fallback value
                 combined = mark_rates.merge(
-                    funding_rates, on='date', how="outer", suffixes=["_mark", "_fund"]
+                    funding_rates, on='date', how="left", suffixes=["_mark", "_fund"]
                     )
                 combined['open_fund'] = combined['open_fund'].fillna(futures_funding_rate)
                 return combined
@@ -2818,7 +2922,8 @@ class Exchange:
         if not df.empty:
             df1 = df[(df['date'] >= open_date) & (df['date'] <= close_date)]
             fees = sum(df1['open_fund'] * df1['open_mark'] * amount)
-
+        if isnan(fees):
+            fees = 0.0
         # Negate fees for longs as funding_fees expects it this way based on live endpoints.
         return fees if is_short else -fees
 
@@ -2994,3 +3099,4 @@ class Exchange:
             # describes the min amt for a tier, and the lowest tier will always go down to 0
         else:
             raise ExchangeError(f"Cannot get maintenance ratio using {self.name}")
+            raise ExchangeError(f"Cannot get maintenance ratio using {self.name}")

freqtrade/exchange/exchange_utils.py

@@ -11,9 +11,9 @@ from ccxt import (DECIMAL_PLACES, ROUND, ROUND_DOWN, ROUND_UP, SIGNIFICANT_DIGIT
 
 from freqtrade.exchange.common import (BAD_EXCHANGES, EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED,
                                        SUPPORTED_EXCHANGES)
+from freqtrade.exchange.exchange_utils_timeframe import timeframe_to_minutes, timeframe_to_prev_date
 from freqtrade.types import ValidExchangesType
 from freqtrade.util import FtPrecise
-from freqtrade.util.datetime_helpers import dt_from_ts, dt_ts
 
 
 CcxtModuleType = Any
@@ -40,21 +40,34 @@ def available_exchanges(ccxt_module: Optional[CcxtModuleType] = None) -> List[st
 
 
 def validate_exchange(exchange: str) -> Tuple[bool, str]:
+    """
+    returns: can_use, reason
+        with Reason including both missing and missing_opt
+    """
     ex_mod = getattr(ccxt, exchange.lower())()
+    result = True
+    reason = ''
     if not ex_mod or not ex_mod.has:
         return False, ''
-    missing = [k for k in EXCHANGE_HAS_REQUIRED if ex_mod.has.get(k) is not True]
+    missing = [
+        k for k, v in EXCHANGE_HAS_REQUIRED.items()
+        if ex_mod.has.get(k) is not True
+        and not (all(ex_mod.has.get(x) for x in v))
+    ]
     if missing:
-        return False, f"missing: {', '.join(missing)}"
+        result = False
+        reason += f"missing: {', '.join(missing)}"
 
     missing_opt = [k for k in EXCHANGE_HAS_OPTIONAL if not ex_mod.has.get(k)]
 
     if exchange.lower() in BAD_EXCHANGES:
-        return False, BAD_EXCHANGES.get(exchange.lower(), '')
+        result = False
-    if missing_opt:
+        reason = BAD_EXCHANGES.get(exchange.lower(), '')
-        return True, f"missing opt: {', '.join(missing_opt)}"
 
-    return True, ''
+    if missing_opt:
+        reason += f"{'. ' if reason else ''}missing opt: {', '.join(missing_opt)}. "
+
+    return result, reason
 
 
 def _build_exchange_list_entry(
@@ -95,57 +108,6 @@ def list_available_exchanges(all_exchanges: bool) -> List[ValidExchangesType]:
     return exchanges_valid
 
 
-def timeframe_to_seconds(timeframe: str) -> int:
-    """
-    Translates the timeframe interval value written in the human readable
-    form ('1m', '5m', '1h', '1d', '1w', etc.) to the number
-    of seconds for one timeframe interval.
-    """
-    return ccxt.Exchange.parse_timeframe(timeframe)
-
-
-def timeframe_to_minutes(timeframe: str) -> int:
-    """
-    Same as timeframe_to_seconds, but returns minutes.
-    """
-    return ccxt.Exchange.parse_timeframe(timeframe) // 60
-
-
-def timeframe_to_msecs(timeframe: str) -> int:
-    """
-    Same as timeframe_to_seconds, but returns milliseconds.
-    """
-    return ccxt.Exchange.parse_timeframe(timeframe) * 1000
-
-
-def timeframe_to_prev_date(timeframe: str, date: Optional[datetime] = None) -> datetime:
-    """
-    Use Timeframe and determine the candle start date for this date.
-    Does not round when given a candle start date.
-    :param timeframe: timeframe in string format (e.g. "5m")
-    :param date: date to use. Defaults to now(utc)
-    :returns: date of previous candle (with utc timezone)
-    """
-    if not date:
-        date = datetime.now(timezone.utc)
-
-    new_timestamp = ccxt.Exchange.round_timeframe(timeframe, dt_ts(date), ROUND_DOWN) // 1000
-    return dt_from_ts(new_timestamp)
-
-
-def timeframe_to_next_date(timeframe: str, date: Optional[datetime] = None) -> datetime:
-    """
-    Use Timeframe and determine next candle.
-    :param timeframe: timeframe in string format (e.g. "5m")
-    :param date: date to use. Defaults to now(utc)
-    :returns: date of next candle (with utc timezone)
-    """
-    if not date:
-        date = datetime.now(timezone.utc)
-    new_timestamp = ccxt.Exchange.round_timeframe(timeframe, dt_ts(date), ROUND_UP) // 1000
-    return dt_from_ts(new_timestamp)
-
-
 def date_minus_candles(
         timeframe: str, candle_count: int, date: Optional[datetime] = None) -> datetime:
     """

freqtrade/exchange/exchange_utils_timeframe.py

@@ -0,0 +1,81 @@
+from datetime import datetime, timezone
+from typing import Optional
+
+import ccxt
+from ccxt import ROUND_DOWN, ROUND_UP
+
+from freqtrade.util.datetime_helpers import dt_from_ts, dt_ts
+
+
+def timeframe_to_seconds(timeframe: str) -> int:
+    """
+    Translates the timeframe interval value written in the human readable
+    form ('1m', '5m', '1h', '1d', '1w', etc.) to the number
+    of seconds for one timeframe interval.
+    """
+    return ccxt.Exchange.parse_timeframe(timeframe)
+
+
+def timeframe_to_minutes(timeframe: str) -> int:
+    """
+    Same as timeframe_to_seconds, but returns minutes.
+    """
+    return ccxt.Exchange.parse_timeframe(timeframe) // 60
+
+
+def timeframe_to_msecs(timeframe: str) -> int:
+    """
+    Same as timeframe_to_seconds, but returns milliseconds.
+    """
+    return ccxt.Exchange.parse_timeframe(timeframe) * 1000
+
+
+def timeframe_to_resample_freq(timeframe: str) -> str:
+    """
+    Translates the timeframe interval value written in the human readable
+    form ('1m', '5m', '1h', '1d', '1w', etc.) to the resample frequency
+    used by pandas ('1T', '5T', '1H', '1D', '1W', etc.)
+    """
+    if timeframe == '1y':
+        return '1YS'
+    timeframe_seconds = timeframe_to_seconds(timeframe)
+    timeframe_minutes = timeframe_seconds // 60
+    resample_interval = f'{timeframe_seconds}s'
+    if 10000 < timeframe_minutes < 43200:
+        resample_interval = '1W-MON'
+    elif timeframe_minutes >= 43200 and timeframe_minutes < 525600:
+        # Monthly candles need special treatment to stick to the 1st of the month
+        resample_interval = f'{timeframe}S'
+    elif timeframe_minutes > 43200:
+        resample_interval = timeframe
+    return resample_interval
+
+
+def timeframe_to_prev_date(timeframe: str, date: Optional[datetime] = None) -> datetime:
+    """
+    Use Timeframe and determine the candle start date for this date.
+    Does not round when given a candle start date.
+    :param timeframe: timeframe in string format (e.g. "5m")
+    :param date: date to use. Defaults to now(utc)
+    :returns: date of previous candle (with utc timezone)
+    """
+    if not date:
+        date = datetime.now(timezone.utc)
+
+    new_timestamp = ccxt.Exchange.round_timeframe(
+        timeframe, dt_ts(date), ROUND_DOWN) // 1000
+    return dt_from_ts(new_timestamp)
+
+
+def timeframe_to_next_date(timeframe: str, date: Optional[datetime] = None) -> datetime:
+    """
+    Use Timeframe and determine next candle.
+    :param timeframe: timeframe in string format (e.g. "5m")
+    :param date: date to use. Defaults to now(utc)
+    :returns: date of next candle (with utc timezone)
+    """
+    if not date:
+        date = datetime.now(timezone.utc)
+    new_timestamp = ccxt.Exchange.round_timeframe(
+        timeframe, dt_ts(date), ROUND_UP) // 1000
+    return dt_from_ts(new_timestamp)

freqtrade/exchange/gate.py

@@ -96,9 +96,7 @@ class Gate(Exchange):
         return trades
 
     def get_order_id_conditional(self, order: Dict[str, Any]) -> str:
-        if self.trading_mode == TradingMode.FUTURES:
+        return safe_value_fallback2(order, order, 'id_stop', 'id')
-            return safe_value_fallback2(order, order, 'id_stop', 'id')
-        return order['id']
 
     def fetch_stoploss_order(self, order_id: str, pair: str, params: Dict = {}) -> Dict:
         order = self.fetch_order(
@@ -106,17 +104,19 @@ class Gate(Exchange):
             pair=pair,
             params={'stop': True}
         )
-        if self.trading_mode == TradingMode.FUTURES:
+        if order.get('status', 'open') == 'closed':
-            if order['status'] == 'closed':
+            # Places a real order - which we need to fetch explicitly.
-                # Places a real order - which we need to fetch explicitly.
+            val = 'trade_id' if self.trading_mode == TradingMode.FUTURES else 'fired_order_id'
-                new_orderid = order.get('info', {}).get('trade_id')
-                if new_orderid:
-                    order1 = self.fetch_order(order_id=new_orderid, pair=pair, params=params)
-                    order1['id_stop'] = order1['id']
-                    order1['id'] = order_id
-                    order1['stopPrice'] = order.get('stopPrice')
 
-                    return order1
+            if new_orderid := order.get('info', {}).get(val):
+                order1 = self.fetch_order(order_id=new_orderid, pair=pair, params=params)
+                order1['id_stop'] = order1['id']
+                order1['id'] = order_id
+                order1['type'] = 'stoploss'
+                order1['stopPrice'] = order.get('stopPrice')
+                order1['status_stop'] = 'triggered'
+
+                return order1
         return order
 
     def cancel_stoploss_order(self, order_id: str, pair: str, params: Dict = {}) -> Dict:

freqtrade/exchange/htx.py

@@ -1,4 +1,4 @@
-""" Huobi exchange subclass """
+""" HTX exchange subclass """
 import logging
 from typing import Dict
 
@@ -9,9 +9,9 @@ from freqtrade.exchange import Exchange
 logger = logging.getLogger(__name__)
 
 
-class Huobi(Exchange):
+class Htx(Exchange):
     """
-    Huobi exchange class. Contains adjustments needed for Freqtrade to work
+    HTX exchange class. Contains adjustments needed for Freqtrade to work
     with this exchange.
     """