Bump version to 2024.7.1

https://github.com/piwheels/packages/issues/464

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-20.04, ubuntu-22.04 ]
+        os: [ "ubuntu-20.04", "ubuntu-22.04", "ubuntu-24.04" ]
         python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
@@ -55,7 +55,7 @@ jobs:
 
     - name: Installation - *nix
       run: |
-        python -m pip install --upgrade pip wheel
+        python -m pip install --upgrade "pip<=24.0" wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
         export TA_LIBRARY_PATH=${HOME}/dependencies/lib
         export TA_INCLUDE_PATH=${HOME}/dependencies/include
@@ -80,6 +80,11 @@ jobs:
         # Allow failure for coveralls
         coveralls || true
 
+    - name: Run json schema extract
+      # This should be kept before the repository check to ensure that the schema is up-to-date
+      run: |
+        python build_helpers/extract_config_json_schema.py
+
     - name: Check for repository changes
       run: |
         if [ -n "$(git status --porcelain)" ]; then
@@ -192,7 +197,7 @@ jobs:
 
     - name: Installation (python)
       run: |
-        python -m pip install --upgrade pip wheel
+        python -m pip install --upgrade "pip<=24.0" wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
         export TA_LIBRARY_PATH=${HOME}/dependencies/lib
         export TA_INCLUDE_PATH=${HOME}/dependencies/include
@@ -318,6 +323,17 @@ jobs:
       run: |
         mypy freqtrade scripts tests
 
+    - name: Run Pester tests (PowerShell)
+      run: |
+        $PSVersionTable
+        Set-PSRepository psgallery -InstallationPolicy trusted
+        Install-Module -Name Pester -RequiredVersion 5.3.1 -Confirm:$false -Force -SkipPublisherCheck
+        $Error.clear()
+        Invoke-Pester -Path "tests" -CI
+        if ($Error.Length -gt 0) {exit 1}
+
+      shell: powershell
+
     - name: Discord notification
       uses: rjstone/discord-webhook-notify@v1
       if: failure() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)
@@ -411,7 +427,7 @@ jobs:
 
     - name: Installation - *nix
       run: |
-        python -m pip install --upgrade pip wheel
+        python -m pip install --upgrade "pip<=24.0" wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
         export TA_LIBRARY_PATH=${HOME}/dependencies/lib
         export TA_INCLUDE_PATH=${HOME}/dependencies/include
@@ -522,12 +538,12 @@ jobs:
 
 
     - name: Publish to PyPI (Test)
-      uses: pypa/gh-action-pypi-publish@v1.8.14
+      uses: pypa/gh-action-pypi-publish@v1.9.0
       with:
         repository-url: https://test.pypi.org/legacy/
 
     - name: Publish to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.8.14
+      uses: pypa/gh-action-pypi-publish@v1.9.0
 
 
   deploy-docker:
@@ -566,11 +582,11 @@ jobs:
           docker version -f '{{.Server.Experimental}}'
 
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v1
+      uses: docker/setup-qemu-action@v3
 
     - name: Set up Docker Buildx
       id: buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v3
 
     - name: Available platforms
       run: echo ${{ steps.buildx.outputs.platforms }}

.pre-commit-config.yaml

@@ -2,24 +2,24 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
   - repo: https://github.com/pycqa/flake8
-    rev: "7.0.0"
+    rev: "7.1.0"
     hooks:
       - id: flake8
         additional_dependencies: [Flake8-pyproject]
         # stages: [push]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.10.0"
+    rev: "v1.11.0"
     hooks:
       - id: mypy
         exclude: build_helpers
         additional_dependencies:
-          - types-cachetools==5.3.0.7
+          - types-cachetools==5.4.0.20240717
           - types-filelock==3.2.7
-          - types-requests==2.32.0.20240523
+          - types-requests==2.32.0.20240712
           - types-tabulate==0.9.0.20240106
           - types-python-dateutil==2.9.0.20240316
-          - SQLAlchemy==2.0.30
+          - SQLAlchemy==2.0.31
         # stages: [push]
 
   - repo: https://github.com/pycqa/isort
@@ -31,7 +31,7 @@ repos:
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.4.5'
+    rev: 'v0.5.4'
     hooks:
       - id: ruff
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.12.3-slim-bookworm as base
+FROM python:3.12.4-slim-bookworm as base
 
 # Setup env
 ENV LANG C.UTF-8
@@ -25,7 +25,7 @@ FROM base as python-deps
 RUN  apt-get update \
   && apt-get -y install build-essential libssl-dev git libffi-dev libgfortran5 pkg-config cmake gcc \
   && apt-get clean \
-  && pip install --upgrade pip wheel
+  && pip install --upgrade "pip<=24.0" wheel
 
 # Install TA-lib
 COPY build_helpers/* /tmp/
@@ -35,7 +35,7 @@ ENV LD_LIBRARY_PATH /usr/local/lib
 # Install dependencies
 COPY --chown=ftuser:ftuser requirements.txt requirements-hyperopt.txt /freqtrade/
 USER ftuser
-RUN  pip install --user --no-cache-dir numpy \
+RUN  pip install --user --no-cache-dir "numpy<2.0" \
   && pip install --user --no-cache-dir -r requirements-hyperopt.txt
 
 # Copy dependencies to runtime-image

build_helpers/extract_config_json_schema.py

@@ -0,0 +1,17 @@
+"""Script to extract the configuration json schema from config_schema.py file."""
+
+from pathlib import Path
+
+import rapidjson
+
+from freqtrade.configuration.config_schema import CONF_SCHEMA
+
+
+def extract_config_json_schema():
+    schema_filename = Path(__file__).parent / "schema.json"
+    with schema_filename.open("w") as f:
+        rapidjson.dump(CONF_SCHEMA, f, indent=2)
+
+
+if __name__ == "__main__":
+    extract_config_json_schema()

build_helpers/install_windows.ps1

@@ -1,6 +1,6 @@
 # vendored Wheels compiled via https://github.com/xmatthias/ta-lib-python/tree/ta_bundled_040
 
-python -m pip install --upgrade pip wheel
+python -m pip install --upgrade "pip<=24.0" wheel
 
 $pyv = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"
 

docker-compose.yml

@@ -1,5 +1,4 @@
 ---
-version: '3'
 services:
   freqtrade:
     image: freqtradeorg/freqtrade:stable

docker/Dockerfile.armhf

@@ -17,7 +17,7 @@ RUN mkdir /freqtrade \
   && chown ftuser:ftuser /freqtrade \
   # Allow sudoers
   && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers \
-  && pip install --upgrade pip
+  && pip install --upgrade "pip<=24.0"
 
 WORKDIR /freqtrade
 

docker/docker-compose-freqai.yml

@@ -1,5 +1,4 @@
 ---
-version: '3'
 services:
   freqtrade:
     image: freqtradeorg/freqtrade:stable_freqaitorch

docker/docker-compose-jupyter.yml

@@ -1,5 +1,4 @@
 ---
-version: '3'
 services:
   ft_jupyterlab:
     build:

docs/advanced-orderflow.md

@@ -0,0 +1,152 @@
+# Orderflow data
+
+This guide walks you through utilizing public trade data for advanced orderflow analysis in Freqtrade.
+
+!!! Warning "Experimental Feature"
+    The orderflow feature is currently in beta and may be subject to changes in future releases. Please report any issues or feedback on the [Freqtrade GitHub repository](https://github.com/freqtrade/freqtrade/issues).
+
+!!! Warning "Performance"
+    Orderflow requires raw trades data. This data is rather large, and can cause a slow initial startup, when freqtrade needs to download the trades data for the last X candles. Additionally, enabling this feature will cause increased memory usage. Please ensure to have sufficient resources available.
+
+## Getting Started
+
+### Enable Public Trades
+
+In your `config.json` file, set the `use_public_trades` option to true under the `exchange` section.
+
+```json
+"exchange": {
+   ...
+   "use_public_trades": true,
+}
+```
+
+### Configure Orderflow Processing
+
+Define your desired settings for orderflow processing within the orderflow section of config.json. Here, you can adjust factors like:
+
+- `cache_size`: How many previous orderflow candles are saved into cache instead of calculated every new candle
+- `max_candles`: Filter how many candles would you like to get trades data for.
+- `scale`: This controls the price bin size for the footprint chart.
+- `stacked_imbalance_range`: Defines the minimum consecutive imbalanced price levels required for consideration.
+- `imbalance_volume`: Filters out imbalances with volume below this threshold.
+- `imbalance_ratio`: Filters out imbalances with a ratio (difference between ask and bid volume) lower than this value.
+
+```json
+"orderflow": {
+    "cache_size": 1000, 
+    "max_candles": 1500, 
+    "scale": 0.5, 
+    "stacked_imbalance_range": 3, //  needs at least this amount of imbalance next to each other
+    "imbalance_volume": 1, //  filters out below
+    "imbalance_ratio": 3 //  filters out ratio lower than
+  },
+```
+
+## Downloading Trade Data for Backtesting
+
+To download historical trade data for backtesting, use the --dl-trades flag with the freqtrade download-data command.
+
+```bash
+freqtrade download-data -p BTC/USDT:USDT --timerange 20230101- --trading-mode futures --timeframes 5m --dl-trades
+```
+
+!!! Warning "Data availability"
+    Not all exchanges provide public trade data. For supported exchanges, freqtrade will warn you if public trade data is not available if you start downloading data with the `--dl-trades` flag.
+
+## Accessing Orderflow Data
+
+Once activated, several new columns become available in your dataframe:
+
+``` python
+
+dataframe["trades"] # Contains information about each individual trade.
+dataframe["orderflow"] # Represents a footprint chart dict (see below)
+dataframe["imbalances"] # Contains information about imbalances in the order flow.
+dataframe["bid"] # Total bid volume 
+dataframe["ask"] # Total ask volume
+dataframe["delta"] # Difference between ask and bid volume.
+dataframe["min_delta"] # Minimum delta within the candle
+dataframe["max_delta"] # Maximum delta within the candle
+dataframe["total_trades"] # Total number of trades
+dataframe["stacked_imbalances_bid"] # Price level of stacked bid imbalance 
+dataframe["stacked_imbalances_ask"] # Price level of stacked ask imbalance  
+```
+
+You can access these columns in your strategy code for further analysis. Here's an example:
+
+``` python
+def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+    # Calculating cumulative delta
+    dataframe["cum_delta"] = cumulative_delta(dataframe["delta"])
+    # Accessing total trades
+    total_trades = dataframe["total_trades"]
+    ...
+
+def cumulative_delta(delta: Series):
+    cumdelta = delta.cumsum()
+    return cumdelta
+
+```
+
+### Footprint chart (`dataframe["orderflow"]`)
+
+This column provides a detailed breakdown of buy and sell orders at different price levels, offering valuable insights into order flow dynamics. The `scale` parameter in your configuration determines the price bin size for this representation
+
+The `orderflow` column contains a dict with the following structure:
+
+``` output
+{
+    "price": {
+        "bid_amount": 0.0,
+        "ask_amount": 0.0,
+        "bid": 0,
+        "ask": 0,
+        "delta": 0.0,
+        "total_volume": 0.0,
+        "total_trades": 0
+    }
+}
+```
+
+#### Orderflow column explanation
+
+- key: Price bin - binned at `scale` intervals
+- `bid_amount`: Total volume bought at each price level.
+- `ask_amount`: Total volume sold at each price level.
+- `bid`: Number of buy orders at each price level.
+- `ask`: Number of sell orders at each price level.
+- `delta`: Difference between ask and bid volume at each price level.
+- `total_volume`: Total volume (ask amount + bid amount) at each price level.
+- `total_trades`: Total number of trades (ask + bid) at each price level.
+
+By leveraging these features, you can gain valuable insights into market sentiment and potential trading opportunities based on order flow analysis.
+
+### Raw trades data (`dataframe["trades"]`)
+
+List with the individual trades that occurred during the candle. This data can be used for more granular analysis of order flow dynamics.
+
+Each individual entry contains a dict with the following keys:
+
+- `timestamp`: Timestamp of the trade.
+- `date`: Date of the trade.
+- `price`: Price of the trade.
+- `amount`: Volume of the trade.
+- `side`: Buy or sell.
+- `id`: Unique identifier for the trade.
+- `cost`: Total cost of the trade (price * amount).
+
+### Imbalances (`dataframe["imbalances"]`)
+
+This column provides a dict with information about imbalances in the order flow. An imbalance occurs when there is a significant difference between the ask and bid volume at a given price level.
+
+Each row looks as follows - with price as index, and the corresponding bid and ask imbalance values as columns
+
+``` output
+{
+    "price": {
+        "bid_imbalance": False,
+        "ask_imbalance": False
+    }
+}
+```

docs/advanced-setup.md

@@ -114,8 +114,46 @@ services:
       --strategy SampleStrategy
 
 ```
+
 You can use whatever naming convention you want, freqtrade1 and 2 are arbitrary. Note, that you will need to use different database files, port mappings and telegram configurations for each instance, as mentioned above. 
 
+## Use a different database system
+
+Freqtrade is using SQLAlchemy, which supports multiple different database systems. As such, a multitude of database systems should be supported.
+Freqtrade does not depend or install any additional database driver. Please refer to the [SQLAlchemy docs](https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls) on installation instructions for the respective database systems.
+
+The following systems have been tested and are known to work with freqtrade:
+
+* sqlite (default)
+* PostgreSQL
+* MariaDB
+
+!!! Warning
+    By using one of the below database systems, you acknowledge that you know how to manage such a system. The freqtrade team will not provide any support with setup or maintenance (or backups) of the below database systems.
+
+### PostgreSQL
+
+Installation:
+`pip install psycopg2-binary`
+
+Usage:
+`... --db-url postgresql+psycopg2://<username>:<password>@localhost:5432/<database>`
+
+Freqtrade will automatically create the tables necessary upon startup.
+
+If you're running different instances of Freqtrade, you must either setup one database per Instance or use different users / schemas for your connections.
+
+### MariaDB / MySQL
+
+Freqtrade supports MariaDB by using SQLAlchemy, which supports multiple different database systems.
+
+Installation:
+`pip install pymysql`
+
+Usage:
+`... --db-url mysql+pymysql://<username>:<password>@localhost:3306/<database>`
+
+
 
 ## Configure the bot running as a systemd service
 

docs/backtesting.md

@@ -253,8 +253,8 @@ A backtesting result will look like that:
 
 ```
 ================================================ BACKTESTING REPORT =================================================
-| Pair     | Entries |   Avg Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration |  Wins Draws Loss   Win%  |
-|:---------|--------:|---------------:|-----------------:|---------------:|:-------------|-------------------------:|
+| Pair     | Trades |   Avg Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration |  Wins Draws Loss   Win%  |
+|----------+--------+----------------+------------------+----------------+--------------+--------------------------|
 | ADA/BTC  |     35 |          -0.11 |      -0.00019428 |          -1.94 | 4:35:00      |    14     0    21   40.0 |
 | ARK/BTC  |     11 |          -0.41 |      -0.00022647 |          -2.26 | 2:03:00      |     3     0     8   27.3 |
 | BTS/BTC  |     32 |           0.31 |       0.00048938 |           4.89 | 5:05:00      |    18     0    14   56.2 |
@@ -275,14 +275,14 @@ A backtesting result will look like that:
 | ZEC/BTC  |     22 |          -0.46 |      -0.00050971 |          -5.09 | 2:22:00      |     7     0    15   31.8 |
 | TOTAL    |    429 |           0.36 |       0.00762792 |          76.20 | 4:12:00      |   186     0   243   43.4 |
 ============================================= LEFT OPEN TRADES REPORT =============================================
-| Pair     |  Entries |   Avg Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Win Draw Loss Win% |
-|:---------|---------:|---------------:|-----------------:|---------------:|:---------------|--------------------:|
+| Pair     |  Trades |   Avg Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Win Draw Loss Win% |
+|----------+---------+----------------+------------------+----------------+----------------+---------------------|
 | ADA/BTC  |       1 |           0.89 |       0.00004434 |           0.44 | 6:00:00        |    1    0    0  100 |
 | LTC/BTC  |       1 |           0.68 |       0.00003421 |           0.34 | 2:00:00        |    1    0    0  100 |
 | TOTAL    |       2 |           0.78 |       0.00007855 |           0.78 | 4:00:00        |    2    0    0  100 |
 ==================== EXIT REASON STATS ====================
 | Exit Reason        |   Exits |  Wins |  Draws |  Losses |
-|:-------------------|--------:|------:|-------:|--------:|
+|--------------------+---------+-------+--------+---------|
 | trailing_stop_loss |     205 |   150 |      0 |      55 |
 | stop_loss          |     166 |     0 |      0 |     166 |
 | exit_signal        |      56 |    36 |      0 |      20 |
@@ -631,8 +631,8 @@ Detailed output for all strategies one after the other will be available, so mak
 
 ```
 ================================================== STRATEGY SUMMARY ===================================================================
-| Strategy    |  Entries |   Avg Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Wins |  Draws | Losses | Drawdown % |
-|:------------|---------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|-------:|-----------:|
+| Strategy    |  Trades |   Avg Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Wins |  Draws | Losses | Drawdown % |
+|-------------+---------+----------------+------------------+----------------+----------------+-------+--------+--------+------------|
 | Strategy1   |     429 |           0.36 |       0.00762792 |          76.20 | 4:12:00        |   186 |      0 |    243 |       45.2 |
 | Strategy2   |    1487 |          -0.13 |      -0.00988917 |         -98.79 | 4:43:00        |   662 |      0 |    825 |     241.68 |
 ```

docs/configuration.md

@@ -204,9 +204,10 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `exchange.uid` | API uid to use for the exchange. Only required when you are in production mode and for exchanges that use uid for API requests.<br>**Keep it in secret, do not disclose publicly.** <br> **Datatype:** String
 | `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Supports regex pairs as `.*/BTC`. Not used by VolumePairList. [More information](plugins.md#pairlists-and-pairlist-handlers). <br> **Datatype:** List
 | `exchange.pair_blacklist` | List of pairs the bot must absolutely avoid for trading and backtesting. [More information](plugins.md#pairlists-and-pairlist-handlers). <br> **Datatype:** List
-| `exchange.ccxt_config` | Additional CCXT parameters passed to both ccxt instances (sync and async). This is usually the correct place for additional ccxt configurations. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation). Please avoid adding exchange secrets here (use the dedicated fields instead), as they may be contained in logs. <br> **Datatype:** Dict
-| `exchange.ccxt_sync_config` | Additional CCXT parameters passed to the regular (sync) ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) <br> **Datatype:** Dict
-| `exchange.ccxt_async_config` | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange  and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) <br> **Datatype:** Dict
+| `exchange.ccxt_config` | Additional CCXT parameters passed to both ccxt instances (sync and async). This is usually the correct place for additional ccxt configurations. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://docs.ccxt.com/#/README?id=overriding-exchange-properties-upon-instantiation). Please avoid adding exchange secrets here (use the dedicated fields instead), as they may be contained in logs. <br> **Datatype:** Dict
+| `exchange.ccxt_sync_config` | Additional CCXT parameters passed to the regular (sync) ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://docs.ccxt.com/#/README?id=overriding-exchange-properties-upon-instantiation) <br> **Datatype:** Dict
+| `exchange.ccxt_async_config` | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange  and are documented in the [ccxt documentation](https://docs.ccxt.com/#/README?id=overriding-exchange-properties-upon-instantiation) <br> **Datatype:** Dict
+| `exchange.enable_ws` | Enable the usage of Websockets for the exchange. <br>[More information](#consuming-exchange-websockets).<br>*Defaults to `true`.* <br> **Datatype:** Boolean
 | `exchange.markets_refresh_interval` | The interval in minutes in which markets are reloaded. <br>*Defaults to `60` minutes.* <br> **Datatype:** Positive Integer
 | `exchange.skip_pair_validation` | Skip pairlist validation on startup.<br>*Defaults to `false`*<br> **Datatype:** Boolean
 | `exchange.skip_open_order_update` | Skips open order updates on startup should the exchange cause problems. Only relevant in live conditions.<br>*Defaults to `false`*<br> **Datatype:** Boolean
@@ -409,6 +410,8 @@ Or another example if your position adjustment assumes it can do 1 additional bu
 
 --8<-- "includes/pricing.md"
 
+## Further Configuration details
+
 ### Understand minimal_roi
 
 The `minimal_roi` configuration parameter is a JSON object where the key is a duration
@@ -614,6 +617,30 @@ Freqtrade supports both Demo and Pro coingecko API keys.
 The Coingecko API key is NOT required for the bot to function correctly.
 It is only used for the conversion of coin to fiat in the Telegram reports, which usually also work without API key.
 
+## Consuming exchange Websockets
+
+Freqtrade can consume websockets through ccxt.pro.
+
+Freqtrade aims ensure data is available at all times.
+Should the websocket connection fail (or be disabled), the bot will fall back to REST API calls.
+
+Should you experience problems you suspect are caused by websockets, you can disable these via the setting `exchange.enable_ws`, which defaults to true.
+
+```jsonc
+"exchange": {
+    // ...
+    "enable_ws": false,
+    // ...
+}
+```
+
+Should you be required to use a proxy, please refer to the [proxy section](#using-proxy-with-freqtrade) for more information.
+
+!!! Info "Rollout"
+    We're implementing this out slowly, ensuring stability of your bots.
+    Currently, usage is limited to ohlcv data streams.
+    It's also limited to a few exchanges, with new exchanges being added on an ongoing basis.
+
 ## Using Dry-run mode
 
 We recommend starting the bot in the Dry-run mode to see how your bot will
@@ -650,9 +677,9 @@ Once you will be happy with your bot performance running in the Dry-run mode, yo
 * API-keys may or may not be provided. Only Read-Only operations (i.e. operations that do not alter account state) on the exchange are performed in dry-run mode.
 * Wallets (`/balance`) are simulated based on `dry_run_wallet`.
 * Orders are simulated, and will not be posted to the exchange.
-* Market orders fill based on orderbook volume the moment the order is placed.
+* Market orders fill based on orderbook volume the moment the order is placed, with a maximum slippage of 5%.
 * Limit orders fill once the price reaches the defined level - or time out based on `unfilledtimeout` settings.
-* Limit orders will be converted to market orders if they cross the price by more than 1%.
+* Limit orders will be converted to market orders if they cross the price by more than 1%, and will be filled immediately based regular market order rules (see point about Market orders above).
 * In combination with `stoploss_on_exchange`, the stop_loss price is assumed to be filled.
 * Open orders (not trades, which are stored in the database) are kept open after bot restarts, with the assumption that they were not filled while being offline.
 
@@ -702,7 +729,7 @@ You should also make sure to read the [Exchanges](exchanges.md) section of the d
 
     **NEVER** share your private configuration file or your exchange keys with anyone!
 
-### Using proxy with Freqtrade
+## Using a proxy with Freqtrade
 
 To use a proxy with freqtrade, export your proxy settings using the variables `"HTTP_PROXY"` and `"HTTPS_PROXY"` set to the appropriate values.
 This will have the proxy settings applied to everything (telegram, coingecko, ...) **except** for exchange requests.
@@ -713,7 +740,7 @@ export HTTPS_PROXY="http://addr:port"
 freqtrade
 ```
 
-#### Proxy exchange requests
+### Proxy exchange requests
 
 To use a proxy for exchange connections - you will have to define the proxies as part of the ccxt configuration.
 
@@ -722,6 +749,7 @@ To use a proxy for exchange connections - you will have to define the proxies as
   "exchange": {
     "ccxt_config": {
       "httpsProxy": "http://addr:port",
+      "wsProxy": "http://addr:port",
     }
   }
 }

docs/developer.md

@@ -22,7 +22,7 @@ This will spin up a local server (usually on port 8000) so you can see if everyt
 ## Developer setup
 
 To configure a development environment, you can either use the provided [DevContainer](#devcontainer-setup), or use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ".
-Alternatively (e.g. if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`.
+Alternatively (e.g. if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -r requirements-dev.txt` - followed by `pip3 install -e .[all]`.
 
 This will install all required tools for development, including `pytest`, `ruff`, `mypy`, and `coveralls`.
 

docs/freqai-running.md

@@ -73,23 +73,26 @@ Backtesting mode requires [downloading the necessary data](#downloading-data-to-
 
 ---
 
-### Saving prediction data
+### Saving backtesting prediction data
 
 To allow for tweaking your strategy (**not** the features!), FreqAI will automatically save the predictions during backtesting so that they can be reused for future backtests and live runs using the same `identifier` model. This provides a performance enhancement geared towards enabling **high-level hyperopting** of entry/exit criteria.
 
-An additional directory called `backtesting_predictions`, which contains all the predictions stored in `hdf` format, will be created in the `unique-id` folder.
+An additional directory called `backtesting_predictions`, which contains all the predictions stored in `feather` format, will be created in the `unique-id` folder.
 
 To change your **features**, you **must** set a new `identifier` in the config to signal to FreqAI to train new models.
 
 To save the models generated during a particular backtest so that you can start a live deployment from one of them instead of training a new model, you must set `save_backtest_models` to `True` in the config.
 
+!!! Note
+    To ensure that the model can be reused, freqAI will call your strategy with a dataframe of length 1. 
+    If your strategy requires more data than this to generate the same features, you can't reuse backtest predictions for live deployment and need to update your `identifier` for each new backtest.
+
 ### Backtest live collected predictions
 
 FreqAI allow you to reuse live historic predictions through the backtest parameter `--freqai-backtest-live-models`. This can be useful when you want to reuse predictions generated in dry/run for comparison or other study.
 
 The `--timerange` parameter must not be informed, as it will be automatically calculated through the data in the historic predictions file.
 
-
 ### Downloading data to cover the full backtest period
 
 For live/dry deployments, FreqAI will download the necessary data automatically. However, to use backtesting functionality, you need to download the necessary data using `download-data` (details [here](data-download.md#data-downloading)). You need to pay careful attention to understanding how much *additional* data needs to be downloaded to ensure that there is a sufficient amount of training data *before* the start of the backtesting time range. The amount of additional data can be roughly estimated by moving the start date of the time range backwards by `train_period_days` and the `startup_candle_count` (see the [parameter table](freqai-parameter-table.md) for detailed descriptions of these parameters) from the beginning of the desired backtesting time range. 

docs/includes/pairlists.md

@@ -373,7 +373,7 @@ Filters low-value coins which would not allow setting stoplosses.
 
 Namely, pairs are blacklisted if a variance of one percent or more in the stop price would be caused by precision rounding on the exchange, i.e. `rounded(stop_price) <= rounded(stop_price * 0.99)`. The idea is to avoid coins with a value VERY close to their lower trading boundary, not allowing setting of proper stoploss.
 
-!!! Tip "PerformanceFilter is pointless for futures trading"
+!!! Tip "PrecisionFilter is pointless for futures trading"
     The above does not apply to shorts. And for longs, in theory the trade will be liquidated first.
 
 !!! Warning "Backtesting"

docs/plotting.md

@@ -2,6 +2,14 @@
 
 This page explains how to plot prices, indicators and profits.
 
+!!! Warning "Deprecated"
+    The commands described in this page (`plot-dataframe`, `plot-profit`) should be considered deprecated and are in maintenance mode.
+    This is mostly for the performance problems even medium sized plots can cause, but also because "store a file and open it in a browser" isn't very intuitive from a UI perspective.
+
+    While there are no immediate plans to remove them, they are not actively maintained - and may be removed short-term should major changes be required to keep them working.
+    
+    Please use [FreqUI](freq-ui.md) for plotting needs, which doesn't struggle with the same performance problems.
+
 ## Installation / Setup
 
 Plotting modules use the Plotly library. You can install / upgrade this by running the following command:

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
 markdown==3.6
 mkdocs==1.6.0
-mkdocs-material==9.5.24
+mkdocs-material==9.5.29
 mdx_truly_sane_lists==1.3
 pymdown-extensions==10.8.1
 jinja2==3.1.4

docs/rest-api.md

@@ -2,7 +2,7 @@
 
 ## FreqUI
 
-FreqUI now has it's own dedicated [documentation section](frequi.md) - please refer to that section for all information regarding the FreqUI.
+FreqUI now has it's own dedicated [documentation section](freq-ui.md) - please refer to that section for all information regarding the FreqUI.
 
 ## Configuration
 
@@ -118,6 +118,14 @@ By default, the script assumes `127.0.0.1` (localhost) and port `8080` to be use
 freqtrade-client --config rest_config.json <command> [optional parameters]
 ```
 
+Commands with many arguments may require keyword arguments (for clarity) - which can be provided as follows:
+
+``` bash
+freqtrade-client --config rest_config.json forceenter BTC/USDT long enter_tag=GutFeeling
+```
+
+This method will work for all arguments - check the "show" command for a list of available parameters.
+
 ??? Note "Programmatic use"
     The `freqtrade-client` package (installable independent of freqtrade) can be used in your own scripts to interact with the freqtrade API.
     to do so, please use the following:

docs/sql_cheatsheet.md

@@ -1,6 +1,13 @@
 # SQL Helper
 
-This page contains some help if you want to edit your sqlite db.
+This page contains some help if you want to query your sqlite db.
+
+!!! Tip "Other Database systems"
+    To use other Database Systems like PostgreSQL or MariaDB, you can use the same queries, but you need to use the respective client for the database system. [Click here](advanced-setup.md#use-a-different-database-system) to learn how to setup a different database system with freqtrade.
+
+!!! Warning
+    If you are not familiar with SQL, you should be very careful when running queries on your database.  
+    Always make sure to have a backup of your database before running any queries.
 
 ## Install sqlite3
 
@@ -43,13 +50,25 @@ sqlite3
 .schema <table_name>
 ```
 
-## Get all trades in the table
+### Get all trades in the table
 
 ```sql
 SELECT * FROM trades;
 ```
 
-## Fix trade still open after a manual exit on the exchange
+## Destructive queries
+
+Queries that write to the database.
+These queries should usually not be necessary as freqtrade tries to handle all database operations itself - or exposes them via API or telegram commands.
+
+!!! Warning
+    Please make sure you have a backup of your database before running any of the below queries.
+
+!!! Danger
+    You should also **never** run any writing query (`update`, `insert`, `delete`) while a bot is connected to the database.
+    This can and will lead to data corruption - most likely, without the possibility of recovery.
+
+### Fix trade still open after a manual exit on the exchange
 
 !!! Warning
     Manually selling a pair on the exchange will not be detected by the bot and it will try to sell anyway. Whenever possible, /forceexit <tradeid> should be used to accomplish the same thing.  
@@ -69,7 +88,7 @@ SET is_open=0,
 WHERE id=<trade_ID_to_update>;
 ```
 
-### Example
+#### Example
 
 ```sql
 UPDATE trades
@@ -82,7 +101,7 @@ SET is_open=0,
 WHERE id=31;
 ```
 
-## Remove trade from the database
+### Remove trade from the database
 
 !!! Tip "Use RPC Methods to delete trades"
     Consider using `/delete <tradeid>` via telegram or rest API. That's the recommended way to deleting trades.
@@ -100,39 +119,3 @@ DELETE FROM trades WHERE id = 31;
 
 !!! Warning
     This will remove this trade from the database. Please make sure you got the correct id and **NEVER** run this query without the `where` clause.
-
-## Use a different database system
-
-Freqtrade is using SQLAlchemy, which supports multiple different database systems. As such, a multitude of database systems should be supported.
-Freqtrade does not depend or install any additional database driver. Please refer to the [SQLAlchemy docs](https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls) on installation instructions for the respective database systems.
-
-The following systems have been tested and are known to work with freqtrade:
-
-* sqlite (default)
-* PostgreSQL
-* MariaDB
-
-!!! Warning
-    By using one of the below database systems, you acknowledge that you know how to manage such a system. The freqtrade team will not provide any support with setup or maintenance (or backups) of the below database systems.
-
-### PostgreSQL
-
-Installation:
-`pip install psycopg2-binary`
-
-Usage:
-`... --db-url postgresql+psycopg2://<username>:<password>@localhost:5432/<database>`
-
-Freqtrade will automatically create the tables necessary upon startup.
-
-If you're running different instances of Freqtrade, you must either setup one database per Instance or use different users / schemas for your connections.
-
-### MariaDB / MySQL
-
-Freqtrade supports MariaDB by using SQLAlchemy, which supports multiple different database systems.
-
-Installation:
-`pip install pymysql`
-
-Usage:
-`... --db-url mysql+pymysql://<username>:<password>@localhost:3306/<database>`

docs/strategy-callbacks.md

@@ -165,7 +165,9 @@ E.g. If the `current_rate` is 200 USD, then returning `0.02` will set the stoplo
 During backtesting, `current_rate` (and `current_profit`) are provided against the candle's high (or low for short trades) - while the resulting stoploss is evaluated against the candle's low (or high for short trades).
 
 The absolute value of the return value is used (the sign is ignored), so returning `0.05` or `-0.05` have the same result, a stoploss 5% below the current price.
-Returning None will be interpreted as "no desire to change", and is the only safe way to return when you'd like to not modify the stoploss.
+Returning `None` will be interpreted as "no desire to change", and is the only safe way to return when you'd like to not modify the stoploss.
+`NaN` and `inf` values are considered invalid and will be ignored (identical to `None`).
+
 
 Stoploss on exchange works similar to `trailing_stop`, and the stoploss on exchange is updated as configured in `stoploss_on_exchange_interval` ([More details about stoploss on exchange](stoploss.md#stop-loss-on-exchangefreqtrade)).
 
@@ -467,7 +469,7 @@ The helper function `stoploss_from_absolute()` can be used to convert from an ab
 
 ??? Example "Returning a stoploss using absolute price from the custom stoploss function"
 
-    If we want to trail a stop price at 2xATR below current price we can call `stoploss_from_absolute(current_rate + (side * candle['atr'] * 2), current_rate, is_short=trade.is_short, leverage=trade.leverage)`.
+    If we want to trail a stop price at 2xATR below current price we can call `stoploss_from_absolute(current_rate + (side * candle['atr'] * 2), current_rate=current_rate, is_short=trade.is_short, leverage=trade.leverage)`.
     For futures, we need to adjust the direction (up or down), as well as adjust for leverage, since the [`custom_stoploss`](strategy-callbacks.md#custom-stoploss) callback  returns the ["risk for this trade"](stoploss.md#stoploss-and-leverage) - not the relative price movement.
 
     ``` python
@@ -492,7 +494,8 @@ The helper function `stoploss_from_absolute()` can be used to convert from an ab
             candle = dataframe.iloc[-1].squeeze()
             side = 1 if trade.is_short else -1
             return stoploss_from_absolute(current_rate + (side * candle['atr'] * 2), 
-                                          current_rate, is_short=trade.is_short,
+                                          current_rate=current_rate, 
+                                          is_short=trade.is_short,
                                           leverage=trade.leverage)
 
     ```

docs/trade-object.md

@@ -13,28 +13,28 @@ The following attributes / properties are available for each individual trade -
 
 |  Attribute | DataType | Description |
 |------------|-------------|-------------|
-`pair`| string | Pair of this trade
-`is_open`| boolean | Is the trade currently open, or has it been concluded
-`open_rate`| float | Rate this trade was entered at (Avg. entry rate in case of trade-adjustments)
-`close_rate`| float | Close rate - only set when is_open = False
-`stake_amount`| float | Amount in Stake (or Quote) currency.
-`amount`| float | Amount in Asset / Base currency that is currently owned.
-`open_date`| datetime | Timestamp when trade was opened **use `open_date_utc` instead**
-`open_date_utc`| datetime | Timestamp when trade was opened - in UTC
-`close_date`| datetime | Timestamp when trade was closed **use `close_date_utc` instead**
-`close_date_utc`| datetime | Timestamp when trade was closed - in UTC
-`close_profit`| float | Relative profit at the time of trade closure. `0.01` == 1%
-`close_profit_abs`| float | Absolute profit (in stake currency) at the time of trade closure.
-`leverage` | float | Leverage used for this trade - defaults to 1.0 in spot markets.
-`enter_tag`| string | Tag provided on entry via the `enter_tag` column in the dataframe
-`is_short` | boolean | True for short trades, False otherwise
-`orders` | Order[] | List of order objects attached to this trade (includes both filled and cancelled orders)
-`date_last_filled_utc` | datetime | Time of the last filled order
-`entry_side` | "buy" / "sell" | Order Side the trade was entered
-`exit_side` | "buy" / "sell" | Order Side that will result in a trade exit / position reduction.
-`trade_direction` | "long" / "short" | Trade direction in text - long or short.
-`nr_of_successful_entries` | int | Number of successful (filled) entry orders
-`nr_of_successful_exits` | int | Number of successful (filled) exit orders
+| `pair` | string | Pair of this trade. |
+| `is_open` | boolean | Is the trade currently open, or has it been concluded. |
+| `open_rate` | float | Rate this trade was entered at (Avg. entry rate in case of trade-adjustments). |
+| `close_rate` | float | Close rate - only set when is_open = False. |
+| `stake_amount` | float | Amount in Stake (or Quote) currency. |
+| `amount` | float | Amount in Asset / Base currency that is currently owned. |
+| `open_date` | datetime | Timestamp when trade was opened **use `open_date_utc` instead** |
+| `open_date_utc` | datetime | Timestamp when trade was opened - in UTC. |
+| `close_date` | datetime | Timestamp when trade was closed **use `close_date_utc` instead** |
+| `close_date_utc` | datetime | Timestamp when trade was closed - in UTC. |
+| `close_profit` | float | Relative profit at the time of trade closure. `0.01` == 1% |
+| `close_profit_abs` | float | Absolute profit (in stake currency) at the time of trade closure. |
+| `leverage` | float | Leverage used for this trade - defaults to 1.0 in spot markets. |
+| `enter_tag` | string | Tag provided on entry via the `enter_tag` column in the dataframe. |
+| `is_short` | boolean | True for short trades, False otherwise. |
+| `orders` | Order[] | List of order objects attached to this trade (includes both filled and cancelled orders). |
+| `date_last_filled_utc` | datetime | Time of the last filled order. |
+| `entry_side` | "buy" / "sell" | Order Side the trade was entered. |
+| `exit_side` | "buy" / "sell" | Order Side that will result in a trade exit / position reduction. |
+| `trade_direction` | "long" / "short" | Trade direction in text - long or short. |
+| `nr_of_successful_entries` | int | Number of successful (filled) entry orders. |
+| `nr_of_successful_exits` | int | Number of successful (filled) exit orders. |
 
 ## Class methods
 

docs/utils.md

@@ -488,7 +488,7 @@ freqtrade test-pairlist --config config.json --quote USDT BTC
 
 `freqtrade convert-db` can be used to convert your database from one system to another (sqlite -> postgres, postgres -> other postgres), migrating all trades, orders and Pairlocks.
 
-Please refer to the [SQL cheatsheet](sql_cheatsheet.md#use-a-different-database-system) to learn about requirements for different database systems.
+Please refer to the [corresponding documentation](advanced-setup.md#use-a-different-database-system) to learn about requirements for different database systems.
 
 ```
 usage: freqtrade convert-db [-h] [--db-url PATH] [--db-url-from PATH]

docs/windows_installation.md

@@ -5,6 +5,30 @@ We **strongly** recommend that Windows users use [Docker](docker_quickstart.md)
 If that is not possible, try using the Windows Linux subsystem (WSL) - for which the Ubuntu instructions should work.
 Otherwise, please follow the instructions below.
 
+All instructions assume that python 3.9+ is installed and available.
+
+## Clone the git repository
+
+First of all clone the repository by running:
+
+``` powershell
+git clone https://github.com/freqtrade/freqtrade.git
+```
+
+Now, choose your installation method, either automatically via script (recommended) or manually following the corresponding instructions.
+
+## Install freqtrade automatically
+
+### Run the installation script
+
+The script will ask you a few questions to determine which parts should be installed.
+
+```powershell
+Set-ExecutionPolicy -ExecutionPolicy Bypass
+cd freqtrade
+. .\setup.ps1
+```
+
 ## Install freqtrade manually
 
 !!! Note "64bit Python version"
@@ -14,13 +38,7 @@ Otherwise, please follow the instructions below.
 !!! Hint
     Using the [Anaconda Distribution](https://www.anaconda.com/distribution/) under Windows can greatly help with installation problems. Check out the [Anaconda installation section](installation.md#installation-with-conda) in the documentation for more information.
 
-### 1. Clone the git repository
-
-```bash
-git clone https://github.com/freqtrade/freqtrade.git
-```
-
-### 2. Install ta-lib
+### Install ta-lib
 
 Install ta-lib according to the [ta-lib documentation](https://github.com/TA-Lib/ta-lib-python#windows).
 

freqtrade/__init__.py

@@ -1,12 +1,12 @@
 """Freqtrade bot"""
 
-__version__ = "2024.5"
+__version__ = "2024.7.1"
 
 if "dev" in __version__:
     from pathlib import Path
 
     try:
-        import subprocess
+        import subprocess  # noqa: S404
 
         freqtrade_basedir = Path(__file__).parent
 
@@ -30,5 +30,5 @@ if "dev" in __version__:
             versionfile = Path("./freqtrade_commit")
             if versionfile.is_file():
                 __version__ = f"docker-{__version__}-{versionfile.read_text()[:8]}"
-        except Exception:
+        except Exception:  # noqa: S110
             pass

freqtrade/commands/arguments.py

@@ -2,10 +2,10 @@
 This module contains the argument manager class
 """
 
-import argparse
+from argparse import ArgumentParser, Namespace, _ArgumentGroup
 from functools import partial
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 from freqtrade.commands.cli_options import AVAILABLE_CLI_OPTIONS
 from freqtrade.constants import DEFAULT_CONFIG
@@ -226,6 +226,19 @@ ARGS_ANALYZE_ENTRIES_EXITS = [
     "analysis_csv_path",
 ]
 
+
+ARGS_STRATEGY_UPDATER = ["strategy_list", "strategy_path", "recursive_strategy_search"]
+
+ARGS_LOOKAHEAD_ANALYSIS = [
+    a
+    for a in ARGS_BACKTEST
+    if a
+    not in ("position_stacking", "use_max_market_positions", "backtest_cache", "backtest_breakdown")
+] + ["minimum_trade_amount", "targeted_trade_amount", "lookahead_analysis_exportfilename"]
+
+ARGS_RECURSIVE_ANALYSIS = ["timeframe", "timerange", "dataformat_ohlcv", "pairs", "startup_candle"]
+
+# Command level configs - keep at the bottom of the above definitions
 NO_CONF_REQURIED = [
     "convert-data",
     "convert-trade-data",
@@ -248,14 +261,6 @@ NO_CONF_REQURIED = [
 
 NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"]
 
-ARGS_STRATEGY_UPDATER = ["strategy_list", "strategy_path", "recursive_strategy_search"]
-
-ARGS_LOOKAHEAD_ANALYSIS = [
-    a for a in ARGS_BACKTEST if a not in ("position_stacking", "use_max_market_positions", "cache")
-] + ["minimum_trade_amount", "targeted_trade_amount", "lookahead_analysis_exportfilename"]
-
-ARGS_RECURSIVE_ANALYSIS = ["timeframe", "timerange", "dataformat_ohlcv", "pairs", "startup_candle"]
-
 
 class Arguments:
     """
@@ -264,7 +269,7 @@ class Arguments:
 
     def __init__(self, args: Optional[List[str]]) -> None:
         self.args = args
-        self._parsed_arg: Optional[argparse.Namespace] = None
+        self._parsed_arg: Optional[Namespace] = None
 
     def get_parsed_arg(self) -> Dict[str, Any]:
         """
@@ -277,7 +282,7 @@ class Arguments:
 
         return vars(self._parsed_arg)
 
-    def _parse_args(self) -> argparse.Namespace:
+    def _parse_args(self) -> Namespace:
         """
         Parses given arguments and returns an argparse Namespace instance.
         """
@@ -306,7 +311,9 @@ class Arguments:
 
         return parsed_arg
 
-    def _build_args(self, optionlist, parser):
+    def _build_args(
+        self, optionlist: List[str], parser: Union[ArgumentParser, _ArgumentGroup]
+    ) -> None:
         for val in optionlist:
             opt = AVAILABLE_CLI_OPTIONS[val]
             parser.add_argument(*opt.cli, dest=val, **opt.kwargs)
@@ -317,16 +324,16 @@ class Arguments:
         :return: None
         """
         # Build shared arguments (as group Common Options)
-        _common_parser = argparse.ArgumentParser(add_help=False)
+        _common_parser = ArgumentParser(add_help=False)
         group = _common_parser.add_argument_group("Common arguments")
         self._build_args(optionlist=ARGS_COMMON, parser=group)
 
-        _strategy_parser = argparse.ArgumentParser(add_help=False)
+        _strategy_parser = ArgumentParser(add_help=False)
         strategy_group = _strategy_parser.add_argument_group("Strategy arguments")
         self._build_args(optionlist=ARGS_STRATEGY, parser=strategy_group)
 
         # Build main command
-        self.parser = argparse.ArgumentParser(
+        self.parser = ArgumentParser(
             prog="freqtrade", description="Free, open source crypto trading bot"
         )
         self._build_args(optionlist=["version"], parser=self.parser)

freqtrade/commands/build_config_commands.py

@@ -187,7 +187,7 @@ def ask_user_config() -> Dict[str, Any]:
                 "Insert Api server Listen Address (0.0.0.0 for docker, "
                 "otherwise best left untouched)"
             ),
-            "default": "127.0.0.1" if not running_in_docker() else "0.0.0.0",
+            "default": "127.0.0.1" if not running_in_docker() else "0.0.0.0",  # noqa: S104
             "when": lambda x: x["api_server"],
         },
         {

freqtrade/commands/data_commands.py

@@ -16,6 +16,7 @@ from freqtrade.exceptions import ConfigurationError
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
+from freqtrade.util import print_rich_table
 from freqtrade.util.migrations import migrate_data
 
 
@@ -119,8 +120,6 @@ def start_list_data(args: Dict[str, Any]) -> None:
 
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
-    from tabulate import tabulate
-
     from freqtrade.data.history import get_datahandler
 
     dhc = get_datahandler(config["datadir"], config["dataformat_ohlcv"])
@@ -131,8 +130,7 @@ def start_list_data(args: Dict[str, Any]) -> None:
 
     if args["pairs"]:
         paircombs = [comb for comb in paircombs if comb[0] in args["pairs"]]
-
-    print(f"Found {len(paircombs)} pair / timeframe combinations.")
+    title = f"Found {len(paircombs)} pair / timeframe combinations."
     if not config.get("show_timerange"):
         groupedpair = defaultdict(list)
         for pair, timeframe, candle_type in sorted(
@@ -141,25 +139,21 @@ def start_list_data(args: Dict[str, Any]) -> None:
             groupedpair[(pair, candle_type)].append(timeframe)
 
         if groupedpair:
-            print(
-                tabulate(
+            print_rich_table(
                 [
                     (pair, ", ".join(timeframes), candle_type)
                     for (pair, candle_type), timeframes in groupedpair.items()
                 ],
-                    headers=("Pair", "Timeframe", "Type"),
-                    tablefmt="psql",
-                    stralign="right",
-                )
+                ("Pair", "Timeframe", "Type"),
+                title,
+                table_kwargs={"min_width": 50},
             )
     else:
         paircombs1 = [
             (pair, timeframe, candle_type, *dhc.ohlcv_data_min_max(pair, timeframe, candle_type))
             for pair, timeframe, candle_type in paircombs
         ]
-
-        print(
-            tabulate(
+        print_rich_table(
             [
                 (
                     pair,
@@ -167,14 +161,13 @@ def start_list_data(args: Dict[str, Any]) -> None:
                     candle_type,
                     start.strftime(DATETIME_PRINT_FORMAT),
                     end.strftime(DATETIME_PRINT_FORMAT),
-                        length,
+                    str(length),
                 )
                 for pair, timeframe, candle_type, start, end, length in sorted(
                     paircombs1, key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2])
                 )
             ],
-                headers=("Pair", "Timeframe", "Type", "From", "To", "Candles"),
-                tablefmt="psql",
-                stralign="right",
-            )
+            ("Pair", "Timeframe", "Type", "From", "To", "Candles"),
+            summary=title,
+            table_kwargs={"min_width": 50},
         )

freqtrade/commands/hyperopt_commands.py

@@ -2,8 +2,6 @@ import logging
 from operator import itemgetter
 from typing import Any, Dict
 
-from colorama import init as colorama_init
-
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.data.btanalysis import get_latest_hyperopt_file
 from freqtrade.enums import RunMode
@@ -18,6 +16,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
     """
     List hyperopt epochs previously evaluated
     """
+    from freqtrade.optimize.hyperopt_output import HyperoptOutput
     from freqtrade.optimize.hyperopt_tools import HyperoptTools
 
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
@@ -35,21 +34,17 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
     # Previous evaluations
     epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config)
 
-    if print_colorized:
-        colorama_init(autoreset=True)
-
     if not export_csv:
         try:
-            print(
-                HyperoptTools.get_result_table(
+            h_out = HyperoptOutput()
+            h_out.add_data(
                 config,
                 epochs,
                 total_epochs,
                 not config.get("hyperopt_list_best", False),
-                    print_colorized,
-                    0,
-                )
             )
+            h_out.print(print_colorized=print_colorized)
+
         except KeyboardInterrupt:
             print("User interrupted..")
 

freqtrade/commands/list_commands.py

@@ -4,9 +4,9 @@ import sys
 from typing import Any, Dict, List, Union
 
 import rapidjson
-from colorama import Fore, Style
-from colorama import init as colorama_init
-from tabulate import tabulate
+from rich.console import Console
+from rich.table import Table
+from rich.text import Text
 
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.enums import RunMode
@@ -14,7 +14,8 @@ from freqtrade.exceptions import ConfigurationError, OperationalException
 from freqtrade.exchange import list_available_exchanges, market_is_active
 from freqtrade.misc import parse_db_uri_for_logging, plural
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
-from freqtrade.types import ValidExchangesType
+from freqtrade.types.valid_exchanges_type import ValidExchangesType
+from freqtrade.util import print_rich_table
 
 
 logger = logging.getLogger(__name__)
@@ -26,71 +27,69 @@ def start_list_exchanges(args: Dict[str, Any]) -> None:
     :param args: Cli args from Arguments()
     :return: None
     """
-    exchanges = list_available_exchanges(args["list_exchanges_all"])
+    available_exchanges: List[ValidExchangesType] = list_available_exchanges(
+        args["list_exchanges_all"]
+    )
 
     if args["print_one_column"]:
-        print("\n".join([e["name"] for e in exchanges]))
+        print("\n".join([e["name"] for e in available_exchanges]))
     else:
-        headers = {
-            "name": "Exchange name",
-            "supported": "Supported",
-            "trade_modes": "Markets",
-            "comment": "Reason",
-        }
-        headers.update({"valid": "Valid"} if args["list_exchanges_all"] else {})
+        if args["list_exchanges_all"]:
+            title = (
+                f"All exchanges supported by the ccxt library "
+                f"({len(available_exchanges)} exchanges):"
+            )
+        else:
+            available_exchanges = [e for e in available_exchanges if e["valid"] is not False]
+            title = f"Exchanges available for Freqtrade ({len(available_exchanges)} exchanges):"
 
-        def build_entry(exchange: ValidExchangesType, valid: bool):
-            valid_entry = {"valid": exchange["valid"]} if valid else {}
-            result: Dict[str, Union[str, bool]] = {
-                "name": exchange["name"],
-                **valid_entry,
-                "supported": "Official" if exchange["supported"] else "",
-                "trade_modes": ", ".join(
-                    (f"{a['margin_mode']} " if a["margin_mode"] else "") + a["trading_mode"]
+        table = Table(title=title)
+
+        table.add_column("Exchange Name")
+        table.add_column("Markets")
+        table.add_column("Reason")
+
+        for exchange in available_exchanges:
+            name = Text(exchange["name"])
+            if exchange["supported"]:
+                name.append(" (Official)", style="italic")
+                name.stylize("green bold")
+
+            trade_modes = Text(
+                ", ".join(
+                    (f"{a.get('margin_mode', '')} {a['trading_mode']}").lstrip()
                     for a in exchange["trade_modes"]
                 ),
-                "comment": exchange["comment"],
-            }
-
-            return result
-
-        if args["list_exchanges_all"]:
-            print("All exchanges supported by the ccxt library:")
-            exchanges = [build_entry(e, True) for e in exchanges]
-        else:
-            print("Exchanges available for Freqtrade:")
-            exchanges = [build_entry(e, False) for e in exchanges if e["valid"] is not False]
-
-        print(
-            tabulate(
-                exchanges,
-                headers=headers,
+                style="",
             )
+            if exchange["dex"]:
+                trade_modes = Text("DEX: ") + trade_modes
+                trade_modes.stylize("bold", 0, 3)
+
+            table.add_row(
+                name,
+                trade_modes,
+                exchange["comment"],
+                style=None if exchange["valid"] else "red",
             )
+            # table.add_row(*[exchange[header] for header in headers])
+
+        console = Console()
+        console.print(table)
 
 
 def _print_objs_tabular(objs: List, print_colorized: bool) -> None:
-    if print_colorized:
-        colorama_init(autoreset=True)
-        red = Fore.RED
-        yellow = Fore.YELLOW
-        reset = Style.RESET_ALL
-    else:
-        red = ""
-        yellow = ""
-        reset = ""
-
     names = [s["name"] for s in objs]
-    objs_to_print = [
+    objs_to_print: List[Dict[str, Union[Text, str]]] = [
         {
-            "name": s["name"] if s["name"] else "--",
+            "name": Text(s["name"] if s["name"] else "--"),
             "location": s["location_rel"],
             "status": (
-                red + "LOAD FAILED" + reset
+                Text("LOAD FAILED", style="bold red")
                 if s["class"] is None
-                else "OK"
+                else Text("OK", style="bold green")
                 if names.count(s["name"]) == 1
-                else yellow + "DUPLICATE NAME" + reset
+                else Text("DUPLICATE NAME", style="bold yellow")
             ),
         }
         for s in objs
@@ -100,11 +99,23 @@ def _print_objs_tabular(objs: List, print_colorized: bool) -> None:
             objs_to_print[idx].update(
                 {
                     "hyperoptable": "Yes" if s["hyperoptable"]["count"] > 0 else "No",
-                    "buy-Params": len(s["hyperoptable"].get("buy", [])),
-                    "sell-Params": len(s["hyperoptable"].get("sell", [])),
+                    "buy-Params": str(len(s["hyperoptable"].get("buy", []))),
+                    "sell-Params": str(len(s["hyperoptable"].get("sell", []))),
                 }
             )
-    print(tabulate(objs_to_print, headers="keys", tablefmt="psql", stralign="right"))
+    table = Table()
+
+    for header in objs_to_print[0].keys():
+        table.add_column(header.capitalize(), justify="right")
+
+    for row in objs_to_print:
+        table.add_row(*[row[header] for header in objs_to_print[0].keys()])
+
+    console = Console(
+        color_system="auto" if print_colorized else None,
+        width=200 if "pytest" in sys.modules else None,
+    )
+    console.print(table)
 
 
 def start_list_strategies(args: Dict[str, Any]) -> None:
@@ -269,9 +280,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
                 writer.writeheader()
                 writer.writerows(tabular_data)
             else:
-                # print data as a table, with the human-readable summary
-                print(f"{summary_str}:")
-                print(tabulate(tabular_data, headers="keys", tablefmt="psql", stralign="right"))
+                print_rich_table(tabular_data, headers, summary_str)
         elif not (
             args.get("print_one_column", False)
             or args.get("list_pairs_print_json", False)

freqtrade/configuration/config_secrets.py

@@ -14,9 +14,13 @@ def sanitize_config(config: Config, *, show_sensitive: bool = False) -> Config:
         return config
     keys_to_remove = [
         "exchange.key",
+        "exchange.apiKey",
         "exchange.secret",
         "exchange.password",
         "exchange.uid",
+        "exchange.accountId",
+        "exchange.walletAddress",
+        "exchange.privateKey",
         "telegram.token",
         "telegram.chat_id",
         "discord.webhook_url",

freqtrade/configuration/config_validation.py

@@ -6,8 +6,16 @@ from typing import Any, Dict
 from jsonschema import Draft4Validator, validators
 from jsonschema.exceptions import ValidationError, best_match
 
-from freqtrade import constants
+from freqtrade.configuration.config_schema import (
+    CONF_SCHEMA,
+    SCHEMA_BACKTEST_REQUIRED,
+    SCHEMA_BACKTEST_REQUIRED_FINAL,
+    SCHEMA_MINIMAL_REQUIRED,
+    SCHEMA_MINIMAL_WEBSERVER,
+    SCHEMA_TRADE_REQUIRED,
+)
 from freqtrade.configuration.deprecated_settings import process_deprecated_setting
+from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
 from freqtrade.enums import RunMode, TradingMode
 from freqtrade.exceptions import ConfigurationError
 
@@ -41,18 +49,18 @@ def validate_config_schema(conf: Dict[str, Any], preliminary: bool = False) -> D
     :param conf: Config in JSON format
     :return: Returns the config if valid, otherwise throw an exception
     """
-    conf_schema = deepcopy(constants.CONF_SCHEMA)
+    conf_schema = deepcopy(CONF_SCHEMA)
     if conf.get("runmode", RunMode.OTHER) in (RunMode.DRY_RUN, RunMode.LIVE):
-        conf_schema["required"] = constants.SCHEMA_TRADE_REQUIRED
+        conf_schema["required"] = SCHEMA_TRADE_REQUIRED
     elif conf.get("runmode", RunMode.OTHER) in (RunMode.BACKTEST, RunMode.HYPEROPT):
         if preliminary:
-            conf_schema["required"] = constants.SCHEMA_BACKTEST_REQUIRED
+            conf_schema["required"] = SCHEMA_BACKTEST_REQUIRED
         else:
-            conf_schema["required"] = constants.SCHEMA_BACKTEST_REQUIRED_FINAL
+            conf_schema["required"] = SCHEMA_BACKTEST_REQUIRED_FINAL
     elif conf.get("runmode", RunMode.OTHER) == RunMode.WEBSERVER:
-        conf_schema["required"] = constants.SCHEMA_MINIMAL_WEBSERVER
+        conf_schema["required"] = SCHEMA_MINIMAL_WEBSERVER
     else:
-        conf_schema["required"] = constants.SCHEMA_MINIMAL_REQUIRED
+        conf_schema["required"] = SCHEMA_MINIMAL_REQUIRED
     try:
         FreqtradeValidator(conf_schema).validate(conf)
         return conf
@@ -83,6 +91,7 @@ def validate_config_consistency(conf: Dict[str, Any], *, preliminary: bool = Fal
     _validate_freqai_include_timeframes(conf, preliminary=preliminary)
     _validate_consumers(conf)
     validate_migrated_strategy_settings(conf)
+    _validate_orderflow(conf)
 
     # validate configuration before returning
     logger.info("Validating configuration ...")
@@ -97,7 +106,7 @@ def _validate_unlimited_amount(conf: Dict[str, Any]) -> None:
     if (
         not conf.get("edge", {}).get("enabled")
         and conf.get("max_open_trades") == float("inf")
-        and conf.get("stake_amount") == constants.UNLIMITED_STAKE_AMOUNT
+        and conf.get("stake_amount") == UNLIMITED_STAKE_AMOUNT
     ):
         raise ConfigurationError("`max_open_trades` and `stake_amount` cannot both be unlimited.")
 
@@ -421,6 +430,14 @@ def _validate_consumers(conf: Dict[str, Any]) -> None:
             )
 
 
+def _validate_orderflow(conf: Dict[str, Any]) -> None:
+    if conf.get("exchange", {}).get("use_public_trades"):
+        if "orderflow" not in conf:
+            raise ConfigurationError(
+                "Orderflow is a required configuration key when using public trades."
+            )
+
+
 def _strategy_settings(conf: Dict[str, Any]) -> None:
     process_deprecated_setting(conf, None, "use_sell_signal", None, "use_exit_signal")
     process_deprecated_setting(conf, None, "sell_profit_only", None, "exit_profit_only")

freqtrade/configuration/configuration.py

@@ -2,6 +2,7 @@
 This module contains the configuration class
 """
 
+import ast
 import logging
 import warnings
 from copy import deepcopy
@@ -301,7 +302,7 @@ class Configuration:
 
         # Edge section:
         if "stoploss_range" in self.args and self.args["stoploss_range"]:
-            txt_range = eval(self.args["stoploss_range"])
+            txt_range = ast.literal_eval(self.args["stoploss_range"])
             config["edge"].update({"stoploss_range_min": txt_range[0]})
             config["edge"].update({"stoploss_range_max": txt_range[1]})
             config["edge"].update({"stoploss_range_step": txt_range[2]})

freqtrade/configuration/directory_operations.py

@@ -38,7 +38,7 @@ def chown_user_directory(directory: Path) -> None:
     """
     if running_in_docker():
         try:
-            import subprocess
+            import subprocess  # noqa: S404
 
             subprocess.check_output(["sudo", "chown", "-R", "ftuser:", str(directory.resolve())])
         except Exception:

freqtrade/constants.py

@@ -4,9 +4,9 @@
 bot constants
 """
 
-from typing import Any, Dict, List, Literal, Tuple
+from typing import Any, Dict, List, Literal, Optional, Tuple
 
-from freqtrade.enums import CandleType, PriceType, RPCMessageType
+from freqtrade.enums import CandleType, PriceType
 
 
 DOCS_LINK = "https://www.freqtrade.io/en/stable"
@@ -68,6 +68,7 @@ DEFAULT_DATAFRAME_COLUMNS = ["date", "open", "high", "low", "close", "volume"]
 # Don't modify sequence of DEFAULT_TRADES_COLUMNS
 # it has wide consequences for stored trades files
 DEFAULT_TRADES_COLUMNS = ["timestamp", "id", "type", "side", "price", "amount", "cost"]
+DEFAULT_ORDERFLOW_COLUMNS = ["level", "bid", "ask", "delta"]
 TRADES_DTYPES = {
     "timestamp": "int64",
     "id": "str",
@@ -171,585 +172,6 @@ MINIMAL_CONFIG = {
     },
 }
 
-__MESSAGE_TYPE_DICT: Dict[str, Dict[str, str]] = {x: {"type": "object"} for x in RPCMessageType}
-
-# Required json-schema for user specified config
-CONF_SCHEMA = {
-    "type": "object",
-    "properties": {
-        "max_open_trades": {"type": ["integer", "number"], "minimum": -1},
-        "new_pairs_days": {"type": "integer", "default": 30},
-        "timeframe": {"type": "string"},
-        "stake_currency": {"type": "string"},
-        "stake_amount": {
-            "type": ["number", "string"],
-            "minimum": 0.0001,
-            "pattern": UNLIMITED_STAKE_AMOUNT,
-        },
-        "tradable_balance_ratio": {"type": "number", "minimum": 0.0, "maximum": 1, "default": 0.99},
-        "available_capital": {
-            "type": "number",
-            "minimum": 0,
-        },
-        "amend_last_stake_amount": {"type": "boolean", "default": False},
-        "last_stake_amount_min_ratio": {
-            "type": "number",
-            "minimum": 0.0,
-            "maximum": 1.0,
-            "default": 0.5,
-        },
-        "fiat_display_currency": {"type": "string", "enum": SUPPORTED_FIAT},
-        "dry_run": {"type": "boolean"},
-        "dry_run_wallet": {"type": "number", "default": DRY_RUN_WALLET},
-        "cancel_open_orders_on_exit": {"type": "boolean", "default": False},
-        "process_only_new_candles": {"type": "boolean"},
-        "minimal_roi": {
-            "type": "object",
-            "patternProperties": {"^[0-9.]+$": {"type": "number"}},
-        },
-        "amount_reserve_percent": {"type": "number", "minimum": 0.0, "maximum": 0.5},
-        "stoploss": {"type": "number", "maximum": 0, "exclusiveMaximum": True},
-        "trailing_stop": {"type": "boolean"},
-        "trailing_stop_positive": {"type": "number", "minimum": 0, "maximum": 1},
-        "trailing_stop_positive_offset": {"type": "number", "minimum": 0, "maximum": 1},
-        "trailing_only_offset_is_reached": {"type": "boolean"},
-        "use_exit_signal": {"type": "boolean"},
-        "exit_profit_only": {"type": "boolean"},
-        "exit_profit_offset": {"type": "number"},
-        "fee": {"type": "number", "minimum": 0, "maximum": 0.1},
-        "ignore_roi_if_entry_signal": {"type": "boolean"},
-        "ignore_buying_expired_candle_after": {"type": "number"},
-        "trading_mode": {"type": "string", "enum": TRADING_MODES},
-        "margin_mode": {"type": "string", "enum": MARGIN_MODES},
-        "reduce_df_footprint": {"type": "boolean", "default": False},
-        "minimum_trade_amount": {"type": "number", "default": 10},
-        "targeted_trade_amount": {"type": "number", "default": 20},
-        "lookahead_analysis_exportfilename": {"type": "string"},
-        "startup_candle": {
-            "type": "array",
-            "uniqueItems": True,
-            "default": [199, 399, 499, 999, 1999],
-        },
-        "liquidation_buffer": {"type": "number", "minimum": 0.0, "maximum": 0.99},
-        "backtest_breakdown": {
-            "type": "array",
-            "items": {"type": "string", "enum": BACKTEST_BREAKDOWNS},
-        },
-        "bot_name": {"type": "string"},
-        "unfilledtimeout": {
-            "type": "object",
-            "properties": {
-                "entry": {"type": "number", "minimum": 1},
-                "exit": {"type": "number", "minimum": 1},
-                "exit_timeout_count": {"type": "number", "minimum": 0, "default": 0},
-                "unit": {"type": "string", "enum": TIMEOUT_UNITS, "default": "minutes"},
-            },
-        },
-        "entry_pricing": {
-            "type": "object",
-            "properties": {
-                "price_last_balance": {
-                    "type": "number",
-                    "minimum": 0,
-                    "maximum": 1,
-                    "exclusiveMaximum": False,
-                },
-                "price_side": {"type": "string", "enum": PRICING_SIDES, "default": "same"},
-                "use_order_book": {"type": "boolean"},
-                "order_book_top": {
-                    "type": "integer",
-                    "minimum": 1,
-                    "maximum": 50,
-                },
-                "check_depth_of_market": {
-                    "type": "object",
-                    "properties": {
-                        "enabled": {"type": "boolean"},
-                        "bids_to_ask_delta": {"type": "number", "minimum": 0},
-                    },
-                },
-            },
-            "required": ["price_side"],
-        },
-        "exit_pricing": {
-            "type": "object",
-            "properties": {
-                "price_side": {"type": "string", "enum": PRICING_SIDES, "default": "same"},
-                "price_last_balance": {
-                    "type": "number",
-                    "minimum": 0,
-                    "maximum": 1,
-                    "exclusiveMaximum": False,
-                },
-                "use_order_book": {"type": "boolean"},
-                "order_book_top": {
-                    "type": "integer",
-                    "minimum": 1,
-                    "maximum": 50,
-                },
-            },
-            "required": ["price_side"],
-        },
-        "custom_price_max_distance_ratio": {"type": "number", "minimum": 0.0},
-        "order_types": {
-            "type": "object",
-            "properties": {
-                "entry": {"type": "string", "enum": ORDERTYPE_POSSIBILITIES},
-                "exit": {"type": "string", "enum": ORDERTYPE_POSSIBILITIES},
-                "force_exit": {"type": "string", "enum": ORDERTYPE_POSSIBILITIES},
-                "force_entry": {"type": "string", "enum": ORDERTYPE_POSSIBILITIES},
-                "emergency_exit": {
-                    "type": "string",
-                    "enum": ORDERTYPE_POSSIBILITIES,
-                    "default": "market",
-                },
-                "stoploss": {"type": "string", "enum": ORDERTYPE_POSSIBILITIES},
-                "stoploss_on_exchange": {"type": "boolean"},
-                "stoploss_price_type": {"type": "string", "enum": STOPLOSS_PRICE_TYPES},
-                "stoploss_on_exchange_interval": {"type": "number"},
-                "stoploss_on_exchange_limit_ratio": {
-                    "type": "number",
-                    "minimum": 0.0,
-                    "maximum": 1.0,
-                },
-            },
-            "required": ["entry", "exit", "stoploss", "stoploss_on_exchange"],
-        },
-        "order_time_in_force": {
-            "type": "object",
-            "properties": {
-                "entry": {"type": "string", "enum": ORDERTIF_POSSIBILITIES},
-                "exit": {"type": "string", "enum": ORDERTIF_POSSIBILITIES},
-            },
-            "required": REQUIRED_ORDERTIF,
-        },
-        "coingecko": {
-            "type": "object",
-            "properties": {
-                "is_demo": {"type": "boolean", "default": True},
-                "api_key": {"type": "string"},
-            },
-            "required": ["is_demo", "api_key"],
-        },
-        "exchange": {"$ref": "#/definitions/exchange"},
-        "edge": {"$ref": "#/definitions/edge"},
-        "freqai": {"$ref": "#/definitions/freqai"},
-        "external_message_consumer": {"$ref": "#/definitions/external_message_consumer"},
-        "experimental": {
-            "type": "object",
-            "properties": {"block_bad_exchanges": {"type": "boolean"}},
-        },
-        "pairlists": {
-            "type": "array",
-            "items": {
-                "type": "object",
-                "properties": {
-                    "method": {"type": "string", "enum": AVAILABLE_PAIRLISTS},
-                },
-                "required": ["method"],
-            },
-        },
-        "protections": {
-            "type": "array",
-            "items": {
-                "type": "object",
-                "properties": {
-                    "method": {"type": "string", "enum": AVAILABLE_PROTECTIONS},
-                    "stop_duration": {"type": "number", "minimum": 0.0},
-                    "stop_duration_candles": {"type": "number", "minimum": 0},
-                    "trade_limit": {"type": "number", "minimum": 1},
-                    "lookback_period": {"type": "number", "minimum": 1},
-                    "lookback_period_candles": {"type": "number", "minimum": 1},
-                },
-                "required": ["method"],
-            },
-        },
-        "telegram": {
-            "type": "object",
-            "properties": {
-                "enabled": {"type": "boolean"},
-                "token": {"type": "string"},
-                "chat_id": {"type": "string"},
-                "allow_custom_messages": {"type": "boolean", "default": True},
-                "balance_dust_level": {"type": "number", "minimum": 0.0},
-                "notification_settings": {
-                    "type": "object",
-                    "default": {},
-                    "properties": {
-                        "status": {"type": "string", "enum": TELEGRAM_SETTING_OPTIONS},
-                        "warning": {"type": "string", "enum": TELEGRAM_SETTING_OPTIONS},
-                        "startup": {"type": "string", "enum": TELEGRAM_SETTING_OPTIONS},
-                        "entry": {"type": "string", "enum": TELEGRAM_SETTING_OPTIONS},
-                        "entry_fill": {
-                            "type": "string",
-                            "enum": TELEGRAM_SETTING_OPTIONS,
-                            "default": "off",
-                        },
-                        "entry_cancel": {
-                            "type": "string",
-                            "enum": TELEGRAM_SETTING_OPTIONS,
-                        },
-                        "exit": {
-                            "type": ["string", "object"],
-                            "additionalProperties": {
-                                "type": "string",
-                                "enum": TELEGRAM_SETTING_OPTIONS,
-                            },
-                        },
-                        "exit_fill": {
-                            "type": "string",
-                            "enum": TELEGRAM_SETTING_OPTIONS,
-                            "default": "on",
-                        },
-                        "exit_cancel": {"type": "string", "enum": TELEGRAM_SETTING_OPTIONS},
-                        "protection_trigger": {
-                            "type": "string",
-                            "enum": TELEGRAM_SETTING_OPTIONS,
-                            "default": "on",
-                        },
-                        "protection_trigger_global": {
-                            "type": "string",
-                            "enum": TELEGRAM_SETTING_OPTIONS,
-                            "default": "on",
-                        },
-                        "show_candle": {
-                            "type": "string",
-                            "enum": ["off", "ohlc"],
-                            "default": "off",
-                        },
-                        "strategy_msg": {
-                            "type": "string",
-                            "enum": TELEGRAM_SETTING_OPTIONS,
-                            "default": "on",
-                        },
-                    },
-                },
-                "reload": {"type": "boolean"},
-            },
-            "required": ["enabled", "token", "chat_id"],
-        },
-        "webhook": {
-            "type": "object",
-            "properties": {
-                "enabled": {"type": "boolean"},
-                "url": {"type": "string"},
-                "format": {"type": "string", "enum": WEBHOOK_FORMAT_OPTIONS, "default": "form"},
-                "retries": {"type": "integer", "minimum": 0},
-                "retry_delay": {"type": "number", "minimum": 0},
-                **__MESSAGE_TYPE_DICT,
-                # **{x: {'type': 'object'} for x in RPCMessageType},
-                # Below -> Deprecated
-                "webhookentry": {"type": "object"},
-                "webhookentrycancel": {"type": "object"},
-                "webhookentryfill": {"type": "object"},
-                "webhookexit": {"type": "object"},
-                "webhookexitcancel": {"type": "object"},
-                "webhookexitfill": {"type": "object"},
-                "webhookstatus": {"type": "object"},
-            },
-        },
-        "discord": {
-            "type": "object",
-            "properties": {
-                "enabled": {"type": "boolean"},
-                "webhook_url": {"type": "string"},
-                "exit_fill": {
-                    "type": "array",
-                    "items": {"type": "object"},
-                    "default": [
-                        {"Trade ID": "{trade_id}"},
-                        {"Exchange": "{exchange}"},
-                        {"Pair": "{pair}"},
-                        {"Direction": "{direction}"},
-                        {"Open rate": "{open_rate}"},
-                        {"Close rate": "{close_rate}"},
-                        {"Amount": "{amount}"},
-                        {"Open date": "{open_date:%Y-%m-%d %H:%M:%S}"},
-                        {"Close date": "{close_date:%Y-%m-%d %H:%M:%S}"},
-                        {"Profit": "{profit_amount} {stake_currency}"},
-                        {"Profitability": "{profit_ratio:.2%}"},
-                        {"Enter tag": "{enter_tag}"},
-                        {"Exit Reason": "{exit_reason}"},
-                        {"Strategy": "{strategy}"},
-                        {"Timeframe": "{timeframe}"},
-                    ],
-                },
-                "entry_fill": {
-                    "type": "array",
-                    "items": {"type": "object"},
-                    "default": [
-                        {"Trade ID": "{trade_id}"},
-                        {"Exchange": "{exchange}"},
-                        {"Pair": "{pair}"},
-                        {"Direction": "{direction}"},
-                        {"Open rate": "{open_rate}"},
-                        {"Amount": "{amount}"},
-                        {"Open date": "{open_date:%Y-%m-%d %H:%M:%S}"},
-                        {"Enter tag": "{enter_tag}"},
-                        {"Strategy": "{strategy} {timeframe}"},
-                    ],
-                },
-            },
-        },
-        "api_server": {
-            "type": "object",
-            "properties": {
-                "enabled": {"type": "boolean"},
-                "listen_ip_address": {"format": "ipv4"},
-                "listen_port": {"type": "integer", "minimum": 1024, "maximum": 65535},
-                "username": {"type": "string"},
-                "password": {"type": "string"},
-                "ws_token": {"type": ["string", "array"], "items": {"type": "string"}},
-                "jwt_secret_key": {"type": "string"},
-                "CORS_origins": {"type": "array", "items": {"type": "string"}},
-                "verbosity": {"type": "string", "enum": ["error", "info"]},
-            },
-            "required": ["enabled", "listen_ip_address", "listen_port", "username", "password"],
-        },
-        "db_url": {"type": "string"},
-        "export": {"type": "string", "enum": EXPORT_OPTIONS, "default": "trades"},
-        "disableparamexport": {"type": "boolean"},
-        "initial_state": {"type": "string", "enum": ["running", "stopped"]},
-        "force_entry_enable": {"type": "boolean"},
-        "disable_dataframe_checks": {"type": "boolean"},
-        "internals": {
-            "type": "object",
-            "default": {},
-            "properties": {
-                "process_throttle_secs": {"type": "integer"},
-                "interval": {"type": "integer"},
-                "sd_notify": {"type": "boolean"},
-            },
-        },
-        "dataformat_ohlcv": {
-            "type": "string",
-            "enum": AVAILABLE_DATAHANDLERS,
-            "default": "feather",
-        },
-        "dataformat_trades": {
-            "type": "string",
-            "enum": AVAILABLE_DATAHANDLERS,
-            "default": "feather",
-        },
-        "position_adjustment_enable": {"type": "boolean"},
-        "max_entry_position_adjustment": {"type": ["integer", "number"], "minimum": -1},
-    },
-    "definitions": {
-        "exchange": {
-            "type": "object",
-            "properties": {
-                "name": {"type": "string"},
-                "key": {"type": "string", "default": ""},
-                "secret": {"type": "string", "default": ""},
-                "password": {"type": "string", "default": ""},
-                "uid": {"type": "string"},
-                "pair_whitelist": {
-                    "type": "array",
-                    "items": {
-                        "type": "string",
-                    },
-                    "uniqueItems": True,
-                },
-                "pair_blacklist": {
-                    "type": "array",
-                    "items": {
-                        "type": "string",
-                    },
-                    "uniqueItems": True,
-                },
-                "unknown_fee_rate": {"type": "number"},
-                "outdated_offset": {"type": "integer", "minimum": 1},
-                "markets_refresh_interval": {"type": "integer"},
-                "ccxt_config": {"type": "object"},
-                "ccxt_async_config": {"type": "object"},
-            },
-            "required": ["name"],
-        },
-        "edge": {
-            "type": "object",
-            "properties": {
-                "enabled": {"type": "boolean"},
-                "process_throttle_secs": {"type": "integer", "minimum": 600},
-                "calculate_since_number_of_days": {"type": "integer"},
-                "allowed_risk": {"type": "number"},
-                "stoploss_range_min": {"type": "number"},
-                "stoploss_range_max": {"type": "number"},
-                "stoploss_range_step": {"type": "number"},
-                "minimum_winrate": {"type": "number"},
-                "minimum_expectancy": {"type": "number"},
-                "min_trade_number": {"type": "number"},
-                "max_trade_duration_minute": {"type": "integer"},
-                "remove_pumps": {"type": "boolean"},
-            },
-            "required": ["process_throttle_secs", "allowed_risk"],
-        },
-        "external_message_consumer": {
-            "type": "object",
-            "properties": {
-                "enabled": {"type": "boolean", "default": False},
-                "producers": {
-                    "type": "array",
-                    "items": {
-                        "type": "object",
-                        "properties": {
-                            "name": {"type": "string"},
-                            "host": {"type": "string"},
-                            "port": {
-                                "type": "integer",
-                                "default": 8080,
-                                "minimum": 0,
-                                "maximum": 65535,
-                            },
-                            "secure": {"type": "boolean", "default": False},
-                            "ws_token": {"type": "string"},
-                        },
-                        "required": ["name", "host", "ws_token"],
-                    },
-                },
-                "wait_timeout": {"type": "integer", "minimum": 0},
-                "sleep_time": {"type": "integer", "minimum": 0},
-                "ping_timeout": {"type": "integer", "minimum": 0},
-                "remove_entry_exit_signals": {"type": "boolean", "default": False},
-                "initial_candle_limit": {
-                    "type": "integer",
-                    "minimum": 0,
-                    "maximum": 1500,
-                    "default": 1500,
-                },
-                "message_size_limit": {  # In megabytes
-                    "type": "integer",
-                    "minimum": 1,
-                    "maximum": 20,
-                    "default": 8,
-                },
-            },
-            "required": ["producers"],
-        },
-        "freqai": {
-            "type": "object",
-            "properties": {
-                "enabled": {"type": "boolean", "default": False},
-                "keras": {"type": "boolean", "default": False},
-                "write_metrics_to_disk": {"type": "boolean", "default": False},
-                "purge_old_models": {"type": ["boolean", "number"], "default": 2},
-                "conv_width": {"type": "integer", "default": 1},
-                "train_period_days": {"type": "integer", "default": 0},
-                "backtest_period_days": {"type": "number", "default": 7},
-                "identifier": {"type": "string", "default": "example"},
-                "feature_parameters": {
-                    "type": "object",
-                    "properties": {
-                        "include_corr_pairlist": {"type": "array"},
-                        "include_timeframes": {"type": "array"},
-                        "label_period_candles": {"type": "integer"},
-                        "include_shifted_candles": {"type": "integer", "default": 0},
-                        "DI_threshold": {"type": "number", "default": 0},
-                        "weight_factor": {"type": "number", "default": 0},
-                        "principal_component_analysis": {"type": "boolean", "default": False},
-                        "use_SVM_to_remove_outliers": {"type": "boolean", "default": False},
-                        "plot_feature_importances": {"type": "integer", "default": 0},
-                        "svm_params": {
-                            "type": "object",
-                            "properties": {
-                                "shuffle": {"type": "boolean", "default": False},
-                                "nu": {"type": "number", "default": 0.1},
-                            },
-                        },
-                        "shuffle_after_split": {"type": "boolean", "default": False},
-                        "buffer_train_data_candles": {"type": "integer", "default": 0},
-                    },
-                    "required": [
-                        "include_timeframes",
-                        "include_corr_pairlist",
-                    ],
-                },
-                "data_split_parameters": {
-                    "type": "object",
-                    "properties": {
-                        "test_size": {"type": "number"},
-                        "random_state": {"type": "integer"},
-                        "shuffle": {"type": "boolean", "default": False},
-                    },
-                },
-                "model_training_parameters": {"type": "object"},
-                "rl_config": {
-                    "type": "object",
-                    "properties": {
-                        "drop_ohlc_from_features": {"type": "boolean", "default": False},
-                        "train_cycles": {"type": "integer"},
-                        "max_trade_duration_candles": {"type": "integer"},
-                        "add_state_info": {"type": "boolean", "default": False},
-                        "max_training_drawdown_pct": {"type": "number", "default": 0.02},
-                        "cpu_count": {"type": "integer", "default": 1},
-                        "model_type": {"type": "string", "default": "PPO"},
-                        "policy_type": {"type": "string", "default": "MlpPolicy"},
-                        "net_arch": {"type": "array", "default": [128, 128]},
-                        "randomize_starting_position": {"type": "boolean", "default": False},
-                        "progress_bar": {"type": "boolean", "default": True},
-                        "model_reward_parameters": {
-                            "type": "object",
-                            "properties": {
-                                "rr": {"type": "number", "default": 1},
-                                "profit_aim": {"type": "number", "default": 0.025},
-                            },
-                        },
-                    },
-                },
-            },
-            "required": [
-                "enabled",
-                "train_period_days",
-                "backtest_period_days",
-                "identifier",
-                "feature_parameters",
-                "data_split_parameters",
-            ],
-        },
-    },
-}
-
-SCHEMA_TRADE_REQUIRED = [
-    "exchange",
-    "timeframe",
-    "max_open_trades",
-    "stake_currency",
-    "stake_amount",
-    "tradable_balance_ratio",
-    "last_stake_amount_min_ratio",
-    "dry_run",
-    "dry_run_wallet",
-    "exit_pricing",
-    "entry_pricing",
-    "stoploss",
-    "minimal_roi",
-    "internals",
-    "dataformat_ohlcv",
-    "dataformat_trades",
-]
-
-SCHEMA_BACKTEST_REQUIRED = [
-    "exchange",
-    "stake_currency",
-    "stake_amount",
-    "dry_run_wallet",
-    "dataformat_ohlcv",
-    "dataformat_trades",
-]
-SCHEMA_BACKTEST_REQUIRED_FINAL = SCHEMA_BACKTEST_REQUIRED + [
-    "stoploss",
-    "minimal_roi",
-    "max_open_trades",
-]
-
-SCHEMA_MINIMAL_REQUIRED = [
-    "exchange",
-    "dry_run",
-    "dataformat_ohlcv",
-    "dataformat_trades",
-]
-SCHEMA_MINIMAL_WEBSERVER = SCHEMA_MINIMAL_REQUIRED + [
-    "api_server",
-]
 
 CANCEL_REASON = {
     "TIMEOUT": "cancelled due to timeout",
@@ -770,6 +192,9 @@ ListPairsWithTimeframes = List[PairWithTimeframe]
 
 # Type for trades list
 TradeList = List[List]
+# ticks, pair, timeframe, CandleType
+TickWithTimeframe = Tuple[str, str, CandleType, Optional[int], Optional[int]]
+ListTicksWithTimeframes = List[TickWithTimeframe]
 
 LongShort = Literal["long", "short"]
 EntryExit = Literal["entry", "exit"]

freqtrade/data/__init__.py

@@ -2,5 +2,8 @@
 Module to handle data operations for freqtrade
 """
 
+from freqtrade.data import converter
+
+
 # limit what's imported when using `from freqtrade.data import *`
 __all__ = ["converter"]

freqtrade/data/btanalysis.py

@@ -185,7 +185,7 @@ def load_and_merge_backtest_result(strategy_name: str, filename: Path, results:
     """
     bt_data = load_backtest_stats(filename)
     k: Literal["metadata", "strategy"]
-    for k in ("metadata", "strategy"):  # type: ignore
+    for k in ("metadata", "strategy"):
         results[k][strategy_name] = bt_data[k][strategy_name]
     results["metadata"][strategy_name]["filename"] = filename.stem
     comparison = bt_data["strategy_comparison"]

freqtrade/data/converter/__init__.py

@@ -8,6 +8,7 @@ from freqtrade.data.converter.converter import (
     trim_dataframe,
     trim_dataframes,
 )
+from freqtrade.data.converter.orderflow import populate_dataframe_with_trades
 from freqtrade.data.converter.trade_converter import (
     convert_trades_format,
     convert_trades_to_ohlcv,
@@ -30,6 +31,7 @@ __all__ = [
     "trim_dataframes",
     "convert_trades_format",
     "convert_trades_to_ohlcv",
+    "populate_dataframe_with_trades",
     "trades_convert_types",
     "trades_df_remove_duplicates",
     "trades_dict_to_list",

freqtrade/data/converter/orderflow.py

@@ -0,0 +1,295 @@
+"""
+Functions to convert orderflow data from public_trades
+"""
+
+import logging
+import time
+import typing
+from collections import OrderedDict
+from datetime import datetime
+from typing import Tuple
+
+import numpy as np
+import pandas as pd
+
+from freqtrade.constants import DEFAULT_ORDERFLOW_COLUMNS
+from freqtrade.enums import RunMode
+from freqtrade.exceptions import DependencyException
+
+
+logger = logging.getLogger(__name__)
+
+
+def _init_dataframe_with_trades_columns(dataframe: pd.DataFrame):
+    """
+    Populates a dataframe with trades columns
+    :param dataframe: Dataframe to populate
+    """
+    # Initialize columns with appropriate dtypes
+    dataframe["trades"] = np.nan
+    dataframe["orderflow"] = np.nan
+    dataframe["imbalances"] = np.nan
+    dataframe["stacked_imbalances_bid"] = np.nan
+    dataframe["stacked_imbalances_ask"] = np.nan
+    dataframe["max_delta"] = np.nan
+    dataframe["min_delta"] = np.nan
+    dataframe["bid"] = np.nan
+    dataframe["ask"] = np.nan
+    dataframe["delta"] = np.nan
+    dataframe["total_trades"] = np.nan
+
+    # Ensure the 'trades' column is of object type
+    dataframe["trades"] = dataframe["trades"].astype(object)
+    dataframe["orderflow"] = dataframe["orderflow"].astype(object)
+    dataframe["imbalances"] = dataframe["imbalances"].astype(object)
+    dataframe["stacked_imbalances_bid"] = dataframe["stacked_imbalances_bid"].astype(object)
+    dataframe["stacked_imbalances_ask"] = dataframe["stacked_imbalances_ask"].astype(object)
+
+
+def _calculate_ohlcv_candle_start_and_end(df: pd.DataFrame, timeframe: str):
+    from freqtrade.exchange import timeframe_to_next_date, timeframe_to_resample_freq
+
+    timeframe_frequency = timeframe_to_resample_freq(timeframe)
+    # calculate ohlcv candle start and end
+    if df is not None and not df.empty:
+        df["datetime"] = pd.to_datetime(df["date"], unit="ms")
+        df["candle_start"] = df["datetime"].dt.floor(timeframe_frequency)
+        # used in _now_is_time_to_refresh_trades
+        df["candle_end"] = df["candle_start"].apply(
+            lambda candle_start: timeframe_to_next_date(timeframe, candle_start)
+        )
+        df.drop(columns=["datetime"], inplace=True)
+
+
+def populate_dataframe_with_trades(
+    cached_grouped_trades: OrderedDict[Tuple[datetime, datetime], pd.DataFrame],
+    config,
+    dataframe: pd.DataFrame,
+    trades: pd.DataFrame,
+) -> Tuple[pd.DataFrame, OrderedDict[Tuple[datetime, datetime], pd.DataFrame]]:
+    """
+    Populates a dataframe with trades
+    :param dataframe: Dataframe to populate
+    :param trades: Trades to populate with
+    :return: Dataframe with trades populated
+    """
+    timeframe = config["timeframe"]
+    config_orderflow = config["orderflow"]
+
+    # create columns for trades
+    _init_dataframe_with_trades_columns(dataframe)
+
+    try:
+        start_time = time.time()
+        # calculate ohlcv candle start and end
+        _calculate_ohlcv_candle_start_and_end(trades, timeframe)
+
+        # get date of earliest max_candles candle
+        max_candles = config_orderflow["max_candles"]
+        start_date = dataframe.tail(max_candles).date.iat[0]
+        # slice of trades that are before current ohlcv candles to make groupby faster
+        trades = trades.loc[trades.candle_start >= start_date]
+        trades.reset_index(inplace=True, drop=True)
+
+        # group trades by candle start
+        trades_grouped_by_candle_start = trades.groupby("candle_start", group_keys=False)
+        # Create Series to hold complex data
+        trades_series = pd.Series(index=dataframe.index, dtype=object)
+        orderflow_series = pd.Series(index=dataframe.index, dtype=object)
+        imbalances_series = pd.Series(index=dataframe.index, dtype=object)
+        stacked_imbalances_bid_series = pd.Series(index=dataframe.index, dtype=object)
+        stacked_imbalances_ask_series = pd.Series(index=dataframe.index, dtype=object)
+
+        trades_grouped_by_candle_start = trades.groupby("candle_start", group_keys=False)
+        for candle_start, trades_grouped_df in trades_grouped_by_candle_start:
+            is_between = candle_start == dataframe["date"]
+            if is_between.any():
+                from freqtrade.exchange import timeframe_to_next_date
+
+                candle_next = timeframe_to_next_date(timeframe, typing.cast(datetime, candle_start))
+                if candle_next not in trades_grouped_by_candle_start.groups:
+                    logger.warning(
+                        f"candle at {candle_start} with {len(trades_grouped_df)} trades "
+                        f"might be unfinished, because no finished trades at {candle_next}"
+                    )
+
+                indices = dataframe.index[is_between].tolist()
+                # Add trades to each candle
+                trades_series.loc[indices] = [
+                    trades_grouped_df.drop(columns=["candle_start", "candle_end"]).to_dict(
+                        orient="records"
+                    )
+                ]
+                # Use caching mechanism
+                if (candle_start, candle_next) in cached_grouped_trades:
+                    cache_entry = cached_grouped_trades[
+                        (typing.cast(datetime, candle_start), candle_next)
+                    ]
+                    # dataframe.loc[is_between] = cache_entry # doesn't take, so we need workaround:
+                    # Create a dictionary of the column values to be assigned
+                    update_dict = {c: cache_entry[c].iat[0] for c in cache_entry.columns}
+                    # Assign the values using the update_dict
+                    dataframe.loc[is_between, update_dict.keys()] = pd.DataFrame(
+                        [update_dict], index=dataframe.loc[is_between].index
+                    )
+                    continue
+
+                # Calculate orderflow for each candle
+                orderflow = trades_to_volumeprofile_with_total_delta_bid_ask(
+                    trades_grouped_df, scale=config_orderflow["scale"]
+                )
+                orderflow_series.loc[indices] = [orderflow.to_dict(orient="index")]
+                # Calculate imbalances for each candle's orderflow
+                imbalances = trades_orderflow_to_imbalances(
+                    orderflow,
+                    imbalance_ratio=config_orderflow["imbalance_ratio"],
+                    imbalance_volume=config_orderflow["imbalance_volume"],
+                )
+                imbalances_series.loc[indices] = [imbalances.to_dict(orient="index")]
+
+                stacked_imbalance_range = config_orderflow["stacked_imbalance_range"]
+                stacked_imbalances_bid_series.loc[indices] = [
+                    stacked_imbalance_bid(
+                        imbalances, stacked_imbalance_range=stacked_imbalance_range
+                    )
+                ]
+                stacked_imbalances_ask_series.loc[indices] = [
+                    stacked_imbalance_ask(
+                        imbalances, stacked_imbalance_range=stacked_imbalance_range
+                    )
+                ]
+
+                bid = np.where(
+                    trades_grouped_df["side"].str.contains("sell"), trades_grouped_df["amount"], 0
+                )
+
+                ask = np.where(
+                    trades_grouped_df["side"].str.contains("buy"), trades_grouped_df["amount"], 0
+                )
+                deltas_per_trade = ask - bid
+                min_delta = deltas_per_trade.cumsum().min()
+                max_delta = deltas_per_trade.cumsum().max()
+                dataframe.loc[indices, "max_delta"] = max_delta
+                dataframe.loc[indices, "min_delta"] = min_delta
+
+                dataframe.loc[indices, "bid"] = bid.sum()
+                dataframe.loc[indices, "ask"] = ask.sum()
+                dataframe.loc[indices, "delta"] = (
+                    dataframe.loc[indices, "ask"] - dataframe.loc[indices, "bid"]
+                )
+                dataframe.loc[indices, "total_trades"] = len(trades_grouped_df)
+
+                # Cache the result
+                cached_grouped_trades[(typing.cast(datetime, candle_start), candle_next)] = (
+                    dataframe.loc[is_between].copy()
+                )
+
+                # Maintain cache size
+                if (
+                    config.get("runmode") in (RunMode.DRY_RUN, RunMode.LIVE)
+                    and len(cached_grouped_trades) > config_orderflow["cache_size"]
+                ):
+                    cached_grouped_trades.popitem(last=False)
+            else:
+                logger.debug(f"Found NO candles for trades starting with {candle_start}")
+        logger.debug(f"trades.groups_keys in {time.time() - start_time} seconds")
+
+        # Merge the complex data Series back into the DataFrame
+        dataframe["trades"] = trades_series
+        dataframe["orderflow"] = orderflow_series
+        dataframe["imbalances"] = imbalances_series
+        dataframe["stacked_imbalances_bid"] = stacked_imbalances_bid_series
+        dataframe["stacked_imbalances_ask"] = stacked_imbalances_ask_series
+
+    except Exception as e:
+        logger.exception("Error populating dataframe with trades")
+        raise DependencyException(e)
+
+    return dataframe, cached_grouped_trades
+
+
+def trades_to_volumeprofile_with_total_delta_bid_ask(
+    trades: pd.DataFrame, scale: float
+) -> pd.DataFrame:
+    """
+    :param trades: dataframe
+    :param scale: scale aka bin size e.g. 0.5
+    :return: trades binned to levels according to scale aka orderflow
+    """
+    df = pd.DataFrame([], columns=DEFAULT_ORDERFLOW_COLUMNS)
+    # create bid, ask where side is sell or buy
+    df["bid_amount"] = np.where(trades["side"].str.contains("sell"), trades["amount"], 0)
+    df["ask_amount"] = np.where(trades["side"].str.contains("buy"), trades["amount"], 0)
+    df["bid"] = np.where(trades["side"].str.contains("sell"), 1, 0)
+    df["ask"] = np.where(trades["side"].str.contains("buy"), 1, 0)
+    # round the prices to the nearest multiple of the scale
+    df["price"] = ((trades["price"] / scale).round() * scale).astype("float64").values
+    if df.empty:
+        df["total"] = np.nan
+        df["delta"] = np.nan
+        return df
+
+    df["delta"] = df["ask_amount"] - df["bid_amount"]
+    df["total_volume"] = df["ask_amount"] + df["bid_amount"]
+    df["total_trades"] = df["ask"] + df["bid"]
+
+    # group to bins aka apply scale
+    df = df.groupby("price").sum(numeric_only=True)
+    return df
+
+
+def trades_orderflow_to_imbalances(df: pd.DataFrame, imbalance_ratio: int, imbalance_volume: int):
+    """
+    :param df: dataframes with bid and ask
+    :param imbalance_ratio: imbalance_ratio e.g. 3
+    :param imbalance_volume: imbalance volume e.g. 10
+    :return: dataframe with bid and ask imbalance
+    """
+    bid = df.bid
+    # compares bid and ask diagonally
+    ask = df.ask.shift(-1)
+    bid_imbalance = (bid / ask) > (imbalance_ratio)
+    # overwrite bid_imbalance with False if volume is not big enough
+    bid_imbalance_filtered = np.where(df.total_volume < imbalance_volume, False, bid_imbalance)
+    ask_imbalance = (ask / bid) > (imbalance_ratio)
+    # overwrite ask_imbalance with False if volume is not big enough
+    ask_imbalance_filtered = np.where(df.total_volume < imbalance_volume, False, ask_imbalance)
+    dataframe = pd.DataFrame(
+        {"bid_imbalance": bid_imbalance_filtered, "ask_imbalance": ask_imbalance_filtered},
+        index=df.index,
+    )
+
+    return dataframe
+
+
+def stacked_imbalance(
+    df: pd.DataFrame, label: str, stacked_imbalance_range: int, should_reverse: bool
+):
+    """
+    y * (y.groupby((y != y.shift()).cumsum()).cumcount() + 1)
+    https://stackoverflow.com/questions/27626542/counting-consecutive-positive-values-in-python-pandas-array
+    """
+    imbalance = df[f"{label}_imbalance"]
+    int_series = pd.Series(np.where(imbalance, 1, 0))
+    stacked = int_series * (
+        int_series.groupby((int_series != int_series.shift()).cumsum()).cumcount() + 1
+    )
+
+    max_stacked_imbalance_idx = stacked.index[stacked >= stacked_imbalance_range]
+    stacked_imbalance_price = np.nan
+    if not max_stacked_imbalance_idx.empty:
+        idx = (
+            max_stacked_imbalance_idx[0]
+            if not should_reverse
+            else np.flipud(max_stacked_imbalance_idx)[0]
+        )
+        stacked_imbalance_price = imbalance.index[idx]
+    return stacked_imbalance_price
+
+
+def stacked_imbalance_ask(df: pd.DataFrame, stacked_imbalance_range: int):
+    return stacked_imbalance(df, "ask", stacked_imbalance_range, should_reverse=True)
+
+
+def stacked_imbalance_bid(df: pd.DataFrame, stacked_imbalance_range: int):
+    return stacked_imbalance(df, "bid", stacked_imbalance_range, should_reverse=False)

freqtrade/data/dataprovider.py

@@ -19,8 +19,8 @@ from freqtrade.constants import (
     ListPairsWithTimeframes,
     PairWithTimeframe,
 )
-from freqtrade.data.history import load_pair_history
-from freqtrade.enums import CandleType, RPCMessageType, RunMode
+from freqtrade.data.history import get_datahandler, load_pair_history
+from freqtrade.enums import CandleType, RPCMessageType, RunMode, TradingMode
 from freqtrade.exceptions import ExchangeError, OperationalException
 from freqtrade.exchange import Exchange, timeframe_to_prev_date, timeframe_to_seconds
 from freqtrade.exchange.types import OrderBook
@@ -445,7 +445,20 @@ class DataProvider:
         if self._exchange is None:
             raise OperationalException(NO_EXCHANGE_EXCEPTION)
         final_pairs = (pairlist + helping_pairs) if helping_pairs else pairlist
+        # refresh latest ohlcv data
         self._exchange.refresh_latest_ohlcv(final_pairs)
+        # refresh latest trades data
+        self.refresh_latest_trades(pairlist)
+
+    def refresh_latest_trades(self, pairlist: ListPairsWithTimeframes) -> None:
+        """
+        Refresh latest trades data (if enabled in config)
+        """
+
+        use_public_trades = self._config.get("exchange", {}).get("use_public_trades", False)
+        if use_public_trades:
+            if self._exchange:
+                self._exchange.refresh_latest_trades(pairlist)
 
     @property
     def available_pairs(self) -> ListPairsWithTimeframes:
@@ -483,6 +496,45 @@ class DataProvider:
         else:
             return DataFrame()
 
+    def trades(
+        self, pair: str, timeframe: Optional[str] = None, copy: bool = True, candle_type: str = ""
+    ) -> DataFrame:
+        """
+        Get candle (TRADES) data for the given pair as DataFrame
+        Please use the `available_pairs` method to verify which pairs are currently cached.
+        This is not meant to be used in callbacks because of lookahead bias.
+        :param pair: pair to get the data for
+        :param timeframe: Timeframe to get data for
+        :param candle_type: '', mark, index, premiumIndex, or funding_rate
+        :param copy: copy dataframe before returning if True.
+                     Use False only for read-only operations (where the dataframe is not modified)
+        """
+        if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
+            if self._exchange is None:
+                raise OperationalException(NO_EXCHANGE_EXCEPTION)
+            _candle_type = (
+                CandleType.from_string(candle_type)
+                if candle_type != ""
+                else self._config["candle_type_def"]
+            )
+            return self._exchange.trades(
+                (pair, timeframe or self._config["timeframe"], _candle_type), copy=copy
+            )
+        elif self.runmode in (RunMode.BACKTEST, RunMode.HYPEROPT):
+            _candle_type = (
+                CandleType.from_string(candle_type)
+                if candle_type != ""
+                else self._config["candle_type_def"]
+            )
+            data_handler = get_datahandler(
+                self._config["datadir"], data_format=self._config["dataformat_trades"]
+            )
+            trades_df = data_handler.trades_load(pair, TradingMode.FUTURES)
+            return trades_df
+
+        else:
+            return DataFrame()
+
     def market(self, pair: str) -> Optional[Dict[str, Any]]:
         """
         Return market data for the pair

freqtrade/data/entryexitanalysis.py

@@ -4,7 +4,6 @@ from typing import List
 
 import joblib
 import pandas as pd
-from tabulate import tabulate
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import Config
@@ -14,6 +13,7 @@ from freqtrade.data.btanalysis import (
     load_backtest_stats,
 )
 from freqtrade.exceptions import OperationalException
+from freqtrade.util import print_df_rich_table
 
 
 logger = logging.getLogger(__name__)
@@ -307,7 +307,7 @@ def _print_table(
         if name is not None:
             print(name)
 
-        print(tabulate(data, headers="keys", tablefmt="psql", showindex=show_index))
+        print_df_rich_table(data, data.keys(), show_index=show_index)
 
 
 def process_entry_exit_reasons(config: Config):

freqtrade/data/history/history_utils.py

@@ -26,8 +26,7 @@ from freqtrade.enums import CandleType, TradingMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import Exchange
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
-from freqtrade.util import dt_ts, format_ms_time
-from freqtrade.util.datetime_helpers import dt_now
+from freqtrade.util import dt_now, dt_ts, format_ms_time, get_progress_tracker
 from freqtrade.util.migrations import migrate_data
 
 
@@ -155,11 +154,9 @@ def refresh_data(
     :param candle_type: Any of the enum CandleType (must match trading mode!)
     """
     data_handler = get_datahandler(datadir, data_format)
-    for idx, pair in enumerate(pairs):
-        process = f"{idx}/{len(pairs)}"
+    for pair in pairs:
         _download_pair_history(
             pair=pair,
-            process=process,
             timeframe=timeframe,
             datadir=datadir,
             timerange=timerange,
@@ -223,7 +220,6 @@ def _download_pair_history(
     datadir: Path,
     exchange: Exchange,
     timeframe: str = "5m",
-    process: str = "",
     new_pairs_days: int = 30,
     data_handler: Optional[IDataHandler] = None,
     timerange: Optional[TimeRange] = None,
@@ -261,7 +257,7 @@ def _download_pair_history(
         )
 
         logger.info(
-            f'({process}) - Download history data for "{pair}", {timeframe}, '
+            f'Download history data for "{pair}", {timeframe}, '
             f"{candle_type} and store in {datadir}. "
             f'From {format_ms_time(since_ms) if since_ms else "start"} to '
             f'{format_ms_time(until_ms) if until_ms else "now"}'
@@ -345,18 +341,24 @@ def refresh_backtest_ohlcv_data(
     pairs_not_available = []
     data_handler = get_datahandler(datadir, data_format)
     candle_type = CandleType.get_default(trading_mode)
-    process = ""
-    for idx, pair in enumerate(pairs, start=1):
+    with get_progress_tracker() as progress:
+        tf_length = len(timeframes) if trading_mode != "futures" else len(timeframes) + 2
+        timeframe_task = progress.add_task("Timeframe", total=tf_length)
+        pair_task = progress.add_task("Downloading data...", total=len(pairs))
+
+        for pair in pairs:
+            progress.update(pair_task, description=f"Downloading {pair}")
+            progress.update(timeframe_task, completed=0)
+
             if pair not in exchange.markets:
                 pairs_not_available.append(pair)
                 logger.info(f"Skipping pair {pair}...")
                 continue
             for timeframe in timeframes:
+                progress.update(timeframe_task, description=f"Timeframe {timeframe}")
                 logger.debug(f"Downloading pair {pair}, {candle_type}, interval {timeframe}.")
-            process = f"{idx}/{len(pairs)}"
                 _download_pair_history(
                     pair=pair,
-                process=process,
                     datadir=datadir,
                     exchange=exchange,
                     timerange=timerange,
@@ -367,6 +369,7 @@ def refresh_backtest_ohlcv_data(
                     erase=erase,
                     prepend=prepend,
                 )
+                progress.update(timeframe_task, advance=1)
             if trading_mode == "futures":
                 # Predefined candletype (and timeframe) depending on exchange
                 # Downloads what is necessary to backtest based on futures data.
@@ -381,7 +384,6 @@ def refresh_backtest_ohlcv_data(
                     logger.debug(f"Downloading pair {pair}, {candle_type_f}, interval {tf}.")
                     _download_pair_history(
                         pair=pair,
-                    process=process,
                         datadir=datadir,
                         exchange=exchange,
                         timerange=timerange,
@@ -392,6 +394,12 @@ def refresh_backtest_ohlcv_data(
                         erase=erase,
                         prepend=prepend,
                     )
+                    progress.update(
+                        timeframe_task, advance=1, description=f"Timeframe {candle_type_f}, {tf}"
+                    )
+
+            progress.update(pair_task, advance=1)
+            progress.update(timeframe_task, description="Timeframe")
 
     return pairs_not_available
 
@@ -480,7 +488,7 @@ def _download_trades_history(
         return True
 
     except Exception:
-        logger.exception(f'Failed to download historic trades for pair: "{pair}". ')
+        logger.exception(f'Failed to download and store historic trades for pair: "{pair}". ')
         return False
 
 
@@ -501,7 +509,10 @@ def refresh_backtest_trades_data(
     """
     pairs_not_available = []
     data_handler = get_datahandler(datadir, data_format=data_format)
+    with get_progress_tracker() as progress:
+        pair_task = progress.add_task("Downloading data...", total=len(pairs))
         for pair in pairs:
+            progress.update(pair_task, description=f"Downloading trades [{pair}]")
             if pair not in exchange.markets:
                 pairs_not_available.append(pair)
                 logger.info(f"Skipping pair {pair}...")
@@ -520,6 +531,8 @@ def refresh_backtest_trades_data(
                 data_handler=data_handler,
                 trading_mode=trading_mode,
             )
+            progress.update(pair_task, advance=1)
+
     return pairs_not_available
 
 
@@ -618,6 +631,11 @@ def download_data_main(config: Config) -> None:
     # Start downloading
     try:
         if config.get("download_trades"):
+            if not exchange.get_option("trades_has_history", True):
+                raise OperationalException(
+                    f"Trade history not available for {exchange.name}. "
+                    "You cannot use --dl-trades for this exchange."
+                )
             pairs_not_available = refresh_backtest_trades_data(
                 exchange,
                 pairs=expanded_pairs,

freqtrade/enums/runmode.py

@@ -1,7 +1,7 @@
 from enum import Enum
 
 
-class RunMode(Enum):
+class RunMode(str, Enum):
     """
     Bot running mode (backtest, hyperopt, ...)
     can be "live", "dry-run", "backtest", "edge", "hyperopt".

freqtrade/exchange/__init__.py

@@ -11,6 +11,7 @@ from freqtrade.exchange.bitpanda import Bitpanda
 from freqtrade.exchange.bitvavo import Bitvavo
 from freqtrade.exchange.bybit import Bybit
 from freqtrade.exchange.coinbasepro import Coinbasepro
+from freqtrade.exchange.cryptocom import Cryptocom
 from freqtrade.exchange.exchange_utils import (
     ROUND_DOWN,
     ROUND_UP,

freqtrade/exchange/binance.py

@@ -28,7 +28,9 @@ class Binance(Exchange):
         "ohlcv_candle_limit": 1000,
         "trades_pagination": "id",
         "trades_pagination_arg": "fromId",
+        "trades_has_history": True,
         "l2_limit_range": [5, 10, 20, 50, 100, 500, 1000],
+        "ws.enabled": True,
     }
     _ft_has_futures: Dict = {
         "stoploss_order_types": {"limit": "stop", "market": "stop_market"},
@@ -41,6 +43,7 @@ class Binance(Exchange):
             PriceType.LAST: "CONTRACT_PRICE",
             PriceType.MARK: "MARK_PRICE",
         },
+        "ws.enabled": False,
     }
 
     _supported_trading_mode_margin_pairs: List[Tuple[TradingMode, MarginMode]] = [

freqtrade/exchange/bingx.py

@@ -20,4 +20,5 @@ class Bingx(Exchange):
         "stoploss_on_exchange": True,
         "stoploss_order_types": {"limit": "limit", "market": "market"},
         "order_time_in_force": ["GTC", "IOC", "PO"],
+        "trades_has_history": False,  # Endpoint doesn't seem to support pagination
     }

freqtrade/exchange/bitmart.py

@@ -18,4 +18,5 @@ class Bitmart(Exchange):
     _ft_has: Dict = {
         "stoploss_on_exchange": False,  # Bitmart API does not support stoploss orders
         "ohlcv_candle_limit": 200,
+        "trades_has_history": False,  # Endpoint doesn't seem to support pagination
     }

freqtrade/exchange/bybit.py

@@ -33,6 +33,8 @@ class Bybit(Exchange):
         "ohlcv_candle_limit": 1000,
         "ohlcv_has_history": True,
         "order_time_in_force": ["GTC", "FOK", "IOC", "PO"],
+        "ws.enabled": True,
+        "trades_has_history": False,  # Endpoint doesn't support pagination
     }
     _ft_has_futures: Dict = {
         "ohlcv_has_history": True,

freqtrade/exchange/check_exchange.py

@@ -47,7 +47,7 @@ def check_exchange(config: Config, check_for_bad: bool = True) -> bool:
             f'{", ".join(available_exchanges())}'
         )
 
-    valid, reason = validate_exchange(exchange)
+    valid, reason, _ = validate_exchange(exchange)
     if not valid:
         if check_for_bad:
             raise OperationalException(

freqtrade/exchange/common.py

@@ -37,7 +37,6 @@ API_FETCH_ORDER_RETRY_COUNT = 5
 
 BAD_EXCHANGES = {
     "bitmex": "Various reasons.",
-    "phemex": "Does not provide history.",
     "probit": "Requires additional, regular calls to `signIn()`.",
     "poloniex": "Does not provide fetch_order endpoint to fetch both open and closed orders.",
 }
@@ -65,6 +64,7 @@ SUPPORTED_EXCHANGES = [
 EXCHANGE_HAS_REQUIRED: Dict[str, List[str]] = {
     # Required / private
     "fetchOrder": ["fetchOpenOrder", "fetchClosedOrder"],
+    "fetchL2OrderBook": ["fetchTicker"],
     "cancelOrder": [],
     "createOrder": [],
     "fetchBalance": [],
@@ -92,6 +92,8 @@ EXCHANGE_HAS_OPTIONAL = [
     # 'fetchMarketLeverageTiers',  # Futures initialization
     # 'fetchOpenOrder', 'fetchClosedOrder',  # replacement for fetchOrder
     # 'fetchOpenOrders', 'fetchClosedOrders',  # 'fetchOrders',  # Refinding balance...
+    # ccxt.pro
+    "watchOHLCV",
 ]
 
 

freqtrade/exchange/cryptocom.py

@@ -0,0 +1,19 @@
+"""Crypto.com exchange subclass"""
+
+import logging
+from typing import Dict
+
+from freqtrade.exchange import Exchange
+
+
+logger = logging.getLogger(__name__)
+
+
+class Cryptocom(Exchange):
+    """Crypto.com exchange class.
+    Contains adjustments needed for Freqtrade to work with this exchange.
+    """
+
+    _ft_has: Dict = {
+        "ohlcv_candle_limit": 300,
+    }

freqtrade/exchange/exchange.py

@@ -14,7 +14,7 @@ from threading import Lock
 from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union
 
 import ccxt
-import ccxt.async_support as ccxt_async
+import ccxt.pro as ccxt_pro
 from cachetools import TTLCache
 from ccxt import TICK_SIZE
 from dateutil import parser
@@ -22,6 +22,7 @@ from pandas import DataFrame, concat
 
 from freqtrade.constants import (
     DEFAULT_AMOUNT_RESERVE_PERCENT,
+    DEFAULT_TRADES_COLUMNS,
     NON_OPEN_EXCHANGE_STATES,
     BidAsk,
     BuySell,
@@ -33,8 +34,22 @@ from freqtrade.constants import (
     OBLiteral,
     PairWithTimeframe,
 )
-from freqtrade.data.converter import clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_dict_to_list
-from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, PriceType, RunMode, TradingMode
+from freqtrade.data.converter import (
+    clean_ohlcv_dataframe,
+    ohlcv_to_dataframe,
+    trades_df_remove_duplicates,
+    trades_dict_to_list,
+    trades_list_to_df,
+)
+from freqtrade.enums import (
+    OPTIMIZE_MODES,
+    TRADE_MODES,
+    CandleType,
+    MarginMode,
+    PriceType,
+    RunMode,
+    TradingMode,
+)
 from freqtrade.exceptions import (
     ConfigurationError,
     DDosProtection,
@@ -56,7 +71,6 @@ from freqtrade.exchange.exchange_utils import (
     ROUND,
     ROUND_DOWN,
     ROUND_UP,
-    CcxtModuleType,
     amount_to_contract_precision,
     amount_to_contracts,
     amount_to_precision,
@@ -73,6 +87,7 @@ from freqtrade.exchange.exchange_utils_timeframe import (
     timeframe_to_prev_date,
     timeframe_to_seconds,
 )
+from freqtrade.exchange.exchange_ws import ExchangeWS
 from freqtrade.exchange.types import OHLCVResponse, OrderBook, Ticker, Tickers
 from freqtrade.misc import (
     chunks,
@@ -83,7 +98,7 @@ from freqtrade.misc import (
 )
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.util import dt_from_ts, dt_now
-from freqtrade.util.datetime_helpers import dt_humanize_delta, dt_ts
+from freqtrade.util.datetime_helpers import dt_humanize_delta, dt_ts, format_ms_time
 from freqtrade.util.periodic_cache import PeriodicCache
 
 
@@ -115,8 +130,10 @@ class Exchange:
         "tickers_have_quoteVolume": True,
         "tickers_have_bid_ask": True,  # bid / ask empty for fetch_tickers
         "tickers_have_price": True,
+        "trades_limit": 1000,  # Limit for 1 call to fetch_trades
         "trades_pagination": "time",  # Possible are "time" or "id"
         "trades_pagination_arg": "since",
+        "trades_has_history": False,
         "l2_limit_range": None,
         "l2_limit_range_required": True,  # Allow Empty L2 limit (kucoin)
         "mark_ohlcv_price": "mark",
@@ -129,6 +146,7 @@ class Exchange:
         "marketOrderRequiresPrice": False,
         "exchange_has_overrides": {},  # Dictionary overriding ccxt's "has".
         # Expected to be in the format {"fetchOHLCV": True} or {"fetchOHLCV": False}
+        "ws.enabled": False,  # Set to true for exchanges with tested websocket support
     }
     _ft_has: Dict = {}
     _ft_has_futures: Dict = {}
@@ -151,7 +169,9 @@ class Exchange:
         :return: None
         """
         self._api: ccxt.Exchange
-        self._api_async: ccxt_async.Exchange = None
+        self._api_async: ccxt_pro.Exchange
+        self._ws_async: ccxt_pro.Exchange = None
+        self._exchange_ws: Optional[ExchangeWS] = None
         self._markets: Dict = {}
         self._trading_fees: Dict[str, Any] = {}
         self._leverage_tiers: Dict[str, List[Dict]] = {}
@@ -182,6 +202,9 @@ class Exchange:
         self._klines: Dict[PairWithTimeframe, DataFrame] = {}
         self._expiring_candle_cache: Dict[Tuple[str, int], PeriodicCache] = {}
 
+        # Holds public_trades
+        self._trades: Dict[PairWithTimeframe, DataFrame] = {}
+
         # Holds all open sell orders for dry_run
         self._dry_run_open_orders: Dict[str, Any] = {}
 
@@ -210,6 +233,8 @@ class Exchange:
         # Assign this directly for easy access
         self._ohlcv_partial_candle = self._ft_has["ohlcv_partial_candle"]
 
+        self._max_trades_limit = self._ft_has["trades_limit"]
+
         self._trades_pagination = self._ft_has["trades_pagination"]
         self._trades_pagination_arg = self._ft_has["trades_pagination_arg"]
 
@@ -218,7 +243,7 @@ class Exchange:
         ccxt_config = deep_merge_dicts(exchange_conf.get("ccxt_config", {}), ccxt_config)
         ccxt_config = deep_merge_dicts(exchange_conf.get("ccxt_sync_config", {}), ccxt_config)
 
-        self._api = self._init_ccxt(exchange_conf, ccxt_kwargs=ccxt_config)
+        self._api = self._init_ccxt(exchange_conf, True, ccxt_config)
 
         ccxt_async_config = self._ccxt_config
         ccxt_async_config = deep_merge_dicts(
@@ -227,13 +252,21 @@ class Exchange:
         ccxt_async_config = deep_merge_dicts(
             exchange_conf.get("ccxt_async_config", {}), ccxt_async_config
         )
-        self._api_async = self._init_ccxt(exchange_conf, ccxt_async, ccxt_kwargs=ccxt_async_config)
+        self._api_async = self._init_ccxt(exchange_conf, False, ccxt_async_config)
+        self._has_watch_ohlcv = self.exchange_has("watchOHLCV") and self._ft_has["ws.enabled"]
+        if (
+            self._config["runmode"] in TRADE_MODES
+            and exchange_conf.get("enable_ws", True)
+            and self._has_watch_ohlcv
+        ):
+            self._ws_async = self._init_ccxt(exchange_conf, False, ccxt_async_config)
+            self._exchange_ws = ExchangeWS(self._config, self._ws_async)
 
         logger.info(f'Using Exchange "{self.name}"')
         self.required_candle_call_count = 1
         if validate:
             # Initial markets load
-            self._load_markets()
+            self.reload_markets(True, load_leverage_tiers=False)
             self.validate_config(config)
             self._startup_candle_count: int = config.get("startup_candle_count", 0)
             self.required_candle_call_count = self.validate_required_startup_candles(
@@ -256,14 +289,24 @@ class Exchange:
         self.close()
 
     def close(self):
+        if self._exchange_ws:
+            self._exchange_ws.cleanup()
         logger.debug("Exchange object destroyed, closing async loop")
         if (
-            self._api_async
+            getattr(self, "_api_async", None)
             and inspect.iscoroutinefunction(self._api_async.close)
             and self._api_async.session
         ):
             logger.debug("Closing async ccxt session.")
             self.loop.run_until_complete(self._api_async.close())
+        if (
+            self._ws_async
+            and inspect.iscoroutinefunction(self._ws_async.close)
+            and self._ws_async.session
+        ):
+            logger.debug("Closing ws ccxt session.")
+            self.loop.run_until_complete(self._ws_async.close())
+
         if self.loop and not self.loop.is_closed():
             self.loop.close()
 
@@ -285,29 +328,38 @@ class Exchange:
         self.validate_trading_mode_and_margin_mode(self.trading_mode, self.margin_mode)
         self.validate_pricing(config["exit_pricing"])
         self.validate_pricing(config["entry_pricing"])
+        self.validate_orderflow(config["exchange"])
 
     def _init_ccxt(
-        self,
-        exchange_config: Dict[str, Any],
-        ccxt_module: CcxtModuleType = ccxt,
-        *,
-        ccxt_kwargs: Dict,
+        self, exchange_config: Dict[str, Any], sync: bool, ccxt_kwargs: Dict[str, Any]
     ) -> ccxt.Exchange:
         """
-        Initialize ccxt with given config and return valid
-        ccxt instance.
+        Initialize ccxt with given config and return valid ccxt instance.
         """
         # Find matching class for the given exchange name
         name = exchange_config["name"]
+        if sync:
+            ccxt_module = ccxt
+        else:
+            ccxt_module = ccxt_pro
+            if not is_exchange_known_ccxt(name, ccxt_module):
+                # Fall back to async if pro doesn't support this exchange
+                import ccxt.async_support as ccxt_async
+
+                ccxt_module = ccxt_async
 
         if not is_exchange_known_ccxt(name, ccxt_module):
             raise OperationalException(f"Exchange {name} is not supported by ccxt")
 
         ex_config = {
-            "apiKey": exchange_config.get("key"),
+            "apiKey": exchange_config.get("apiKey", exchange_config.get("key")),
             "secret": exchange_config.get("secret"),
             "password": exchange_config.get("password"),
             "uid": exchange_config.get("uid", ""),
+            "accountId": exchange_config.get("accountId", ""),
+            # DEX attributes:
+            "walletAddress": exchange_config.get("walletAddress"),
+            "privateKey": exchange_config.get("privateKey"),
         }
         if ccxt_kwargs:
             logger.info("Applying additional ccxt config: %s", ccxt_kwargs)
@@ -354,7 +406,7 @@ class Exchange:
         """exchange ccxt markets"""
         if not self._markets:
             logger.info("Markets were not loaded. Loading them now..")
-            self._load_markets()
+            self.reload_markets(True)
         return self._markets
 
     @property
@@ -482,6 +534,15 @@ class Exchange:
         else:
             return DataFrame()
 
+    def trades(self, pair_interval: PairWithTimeframe, copy: bool = True) -> DataFrame:
+        if pair_interval in self._trades:
+            if copy:
+                return self._trades[pair_interval].copy()
+            else:
+                return self._trades[pair_interval]
+        else:
+            return DataFrame()
+
     def get_contract_size(self, pair: str) -> Optional[float]:
         if self.trading_mode == TradingMode.FUTURES:
             market = self.markets.get(pair, {})
@@ -530,30 +591,33 @@ class Exchange:
             amount, self.get_precision_amount(pair), self.precisionMode, contract_size
         )
 
-    def _load_async_markets(self, reload: bool = False) -> None:
+    def ws_connection_reset(self):
+        """
+        called at regular intervals to reset the websocket connection
+        """
+        if self._exchange_ws:
+            self._exchange_ws.reset_connections()
+
+    def _load_async_markets(self, reload: bool = False) -> Dict[str, Any]:
         try:
-            if self._api_async:
-                self.loop.run_until_complete(self._api_async.load_markets(reload=reload, params={}))
+            markets = self.loop.run_until_complete(
+                self._api_async.load_markets(reload=reload, params={})
+            )
 
-        except (asyncio.TimeoutError, ccxt.BaseError) as e:
-            logger.warning("Could not load async markets. Reason: %s", e)
-            return
+            if isinstance(markets, Exception):
+                raise markets
+            return markets
+        except asyncio.TimeoutError as e:
+            logger.warning("Could not load markets. Reason: %s", e)
+            raise TemporaryError from e
 
-    def _load_markets(self) -> None:
-        """Initialize markets both sync and async"""
-        try:
-            self._markets = self._api.load_markets(params={})
-            self._load_async_markets()
-            self._last_markets_refresh = dt_ts()
-            if self._ft_has["needs_trading_fees"]:
-                self._trading_fees = self.fetch_trading_fees()
+    def reload_markets(self, force: bool = False, *, load_leverage_tiers: bool = True) -> None:
+        """
+        Reload / Initialize markets both sync and async if refresh interval has passed
 
-        except ccxt.BaseError:
-            logger.exception("Unable to initialize markets.")
-
-    def reload_markets(self, force: bool = False) -> None:
-        """Reload markets both sync and async if refresh interval has passed"""
+        """
         # Check whether markets have to be reloaded
+        is_initial = self._last_markets_refresh == 0
         if (
             not force
             and self._last_markets_refresh > 0
@@ -562,13 +626,24 @@ class Exchange:
             return None
         logger.debug("Performing scheduled market reload..")
         try:
-            self._markets = self._api.load_markets(reload=True, params={})
-            # Also reload async markets to avoid issues with newly listed pairs
-            self._load_async_markets(reload=True)
+            # Reload async markets, then assign them to sync api
+            self._markets = self._load_async_markets(reload=True)
+            self._api.set_markets(self._api_async.markets, self._api_async.currencies)
+            # Assign options array, as it contains some temporary information from the exchange.
+            self._api.options = self._api_async.options
+            if self._exchange_ws:
+                # Set markets to avoid reloading on websocket api
+                self._ws_async.set_markets(self._api.markets, self._api.currencies)
+                self._ws_async.options = self._api.options
             self._last_markets_refresh = dt_ts()
+
+            if is_initial and self._ft_has["needs_trading_fees"]:
+                self._trading_fees = self.fetch_trading_fees()
+
+            if load_leverage_tiers and self.trading_mode == TradingMode.FUTURES:
                 self.fill_leverage_tiers()
-        except ccxt.BaseError:
-            logger.exception("Could not reload markets.")
+        except (ccxt.BaseError, TemporaryError):
+            logger.exception("Could not load markets.")
 
     def validate_stakecurrency(self, stake_currency: str) -> None:
         """
@@ -721,6 +796,14 @@ class Exchange:
                 f"Time in force policies are not supported for {self.name} yet."
             )
 
+    def validate_orderflow(self, exchange: Dict) -> None:
+        if exchange.get("use_public_trades", False) and (
+            not self.exchange_has("fetchTrades") or not self._ft_has["trades_has_history"]
+        ):
+            raise ConfigurationError(
+                f"Trade data not available for {self.name}. Can't use orderflow feature."
+            )
+
     def validate_required_startup_candles(self, startup_candles: int, timeframe: str) -> int:
         """
         Checks if required startup_candles is more than ohlcv_candle_limit().
@@ -793,7 +876,7 @@ class Exchange:
         """
         if endpoint in self._ft_has.get("exchange_has_overrides", {}):
             return self._ft_has["exchange_has_overrides"][endpoint]
-        return endpoint in self._api.has and self._api.has[endpoint]
+        return endpoint in self._api_async.has and self._api_async.has[endpoint]
 
     def get_precision_amount(self, pair: str) -> Optional[float]:
         """
@@ -2017,7 +2100,7 @@ class Exchange:
     def get_fee(
         self,
         symbol: str,
-        type: str = "",
+        order_type: str = "",
         side: str = "",
         amount: float = 1,
         price: float = 1,
@@ -2026,13 +2109,13 @@ class Exchange:
         """
         Retrieve fee from exchange
         :param symbol: Pair
-        :param type: Type of order (market, limit, ...)
+        :param order_type: Type of order (market, limit, ...)
         :param side: Side of order (buy, sell)
         :param amount: Amount of order
         :param price: Price of order
         :param taker_or_maker: 'maker' or 'taker' (ignored if "type" is provided)
         """
-        if type and type == "market":
+        if order_type and order_type == "market":
             taker_or_maker = "taker"
         try:
             if self._config["dry_run"] and self._config.get("fee", None) is not None:
@@ -2043,7 +2126,7 @@ class Exchange:
 
             return self._api.calculate_fee(
                 symbol=symbol,
-                type=type,
+                type=order_type,
                 side=side,
                 amount=amount,
                 price=price,
@@ -2226,9 +2309,40 @@ class Exchange:
         cache: bool,
     ) -> Coroutine[Any, Any, OHLCVResponse]:
         not_all_data = cache and self.required_candle_call_count > 1
+        if cache and candle_type in (CandleType.SPOT, CandleType.FUTURES):
+            if self._has_watch_ohlcv and self._exchange_ws:
+                # Subscribe to websocket
+                self._exchange_ws.schedule_ohlcv(pair, timeframe, candle_type)
+
         if cache and (pair, timeframe, candle_type) in self._klines:
             candle_limit = self.ohlcv_candle_limit(timeframe, candle_type)
-            min_date = date_minus_candles(timeframe, candle_limit - 5).timestamp()
+            min_date = int(date_minus_candles(timeframe, candle_limit - 5).timestamp())
+
+            if self._exchange_ws:
+                candle_date = int(timeframe_to_prev_date(timeframe).timestamp() * 1000)
+                prev_candle_date = int(date_minus_candles(timeframe, 1).timestamp() * 1000)
+                candles = self._exchange_ws.ccxt_object.ohlcvs.get(pair, {}).get(timeframe)
+                half_candle = int(candle_date - (candle_date - prev_candle_date) * 0.5)
+                last_refresh_time = int(
+                    self._exchange_ws.klines_last_refresh.get((pair, timeframe, candle_type), 0)
+                )
+
+                if (
+                    candles
+                    and candles[-1][0] >= prev_candle_date
+                    and last_refresh_time >= half_candle
+                ):
+                    # Usable result, candle contains the previous candle.
+                    # Also, we check if the last refresh time is no more than half the candle ago.
+                    logger.debug(f"reuse watch result for {pair}, {timeframe}, {last_refresh_time}")
+
+                    return self._exchange_ws.get_ohlcv(pair, timeframe, candle_type, candle_date)
+                logger.info(
+                    f"Failed to reuse watch {pair}, {timeframe}, {candle_date < last_refresh_time},"
+                    f" {candle_date}, {last_refresh_time}, "
+                    f"{format_ms_time(candle_date)}, {format_ms_time(last_refresh_time)} "
+                )
+
             # Check if 1 call can get us updated candles without hole in the data.
             if min_date < self._pairs_last_refresh_time.get((pair, timeframe, candle_type), 0):
                 # Cache can be used - do one-off call.
@@ -2261,7 +2375,7 @@ class Exchange:
 
     def _build_ohlcv_dl_jobs(
         self, pair_list: ListPairsWithTimeframes, since_ms: Optional[int], cache: bool
-    ) -> Tuple[List[Coroutine], List[Tuple[str, str, CandleType]]]:
+    ) -> Tuple[List[Coroutine], List[PairWithTimeframe]]:
         """
         Build Coroutines to execute as part of refresh_latest_ohlcv
         """
@@ -2517,6 +2631,171 @@ class Exchange:
         data = [[x["timestamp"], x["fundingRate"], 0, 0, 0, 0] for x in data]
         return data
 
+    # fetch Trade data stuff
+
+    def needed_candle_for_trades_ms(self, timeframe: str, candle_type: CandleType) -> int:
+        candle_limit = self.ohlcv_candle_limit(timeframe, candle_type)
+        tf_s = timeframe_to_seconds(timeframe)
+        candles_fetched = candle_limit * self.required_candle_call_count
+
+        max_candles = self._config["orderflow"]["max_candles"]
+
+        required_candles = min(max_candles, candles_fetched)
+        move_to = (
+            tf_s * candle_limit * required_candles
+            if required_candles > candle_limit
+            else (max_candles + 1) * tf_s
+        )
+
+        now = timeframe_to_next_date(timeframe)
+        return int((now - timedelta(seconds=move_to)).timestamp() * 1000)
+
+    def _process_trades_df(
+        self,
+        pair: str,
+        timeframe: str,
+        c_type: CandleType,
+        ticks: List[List],
+        cache: bool,
+        first_required_candle_date: int,
+    ) -> DataFrame:
+        # keeping parsed dataframe in cache
+        trades_df = trades_list_to_df(ticks, True)
+
+        if cache:
+            if (pair, timeframe, c_type) in self._trades:
+                old = self._trades[(pair, timeframe, c_type)]
+                # Reassign so we return the updated, combined df
+                combined_df = concat([old, trades_df], axis=0)
+                logger.debug(f"Clean duplicated ticks from Trades data {pair}")
+                trades_df = DataFrame(
+                    trades_df_remove_duplicates(combined_df), columns=combined_df.columns
+                )
+                # Age out old candles
+                trades_df = trades_df[first_required_candle_date < trades_df["timestamp"]]
+                trades_df = trades_df.reset_index(drop=True)
+            self._trades[(pair, timeframe, c_type)] = trades_df
+        return trades_df
+
+    def refresh_latest_trades(
+        self,
+        pair_list: ListPairsWithTimeframes,
+        *,
+        cache: bool = True,
+    ) -> Dict[PairWithTimeframe, DataFrame]:
+        """
+        Refresh in-memory TRADES asynchronously and set `_trades` with the result
+        Loops asynchronously over pair_list and downloads all pairs async (semi-parallel).
+        Only used in the dataprovider.refresh() method.
+        :param pair_list: List of 3 element tuples containing (pair, timeframe, candle_type)
+        :param cache: Assign result to _trades. Useful for one-off downloads like for pairlists
+        :return: Dict of [{(pair, timeframe): Dataframe}]
+        """
+        from freqtrade.data.history import get_datahandler
+
+        data_handler = get_datahandler(
+            self._config["datadir"], data_format=self._config["dataformat_trades"]
+        )
+        logger.debug("Refreshing TRADES data for %d pairs", len(pair_list))
+        since_ms = None
+        results_df = {}
+        for pair, timeframe, candle_type in set(pair_list):
+            new_ticks: List = []
+            all_stored_ticks_df = DataFrame(columns=DEFAULT_TRADES_COLUMNS + ["date"])
+            first_candle_ms = self.needed_candle_for_trades_ms(timeframe, candle_type)
+            # refresh, if
+            # a. not in _trades
+            # b. no cache used
+            # c. need new data
+            is_in_cache = (pair, timeframe, candle_type) in self._trades
+            if (
+                not is_in_cache
+                or not cache
+                or self._now_is_time_to_refresh_trades(pair, timeframe, candle_type)
+            ):
+                logger.debug(f"Refreshing TRADES data for {pair}")
+                # fetch trades since latest _trades and
+                # store together with existing trades
+                try:
+                    until = None
+                    from_id = None
+                    if is_in_cache:
+                        from_id = self._trades[(pair, timeframe, candle_type)].iloc[-1]["id"]
+                        until = dt_ts()  # now
+
+                    else:
+                        until = int(timeframe_to_prev_date(timeframe).timestamp()) * 1000
+                        all_stored_ticks_df = data_handler.trades_load(
+                            f"{pair}-cached", self.trading_mode
+                        )
+
+                        if not all_stored_ticks_df.empty:
+                            if (
+                                all_stored_ticks_df.iloc[-1]["timestamp"] > first_candle_ms
+                                and all_stored_ticks_df.iloc[0]["timestamp"] <= first_candle_ms
+                            ):
+                                # Use cache and populate further
+                                last_cached_ms = all_stored_ticks_df.iloc[-1]["timestamp"]
+                                from_id = all_stored_ticks_df.iloc[-1]["id"]
+                                # only use cached if it's closer than first_candle_ms
+                                since_ms = (
+                                    last_cached_ms
+                                    if last_cached_ms > first_candle_ms
+                                    else first_candle_ms
+                                )
+                            else:
+                                # Skip cache, it's too old
+                                all_stored_ticks_df = DataFrame(
+                                    columns=DEFAULT_TRADES_COLUMNS + ["date"]
+                                )
+
+                    # from_id overrules with exchange set to id paginate
+                    [_, new_ticks] = self.get_historic_trades(
+                        pair,
+                        since=since_ms if since_ms else first_candle_ms,
+                        until=until,
+                        from_id=from_id,
+                    )
+
+                except Exception:
+                    logger.exception(f"Refreshing TRADES data for {pair} failed")
+                    continue
+
+                if new_ticks:
+                    all_stored_ticks_list = all_stored_ticks_df[
+                        DEFAULT_TRADES_COLUMNS
+                    ].values.tolist()
+                    all_stored_ticks_list.extend(new_ticks)
+                    trades_df = self._process_trades_df(
+                        pair,
+                        timeframe,
+                        candle_type,
+                        all_stored_ticks_list,
+                        cache,
+                        first_required_candle_date=first_candle_ms,
+                    )
+                    results_df[(pair, timeframe, candle_type)] = trades_df
+                    data_handler.trades_store(
+                        f"{pair}-cached", trades_df[DEFAULT_TRADES_COLUMNS], self.trading_mode
+                    )
+
+                else:
+                    logger.error(f"No new ticks for {pair}")
+
+        return results_df
+
+    def _now_is_time_to_refresh_trades(
+        self, pair: str, timeframe: str, candle_type: CandleType
+    ) -> bool:  # Timeframe in seconds
+        trades = self.trades((pair, timeframe, candle_type), False)
+        pair_last_refreshed = int(trades.iloc[-1]["timestamp"])
+        full_candle = (
+            int(timeframe_to_next_date(timeframe, dt_from_ts(pair_last_refreshed)).timestamp())
+            * 1000
+        )
+        now = dt_ts()
+        return full_candle <= now
+
     # Fetch historic trades
 
     @retrier_async
@@ -2531,10 +2810,11 @@ class Exchange:
         returns: List of dicts containing trades, the next iteration value (new "since" or trade_id)
         """
         try:
+            trades_limit = self._max_trades_limit
             # fetch trades asynchronously
             if params:
                 logger.debug("Fetching trades for pair %s, params: %s ", pair, params)
-                trades = await self._api_async.fetch_trades(pair, params=params, limit=1000)
+                trades = await self._api_async.fetch_trades(pair, params=params, limit=trades_limit)
             else:
                 logger.debug(
                     "Fetching trades for pair %s, since %s %s...",
@@ -2542,7 +2822,7 @@ class Exchange:
                     since,
                     "(" + dt_from_ts(since).isoformat() + ") " if since is not None else "",
                 )
-                trades = await self._api_async.fetch_trades(pair, since=since, limit=1000)
+                trades = await self._api_async.fetch_trades(pair, since=since, limit=trades_limit)
             trades = self._trades_contracts_to_amount(trades)
             pagination_value = self._get_trade_pagination_next_value(trades)
             return trades_dict_to_list(trades), pagination_value
@@ -3337,13 +3617,12 @@ class Exchange:
     def get_maintenance_ratio_and_amt(
         self,
         pair: str,
-        nominal_value: float,
+        notional_value: float,
     ) -> Tuple[float, Optional[float]]:
         """
         Important: Must be fetching data from cached values as this is used by backtesting!
         :param pair: Market symbol
-        :param nominal_value: The total trade amount in quote currency including leverage
-        maintenance amount only on Binance
+        :param notional_value: The total trade amount in quote currency
         :return: (maintenance margin ratio, maintenance amount)
         """
 
@@ -3360,7 +3639,7 @@ class Exchange:
             pair_tiers = self._leverage_tiers[pair]
 
             for tier in reversed(pair_tiers):
-                if nominal_value >= tier["minNotional"]:
+                if notional_value >= tier["minNotional"]:
                     return (tier["maintenanceMarginRate"], tier["maintAmt"])
 
             raise ExchangeError("nominal value can not be lower than 0")
@@ -3368,4 +3647,3 @@ class Exchange:
             # describes the min amt for a tier, and the lowest tier will always go down to 0
         else:
             raise ExchangeError(f"Cannot get maintenance ratio using {self.name}")
-            raise ExchangeError(f"Cannot get maintenance ratio using {self.name}")

freqtrade/exchange/exchange_utils.py

@@ -53,16 +53,22 @@ def available_exchanges(ccxt_module: Optional[CcxtModuleType] = None) -> List[st
     return [x for x in exchanges if validate_exchange(x)[0]]
 
 
-def validate_exchange(exchange: str) -> Tuple[bool, str]:
+def validate_exchange(exchange: str) -> Tuple[bool, str, bool]:
     """
     returns: can_use, reason
         with Reason including both missing and missing_opt
     """
-    ex_mod = getattr(ccxt, exchange.lower())()
+    try:
+        ex_mod = getattr(ccxt.pro, exchange.lower())()
+    except AttributeError:
+        ex_mod = getattr(ccxt.async_support, exchange.lower())()
+
+    if not ex_mod or not ex_mod.has:
+        return False, "", False
+
     result = True
     reason = ""
-    if not ex_mod or not ex_mod.has:
-        return False, ""
+    is_dex = getattr(ex_mod, "dex", False)
     missing = [
         k
         for k, v in EXCHANGE_HAS_REQUIRED.items()
@@ -81,18 +87,19 @@ def validate_exchange(exchange: str) -> Tuple[bool, str]:
     if missing_opt:
         reason += f"{'. ' if reason else ''}missing opt: {', '.join(missing_opt)}. "
 
-    return result, reason
+    return result, reason, is_dex
 
 
 def _build_exchange_list_entry(
     exchange_name: str, exchangeClasses: Dict[str, Any]
 ) -> ValidExchangesType:
-    valid, comment = validate_exchange(exchange_name)
+    valid, comment, is_dex = validate_exchange(exchange_name)
     result: ValidExchangesType = {
         "name": exchange_name,
         "valid": valid,
         "supported": exchange_name.lower() in SUPPORTED_EXCHANGES,
         "comment": comment,
+        "dex": is_dex,
         "trade_modes": [{"trading_mode": "spot", "margin_mode": ""}],
     }
     if resolved := exchangeClasses.get(exchange_name.lower()):

freqtrade/exchange/exchange_ws.py

@@ -0,0 +1,195 @@
+import asyncio
+import logging
+import time
+from copy import deepcopy
+from functools import partial
+from threading import Thread
+from typing import Dict, Set
+
+import ccxt
+
+from freqtrade.constants import Config, PairWithTimeframe
+from freqtrade.enums.candletype import CandleType
+from freqtrade.exchange.exchange import timeframe_to_seconds
+from freqtrade.exchange.types import OHLCVResponse
+from freqtrade.util import dt_ts, format_ms_time
+
+
+logger = logging.getLogger(__name__)
+
+
+class ExchangeWS:
+    def __init__(self, config: Config, ccxt_object: ccxt.Exchange) -> None:
+        self.config = config
+        self.ccxt_object = ccxt_object
+        self._background_tasks: Set[asyncio.Task] = set()
+
+        self._klines_watching: Set[PairWithTimeframe] = set()
+        self._klines_scheduled: Set[PairWithTimeframe] = set()
+        self.klines_last_refresh: Dict[PairWithTimeframe, float] = {}
+        self.klines_last_request: Dict[PairWithTimeframe, float] = {}
+        self._thread = Thread(name="ccxt_ws", target=self._start_forever)
+        self._thread.start()
+        self.__cleanup_called = False
+
+    def _start_forever(self) -> None:
+        self._loop = asyncio.new_event_loop()
+        try:
+            self._loop.run_forever()
+        finally:
+            if self._loop.is_running():
+                self._loop.stop()
+
+    def cleanup(self) -> None:
+        logger.debug("Cleanup called - stopping")
+        self._klines_watching.clear()
+        for task in self._background_tasks:
+            task.cancel()
+        if hasattr(self, "_loop") and not self._loop.is_closed():
+            self.reset_connections()
+
+            self._loop.call_soon_threadsafe(self._loop.stop)
+            time.sleep(0.1)
+            if not self._loop.is_closed():
+                self._loop.close()
+
+        self._thread.join()
+        logger.debug("Stopped")
+
+    def reset_connections(self) -> None:
+        """
+        Reset all connections - avoids "connection-reset" errors that happen after ~9 days
+        """
+        if hasattr(self, "_loop") and not self._loop.is_closed():
+            logger.info("Resetting WS connections.")
+            asyncio.run_coroutine_threadsafe(self._cleanup_async(), loop=self._loop)
+            while not self.__cleanup_called:
+                time.sleep(0.1)
+        self.__cleanup_called = False
+
+    async def _cleanup_async(self) -> None:
+        try:
+            await self.ccxt_object.close()
+            # Clear the cache.
+            # Not doing this will cause problems on startup with dynamic pairlists
+            self.ccxt_object.ohlcvs.clear()
+        except Exception:
+            logger.exception("Exception in _cleanup_async")
+        finally:
+            self.__cleanup_called = True
+
+    def _pop_history(self, paircomb: PairWithTimeframe) -> None:
+        """
+        Remove history for a pair/timeframe combination from ccxt cache
+        """
+        self.ccxt_object.ohlcvs.get(paircomb[0], {}).pop(paircomb[1], None)
+
+    def cleanup_expired(self) -> None:
+        """
+        Remove pairs from watchlist if they've not been requested within
+        the last timeframe (+ offset)
+        """
+        changed = False
+        for p in list(self._klines_watching):
+            _, timeframe, _ = p
+            timeframe_s = timeframe_to_seconds(timeframe)
+            last_refresh = self.klines_last_request.get(p, 0)
+            if last_refresh > 0 and (dt_ts() - last_refresh) > ((timeframe_s + 20) * 1000):
+                logger.info(f"Removing {p} from websocket watchlist.")
+                self._klines_watching.discard(p)
+                # Pop history to avoid getting stale data
+                self._pop_history(p)
+                changed = True
+        if changed:
+            logger.info(f"Removal done: new watch list ({len(self._klines_watching)})")
+
+    async def _schedule_while_true(self) -> None:
+        # For the ones we should be watching
+        for p in self._klines_watching:
+            # Check if they're already scheduled
+            if p not in self._klines_scheduled:
+                self._klines_scheduled.add(p)
+                pair, timeframe, candle_type = p
+                task = asyncio.create_task(
+                    self._continuously_async_watch_ohlcv(pair, timeframe, candle_type)
+                )
+                self._background_tasks.add(task)
+                task.add_done_callback(
+                    partial(
+                        self._continuous_stopped,
+                        pair=pair,
+                        timeframe=timeframe,
+                        candle_type=candle_type,
+                    )
+                )
+
+    def _continuous_stopped(
+        self, task: asyncio.Task, pair: str, timeframe: str, candle_type: CandleType
+    ):
+        self._background_tasks.discard(task)
+        result = "done"
+        if task.cancelled():
+            result = "cancelled"
+        else:
+            if (result1 := task.result()) is not None:
+                result = str(result1)
+
+        logger.info(f"{pair}, {timeframe}, {candle_type} - Task finished - {result}")
+        self._klines_scheduled.discard((pair, timeframe, candle_type))
+        self._pop_history((pair, timeframe, candle_type))
+
+    async def _continuously_async_watch_ohlcv(
+        self, pair: str, timeframe: str, candle_type: CandleType
+    ) -> None:
+        try:
+            while (pair, timeframe, candle_type) in self._klines_watching:
+                start = dt_ts()
+                data = await self.ccxt_object.watch_ohlcv(pair, timeframe)
+                self.klines_last_refresh[(pair, timeframe, candle_type)] = dt_ts()
+                logger.debug(
+                    f"watch done {pair}, {timeframe}, data {len(data)} "
+                    f"in {dt_ts() - start:.2f}s"
+                )
+        except ccxt.ExchangeClosedByUser:
+            logger.debug("Exchange connection closed by user")
+        except ccxt.BaseError:
+            logger.exception(f"Exception in continuously_async_watch_ohlcv for {pair}, {timeframe}")
+        finally:
+            self._klines_watching.discard((pair, timeframe, candle_type))
+
+    def schedule_ohlcv(self, pair: str, timeframe: str, candle_type: CandleType) -> None:
+        """
+        Schedule a pair/timeframe combination to be watched
+        """
+        self._klines_watching.add((pair, timeframe, candle_type))
+        self.klines_last_request[(pair, timeframe, candle_type)] = dt_ts()
+        # asyncio.run_coroutine_threadsafe(self.schedule_schedule(), loop=self._loop)
+        asyncio.run_coroutine_threadsafe(self._schedule_while_true(), loop=self._loop)
+        self.cleanup_expired()
+
+    async def get_ohlcv(
+        self,
+        pair: str,
+        timeframe: str,
+        candle_type: CandleType,
+        candle_date: int,
+    ) -> OHLCVResponse:
+        """
+        Returns cached klines from ccxt's "watch" cache.
+        :param candle_date: timestamp of the end-time of the candle.
+        """
+        # Deepcopy the response - as it might be modified in the background as new messages arrive
+        candles = deepcopy(self.ccxt_object.ohlcvs.get(pair, {}).get(timeframe))
+        refresh_date = self.klines_last_refresh[(pair, timeframe, candle_type)]
+        drop_hint = False
+        if refresh_date > candle_date:
+            # Refreshed after candle was complete.
+            # logger.info(f"{candles[-1][0]} >= {candle_date}")
+            drop_hint = candles[-1][0] >= candle_date
+        logger.debug(
+            f"watch result for {pair}, {timeframe} with length {len(candles)}, "
+            f"{format_ms_time(candles[-1][0])}, "
+            f"lref={format_ms_time(refresh_date)}, "
+            f"candle_date={format_ms_time(candle_date)}, {drop_hint=}"
+        )
+        return pair, timeframe, candle_type, candles, drop_hint

freqtrade/exchange/gate.py

@@ -31,6 +31,7 @@ class Gate(Exchange):
         "stop_price_param": "stopPrice",
         "stop_price_prop": "stopPrice",
         "marketOrderRequiresPrice": True,
+        "trades_has_history": False,  # Endpoint would support this - but ccxt doesn't.
     }
 
     _ft_has_futures: Dict = {

freqtrade/exchange/htx.py

@@ -28,6 +28,7 @@ class Htx(Exchange):
             "1w": 500,
             "1M": 500,
         },
+        "trades_has_history": False,  # Endpoint doesn't have a "since" parameter
     }
 
     def _get_stop_params(self, side: BuySell, ordertype: str, stop_price: float) -> Dict:

freqtrade/exchange/hyperliquid.py

@@ -0,0 +1,24 @@
+"""Hyperliquid exchange subclass"""
+
+import logging
+from typing import Dict
+
+from freqtrade.exchange import Exchange
+
+
+logger = logging.getLogger(__name__)
+
+
+class Hyperliquid(Exchange):
+    """Hyperliquid exchange class.
+    Contains adjustments needed for Freqtrade to work with this exchange.
+    """
+
+    _ft_has: Dict = {
+        # Only the most recent 5000 candles are available according to the
+        # exchange's API documentation.
+        "ohlcv_has_history": True,
+        "ohlcv_candle_limit": 5000,
+        "trades_has_history": False,  # Trades endpoint doesn't seem available.
+        "exchange_has_overrides": {"fetchTrades": False},
+    }

freqtrade/exchange/kraken.py

@@ -31,6 +31,7 @@ class Kraken(Exchange):
         "trades_pagination": "id",
         "trades_pagination_arg": "since",
         "trades_pagination_overlap": False,
+        "trades_has_history": True,
         "mark_ohlcv_timeframe": "4h",
     }
 

freqtrade/exchange/okx.py

@@ -33,6 +33,7 @@ class Okx(Exchange):
         "funding_fee_timeframe": "8h",
         "stoploss_order_types": {"limit": "limit"},
         "stoploss_on_exchange": True,
+        "trades_has_history": False,  # Endpoint doesn't have a "since" parameter
     }
     _ft_has_futures: Dict = {
         "tickers_have_quoteVolume": False,

freqtrade/freqai/RL/BaseEnvironment.py

@@ -52,7 +52,7 @@ class BaseEnvironment(gym.Env):
         reward_kwargs: dict = {},
         window_size=10,
         starting_point=True,
-        id: str = "baseenv-1",
+        id: str = "baseenv-1",  # noqa: A002
         seed: int = 1,
         config: dict = {},
         live: bool = False,

freqtrade/freqai/data_drawer.py

@@ -238,9 +238,9 @@ class FreqaiDataDrawer:
                     metadata, fp, default=self.np_encoder, number_mode=rapidjson.NM_NATIVE
                 )
 
-    def np_encoder(self, object):
-        if isinstance(object, np.generic):
-            return object.item()
+    def np_encoder(self, obj):
+        if isinstance(obj, np.generic):
+            return obj.item()
 
     def get_pair_dict_info(self, pair: str) -> Tuple[str, int]:
         """
@@ -448,8 +448,8 @@ class FreqaiDataDrawer:
 
         delete_dict: Dict[str, Any] = {}
 
-        for dir in model_folders:
-            result = pattern.match(str(dir.name))
+        for directory in model_folders:
+            result = pattern.match(str(directory.name))
             if result is None:
                 continue
             coin = result.group(1)
@@ -458,10 +458,10 @@ class FreqaiDataDrawer:
             if coin not in delete_dict:
                 delete_dict[coin] = {}
                 delete_dict[coin]["num_folders"] = 1
-                delete_dict[coin]["timestamps"] = {int(timestamp): dir}
+                delete_dict[coin]["timestamps"] = {int(timestamp): directory}
             else:
                 delete_dict[coin]["num_folders"] += 1
-                delete_dict[coin]["timestamps"][int(timestamp)] = dir
+                delete_dict[coin]["timestamps"][int(timestamp)] = directory
 
         for coin in delete_dict:
             if delete_dict[coin]["num_folders"] > num_keep:
@@ -612,9 +612,9 @@ class FreqaiDataDrawer:
         elif self.model_type == "pytorch":
             import torch
 
-            zip = torch.load(dk.data_path / f"{dk.model_filename}_model.zip")
-            model = zip["pytrainer"]
-            model = model.load_from_checkpoint(zip)
+            zipfile = torch.load(dk.data_path / f"{dk.model_filename}_model.zip")
+            model = zipfile["pytrainer"]
+            model = model.load_from_checkpoint(zipfile)
 
         if not model:
             raise OperationalException(

freqtrade/freqai/tensorboard/tensorboard.py

@@ -45,10 +45,10 @@ class TensorBoardCallback(BaseTensorBoardCallback):
             return False
 
         evals = ["validation", "train"]
-        for metric, eval in zip(evals_log.items(), evals):
+        for metric, eval_ in zip(evals_log.items(), evals):
             for metric_name, log in metric[1].items():
                 score = log[-1][0] if isinstance(log[-1], tuple) else log[-1]
-                self.writer.add_scalar(f"{eval}-{metric_name}", score, epoch)
+                self.writer.add_scalar(f"{eval_}-{metric_name}", score, epoch)
 
         return False
 

freqtrade/freqai/torch/PyTorchModelTrainer.py

@@ -148,7 +148,8 @@ class PyTorchModelTrainer(PyTorchTrainerInterface):
         the motivation here is that `n_steps` is easier to optimize and keep stable,
         across different n_obs - the number of data points.
         """
-        assert isinstance(self.n_steps, int), "Either `n_steps` or `n_epochs` should be set."
+        if not isinstance(self.n_steps, int):
+            raise ValueError("Either `n_steps` or `n_epochs` should be set.")
         n_batches = n_obs // self.batch_size
         n_epochs = max(self.n_steps // n_batches, 1)
         if n_epochs <= 10:

freqtrade/freqtradebot.py

@@ -168,6 +168,8 @@ class FreqtradeBot(LoggingMixin):
                     t = str(time(time_slot, minutes, 2))
                     self._schedule.every().day.at(t).do(update)
 
+        self._schedule.every().day.at("00:02").do(self.exchange.ws_connection_reset)
+
         self.strategy.ft_bot_start()
         # Initialize protections AFTER bot start - otherwise parameters are not loaded.
         self.protections = ProtectionManager(self.config, self.strategy.protections)
@@ -217,7 +219,7 @@ class FreqtradeBot(LoggingMixin):
         except Exception:
             # Exceptions here will be happening if the db disappeared.
             # At which point we can no longer commit anyway.
-            pass
+            logger.exception("Error during cleanup")
 
     def startup(self) -> None:
         """
@@ -289,7 +291,6 @@ class FreqtradeBot(LoggingMixin):
         # Then looking for entry opportunities
         if self.get_free_open_trades():
             self.enter_positions()
-        if self.trading_mode == TradingMode.FUTURES:
         self._schedule.run_pending()
         Trade.commit()
         self.rpc.process_msg_queue(self.dataprovider._msg_queue)
@@ -2369,6 +2370,18 @@ class FreqtradeBot(LoggingMixin):
             trade, order, order_obj, order_amount, order.get("trades", [])
         )
 
+    def _trades_valid_for_fee(self, trades: List[Dict[str, Any]]) -> bool:
+        """
+        Check if trades are valid for fee detection.
+        :return: True if trades are valid for fee detection, False otherwise
+        """
+        if not trades:
+            return False
+        # We expect amount and cost to be present in all trade objects.
+        if any(trade.get("amount") is None or trade.get("cost") is None for trade in trades):
+            return False
+        return True
+
     def fee_detection_from_trades(
         self, trade: Trade, order: Dict, order_obj: Order, order_amount: float, trades: List
     ) -> Optional[float]:
@@ -2376,7 +2389,7 @@ class FreqtradeBot(LoggingMixin):
         fee-detection fallback to Trades.
         Either uses provided trades list or the result of fetch_my_trades to get correct fee.
         """
-        if not trades:
+        if not self._trades_valid_for_fee(trades):
             trades = self.exchange.get_trades_for_order(
                 self.exchange.get_order_id_conditional(order), trade.pair, order_obj.order_date
             )

freqtrade/misc.py

@@ -33,7 +33,7 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool =
         if log:
             logger.info(f'dumping json to "{filename}"')
 
-        with gzip.open(filename, "w") as fpz:
+        with gzip.open(filename, "wt", encoding="utf-8") as fpz:
             rapidjson.dump(data, fpz, default=str, number_mode=rapidjson.NM_NATIVE)
     else:
         if log:
@@ -60,7 +60,7 @@ def file_dump_joblib(filename: Path, data: Any, log: bool = True) -> None:
     logger.debug(f'done joblib dump to "{filename}"')
 
 
-def json_load(datafile: Union[gzip.GzipFile, TextIO]) -> Any:
+def json_load(datafile: TextIO) -> Any:
     """
     load data with rapidjson
     Use this to have a consistent experience,
@@ -77,7 +77,7 @@ def file_load_json(file: Path):
     # Try gzip file first, otherwise regular json file.
     if gzipfile.is_file():
         logger.debug(f"Loading historical data from file {gzipfile}")
-        with gzip.open(gzipfile) as datafile:
+        with gzip.open(gzipfile, "rt", encoding="utf-8") as datafile:
             pairdata = json_load(datafile)
     elif file.is_file():
         logger.debug(f"Loading historical data from file {file}")

freqtrade/optimize/analysis/lookahead_helpers.py

@@ -4,11 +4,13 @@ from pathlib import Path
 from typing import Any, Dict, List
 
 import pandas as pd
+from rich.text import Text
 
 from freqtrade.constants import Config
 from freqtrade.exceptions import OperationalException
 from freqtrade.optimize.analysis.lookahead import LookaheadAnalysis
 from freqtrade.resolvers import StrategyResolver
+from freqtrade.util import print_rich_table
 
 
 logger = logging.getLogger(__name__)
@@ -53,18 +55,18 @@ class LookaheadAnalysisSubFunctions:
                     [
                         inst.strategy_obj["location"].parts[-1],
                         inst.strategy_obj["name"],
-                        inst.current_analysis.has_bias,
+                        Text("Yes", style="bold red")
+                        if inst.current_analysis.has_bias
+                        else Text("No", style="bold green"),
                         inst.current_analysis.total_signals,
                         inst.current_analysis.false_entry_signals,
                         inst.current_analysis.false_exit_signals,
                         ", ".join(inst.current_analysis.false_indicators),
                     ]
                 )
-        from tabulate import tabulate
 
-        table = tabulate(data, headers=headers, tablefmt="orgtbl")
-        print(table)
-        return table, headers, data
+        print_rich_table(data, headers, summary="Lookahead Analysis")
+        return data
 
     @staticmethod
     def export_to_csv(config: Dict[str, Any], lookahead_analysis: List[LookaheadAnalysis]):

freqtrade/optimize/analysis/recursive_helpers.py

@@ -7,6 +7,7 @@ from freqtrade.constants import Config
 from freqtrade.exceptions import OperationalException
 from freqtrade.optimize.analysis.recursive import RecursiveAnalysis
 from freqtrade.resolvers import StrategyResolver
+from freqtrade.util import print_rich_table
 
 
 logger = logging.getLogger(__name__)
@@ -16,9 +17,9 @@ class RecursiveAnalysisSubFunctions:
     @staticmethod
     def text_table_recursive_analysis_instances(recursive_instances: List[RecursiveAnalysis]):
         startups = recursive_instances[0]._startup_candle
-        headers = ["indicators"]
+        headers = ["Indicators"]
         for candle in startups:
-            headers.append(candle)
+            headers.append(str(candle))
 
         data = []
         for inst in recursive_instances:
@@ -30,13 +31,11 @@ class RecursiveAnalysisSubFunctions:
                     data.append(temp_data)
 
         if len(data) > 0:
-            from tabulate import tabulate
+            print_rich_table(data, headers, summary="Recursive Analysis")
 
-            table = tabulate(data, headers=headers, tablefmt="orgtbl")
-            print(table)
-            return table, headers, data
+            return data
 
-        return None, None, data
+        return data
 
     @staticmethod
     def calculate_config_overrides(config: Config):

freqtrade/optimize/backtest_caching.py

@@ -13,7 +13,7 @@ def get_strategy_run_id(strategy) -> str:
     :param strategy: strategy object.
     :return: hex string id.
     """
-    digest = hashlib.sha1()
+    digest = hashlib.sha1()  # noqa: S324
     config = deepcopy(strategy.config)
 
     # Options that have no impact on results of individual backtest.

freqtrade/optimize/backtesting.py

@@ -217,8 +217,6 @@ class Backtesting:
             raise OperationalException(
                 "VolumePairList not allowed for backtesting. Please use StaticPairList instead."
             )
-        if "PerformanceFilter" in self.pairlists.name_list:
-            raise OperationalException("PerformanceFilter not allowed for backtesting.")
 
         if len(self.strategylist) > 1 and "PrecisionFilter" in self.pairlists.name_list:
             raise OperationalException(
@@ -467,25 +465,25 @@ class Backtesting:
         return data
 
     def _get_close_rate(
-        self, row: Tuple, trade: LocalTrade, exit: ExitCheckTuple, trade_dur: int
+        self, row: Tuple, trade: LocalTrade, exit_: ExitCheckTuple, trade_dur: int
     ) -> float:
         """
         Get close rate for backtesting result
         """
         # Special handling if high or low hit STOP_LOSS or ROI
-        if exit.exit_type in (
+        if exit_.exit_type in (
             ExitType.STOP_LOSS,
             ExitType.TRAILING_STOP_LOSS,
             ExitType.LIQUIDATION,
         ):
-            return self._get_close_rate_for_stoploss(row, trade, exit, trade_dur)
-        elif exit.exit_type == (ExitType.ROI):
-            return self._get_close_rate_for_roi(row, trade, exit, trade_dur)
+            return self._get_close_rate_for_stoploss(row, trade, exit_, trade_dur)
+        elif exit_.exit_type == (ExitType.ROI):
+            return self._get_close_rate_for_roi(row, trade, exit_, trade_dur)
         else:
             return row[OPEN_IDX]
 
     def _get_close_rate_for_stoploss(
-        self, row: Tuple, trade: LocalTrade, exit: ExitCheckTuple, trade_dur: int
+        self, row: Tuple, trade: LocalTrade, exit_: ExitCheckTuple, trade_dur: int
     ) -> float:
         # our stoploss was already lower than candle high,
         # possibly due to a cancelled trade exit.
@@ -493,7 +491,7 @@ class Backtesting:
         is_short = trade.is_short or False
         leverage = trade.leverage or 1.0
         side_1 = -1 if is_short else 1
-        if exit.exit_type == ExitType.LIQUIDATION and trade.liquidation_price:
+        if exit_.exit_type == ExitType.LIQUIDATION and trade.liquidation_price:
             stoploss_value = trade.liquidation_price
         else:
             stoploss_value = trade.stop_loss
@@ -508,7 +506,7 @@ class Backtesting:
         # Special case: trailing triggers within same candle as trade opened. Assume most
         # pessimistic price movement, which is moving just enough to arm stoploss and
         # immediately going down to stop price.
-        if exit.exit_type == ExitType.TRAILING_STOP_LOSS and trade_dur == 0:
+        if exit_.exit_type == ExitType.TRAILING_STOP_LOSS and trade_dur == 0:
             if (
                 not self.strategy.use_custom_stoploss
                 and self.strategy.trailing_stop
@@ -539,7 +537,7 @@ class Backtesting:
         return stoploss_value
 
     def _get_close_rate_for_roi(
-        self, row: Tuple, trade: LocalTrade, exit: ExitCheckTuple, trade_dur: int
+        self, row: Tuple, trade: LocalTrade, exit_: ExitCheckTuple, trade_dur: int
     ) -> float:
         is_short = trade.is_short or False
         leverage = trade.leverage or 1.0

freqtrade/optimize/edge_cli.py

@@ -52,4 +52,4 @@ class EdgeCli:
         result = self.edge.calculate(self.config["exchange"]["pair_whitelist"])
         if result:
             print("")  # blank line for readability
-            print(generate_edge_table(self.edge._cached_pairs))
+            generate_edge_table(self.edge._cached_pairs)

freqtrade/optimize/hyperopt.py

@@ -14,19 +14,11 @@ from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
 import rapidjson
-from colorama import init as colorama_init
 from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects
 from joblib.externals import cloudpickle
 from pandas import DataFrame
-from rich.progress import (
-    BarColumn,
-    MofNCompleteColumn,
-    Progress,
-    TaskProgressColumn,
-    TextColumn,
-    TimeElapsedColumn,
-    TimeRemainingColumn,
-)
+from rich.align import Align
+from rich.console import Console
 
 from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN, Config
 from freqtrade.data.converter import trim_dataframes
@@ -40,6 +32,7 @@ from freqtrade.optimize.backtesting import Backtesting
 # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
 from freqtrade.optimize.hyperopt_auto import HyperOptAuto
 from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss
+from freqtrade.optimize.hyperopt_output import HyperoptOutput
 from freqtrade.optimize.hyperopt_tools import (
     HyperoptStateContainer,
     HyperoptTools,
@@ -47,6 +40,7 @@ from freqtrade.optimize.hyperopt_tools import (
 )
 from freqtrade.optimize.optimize_reports import generate_strategy_stats
 from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver
+from freqtrade.util import get_progress_tracker
 
 
 # Suppress scikit-learn FutureWarnings from skopt
@@ -86,6 +80,8 @@ class Hyperopt:
         self.max_open_trades_space: List[Dimension] = []
         self.dimensions: List[Dimension] = []
 
+        self._hyper_out: HyperoptOutput = HyperoptOutput()
+
         self.config = config
         self.min_date: datetime
         self.max_date: datetime
@@ -260,7 +256,7 @@ class Hyperopt:
             result["max_open_trades"] = {"max_open_trades": strategy.max_open_trades}
         return result
 
-    def print_results(self, results) -> None:
+    def print_results(self, results: Dict[str, Any]) -> None:
         """
         Log results if it is better than any previous evaluation
         TODO: this should be moved to HyperoptTools too
@@ -268,17 +264,12 @@ class Hyperopt:
         is_best = results["is_best"]
 
         if self.print_all or is_best:
-            print(
-                HyperoptTools.get_result_table(
+            self._hyper_out.add_data(
                 self.config,
-                    results,
+                [results],
                 self.total_epochs,
                 self.print_all,
-                    self.print_colorized,
-                    self.hyperopt_table_header,
             )
-            )
-            self.hyperopt_table_header = 2
 
     def init_spaces(self):
         """
@@ -489,7 +480,7 @@ class Hyperopt:
         )
 
     def _set_random_state(self, random_state: Optional[int]) -> int:
-        return random_state or random.randint(1, 2**16 - 1)
+        return random_state or random.randint(1, 2**16 - 1)  # noqa: S311
 
     def advise_and_trim(self, data: Dict[str, DataFrame]) -> Dict[str, DataFrame]:
         preprocessed = self.backtesting.strategy.advise_all_indicators(data)
@@ -626,25 +617,18 @@ class Hyperopt:
 
         self.opt = self.get_optimizer(self.dimensions, config_jobs)
 
-        if self.print_colorized:
-            colorama_init(autoreset=True)
-
         try:
             with Parallel(n_jobs=config_jobs) as parallel:
                 jobs = parallel._effective_n_jobs()
                 logger.info(f"Effective number of parallel workers used: {jobs}")
+                console = Console(
+                    color_system="auto" if self.print_colorized else None,
+                )
 
                 # Define progressbar
-                with Progress(
-                    TextColumn("[progress.description]{task.description}"),
-                    BarColumn(bar_width=None),
-                    MofNCompleteColumn(),
-                    TaskProgressColumn(),
-                    "•",
-                    TimeElapsedColumn(),
-                    "•",
-                    TimeRemainingColumn(),
-                    expand=True,
+                with get_progress_tracker(
+                    console=console,
+                    cust_objs=[Align.center(self._hyper_out.table)],
                 ) as pbar:
                     task = pbar.add_task("Epochs", total=self.total_epochs)
 

freqtrade/optimize/hyperopt_output.py

@@ -0,0 +1,123 @@
+import sys
+from typing import List, Optional, Union
+
+from rich.console import Console
+from rich.table import Table
+from rich.text import Text
+
+from freqtrade.constants import Config
+from freqtrade.optimize.optimize_reports import generate_wins_draws_losses
+from freqtrade.util import fmt_coin
+
+
+class HyperoptOutput:
+    def __init__(self):
+        self.table = Table(
+            title="Hyperopt results",
+        )
+        # Headers
+        self.table.add_column("Best", justify="left")
+        self.table.add_column("Epoch", justify="right")
+        self.table.add_column("Trades", justify="right")
+        self.table.add_column("Win  Draw  Loss  Win%", justify="right")
+        self.table.add_column("Avg profit", justify="right")
+        self.table.add_column("Profit", justify="right")
+        self.table.add_column("Avg duration", justify="right")
+        self.table.add_column("Objective", justify="right")
+        self.table.add_column("Max Drawdown (Acct)", justify="right")
+
+    def _add_row(self, data: List[Union[str, Text]]):
+        """Add single row"""
+        row_to_add: List[Union[str, Text]] = [r if isinstance(r, Text) else str(r) for r in data]
+
+        self.table.add_row(*row_to_add)
+
+    def _add_rows(self, data: List[List[Union[str, Text]]]):
+        """add multiple rows"""
+        for row in data:
+            self._add_row(row)
+
+    def print(self, console: Optional[Console] = None, *, print_colorized=True):
+        if not console:
+            console = Console(
+                color_system="auto" if print_colorized else None,
+                width=200 if "pytest" in sys.modules else None,
+            )
+
+        console.print(self.table)
+
+    def add_data(
+        self,
+        config: Config,
+        results: list,
+        total_epochs: int,
+        highlight_best: bool,
+    ) -> None:
+        """Format one or multiple rows and add them"""
+        stake_currency = config["stake_currency"]
+
+        for r in results:
+            self.table.add_row(
+                *[
+                    # "Best":
+                    (
+                        ("*" if r["is_initial_point"] or r["is_random"] else "")
+                        + (" Best" if r["is_best"] else "")
+                    ).lstrip(),
+                    # "Epoch":
+                    f"{r['current_epoch']}/{total_epochs}",
+                    # "Trades":
+                    str(r["results_metrics"]["total_trades"]),
+                    # "Win  Draw  Loss  Win%":
+                    generate_wins_draws_losses(
+                        r["results_metrics"]["wins"],
+                        r["results_metrics"]["draws"],
+                        r["results_metrics"]["losses"],
+                    ),
+                    # "Avg profit":
+                    f"{r['results_metrics']['profit_mean']:.2%}"
+                    if r["results_metrics"]["profit_mean"] is not None
+                    else "--",
+                    # "Profit":
+                    Text(
+                        "{} {}".format(
+                            fmt_coin(
+                                r["results_metrics"]["profit_total_abs"],
+                                stake_currency,
+                                keep_trailing_zeros=True,
+                            ),
+                            f"({r['results_metrics']['profit_total']:,.2%})".rjust(10, " "),
+                        )
+                        if r["results_metrics"].get("profit_total_abs", 0) != 0.0
+                        else "--",
+                        style=(
+                            "green"
+                            if r["results_metrics"].get("profit_total_abs", 0) > 0
+                            else "red"
+                        )
+                        if not r["is_best"]
+                        else "",
+                    ),
+                    # "Avg duration":
+                    str(r["results_metrics"]["holding_avg"]),
+                    # "Objective":
+                    f"{r['loss']:,.5f}" if r["loss"] != 100000 else "N/A",
+                    # "Max Drawdown (Acct)":
+                    "{} {}".format(
+                        fmt_coin(
+                            r["results_metrics"]["max_drawdown_abs"],
+                            stake_currency,
+                            keep_trailing_zeros=True,
+                        ),
+                        (f"({r['results_metrics']['max_drawdown_account']:,.2%})").rjust(10, " "),
+                    )
+                    if r["results_metrics"]["max_drawdown_account"] != 0.0
+                    else "--",
+                ],
+                style=" ".join(
+                    [
+                        "bold gold1" if r["is_best"] and highlight_best else "",
+                        "italic " if r["is_initial_point"] else "",
+                    ]
+                ),
+            )

freqtrade/optimize/hyperopt_tools.py

@@ -5,10 +5,7 @@ from pathlib import Path
 from typing import Any, Dict, Iterator, List, Optional, Tuple
 
 import numpy as np
-import pandas as pd
 import rapidjson
-import tabulate
-from colorama import Fore, Style
 from pandas import isna, json_normalize
 
 from freqtrade.constants import FTHYPT_FILEVERSION, Config
@@ -16,8 +13,6 @@ from freqtrade.enums import HyperoptState
 from freqtrade.exceptions import OperationalException
 from freqtrade.misc import deep_merge_dicts, round_dict, safe_value_fallback2
 from freqtrade.optimize.hyperopt_epoch_filters import hyperopt_filter_epochs
-from freqtrade.optimize.optimize_reports import generate_wins_draws_losses
-from freqtrade.util import fmt_coin
 
 
 logger = logging.getLogger(__name__)
@@ -357,175 +352,6 @@ class HyperoptTools:
             + f"Objective: {results['loss']:.5f}"
         )
 
-    @staticmethod
-    def prepare_trials_columns(trials: pd.DataFrame) -> pd.DataFrame:
-        trials["Best"] = ""
-
-        if "results_metrics.winsdrawslosses" not in trials.columns:
-            # Ensure compatibility with older versions of hyperopt results
-            trials["results_metrics.winsdrawslosses"] = "N/A"
-
-        has_account_drawdown = "results_metrics.max_drawdown_account" in trials.columns
-        if not has_account_drawdown:
-            # Ensure compatibility with older versions of hyperopt results
-            trials["results_metrics.max_drawdown_account"] = None
-        if "is_random" not in trials.columns:
-            trials["is_random"] = False
-
-        # New mode, using backtest result for metrics
-        trials["results_metrics.winsdrawslosses"] = trials.apply(
-            lambda x: generate_wins_draws_losses(
-                x["results_metrics.wins"], x["results_metrics.draws"], x["results_metrics.losses"]
-            ),
-            axis=1,
-        )
-
-        trials = trials[
-            [
-                "Best",
-                "current_epoch",
-                "results_metrics.total_trades",
-                "results_metrics.winsdrawslosses",
-                "results_metrics.profit_mean",
-                "results_metrics.profit_total_abs",
-                "results_metrics.profit_total",
-                "results_metrics.holding_avg",
-                "results_metrics.max_drawdown_account",
-                "results_metrics.max_drawdown_abs",
-                "loss",
-                "is_initial_point",
-                "is_random",
-                "is_best",
-            ]
-        ]
-
-        trials.columns = [
-            "Best",
-            "Epoch",
-            "Trades",
-            " Win  Draw  Loss  Win%",
-            "Avg profit",
-            "Total profit",
-            "Profit",
-            "Avg duration",
-            "max_drawdown_account",
-            "max_drawdown_abs",
-            "Objective",
-            "is_initial_point",
-            "is_random",
-            "is_best",
-        ]
-
-        return trials
-
-    @staticmethod
-    def get_result_table(
-        config: Config,
-        results: list,
-        total_epochs: int,
-        highlight_best: bool,
-        print_colorized: bool,
-        remove_header: int,
-    ) -> str:
-        """
-        Log result table
-        """
-        if not results:
-            return ""
-
-        tabulate.PRESERVE_WHITESPACE = True
-        trials = json_normalize(results, max_level=1)
-
-        trials = HyperoptTools.prepare_trials_columns(trials)
-
-        trials["is_profit"] = False
-        trials.loc[trials["is_initial_point"] | trials["is_random"], "Best"] = "*     "
-        trials.loc[trials["is_best"], "Best"] = "Best"
-        trials.loc[
-            (trials["is_initial_point"] | trials["is_random"]) & trials["is_best"], "Best"
-        ] = "* Best"
-        trials.loc[trials["Total profit"] > 0, "is_profit"] = True
-        trials["Trades"] = trials["Trades"].astype(str)
-        # perc_multi = 1 if legacy_mode else 100
-        trials["Epoch"] = trials["Epoch"].apply(
-            lambda x: "{}/{}".format(str(x).rjust(len(str(total_epochs)), " "), total_epochs)
-        )
-        trials["Avg profit"] = trials["Avg profit"].apply(
-            lambda x: f"{x:,.2%}".rjust(7, " ") if not isna(x) else "--".rjust(7, " ")
-        )
-        trials["Avg duration"] = trials["Avg duration"].apply(
-            lambda x: (
-                f"{x:,.1f} m".rjust(7, " ")
-                if isinstance(x, float)
-                else f"{x}"
-                if not isna(x)
-                else "--".rjust(7, " ")
-            )
-        )
-        trials["Objective"] = trials["Objective"].apply(
-            lambda x: f"{x:,.5f}".rjust(8, " ") if x != 100000 else "N/A".rjust(8, " ")
-        )
-
-        stake_currency = config["stake_currency"]
-
-        trials["Max Drawdown (Acct)"] = trials.apply(
-            lambda x: (
-                "{} {}".format(
-                    fmt_coin(x["max_drawdown_abs"], stake_currency, keep_trailing_zeros=True),
-                    (f"({x['max_drawdown_account']:,.2%})").rjust(10, " "),
-                ).rjust(25 + len(stake_currency))
-                if x["max_drawdown_account"] != 0.0
-                else "--".rjust(25 + len(stake_currency))
-            ),
-            axis=1,
-        )
-
-        trials = trials.drop(columns=["max_drawdown_abs", "max_drawdown_account"])
-
-        trials["Profit"] = trials.apply(
-            lambda x: (
-                "{} {}".format(
-                    fmt_coin(x["Total profit"], stake_currency, keep_trailing_zeros=True),
-                    f"({x['Profit']:,.2%})".rjust(10, " "),
-                ).rjust(25 + len(stake_currency))
-                if x["Total profit"] != 0.0
-                else "--".rjust(25 + len(stake_currency))
-            ),
-            axis=1,
-        )
-        trials = trials.drop(columns=["Total profit"])
-
-        if print_colorized:
-            trials2 = trials.astype(str)
-            for i in range(len(trials)):
-                if trials.loc[i]["is_profit"]:
-                    for j in range(len(trials.loc[i]) - 3):
-                        trials2.iat[i, j] = f"{Fore.GREEN}{str(trials.iloc[i, j])}{Fore.RESET}"
-                if trials.loc[i]["is_best"] and highlight_best:
-                    for j in range(len(trials.loc[i]) - 3):
-                        trials2.iat[i, j] = (
-                            f"{Style.BRIGHT}{str(trials.iloc[i, j])}{Style.RESET_ALL}"
-                        )
-            trials = trials2
-            del trials2
-        trials = trials.drop(columns=["is_initial_point", "is_best", "is_profit", "is_random"])
-        if remove_header > 0:
-            table = tabulate.tabulate(
-                trials.to_dict(orient="list"), tablefmt="orgtbl", headers="keys", stralign="right"
-            )
-
-            table = table.split("\n", remove_header)[remove_header]
-        elif remove_header < 0:
-            table = tabulate.tabulate(
-                trials.to_dict(orient="list"), tablefmt="psql", headers="keys", stralign="right"
-            )
-            table = "\n".join(table.split("\n")[0:remove_header])
-        else:
-            table = tabulate.tabulate(
-                trials.to_dict(orient="list"), tablefmt="psql", headers="keys", stralign="right"
-            )
-        return table
-
     @staticmethod
     def export_csv_file(config: Config, results: list, csv_file: str) -> None:
         """

freqtrade/optimize/optimize_reports/bt_output.py

@@ -1,12 +1,10 @@
 import logging
-from typing import Any, Dict, List
-
-from tabulate import tabulate
+from typing import Any, Dict, List, Literal, Union
 
 from freqtrade.constants import UNLIMITED_STAKE_AMOUNT, Config
 from freqtrade.optimize.optimize_reports.optimize_reports import generate_periodic_breakdown_stats
 from freqtrade.types import BacktestResultType
-from freqtrade.util import decimals_per_coin, fmt_coin
+from freqtrade.util import decimals_per_coin, fmt_coin, print_rich_table
 
 
 logger = logging.getLogger(__name__)
@@ -20,13 +18,13 @@ def _get_line_floatfmt(stake_currency: str) -> List[str]:
 
 
 def _get_line_header(
-    first_column: str, stake_currency: str, direction: str = "Entries"
+    first_column: Union[str, List[str]], stake_currency: str, direction: str = "Trades"
 ) -> List[str]:
     """
     Generate header lines (goes in line with _generate_result_line())
     """
     return [
-        first_column,
+        *([first_column] if isinstance(first_column, str) else first_column),
         direction,
         "Avg Profit %",
         f"Tot Profit {stake_currency}",
@@ -46,22 +44,23 @@ def generate_wins_draws_losses(wins, draws, losses):
     return f"{wins:>4}  {draws:>4}  {losses:>4}  {wl_ratio:>4}"
 
 
-def text_table_bt_results(pair_results: List[Dict[str, Any]], stake_currency: str) -> str:
+def text_table_bt_results(
+    pair_results: List[Dict[str, Any]], stake_currency: str, title: str
+) -> None:
     """
     Generates and returns a text table for the given backtest data and the results dataframe
     :param pair_results: List of Dictionaries - one entry per pair + final TOTAL row
     :param stake_currency: stake-currency - used to correctly name headers
-    :return: pretty printed table with tabulate as string
+    :param title: Title of the table
     """
 
-    headers = _get_line_header("Pair", stake_currency)
-    floatfmt = _get_line_floatfmt(stake_currency)
+    headers = _get_line_header("Pair", stake_currency, "Trades")
     output = [
         [
             t["key"],
             t["trades"],
             t["profit_mean_pct"],
-            t["profit_total_abs"],
+            f"{t['profit_total_abs']:.{decimals_per_coin(stake_currency)}f}",
             t["profit_total_pct"],
             t["duration_avg"],
             generate_wins_draws_losses(t["wins"], t["draws"], t["losses"]),
@@ -69,34 +68,50 @@ def text_table_bt_results(pair_results: List[Dict[str, Any]], stake_currency: st
         for t in pair_results
     ]
     # Ignore type as floatfmt does allow tuples but mypy does not know that
-    return tabulate(output, headers=headers, floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")
+    print_rich_table(output, headers, summary=title)
 
 
-def text_table_tags(tag_type: str, tag_results: List[Dict[str, Any]], stake_currency: str) -> str:
+def text_table_tags(
+    tag_type: Literal["enter_tag", "exit_tag", "mix_tag"],
+    tag_results: List[Dict[str, Any]],
+    stake_currency: str,
+) -> None:
     """
     Generates and returns a text table for the given backtest data and the results dataframe
     :param pair_results: List of Dictionaries - one entry per pair + final TOTAL row
     :param stake_currency: stake-currency - used to correctly name headers
-    :return: pretty printed table with tabulate as string
     """
-    fallback: str = ""
-    if tag_type == "enter_tag":
-        headers = _get_line_header("TAG", stake_currency)
-    else:
-        headers = _get_line_header("Exit Reason", stake_currency, "Exits")
-        fallback = "exit_reason"
-
     floatfmt = _get_line_floatfmt(stake_currency)
+    fallback: str = ""
+    is_list = False
+    if tag_type == "enter_tag":
+        title = "Enter Tag"
+        headers = _get_line_header(title, stake_currency, "Entries")
+    elif tag_type == "exit_tag":
+        title = "Exit Reason"
+        headers = _get_line_header(title, stake_currency, "Exits")
+        fallback = "exit_reason"
+    else:
+        # Mix tag
+        title = "Mixed Tag"
+        headers = _get_line_header(["Enter Tag", "Exit Reason"], stake_currency, "Trades")
+        floatfmt.insert(0, "s")
+        is_list = True
+
     output = [
         [
+            *(
                 (
-                t["key"]
+                    (t["key"] if isinstance(t["key"], list) else [t["key"], ""])
+                    if is_list
+                    else [t["key"]]
+                )
                 if t.get("key") is not None and len(str(t["key"])) > 0
-                else t.get(fallback, "OTHER")
+                else [t.get(fallback, "OTHER")]
             ),
             t["trades"],
             t["profit_mean_pct"],
-            t["profit_total_abs"],
+            f"{t['profit_total_abs']:.{decimals_per_coin(stake_currency)}f}",
             t["profit_total_pct"],
             t.get("duration_avg"),
             generate_wins_draws_losses(t["wins"], t["draws"], t["losses"]),
@@ -104,17 +119,16 @@ def text_table_tags(tag_type: str, tag_results: List[Dict[str, Any]], stake_curr
         for t in tag_results
     ]
     # Ignore type as floatfmt does allow tuples but mypy does not know that
-    return tabulate(output, headers=headers, floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")
+    print_rich_table(output, headers, summary=f"{title.upper()} STATS")
 
 
 def text_table_periodic_breakdown(
     days_breakdown_stats: List[Dict[str, Any]], stake_currency: str, period: str
-) -> str:
+) -> None:
     """
     Generate small table with Backtest results by days
     :param days_breakdown_stats: Days breakdown metrics
     :param stake_currency: Stakecurrency used
-    :return: pretty printed table with tabulate as string
     """
     headers = [
         period.capitalize(),
@@ -133,18 +147,16 @@ def text_table_periodic_breakdown(
         ]
         for d in days_breakdown_stats
     ]
-    return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right")
+    print_rich_table(output, headers, summary=f"{period.upper()} BREAKDOWN")
 
 
-def text_table_strategy(strategy_results, stake_currency: str) -> str:
+def text_table_strategy(strategy_results, stake_currency: str, title: str):
     """
     Generate summary table per strategy
     :param strategy_results: Dict of <Strategyname: DataFrame> containing results for all strategies
     :param stake_currency: stake-currency - used to correctly name headers
-    :return: pretty printed table with tabulate as string
     """
-    floatfmt = _get_line_floatfmt(stake_currency)
-    headers = _get_line_header("Strategy", stake_currency)
+    headers = _get_line_header("Strategy", stake_currency, "Trades")
     # _get_line_header() is also used for per-pair summary. Per-pair drawdown is mostly useless
     # therefore we slip this column in only for strategy summary here.
     headers.append("Drawdown")
@@ -167,8 +179,8 @@ def text_table_strategy(strategy_results, stake_currency: str) -> str:
         [
             t["key"],
             t["trades"],
-            t["profit_mean_pct"],
-            t["profit_total_abs"],
+            f"{t['profit_mean_pct']:.2f}",
+            f"{t['profit_total_abs']:.{decimals_per_coin(stake_currency)}f}",
             t["profit_total_pct"],
             t["duration_avg"],
             generate_wins_draws_losses(t["wins"], t["draws"], t["losses"]),
@@ -176,11 +188,10 @@ def text_table_strategy(strategy_results, stake_currency: str) -> str:
         ]
         for t, drawdown in zip(strategy_results, drawdown)
     ]
-    # Ignore type as floatfmt does allow tuples but mypy does not know that
-    return tabulate(output, headers=headers, floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")
+    print_rich_table(output, headers, summary=title)
 
 
-def text_table_add_metrics(strat_results: Dict) -> str:
+def text_table_add_metrics(strat_results: Dict) -> None:
     if len(strat_results["trades"]) > 0:
         best_trade = max(strat_results["trades"], key=lambda x: x["profit_ratio"])
         worst_trade = min(strat_results["trades"], key=lambda x: x["profit_ratio"])
@@ -362,8 +373,8 @@ def text_table_add_metrics(strat_results: Dict) -> str:
             *drawdown_metrics,
             ("Market change", f"{strat_results['market_change']:.2%}"),
         ]
+        print_rich_table(metrics, ["Metric", "Value"], summary="SUMMARY METRICS", justify="left")
 
-        return tabulate(metrics, headers=["Metric", "Value"], tablefmt="orgtbl")
     else:
         start_balance = fmt_coin(strat_results["starting_balance"], strat_results["stake_currency"])
         stake_amount = (
@@ -377,7 +388,21 @@ def text_table_add_metrics(strat_results: Dict) -> str:
             f"Your starting balance was {start_balance}, "
             f"and your stake was {stake_amount}."
         )
-        return message
+        print(message)
+
+
+def _show_tag_subresults(results: Dict[str, Any], stake_currency: str):
+    """
+    Print tag subresults (enter_tag, exit_reason_summary, mix_tag_stats)
+    """
+    if (enter_tags := results.get("results_per_enter_tag")) is not None:
+        text_table_tags("enter_tag", enter_tags, stake_currency)
+
+    if (exit_reasons := results.get("exit_reason_summary")) is not None:
+        text_table_tags("exit_tag", exit_reasons, stake_currency)
+
+    if (mix_tag := results.get("mix_tag_stats")) is not None:
+        text_table_tags("mix_tag", mix_tag, stake_currency)
 
 
 def show_backtest_result(
@@ -388,29 +413,14 @@ def show_backtest_result(
     """
     # Print results
     print(f"Result for strategy {strategy}")
-    table = text_table_bt_results(results["results_per_pair"], stake_currency=stake_currency)
-    if isinstance(table, str):
-        print(" BACKTESTING REPORT ".center(len(table.splitlines()[0]), "="))
-    print(table)
+    text_table_bt_results(
+        results["results_per_pair"], stake_currency=stake_currency, title="BACKTESTING REPORT"
+    )
+    text_table_bt_results(
+        results["left_open_trades"], stake_currency=stake_currency, title="LEFT OPEN TRADES REPORT"
+    )
 
-    table = text_table_bt_results(results["left_open_trades"], stake_currency=stake_currency)
-    if isinstance(table, str) and len(table) > 0:
-        print(" LEFT OPEN TRADES REPORT ".center(len(table.splitlines()[0]), "="))
-    print(table)
-
-    if (enter_tags := results.get("results_per_enter_tag")) is not None:
-        table = text_table_tags("enter_tag", enter_tags, stake_currency)
-
-        if isinstance(table, str) and len(table) > 0:
-            print(" ENTER TAG STATS ".center(len(table.splitlines()[0]), "="))
-        print(table)
-
-    if (exit_reasons := results.get("exit_reason_summary")) is not None:
-        table = text_table_tags("exit_tag", exit_reasons, stake_currency)
-
-        if isinstance(table, str) and len(table) > 0:
-            print(" EXIT REASON STATS ".center(len(table.splitlines()[0]), "="))
-        print(table)
+    _show_tag_subresults(results, stake_currency)
 
     for period in backtest_breakdown:
         if period in results.get("periodic_breakdown", {}):
@@ -419,20 +429,11 @@ def show_backtest_result(
             days_breakdown_stats = generate_periodic_breakdown_stats(
                 trade_list=results["trades"], period=period
             )
-        table = text_table_periodic_breakdown(
+        text_table_periodic_breakdown(
             days_breakdown_stats=days_breakdown_stats, stake_currency=stake_currency, period=period
         )
-        if isinstance(table, str) and len(table) > 0:
-            print(f" {period.upper()} BREAKDOWN ".center(len(table.splitlines()[0]), "="))
-        print(table)
 
-    table = text_table_add_metrics(results)
-    if isinstance(table, str) and len(table) > 0:
-        print(" SUMMARY METRICS ".center(len(table.splitlines()[0]), "="))
-    print(table)
-
-    if isinstance(table, str) and len(table) > 0:
-        print("=" * len(table.splitlines()[0]))
+    text_table_add_metrics(results)
 
     print()
 
@@ -448,15 +449,13 @@ def show_backtest_results(config: Config, backtest_stats: BacktestResultType):
     if len(backtest_stats["strategy"]) > 0:
         # Print Strategy summary table
 
-        table = text_table_strategy(backtest_stats["strategy_comparison"], stake_currency)
         print(
             f"Backtested {results['backtest_start']} -> {results['backtest_end']} |"
             f" Max open trades : {results['max_open_trades']}"
         )
-        print(" STRATEGY SUMMARY ".center(len(table.splitlines()[0]), "="))
-        print(table)
-        print("=" * len(table.splitlines()[0]))
-        print("\nFor more details, please look at the detail tables above")
+        text_table_strategy(
+            backtest_stats["strategy_comparison"], stake_currency, "STRATEGY SUMMARY"
+        )
 
 
 def show_sorted_pairlist(config: Config, backtest_stats: BacktestResultType):
@@ -469,8 +468,7 @@ def show_sorted_pairlist(config: Config, backtest_stats: BacktestResultType):
             print("]")
 
 
-def generate_edge_table(results: dict) -> str:
-    floatfmt = ("s", ".10g", ".2f", ".2f", ".2f", ".2f", "d", "d", "d")
+def generate_edge_table(results: dict) -> None:
     tabular_data = []
     headers = [
         "Pair",
@@ -488,17 +486,13 @@ def generate_edge_table(results: dict) -> str:
             tabular_data.append(
                 [
                     result[0],
-                    result[1].stoploss,
-                    result[1].winrate,
-                    result[1].risk_reward_ratio,
-                    result[1].required_risk_reward,
-                    result[1].expectancy,
+                    f"{result[1].stoploss:.10g}",
+                    f"{result[1].winrate:.2f}",
+                    f"{result[1].risk_reward_ratio:.2f}",
+                    f"{result[1].required_risk_reward:.2f}",
+                    f"{result[1].expectancy:.2f}",
                     result[1].nb_trades,
                     round(result[1].avg_trade_duration),
                 ]
             )
-
-    # Ignore type as floatfmt does allow tuples but mypy does not know that
-    return tabulate(
-        tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="orgtbl", stralign="right"
-    )
+    print_rich_table(tabular_data, headers, summary="EDGE TABLE")

freqtrade/optimize/optimize_reports/optimize_reports.py

@@ -1,7 +1,7 @@
 import logging
 from copy import deepcopy
 from datetime import datetime, timedelta, timezone
-from typing import Any, Dict, List, Tuple, Union
+from typing import Any, Dict, List, Literal, Tuple, Union
 
 import numpy as np
 from pandas import DataFrame, Series, concat, to_datetime
@@ -68,7 +68,9 @@ def generate_rejected_signals(
     return rejected_candles_only
 
 
-def _generate_result_line(result: DataFrame, starting_balance: int, first_column: str) -> Dict:
+def _generate_result_line(
+    result: DataFrame, starting_balance: int, first_column: Union[str, List[str]]
+) -> Dict:
     """
     Generate one result dict, with "first_column" as key.
     """
@@ -141,7 +143,10 @@ def generate_pair_metrics(
 
 
 def generate_tag_metrics(
-    tag_type: str, starting_balance: int, results: DataFrame, skip_nan: bool = False
+    tag_type: Union[Literal["enter_tag", "exit_reason"], List[Literal["enter_tag", "exit_reason"]]],
+    starting_balance: int,
+    results: DataFrame,
+    skip_nan: bool = False,
 ) -> List[Dict]:
     """
     Generates and returns a list of metrics for the given tag trades and the results dataframe
@@ -153,13 +158,14 @@ def generate_tag_metrics(
 
     tabular_data = []
 
-    if tag_type in results.columns:
-        for tag, count in results[tag_type].value_counts().items():
-            result = results[results[tag_type] == tag]
-            if skip_nan and result["profit_abs"].isnull().all():
+    if all(
+        tag in results.columns for tag in (tag_type if isinstance(tag_type, list) else [tag_type])
+    ):
+        for tags, group in results.groupby(tag_type):
+            if skip_nan and group["profit_abs"].isnull().all():
                 continue
 
-            tabular_data.append(_generate_result_line(result, starting_balance, tag))
+            tabular_data.append(_generate_result_line(group, starting_balance, tags))
 
         # Sort by total profit %:
         tabular_data = sorted(tabular_data, key=lambda k: k["profit_total_abs"], reverse=True)
@@ -378,12 +384,18 @@ def generate_strategy_stats(
         skip_nan=False,
     )
 
-    enter_tag_results = generate_tag_metrics(
+    enter_tag_stats = generate_tag_metrics(
         "enter_tag", starting_balance=start_balance, results=results, skip_nan=False
     )
     exit_reason_stats = generate_tag_metrics(
         "exit_reason", starting_balance=start_balance, results=results, skip_nan=False
     )
+    mix_tag_stats = generate_tag_metrics(
+        ["enter_tag", "exit_reason"],
+        starting_balance=start_balance,
+        results=results,
+        skip_nan=False,
+    )
     left_open_results = generate_pair_metrics(
         pairlist,
         stake_currency=stake_currency,
@@ -425,8 +437,9 @@ def generate_strategy_stats(
         "best_pair": best_pair,
         "worst_pair": worst_pair,
         "results_per_pair": pair_results,
-        "results_per_enter_tag": enter_tag_results,
+        "results_per_enter_tag": enter_tag_stats,
         "exit_reason_summary": exit_reason_stats,
+        "mix_tag_stats": mix_tag_stats,
         "left_open_trades": left_open_results,
         "total_trades": len(results),
         "trade_count_long": len(results.loc[~results["is_short"]]),

freqtrade/persistence/models.py

@@ -32,12 +32,12 @@ def get_request_or_thread_id() -> Optional[str]:
     """
     Helper method to get either async context (for fastapi requests), or thread id
     """
-    id = _request_id_ctx_var.get()
-    if id is None:
+    request_id = _request_id_ctx_var.get()
+    if request_id is None:
         # when not in request context - use thread id
-        id = str(threading.current_thread().ident)
+        request_id = str(threading.current_thread().ident)
 
-    return id
+    return request_id
 
 
 _SQL_DOCS_URL = "http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls"

freqtrade/persistence/trade_model.py

@@ -2012,7 +2012,7 @@ class Trade(ModelBase, LocalTrade):
         ).all()
 
         resp: List[Dict] = []
-        for id, enter_tag, exit_reason, profit, profit_abs, count in mix_tag_perf:
+        for _, enter_tag, exit_reason, profit, profit_abs, count in mix_tag_perf:
             enter_tag = enter_tag if enter_tag is not None else "Other"
             exit_reason = exit_reason if exit_reason is not None else "Other"
 

freqtrade/plugins/pairlist/AgeFilter.py

@@ -5,15 +5,15 @@ Minimum age (days listed) pair list filter
 import logging
 from copy import deepcopy
 from datetime import timedelta
-from typing import Any, Dict, List, Optional
+from typing import Dict, List, Optional
 
 from pandas import DataFrame
 
-from freqtrade.constants import Config, ListPairsWithTimeframes
+from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.misc import plural
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import PeriodicCache, dt_floor_day, dt_now, dt_ts
 
 
@@ -21,24 +21,19 @@ logger = logging.getLogger(__name__)
 
 
 class AgeFilter(IPairList):
-    def __init__(
-        self,
-        exchange,
-        pairlistmanager,
-        config: Config,
-        pairlistconfig: Dict[str, Any],
-        pairlist_pos: int,
-    ) -> None:
-        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
+    supports_backtesting = SupportsBacktesting.NO
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
 
         # Checked symbols cache (dictionary of ticker symbol => timestamp)
         self._symbolsChecked: Dict[str, int] = {}
         self._symbolsCheckFailed = PeriodicCache(maxsize=1000, ttl=86_400)
 
-        self._min_days_listed = pairlistconfig.get("min_days_listed", 10)
-        self._max_days_listed = pairlistconfig.get("max_days_listed")
+        self._min_days_listed = self._pairlistconfig.get("min_days_listed", 10)
+        self._max_days_listed = self._pairlistconfig.get("max_days_listed")
 
-        candle_limit = exchange.ohlcv_candle_limit("1d", self._config["candle_type_def"])
+        candle_limit = self._exchange.ohlcv_candle_limit("1d", self._config["candle_type_def"])
         if self._min_days_listed < 1:
             raise OperationalException("AgeFilter requires min_days_listed to be >= 1")
         if self._min_days_listed > candle_limit:

freqtrade/plugins/pairlist/FullTradesFilter.py

@@ -3,27 +3,18 @@ Full trade slots pair list filter
 """
 
 import logging
-from typing import Any, Dict, List
+from typing import List
 
-from freqtrade.constants import Config
 from freqtrade.exchange.types import Tickers
 from freqtrade.persistence import Trade
-from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.plugins.pairlist.IPairList import IPairList, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class FullTradesFilter(IPairList):
-    def __init__(
-        self,
-        exchange,
-        pairlistmanager,
-        config: Config,
-        pairlistconfig: Dict[str, Any],
-        pairlist_pos: int,
-    ) -> None:
-        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
+    supports_backtesting = SupportsBacktesting.NO_ACTION
 
     @property
     def needstickers(self) -> bool:

freqtrade/plugins/pairlist/IPairList.py

@@ -3,8 +3,9 @@ PairList Handler base class
 """
 
 import logging
-from abc import ABC, abstractmethod, abstractproperty
+from abc import ABC, abstractmethod
 from copy import deepcopy
+from enum import Enum
 from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
 
 from freqtrade.constants import Config
@@ -51,8 +52,20 @@ PairlistParameter = Union[
 ]
 
 
+class SupportsBacktesting(str, Enum):
+    """
+    Enum to indicate if a Pairlist Handler supports backtesting.
+    """
+
+    YES = "yes"
+    NO = "no"
+    NO_ACTION = "no_action"
+    BIASED = "biased"
+
+
 class IPairList(LoggingMixin, ABC):
     is_pairlist_generator = False
+    supports_backtesting: SupportsBacktesting = SupportsBacktesting.NO
 
     def __init__(
         self,
@@ -87,7 +100,8 @@ class IPairList(LoggingMixin, ABC):
         """
         return self.__class__.__name__
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def needstickers(self) -> bool:
         """
         Boolean property defining if tickers are necessary.

freqtrade/plugins/pairlist/MarketCapPairList.py

@@ -5,14 +5,13 @@ Provides dynamic pair list based on Market Cap
 """
 
 import logging
-from typing import Any, Dict, List
+from typing import Dict, List
 
 from cachetools import TTLCache
 
-from freqtrade.constants import Config
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util.coin_gecko import FtCoinGeckoApi
 
 
@@ -21,16 +20,10 @@ logger = logging.getLogger(__name__)
 
 class MarketCapPairList(IPairList):
     is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.BIASED
 
-    def __init__(
-        self,
-        exchange,
-        pairlistmanager,
-        config: Config,
-        pairlistconfig: Dict[str, Any],
-        pairlist_pos: int,
-    ) -> None:
-        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
 
         if "number_assets" not in self._pairlistconfig:
             raise OperationalException(
@@ -38,14 +31,14 @@ class MarketCapPairList(IPairList):
                 'for "pairlist.config.number_assets"'
             )
 
-        self._stake_currency = config["stake_currency"]
+        self._stake_currency = self._config["stake_currency"]
         self._number_assets = self._pairlistconfig["number_assets"]
         self._max_rank = self._pairlistconfig.get("max_rank", 30)
         self._refresh_period = self._pairlistconfig.get("refresh_period", 86400)
         self._marketcap_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
         self._def_candletype = self._config["candle_type_def"]
 
-        _coingecko_config = config.get("coingecko", {})
+        _coingecko_config = self._config.get("coingecko", {})
 
         self._coingecko: FtCoinGeckoApi = FtCoinGeckoApi(
             api_key=_coingecko_config.get("api_key", ""),

freqtrade/plugins/pairlist/OffsetFilter.py

@@ -3,30 +3,24 @@ Offset pair list filter
 """
 
 import logging
-from typing import Any, Dict, List
+from typing import Dict, List
 
-from freqtrade.constants import Config
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class OffsetFilter(IPairList):
-    def __init__(
-        self,
-        exchange,
-        pairlistmanager,
-        config: Config,
-        pairlistconfig: Dict[str, Any],
-        pairlist_pos: int,
-    ) -> None:
-        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
+    supports_backtesting = SupportsBacktesting.YES
 
-        self._offset = pairlistconfig.get("offset", 0)
-        self._number_pairs = pairlistconfig.get("number_assets", 0)
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+
+        self._offset = self._pairlistconfig.get("offset", 0)
+        self._number_pairs = self._pairlistconfig.get("number_assets", 0)
 
         if self._offset < 0:
             raise OperationalException("OffsetFilter requires offset to be >= 0")

freqtrade/plugins/pairlist/PerformanceFilter.py

@@ -3,32 +3,26 @@ Performance pair list filter
 """
 
 import logging
-from typing import Any, Dict, List
+from typing import Dict, List
 
 import pandas as pd
 
-from freqtrade.constants import Config
 from freqtrade.exchange.types import Tickers
 from freqtrade.persistence import Trade
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class PerformanceFilter(IPairList):
-    def __init__(
-        self,
-        exchange,
-        pairlistmanager,
-        config: Config,
-        pairlistconfig: Dict[str, Any],
-        pairlist_pos: int,
-    ) -> None:
-        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
+    supports_backtesting = SupportsBacktesting.NO_ACTION
 
-        self._minutes = pairlistconfig.get("minutes", 0)
-        self._min_profit = pairlistconfig.get("min_profit")
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+
+        self._minutes = self._pairlistconfig.get("minutes", 0)
+        self._min_profit = self._pairlistconfig.get("min_profit")
 
     @property
     def needstickers(self) -> bool: