Merge pull request #10464 from freqtrade/new_release

New release 2024.7

.github/workflows/ci.yml

@@ -80,6 +80,11 @@ jobs:
         # Allow failure for coveralls
         coveralls || true
 
+    - name: Run json schema extract
+      # This should be kept before the repository check to ensure that the schema is up-to-date
+      run: |
+        python build_helpers/extract_config_json_schema.py
+
     - name: Check for repository changes
       run: |
         if [ -n "$(git status --porcelain)" ]; then

.pre-commit-config.yaml

@@ -9,14 +9,14 @@ repos:
         # stages: [push]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.10.0"
+    rev: "v1.11.0"
     hooks:
       - id: mypy
         exclude: build_helpers
         additional_dependencies:
-          - types-cachetools==5.3.0.7
+          - types-cachetools==5.4.0.20240717
           - types-filelock==3.2.7
-          - types-requests==2.32.0.20240622
+          - types-requests==2.32.0.20240712
           - types-tabulate==0.9.0.20240106
           - types-python-dateutil==2.9.0.20240316
           - SQLAlchemy==2.0.31
@@ -31,7 +31,7 @@ repos:
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.4.10'
+    rev: 'v0.5.4'
     hooks:
       - id: ruff
 

build_helpers/extract_config_json_schema.py

@@ -0,0 +1,17 @@
+"""Script to extract the configuration json schema from config_schema.py file."""
+
+from pathlib import Path
+
+import rapidjson
+
+from freqtrade.configuration.config_schema import CONF_SCHEMA
+
+
+def extract_config_json_schema():
+    schema_filename = Path(__file__).parent / "schema.json"
+    with schema_filename.open("w") as f:
+        rapidjson.dump(CONF_SCHEMA, f, indent=2)
+
+
+if __name__ == "__main__":
+    extract_config_json_schema()

docker-compose.yml

@@ -1,5 +1,4 @@
 ---
-version: '3'
 services:
   freqtrade:
     image: freqtradeorg/freqtrade:stable

docker/docker-compose-freqai.yml

@@ -1,5 +1,4 @@
 ---
-version: '3'
 services:
   freqtrade:
     image: freqtradeorg/freqtrade:stable_freqaitorch

docker/docker-compose-jupyter.yml

@@ -1,5 +1,4 @@
 ---
-version: '3'
 services:
   ft_jupyterlab:
     build:

docs/advanced-orderflow.md

@@ -0,0 +1,152 @@
+# Orderflow data
+
+This guide walks you through utilizing public trade data for advanced orderflow analysis in Freqtrade.
+
+!!! Warning "Experimental Feature"
+    The orderflow feature is currently in beta and may be subject to changes in future releases. Please report any issues or feedback on the [Freqtrade GitHub repository](https://github.com/freqtrade/freqtrade/issues).
+
+!!! Warning "Performance"
+    Orderflow requires raw trades data. This data is rather large, and can cause a slow initial startup, when freqtrade needs to download the trades data for the last X candles. Additionally, enabling this feature will cause increased memory usage. Please ensure to have sufficient resources available.
+
+## Getting Started
+
+### Enable Public Trades
+
+In your `config.json` file, set the `use_public_trades` option to true under the `exchange` section.
+
+```json
+"exchange": {
+   ...
+   "use_public_trades": true,
+}
+```
+
+### Configure Orderflow Processing
+
+Define your desired settings for orderflow processing within the orderflow section of config.json. Here, you can adjust factors like:
+
+- `cache_size`: How many previous orderflow candles are saved into cache instead of calculated every new candle
+- `max_candles`: Filter how many candles would you like to get trades data for.
+- `scale`: This controls the price bin size for the footprint chart.
+- `stacked_imbalance_range`: Defines the minimum consecutive imbalanced price levels required for consideration.
+- `imbalance_volume`: Filters out imbalances with volume below this threshold.
+- `imbalance_ratio`: Filters out imbalances with a ratio (difference between ask and bid volume) lower than this value.
+
+```json
+"orderflow": {
+    "cache_size": 1000, 
+    "max_candles": 1500, 
+    "scale": 0.5, 
+    "stacked_imbalance_range": 3, //  needs at least this amount of imbalance next to each other
+    "imbalance_volume": 1, //  filters out below
+    "imbalance_ratio": 3 //  filters out ratio lower than
+  },
+```
+
+## Downloading Trade Data for Backtesting
+
+To download historical trade data for backtesting, use the --dl-trades flag with the freqtrade download-data command.
+
+```bash
+freqtrade download-data -p BTC/USDT:USDT --timerange 20230101- --trading-mode futures --timeframes 5m --dl-trades
+```
+
+!!! Warning "Data availability"
+    Not all exchanges provide public trade data. For supported exchanges, freqtrade will warn you if public trade data is not available if you start downloading data with the `--dl-trades` flag.
+
+## Accessing Orderflow Data
+
+Once activated, several new columns become available in your dataframe:
+
+``` python
+
+dataframe["trades"] # Contains information about each individual trade.
+dataframe["orderflow"] # Represents a footprint chart dict (see below)
+dataframe["imbalances"] # Contains information about imbalances in the order flow.
+dataframe["bid"] # Total bid volume 
+dataframe["ask"] # Total ask volume
+dataframe["delta"] # Difference between ask and bid volume.
+dataframe["min_delta"] # Minimum delta within the candle
+dataframe["max_delta"] # Maximum delta within the candle
+dataframe["total_trades"] # Total number of trades
+dataframe["stacked_imbalances_bid"] # Price level of stacked bid imbalance 
+dataframe["stacked_imbalances_ask"] # Price level of stacked ask imbalance  
+```
+
+You can access these columns in your strategy code for further analysis. Here's an example:
+
+``` python
+def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+    # Calculating cumulative delta
+    dataframe["cum_delta"] = cumulative_delta(dataframe["delta"])
+    # Accessing total trades
+    total_trades = dataframe["total_trades"]
+    ...
+
+def cumulative_delta(delta: Series):
+    cumdelta = delta.cumsum()
+    return cumdelta
+
+```
+
+### Footprint chart (`dataframe["orderflow"]`)
+
+This column provides a detailed breakdown of buy and sell orders at different price levels, offering valuable insights into order flow dynamics. The `scale` parameter in your configuration determines the price bin size for this representation
+
+The `orderflow` column contains a dict with the following structure:
+
+``` output
+{
+    "price": {
+        "bid_amount": 0.0,
+        "ask_amount": 0.0,
+        "bid": 0,
+        "ask": 0,
+        "delta": 0.0,
+        "total_volume": 0.0,
+        "total_trades": 0
+    }
+}
+```
+
+#### Orderflow column explanation
+
+- key: Price bin - binned at `scale` intervals
+- `bid_amount`: Total volume bought at each price level.
+- `ask_amount`: Total volume sold at each price level.
+- `bid`: Number of buy orders at each price level.
+- `ask`: Number of sell orders at each price level.
+- `delta`: Difference between ask and bid volume at each price level.
+- `total_volume`: Total volume (ask amount + bid amount) at each price level.
+- `total_trades`: Total number of trades (ask + bid) at each price level.
+
+By leveraging these features, you can gain valuable insights into market sentiment and potential trading opportunities based on order flow analysis.
+
+### Raw trades data (`dataframe["trades"]`)
+
+List with the individual trades that occurred during the candle. This data can be used for more granular analysis of order flow dynamics.
+
+Each individual entry contains a dict with the following keys:
+
+- `timestamp`: Timestamp of the trade.
+- `date`: Date of the trade.
+- `price`: Price of the trade.
+- `amount`: Volume of the trade.
+- `side`: Buy or sell.
+- `id`: Unique identifier for the trade.
+- `cost`: Total cost of the trade (price * amount).
+
+### Imbalances (`dataframe["imbalances"]`)
+
+This column provides a dict with information about imbalances in the order flow. An imbalance occurs when there is a significant difference between the ask and bid volume at a given price level.
+
+Each row looks as follows - with price as index, and the corresponding bid and ask imbalance values as columns
+
+``` output
+{
+    "price": {
+        "bid_imbalance": False,
+        "ask_imbalance": False
+    }
+}
+```

docs/advanced-setup.md

@@ -114,8 +114,46 @@ services:
       --strategy SampleStrategy
 
 ```
+
 You can use whatever naming convention you want, freqtrade1 and 2 are arbitrary. Note, that you will need to use different database files, port mappings and telegram configurations for each instance, as mentioned above. 
 
+## Use a different database system
+
+Freqtrade is using SQLAlchemy, which supports multiple different database systems. As such, a multitude of database systems should be supported.
+Freqtrade does not depend or install any additional database driver. Please refer to the [SQLAlchemy docs](https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls) on installation instructions for the respective database systems.
+
+The following systems have been tested and are known to work with freqtrade:
+
+* sqlite (default)
+* PostgreSQL
+* MariaDB
+
+!!! Warning
+    By using one of the below database systems, you acknowledge that you know how to manage such a system. The freqtrade team will not provide any support with setup or maintenance (or backups) of the below database systems.
+
+### PostgreSQL
+
+Installation:
+`pip install psycopg2-binary`
+
+Usage:
+`... --db-url postgresql+psycopg2://<username>:<password>@localhost:5432/<database>`
+
+Freqtrade will automatically create the tables necessary upon startup.
+
+If you're running different instances of Freqtrade, you must either setup one database per Instance or use different users / schemas for your connections.
+
+### MariaDB / MySQL
+
+Freqtrade supports MariaDB by using SQLAlchemy, which supports multiple different database systems.
+
+Installation:
+`pip install pymysql`
+
+Usage:
+`... --db-url mysql+pymysql://<username>:<password>@localhost:3306/<database>`
+
+
 
 ## Configure the bot running as a systemd service
 

docs/configuration.md

@@ -204,9 +204,10 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `exchange.uid` | API uid to use for the exchange. Only required when you are in production mode and for exchanges that use uid for API requests.<br>**Keep it in secret, do not disclose publicly.** <br> **Datatype:** String
 | `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Supports regex pairs as `.*/BTC`. Not used by VolumePairList. [More information](plugins.md#pairlists-and-pairlist-handlers). <br> **Datatype:** List
 | `exchange.pair_blacklist` | List of pairs the bot must absolutely avoid for trading and backtesting. [More information](plugins.md#pairlists-and-pairlist-handlers). <br> **Datatype:** List
-| `exchange.ccxt_config` | Additional CCXT parameters passed to both ccxt instances (sync and async). This is usually the correct place for additional ccxt configurations. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation). Please avoid adding exchange secrets here (use the dedicated fields instead), as they may be contained in logs. <br> **Datatype:** Dict
-| `exchange.ccxt_sync_config` | Additional CCXT parameters passed to the regular (sync) ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) <br> **Datatype:** Dict
-| `exchange.ccxt_async_config` | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange  and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) <br> **Datatype:** Dict
+| `exchange.ccxt_config` | Additional CCXT parameters passed to both ccxt instances (sync and async). This is usually the correct place for additional ccxt configurations. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://docs.ccxt.com/#/README?id=overriding-exchange-properties-upon-instantiation). Please avoid adding exchange secrets here (use the dedicated fields instead), as they may be contained in logs. <br> **Datatype:** Dict
+| `exchange.ccxt_sync_config` | Additional CCXT parameters passed to the regular (sync) ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://docs.ccxt.com/#/README?id=overriding-exchange-properties-upon-instantiation) <br> **Datatype:** Dict
+| `exchange.ccxt_async_config` | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange  and are documented in the [ccxt documentation](https://docs.ccxt.com/#/README?id=overriding-exchange-properties-upon-instantiation) <br> **Datatype:** Dict
+| `exchange.enable_ws` | Enable the usage of Websockets for the exchange. <br>[More information](#consuming-exchange-websockets).<br>*Defaults to `true`.* <br> **Datatype:** Boolean
 | `exchange.markets_refresh_interval` | The interval in minutes in which markets are reloaded. <br>*Defaults to `60` minutes.* <br> **Datatype:** Positive Integer
 | `exchange.skip_pair_validation` | Skip pairlist validation on startup.<br>*Defaults to `false`*<br> **Datatype:** Boolean
 | `exchange.skip_open_order_update` | Skips open order updates on startup should the exchange cause problems. Only relevant in live conditions.<br>*Defaults to `false`*<br> **Datatype:** Boolean
@@ -409,6 +410,8 @@ Or another example if your position adjustment assumes it can do 1 additional bu
 
 --8<-- "includes/pricing.md"
 
+## Further Configuration details
+
 ### Understand minimal_roi
 
 The `minimal_roi` configuration parameter is a JSON object where the key is a duration
@@ -614,6 +617,30 @@ Freqtrade supports both Demo and Pro coingecko API keys.
 The Coingecko API key is NOT required for the bot to function correctly.
 It is only used for the conversion of coin to fiat in the Telegram reports, which usually also work without API key.
 
+## Consuming exchange Websockets
+
+Freqtrade can consume websockets through ccxt.pro.
+
+Freqtrade aims ensure data is available at all times.
+Should the websocket connection fail (or be disabled), the bot will fall back to REST API calls.
+
+Should you experience problems you suspect are caused by websockets, you can disable these via the setting `exchange.enable_ws`, which defaults to true.
+
+```jsonc
+"exchange": {
+    // ...
+    "enable_ws": false,
+    // ...
+}
+```
+
+Should you be required to use a proxy, please refer to the [proxy section](#using-proxy-with-freqtrade) for more information.
+
+!!! Info "Rollout"
+    We're implementing this out slowly, ensuring stability of your bots.
+    Currently, usage is limited to ohlcv data streams.
+    It's also limited to a few exchanges, with new exchanges being added on an ongoing basis.
+
 ## Using Dry-run mode
 
 We recommend starting the bot in the Dry-run mode to see how your bot will
@@ -650,9 +677,9 @@ Once you will be happy with your bot performance running in the Dry-run mode, yo
 * API-keys may or may not be provided. Only Read-Only operations (i.e. operations that do not alter account state) on the exchange are performed in dry-run mode.
 * Wallets (`/balance`) are simulated based on `dry_run_wallet`.
 * Orders are simulated, and will not be posted to the exchange.
-* Market orders fill based on orderbook volume the moment the order is placed.
+* Market orders fill based on orderbook volume the moment the order is placed, with a maximum slippage of 5%.
 * Limit orders fill once the price reaches the defined level - or time out based on `unfilledtimeout` settings.
-* Limit orders will be converted to market orders if they cross the price by more than 1%.
+* Limit orders will be converted to market orders if they cross the price by more than 1%, and will be filled immediately based regular market order rules (see point about Market orders above).
 * In combination with `stoploss_on_exchange`, the stop_loss price is assumed to be filled.
 * Open orders (not trades, which are stored in the database) are kept open after bot restarts, with the assumption that they were not filled while being offline.
 
@@ -702,7 +729,7 @@ You should also make sure to read the [Exchanges](exchanges.md) section of the d
 
     **NEVER** share your private configuration file or your exchange keys with anyone!
 
-### Using proxy with Freqtrade
+## Using a proxy with Freqtrade
 
 To use a proxy with freqtrade, export your proxy settings using the variables `"HTTP_PROXY"` and `"HTTPS_PROXY"` set to the appropriate values.
 This will have the proxy settings applied to everything (telegram, coingecko, ...) **except** for exchange requests.
@@ -713,7 +740,7 @@ export HTTPS_PROXY="http://addr:port"
 freqtrade
 ```
 
-#### Proxy exchange requests
+### Proxy exchange requests
 
 To use a proxy for exchange connections - you will have to define the proxies as part of the ccxt configuration.
 
@@ -722,6 +749,7 @@ To use a proxy for exchange connections - you will have to define the proxies as
   "exchange": {
     "ccxt_config": {
       "httpsProxy": "http://addr:port",
+      "wsProxy": "http://addr:port",
     }
   }
 }

docs/developer.md

@@ -22,7 +22,7 @@ This will spin up a local server (usually on port 8000) so you can see if everyt
 ## Developer setup
 
 To configure a development environment, you can either use the provided [DevContainer](#devcontainer-setup), or use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ".
-Alternatively (e.g. if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`.
+Alternatively (e.g. if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -r requirements-dev.txt` - followed by `pip3 install -e .[all]`.
 
 This will install all required tools for development, including `pytest`, `ruff`, `mypy`, and `coveralls`.
 

docs/freqai-running.md

@@ -73,23 +73,26 @@ Backtesting mode requires [downloading the necessary data](#downloading-data-to-
 
 ---
 
-### Saving prediction data
+### Saving backtesting prediction data
 
 To allow for tweaking your strategy (**not** the features!), FreqAI will automatically save the predictions during backtesting so that they can be reused for future backtests and live runs using the same `identifier` model. This provides a performance enhancement geared towards enabling **high-level hyperopting** of entry/exit criteria.
 
-An additional directory called `backtesting_predictions`, which contains all the predictions stored in `hdf` format, will be created in the `unique-id` folder.
+An additional directory called `backtesting_predictions`, which contains all the predictions stored in `feather` format, will be created in the `unique-id` folder.
 
 To change your **features**, you **must** set a new `identifier` in the config to signal to FreqAI to train new models.
 
 To save the models generated during a particular backtest so that you can start a live deployment from one of them instead of training a new model, you must set `save_backtest_models` to `True` in the config.
 
+!!! Note
+    To ensure that the model can be reused, freqAI will call your strategy with a dataframe of length 1. 
+    If your strategy requires more data than this to generate the same features, you can't reuse backtest predictions for live deployment and need to update your `identifier` for each new backtest.
+
 ### Backtest live collected predictions
 
 FreqAI allow you to reuse live historic predictions through the backtest parameter `--freqai-backtest-live-models`. This can be useful when you want to reuse predictions generated in dry/run for comparison or other study.
 
 The `--timerange` parameter must not be informed, as it will be automatically calculated through the data in the historic predictions file.
 
-
 ### Downloading data to cover the full backtest period
 
 For live/dry deployments, FreqAI will download the necessary data automatically. However, to use backtesting functionality, you need to download the necessary data using `download-data` (details [here](data-download.md#data-downloading)). You need to pay careful attention to understanding how much *additional* data needs to be downloaded to ensure that there is a sufficient amount of training data *before* the start of the backtesting time range. The amount of additional data can be roughly estimated by moving the start date of the time range backwards by `train_period_days` and the `startup_candle_count` (see the [parameter table](freqai-parameter-table.md) for detailed descriptions of these parameters) from the beginning of the desired backtesting time range. 

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
 markdown==3.6
 mkdocs==1.6.0
-mkdocs-material==9.5.27
+mkdocs-material==9.5.29
 mdx_truly_sane_lists==1.3
 pymdown-extensions==10.8.1
 jinja2==3.1.4

docs/rest-api.md

@@ -2,7 +2,7 @@
 
 ## FreqUI
 
-FreqUI now has it's own dedicated [documentation section](frequi.md) - please refer to that section for all information regarding the FreqUI.
+FreqUI now has it's own dedicated [documentation section](freq-ui.md) - please refer to that section for all information regarding the FreqUI.
 
 ## Configuration
 

docs/sql_cheatsheet.md

@@ -1,6 +1,13 @@
 # SQL Helper
 
-This page contains some help if you want to edit your sqlite db.
+This page contains some help if you want to query your sqlite db.
+
+!!! Tip "Other Database systems"
+    To use other Database Systems like PostgreSQL or MariaDB, you can use the same queries, but you need to use the respective client for the database system. [Click here](advanced-setup.md#use-a-different-database-system) to learn how to setup a different database system with freqtrade.
+
+!!! Warning
+    If you are not familiar with SQL, you should be very careful when running queries on your database.  
+    Always make sure to have a backup of your database before running any queries.
 
 ## Install sqlite3
 
@@ -43,13 +50,25 @@ sqlite3
 .schema <table_name>
 ```
 
-## Get all trades in the table
+### Get all trades in the table
 
 ```sql
 SELECT * FROM trades;
 ```
 
-## Fix trade still open after a manual exit on the exchange
+## Destructive queries
+
+Queries that write to the database.
+These queries should usually not be necessary as freqtrade tries to handle all database operations itself - or exposes them via API or telegram commands.
+
+!!! Warning
+    Please make sure you have a backup of your database before running any of the below queries.
+
+!!! Danger
+    You should also **never** run any writing query (`update`, `insert`, `delete`) while a bot is connected to the database.
+    This can and will lead to data corruption - most likely, without the possibility of recovery.
+
+### Fix trade still open after a manual exit on the exchange
 
 !!! Warning
     Manually selling a pair on the exchange will not be detected by the bot and it will try to sell anyway. Whenever possible, /forceexit <tradeid> should be used to accomplish the same thing.  
@@ -69,7 +88,7 @@ SET is_open=0,
 WHERE id=<trade_ID_to_update>;
 ```
 
-### Example
+#### Example
 
 ```sql
 UPDATE trades
@@ -82,7 +101,7 @@ SET is_open=0,
 WHERE id=31;
 ```
 
-## Remove trade from the database
+### Remove trade from the database
 
 !!! Tip "Use RPC Methods to delete trades"
     Consider using `/delete <tradeid>` via telegram or rest API. That's the recommended way to deleting trades.
@@ -100,39 +119,3 @@ DELETE FROM trades WHERE id = 31;
 
 !!! Warning
     This will remove this trade from the database. Please make sure you got the correct id and **NEVER** run this query without the `where` clause.
-
-## Use a different database system
-
-Freqtrade is using SQLAlchemy, which supports multiple different database systems. As such, a multitude of database systems should be supported.
-Freqtrade does not depend or install any additional database driver. Please refer to the [SQLAlchemy docs](https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls) on installation instructions for the respective database systems.
-
-The following systems have been tested and are known to work with freqtrade:
-
-* sqlite (default)
-* PostgreSQL
-* MariaDB
-
-!!! Warning
-    By using one of the below database systems, you acknowledge that you know how to manage such a system. The freqtrade team will not provide any support with setup or maintenance (or backups) of the below database systems.
-
-### PostgreSQL
-
-Installation:
-`pip install psycopg2-binary`
-
-Usage:
-`... --db-url postgresql+psycopg2://<username>:<password>@localhost:5432/<database>`
-
-Freqtrade will automatically create the tables necessary upon startup.
-
-If you're running different instances of Freqtrade, you must either setup one database per Instance or use different users / schemas for your connections.
-
-### MariaDB / MySQL
-
-Freqtrade supports MariaDB by using SQLAlchemy, which supports multiple different database systems.
-
-Installation:
-`pip install pymysql`
-
-Usage:
-`... --db-url mysql+pymysql://<username>:<password>@localhost:3306/<database>`

docs/utils.md

@@ -488,7 +488,7 @@ freqtrade test-pairlist --config config.json --quote USDT BTC
 
 `freqtrade convert-db` can be used to convert your database from one system to another (sqlite -> postgres, postgres -> other postgres), migrating all trades, orders and Pairlocks.
 
-Please refer to the [SQL cheatsheet](sql_cheatsheet.md#use-a-different-database-system) to learn about requirements for different database systems.
+Please refer to the [corresponding documentation](advanced-setup.md#use-a-different-database-system) to learn about requirements for different database systems.
 
 ```
 usage: freqtrade convert-db [-h] [--db-url PATH] [--db-url-from PATH]

freqtrade/__init__.py

@@ -1,12 +1,12 @@
 """Freqtrade bot"""
 
-__version__ = "2024.6"
+__version__ = "2024.7"
 
 if "dev" in __version__:
     from pathlib import Path
 
     try:
-        import subprocess
+        import subprocess  # noqa: S404
 
         freqtrade_basedir = Path(__file__).parent
 

freqtrade/commands/arguments.py

@@ -2,10 +2,10 @@
 This module contains the argument manager class
 """
 
-import argparse
+from argparse import ArgumentParser, Namespace, _ArgumentGroup
 from functools import partial
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 from freqtrade.commands.cli_options import AVAILABLE_CLI_OPTIONS
 from freqtrade.constants import DEFAULT_CONFIG
@@ -226,6 +226,19 @@ ARGS_ANALYZE_ENTRIES_EXITS = [
     "analysis_csv_path",
 ]
 
+
+ARGS_STRATEGY_UPDATER = ["strategy_list", "strategy_path", "recursive_strategy_search"]
+
+ARGS_LOOKAHEAD_ANALYSIS = [
+    a
+    for a in ARGS_BACKTEST
+    if a
+    not in ("position_stacking", "use_max_market_positions", "backtest_cache", "backtest_breakdown")
+] + ["minimum_trade_amount", "targeted_trade_amount", "lookahead_analysis_exportfilename"]
+
+ARGS_RECURSIVE_ANALYSIS = ["timeframe", "timerange", "dataformat_ohlcv", "pairs", "startup_candle"]
+
+# Command level configs - keep at the bottom of the above definitions
 NO_CONF_REQURIED = [
     "convert-data",
     "convert-trade-data",
@@ -248,14 +261,6 @@ NO_CONF_REQURIED = [
 
 NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"]
 
-ARGS_STRATEGY_UPDATER = ["strategy_list", "strategy_path", "recursive_strategy_search"]
-
-ARGS_LOOKAHEAD_ANALYSIS = [
-    a for a in ARGS_BACKTEST if a not in ("position_stacking", "use_max_market_positions", "cache")
-] + ["minimum_trade_amount", "targeted_trade_amount", "lookahead_analysis_exportfilename"]
-
-ARGS_RECURSIVE_ANALYSIS = ["timeframe", "timerange", "dataformat_ohlcv", "pairs", "startup_candle"]
-
 
 class Arguments:
     """
@@ -264,7 +269,7 @@ class Arguments:
 
     def __init__(self, args: Optional[List[str]]) -> None:
         self.args = args
-        self._parsed_arg: Optional[argparse.Namespace] = None
+        self._parsed_arg: Optional[Namespace] = None
 
     def get_parsed_arg(self) -> Dict[str, Any]:
         """
@@ -277,7 +282,7 @@ class Arguments:
 
         return vars(self._parsed_arg)
 
-    def _parse_args(self) -> argparse.Namespace:
+    def _parse_args(self) -> Namespace:
         """
         Parses given arguments and returns an argparse Namespace instance.
         """
@@ -306,7 +311,9 @@ class Arguments:
 
         return parsed_arg
 
-    def _build_args(self, optionlist, parser):
+    def _build_args(
+        self, optionlist: List[str], parser: Union[ArgumentParser, _ArgumentGroup]
+    ) -> None:
         for val in optionlist:
             opt = AVAILABLE_CLI_OPTIONS[val]
             parser.add_argument(*opt.cli, dest=val, **opt.kwargs)
@@ -317,16 +324,16 @@ class Arguments:
         :return: None
         """
         # Build shared arguments (as group Common Options)
-        _common_parser = argparse.ArgumentParser(add_help=False)
+        _common_parser = ArgumentParser(add_help=False)
         group = _common_parser.add_argument_group("Common arguments")
         self._build_args(optionlist=ARGS_COMMON, parser=group)
 
-        _strategy_parser = argparse.ArgumentParser(add_help=False)
+        _strategy_parser = ArgumentParser(add_help=False)
         strategy_group = _strategy_parser.add_argument_group("Strategy arguments")
         self._build_args(optionlist=ARGS_STRATEGY, parser=strategy_group)
 
         # Build main command
-        self.parser = argparse.ArgumentParser(
+        self.parser = ArgumentParser(
             prog="freqtrade", description="Free, open source crypto trading bot"
         )
         self._build_args(optionlist=["version"], parser=self.parser)

freqtrade/commands/data_commands.py

@@ -16,6 +16,7 @@ from freqtrade.exceptions import ConfigurationError
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
+from freqtrade.util import print_rich_table
 from freqtrade.util.migrations import migrate_data
 
 
@@ -119,8 +120,6 @@ def start_list_data(args: Dict[str, Any]) -> None:
 
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
-    from tabulate import tabulate
-
     from freqtrade.data.history import get_datahandler
 
     dhc = get_datahandler(config["datadir"], config["dataformat_ohlcv"])
@@ -131,8 +130,7 @@ def start_list_data(args: Dict[str, Any]) -> None:
 
     if args["pairs"]:
         paircombs = [comb for comb in paircombs if comb[0] in args["pairs"]]
-
-    print(f"Found {len(paircombs)} pair / timeframe combinations.")
+    title = f"Found {len(paircombs)} pair / timeframe combinations."
     if not config.get("show_timerange"):
         groupedpair = defaultdict(list)
         for pair, timeframe, candle_type in sorted(
@@ -141,40 +139,35 @@ def start_list_data(args: Dict[str, Any]) -> None:
             groupedpair[(pair, candle_type)].append(timeframe)
 
         if groupedpair:
-            print(
-                tabulate(
-                    [
-                        (pair, ", ".join(timeframes), candle_type)
-                        for (pair, candle_type), timeframes in groupedpair.items()
-                    ],
-                    headers=("Pair", "Timeframe", "Type"),
-                    tablefmt="psql",
-                    stralign="right",
-                )
+            print_rich_table(
+                [
+                    (pair, ", ".join(timeframes), candle_type)
+                    for (pair, candle_type), timeframes in groupedpair.items()
+                ],
+                ("Pair", "Timeframe", "Type"),
+                title,
+                table_kwargs={"min_width": 50},
             )
     else:
         paircombs1 = [
             (pair, timeframe, candle_type, *dhc.ohlcv_data_min_max(pair, timeframe, candle_type))
             for pair, timeframe, candle_type in paircombs
         ]
-
-        print(
-            tabulate(
-                [
-                    (
-                        pair,
-                        timeframe,
-                        candle_type,
-                        start.strftime(DATETIME_PRINT_FORMAT),
-                        end.strftime(DATETIME_PRINT_FORMAT),
-                        length,
-                    )
-                    for pair, timeframe, candle_type, start, end, length in sorted(
-                        paircombs1, key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2])
-                    )
-                ],
-                headers=("Pair", "Timeframe", "Type", "From", "To", "Candles"),
-                tablefmt="psql",
-                stralign="right",
-            )
+        print_rich_table(
+            [
+                (
+                    pair,
+                    timeframe,
+                    candle_type,
+                    start.strftime(DATETIME_PRINT_FORMAT),
+                    end.strftime(DATETIME_PRINT_FORMAT),
+                    str(length),
+                )
+                for pair, timeframe, candle_type, start, end, length in sorted(
+                    paircombs1, key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2])
+                )
+            ],
+            ("Pair", "Timeframe", "Type", "From", "To", "Candles"),
+            summary=title,
+            table_kwargs={"min_width": 50},
         )

freqtrade/commands/hyperopt_commands.py

@@ -2,8 +2,6 @@ import logging
 from operator import itemgetter
 from typing import Any, Dict
 
-from colorama import init as colorama_init
-
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.data.btanalysis import get_latest_hyperopt_file
 from freqtrade.enums import RunMode
@@ -18,6 +16,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
     """
     List hyperopt epochs previously evaluated
     """
+    from freqtrade.optimize.hyperopt_output import HyperoptOutput
     from freqtrade.optimize.hyperopt_tools import HyperoptTools
 
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
@@ -35,21 +34,17 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
     # Previous evaluations
     epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config)
 
-    if print_colorized:
-        colorama_init(autoreset=True)
-
     if not export_csv:
         try:
-            print(
-                HyperoptTools.get_result_table(
-                    config,
-                    epochs,
-                    total_epochs,
-                    not config.get("hyperopt_list_best", False),
-                    print_colorized,
-                    0,
-                )
+            h_out = HyperoptOutput()
+            h_out.add_data(
+                config,
+                epochs,
+                total_epochs,
+                not config.get("hyperopt_list_best", False),
             )
+            h_out.print(print_colorized=print_colorized)
+
         except KeyboardInterrupt:
             print("User interrupted..")
 

freqtrade/commands/list_commands.py

@@ -4,9 +4,9 @@ import sys
 from typing import Any, Dict, List, Union
 
 import rapidjson
-from colorama import Fore, Style
-from colorama import init as colorama_init
-from tabulate import tabulate
+from rich.console import Console
+from rich.table import Table
+from rich.text import Text
 
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.enums import RunMode
@@ -14,7 +14,8 @@ from freqtrade.exceptions import ConfigurationError, OperationalException
 from freqtrade.exchange import list_available_exchanges, market_is_active
 from freqtrade.misc import parse_db_uri_for_logging, plural
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
-from freqtrade.types import ValidExchangesType
+from freqtrade.types.valid_exchanges_type import ValidExchangesType
+from freqtrade.util import print_rich_table
 
 
 logger = logging.getLogger(__name__)
@@ -26,71 +27,69 @@ def start_list_exchanges(args: Dict[str, Any]) -> None:
     :param args: Cli args from Arguments()
     :return: None
     """
-    exchanges = list_available_exchanges(args["list_exchanges_all"])
+    available_exchanges: List[ValidExchangesType] = list_available_exchanges(
+        args["list_exchanges_all"]
+    )
 
     if args["print_one_column"]:
-        print("\n".join([e["name"] for e in exchanges]))
+        print("\n".join([e["name"] for e in available_exchanges]))
     else:
-        headers = {
-            "name": "Exchange name",
-            "supported": "Supported",
-            "trade_modes": "Markets",
-            "comment": "Reason",
-        }
-        headers.update({"valid": "Valid"} if args["list_exchanges_all"] else {})
+        if args["list_exchanges_all"]:
+            title = (
+                f"All exchanges supported by the ccxt library "
+                f"({len(available_exchanges)} exchanges):"
+            )
+        else:
+            available_exchanges = [e for e in available_exchanges if e["valid"] is not False]
+            title = f"Exchanges available for Freqtrade ({len(available_exchanges)} exchanges):"
 
-        def build_entry(exchange: ValidExchangesType, valid: bool):
-            valid_entry = {"valid": exchange["valid"]} if valid else {}
-            result: Dict[str, Union[str, bool]] = {
-                "name": exchange["name"],
-                **valid_entry,
-                "supported": "Official" if exchange["supported"] else "",
-                "trade_modes": ", ".join(
-                    (f"{a['margin_mode']} " if a["margin_mode"] else "") + a["trading_mode"]
+        table = Table(title=title)
+
+        table.add_column("Exchange Name")
+        table.add_column("Markets")
+        table.add_column("Reason")
+
+        for exchange in available_exchanges:
+            name = Text(exchange["name"])
+            if exchange["supported"]:
+                name.append(" (Official)", style="italic")
+                name.stylize("green bold")
+
+            trade_modes = Text(
+                ", ".join(
+                    (f"{a.get('margin_mode', '')} {a['trading_mode']}").lstrip()
                     for a in exchange["trade_modes"]
                 ),
-                "comment": exchange["comment"],
-            }
-
-            return result
-
-        if args["list_exchanges_all"]:
-            print("All exchanges supported by the ccxt library:")
-            exchanges = [build_entry(e, True) for e in exchanges]
-        else:
-            print("Exchanges available for Freqtrade:")
-            exchanges = [build_entry(e, False) for e in exchanges if e["valid"] is not False]
-
-        print(
-            tabulate(
-                exchanges,
-                headers=headers,
+                style="",
             )
-        )
+            if exchange["dex"]:
+                trade_modes = Text("DEX: ") + trade_modes
+                trade_modes.stylize("bold", 0, 3)
+
+            table.add_row(
+                name,
+                trade_modes,
+                exchange["comment"],
+                style=None if exchange["valid"] else "red",
+            )
+            # table.add_row(*[exchange[header] for header in headers])
+
+        console = Console()
+        console.print(table)
 
 
 def _print_objs_tabular(objs: List, print_colorized: bool) -> None:
-    if print_colorized:
-        colorama_init(autoreset=True)
-        red = Fore.RED
-        yellow = Fore.YELLOW
-        reset = Style.RESET_ALL
-    else:
-        red = ""
-        yellow = ""
-        reset = ""
-
     names = [s["name"] for s in objs]
-    objs_to_print = [
+    objs_to_print: List[Dict[str, Union[Text, str]]] = [
         {
-            "name": s["name"] if s["name"] else "--",
+            "name": Text(s["name"] if s["name"] else "--"),
             "location": s["location_rel"],
             "status": (
-                red + "LOAD FAILED" + reset
+                Text("LOAD FAILED", style="bold red")
                 if s["class"] is None
-                else "OK"
+                else Text("OK", style="bold green")
                 if names.count(s["name"]) == 1
-                else yellow + "DUPLICATE NAME" + reset
+                else Text("DUPLICATE NAME", style="bold yellow")
             ),
         }
         for s in objs
@@ -100,11 +99,23 @@ def _print_objs_tabular(objs: List, print_colorized: bool) -> None:
             objs_to_print[idx].update(
                 {
                     "hyperoptable": "Yes" if s["hyperoptable"]["count"] > 0 else "No",
-                    "buy-Params": len(s["hyperoptable"].get("buy", [])),
-                    "sell-Params": len(s["hyperoptable"].get("sell", [])),
+                    "buy-Params": str(len(s["hyperoptable"].get("buy", []))),
+                    "sell-Params": str(len(s["hyperoptable"].get("sell", []))),
                 }
             )
-    print(tabulate(objs_to_print, headers="keys", tablefmt="psql", stralign="right"))
+    table = Table()
+
+    for header in objs_to_print[0].keys():
+        table.add_column(header.capitalize(), justify="right")
+
+    for row in objs_to_print:
+        table.add_row(*[row[header] for header in objs_to_print[0].keys()])
+
+    console = Console(
+        color_system="auto" if print_colorized else None,
+        width=200 if "pytest" in sys.modules else None,
+    )
+    console.print(table)
 
 
 def start_list_strategies(args: Dict[str, Any]) -> None:
@@ -269,9 +280,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
                 writer.writeheader()
                 writer.writerows(tabular_data)
             else:
-                # print data as a table, with the human-readable summary
-                print(f"{summary_str}:")
-                print(tabulate(tabular_data, headers="keys", tablefmt="psql", stralign="right"))
+                print_rich_table(tabular_data, headers, summary_str)
         elif not (
             args.get("print_one_column", False)
             or args.get("list_pairs_print_json", False)

freqtrade/configuration/config_secrets.py

@@ -14,9 +14,13 @@ def sanitize_config(config: Config, *, show_sensitive: bool = False) -> Config:
         return config
     keys_to_remove = [
         "exchange.key",
+        "exchange.apiKey",
         "exchange.secret",
         "exchange.password",
         "exchange.uid",
+        "exchange.accountId",
+        "exchange.walletAddress",
+        "exchange.privateKey",
         "telegram.token",
         "telegram.chat_id",
         "discord.webhook_url",

freqtrade/configuration/config_validation.py

@@ -6,8 +6,16 @@ from typing import Any, Dict
 from jsonschema import Draft4Validator, validators
 from jsonschema.exceptions import ValidationError, best_match
 
-from freqtrade import constants
+from freqtrade.configuration.config_schema import (
+    CONF_SCHEMA,
+    SCHEMA_BACKTEST_REQUIRED,
+    SCHEMA_BACKTEST_REQUIRED_FINAL,
+    SCHEMA_MINIMAL_REQUIRED,
+    SCHEMA_MINIMAL_WEBSERVER,
+    SCHEMA_TRADE_REQUIRED,
+)
 from freqtrade.configuration.deprecated_settings import process_deprecated_setting
+from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
 from freqtrade.enums import RunMode, TradingMode
 from freqtrade.exceptions import ConfigurationError
 
@@ -41,18 +49,18 @@ def validate_config_schema(conf: Dict[str, Any], preliminary: bool = False) -> D
     :param conf: Config in JSON format
     :return: Returns the config if valid, otherwise throw an exception
     """
-    conf_schema = deepcopy(constants.CONF_SCHEMA)
+    conf_schema = deepcopy(CONF_SCHEMA)
     if conf.get("runmode", RunMode.OTHER) in (RunMode.DRY_RUN, RunMode.LIVE):
-        conf_schema["required"] = constants.SCHEMA_TRADE_REQUIRED
+        conf_schema["required"] = SCHEMA_TRADE_REQUIRED
     elif conf.get("runmode", RunMode.OTHER) in (RunMode.BACKTEST, RunMode.HYPEROPT):
         if preliminary:
-            conf_schema["required"] = constants.SCHEMA_BACKTEST_REQUIRED
+            conf_schema["required"] = SCHEMA_BACKTEST_REQUIRED
         else:
-            conf_schema["required"] = constants.SCHEMA_BACKTEST_REQUIRED_FINAL
+            conf_schema["required"] = SCHEMA_BACKTEST_REQUIRED_FINAL
     elif conf.get("runmode", RunMode.OTHER) == RunMode.WEBSERVER:
-        conf_schema["required"] = constants.SCHEMA_MINIMAL_WEBSERVER
+        conf_schema["required"] = SCHEMA_MINIMAL_WEBSERVER
     else:
-        conf_schema["required"] = constants.SCHEMA_MINIMAL_REQUIRED
+        conf_schema["required"] = SCHEMA_MINIMAL_REQUIRED
     try:
         FreqtradeValidator(conf_schema).validate(conf)
         return conf
@@ -83,6 +91,7 @@ def validate_config_consistency(conf: Dict[str, Any], *, preliminary: bool = Fal
     _validate_freqai_include_timeframes(conf, preliminary=preliminary)
     _validate_consumers(conf)
     validate_migrated_strategy_settings(conf)
+    _validate_orderflow(conf)
 
     # validate configuration before returning
     logger.info("Validating configuration ...")
@@ -97,7 +106,7 @@ def _validate_unlimited_amount(conf: Dict[str, Any]) -> None:
     if (
         not conf.get("edge", {}).get("enabled")
         and conf.get("max_open_trades") == float("inf")
-        and conf.get("stake_amount") == constants.UNLIMITED_STAKE_AMOUNT
+        and conf.get("stake_amount") == UNLIMITED_STAKE_AMOUNT
     ):
         raise ConfigurationError("`max_open_trades` and `stake_amount` cannot both be unlimited.")
 
@@ -421,6 +430,14 @@ def _validate_consumers(conf: Dict[str, Any]) -> None:
             )
 
 
+def _validate_orderflow(conf: Dict[str, Any]) -> None:
+    if conf.get("exchange", {}).get("use_public_trades"):
+        if "orderflow" not in conf:
+            raise ConfigurationError(
+                "Orderflow is a required configuration key when using public trades."
+            )
+
+
 def _strategy_settings(conf: Dict[str, Any]) -> None:
     process_deprecated_setting(conf, None, "use_sell_signal", None, "use_exit_signal")
     process_deprecated_setting(conf, None, "sell_profit_only", None, "exit_profit_only")

freqtrade/configuration/directory_operations.py

@@ -38,7 +38,7 @@ def chown_user_directory(directory: Path) -> None:
     """
     if running_in_docker():
         try:
-            import subprocess
+            import subprocess  # noqa: S404
 
             subprocess.check_output(["sudo", "chown", "-R", "ftuser:", str(directory.resolve())])
         except Exception:

freqtrade/constants.py

@@ -4,9 +4,9 @@
 bot constants
 """
 
-from typing import Any, Dict, List, Literal, Tuple
+from typing import Any, Dict, List, Literal, Optional, Tuple
 
-from freqtrade.enums import CandleType, PriceType, RPCMessageType
+from freqtrade.enums import CandleType, PriceType
 
 
 DOCS_LINK = "https://www.freqtrade.io/en/stable"
@@ -68,6 +68,7 @@ DEFAULT_DATAFRAME_COLUMNS = ["date", "open", "high", "low", "close", "volume"]
 # Don't modify sequence of DEFAULT_TRADES_COLUMNS
 # it has wide consequences for stored trades files
 DEFAULT_TRADES_COLUMNS = ["timestamp", "id", "type", "side", "price", "amount", "cost"]
+DEFAULT_ORDERFLOW_COLUMNS = ["level", "bid", "ask", "delta"]
 TRADES_DTYPES = {
     "timestamp": "int64",
     "id": "str",
@@ -171,585 +172,6 @@ MINIMAL_CONFIG = {
     },
 }
 
-__MESSAGE_TYPE_DICT: Dict[str, Dict[str, str]] = {x: {"type": "object"} for x in RPCMessageType}
-
-# Required json-schema for user specified config
-CONF_SCHEMA = {
-    "type": "object",
-    "properties": {
-        "max_open_trades": {"type": ["integer", "number"], "minimum": -1},
-        "new_pairs_days": {"type": "integer", "default": 30},
-        "timeframe": {"type": "string"},
-        "stake_currency": {"type": "string"},
-        "stake_amount": {
-            "type": ["number", "string"],
-            "minimum": 0.0001,
-            "pattern": UNLIMITED_STAKE_AMOUNT,
-        },
-        "tradable_balance_ratio": {"type": "number", "minimum": 0.0, "maximum": 1, "default": 0.99},
-        "available_capital": {
-            "type": "number",
-            "minimum": 0,
-        },
-        "amend_last_stake_amount": {"type": "boolean", "default": False},
-        "last_stake_amount_min_ratio": {
-            "type": "number",
-            "minimum": 0.0,
-            "maximum": 1.0,
-            "default": 0.5,
-        },
-        "fiat_display_currency": {"type": "string", "enum": SUPPORTED_FIAT},
-        "dry_run": {"type": "boolean"},
-        "dry_run_wallet": {"type": "number", "default": DRY_RUN_WALLET},
-        "cancel_open_orders_on_exit": {"type": "boolean", "default": False},
-        "process_only_new_candles": {"type": "boolean"},
-        "minimal_roi": {
-            "type": "object",
-            "patternProperties": {"^[0-9.]+$": {"type": "number"}},
-        },
-        "amount_reserve_percent": {"type": "number", "minimum": 0.0, "maximum": 0.5},
-        "stoploss": {"type": "number", "maximum": 0, "exclusiveMaximum": True},
-        "trailing_stop": {"type": "boolean"},
-        "trailing_stop_positive": {"type": "number", "minimum": 0, "maximum": 1},
-        "trailing_stop_positive_offset": {"type": "number", "minimum": 0, "maximum": 1},
-        "trailing_only_offset_is_reached": {"type": "boolean"},
-        "use_exit_signal": {"type": "boolean"},
-        "exit_profit_only": {"type": "boolean"},
-        "exit_profit_offset": {"type": "number"},
-        "fee": {"type": "number", "minimum": 0, "maximum": 0.1},
-        "ignore_roi_if_entry_signal": {"type": "boolean"},
-        "ignore_buying_expired_candle_after": {"type": "number"},
-        "trading_mode": {"type": "string", "enum": TRADING_MODES},
-        "margin_mode": {"type": "string", "enum": MARGIN_MODES},
-        "reduce_df_footprint": {"type": "boolean", "default": False},
-        "minimum_trade_amount": {"type": "number", "default": 10},
-        "targeted_trade_amount": {"type": "number", "default": 20},
-        "lookahead_analysis_exportfilename": {"type": "string"},
-        "startup_candle": {
-            "type": "array",
-            "uniqueItems": True,
-            "default": [199, 399, 499, 999, 1999],
-        },
-        "liquidation_buffer": {"type": "number", "minimum": 0.0, "maximum": 0.99},
-        "backtest_breakdown": {
-            "type": "array",
-            "items": {"type": "string", "enum": BACKTEST_BREAKDOWNS},
-        },
-        "bot_name": {"type": "string"},
-        "unfilledtimeout": {
-            "type": "object",
-            "properties": {
-                "entry": {"type": "number", "minimum": 1},
-                "exit": {"type": "number", "minimum": 1},
-                "exit_timeout_count": {"type": "number", "minimum": 0, "default": 0},
-                "unit": {"type": "string", "enum": TIMEOUT_UNITS, "default": "minutes"},
-            },
-        },
-        "entry_pricing": {
-            "type": "object",
-            "properties": {
-                "price_last_balance": {
-                    "type": "number",
-                    "minimum": 0,
-                    "maximum": 1,
-                    "exclusiveMaximum": False,
-                },
-                "price_side": {"type": "string", "enum": PRICING_SIDES, "default": "same"},
-                "use_order_book": {"type": "boolean"},
-                "order_book_top": {
-                    "type": "integer",
-                    "minimum": 1,
-                    "maximum": 50,
-                },
-                "check_depth_of_market": {
-                    "type": "object",
-                    "properties": {
-                        "enabled": {"type": "boolean"},
-                        "bids_to_ask_delta": {"type": "number", "minimum": 0},
-                    },
-                },
-            },
-            "required": ["price_side"],
-        },
-        "exit_pricing": {
-            "type": "object",
-            "properties": {
-                "price_side": {"type": "string", "enum": PRICING_SIDES, "default": "same"},
-                "price_last_balance": {
-                    "type": "number",
-                    "minimum": 0,
-                    "maximum": 1,
-                    "exclusiveMaximum": False,
-                },
-                "use_order_book": {"type": "boolean"},
-                "order_book_top": {
-                    "type": "integer",
-                    "minimum": 1,
-                    "maximum": 50,
-                },
-            },
-            "required": ["price_side"],
-        },
-        "custom_price_max_distance_ratio": {"type": "number", "minimum": 0.0},
-        "order_types": {
-            "type": "object",
-            "properties": {
-                "entry": {"type": "string", "enum": ORDERTYPE_POSSIBILITIES},
-                "exit": {"type": "string", "enum": ORDERTYPE_POSSIBILITIES},
-                "force_exit": {"type": "string", "enum": ORDERTYPE_POSSIBILITIES},
-                "force_entry": {"type": "string", "enum": ORDERTYPE_POSSIBILITIES},
-                "emergency_exit": {
-                    "type": "string",
-                    "enum": ORDERTYPE_POSSIBILITIES,
-                    "default": "market",
-                },
-                "stoploss": {"type": "string", "enum": ORDERTYPE_POSSIBILITIES},
-                "stoploss_on_exchange": {"type": "boolean"},
-                "stoploss_price_type": {"type": "string", "enum": STOPLOSS_PRICE_TYPES},
-                "stoploss_on_exchange_interval": {"type": "number"},
-                "stoploss_on_exchange_limit_ratio": {
-                    "type": "number",
-                    "minimum": 0.0,
-                    "maximum": 1.0,
-                },
-            },
-            "required": ["entry", "exit", "stoploss", "stoploss_on_exchange"],
-        },
-        "order_time_in_force": {
-            "type": "object",
-            "properties": {
-                "entry": {"type": "string", "enum": ORDERTIF_POSSIBILITIES},
-                "exit": {"type": "string", "enum": ORDERTIF_POSSIBILITIES},
-            },
-            "required": REQUIRED_ORDERTIF,
-        },
-        "coingecko": {
-            "type": "object",
-            "properties": {
-                "is_demo": {"type": "boolean", "default": True},
-                "api_key": {"type": "string"},
-            },
-            "required": ["is_demo", "api_key"],
-        },
-        "exchange": {"$ref": "#/definitions/exchange"},
-        "edge": {"$ref": "#/definitions/edge"},
-        "freqai": {"$ref": "#/definitions/freqai"},
-        "external_message_consumer": {"$ref": "#/definitions/external_message_consumer"},
-        "experimental": {
-            "type": "object",
-            "properties": {"block_bad_exchanges": {"type": "boolean"}},
-        },
-        "pairlists": {
-            "type": "array",
-            "items": {
-                "type": "object",
-                "properties": {
-                    "method": {"type": "string", "enum": AVAILABLE_PAIRLISTS},
-                },
-                "required": ["method"],
-            },
-        },
-        "protections": {
-            "type": "array",
-            "items": {
-                "type": "object",
-                "properties": {
-                    "method": {"type": "string", "enum": AVAILABLE_PROTECTIONS},
-                    "stop_duration": {"type": "number", "minimum": 0.0},
-                    "stop_duration_candles": {"type": "number", "minimum": 0},
-                    "trade_limit": {"type": "number", "minimum": 1},
-                    "lookback_period": {"type": "number", "minimum": 1},
-                    "lookback_period_candles": {"type": "number", "minimum": 1},
-                },
-                "required": ["method"],
-            },
-        },
-        "telegram": {
-            "type": "object",
-            "properties": {
-                "enabled": {"type": "boolean"},
-                "token": {"type": "string"},
-                "chat_id": {"type": "string"},
-                "allow_custom_messages": {"type": "boolean", "default": True},
-                "balance_dust_level": {"type": "number", "minimum": 0.0},
-                "notification_settings": {
-                    "type": "object",
-                    "default": {},
-                    "properties": {
-                        "status": {"type": "string", "enum": TELEGRAM_SETTING_OPTIONS},
-                        "warning": {"type": "string", "enum": TELEGRAM_SETTING_OPTIONS},
-                        "startup": {"type": "string", "enum": TELEGRAM_SETTING_OPTIONS},
-                        "entry": {"type": "string", "enum": TELEGRAM_SETTING_OPTIONS},
-                        "entry_fill": {
-                            "type": "string",
-                            "enum": TELEGRAM_SETTING_OPTIONS,
-                            "default": "off",
-                        },
-                        "entry_cancel": {
-                            "type": "string",
-                            "enum": TELEGRAM_SETTING_OPTIONS,
-                        },
-                        "exit": {
-                            "type": ["string", "object"],
-                            "additionalProperties": {
-                                "type": "string",
-                                "enum": TELEGRAM_SETTING_OPTIONS,
-                            },
-                        },
-                        "exit_fill": {
-                            "type": "string",
-                            "enum": TELEGRAM_SETTING_OPTIONS,
-                            "default": "on",
-                        },
-                        "exit_cancel": {"type": "string", "enum": TELEGRAM_SETTING_OPTIONS},
-                        "protection_trigger": {
-                            "type": "string",
-                            "enum": TELEGRAM_SETTING_OPTIONS,
-                            "default": "on",
-                        },
-                        "protection_trigger_global": {
-                            "type": "string",
-                            "enum": TELEGRAM_SETTING_OPTIONS,
-                            "default": "on",
-                        },
-                        "show_candle": {
-                            "type": "string",
-                            "enum": ["off", "ohlc"],
-                            "default": "off",
-                        },
-                        "strategy_msg": {
-                            "type": "string",
-                            "enum": TELEGRAM_SETTING_OPTIONS,
-                            "default": "on",
-                        },
-                    },
-                },
-                "reload": {"type": "boolean"},
-            },
-            "required": ["enabled", "token", "chat_id"],
-        },
-        "webhook": {
-            "type": "object",
-            "properties": {
-                "enabled": {"type": "boolean"},
-                "url": {"type": "string"},
-                "format": {"type": "string", "enum": WEBHOOK_FORMAT_OPTIONS, "default": "form"},
-                "retries": {"type": "integer", "minimum": 0},
-                "retry_delay": {"type": "number", "minimum": 0},
-                **__MESSAGE_TYPE_DICT,
-                # **{x: {'type': 'object'} for x in RPCMessageType},
-                # Below -> Deprecated
-                "webhookentry": {"type": "object"},
-                "webhookentrycancel": {"type": "object"},
-                "webhookentryfill": {"type": "object"},
-                "webhookexit": {"type": "object"},
-                "webhookexitcancel": {"type": "object"},
-                "webhookexitfill": {"type": "object"},
-                "webhookstatus": {"type": "object"},
-            },
-        },
-        "discord": {
-            "type": "object",
-            "properties": {
-                "enabled": {"type": "boolean"},
-                "webhook_url": {"type": "string"},
-                "exit_fill": {
-                    "type": "array",
-                    "items": {"type": "object"},
-                    "default": [
-                        {"Trade ID": "{trade_id}"},
-                        {"Exchange": "{exchange}"},
-                        {"Pair": "{pair}"},
-                        {"Direction": "{direction}"},
-                        {"Open rate": "{open_rate}"},
-                        {"Close rate": "{close_rate}"},
-                        {"Amount": "{amount}"},
-                        {"Open date": "{open_date:%Y-%m-%d %H:%M:%S}"},
-                        {"Close date": "{close_date:%Y-%m-%d %H:%M:%S}"},
-                        {"Profit": "{profit_amount} {stake_currency}"},
-                        {"Profitability": "{profit_ratio:.2%}"},
-                        {"Enter tag": "{enter_tag}"},
-                        {"Exit Reason": "{exit_reason}"},
-                        {"Strategy": "{strategy}"},
-                        {"Timeframe": "{timeframe}"},
-                    ],
-                },
-                "entry_fill": {
-                    "type": "array",
-                    "items": {"type": "object"},
-                    "default": [
-                        {"Trade ID": "{trade_id}"},
-                        {"Exchange": "{exchange}"},
-                        {"Pair": "{pair}"},
-                        {"Direction": "{direction}"},
-                        {"Open rate": "{open_rate}"},
-                        {"Amount": "{amount}"},
-                        {"Open date": "{open_date:%Y-%m-%d %H:%M:%S}"},
-                        {"Enter tag": "{enter_tag}"},
-                        {"Strategy": "{strategy} {timeframe}"},
-                    ],
-                },
-            },
-        },
-        "api_server": {
-            "type": "object",
-            "properties": {
-                "enabled": {"type": "boolean"},
-                "listen_ip_address": {"format": "ipv4"},
-                "listen_port": {"type": "integer", "minimum": 1024, "maximum": 65535},
-                "username": {"type": "string"},
-                "password": {"type": "string"},
-                "ws_token": {"type": ["string", "array"], "items": {"type": "string"}},
-                "jwt_secret_key": {"type": "string"},
-                "CORS_origins": {"type": "array", "items": {"type": "string"}},
-                "verbosity": {"type": "string", "enum": ["error", "info"]},
-            },
-            "required": ["enabled", "listen_ip_address", "listen_port", "username", "password"],
-        },
-        "db_url": {"type": "string"},
-        "export": {"type": "string", "enum": EXPORT_OPTIONS, "default": "trades"},
-        "disableparamexport": {"type": "boolean"},
-        "initial_state": {"type": "string", "enum": ["running", "stopped"]},
-        "force_entry_enable": {"type": "boolean"},
-        "disable_dataframe_checks": {"type": "boolean"},
-        "internals": {
-            "type": "object",
-            "default": {},
-            "properties": {
-                "process_throttle_secs": {"type": "integer"},
-                "interval": {"type": "integer"},
-                "sd_notify": {"type": "boolean"},
-            },
-        },
-        "dataformat_ohlcv": {
-            "type": "string",
-            "enum": AVAILABLE_DATAHANDLERS,
-            "default": "feather",
-        },
-        "dataformat_trades": {
-            "type": "string",
-            "enum": AVAILABLE_DATAHANDLERS,
-            "default": "feather",
-        },
-        "position_adjustment_enable": {"type": "boolean"},
-        "max_entry_position_adjustment": {"type": ["integer", "number"], "minimum": -1},
-    },
-    "definitions": {
-        "exchange": {
-            "type": "object",
-            "properties": {
-                "name": {"type": "string"},
-                "key": {"type": "string", "default": ""},
-                "secret": {"type": "string", "default": ""},
-                "password": {"type": "string", "default": ""},
-                "uid": {"type": "string"},
-                "pair_whitelist": {
-                    "type": "array",
-                    "items": {
-                        "type": "string",
-                    },
-                    "uniqueItems": True,
-                },
-                "pair_blacklist": {
-                    "type": "array",
-                    "items": {
-                        "type": "string",
-                    },
-                    "uniqueItems": True,
-                },
-                "unknown_fee_rate": {"type": "number"},
-                "outdated_offset": {"type": "integer", "minimum": 1},
-                "markets_refresh_interval": {"type": "integer"},
-                "ccxt_config": {"type": "object"},
-                "ccxt_async_config": {"type": "object"},
-            },
-            "required": ["name"],
-        },
-        "edge": {
-            "type": "object",
-            "properties": {
-                "enabled": {"type": "boolean"},
-                "process_throttle_secs": {"type": "integer", "minimum": 600},
-                "calculate_since_number_of_days": {"type": "integer"},
-                "allowed_risk": {"type": "number"},
-                "stoploss_range_min": {"type": "number"},
-                "stoploss_range_max": {"type": "number"},
-                "stoploss_range_step": {"type": "number"},
-                "minimum_winrate": {"type": "number"},
-                "minimum_expectancy": {"type": "number"},
-                "min_trade_number": {"type": "number"},
-                "max_trade_duration_minute": {"type": "integer"},
-                "remove_pumps": {"type": "boolean"},
-            },
-            "required": ["process_throttle_secs", "allowed_risk"],
-        },
-        "external_message_consumer": {
-            "type": "object",
-            "properties": {
-                "enabled": {"type": "boolean", "default": False},
-                "producers": {
-                    "type": "array",
-                    "items": {
-                        "type": "object",
-                        "properties": {
-                            "name": {"type": "string"},
-                            "host": {"type": "string"},
-                            "port": {
-                                "type": "integer",
-                                "default": 8080,
-                                "minimum": 0,
-                                "maximum": 65535,
-                            },
-                            "secure": {"type": "boolean", "default": False},
-                            "ws_token": {"type": "string"},
-                        },
-                        "required": ["name", "host", "ws_token"],
-                    },
-                },
-                "wait_timeout": {"type": "integer", "minimum": 0},
-                "sleep_time": {"type": "integer", "minimum": 0},
-                "ping_timeout": {"type": "integer", "minimum": 0},
-                "remove_entry_exit_signals": {"type": "boolean", "default": False},
-                "initial_candle_limit": {
-                    "type": "integer",
-                    "minimum": 0,
-                    "maximum": 1500,
-                    "default": 1500,
-                },
-                "message_size_limit": {  # In megabytes
-                    "type": "integer",
-                    "minimum": 1,
-                    "maximum": 20,
-                    "default": 8,
-                },
-            },
-            "required": ["producers"],
-        },
-        "freqai": {
-            "type": "object",
-            "properties": {
-                "enabled": {"type": "boolean", "default": False},
-                "keras": {"type": "boolean", "default": False},
-                "write_metrics_to_disk": {"type": "boolean", "default": False},
-                "purge_old_models": {"type": ["boolean", "number"], "default": 2},
-                "conv_width": {"type": "integer", "default": 1},
-                "train_period_days": {"type": "integer", "default": 0},
-                "backtest_period_days": {"type": "number", "default": 7},
-                "identifier": {"type": "string", "default": "example"},
-                "feature_parameters": {
-                    "type": "object",
-                    "properties": {
-                        "include_corr_pairlist": {"type": "array"},
-                        "include_timeframes": {"type": "array"},
-                        "label_period_candles": {"type": "integer"},
-                        "include_shifted_candles": {"type": "integer", "default": 0},
-                        "DI_threshold": {"type": "number", "default": 0},
-                        "weight_factor": {"type": "number", "default": 0},
-                        "principal_component_analysis": {"type": "boolean", "default": False},
-                        "use_SVM_to_remove_outliers": {"type": "boolean", "default": False},
-                        "plot_feature_importances": {"type": "integer", "default": 0},
-                        "svm_params": {
-                            "type": "object",
-                            "properties": {
-                                "shuffle": {"type": "boolean", "default": False},
-                                "nu": {"type": "number", "default": 0.1},
-                            },
-                        },
-                        "shuffle_after_split": {"type": "boolean", "default": False},
-                        "buffer_train_data_candles": {"type": "integer", "default": 0},
-                    },
-                    "required": [
-                        "include_timeframes",
-                        "include_corr_pairlist",
-                    ],
-                },
-                "data_split_parameters": {
-                    "type": "object",
-                    "properties": {
-                        "test_size": {"type": "number"},
-                        "random_state": {"type": "integer"},
-                        "shuffle": {"type": "boolean", "default": False},
-                    },
-                },
-                "model_training_parameters": {"type": "object"},
-                "rl_config": {
-                    "type": "object",
-                    "properties": {
-                        "drop_ohlc_from_features": {"type": "boolean", "default": False},
-                        "train_cycles": {"type": "integer"},
-                        "max_trade_duration_candles": {"type": "integer"},
-                        "add_state_info": {"type": "boolean", "default": False},
-                        "max_training_drawdown_pct": {"type": "number", "default": 0.02},
-                        "cpu_count": {"type": "integer", "default": 1},
-                        "model_type": {"type": "string", "default": "PPO"},
-                        "policy_type": {"type": "string", "default": "MlpPolicy"},
-                        "net_arch": {"type": "array", "default": [128, 128]},
-                        "randomize_starting_position": {"type": "boolean", "default": False},
-                        "progress_bar": {"type": "boolean", "default": True},
-                        "model_reward_parameters": {
-                            "type": "object",
-                            "properties": {
-                                "rr": {"type": "number", "default": 1},
-                                "profit_aim": {"type": "number", "default": 0.025},
-                            },
-                        },
-                    },
-                },
-            },
-            "required": [
-                "enabled",
-                "train_period_days",
-                "backtest_period_days",
-                "identifier",
-                "feature_parameters",
-                "data_split_parameters",
-            ],
-        },
-    },
-}
-
-SCHEMA_TRADE_REQUIRED = [
-    "exchange",
-    "timeframe",
-    "max_open_trades",
-    "stake_currency",
-    "stake_amount",
-    "tradable_balance_ratio",
-    "last_stake_amount_min_ratio",
-    "dry_run",
-    "dry_run_wallet",
-    "exit_pricing",
-    "entry_pricing",
-    "stoploss",
-    "minimal_roi",
-    "internals",
-    "dataformat_ohlcv",
-    "dataformat_trades",
-]
-
-SCHEMA_BACKTEST_REQUIRED = [
-    "exchange",
-    "stake_currency",
-    "stake_amount",
-    "dry_run_wallet",
-    "dataformat_ohlcv",
-    "dataformat_trades",
-]
-SCHEMA_BACKTEST_REQUIRED_FINAL = SCHEMA_BACKTEST_REQUIRED + [
-    "stoploss",
-    "minimal_roi",
-    "max_open_trades",
-]
-
-SCHEMA_MINIMAL_REQUIRED = [
-    "exchange",
-    "dry_run",
-    "dataformat_ohlcv",
-    "dataformat_trades",
-]
-SCHEMA_MINIMAL_WEBSERVER = SCHEMA_MINIMAL_REQUIRED + [
-    "api_server",
-]
 
 CANCEL_REASON = {
     "TIMEOUT": "cancelled due to timeout",
@@ -770,6 +192,9 @@ ListPairsWithTimeframes = List[PairWithTimeframe]
 
 # Type for trades list
 TradeList = List[List]
+# ticks, pair, timeframe, CandleType
+TickWithTimeframe = Tuple[str, str, CandleType, Optional[int], Optional[int]]
+ListTicksWithTimeframes = List[TickWithTimeframe]
 
 LongShort = Literal["long", "short"]
 EntryExit = Literal["entry", "exit"]

freqtrade/data/__init__.py

@@ -2,5 +2,8 @@
 Module to handle data operations for freqtrade
 """
 
+from freqtrade.data import converter
+
+
 # limit what's imported when using `from freqtrade.data import *`
 __all__ = ["converter"]

freqtrade/data/btanalysis.py

@@ -185,7 +185,7 @@ def load_and_merge_backtest_result(strategy_name: str, filename: Path, results:
     """
     bt_data = load_backtest_stats(filename)
     k: Literal["metadata", "strategy"]
-    for k in ("metadata", "strategy"):  # type: ignore
+    for k in ("metadata", "strategy"):
         results[k][strategy_name] = bt_data[k][strategy_name]
     results["metadata"][strategy_name]["filename"] = filename.stem
     comparison = bt_data["strategy_comparison"]

freqtrade/data/converter/__init__.py

@@ -8,6 +8,7 @@ from freqtrade.data.converter.converter import (
     trim_dataframe,
     trim_dataframes,
 )
+from freqtrade.data.converter.orderflow import populate_dataframe_with_trades
 from freqtrade.data.converter.trade_converter import (
     convert_trades_format,
     convert_trades_to_ohlcv,
@@ -30,6 +31,7 @@ __all__ = [
     "trim_dataframes",
     "convert_trades_format",
     "convert_trades_to_ohlcv",
+    "populate_dataframe_with_trades",
     "trades_convert_types",
     "trades_df_remove_duplicates",
     "trades_dict_to_list",

freqtrade/data/converter/orderflow.py

@@ -0,0 +1,295 @@
+"""
+Functions to convert orderflow data from public_trades
+"""
+
+import logging
+import time
+import typing
+from collections import OrderedDict
+from datetime import datetime
+from typing import Tuple
+
+import numpy as np
+import pandas as pd
+
+from freqtrade.constants import DEFAULT_ORDERFLOW_COLUMNS
+from freqtrade.enums import RunMode
+from freqtrade.exceptions import DependencyException
+
+
+logger = logging.getLogger(__name__)
+
+
+def _init_dataframe_with_trades_columns(dataframe: pd.DataFrame):
+    """
+    Populates a dataframe with trades columns
+    :param dataframe: Dataframe to populate
+    """
+    # Initialize columns with appropriate dtypes
+    dataframe["trades"] = np.nan
+    dataframe["orderflow"] = np.nan
+    dataframe["imbalances"] = np.nan
+    dataframe["stacked_imbalances_bid"] = np.nan
+    dataframe["stacked_imbalances_ask"] = np.nan
+    dataframe["max_delta"] = np.nan
+    dataframe["min_delta"] = np.nan
+    dataframe["bid"] = np.nan
+    dataframe["ask"] = np.nan
+    dataframe["delta"] = np.nan
+    dataframe["total_trades"] = np.nan
+
+    # Ensure the 'trades' column is of object type
+    dataframe["trades"] = dataframe["trades"].astype(object)
+    dataframe["orderflow"] = dataframe["orderflow"].astype(object)
+    dataframe["imbalances"] = dataframe["imbalances"].astype(object)
+    dataframe["stacked_imbalances_bid"] = dataframe["stacked_imbalances_bid"].astype(object)
+    dataframe["stacked_imbalances_ask"] = dataframe["stacked_imbalances_ask"].astype(object)
+
+
+def _calculate_ohlcv_candle_start_and_end(df: pd.DataFrame, timeframe: str):
+    from freqtrade.exchange import timeframe_to_next_date, timeframe_to_resample_freq
+
+    timeframe_frequency = timeframe_to_resample_freq(timeframe)
+    # calculate ohlcv candle start and end
+    if df is not None and not df.empty:
+        df["datetime"] = pd.to_datetime(df["date"], unit="ms")
+        df["candle_start"] = df["datetime"].dt.floor(timeframe_frequency)
+        # used in _now_is_time_to_refresh_trades
+        df["candle_end"] = df["candle_start"].apply(
+            lambda candle_start: timeframe_to_next_date(timeframe, candle_start)
+        )
+        df.drop(columns=["datetime"], inplace=True)
+
+
+def populate_dataframe_with_trades(
+    cached_grouped_trades: OrderedDict[Tuple[datetime, datetime], pd.DataFrame],
+    config,
+    dataframe: pd.DataFrame,
+    trades: pd.DataFrame,
+) -> Tuple[pd.DataFrame, OrderedDict[Tuple[datetime, datetime], pd.DataFrame]]:
+    """
+    Populates a dataframe with trades
+    :param dataframe: Dataframe to populate
+    :param trades: Trades to populate with
+    :return: Dataframe with trades populated
+    """
+    timeframe = config["timeframe"]
+    config_orderflow = config["orderflow"]
+
+    # create columns for trades
+    _init_dataframe_with_trades_columns(dataframe)
+
+    try:
+        start_time = time.time()
+        # calculate ohlcv candle start and end
+        _calculate_ohlcv_candle_start_and_end(trades, timeframe)
+
+        # get date of earliest max_candles candle
+        max_candles = config_orderflow["max_candles"]
+        start_date = dataframe.tail(max_candles).date.iat[0]
+        # slice of trades that are before current ohlcv candles to make groupby faster
+        trades = trades.loc[trades.candle_start >= start_date]
+        trades.reset_index(inplace=True, drop=True)
+
+        # group trades by candle start
+        trades_grouped_by_candle_start = trades.groupby("candle_start", group_keys=False)
+        # Create Series to hold complex data
+        trades_series = pd.Series(index=dataframe.index, dtype=object)
+        orderflow_series = pd.Series(index=dataframe.index, dtype=object)
+        imbalances_series = pd.Series(index=dataframe.index, dtype=object)
+        stacked_imbalances_bid_series = pd.Series(index=dataframe.index, dtype=object)
+        stacked_imbalances_ask_series = pd.Series(index=dataframe.index, dtype=object)
+
+        trades_grouped_by_candle_start = trades.groupby("candle_start", group_keys=False)
+        for candle_start, trades_grouped_df in trades_grouped_by_candle_start:
+            is_between = candle_start == dataframe["date"]
+            if is_between.any():
+                from freqtrade.exchange import timeframe_to_next_date
+
+                candle_next = timeframe_to_next_date(timeframe, typing.cast(datetime, candle_start))
+                if candle_next not in trades_grouped_by_candle_start.groups:
+                    logger.warning(
+                        f"candle at {candle_start} with {len(trades_grouped_df)} trades "
+                        f"might be unfinished, because no finished trades at {candle_next}"
+                    )
+
+                indices = dataframe.index[is_between].tolist()
+                # Add trades to each candle
+                trades_series.loc[indices] = [
+                    trades_grouped_df.drop(columns=["candle_start", "candle_end"]).to_dict(
+                        orient="records"
+                    )
+                ]
+                # Use caching mechanism
+                if (candle_start, candle_next) in cached_grouped_trades:
+                    cache_entry = cached_grouped_trades[
+                        (typing.cast(datetime, candle_start), candle_next)
+                    ]
+                    # dataframe.loc[is_between] = cache_entry # doesn't take, so we need workaround:
+                    # Create a dictionary of the column values to be assigned
+                    update_dict = {c: cache_entry[c].iat[0] for c in cache_entry.columns}
+                    # Assign the values using the update_dict
+                    dataframe.loc[is_between, update_dict.keys()] = pd.DataFrame(
+                        [update_dict], index=dataframe.loc[is_between].index
+                    )
+                    continue
+
+                # Calculate orderflow for each candle
+                orderflow = trades_to_volumeprofile_with_total_delta_bid_ask(
+                    trades_grouped_df, scale=config_orderflow["scale"]
+                )
+                orderflow_series.loc[indices] = [orderflow.to_dict(orient="index")]
+                # Calculate imbalances for each candle's orderflow
+                imbalances = trades_orderflow_to_imbalances(
+                    orderflow,
+                    imbalance_ratio=config_orderflow["imbalance_ratio"],
+                    imbalance_volume=config_orderflow["imbalance_volume"],
+                )
+                imbalances_series.loc[indices] = [imbalances.to_dict(orient="index")]
+
+                stacked_imbalance_range = config_orderflow["stacked_imbalance_range"]
+                stacked_imbalances_bid_series.loc[indices] = [
+                    stacked_imbalance_bid(
+                        imbalances, stacked_imbalance_range=stacked_imbalance_range
+                    )
+                ]
+                stacked_imbalances_ask_series.loc[indices] = [
+                    stacked_imbalance_ask(
+                        imbalances, stacked_imbalance_range=stacked_imbalance_range
+                    )
+                ]
+
+                bid = np.where(
+                    trades_grouped_df["side"].str.contains("sell"), trades_grouped_df["amount"], 0
+                )
+
+                ask = np.where(
+                    trades_grouped_df["side"].str.contains("buy"), trades_grouped_df["amount"], 0
+                )
+                deltas_per_trade = ask - bid
+                min_delta = deltas_per_trade.cumsum().min()
+                max_delta = deltas_per_trade.cumsum().max()
+                dataframe.loc[indices, "max_delta"] = max_delta
+                dataframe.loc[indices, "min_delta"] = min_delta
+
+                dataframe.loc[indices, "bid"] = bid.sum()
+                dataframe.loc[indices, "ask"] = ask.sum()
+                dataframe.loc[indices, "delta"] = (
+                    dataframe.loc[indices, "ask"] - dataframe.loc[indices, "bid"]
+                )
+                dataframe.loc[indices, "total_trades"] = len(trades_grouped_df)
+
+                # Cache the result
+                cached_grouped_trades[(typing.cast(datetime, candle_start), candle_next)] = (
+                    dataframe.loc[is_between].copy()
+                )
+
+                # Maintain cache size
+                if (
+                    config.get("runmode") in (RunMode.DRY_RUN, RunMode.LIVE)
+                    and len(cached_grouped_trades) > config_orderflow["cache_size"]
+                ):
+                    cached_grouped_trades.popitem(last=False)
+            else:
+                logger.debug(f"Found NO candles for trades starting with {candle_start}")
+        logger.debug(f"trades.groups_keys in {time.time() - start_time} seconds")
+
+        # Merge the complex data Series back into the DataFrame
+        dataframe["trades"] = trades_series
+        dataframe["orderflow"] = orderflow_series
+        dataframe["imbalances"] = imbalances_series
+        dataframe["stacked_imbalances_bid"] = stacked_imbalances_bid_series
+        dataframe["stacked_imbalances_ask"] = stacked_imbalances_ask_series
+
+    except Exception as e:
+        logger.exception("Error populating dataframe with trades")
+        raise DependencyException(e)
+
+    return dataframe, cached_grouped_trades
+
+
+def trades_to_volumeprofile_with_total_delta_bid_ask(
+    trades: pd.DataFrame, scale: float
+) -> pd.DataFrame:
+    """
+    :param trades: dataframe
+    :param scale: scale aka bin size e.g. 0.5
+    :return: trades binned to levels according to scale aka orderflow
+    """
+    df = pd.DataFrame([], columns=DEFAULT_ORDERFLOW_COLUMNS)
+    # create bid, ask where side is sell or buy
+    df["bid_amount"] = np.where(trades["side"].str.contains("sell"), trades["amount"], 0)
+    df["ask_amount"] = np.where(trades["side"].str.contains("buy"), trades["amount"], 0)
+    df["bid"] = np.where(trades["side"].str.contains("sell"), 1, 0)
+    df["ask"] = np.where(trades["side"].str.contains("buy"), 1, 0)
+    # round the prices to the nearest multiple of the scale
+    df["price"] = ((trades["price"] / scale).round() * scale).astype("float64").values
+    if df.empty:
+        df["total"] = np.nan
+        df["delta"] = np.nan
+        return df
+
+    df["delta"] = df["ask_amount"] - df["bid_amount"]
+    df["total_volume"] = df["ask_amount"] + df["bid_amount"]
+    df["total_trades"] = df["ask"] + df["bid"]
+
+    # group to bins aka apply scale
+    df = df.groupby("price").sum(numeric_only=True)
+    return df
+
+
+def trades_orderflow_to_imbalances(df: pd.DataFrame, imbalance_ratio: int, imbalance_volume: int):
+    """
+    :param df: dataframes with bid and ask
+    :param imbalance_ratio: imbalance_ratio e.g. 3
+    :param imbalance_volume: imbalance volume e.g. 10
+    :return: dataframe with bid and ask imbalance
+    """
+    bid = df.bid
+    # compares bid and ask diagonally
+    ask = df.ask.shift(-1)
+    bid_imbalance = (bid / ask) > (imbalance_ratio)
+    # overwrite bid_imbalance with False if volume is not big enough
+    bid_imbalance_filtered = np.where(df.total_volume < imbalance_volume, False, bid_imbalance)
+    ask_imbalance = (ask / bid) > (imbalance_ratio)
+    # overwrite ask_imbalance with False if volume is not big enough
+    ask_imbalance_filtered = np.where(df.total_volume < imbalance_volume, False, ask_imbalance)
+    dataframe = pd.DataFrame(
+        {"bid_imbalance": bid_imbalance_filtered, "ask_imbalance": ask_imbalance_filtered},
+        index=df.index,
+    )
+
+    return dataframe
+
+
+def stacked_imbalance(
+    df: pd.DataFrame, label: str, stacked_imbalance_range: int, should_reverse: bool
+):
+    """
+    y * (y.groupby((y != y.shift()).cumsum()).cumcount() + 1)
+    https://stackoverflow.com/questions/27626542/counting-consecutive-positive-values-in-python-pandas-array
+    """
+    imbalance = df[f"{label}_imbalance"]
+    int_series = pd.Series(np.where(imbalance, 1, 0))
+    stacked = int_series * (
+        int_series.groupby((int_series != int_series.shift()).cumsum()).cumcount() + 1
+    )
+
+    max_stacked_imbalance_idx = stacked.index[stacked >= stacked_imbalance_range]
+    stacked_imbalance_price = np.nan
+    if not max_stacked_imbalance_idx.empty:
+        idx = (
+            max_stacked_imbalance_idx[0]
+            if not should_reverse
+            else np.flipud(max_stacked_imbalance_idx)[0]
+        )
+        stacked_imbalance_price = imbalance.index[idx]
+    return stacked_imbalance_price
+
+
+def stacked_imbalance_ask(df: pd.DataFrame, stacked_imbalance_range: int):
+    return stacked_imbalance(df, "ask", stacked_imbalance_range, should_reverse=True)
+
+
+def stacked_imbalance_bid(df: pd.DataFrame, stacked_imbalance_range: int):
+    return stacked_imbalance(df, "bid", stacked_imbalance_range, should_reverse=False)

freqtrade/data/dataprovider.py

@@ -19,8 +19,8 @@ from freqtrade.constants import (
     ListPairsWithTimeframes,
     PairWithTimeframe,
 )
-from freqtrade.data.history import load_pair_history
-from freqtrade.enums import CandleType, RPCMessageType, RunMode
+from freqtrade.data.history import get_datahandler, load_pair_history
+from freqtrade.enums import CandleType, RPCMessageType, RunMode, TradingMode
 from freqtrade.exceptions import ExchangeError, OperationalException
 from freqtrade.exchange import Exchange, timeframe_to_prev_date, timeframe_to_seconds
 from freqtrade.exchange.types import OrderBook
@@ -445,7 +445,20 @@ class DataProvider:
         if self._exchange is None:
             raise OperationalException(NO_EXCHANGE_EXCEPTION)
         final_pairs = (pairlist + helping_pairs) if helping_pairs else pairlist
+        # refresh latest ohlcv data
         self._exchange.refresh_latest_ohlcv(final_pairs)
+        # refresh latest trades data
+        self.refresh_latest_trades(pairlist)
+
+    def refresh_latest_trades(self, pairlist: ListPairsWithTimeframes) -> None:
+        """
+        Refresh latest trades data (if enabled in config)
+        """
+
+        use_public_trades = self._config.get("exchange", {}).get("use_public_trades", False)
+        if use_public_trades:
+            if self._exchange:
+                self._exchange.refresh_latest_trades(pairlist)
 
     @property
     def available_pairs(self) -> ListPairsWithTimeframes:
@@ -483,6 +496,45 @@ class DataProvider:
         else:
             return DataFrame()
 
+    def trades(
+        self, pair: str, timeframe: Optional[str] = None, copy: bool = True, candle_type: str = ""
+    ) -> DataFrame:
+        """
+        Get candle (TRADES) data for the given pair as DataFrame
+        Please use the `available_pairs` method to verify which pairs are currently cached.
+        This is not meant to be used in callbacks because of lookahead bias.
+        :param pair: pair to get the data for
+        :param timeframe: Timeframe to get data for
+        :param candle_type: '', mark, index, premiumIndex, or funding_rate
+        :param copy: copy dataframe before returning if True.
+                     Use False only for read-only operations (where the dataframe is not modified)
+        """
+        if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
+            if self._exchange is None:
+                raise OperationalException(NO_EXCHANGE_EXCEPTION)
+            _candle_type = (
+                CandleType.from_string(candle_type)
+                if candle_type != ""
+                else self._config["candle_type_def"]
+            )
+            return self._exchange.trades(
+                (pair, timeframe or self._config["timeframe"], _candle_type), copy=copy
+            )
+        elif self.runmode in (RunMode.BACKTEST, RunMode.HYPEROPT):
+            _candle_type = (
+                CandleType.from_string(candle_type)
+                if candle_type != ""
+                else self._config["candle_type_def"]
+            )
+            data_handler = get_datahandler(
+                self._config["datadir"], data_format=self._config["dataformat_trades"]
+            )
+            trades_df = data_handler.trades_load(pair, TradingMode.FUTURES)
+            return trades_df
+
+        else:
+            return DataFrame()
+
     def market(self, pair: str) -> Optional[Dict[str, Any]]:
         """
         Return market data for the pair

freqtrade/data/entryexitanalysis.py

@@ -4,7 +4,6 @@ from typing import List
 
 import joblib
 import pandas as pd
-from tabulate import tabulate
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import Config
@@ -14,6 +13,7 @@ from freqtrade.data.btanalysis import (
     load_backtest_stats,
 )
 from freqtrade.exceptions import OperationalException
+from freqtrade.util import print_df_rich_table
 
 
 logger = logging.getLogger(__name__)
@@ -307,7 +307,7 @@ def _print_table(
         if name is not None:
             print(name)
 
-        print(tabulate(data, headers="keys", tablefmt="psql", showindex=show_index))
+        print_df_rich_table(data, data.keys(), show_index=show_index)
 
 
 def process_entry_exit_reasons(config: Config):

freqtrade/data/history/history_utils.py

@@ -26,8 +26,7 @@ from freqtrade.enums import CandleType, TradingMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import Exchange
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
-from freqtrade.util import dt_ts, format_ms_time
-from freqtrade.util.datetime_helpers import dt_now
+from freqtrade.util import dt_now, dt_ts, format_ms_time, get_progress_tracker
 from freqtrade.util.migrations import migrate_data
 
 
@@ -155,11 +154,9 @@ def refresh_data(
     :param candle_type: Any of the enum CandleType (must match trading mode!)
     """
     data_handler = get_datahandler(datadir, data_format)
-    for idx, pair in enumerate(pairs):
-        process = f"{idx}/{len(pairs)}"
+    for pair in pairs:
         _download_pair_history(
             pair=pair,
-            process=process,
             timeframe=timeframe,
             datadir=datadir,
             timerange=timerange,
@@ -223,7 +220,6 @@ def _download_pair_history(
     datadir: Path,
     exchange: Exchange,
     timeframe: str = "5m",
-    process: str = "",
     new_pairs_days: int = 30,
     data_handler: Optional[IDataHandler] = None,
     timerange: Optional[TimeRange] = None,
@@ -261,7 +257,7 @@ def _download_pair_history(
         )
 
         logger.info(
-            f'({process}) - Download history data for "{pair}", {timeframe}, '
+            f'Download history data for "{pair}", {timeframe}, '
             f"{candle_type} and store in {datadir}. "
             f'From {format_ms_time(since_ms) if since_ms else "start"} to '
             f'{format_ms_time(until_ms) if until_ms else "now"}'
@@ -345,53 +341,65 @@ def refresh_backtest_ohlcv_data(
     pairs_not_available = []
     data_handler = get_datahandler(datadir, data_format)
     candle_type = CandleType.get_default(trading_mode)
-    process = ""
-    for idx, pair in enumerate(pairs, start=1):
-        if pair not in exchange.markets:
-            pairs_not_available.append(pair)
-            logger.info(f"Skipping pair {pair}...")
-            continue
-        for timeframe in timeframes:
-            logger.debug(f"Downloading pair {pair}, {candle_type}, interval {timeframe}.")
-            process = f"{idx}/{len(pairs)}"
-            _download_pair_history(
-                pair=pair,
-                process=process,
-                datadir=datadir,
-                exchange=exchange,
-                timerange=timerange,
-                data_handler=data_handler,
-                timeframe=str(timeframe),
-                new_pairs_days=new_pairs_days,
-                candle_type=candle_type,
-                erase=erase,
-                prepend=prepend,
-            )
-        if trading_mode == "futures":
-            # Predefined candletype (and timeframe) depending on exchange
-            # Downloads what is necessary to backtest based on futures data.
-            tf_mark = exchange.get_option("mark_ohlcv_timeframe")
-            tf_funding_rate = exchange.get_option("funding_fee_timeframe")
+    with get_progress_tracker() as progress:
+        tf_length = len(timeframes) if trading_mode != "futures" else len(timeframes) + 2
+        timeframe_task = progress.add_task("Timeframe", total=tf_length)
+        pair_task = progress.add_task("Downloading data...", total=len(pairs))
 
-            fr_candle_type = CandleType.from_string(exchange.get_option("mark_ohlcv_price"))
-            # All exchanges need FundingRate for futures trading.
-            # The timeframe is aligned to the mark-price timeframe.
-            combs = ((CandleType.FUNDING_RATE, tf_funding_rate), (fr_candle_type, tf_mark))
-            for candle_type_f, tf in combs:
-                logger.debug(f"Downloading pair {pair}, {candle_type_f}, interval {tf}.")
+        for pair in pairs:
+            progress.update(pair_task, description=f"Downloading {pair}")
+            progress.update(timeframe_task, completed=0)
+
+            if pair not in exchange.markets:
+                pairs_not_available.append(pair)
+                logger.info(f"Skipping pair {pair}...")
+                continue
+            for timeframe in timeframes:
+                progress.update(timeframe_task, description=f"Timeframe {timeframe}")
+                logger.debug(f"Downloading pair {pair}, {candle_type}, interval {timeframe}.")
                 _download_pair_history(
                     pair=pair,
-                    process=process,
                     datadir=datadir,
                     exchange=exchange,
                     timerange=timerange,
                     data_handler=data_handler,
-                    timeframe=str(tf),
+                    timeframe=str(timeframe),
                     new_pairs_days=new_pairs_days,
-                    candle_type=candle_type_f,
+                    candle_type=candle_type,
                     erase=erase,
                     prepend=prepend,
                 )
+                progress.update(timeframe_task, advance=1)
+            if trading_mode == "futures":
+                # Predefined candletype (and timeframe) depending on exchange
+                # Downloads what is necessary to backtest based on futures data.
+                tf_mark = exchange.get_option("mark_ohlcv_timeframe")
+                tf_funding_rate = exchange.get_option("funding_fee_timeframe")
+
+                fr_candle_type = CandleType.from_string(exchange.get_option("mark_ohlcv_price"))
+                # All exchanges need FundingRate for futures trading.
+                # The timeframe is aligned to the mark-price timeframe.
+                combs = ((CandleType.FUNDING_RATE, tf_funding_rate), (fr_candle_type, tf_mark))
+                for candle_type_f, tf in combs:
+                    logger.debug(f"Downloading pair {pair}, {candle_type_f}, interval {tf}.")
+                    _download_pair_history(
+                        pair=pair,
+                        datadir=datadir,
+                        exchange=exchange,
+                        timerange=timerange,
+                        data_handler=data_handler,
+                        timeframe=str(tf),
+                        new_pairs_days=new_pairs_days,
+                        candle_type=candle_type_f,
+                        erase=erase,
+                        prepend=prepend,
+                    )
+                    progress.update(
+                        timeframe_task, advance=1, description=f"Timeframe {candle_type_f}, {tf}"
+                    )
+
+            progress.update(pair_task, advance=1)
+            progress.update(timeframe_task, description="Timeframe")
 
     return pairs_not_available
 
@@ -480,7 +488,7 @@ def _download_trades_history(
         return True
 
     except Exception:
-        logger.exception(f'Failed to download historic trades for pair: "{pair}". ')
+        logger.exception(f'Failed to download and store historic trades for pair: "{pair}". ')
         return False
 
 
@@ -501,25 +509,30 @@ def refresh_backtest_trades_data(
     """
     pairs_not_available = []
     data_handler = get_datahandler(datadir, data_format=data_format)
-    for pair in pairs:
-        if pair not in exchange.markets:
-            pairs_not_available.append(pair)
-            logger.info(f"Skipping pair {pair}...")
-            continue
+    with get_progress_tracker() as progress:
+        pair_task = progress.add_task("Downloading data...", total=len(pairs))
+        for pair in pairs:
+            progress.update(pair_task, description=f"Downloading trades [{pair}]")
+            if pair not in exchange.markets:
+                pairs_not_available.append(pair)
+                logger.info(f"Skipping pair {pair}...")
+                continue
 
-        if erase:
-            if data_handler.trades_purge(pair, trading_mode):
-                logger.info(f"Deleting existing data for pair {pair}.")
+            if erase:
+                if data_handler.trades_purge(pair, trading_mode):
+                    logger.info(f"Deleting existing data for pair {pair}.")
+
+            logger.info(f"Downloading trades for pair {pair}.")
+            _download_trades_history(
+                exchange=exchange,
+                pair=pair,
+                new_pairs_days=new_pairs_days,
+                timerange=timerange,
+                data_handler=data_handler,
+                trading_mode=trading_mode,
+            )
+            progress.update(pair_task, advance=1)
 
-        logger.info(f"Downloading trades for pair {pair}.")
-        _download_trades_history(
-            exchange=exchange,
-            pair=pair,
-            new_pairs_days=new_pairs_days,
-            timerange=timerange,
-            data_handler=data_handler,
-            trading_mode=trading_mode,
-        )
     return pairs_not_available
 
 

freqtrade/exchange/__init__.py

@@ -11,6 +11,7 @@ from freqtrade.exchange.bitpanda import Bitpanda
 from freqtrade.exchange.bitvavo import Bitvavo
 from freqtrade.exchange.bybit import Bybit
 from freqtrade.exchange.coinbasepro import Coinbasepro
+from freqtrade.exchange.cryptocom import Cryptocom
 from freqtrade.exchange.exchange_utils import (
     ROUND_DOWN,
     ROUND_UP,

freqtrade/exchange/binance.py

@@ -30,6 +30,7 @@ class Binance(Exchange):
         "trades_pagination_arg": "fromId",
         "trades_has_history": True,
         "l2_limit_range": [5, 10, 20, 50, 100, 500, 1000],
+        "ws.enabled": True,
     }
     _ft_has_futures: Dict = {
         "stoploss_order_types": {"limit": "stop", "market": "stop_market"},
@@ -42,6 +43,7 @@ class Binance(Exchange):
             PriceType.LAST: "CONTRACT_PRICE",
             PriceType.MARK: "MARK_PRICE",
         },
+        "ws.enabled": False,
     }
 
     _supported_trading_mode_margin_pairs: List[Tuple[TradingMode, MarginMode]] = [

freqtrade/exchange/bybit.py

@@ -33,6 +33,7 @@ class Bybit(Exchange):
         "ohlcv_candle_limit": 1000,
         "ohlcv_has_history": True,
         "order_time_in_force": ["GTC", "FOK", "IOC", "PO"],
+        "ws.enabled": True,
         "trades_has_history": False,  # Endpoint doesn't support pagination
     }
     _ft_has_futures: Dict = {

freqtrade/exchange/check_exchange.py

@@ -47,7 +47,7 @@ def check_exchange(config: Config, check_for_bad: bool = True) -> bool:
             f'{", ".join(available_exchanges())}'
         )
 
-    valid, reason = validate_exchange(exchange)
+    valid, reason, _ = validate_exchange(exchange)
     if not valid:
         if check_for_bad:
             raise OperationalException(

freqtrade/exchange/common.py

@@ -37,7 +37,6 @@ API_FETCH_ORDER_RETRY_COUNT = 5
 
 BAD_EXCHANGES = {
     "bitmex": "Various reasons.",
-    "phemex": "Does not provide history.",
     "probit": "Requires additional, regular calls to `signIn()`.",
     "poloniex": "Does not provide fetch_order endpoint to fetch both open and closed orders.",
 }
@@ -65,6 +64,7 @@ SUPPORTED_EXCHANGES = [
 EXCHANGE_HAS_REQUIRED: Dict[str, List[str]] = {
     # Required / private
     "fetchOrder": ["fetchOpenOrder", "fetchClosedOrder"],
+    "fetchL2OrderBook": ["fetchTicker"],
     "cancelOrder": [],
     "createOrder": [],
     "fetchBalance": [],
@@ -92,6 +92,8 @@ EXCHANGE_HAS_OPTIONAL = [
     # 'fetchMarketLeverageTiers',  # Futures initialization
     # 'fetchOpenOrder', 'fetchClosedOrder',  # replacement for fetchOrder
     # 'fetchOpenOrders', 'fetchClosedOrders',  # 'fetchOrders',  # Refinding balance...
+    # ccxt.pro
+    "watchOHLCV",
 ]
 
 

freqtrade/exchange/cryptocom.py

@@ -0,0 +1,19 @@
+"""Crypto.com exchange subclass"""
+
+import logging
+from typing import Dict
+
+from freqtrade.exchange import Exchange
+
+
+logger = logging.getLogger(__name__)
+
+
+class Cryptocom(Exchange):
+    """Crypto.com exchange class.
+    Contains adjustments needed for Freqtrade to work with this exchange.
+    """
+
+    _ft_has: Dict = {
+        "ohlcv_candle_limit": 300,
+    }

freqtrade/exchange/exchange.py

@@ -14,7 +14,7 @@ from threading import Lock
 from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union
 
 import ccxt
-import ccxt.async_support as ccxt_async
+import ccxt.pro as ccxt_pro
 from cachetools import TTLCache
 from ccxt import TICK_SIZE
 from dateutil import parser
@@ -22,6 +22,7 @@ from pandas import DataFrame, concat
 
 from freqtrade.constants import (
     DEFAULT_AMOUNT_RESERVE_PERCENT,
+    DEFAULT_TRADES_COLUMNS,
     NON_OPEN_EXCHANGE_STATES,
     BidAsk,
     BuySell,
@@ -33,8 +34,22 @@ from freqtrade.constants import (
     OBLiteral,
     PairWithTimeframe,
 )
-from freqtrade.data.converter import clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_dict_to_list
-from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, PriceType, RunMode, TradingMode
+from freqtrade.data.converter import (
+    clean_ohlcv_dataframe,
+    ohlcv_to_dataframe,
+    trades_df_remove_duplicates,
+    trades_dict_to_list,
+    trades_list_to_df,
+)
+from freqtrade.enums import (
+    OPTIMIZE_MODES,
+    TRADE_MODES,
+    CandleType,
+    MarginMode,
+    PriceType,
+    RunMode,
+    TradingMode,
+)
 from freqtrade.exceptions import (
     ConfigurationError,
     DDosProtection,
@@ -56,7 +71,6 @@ from freqtrade.exchange.exchange_utils import (
     ROUND,
     ROUND_DOWN,
     ROUND_UP,
-    CcxtModuleType,
     amount_to_contract_precision,
     amount_to_contracts,
     amount_to_precision,
@@ -73,6 +87,7 @@ from freqtrade.exchange.exchange_utils_timeframe import (
     timeframe_to_prev_date,
     timeframe_to_seconds,
 )
+from freqtrade.exchange.exchange_ws import ExchangeWS
 from freqtrade.exchange.types import OHLCVResponse, OrderBook, Ticker, Tickers
 from freqtrade.misc import (
     chunks,
@@ -83,7 +98,7 @@ from freqtrade.misc import (
 )
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.util import dt_from_ts, dt_now
-from freqtrade.util.datetime_helpers import dt_humanize_delta, dt_ts
+from freqtrade.util.datetime_helpers import dt_humanize_delta, dt_ts, format_ms_time
 from freqtrade.util.periodic_cache import PeriodicCache
 
 
@@ -115,6 +130,7 @@ class Exchange:
         "tickers_have_quoteVolume": True,
         "tickers_have_bid_ask": True,  # bid / ask empty for fetch_tickers
         "tickers_have_price": True,
+        "trades_limit": 1000,  # Limit for 1 call to fetch_trades
         "trades_pagination": "time",  # Possible are "time" or "id"
         "trades_pagination_arg": "since",
         "trades_has_history": False,
@@ -130,6 +146,7 @@ class Exchange:
         "marketOrderRequiresPrice": False,
         "exchange_has_overrides": {},  # Dictionary overriding ccxt's "has".
         # Expected to be in the format {"fetchOHLCV": True} or {"fetchOHLCV": False}
+        "ws.enabled": False,  # Set to true for exchanges with tested websocket support
     }
     _ft_has: Dict = {}
     _ft_has_futures: Dict = {}
@@ -152,7 +169,9 @@ class Exchange:
         :return: None
         """
         self._api: ccxt.Exchange
-        self._api_async: ccxt_async.Exchange
+        self._api_async: ccxt_pro.Exchange
+        self._ws_async: ccxt_pro.Exchange = None
+        self._exchange_ws: Optional[ExchangeWS] = None
         self._markets: Dict = {}
         self._trading_fees: Dict[str, Any] = {}
         self._leverage_tiers: Dict[str, List[Dict]] = {}
@@ -183,6 +202,9 @@ class Exchange:
         self._klines: Dict[PairWithTimeframe, DataFrame] = {}
         self._expiring_candle_cache: Dict[Tuple[str, int], PeriodicCache] = {}
 
+        # Holds public_trades
+        self._trades: Dict[PairWithTimeframe, DataFrame] = {}
+
         # Holds all open sell orders for dry_run
         self._dry_run_open_orders: Dict[str, Any] = {}
 
@@ -211,6 +233,8 @@ class Exchange:
         # Assign this directly for easy access
         self._ohlcv_partial_candle = self._ft_has["ohlcv_partial_candle"]
 
+        self._max_trades_limit = self._ft_has["trades_limit"]
+
         self._trades_pagination = self._ft_has["trades_pagination"]
         self._trades_pagination_arg = self._ft_has["trades_pagination_arg"]
 
@@ -219,7 +243,7 @@ class Exchange:
         ccxt_config = deep_merge_dicts(exchange_conf.get("ccxt_config", {}), ccxt_config)
         ccxt_config = deep_merge_dicts(exchange_conf.get("ccxt_sync_config", {}), ccxt_config)
 
-        self._api = self._init_ccxt(exchange_conf, ccxt_kwargs=ccxt_config)
+        self._api = self._init_ccxt(exchange_conf, True, ccxt_config)
 
         ccxt_async_config = self._ccxt_config
         ccxt_async_config = deep_merge_dicts(
@@ -228,7 +252,15 @@ class Exchange:
         ccxt_async_config = deep_merge_dicts(
             exchange_conf.get("ccxt_async_config", {}), ccxt_async_config
         )
-        self._api_async = self._init_ccxt(exchange_conf, ccxt_async, ccxt_kwargs=ccxt_async_config)
+        self._api_async = self._init_ccxt(exchange_conf, False, ccxt_async_config)
+        self._has_watch_ohlcv = self.exchange_has("watchOHLCV") and self._ft_has["ws.enabled"]
+        if (
+            self._config["runmode"] in TRADE_MODES
+            and exchange_conf.get("enable_ws", True)
+            and self._has_watch_ohlcv
+        ):
+            self._ws_async = self._init_ccxt(exchange_conf, False, ccxt_async_config)
+            self._exchange_ws = ExchangeWS(self._config, self._ws_async)
 
         logger.info(f'Using Exchange "{self.name}"')
         self.required_candle_call_count = 1
@@ -257,6 +289,8 @@ class Exchange:
         self.close()
 
     def close(self):
+        if self._exchange_ws:
+            self._exchange_ws.cleanup()
         logger.debug("Exchange object destroyed, closing async loop")
         if (
             getattr(self, "_api_async", None)
@@ -265,6 +299,14 @@ class Exchange:
         ):
             logger.debug("Closing async ccxt session.")
             self.loop.run_until_complete(self._api_async.close())
+        if (
+            self._ws_async
+            and inspect.iscoroutinefunction(self._ws_async.close)
+            and self._ws_async.session
+        ):
+            logger.debug("Closing ws ccxt session.")
+            self.loop.run_until_complete(self._ws_async.close())
+
         if self.loop and not self.loop.is_closed():
             self.loop.close()
 
@@ -286,29 +328,38 @@ class Exchange:
         self.validate_trading_mode_and_margin_mode(self.trading_mode, self.margin_mode)
         self.validate_pricing(config["exit_pricing"])
         self.validate_pricing(config["entry_pricing"])
+        self.validate_orderflow(config["exchange"])
 
     def _init_ccxt(
-        self,
-        exchange_config: Dict[str, Any],
-        ccxt_module: CcxtModuleType = ccxt,
-        *,
-        ccxt_kwargs: Dict,
+        self, exchange_config: Dict[str, Any], sync: bool, ccxt_kwargs: Dict[str, Any]
     ) -> ccxt.Exchange:
         """
-        Initialize ccxt with given config and return valid
-        ccxt instance.
+        Initialize ccxt with given config and return valid ccxt instance.
         """
         # Find matching class for the given exchange name
         name = exchange_config["name"]
+        if sync:
+            ccxt_module = ccxt
+        else:
+            ccxt_module = ccxt_pro
+            if not is_exchange_known_ccxt(name, ccxt_module):
+                # Fall back to async if pro doesn't support this exchange
+                import ccxt.async_support as ccxt_async
+
+                ccxt_module = ccxt_async
 
         if not is_exchange_known_ccxt(name, ccxt_module):
             raise OperationalException(f"Exchange {name} is not supported by ccxt")
 
         ex_config = {
-            "apiKey": exchange_config.get("key"),
+            "apiKey": exchange_config.get("apiKey", exchange_config.get("key")),
             "secret": exchange_config.get("secret"),
             "password": exchange_config.get("password"),
             "uid": exchange_config.get("uid", ""),
+            "accountId": exchange_config.get("accountId", ""),
+            # DEX attributes:
+            "walletAddress": exchange_config.get("walletAddress"),
+            "privateKey": exchange_config.get("privateKey"),
         }
         if ccxt_kwargs:
             logger.info("Applying additional ccxt config: %s", ccxt_kwargs)
@@ -483,6 +534,15 @@ class Exchange:
         else:
             return DataFrame()
 
+    def trades(self, pair_interval: PairWithTimeframe, copy: bool = True) -> DataFrame:
+        if pair_interval in self._trades:
+            if copy:
+                return self._trades[pair_interval].copy()
+            else:
+                return self._trades[pair_interval]
+        else:
+            return DataFrame()
+
     def get_contract_size(self, pair: str) -> Optional[float]:
         if self.trading_mode == TradingMode.FUTURES:
             market = self.markets.get(pair, {})
@@ -531,6 +591,13 @@ class Exchange:
             amount, self.get_precision_amount(pair), self.precisionMode, contract_size
         )
 
+    def ws_connection_reset(self):
+        """
+        called at regular intervals to reset the websocket connection
+        """
+        if self._exchange_ws:
+            self._exchange_ws.reset_connections()
+
     def _load_async_markets(self, reload: bool = False) -> Dict[str, Any]:
         try:
             markets = self.loop.run_until_complete(
@@ -562,6 +629,12 @@ class Exchange:
             # Reload async markets, then assign them to sync api
             self._markets = self._load_async_markets(reload=True)
             self._api.set_markets(self._api_async.markets, self._api_async.currencies)
+            # Assign options array, as it contains some temporary information from the exchange.
+            self._api.options = self._api_async.options
+            if self._exchange_ws:
+                # Set markets to avoid reloading on websocket api
+                self._ws_async.set_markets(self._api.markets, self._api.currencies)
+                self._ws_async.options = self._api.options
             self._last_markets_refresh = dt_ts()
 
             if is_initial and self._ft_has["needs_trading_fees"]:
@@ -723,6 +796,14 @@ class Exchange:
                 f"Time in force policies are not supported for {self.name} yet."
             )
 
+    def validate_orderflow(self, exchange: Dict) -> None:
+        if exchange.get("use_public_trades", False) and (
+            not self.exchange_has("fetchTrades") or not self._ft_has["trades_has_history"]
+        ):
+            raise ConfigurationError(
+                f"Trade data not available for {self.name}. Can't use orderflow feature."
+            )
+
     def validate_required_startup_candles(self, startup_candles: int, timeframe: str) -> int:
         """
         Checks if required startup_candles is more than ohlcv_candle_limit().
@@ -795,7 +876,7 @@ class Exchange:
         """
         if endpoint in self._ft_has.get("exchange_has_overrides", {}):
             return self._ft_has["exchange_has_overrides"][endpoint]
-        return endpoint in self._api.has and self._api.has[endpoint]
+        return endpoint in self._api_async.has and self._api_async.has[endpoint]
 
     def get_precision_amount(self, pair: str) -> Optional[float]:
         """
@@ -2019,7 +2100,7 @@ class Exchange:
     def get_fee(
         self,
         symbol: str,
-        type: str = "",
+        order_type: str = "",
         side: str = "",
         amount: float = 1,
         price: float = 1,
@@ -2028,13 +2109,13 @@ class Exchange:
         """
         Retrieve fee from exchange
         :param symbol: Pair
-        :param type: Type of order (market, limit, ...)
+        :param order_type: Type of order (market, limit, ...)
         :param side: Side of order (buy, sell)
         :param amount: Amount of order
         :param price: Price of order
         :param taker_or_maker: 'maker' or 'taker' (ignored if "type" is provided)
         """
-        if type and type == "market":
+        if order_type and order_type == "market":
             taker_or_maker = "taker"
         try:
             if self._config["dry_run"] and self._config.get("fee", None) is not None:
@@ -2045,7 +2126,7 @@ class Exchange:
 
             return self._api.calculate_fee(
                 symbol=symbol,
-                type=type,
+                type=order_type,
                 side=side,
                 amount=amount,
                 price=price,
@@ -2228,9 +2309,40 @@ class Exchange:
         cache: bool,
     ) -> Coroutine[Any, Any, OHLCVResponse]:
         not_all_data = cache and self.required_candle_call_count > 1
+        if cache and candle_type in (CandleType.SPOT, CandleType.FUTURES):
+            if self._has_watch_ohlcv and self._exchange_ws:
+                # Subscribe to websocket
+                self._exchange_ws.schedule_ohlcv(pair, timeframe, candle_type)
+
         if cache and (pair, timeframe, candle_type) in self._klines:
             candle_limit = self.ohlcv_candle_limit(timeframe, candle_type)
-            min_date = date_minus_candles(timeframe, candle_limit - 5).timestamp()
+            min_date = int(date_minus_candles(timeframe, candle_limit - 5).timestamp())
+
+            if self._exchange_ws:
+                candle_date = int(timeframe_to_prev_date(timeframe).timestamp() * 1000)
+                prev_candle_date = int(date_minus_candles(timeframe, 1).timestamp() * 1000)
+                candles = self._exchange_ws.ccxt_object.ohlcvs.get(pair, {}).get(timeframe)
+                half_candle = int(candle_date - (candle_date - prev_candle_date) * 0.5)
+                last_refresh_time = int(
+                    self._exchange_ws.klines_last_refresh.get((pair, timeframe, candle_type), 0)
+                )
+
+                if (
+                    candles
+                    and candles[-1][0] >= prev_candle_date
+                    and last_refresh_time >= half_candle
+                ):
+                    # Usable result, candle contains the previous candle.
+                    # Also, we check if the last refresh time is no more than half the candle ago.
+                    logger.debug(f"reuse watch result for {pair}, {timeframe}, {last_refresh_time}")
+
+                    return self._exchange_ws.get_ohlcv(pair, timeframe, candle_type, candle_date)
+                logger.info(
+                    f"Failed to reuse watch {pair}, {timeframe}, {candle_date < last_refresh_time},"
+                    f" {candle_date}, {last_refresh_time}, "
+                    f"{format_ms_time(candle_date)}, {format_ms_time(last_refresh_time)} "
+                )
+
             # Check if 1 call can get us updated candles without hole in the data.
             if min_date < self._pairs_last_refresh_time.get((pair, timeframe, candle_type), 0):
                 # Cache can be used - do one-off call.
@@ -2263,7 +2375,7 @@ class Exchange:
 
     def _build_ohlcv_dl_jobs(
         self, pair_list: ListPairsWithTimeframes, since_ms: Optional[int], cache: bool
-    ) -> Tuple[List[Coroutine], List[Tuple[str, str, CandleType]]]:
+    ) -> Tuple[List[Coroutine], List[PairWithTimeframe]]:
         """
         Build Coroutines to execute as part of refresh_latest_ohlcv
         """
@@ -2519,6 +2631,171 @@ class Exchange:
         data = [[x["timestamp"], x["fundingRate"], 0, 0, 0, 0] for x in data]
         return data
 
+    # fetch Trade data stuff
+
+    def needed_candle_for_trades_ms(self, timeframe: str, candle_type: CandleType) -> int:
+        candle_limit = self.ohlcv_candle_limit(timeframe, candle_type)
+        tf_s = timeframe_to_seconds(timeframe)
+        candles_fetched = candle_limit * self.required_candle_call_count
+
+        max_candles = self._config["orderflow"]["max_candles"]
+
+        required_candles = min(max_candles, candles_fetched)
+        move_to = (
+            tf_s * candle_limit * required_candles
+            if required_candles > candle_limit
+            else (max_candles + 1) * tf_s
+        )
+
+        now = timeframe_to_next_date(timeframe)
+        return int((now - timedelta(seconds=move_to)).timestamp() * 1000)
+
+    def _process_trades_df(
+        self,
+        pair: str,
+        timeframe: str,
+        c_type: CandleType,
+        ticks: List[List],
+        cache: bool,
+        first_required_candle_date: int,
+    ) -> DataFrame:
+        # keeping parsed dataframe in cache
+        trades_df = trades_list_to_df(ticks, True)
+
+        if cache:
+            if (pair, timeframe, c_type) in self._trades:
+                old = self._trades[(pair, timeframe, c_type)]
+                # Reassign so we return the updated, combined df
+                combined_df = concat([old, trades_df], axis=0)
+                logger.debug(f"Clean duplicated ticks from Trades data {pair}")
+                trades_df = DataFrame(
+                    trades_df_remove_duplicates(combined_df), columns=combined_df.columns
+                )
+                # Age out old candles
+                trades_df = trades_df[first_required_candle_date < trades_df["timestamp"]]
+                trades_df = trades_df.reset_index(drop=True)
+            self._trades[(pair, timeframe, c_type)] = trades_df
+        return trades_df
+
+    def refresh_latest_trades(
+        self,
+        pair_list: ListPairsWithTimeframes,
+        *,
+        cache: bool = True,
+    ) -> Dict[PairWithTimeframe, DataFrame]:
+        """
+        Refresh in-memory TRADES asynchronously and set `_trades` with the result
+        Loops asynchronously over pair_list and downloads all pairs async (semi-parallel).
+        Only used in the dataprovider.refresh() method.
+        :param pair_list: List of 3 element tuples containing (pair, timeframe, candle_type)
+        :param cache: Assign result to _trades. Useful for one-off downloads like for pairlists
+        :return: Dict of [{(pair, timeframe): Dataframe}]
+        """
+        from freqtrade.data.history import get_datahandler
+
+        data_handler = get_datahandler(
+            self._config["datadir"], data_format=self._config["dataformat_trades"]
+        )
+        logger.debug("Refreshing TRADES data for %d pairs", len(pair_list))
+        since_ms = None
+        results_df = {}
+        for pair, timeframe, candle_type in set(pair_list):
+            new_ticks: List = []
+            all_stored_ticks_df = DataFrame(columns=DEFAULT_TRADES_COLUMNS + ["date"])
+            first_candle_ms = self.needed_candle_for_trades_ms(timeframe, candle_type)
+            # refresh, if
+            # a. not in _trades
+            # b. no cache used
+            # c. need new data
+            is_in_cache = (pair, timeframe, candle_type) in self._trades
+            if (
+                not is_in_cache
+                or not cache
+                or self._now_is_time_to_refresh_trades(pair, timeframe, candle_type)
+            ):
+                logger.debug(f"Refreshing TRADES data for {pair}")
+                # fetch trades since latest _trades and
+                # store together with existing trades
+                try:
+                    until = None
+                    from_id = None
+                    if is_in_cache:
+                        from_id = self._trades[(pair, timeframe, candle_type)].iloc[-1]["id"]
+                        until = dt_ts()  # now
+
+                    else:
+                        until = int(timeframe_to_prev_date(timeframe).timestamp()) * 1000
+                        all_stored_ticks_df = data_handler.trades_load(
+                            f"{pair}-cached", self.trading_mode
+                        )
+
+                        if not all_stored_ticks_df.empty:
+                            if (
+                                all_stored_ticks_df.iloc[-1]["timestamp"] > first_candle_ms
+                                and all_stored_ticks_df.iloc[0]["timestamp"] <= first_candle_ms
+                            ):
+                                # Use cache and populate further
+                                last_cached_ms = all_stored_ticks_df.iloc[-1]["timestamp"]
+                                from_id = all_stored_ticks_df.iloc[-1]["id"]
+                                # only use cached if it's closer than first_candle_ms
+                                since_ms = (
+                                    last_cached_ms
+                                    if last_cached_ms > first_candle_ms
+                                    else first_candle_ms
+                                )
+                            else:
+                                # Skip cache, it's too old
+                                all_stored_ticks_df = DataFrame(
+                                    columns=DEFAULT_TRADES_COLUMNS + ["date"]
+                                )
+
+                    # from_id overrules with exchange set to id paginate
+                    [_, new_ticks] = self.get_historic_trades(
+                        pair,
+                        since=since_ms if since_ms else first_candle_ms,
+                        until=until,
+                        from_id=from_id,
+                    )
+
+                except Exception:
+                    logger.exception(f"Refreshing TRADES data for {pair} failed")
+                    continue
+
+                if new_ticks:
+                    all_stored_ticks_list = all_stored_ticks_df[
+                        DEFAULT_TRADES_COLUMNS
+                    ].values.tolist()
+                    all_stored_ticks_list.extend(new_ticks)
+                    trades_df = self._process_trades_df(
+                        pair,
+                        timeframe,
+                        candle_type,
+                        all_stored_ticks_list,
+                        cache,
+                        first_required_candle_date=first_candle_ms,
+                    )
+                    results_df[(pair, timeframe, candle_type)] = trades_df
+                    data_handler.trades_store(
+                        f"{pair}-cached", trades_df[DEFAULT_TRADES_COLUMNS], self.trading_mode
+                    )
+
+                else:
+                    logger.error(f"No new ticks for {pair}")
+
+        return results_df
+
+    def _now_is_time_to_refresh_trades(
+        self, pair: str, timeframe: str, candle_type: CandleType
+    ) -> bool:  # Timeframe in seconds
+        trades = self.trades((pair, timeframe, candle_type), False)
+        pair_last_refreshed = int(trades.iloc[-1]["timestamp"])
+        full_candle = (
+            int(timeframe_to_next_date(timeframe, dt_from_ts(pair_last_refreshed)).timestamp())
+            * 1000
+        )
+        now = dt_ts()
+        return full_candle <= now
+
     # Fetch historic trades
 
     @retrier_async
@@ -2533,10 +2810,11 @@ class Exchange:
         returns: List of dicts containing trades, the next iteration value (new "since" or trade_id)
         """
         try:
+            trades_limit = self._max_trades_limit
             # fetch trades asynchronously
             if params:
                 logger.debug("Fetching trades for pair %s, params: %s ", pair, params)
-                trades = await self._api_async.fetch_trades(pair, params=params, limit=1000)
+                trades = await self._api_async.fetch_trades(pair, params=params, limit=trades_limit)
             else:
                 logger.debug(
                     "Fetching trades for pair %s, since %s %s...",
@@ -2544,7 +2822,7 @@ class Exchange:
                     since,
                     "(" + dt_from_ts(since).isoformat() + ") " if since is not None else "",
                 )
-                trades = await self._api_async.fetch_trades(pair, since=since, limit=1000)
+                trades = await self._api_async.fetch_trades(pair, since=since, limit=trades_limit)
             trades = self._trades_contracts_to_amount(trades)
             pagination_value = self._get_trade_pagination_next_value(trades)
             return trades_dict_to_list(trades), pagination_value
@@ -3339,13 +3617,12 @@ class Exchange:
     def get_maintenance_ratio_and_amt(
         self,
         pair: str,
-        nominal_value: float,
+        notional_value: float,
     ) -> Tuple[float, Optional[float]]:
         """
         Important: Must be fetching data from cached values as this is used by backtesting!
         :param pair: Market symbol
-        :param nominal_value: The total trade amount in quote currency including leverage
-        maintenance amount only on Binance
+        :param notional_value: The total trade amount in quote currency
         :return: (maintenance margin ratio, maintenance amount)
         """
 
@@ -3362,7 +3639,7 @@ class Exchange:
             pair_tiers = self._leverage_tiers[pair]
 
             for tier in reversed(pair_tiers):
-                if nominal_value >= tier["minNotional"]:
+                if notional_value >= tier["minNotional"]:
                     return (tier["maintenanceMarginRate"], tier["maintAmt"])
 
             raise ExchangeError("nominal value can not be lower than 0")
@@ -3370,4 +3647,3 @@ class Exchange:
             # describes the min amt for a tier, and the lowest tier will always go down to 0
         else:
             raise ExchangeError(f"Cannot get maintenance ratio using {self.name}")
-            raise ExchangeError(f"Cannot get maintenance ratio using {self.name}")

freqtrade/exchange/exchange_utils.py

@@ -53,16 +53,22 @@ def available_exchanges(ccxt_module: Optional[CcxtModuleType] = None) -> List[st
     return [x for x in exchanges if validate_exchange(x)[0]]
 
 
-def validate_exchange(exchange: str) -> Tuple[bool, str]:
+def validate_exchange(exchange: str) -> Tuple[bool, str, bool]:
     """
     returns: can_use, reason
         with Reason including both missing and missing_opt
     """
-    ex_mod = getattr(ccxt, exchange.lower())()
+    try:
+        ex_mod = getattr(ccxt.pro, exchange.lower())()
+    except AttributeError:
+        ex_mod = getattr(ccxt.async_support, exchange.lower())()
+
+    if not ex_mod or not ex_mod.has:
+        return False, "", False
+
     result = True
     reason = ""
-    if not ex_mod or not ex_mod.has:
-        return False, ""
+    is_dex = getattr(ex_mod, "dex", False)
     missing = [
         k
         for k, v in EXCHANGE_HAS_REQUIRED.items()
@@ -81,18 +87,19 @@ def validate_exchange(exchange: str) -> Tuple[bool, str]:
     if missing_opt:
         reason += f"{'. ' if reason else ''}missing opt: {', '.join(missing_opt)}. "
 
-    return result, reason
+    return result, reason, is_dex
 
 
 def _build_exchange_list_entry(
     exchange_name: str, exchangeClasses: Dict[str, Any]
 ) -> ValidExchangesType:
-    valid, comment = validate_exchange(exchange_name)
+    valid, comment, is_dex = validate_exchange(exchange_name)
     result: ValidExchangesType = {
         "name": exchange_name,
         "valid": valid,
         "supported": exchange_name.lower() in SUPPORTED_EXCHANGES,
         "comment": comment,
+        "dex": is_dex,
         "trade_modes": [{"trading_mode": "spot", "margin_mode": ""}],
     }
     if resolved := exchangeClasses.get(exchange_name.lower()):

freqtrade/exchange/exchange_ws.py

@@ -0,0 +1,195 @@
+import asyncio
+import logging
+import time
+from copy import deepcopy
+from functools import partial
+from threading import Thread
+from typing import Dict, Set
+
+import ccxt
+
+from freqtrade.constants import Config, PairWithTimeframe
+from freqtrade.enums.candletype import CandleType
+from freqtrade.exchange.exchange import timeframe_to_seconds
+from freqtrade.exchange.types import OHLCVResponse
+from freqtrade.util import dt_ts, format_ms_time
+
+
+logger = logging.getLogger(__name__)
+
+
+class ExchangeWS:
+    def __init__(self, config: Config, ccxt_object: ccxt.Exchange) -> None:
+        self.config = config
+        self.ccxt_object = ccxt_object
+        self._background_tasks: Set[asyncio.Task] = set()
+
+        self._klines_watching: Set[PairWithTimeframe] = set()
+        self._klines_scheduled: Set[PairWithTimeframe] = set()
+        self.klines_last_refresh: Dict[PairWithTimeframe, float] = {}
+        self.klines_last_request: Dict[PairWithTimeframe, float] = {}
+        self._thread = Thread(name="ccxt_ws", target=self._start_forever)
+        self._thread.start()
+        self.__cleanup_called = False
+
+    def _start_forever(self) -> None:
+        self._loop = asyncio.new_event_loop()
+        try:
+            self._loop.run_forever()
+        finally:
+            if self._loop.is_running():
+                self._loop.stop()
+
+    def cleanup(self) -> None:
+        logger.debug("Cleanup called - stopping")
+        self._klines_watching.clear()
+        for task in self._background_tasks:
+            task.cancel()
+        if hasattr(self, "_loop") and not self._loop.is_closed():
+            self.reset_connections()
+
+            self._loop.call_soon_threadsafe(self._loop.stop)
+            time.sleep(0.1)
+            if not self._loop.is_closed():
+                self._loop.close()
+
+        self._thread.join()
+        logger.debug("Stopped")
+
+    def reset_connections(self) -> None:
+        """
+        Reset all connections - avoids "connection-reset" errors that happen after ~9 days
+        """
+        if hasattr(self, "_loop") and not self._loop.is_closed():
+            logger.info("Resetting WS connections.")
+            asyncio.run_coroutine_threadsafe(self._cleanup_async(), loop=self._loop)
+            while not self.__cleanup_called:
+                time.sleep(0.1)
+        self.__cleanup_called = False
+
+    async def _cleanup_async(self) -> None:
+        try:
+            await self.ccxt_object.close()
+            # Clear the cache.
+            # Not doing this will cause problems on startup with dynamic pairlists
+            self.ccxt_object.ohlcvs.clear()
+        except Exception:
+            logger.exception("Exception in _cleanup_async")
+        finally:
+            self.__cleanup_called = True
+
+    def _pop_history(self, paircomb: PairWithTimeframe) -> None:
+        """
+        Remove history for a pair/timeframe combination from ccxt cache
+        """
+        self.ccxt_object.ohlcvs.get(paircomb[0], {}).pop(paircomb[1], None)
+
+    def cleanup_expired(self) -> None:
+        """
+        Remove pairs from watchlist if they've not been requested within
+        the last timeframe (+ offset)
+        """
+        changed = False
+        for p in list(self._klines_watching):
+            _, timeframe, _ = p
+            timeframe_s = timeframe_to_seconds(timeframe)
+            last_refresh = self.klines_last_request.get(p, 0)
+            if last_refresh > 0 and (dt_ts() - last_refresh) > ((timeframe_s + 20) * 1000):
+                logger.info(f"Removing {p} from websocket watchlist.")
+                self._klines_watching.discard(p)
+                # Pop history to avoid getting stale data
+                self._pop_history(p)
+                changed = True
+        if changed:
+            logger.info(f"Removal done: new watch list ({len(self._klines_watching)})")
+
+    async def _schedule_while_true(self) -> None:
+        # For the ones we should be watching
+        for p in self._klines_watching:
+            # Check if they're already scheduled
+            if p not in self._klines_scheduled:
+                self._klines_scheduled.add(p)
+                pair, timeframe, candle_type = p
+                task = asyncio.create_task(
+                    self._continuously_async_watch_ohlcv(pair, timeframe, candle_type)
+                )
+                self._background_tasks.add(task)
+                task.add_done_callback(
+                    partial(
+                        self._continuous_stopped,
+                        pair=pair,
+                        timeframe=timeframe,
+                        candle_type=candle_type,
+                    )
+                )
+
+    def _continuous_stopped(
+        self, task: asyncio.Task, pair: str, timeframe: str, candle_type: CandleType
+    ):
+        self._background_tasks.discard(task)
+        result = "done"
+        if task.cancelled():
+            result = "cancelled"
+        else:
+            if (result1 := task.result()) is not None:
+                result = str(result1)
+
+        logger.info(f"{pair}, {timeframe}, {candle_type} - Task finished - {result}")
+        self._klines_scheduled.discard((pair, timeframe, candle_type))
+        self._pop_history((pair, timeframe, candle_type))
+
+    async def _continuously_async_watch_ohlcv(
+        self, pair: str, timeframe: str, candle_type: CandleType
+    ) -> None:
+        try:
+            while (pair, timeframe, candle_type) in self._klines_watching:
+                start = dt_ts()
+                data = await self.ccxt_object.watch_ohlcv(pair, timeframe)
+                self.klines_last_refresh[(pair, timeframe, candle_type)] = dt_ts()
+                logger.debug(
+                    f"watch done {pair}, {timeframe}, data {len(data)} "
+                    f"in {dt_ts() - start:.2f}s"
+                )
+        except ccxt.ExchangeClosedByUser:
+            logger.debug("Exchange connection closed by user")
+        except ccxt.BaseError:
+            logger.exception(f"Exception in continuously_async_watch_ohlcv for {pair}, {timeframe}")
+        finally:
+            self._klines_watching.discard((pair, timeframe, candle_type))
+
+    def schedule_ohlcv(self, pair: str, timeframe: str, candle_type: CandleType) -> None:
+        """
+        Schedule a pair/timeframe combination to be watched
+        """
+        self._klines_watching.add((pair, timeframe, candle_type))
+        self.klines_last_request[(pair, timeframe, candle_type)] = dt_ts()
+        # asyncio.run_coroutine_threadsafe(self.schedule_schedule(), loop=self._loop)
+        asyncio.run_coroutine_threadsafe(self._schedule_while_true(), loop=self._loop)
+        self.cleanup_expired()
+
+    async def get_ohlcv(
+        self,
+        pair: str,
+        timeframe: str,
+        candle_type: CandleType,
+        candle_date: int,
+    ) -> OHLCVResponse:
+        """
+        Returns cached klines from ccxt's "watch" cache.
+        :param candle_date: timestamp of the end-time of the candle.
+        """
+        # Deepcopy the response - as it might be modified in the background as new messages arrive
+        candles = deepcopy(self.ccxt_object.ohlcvs.get(pair, {}).get(timeframe))
+        refresh_date = self.klines_last_refresh[(pair, timeframe, candle_type)]
+        drop_hint = False
+        if refresh_date > candle_date:
+            # Refreshed after candle was complete.
+            # logger.info(f"{candles[-1][0]} >= {candle_date}")
+            drop_hint = candles[-1][0] >= candle_date
+        logger.debug(
+            f"watch result for {pair}, {timeframe} with length {len(candles)}, "
+            f"{format_ms_time(candles[-1][0])}, "
+            f"lref={format_ms_time(refresh_date)}, "
+            f"candle_date={format_ms_time(candle_date)}, {drop_hint=}"
+        )
+        return pair, timeframe, candle_type, candles, drop_hint

freqtrade/exchange/hyperliquid.py

@@ -0,0 +1,24 @@
+"""Hyperliquid exchange subclass"""
+
+import logging
+from typing import Dict
+
+from freqtrade.exchange import Exchange
+
+
+logger = logging.getLogger(__name__)
+
+
+class Hyperliquid(Exchange):
+    """Hyperliquid exchange class.
+    Contains adjustments needed for Freqtrade to work with this exchange.
+    """
+
+    _ft_has: Dict = {
+        # Only the most recent 5000 candles are available according to the
+        # exchange's API documentation.
+        "ohlcv_has_history": True,
+        "ohlcv_candle_limit": 5000,
+        "trades_has_history": False,  # Trades endpoint doesn't seem available.
+        "exchange_has_overrides": {"fetchTrades": False},
+    }

freqtrade/freqai/RL/BaseEnvironment.py

@@ -52,7 +52,7 @@ class BaseEnvironment(gym.Env):
         reward_kwargs: dict = {},
         window_size=10,
         starting_point=True,
-        id: str = "baseenv-1",
+        id: str = "baseenv-1",  # noqa: A002
         seed: int = 1,
         config: dict = {},
         live: bool = False,

freqtrade/freqai/data_drawer.py

@@ -238,9 +238,9 @@ class FreqaiDataDrawer:
                     metadata, fp, default=self.np_encoder, number_mode=rapidjson.NM_NATIVE
                 )
 
-    def np_encoder(self, object):
-        if isinstance(object, np.generic):
-            return object.item()
+    def np_encoder(self, obj):
+        if isinstance(obj, np.generic):
+            return obj.item()
 
     def get_pair_dict_info(self, pair: str) -> Tuple[str, int]:
         """
@@ -448,8 +448,8 @@ class FreqaiDataDrawer:
 
         delete_dict: Dict[str, Any] = {}
 
-        for dir in model_folders:
-            result = pattern.match(str(dir.name))
+        for directory in model_folders:
+            result = pattern.match(str(directory.name))
             if result is None:
                 continue
             coin = result.group(1)
@@ -458,10 +458,10 @@ class FreqaiDataDrawer:
             if coin not in delete_dict:
                 delete_dict[coin] = {}
                 delete_dict[coin]["num_folders"] = 1
-                delete_dict[coin]["timestamps"] = {int(timestamp): dir}
+                delete_dict[coin]["timestamps"] = {int(timestamp): directory}
             else:
                 delete_dict[coin]["num_folders"] += 1
-                delete_dict[coin]["timestamps"][int(timestamp)] = dir
+                delete_dict[coin]["timestamps"][int(timestamp)] = directory
 
         for coin in delete_dict:
             if delete_dict[coin]["num_folders"] > num_keep:
@@ -612,9 +612,9 @@ class FreqaiDataDrawer:
         elif self.model_type == "pytorch":
             import torch
 
-            zip = torch.load(dk.data_path / f"{dk.model_filename}_model.zip")
-            model = zip["pytrainer"]
-            model = model.load_from_checkpoint(zip)
+            zipfile = torch.load(dk.data_path / f"{dk.model_filename}_model.zip")
+            model = zipfile["pytrainer"]
+            model = model.load_from_checkpoint(zipfile)
 
         if not model:
             raise OperationalException(

freqtrade/freqai/tensorboard/tensorboard.py

@@ -45,10 +45,10 @@ class TensorBoardCallback(BaseTensorBoardCallback):
             return False
 
         evals = ["validation", "train"]
-        for metric, eval in zip(evals_log.items(), evals):
+        for metric, eval_ in zip(evals_log.items(), evals):
             for metric_name, log in metric[1].items():
                 score = log[-1][0] if isinstance(log[-1], tuple) else log[-1]
-                self.writer.add_scalar(f"{eval}-{metric_name}", score, epoch)
+                self.writer.add_scalar(f"{eval_}-{metric_name}", score, epoch)
 
         return False
 

freqtrade/freqtradebot.py

@@ -168,6 +168,8 @@ class FreqtradeBot(LoggingMixin):
                     t = str(time(time_slot, minutes, 2))
                     self._schedule.every().day.at(t).do(update)
 
+        self._schedule.every().day.at("00:02").do(self.exchange.ws_connection_reset)
+
         self.strategy.ft_bot_start()
         # Initialize protections AFTER bot start - otherwise parameters are not loaded.
         self.protections = ProtectionManager(self.config, self.strategy.protections)
@@ -289,8 +291,7 @@ class FreqtradeBot(LoggingMixin):
         # Then looking for entry opportunities
         if self.get_free_open_trades():
             self.enter_positions()
-        if self.trading_mode == TradingMode.FUTURES:
-            self._schedule.run_pending()
+        self._schedule.run_pending()
         Trade.commit()
         self.rpc.process_msg_queue(self.dataprovider._msg_queue)
         self.last_process = datetime.now(timezone.utc)
@@ -2369,6 +2370,18 @@ class FreqtradeBot(LoggingMixin):
             trade, order, order_obj, order_amount, order.get("trades", [])
         )
 
+    def _trades_valid_for_fee(self, trades: List[Dict[str, Any]]) -> bool:
+        """
+        Check if trades are valid for fee detection.
+        :return: True if trades are valid for fee detection, False otherwise
+        """
+        if not trades:
+            return False
+        # We expect amount and cost to be present in all trade objects.
+        if any(trade.get("amount") is None or trade.get("cost") is None for trade in trades):
+            return False
+        return True
+
     def fee_detection_from_trades(
         self, trade: Trade, order: Dict, order_obj: Order, order_amount: float, trades: List
     ) -> Optional[float]:
@@ -2376,7 +2389,7 @@ class FreqtradeBot(LoggingMixin):
         fee-detection fallback to Trades.
         Either uses provided trades list or the result of fetch_my_trades to get correct fee.
         """
-        if not trades:
+        if not self._trades_valid_for_fee(trades):
             trades = self.exchange.get_trades_for_order(
                 self.exchange.get_order_id_conditional(order), trade.pair, order_obj.order_date
             )

freqtrade/misc.py

@@ -33,7 +33,7 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool =
         if log:
             logger.info(f'dumping json to "{filename}"')
 
-        with gzip.open(filename, "w") as fpz:
+        with gzip.open(filename, "wt", encoding="utf-8") as fpz:
             rapidjson.dump(data, fpz, default=str, number_mode=rapidjson.NM_NATIVE)
     else:
         if log:
@@ -60,7 +60,7 @@ def file_dump_joblib(filename: Path, data: Any, log: bool = True) -> None:
     logger.debug(f'done joblib dump to "{filename}"')
 
 
-def json_load(datafile: Union[gzip.GzipFile, TextIO]) -> Any:
+def json_load(datafile: TextIO) -> Any:
     """
     load data with rapidjson
     Use this to have a consistent experience,
@@ -77,7 +77,7 @@ def file_load_json(file: Path):
     # Try gzip file first, otherwise regular json file.
     if gzipfile.is_file():
         logger.debug(f"Loading historical data from file {gzipfile}")
-        with gzip.open(gzipfile) as datafile:
+        with gzip.open(gzipfile, "rt", encoding="utf-8") as datafile:
             pairdata = json_load(datafile)
     elif file.is_file():
         logger.debug(f"Loading historical data from file {file}")

freqtrade/optimize/analysis/lookahead_helpers.py

@@ -4,11 +4,13 @@ from pathlib import Path
 from typing import Any, Dict, List
 
 import pandas as pd
+from rich.text import Text
 
 from freqtrade.constants import Config
 from freqtrade.exceptions import OperationalException
 from freqtrade.optimize.analysis.lookahead import LookaheadAnalysis
 from freqtrade.resolvers import StrategyResolver
+from freqtrade.util import print_rich_table
 
 
 logger = logging.getLogger(__name__)
@@ -53,18 +55,18 @@ class LookaheadAnalysisSubFunctions:
                     [
                         inst.strategy_obj["location"].parts[-1],
                         inst.strategy_obj["name"],
-                        inst.current_analysis.has_bias,
+                        Text("Yes", style="bold red")
+                        if inst.current_analysis.has_bias
+                        else Text("No", style="bold green"),
                         inst.current_analysis.total_signals,
                         inst.current_analysis.false_entry_signals,
                         inst.current_analysis.false_exit_signals,
                         ", ".join(inst.current_analysis.false_indicators),
                     ]
                 )
-        from tabulate import tabulate
 
-        table = tabulate(data, headers=headers, tablefmt="orgtbl")
-        print(table)
-        return table, headers, data
+        print_rich_table(data, headers, summary="Lookahead Analysis")
+        return data
 
     @staticmethod
     def export_to_csv(config: Dict[str, Any], lookahead_analysis: List[LookaheadAnalysis]):

freqtrade/optimize/analysis/recursive_helpers.py

@@ -7,6 +7,7 @@ from freqtrade.constants import Config
 from freqtrade.exceptions import OperationalException
 from freqtrade.optimize.analysis.recursive import RecursiveAnalysis
 from freqtrade.resolvers import StrategyResolver
+from freqtrade.util import print_rich_table
 
 
 logger = logging.getLogger(__name__)
@@ -16,9 +17,9 @@ class RecursiveAnalysisSubFunctions:
     @staticmethod
     def text_table_recursive_analysis_instances(recursive_instances: List[RecursiveAnalysis]):
         startups = recursive_instances[0]._startup_candle
-        headers = ["indicators"]
+        headers = ["Indicators"]
         for candle in startups:
-            headers.append(candle)
+            headers.append(str(candle))
 
         data = []
         for inst in recursive_instances:
@@ -30,13 +31,11 @@ class RecursiveAnalysisSubFunctions:
                     data.append(temp_data)
 
         if len(data) > 0:
-            from tabulate import tabulate
+            print_rich_table(data, headers, summary="Recursive Analysis")
 
-            table = tabulate(data, headers=headers, tablefmt="orgtbl")
-            print(table)
-            return table, headers, data
+            return data
 
-        return None, None, data
+        return data
 
     @staticmethod
     def calculate_config_overrides(config: Config):

freqtrade/optimize/backtesting.py

@@ -217,8 +217,6 @@ class Backtesting:
             raise OperationalException(
                 "VolumePairList not allowed for backtesting. Please use StaticPairList instead."
             )
-        if "PerformanceFilter" in self.pairlists.name_list:
-            raise OperationalException("PerformanceFilter not allowed for backtesting.")
 
         if len(self.strategylist) > 1 and "PrecisionFilter" in self.pairlists.name_list:
             raise OperationalException(
@@ -467,25 +465,25 @@ class Backtesting:
         return data
 
     def _get_close_rate(
-        self, row: Tuple, trade: LocalTrade, exit: ExitCheckTuple, trade_dur: int
+        self, row: Tuple, trade: LocalTrade, exit_: ExitCheckTuple, trade_dur: int
     ) -> float:
         """
         Get close rate for backtesting result
         """
         # Special handling if high or low hit STOP_LOSS or ROI
-        if exit.exit_type in (
+        if exit_.exit_type in (
             ExitType.STOP_LOSS,
             ExitType.TRAILING_STOP_LOSS,
             ExitType.LIQUIDATION,
         ):
-            return self._get_close_rate_for_stoploss(row, trade, exit, trade_dur)
-        elif exit.exit_type == (ExitType.ROI):
-            return self._get_close_rate_for_roi(row, trade, exit, trade_dur)
+            return self._get_close_rate_for_stoploss(row, trade, exit_, trade_dur)
+        elif exit_.exit_type == (ExitType.ROI):
+            return self._get_close_rate_for_roi(row, trade, exit_, trade_dur)
         else:
             return row[OPEN_IDX]
 
     def _get_close_rate_for_stoploss(
-        self, row: Tuple, trade: LocalTrade, exit: ExitCheckTuple, trade_dur: int
+        self, row: Tuple, trade: LocalTrade, exit_: ExitCheckTuple, trade_dur: int
     ) -> float:
         # our stoploss was already lower than candle high,
         # possibly due to a cancelled trade exit.
@@ -493,7 +491,7 @@ class Backtesting:
         is_short = trade.is_short or False
         leverage = trade.leverage or 1.0
         side_1 = -1 if is_short else 1
-        if exit.exit_type == ExitType.LIQUIDATION and trade.liquidation_price:
+        if exit_.exit_type == ExitType.LIQUIDATION and trade.liquidation_price:
             stoploss_value = trade.liquidation_price
         else:
             stoploss_value = trade.stop_loss
@@ -508,7 +506,7 @@ class Backtesting:
         # Special case: trailing triggers within same candle as trade opened. Assume most
         # pessimistic price movement, which is moving just enough to arm stoploss and
         # immediately going down to stop price.
-        if exit.exit_type == ExitType.TRAILING_STOP_LOSS and trade_dur == 0:
+        if exit_.exit_type == ExitType.TRAILING_STOP_LOSS and trade_dur == 0:
             if (
                 not self.strategy.use_custom_stoploss
                 and self.strategy.trailing_stop
@@ -539,7 +537,7 @@ class Backtesting:
         return stoploss_value
 
     def _get_close_rate_for_roi(
-        self, row: Tuple, trade: LocalTrade, exit: ExitCheckTuple, trade_dur: int
+        self, row: Tuple, trade: LocalTrade, exit_: ExitCheckTuple, trade_dur: int
     ) -> float:
         is_short = trade.is_short or False
         leverage = trade.leverage or 1.0

freqtrade/optimize/edge_cli.py

@@ -52,4 +52,4 @@ class EdgeCli:
         result = self.edge.calculate(self.config["exchange"]["pair_whitelist"])
         if result:
             print("")  # blank line for readability
-            print(generate_edge_table(self.edge._cached_pairs))
+            generate_edge_table(self.edge._cached_pairs)

freqtrade/optimize/hyperopt.py

@@ -14,19 +14,11 @@ from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
 import rapidjson
-from colorama import init as colorama_init
 from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects
 from joblib.externals import cloudpickle
 from pandas import DataFrame
-from rich.progress import (
-    BarColumn,
-    MofNCompleteColumn,
-    Progress,
-    TaskProgressColumn,
-    TextColumn,
-    TimeElapsedColumn,
-    TimeRemainingColumn,
-)
+from rich.align import Align
+from rich.console import Console
 
 from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN, Config
 from freqtrade.data.converter import trim_dataframes
@@ -40,6 +32,7 @@ from freqtrade.optimize.backtesting import Backtesting
 # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
 from freqtrade.optimize.hyperopt_auto import HyperOptAuto
 from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss
+from freqtrade.optimize.hyperopt_output import HyperoptOutput
 from freqtrade.optimize.hyperopt_tools import (
     HyperoptStateContainer,
     HyperoptTools,
@@ -47,6 +40,7 @@ from freqtrade.optimize.hyperopt_tools import (
 )
 from freqtrade.optimize.optimize_reports import generate_strategy_stats
 from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver
+from freqtrade.util import get_progress_tracker
 
 
 # Suppress scikit-learn FutureWarnings from skopt
@@ -86,6 +80,8 @@ class Hyperopt:
         self.max_open_trades_space: List[Dimension] = []
         self.dimensions: List[Dimension] = []
 
+        self._hyper_out: HyperoptOutput = HyperoptOutput()
+
         self.config = config
         self.min_date: datetime
         self.max_date: datetime
@@ -260,7 +256,7 @@ class Hyperopt:
             result["max_open_trades"] = {"max_open_trades": strategy.max_open_trades}
         return result
 
-    def print_results(self, results) -> None:
+    def print_results(self, results: Dict[str, Any]) -> None:
         """
         Log results if it is better than any previous evaluation
         TODO: this should be moved to HyperoptTools too
@@ -268,17 +264,12 @@ class Hyperopt:
         is_best = results["is_best"]
 
         if self.print_all or is_best:
-            print(
-                HyperoptTools.get_result_table(
-                    self.config,
-                    results,
-                    self.total_epochs,
-                    self.print_all,
-                    self.print_colorized,
-                    self.hyperopt_table_header,
-                )
+            self._hyper_out.add_data(
+                self.config,
+                [results],
+                self.total_epochs,
+                self.print_all,
             )
-            self.hyperopt_table_header = 2
 
     def init_spaces(self):
         """
@@ -626,25 +617,18 @@ class Hyperopt:
 
         self.opt = self.get_optimizer(self.dimensions, config_jobs)
 
-        if self.print_colorized:
-            colorama_init(autoreset=True)
-
         try:
             with Parallel(n_jobs=config_jobs) as parallel:
                 jobs = parallel._effective_n_jobs()
                 logger.info(f"Effective number of parallel workers used: {jobs}")
+                console = Console(
+                    color_system="auto" if self.print_colorized else None,
+                )
 
                 # Define progressbar
-                with Progress(
-                    TextColumn("[progress.description]{task.description}"),
-                    BarColumn(bar_width=None),
-                    MofNCompleteColumn(),
-                    TaskProgressColumn(),
-                    "•",
-                    TimeElapsedColumn(),
-                    "•",
-                    TimeRemainingColumn(),
-                    expand=True,
+                with get_progress_tracker(
+                    console=console,
+                    cust_objs=[Align.center(self._hyper_out.table)],
                 ) as pbar:
                     task = pbar.add_task("Epochs", total=self.total_epochs)
 

freqtrade/optimize/hyperopt_output.py

@@ -0,0 +1,123 @@
+import sys
+from typing import List, Optional, Union
+
+from rich.console import Console
+from rich.table import Table
+from rich.text import Text
+
+from freqtrade.constants import Config
+from freqtrade.optimize.optimize_reports import generate_wins_draws_losses
+from freqtrade.util import fmt_coin
+
+
+class HyperoptOutput:
+    def __init__(self):
+        self.table = Table(
+            title="Hyperopt results",
+        )
+        # Headers
+        self.table.add_column("Best", justify="left")
+        self.table.add_column("Epoch", justify="right")
+        self.table.add_column("Trades", justify="right")
+        self.table.add_column("Win  Draw  Loss  Win%", justify="right")
+        self.table.add_column("Avg profit", justify="right")
+        self.table.add_column("Profit", justify="right")
+        self.table.add_column("Avg duration", justify="right")
+        self.table.add_column("Objective", justify="right")
+        self.table.add_column("Max Drawdown (Acct)", justify="right")
+
+    def _add_row(self, data: List[Union[str, Text]]):
+        """Add single row"""
+        row_to_add: List[Union[str, Text]] = [r if isinstance(r, Text) else str(r) for r in data]
+
+        self.table.add_row(*row_to_add)
+
+    def _add_rows(self, data: List[List[Union[str, Text]]]):
+        """add multiple rows"""
+        for row in data:
+            self._add_row(row)
+
+    def print(self, console: Optional[Console] = None, *, print_colorized=True):
+        if not console:
+            console = Console(
+                color_system="auto" if print_colorized else None,
+                width=200 if "pytest" in sys.modules else None,
+            )
+
+        console.print(self.table)
+
+    def add_data(
+        self,
+        config: Config,
+        results: list,
+        total_epochs: int,
+        highlight_best: bool,
+    ) -> None:
+        """Format one or multiple rows and add them"""
+        stake_currency = config["stake_currency"]
+
+        for r in results:
+            self.table.add_row(
+                *[
+                    # "Best":
+                    (
+                        ("*" if r["is_initial_point"] or r["is_random"] else "")
+                        + (" Best" if r["is_best"] else "")
+                    ).lstrip(),
+                    # "Epoch":
+                    f"{r['current_epoch']}/{total_epochs}",
+                    # "Trades":
+                    str(r["results_metrics"]["total_trades"]),
+                    # "Win  Draw  Loss  Win%":
+                    generate_wins_draws_losses(
+                        r["results_metrics"]["wins"],
+                        r["results_metrics"]["draws"],
+                        r["results_metrics"]["losses"],
+                    ),
+                    # "Avg profit":
+                    f"{r['results_metrics']['profit_mean']:.2%}"
+                    if r["results_metrics"]["profit_mean"] is not None
+                    else "--",
+                    # "Profit":
+                    Text(
+                        "{} {}".format(
+                            fmt_coin(
+                                r["results_metrics"]["profit_total_abs"],
+                                stake_currency,
+                                keep_trailing_zeros=True,
+                            ),
+                            f"({r['results_metrics']['profit_total']:,.2%})".rjust(10, " "),
+                        )
+                        if r["results_metrics"].get("profit_total_abs", 0) != 0.0
+                        else "--",
+                        style=(
+                            "green"
+                            if r["results_metrics"].get("profit_total_abs", 0) > 0
+                            else "red"
+                        )
+                        if not r["is_best"]
+                        else "",
+                    ),
+                    # "Avg duration":
+                    str(r["results_metrics"]["holding_avg"]),
+                    # "Objective":
+                    f"{r['loss']:,.5f}" if r["loss"] != 100000 else "N/A",
+                    # "Max Drawdown (Acct)":
+                    "{} {}".format(
+                        fmt_coin(
+                            r["results_metrics"]["max_drawdown_abs"],
+                            stake_currency,
+                            keep_trailing_zeros=True,
+                        ),
+                        (f"({r['results_metrics']['max_drawdown_account']:,.2%})").rjust(10, " "),
+                    )
+                    if r["results_metrics"]["max_drawdown_account"] != 0.0
+                    else "--",
+                ],
+                style=" ".join(
+                    [
+                        "bold gold1" if r["is_best"] and highlight_best else "",
+                        "italic " if r["is_initial_point"] else "",
+                    ]
+                ),
+            )

freqtrade/optimize/hyperopt_tools.py

@@ -5,10 +5,7 @@ from pathlib import Path
 from typing import Any, Dict, Iterator, List, Optional, Tuple
 
 import numpy as np
-import pandas as pd
 import rapidjson
-import tabulate
-from colorama import Fore, Style
 from pandas import isna, json_normalize
 
 from freqtrade.constants import FTHYPT_FILEVERSION, Config
@@ -16,8 +13,6 @@ from freqtrade.enums import HyperoptState
 from freqtrade.exceptions import OperationalException
 from freqtrade.misc import deep_merge_dicts, round_dict, safe_value_fallback2
 from freqtrade.optimize.hyperopt_epoch_filters import hyperopt_filter_epochs
-from freqtrade.optimize.optimize_reports import generate_wins_draws_losses
-from freqtrade.util import fmt_coin
 
 
 logger = logging.getLogger(__name__)
@@ -357,175 +352,6 @@ class HyperoptTools:
             + f"Objective: {results['loss']:.5f}"
         )
 
-    @staticmethod
-    def prepare_trials_columns(trials: pd.DataFrame) -> pd.DataFrame:
-        trials["Best"] = ""
-
-        if "results_metrics.winsdrawslosses" not in trials.columns:
-            # Ensure compatibility with older versions of hyperopt results
-            trials["results_metrics.winsdrawslosses"] = "N/A"
-
-        has_account_drawdown = "results_metrics.max_drawdown_account" in trials.columns
-        if not has_account_drawdown:
-            # Ensure compatibility with older versions of hyperopt results
-            trials["results_metrics.max_drawdown_account"] = None
-        if "is_random" not in trials.columns:
-            trials["is_random"] = False
-
-        # New mode, using backtest result for metrics
-        trials["results_metrics.winsdrawslosses"] = trials.apply(
-            lambda x: generate_wins_draws_losses(
-                x["results_metrics.wins"], x["results_metrics.draws"], x["results_metrics.losses"]
-            ),
-            axis=1,
-        )
-
-        trials = trials[
-            [
-                "Best",
-                "current_epoch",
-                "results_metrics.total_trades",
-                "results_metrics.winsdrawslosses",
-                "results_metrics.profit_mean",
-                "results_metrics.profit_total_abs",
-                "results_metrics.profit_total",
-                "results_metrics.holding_avg",
-                "results_metrics.max_drawdown_account",
-                "results_metrics.max_drawdown_abs",
-                "loss",
-                "is_initial_point",
-                "is_random",
-                "is_best",
-            ]
-        ]
-
-        trials.columns = [
-            "Best",
-            "Epoch",
-            "Trades",
-            " Win  Draw  Loss  Win%",
-            "Avg profit",
-            "Total profit",
-            "Profit",
-            "Avg duration",
-            "max_drawdown_account",
-            "max_drawdown_abs",
-            "Objective",
-            "is_initial_point",
-            "is_random",
-            "is_best",
-        ]
-
-        return trials
-
-    @staticmethod
-    def get_result_table(
-        config: Config,
-        results: list,
-        total_epochs: int,
-        highlight_best: bool,
-        print_colorized: bool,
-        remove_header: int,
-    ) -> str:
-        """
-        Log result table
-        """
-        if not results:
-            return ""
-
-        tabulate.PRESERVE_WHITESPACE = True
-        trials = json_normalize(results, max_level=1)
-
-        trials = HyperoptTools.prepare_trials_columns(trials)
-
-        trials["is_profit"] = False
-        trials.loc[trials["is_initial_point"] | trials["is_random"], "Best"] = "*     "
-        trials.loc[trials["is_best"], "Best"] = "Best"
-        trials.loc[
-            (trials["is_initial_point"] | trials["is_random"]) & trials["is_best"], "Best"
-        ] = "* Best"
-        trials.loc[trials["Total profit"] > 0, "is_profit"] = True
-        trials["Trades"] = trials["Trades"].astype(str)
-        # perc_multi = 1 if legacy_mode else 100
-        trials["Epoch"] = trials["Epoch"].apply(
-            lambda x: "{}/{}".format(str(x).rjust(len(str(total_epochs)), " "), total_epochs)
-        )
-        trials["Avg profit"] = trials["Avg profit"].apply(
-            lambda x: f"{x:,.2%}".rjust(7, " ") if not isna(x) else "--".rjust(7, " ")
-        )
-        trials["Avg duration"] = trials["Avg duration"].apply(
-            lambda x: (
-                f"{x:,.1f} m".rjust(7, " ")
-                if isinstance(x, float)
-                else f"{x}"
-                if not isna(x)
-                else "--".rjust(7, " ")
-            )
-        )
-        trials["Objective"] = trials["Objective"].apply(
-            lambda x: f"{x:,.5f}".rjust(8, " ") if x != 100000 else "N/A".rjust(8, " ")
-        )
-
-        stake_currency = config["stake_currency"]
-
-        trials["Max Drawdown (Acct)"] = trials.apply(
-            lambda x: (
-                "{} {}".format(
-                    fmt_coin(x["max_drawdown_abs"], stake_currency, keep_trailing_zeros=True),
-                    (f"({x['max_drawdown_account']:,.2%})").rjust(10, " "),
-                ).rjust(25 + len(stake_currency))
-                if x["max_drawdown_account"] != 0.0
-                else "--".rjust(25 + len(stake_currency))
-            ),
-            axis=1,
-        )
-
-        trials = trials.drop(columns=["max_drawdown_abs", "max_drawdown_account"])
-
-        trials["Profit"] = trials.apply(
-            lambda x: (
-                "{} {}".format(
-                    fmt_coin(x["Total profit"], stake_currency, keep_trailing_zeros=True),
-                    f"({x['Profit']:,.2%})".rjust(10, " "),
-                ).rjust(25 + len(stake_currency))
-                if x["Total profit"] != 0.0
-                else "--".rjust(25 + len(stake_currency))
-            ),
-            axis=1,
-        )
-        trials = trials.drop(columns=["Total profit"])
-
-        if print_colorized:
-            trials2 = trials.astype(str)
-            for i in range(len(trials)):
-                if trials.loc[i]["is_profit"]:
-                    for j in range(len(trials.loc[i]) - 3):
-                        trials2.iat[i, j] = f"{Fore.GREEN}{str(trials.iloc[i, j])}{Fore.RESET}"
-                if trials.loc[i]["is_best"] and highlight_best:
-                    for j in range(len(trials.loc[i]) - 3):
-                        trials2.iat[i, j] = (
-                            f"{Style.BRIGHT}{str(trials.iloc[i, j])}{Style.RESET_ALL}"
-                        )
-            trials = trials2
-            del trials2
-        trials = trials.drop(columns=["is_initial_point", "is_best", "is_profit", "is_random"])
-        if remove_header > 0:
-            table = tabulate.tabulate(
-                trials.to_dict(orient="list"), tablefmt="orgtbl", headers="keys", stralign="right"
-            )
-
-            table = table.split("\n", remove_header)[remove_header]
-        elif remove_header < 0:
-            table = tabulate.tabulate(
-                trials.to_dict(orient="list"), tablefmt="psql", headers="keys", stralign="right"
-            )
-            table = "\n".join(table.split("\n")[0:remove_header])
-        else:
-            table = tabulate.tabulate(
-                trials.to_dict(orient="list"), tablefmt="psql", headers="keys", stralign="right"
-            )
-        return table
-
     @staticmethod
     def export_csv_file(config: Config, results: list, csv_file: str) -> None:
         """

freqtrade/optimize/optimize_reports/bt_output.py

@@ -1,12 +1,10 @@
 import logging
-from typing import Any, Dict, List, Union
-
-from tabulate import tabulate
+from typing import Any, Dict, List, Literal, Union
 
 from freqtrade.constants import UNLIMITED_STAKE_AMOUNT, Config
 from freqtrade.optimize.optimize_reports.optimize_reports import generate_periodic_breakdown_stats
 from freqtrade.types import BacktestResultType
-from freqtrade.util import decimals_per_coin, fmt_coin
+from freqtrade.util import decimals_per_coin, fmt_coin, print_rich_table
 
 
 logger = logging.getLogger(__name__)
@@ -46,22 +44,23 @@ def generate_wins_draws_losses(wins, draws, losses):
     return f"{wins:>4}  {draws:>4}  {losses:>4}  {wl_ratio:>4}"
 
 
-def text_table_bt_results(pair_results: List[Dict[str, Any]], stake_currency: str) -> str:
+def text_table_bt_results(
+    pair_results: List[Dict[str, Any]], stake_currency: str, title: str
+) -> None:
     """
     Generates and returns a text table for the given backtest data and the results dataframe
     :param pair_results: List of Dictionaries - one entry per pair + final TOTAL row
     :param stake_currency: stake-currency - used to correctly name headers
-    :return: pretty printed table with tabulate as string
+    :param title: Title of the table
     """
 
     headers = _get_line_header("Pair", stake_currency, "Trades")
-    floatfmt = _get_line_floatfmt(stake_currency)
     output = [
         [
             t["key"],
             t["trades"],
             t["profit_mean_pct"],
-            t["profit_total_abs"],
+            f"{t['profit_total_abs']:.{decimals_per_coin(stake_currency)}f}",
             t["profit_total_pct"],
             t["duration_avg"],
             generate_wins_draws_losses(t["wins"], t["draws"], t["losses"]),
@@ -69,26 +68,32 @@ def text_table_bt_results(pair_results: List[Dict[str, Any]], stake_currency: st
         for t in pair_results
     ]
     # Ignore type as floatfmt does allow tuples but mypy does not know that
-    return tabulate(output, headers=headers, floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")
+    print_rich_table(output, headers, summary=title)
 
 
-def text_table_tags(tag_type: str, tag_results: List[Dict[str, Any]], stake_currency: str) -> str:
+def text_table_tags(
+    tag_type: Literal["enter_tag", "exit_tag", "mix_tag"],
+    tag_results: List[Dict[str, Any]],
+    stake_currency: str,
+) -> None:
     """
     Generates and returns a text table for the given backtest data and the results dataframe
     :param pair_results: List of Dictionaries - one entry per pair + final TOTAL row
     :param stake_currency: stake-currency - used to correctly name headers
-    :return: pretty printed table with tabulate as string
     """
     floatfmt = _get_line_floatfmt(stake_currency)
     fallback: str = ""
     is_list = False
     if tag_type == "enter_tag":
-        headers = _get_line_header("Enter Tag", stake_currency, "Entries")
+        title = "Enter Tag"
+        headers = _get_line_header(title, stake_currency, "Entries")
     elif tag_type == "exit_tag":
-        headers = _get_line_header("Exit Reason", stake_currency, "Exits")
+        title = "Exit Reason"
+        headers = _get_line_header(title, stake_currency, "Exits")
         fallback = "exit_reason"
     else:
         # Mix tag
+        title = "Mixed Tag"
         headers = _get_line_header(["Enter Tag", "Exit Reason"], stake_currency, "Trades")
         floatfmt.insert(0, "s")
         is_list = True
@@ -106,7 +111,7 @@ def text_table_tags(tag_type: str, tag_results: List[Dict[str, Any]], stake_curr
             ),
             t["trades"],
             t["profit_mean_pct"],
-            t["profit_total_abs"],
+            f"{t['profit_total_abs']:.{decimals_per_coin(stake_currency)}f}",
             t["profit_total_pct"],
             t.get("duration_avg"),
             generate_wins_draws_losses(t["wins"], t["draws"], t["losses"]),
@@ -114,17 +119,16 @@ def text_table_tags(tag_type: str, tag_results: List[Dict[str, Any]], stake_curr
         for t in tag_results
     ]
     # Ignore type as floatfmt does allow tuples but mypy does not know that
-    return tabulate(output, headers=headers, floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")
+    print_rich_table(output, headers, summary=f"{title.upper()} STATS")
 
 
 def text_table_periodic_breakdown(
     days_breakdown_stats: List[Dict[str, Any]], stake_currency: str, period: str
-) -> str:
+) -> None:
     """
     Generate small table with Backtest results by days
     :param days_breakdown_stats: Days breakdown metrics
     :param stake_currency: Stakecurrency used
-    :return: pretty printed table with tabulate as string
     """
     headers = [
         period.capitalize(),
@@ -143,17 +147,15 @@ def text_table_periodic_breakdown(
         ]
         for d in days_breakdown_stats
     ]
-    return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right")
+    print_rich_table(output, headers, summary=f"{period.upper()} BREAKDOWN")
 
 
-def text_table_strategy(strategy_results, stake_currency: str) -> str:
+def text_table_strategy(strategy_results, stake_currency: str, title: str):
     """
     Generate summary table per strategy
     :param strategy_results: Dict of <Strategyname: DataFrame> containing results for all strategies
     :param stake_currency: stake-currency - used to correctly name headers
-    :return: pretty printed table with tabulate as string
     """
-    floatfmt = _get_line_floatfmt(stake_currency)
     headers = _get_line_header("Strategy", stake_currency, "Trades")
     # _get_line_header() is also used for per-pair summary. Per-pair drawdown is mostly useless
     # therefore we slip this column in only for strategy summary here.
@@ -177,8 +179,8 @@ def text_table_strategy(strategy_results, stake_currency: str) -> str:
         [
             t["key"],
             t["trades"],
-            t["profit_mean_pct"],
-            t["profit_total_abs"],
+            f"{t['profit_mean_pct']:.2f}",
+            f"{t['profit_total_abs']:.{decimals_per_coin(stake_currency)}f}",
             t["profit_total_pct"],
             t["duration_avg"],
             generate_wins_draws_losses(t["wins"], t["draws"], t["losses"]),
@@ -186,11 +188,10 @@ def text_table_strategy(strategy_results, stake_currency: str) -> str:
         ]
         for t, drawdown in zip(strategy_results, drawdown)
     ]
-    # Ignore type as floatfmt does allow tuples but mypy does not know that
-    return tabulate(output, headers=headers, floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")
+    print_rich_table(output, headers, summary=title)
 
 
-def text_table_add_metrics(strat_results: Dict) -> str:
+def text_table_add_metrics(strat_results: Dict) -> None:
     if len(strat_results["trades"]) > 0:
         best_trade = max(strat_results["trades"], key=lambda x: x["profit_ratio"])
         worst_trade = min(strat_results["trades"], key=lambda x: x["profit_ratio"])
@@ -372,8 +373,8 @@ def text_table_add_metrics(strat_results: Dict) -> str:
             *drawdown_metrics,
             ("Market change", f"{strat_results['market_change']:.2%}"),
         ]
+        print_rich_table(metrics, ["Metric", "Value"], summary="SUMMARY METRICS", justify="left")
 
-        return tabulate(metrics, headers=["Metric", "Value"], tablefmt="orgtbl")
     else:
         start_balance = fmt_coin(strat_results["starting_balance"], strat_results["stake_currency"])
         stake_amount = (
@@ -387,7 +388,7 @@ def text_table_add_metrics(strat_results: Dict) -> str:
             f"Your starting balance was {start_balance}, "
             f"and your stake was {stake_amount}."
         )
-        return message
+        print(message)
 
 
 def _show_tag_subresults(results: Dict[str, Any], stake_currency: str):
@@ -395,25 +396,13 @@ def _show_tag_subresults(results: Dict[str, Any], stake_currency: str):
     Print tag subresults (enter_tag, exit_reason_summary, mix_tag_stats)
     """
     if (enter_tags := results.get("results_per_enter_tag")) is not None:
-        table = text_table_tags("enter_tag", enter_tags, stake_currency)
-
-        if isinstance(table, str) and len(table) > 0:
-            print(" ENTER TAG STATS ".center(len(table.splitlines()[0]), "="))
-        print(table)
+        text_table_tags("enter_tag", enter_tags, stake_currency)
 
     if (exit_reasons := results.get("exit_reason_summary")) is not None:
-        table = text_table_tags("exit_tag", exit_reasons, stake_currency)
-
-        if isinstance(table, str) and len(table) > 0:
-            print(" EXIT REASON STATS ".center(len(table.splitlines()[0]), "="))
-        print(table)
+        text_table_tags("exit_tag", exit_reasons, stake_currency)
 
     if (mix_tag := results.get("mix_tag_stats")) is not None:
-        table = text_table_tags("mix_tag", mix_tag, stake_currency)
-
-        if isinstance(table, str) and len(table) > 0:
-            print(" MIXED TAG STATS ".center(len(table.splitlines()[0]), "="))
-        print(table)
+        text_table_tags("mix_tag", mix_tag, stake_currency)
 
 
 def show_backtest_result(
@@ -424,15 +413,12 @@ def show_backtest_result(
     """
     # Print results
     print(f"Result for strategy {strategy}")
-    table = text_table_bt_results(results["results_per_pair"], stake_currency=stake_currency)
-    if isinstance(table, str):
-        print(" BACKTESTING REPORT ".center(len(table.splitlines()[0]), "="))
-    print(table)
-
-    table = text_table_bt_results(results["left_open_trades"], stake_currency=stake_currency)
-    if isinstance(table, str) and len(table) > 0:
-        print(" LEFT OPEN TRADES REPORT ".center(len(table.splitlines()[0]), "="))
-    print(table)
+    text_table_bt_results(
+        results["results_per_pair"], stake_currency=stake_currency, title="BACKTESTING REPORT"
+    )
+    text_table_bt_results(
+        results["left_open_trades"], stake_currency=stake_currency, title="LEFT OPEN TRADES REPORT"
+    )
 
     _show_tag_subresults(results, stake_currency)
 
@@ -443,20 +429,11 @@ def show_backtest_result(
             days_breakdown_stats = generate_periodic_breakdown_stats(
                 trade_list=results["trades"], period=period
             )
-        table = text_table_periodic_breakdown(
+        text_table_periodic_breakdown(
             days_breakdown_stats=days_breakdown_stats, stake_currency=stake_currency, period=period
         )
-        if isinstance(table, str) and len(table) > 0:
-            print(f" {period.upper()} BREAKDOWN ".center(len(table.splitlines()[0]), "="))
-        print(table)
 
-    table = text_table_add_metrics(results)
-    if isinstance(table, str) and len(table) > 0:
-        print(" SUMMARY METRICS ".center(len(table.splitlines()[0]), "="))
-    print(table)
-
-    if isinstance(table, str) and len(table) > 0:
-        print("=" * len(table.splitlines()[0]))
+    text_table_add_metrics(results)
 
     print()
 
@@ -472,15 +449,13 @@ def show_backtest_results(config: Config, backtest_stats: BacktestResultType):
     if len(backtest_stats["strategy"]) > 0:
         # Print Strategy summary table
 
-        table = text_table_strategy(backtest_stats["strategy_comparison"], stake_currency)
         print(
             f"Backtested {results['backtest_start']} -> {results['backtest_end']} |"
             f" Max open trades : {results['max_open_trades']}"
         )
-        print(" STRATEGY SUMMARY ".center(len(table.splitlines()[0]), "="))
-        print(table)
-        print("=" * len(table.splitlines()[0]))
-        print("\nFor more details, please look at the detail tables above")
+        text_table_strategy(
+            backtest_stats["strategy_comparison"], stake_currency, "STRATEGY SUMMARY"
+        )
 
 
 def show_sorted_pairlist(config: Config, backtest_stats: BacktestResultType):
@@ -493,8 +468,7 @@ def show_sorted_pairlist(config: Config, backtest_stats: BacktestResultType):
             print("]")
 
 
-def generate_edge_table(results: dict) -> str:
-    floatfmt = ("s", ".10g", ".2f", ".2f", ".2f", ".2f", "d", "d", "d")
+def generate_edge_table(results: dict) -> None:
     tabular_data = []
     headers = [
         "Pair",
@@ -512,17 +486,13 @@ def generate_edge_table(results: dict) -> str:
             tabular_data.append(
                 [
                     result[0],
-                    result[1].stoploss,
-                    result[1].winrate,
-                    result[1].risk_reward_ratio,
-                    result[1].required_risk_reward,
-                    result[1].expectancy,
+                    f"{result[1].stoploss:.10g}",
+                    f"{result[1].winrate:.2f}",
+                    f"{result[1].risk_reward_ratio:.2f}",
+                    f"{result[1].required_risk_reward:.2f}",
+                    f"{result[1].expectancy:.2f}",
                     result[1].nb_trades,
                     round(result[1].avg_trade_duration),
                 ]
             )
-
-    # Ignore type as floatfmt does allow tuples but mypy does not know that
-    return tabulate(
-        tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="orgtbl", stralign="right"
-    )
+    print_rich_table(tabular_data, headers, summary="EDGE TABLE")

freqtrade/persistence/models.py

@@ -32,12 +32,12 @@ def get_request_or_thread_id() -> Optional[str]:
     """
     Helper method to get either async context (for fastapi requests), or thread id
     """
-    id = _request_id_ctx_var.get()
-    if id is None:
+    request_id = _request_id_ctx_var.get()
+    if request_id is None:
         # when not in request context - use thread id
-        id = str(threading.current_thread().ident)
+        request_id = str(threading.current_thread().ident)
 
-    return id
+    return request_id
 
 
 _SQL_DOCS_URL = "http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls"

freqtrade/persistence/trade_model.py

@@ -2012,7 +2012,7 @@ class Trade(ModelBase, LocalTrade):
         ).all()
 
         resp: List[Dict] = []
-        for id, enter_tag, exit_reason, profit, profit_abs, count in mix_tag_perf:
+        for _, enter_tag, exit_reason, profit, profit_abs, count in mix_tag_perf:
             enter_tag = enter_tag if enter_tag is not None else "Other"
             exit_reason = exit_reason if exit_reason is not None else "Other"
 

freqtrade/plugins/pairlist/AgeFilter.py

@@ -13,7 +13,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.misc import plural
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import PeriodicCache, dt_floor_day, dt_now, dt_ts
 
 
@@ -21,6 +21,8 @@ logger = logging.getLogger(__name__)
 
 
 class AgeFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/FullTradesFilter.py

@@ -7,13 +7,15 @@ from typing import List
 
 from freqtrade.exchange.types import Tickers
 from freqtrade.persistence import Trade
-from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.plugins.pairlist.IPairList import IPairList, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class FullTradesFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO_ACTION
+
     @property
     def needstickers(self) -> bool:
         """

freqtrade/plugins/pairlist/IPairList.py

@@ -5,6 +5,7 @@ PairList Handler base class
 import logging
 from abc import ABC, abstractmethod
 from copy import deepcopy
+from enum import Enum
 from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
 
 from freqtrade.constants import Config
@@ -51,8 +52,20 @@ PairlistParameter = Union[
 ]
 
 
+class SupportsBacktesting(str, Enum):
+    """
+    Enum to indicate if a Pairlist Handler supports backtesting.
+    """
+
+    YES = "yes"
+    NO = "no"
+    NO_ACTION = "no_action"
+    BIASED = "biased"
+
+
 class IPairList(LoggingMixin, ABC):
     is_pairlist_generator = False
+    supports_backtesting: SupportsBacktesting = SupportsBacktesting.NO
 
     def __init__(
         self,

freqtrade/plugins/pairlist/MarketCapPairList.py

@@ -11,7 +11,7 @@ from cachetools import TTLCache
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util.coin_gecko import FtCoinGeckoApi
 
 
@@ -20,6 +20,7 @@ logger = logging.getLogger(__name__)
 
 class MarketCapPairList(IPairList):
     is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.BIASED
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/OffsetFilter.py

@@ -7,13 +7,15 @@ from typing import Dict, List
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class OffsetFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.YES
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/PerformanceFilter.py

@@ -9,13 +9,15 @@ import pandas as pd
 
 from freqtrade.exchange.types import Tickers
 from freqtrade.persistence import Trade
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class PerformanceFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO_ACTION
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/PrecisionFilter.py

@@ -8,13 +8,15 @@ from typing import Optional
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import ROUND_UP
 from freqtrade.exchange.types import Ticker
-from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.plugins.pairlist.IPairList import IPairList, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class PrecisionFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.BIASED
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/PriceFilter.py

@@ -7,13 +7,15 @@ from typing import Dict, Optional
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Ticker
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class PriceFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.BIASED
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/ProducerPairList.py

@@ -9,7 +9,7 @@ from typing import Dict, List, Optional
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
@@ -31,6 +31,7 @@ class ProducerPairList(IPairList):
     """
 
     is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.NO
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/RemotePairList.py

@@ -16,7 +16,7 @@ from freqtrade import __version__
 from freqtrade.configuration.load_config import CONFIG_PARSE_MODE
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 
 
@@ -25,6 +25,8 @@ logger = logging.getLogger(__name__)
 
 class RemotePairList(IPairList):
     is_pairlist_generator = True
+    # Potential winner bias
+    supports_backtesting = SupportsBacktesting.BIASED
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/ShuffleFilter.py

@@ -9,7 +9,7 @@ from typing import Dict, List, Literal
 from freqtrade.enums import RunMode
 from freqtrade.exchange import timeframe_to_seconds
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util.periodic_cache import PeriodicCache
 
 
@@ -19,6 +19,8 @@ ShuffleValues = Literal["candle", "iteration"]
 
 
 class ShuffleFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.YES
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/SpreadFilter.py

@@ -7,13 +7,15 @@ from typing import Dict, Optional
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Ticker
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class SpreadFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/StaticPairList.py

@@ -9,7 +9,7 @@ from copy import deepcopy
 from typing import Dict, List
 
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
@@ -17,6 +17,7 @@ logger = logging.getLogger(__name__)
 
 class StaticPairList(IPairList):
     is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.YES
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/VolatilityFilter.py

@@ -15,7 +15,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.misc import plural
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import dt_floor_day, dt_now, dt_ts
 
 
@@ -27,6 +27,8 @@ class VolatilityFilter(IPairList):
     Filters pairs by volatility
     """
 
+    supports_backtesting = SupportsBacktesting.NO
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/VolumePairList.py

@@ -14,7 +14,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import dt_now, format_ms_time
 
 
@@ -26,6 +26,7 @@ SORT_VALUES = ["quoteVolume"]
 
 class VolumePairList(IPairList):
     is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.NO
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/rangestabilityfilter.py

@@ -13,7 +13,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.misc import plural
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import dt_floor_day, dt_now, dt_ts
 
 
@@ -21,6 +21,8 @@ logger = logging.getLogger(__name__)
 
 
 class RangeStabilityFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlistmanager.py

@@ -11,10 +11,11 @@ from cachetools import TTLCache, cached
 from freqtrade.constants import Config, ListPairsWithTimeframes
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.enums import CandleType
+from freqtrade.enums.runmode import RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.mixins import LoggingMixin
-from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.plugins.pairlist.IPairList import IPairList, SupportsBacktesting
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.resolvers import PairListResolver
 
@@ -57,9 +58,44 @@ class PairListManager(LoggingMixin):
                 f"{invalid}."
             )
 
+        self._check_backtest()
+
         refresh_period = config.get("pairlist_refresh_period", 3600)
         LoggingMixin.__init__(self, logger, refresh_period)
 
+    def _check_backtest(self) -> None:
+        if self._config["runmode"] not in (RunMode.BACKTEST, RunMode.EDGE, RunMode.HYPEROPT):
+            return
+
+        pairlist_errors: List[str] = []
+        noaction_pairlists: List[str] = []
+        biased_pairlists: List[str] = []
+        for pairlist_handler in self._pairlist_handlers:
+            if pairlist_handler.supports_backtesting == SupportsBacktesting.NO:
+                pairlist_errors.append(pairlist_handler.name)
+            if pairlist_handler.supports_backtesting == SupportsBacktesting.NO_ACTION:
+                noaction_pairlists.append(pairlist_handler.name)
+            if pairlist_handler.supports_backtesting == SupportsBacktesting.BIASED:
+                biased_pairlists.append(pairlist_handler.name)
+
+        if noaction_pairlists:
+            logger.warning(
+                f"Pairlist Handlers {', '.join(noaction_pairlists)} do not generate "
+                "any changes during backtesting. While it's safe to leave them enabled, they will "
+                "not behave like in dry/live modes. "
+            )
+
+        if biased_pairlists:
+            logger.warning(
+                f"Pairlist Handlers {', '.join(biased_pairlists)} will introduce a lookahead bias "
+                "to your backtest results, as they use today's data - which inheritly suffers from "
+                "'winner bias'."
+            )
+        if pairlist_errors:
+            raise OperationalException(
+                f"Pairlist Handlers {', '.join(pairlist_errors)} do not support backtesting."
+            )
+
     @property
     def whitelist(self) -> List[str]:
         """The current whitelist"""

freqtrade/resolvers/iresolver.py

@@ -63,8 +63,8 @@ class IResolver:
 
         # Add extra directory to the top of the search paths
         if extra_dirs:
-            for dir in extra_dirs:
-                abs_paths.insert(0, Path(dir).resolve())
+            for directory in extra_dirs:
+                abs_paths.insert(0, Path(directory).resolve())
 
         if cls.extra_path and (extra := config.get(cls.extra_path)):
             abs_paths.insert(0, Path(extra).resolve())

freqtrade/resolvers/strategy_resolver.py

@@ -311,9 +311,9 @@ def warn_deprecated_setting(strategy: IStrategy, old: str, new: str, error=False
         setattr(strategy, new, getattr(strategy, f"{old}"))
 
 
-def check_override(object, parentclass, attribute):
+def check_override(obj, parentclass, attribute: str):
     """
     Checks if a object overrides the parent class attribute.
     :returns: True if the object is overridden.
     """
-    return getattr(type(object), attribute) != getattr(parentclass, attribute)
+    return getattr(type(obj), attribute) != getattr(parentclass, attribute)

freqtrade/rpc/api_server/uvicorn_threaded.py

@@ -1,4 +1,3 @@
-import contextlib
 import threading
 import time
 
@@ -53,7 +52,6 @@ class UvicornServer(uvicorn.Server):
             loop = asyncio.new_event_loop()
         loop.run_until_complete(self.serve(sockets=sockets))
 
-    @contextlib.contextmanager
     def run_in_thread(self):
         self.thread = threading.Thread(target=self.run, name="FTUvicorn")
         self.thread.start()

freqtrade/rpc/rpc.py

@@ -11,7 +11,7 @@ from typing import Any, Dict, Generator, List, Optional, Sequence, Tuple, Union
 import psutil
 from dateutil.relativedelta import relativedelta
 from dateutil.tz import tzlocal
-from numpy import NAN, inf, int64, mean
+from numpy import inf, int64, mean, nan
 from pandas import DataFrame, NaT
 from sqlalchemy import func, select
 
@@ -204,9 +204,9 @@ class RPC:
                             trade.pair, side="exit", is_short=trade.is_short, refresh=False
                         )
                     except (ExchangeError, PricingError):
-                        current_rate = NAN
+                        current_rate = nan
                     if len(trade.select_filled_orders(trade.entry_side)) > 0:
-                        current_profit = current_profit_abs = current_profit_fiat = NAN
+                        current_profit = current_profit_abs = current_profit_fiat = nan
                         if not isnan(current_rate):
                             prof = trade.calculate_profit(current_rate)
                             current_profit = prof.profit_ratio
@@ -277,7 +277,7 @@ class RPC:
             raise RPCException("no active trade")
         else:
             trades_list = []
-            fiat_profit_sum = NAN
+            fiat_profit_sum = nan
             for trade in trades:
                 # calculate profit and send message to user
                 try:
@@ -285,9 +285,9 @@ class RPC:
                         trade.pair, side="exit", is_short=trade.is_short, refresh=False
                     )
                 except (PricingError, ExchangeError):
-                    current_rate = NAN
-                    trade_profit = NAN
-                    profit_str = f"{NAN:.2%}"
+                    current_rate = nan
+                    trade_profit = nan
+                    profit_str = f"{nan:.2%}"
                 else:
                     if trade.nr_of_successful_entries > 0:
                         profit = trade.calculate_profit(current_rate)
@@ -533,9 +533,9 @@ class RPC:
                         trade.pair, side="exit", is_short=trade.is_short, refresh=False
                     )
                 except (PricingError, ExchangeError):
-                    current_rate = NAN
-                    profit_ratio = NAN
-                    profit_abs = NAN
+                    current_rate = nan
+                    profit_ratio = nan
+                    profit_abs = nan
                 else:
                     _profit = trade.calculate_profit(trade.close_rate or current_rate)
 
@@ -1317,7 +1317,7 @@ class RPC:
                 # replace NaT with `None`
                 dataframe[date_column] = dataframe[date_column].astype(object).replace({NaT: None})
 
-            dataframe = dataframe.replace({inf: None, -inf: None, NAN: None})
+            dataframe = dataframe.replace({inf: None, -inf: None, nan: None})
 
         res = {
             "pair": pair,

freqtrade/rpc/telegram.py

@@ -1401,19 +1401,21 @@ class Telegram(RPCHandler):
             nrecent = int(context.args[0]) if context.args else 10
         except (TypeError, ValueError, IndexError):
             nrecent = 10
+        nonspot = self._config.get("trading_mode", TradingMode.SPOT) != TradingMode.SPOT
         trades = self._rpc._rpc_trade_history(nrecent)
         trades_tab = tabulate(
             [
                 [
                     dt_humanize_delta(dt_from_ts(trade["close_timestamp"])),
-                    trade["pair"] + " (#" + str(trade["trade_id"]) + ")",
+                    f"{trade['pair']} (#{trade['trade_id']}"
+                    f"{(' ' + ('S' if trade['is_short'] else 'L')) if nonspot else ''})",
                     f"{(trade['close_profit']):.2%} ({trade['close_profit_abs']})",
                 ]
                 for trade in trades["trades"]
             ],
             headers=[
                 "Close Date",
-                "Pair (ID)",
+                "Pair (ID L/S)" if nonspot else "Pair (ID)",
                 f"Profit ({stake_cur})",
             ],
             tablefmt="simple",

freqtrade/strategy/interface.py

@@ -5,6 +5,7 @@ This module defines the interface to apply for strategies
 
 import logging
 from abc import ABC, abstractmethod
+from collections import OrderedDict
 from datetime import datetime, timedelta, timezone
 from math import isinf, isnan
 from typing import Dict, List, Optional, Tuple, Union
@@ -12,6 +13,7 @@ from typing import Dict, List, Optional, Tuple, Union
 from pandas import DataFrame
 
 from freqtrade.constants import CUSTOM_TAG_MAX_LENGTH, Config, IntOrInf, ListPairsWithTimeframes
+from freqtrade.data.converter import populate_dataframe_with_trades
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.enums import (
     CandleType,
@@ -139,6 +141,11 @@ class IStrategy(ABC, HyperStrategyMixin):
     # A self set parameter that represents the market direction. filled from configuration
     market_direction: MarketDirection = MarketDirection.NONE
 
+    # Global cache dictionary
+    _cached_grouped_trades_per_pair: Dict[
+        str, OrderedDict[Tuple[datetime, datetime], DataFrame]
+    ] = {}
+
     def __init__(self, config: Config) -> None:
         self.config = config
         # Dict to determine if analysis is necessary
@@ -1040,6 +1047,7 @@ class IStrategy(ABC, HyperStrategyMixin):
         dataframe = self.advise_indicators(dataframe, metadata)
         dataframe = self.advise_entry(dataframe, metadata)
         dataframe = self.advise_exit(dataframe, metadata)
+        logger.debug("TA Analysis Ended")
         return dataframe
 
     def _analyze_ticker_internal(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
@@ -1594,6 +1602,29 @@ class IStrategy(ABC, HyperStrategyMixin):
         dataframe = self.advise_exit(dataframe, metadata)
         return dataframe
 
+    def _if_enabled_populate_trades(self, dataframe: DataFrame, metadata: dict):
+        use_public_trades = self.config.get("exchange", {}).get("use_public_trades", False)
+        if use_public_trades:
+            trades = self.dp.trades(pair=metadata["pair"], copy=False)
+
+            config = self.config
+            config["timeframe"] = self.timeframe
+            pair = metadata["pair"]
+            # TODO: slice trades to size of dataframe for faster backtesting
+            cached_grouped_trades: OrderedDict[Tuple[datetime, datetime], DataFrame] = (
+                self._cached_grouped_trades_per_pair.get(pair, OrderedDict())
+            )
+            dataframe, cached_grouped_trades = populate_dataframe_with_trades(
+                cached_grouped_trades, config, dataframe, trades
+            )
+
+            # dereference old cache
+            if pair in self._cached_grouped_trades_per_pair:
+                del self._cached_grouped_trades_per_pair[pair]
+            self._cached_grouped_trades_per_pair[pair] = cached_grouped_trades
+
+            logger.debug("Populated dataframe with trades.")
+
     def advise_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         """
         Populate indicators that will be used in the Buy, Sell, short, exit_short strategy
@@ -1610,6 +1641,7 @@ class IStrategy(ABC, HyperStrategyMixin):
                 self, dataframe, metadata, inf_data, populate_fn
             )
 
+        self._if_enabled_populate_trades(dataframe, metadata)
         return self.populate_indicators(dataframe, metadata)
 
     def advise_entry(self, dataframe: DataFrame, metadata: dict) -> DataFrame:

freqtrade/types/valid_exchanges_type.py

@@ -14,4 +14,5 @@ class ValidExchangesType(TypedDict):
     valid: bool
     supported: bool
     comment: str
+    dex: bool
     trade_modes: List[TradeModeType]

freqtrade/util/__init__.py

@@ -15,6 +15,9 @@ from freqtrade.util.formatters import decimals_per_coin, fmt_coin, round_value
 from freqtrade.util.ft_precise import FtPrecise
 from freqtrade.util.measure_time import MeasureTime
 from freqtrade.util.periodic_cache import PeriodicCache
+from freqtrade.util.progress_tracker import get_progress_tracker  # noqa F401
+from freqtrade.util.rich_progress import CustomProgress
+from freqtrade.util.rich_tables import print_df_rich_table, print_rich_table
 from freqtrade.util.template_renderer import render_template, render_template_with_fallback  # noqa
 
 
@@ -36,4 +39,7 @@ __all__ = [
     "round_value",
     "fmt_coin",
     "MeasureTime",
+    "print_rich_table",
+    "print_df_rich_table",
+    "CustomProgress",
 ]

freqtrade/util/progress_tracker.py

@@ -0,0 +1,28 @@
+from rich.progress import (
+    BarColumn,
+    MofNCompleteColumn,
+    TaskProgressColumn,
+    TextColumn,
+    TimeElapsedColumn,
+    TimeRemainingColumn,
+)
+
+from freqtrade.util.rich_progress import CustomProgress
+
+
+def get_progress_tracker(**kwargs):
+    """
+    Get progress Bar with custom columns.
+    """
+    return CustomProgress(
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(bar_width=None),
+        MofNCompleteColumn(),
+        TaskProgressColumn(),
+        "•",
+        TimeElapsedColumn(),
+        "•",
+        TimeRemainingColumn(),
+        expand=True,
+        **kwargs,
+    )

freqtrade/util/rich_progress.py

@@ -0,0 +1,14 @@
+from typing import Union
+
+from rich.console import ConsoleRenderable, Group, RichCast
+from rich.progress import Progress
+
+
+class CustomProgress(Progress):
+    def __init__(self, *args, cust_objs=[], **kwargs) -> None:
+        self._cust_objs = cust_objs
+        super().__init__(*args, **kwargs)
+
+    def get_renderable(self) -> Union[ConsoleRenderable, RichCast, str]:
+        renderable = Group(*self._cust_objs, *self.get_renderables())
+        return renderable

freqtrade/util/rich_tables.py

@@ -0,0 +1,77 @@
+import sys
+from typing import Any, Dict, List, Optional, Sequence, Union
+
+from pandas import DataFrame
+from rich.console import Console
+from rich.table import Column, Table
+from rich.text import Text
+
+
+TextOrString = Union[str, Text]
+
+
+def print_rich_table(
+    tabular_data: Sequence[Union[Dict[str, Any], Sequence[TextOrString]]],
+    headers: Sequence[str],
+    summary: Optional[str] = None,
+    *,
+    justify="right",
+    table_kwargs: Optional[Dict[str, Any]] = None,
+) -> None:
+    table = Table(
+        *[c if isinstance(c, Column) else Column(c, justify=justify) for c in headers],
+        title=summary,
+        **(table_kwargs or {}),
+    )
+
+    for row in tabular_data:
+        if isinstance(row, dict):
+            table.add_row(
+                *[
+                    row[header] if isinstance(row[header], Text) else str(row[header])
+                    for header in headers
+                ]
+            )
+
+        else:
+            row_to_add: List[Union[str, Text]] = [r if isinstance(r, Text) else str(r) for r in row]
+            table.add_row(*row_to_add)
+
+    console = Console(
+        width=200 if "pytest" in sys.modules else None,
+    )
+    console.print(table)
+
+
+def _format_value(value: Any, *, floatfmt: str) -> str:
+    if isinstance(value, float):
+        return f"{value:{floatfmt}}"
+    return str(value)
+
+
+def print_df_rich_table(
+    tabular_data: DataFrame,
+    headers: Sequence[str],
+    summary: Optional[str] = None,
+    *,
+    show_index=False,
+    index_name: Optional[str] = None,
+    table_kwargs: Optional[Dict[str, Any]] = None,
+) -> None:
+    table = Table(title=summary, **(table_kwargs or {}))
+
+    if show_index:
+        index_name = str(index_name) if index_name else tabular_data.index.name
+        table.add_column(index_name)
+
+    for header in headers:
+        table.add_column(header, justify="right")
+
+    for value_list in tabular_data.itertuples(index=show_index):
+        row = [_format_value(x, floatfmt=".3f") for x in value_list]
+        table.add_row(*row)
+
+    console = Console(
+        width=200 if "pytest" in sys.modules else None,
+    )
+    console.print(table)