Merge pull request #10529 from freqtrade/feat/list-data-trades

add --trades to list-data command

README.md

@@ -86,41 +86,50 @@ For further (native) installation methods, please refer to the [Installation doc
 
 ```
 usage: freqtrade [-h] [-V]
-                 {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver}
+                 {trade,create-userdir,new-config,show-config,new-strategy,download-data,convert-data,convert-trade-data,trades-to-ohlcv,list-data,backtesting,backtesting-show,backtesting-analysis,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-markets,list-pairs,list-strategies,list-freqaimodels,list-timeframes,show-trades,test-pairlist,convert-db,install-ui,plot-dataframe,plot-profit,webserver,strategy-updater,lookahead-analysis,recursive-analysis}
                  ...
 
 Free, open source crypto trading bot
 
 positional arguments:
-  {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver}
+  {trade,create-userdir,new-config,show-config,new-strategy,download-data,convert-data,convert-trade-data,trades-to-ohlcv,list-data,backtesting,backtesting-show,backtesting-analysis,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-markets,list-pairs,list-strategies,list-freqaimodels,list-timeframes,show-trades,test-pairlist,convert-db,install-ui,plot-dataframe,plot-profit,webserver,strategy-updater,lookahead-analysis,recursive-analysis}
     trade               Trade module.
     create-userdir      Create user-data directory.
     new-config          Create new config
+    show-config         Show resolved config
     new-strategy        Create new strategy
     download-data       Download backtesting data.
     convert-data        Convert candle (OHLCV) data from one format to
                         another.
     convert-trade-data  Convert trade data from one format to another.
+    trades-to-ohlcv     Convert trade data to OHLCV data.
     list-data           List downloaded data.
     backtesting         Backtesting module.
+    backtesting-show    Show past Backtest results
+    backtesting-analysis
+                        Backtest Analysis module.
     edge                Edge module.
     hyperopt            Hyperopt module.
     hyperopt-list       List Hyperopt results
     hyperopt-show       Show details of Hyperopt results
     list-exchanges      Print available exchanges.
-    list-hyperopts      Print available hyperopt classes.
     list-markets        Print markets on exchange.
     list-pairs          Print pairs on exchange.
     list-strategies     Print available strategies.
+    list-freqaimodels   Print available freqAI models.
     list-timeframes     Print available timeframes for the exchange.
     show-trades         Show trades.
     test-pairlist       Test your pairlist configuration.
+    convert-db          Migrate database to different system
     install-ui          Install FreqUI
     plot-dataframe      Plot candles with indicators.
     plot-profit         Generate plot showing profits.
     webserver           Webserver module.
+    strategy-updater    updates outdated strategy files to the current version
+    lookahead-analysis  Check for potential look ahead bias.
+    recursive-analysis  Check for potential recursive formula issue.
 
-optional arguments:
+options:
   -h, --help            show this help message and exit
   -V, --version         show program's version number and exit
 

docs/bot-usage.md

@@ -12,41 +12,50 @@ This page explains the different parameters of the bot and how to run it.
 
 ```
 usage: freqtrade [-h] [-V]
-                 {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver}
+                 {trade,create-userdir,new-config,show-config,new-strategy,download-data,convert-data,convert-trade-data,trades-to-ohlcv,list-data,backtesting,backtesting-show,backtesting-analysis,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-markets,list-pairs,list-strategies,list-freqaimodels,list-timeframes,show-trades,test-pairlist,convert-db,install-ui,plot-dataframe,plot-profit,webserver,strategy-updater,lookahead-analysis,recursive-analysis}
                  ...
 
 Free, open source crypto trading bot
 
 positional arguments:
-  {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver}
+  {trade,create-userdir,new-config,show-config,new-strategy,download-data,convert-data,convert-trade-data,trades-to-ohlcv,list-data,backtesting,backtesting-show,backtesting-analysis,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-markets,list-pairs,list-strategies,list-freqaimodels,list-timeframes,show-trades,test-pairlist,convert-db,install-ui,plot-dataframe,plot-profit,webserver,strategy-updater,lookahead-analysis,recursive-analysis}
     trade               Trade module.
     create-userdir      Create user-data directory.
     new-config          Create new config
+    show-config         Show resolved config
     new-strategy        Create new strategy
     download-data       Download backtesting data.
     convert-data        Convert candle (OHLCV) data from one format to
                         another.
     convert-trade-data  Convert trade data from one format to another.
+    trades-to-ohlcv     Convert trade data to OHLCV data.
     list-data           List downloaded data.
     backtesting         Backtesting module.
+    backtesting-show    Show past Backtest results
+    backtesting-analysis
+                        Backtest Analysis module.
     edge                Edge module.
     hyperopt            Hyperopt module.
     hyperopt-list       List Hyperopt results
     hyperopt-show       Show details of Hyperopt results
     list-exchanges      Print available exchanges.
-    list-hyperopts      Print available hyperopt classes.
     list-markets        Print markets on exchange.
     list-pairs          Print pairs on exchange.
     list-strategies     Print available strategies.
+    list-freqaimodels   Print available freqAI models.
     list-timeframes     Print available timeframes for the exchange.
     show-trades         Show trades.
     test-pairlist       Test your pairlist configuration.
+    convert-db          Migrate database to different system
     install-ui          Install FreqUI
     plot-dataframe      Plot candles with indicators.
     plot-profit         Generate plot showing profits.
     webserver           Webserver module.
+    strategy-updater    updates outdated strategy files to the current version
+    lookahead-analysis  Check for potential look ahead bias.
+    recursive-analysis  Check for potential recursive formula issue.
 
-optional arguments:
+options:
   -h, --help            show this help message and exit
   -V, --version         show program's version number and exit
 

docs/data-download.md

@@ -423,7 +423,8 @@ You can get a list of downloaded data using the `list-data` sub-command.
 usage: freqtrade list-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
                            [--userdir PATH] [--exchange EXCHANGE]
                            [--data-format-ohlcv {json,jsongz,hdf5,feather,parquet}]
-                           [-p PAIRS [PAIRS ...]]
+                           [--data-format-trades {json,jsongz,hdf5,feather,parquet}]
+                           [--trades] [-p PAIRS [PAIRS ...]]
                            [--trading-mode {spot,margin,futures}]
                            [--show-timerange]
 
@@ -433,6 +434,10 @@ options:
   --data-format-ohlcv {json,jsongz,hdf5,feather,parquet}
                         Storage format for downloaded candle (OHLCV) data.
                         (default: `feather`).
+  --data-format-trades {json,jsongz,hdf5,feather,parquet}
+                        Storage format for downloaded trades data. (default:
+                        `feather`).
+  --trades              Work on trades data instead of OHLCV data.
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
                         Limit command to these pairs. Pairs are space-
                         separated.
@@ -465,13 +470,29 @@ Common arguments:
 ```bash
 > freqtrade list-data --userdir ~/.freqtrade/user_data/
 
-Found 33 pair / timeframe combinations.
+              Found 33 pair / timeframe combinations.
-pairs       timeframe
+┏━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━┓
-----------  -----------------------------------------
+┃          Pair ┃                                 Timeframe ┃ Type ┃
-ADA/BTC     5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d
+┡━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━┩
-ADA/ETH     5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d
+│       ADA/BTC │     5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d │ spot │
-ETH/BTC     5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d
+│       ADA/ETH │     5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d │ spot │
-ETH/USDT    5m, 15m, 30m, 1h, 2h, 4h
+│       ETH/BTC │     5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d │ spot │
+│      ETH/USDT │                  5m, 15m, 30m, 1h, 2h, 4h │ spot │
+└───────────────┴───────────────────────────────────────────┴──────┘
+
+```
+
+Show all trades data including from/to timerange
+
+``` bash
+> freqtrade list-data --show --trades
+                     Found trades data for 1 pair.                     
+┏━━━━━━━━━┳━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┓
+┃    Pair ┃ Type ┃                From ┃                  To ┃ Trades ┃
+┡━━━━━━━━━╇━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━┩
+│ XRP/ETH │ spot │ 2019-10-11 00:00:11 │ 2019-10-13 11:19:28 │  12477 │
+└─────────┴──────┴─────────────────────┴─────────────────────┴────────┘
+
 ```
 
 ## Trades (tick) data

freqtrade/commands/__init__.py

@@ -15,6 +15,7 @@ from freqtrade.commands.data_commands import (
     start_convert_trades,
     start_download_data,
     start_list_data,
+    start_list_trades_data,
 )
 from freqtrade.commands.db_commands import start_convert_db
 from freqtrade.commands.deploy_commands import (

freqtrade/commands/arguments.py

@@ -132,7 +132,15 @@ ARGS_CONVERT_TRADES = [
     "trading_mode",
 ]
 
-ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode", "show_timerange"]
+ARGS_LIST_DATA = [
+    "exchange",
+    "dataformat_ohlcv",
+    "dataformat_trades",
+    "trades",
+    "pairs",
+    "trading_mode",
+    "show_timerange",
+]
 
 ARGS_DOWNLOAD_DATA = [
     "pairs",

freqtrade/commands/cli_options.py

@@ -446,8 +446,12 @@ AVAILABLE_CLI_OPTIONS = {
     ),
     "download_trades": Arg(
         "--dl-trades",
-        help="Download trades instead of OHLCV data. The bot will resample trades to the "
+        help="Download trades instead of OHLCV data.",
-        "desired timeframe as specified as --timeframes/-t.",
+        action="store_true",
+    ),
+    "trades": Arg(
+        "--trades",
+        help="Work on trades data instead of OHLCV data.",
         action="store_true",
     ),
     "convert_trades": Arg(

freqtrade/commands/data_commands.py

@@ -14,6 +14,7 @@ from freqtrade.data.history import download_data_main
 from freqtrade.enums import CandleType, RunMode, TradingMode
 from freqtrade.exceptions import ConfigurationError
 from freqtrade.exchange import timeframe_to_minutes
+from freqtrade.misc import plural
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
 from freqtrade.util import print_rich_table
@@ -115,9 +116,13 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
 
 def start_list_data(args: Dict[str, Any]) -> None:
     """
-    List available backtest data
+    List available OHLCV data
     """
 
+    if args["trades"]:
+        start_list_trades_data(args)
+        return
+
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
     from freqtrade.data.history import get_datahandler
@@ -127,7 +132,6 @@ def start_list_data(args: Dict[str, Any]) -> None:
     paircombs = dhc.ohlcv_get_available_data(
         config["datadir"], config.get("trading_mode", TradingMode.SPOT)
     )
-
     if args["pairs"]:
         paircombs = [comb for comb in paircombs if comb[0] in args["pairs"]]
     title = f"Found {len(paircombs)} pair / timeframe combinations."
@@ -171,3 +175,51 @@ def start_list_data(args: Dict[str, Any]) -> None:
             summary=title,
             table_kwargs={"min_width": 50},
         )
+
+
+def start_list_trades_data(args: Dict[str, Any]) -> None:
+    """
+    List available Trades data
+    """
+
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    from freqtrade.data.history import get_datahandler
+
+    dhc = get_datahandler(config["datadir"], config["dataformat_trades"])
+
+    paircombs = dhc.trades_get_available_data(
+        config["datadir"], config.get("trading_mode", TradingMode.SPOT)
+    )
+
+    if args["pairs"]:
+        paircombs = [comb for comb in paircombs if comb in args["pairs"]]
+
+    title = f"Found trades data for {len(paircombs)} {plural(len(paircombs), 'pair')}."
+    if not config.get("show_timerange"):
+        print_rich_table(
+            [(pair, config.get("candle_type_def", CandleType.SPOT)) for pair in sorted(paircombs)],
+            ("Pair", "Type"),
+            title,
+            table_kwargs={"min_width": 50},
+        )
+    else:
+        paircombs1 = [
+            (pair, *dhc.trades_data_min_max(pair, config.get("trading_mode", TradingMode.SPOT)))
+            for pair in paircombs
+        ]
+        print_rich_table(
+            [
+                (
+                    pair,
+                    config.get("candle_type_def", CandleType.SPOT),
+                    start.strftime(DATETIME_PRINT_FORMAT),
+                    end.strftime(DATETIME_PRINT_FORMAT),
+                    str(length),
+                )
+                for pair, start, end, length in sorted(paircombs1, key=lambda x: (x[0]))
+            ],
+            ("Pair", "Type", "From", "To", "Trades"),
+            summary=title,
+            table_kwargs={"min_width": 50},
+        )

freqtrade/data/history/datahandlers/idatahandler.py

@@ -12,7 +12,7 @@ from datetime import datetime, timezone
 from pathlib import Path
 from typing import List, Optional, Tuple, Type
 
-from pandas import DataFrame
+from pandas import DataFrame, to_datetime
 
 from freqtrade import misc
 from freqtrade.configuration import TimeRange
@@ -32,6 +32,7 @@ logger = logging.getLogger(__name__)
 
 class IDataHandler(ABC):
     _OHLCV_REGEX = r"^([a-zA-Z_\d-]+)\-(\d+[a-zA-Z]{1,2})\-?([a-zA-Z_]*)?(?=\.)"
+    _TRADES_REGEX = r"^([a-zA-Z_\d-]+)\-(trades)?(?=\.)"
 
     def __init__(self, datadir: Path) -> None:
         self._datadir = datadir
@@ -166,6 +167,50 @@ class IDataHandler(ABC):
         :param candle_type: Any of the enum CandleType (must match trading mode!)
         """
 
+    @classmethod
+    def trades_get_available_data(cls, datadir: Path, trading_mode: TradingMode) -> List[str]:
+        """
+        Returns a list of all pairs with ohlcv data available in this datadir
+        :param datadir: Directory to search for ohlcv files
+        :param trading_mode: trading-mode to be used
+        :return: List of Tuples of (pair, timeframe, CandleType)
+        """
+        if trading_mode == TradingMode.FUTURES:
+            datadir = datadir.joinpath("futures")
+        _tmp = [
+            re.search(cls._TRADES_REGEX, p.name)
+            for p in datadir.glob(f"*.{cls._get_file_extension()}")
+        ]
+        return [
+            cls.rebuild_pair_from_filename(match[1])
+            for match in _tmp
+            if match and len(match.groups()) > 1
+        ]
+
+    def trades_data_min_max(
+        self,
+        pair: str,
+        trading_mode: TradingMode,
+    ) -> Tuple[datetime, datetime, int]:
+        """
+        Returns the min and max timestamp for the given pair's trades data.
+        :param pair: Pair to get min/max for
+        :param trading_mode: Trading mode to use (used to determine the filename)
+        :return: (min, max, len)
+        """
+        df = self._trades_load(pair, trading_mode)
+        if df.empty:
+            return (
+                datetime.fromtimestamp(0, tz=timezone.utc),
+                datetime.fromtimestamp(0, tz=timezone.utc),
+                0,
+            )
+        return (
+            to_datetime(df.iloc[0]["timestamp"], unit="ms", utc=True).to_pydatetime(),
+            to_datetime(df.iloc[-1]["timestamp"], unit="ms", utc=True).to_pydatetime(),
+            len(df),
+        )
+
     @classmethod
     def trades_get_pairs(cls, datadir: Path) -> List[str]:
         """

tests/commands/test_commands.py

@@ -1692,6 +1692,54 @@ def test_start_list_data(testdatadir, capsys):
     )
 
 
+def test_start_list_trades_data(testdatadir, capsys):
+    args = [
+        "list-data",
+        "--datadir",
+        str(testdatadir),
+        "--trades",
+    ]
+    pargs = get_args(args)
+    pargs["config"] = None
+    start_list_data(pargs)
+    captured = capsys.readouterr()
+    assert "Found trades data for 1 pair." in captured.out
+    assert re.search(r".*Pair.*Type.*\n", captured.out)
+    assert re.search(r"\n.* XRP/ETH .* spot |\n", captured.out)
+
+    args = [
+        "list-data",
+        "--datadir",
+        str(testdatadir),
+        "--trades",
+        "--show-timerange",
+    ]
+    pargs = get_args(args)
+    pargs["config"] = None
+    start_list_data(pargs)
+    captured = capsys.readouterr()
+    assert "Found trades data for 1 pair." in captured.out
+    assert re.search(r".*Pair.*Type.*From.*To.*Trades.*\n", captured.out)
+    assert re.search(
+        r"\n.* XRP/ETH .* spot .* 2019-10-11 00:00:01 .* 2019-10-13 11:19:28 .* 12477 .*|\n",
+        captured.out,
+    )
+
+    args = [
+        "list-data",
+        "--datadir",
+        str(testdatadir),
+        "--trades",
+        "--trading-mode",
+        "futures",
+    ]
+    pargs = get_args(args)
+    pargs["config"] = None
+    start_list_data(pargs)
+    captured = capsys.readouterr()
+    assert "Found trades data for 0 pairs." in captured.out
+
+
 @pytest.mark.usefixtures("init_persistence")
 def test_show_trades(mocker, fee, capsys, caplog):
     mocker.patch("freqtrade.persistence.init_db")

tests/data/test_datahandler.py

@@ -519,6 +519,39 @@ def test_datahandler_trades_purge(mocker, testdatadir, datahandler):
     assert unlinkmock.call_count == 1
 
 
+def test_datahandler_trades_get_available_data(testdatadir):
+    paircombs = FeatherDataHandler.trades_get_available_data(testdatadir, TradingMode.SPOT)
+    # Convert to set to avoid failures due to sorting
+    assert set(paircombs) == {"XRP/ETH"}
+
+    paircombs = FeatherDataHandler.trades_get_available_data(testdatadir, TradingMode.FUTURES)
+    # Convert to set to avoid failures due to sorting
+    assert set(paircombs) == set()
+
+    paircombs = JsonGzDataHandler.trades_get_available_data(testdatadir, TradingMode.SPOT)
+    assert set(paircombs) == {"XRP/ETH", "XRP/OLD"}
+    paircombs = HDF5DataHandler.trades_get_available_data(testdatadir, TradingMode.SPOT)
+    assert set(paircombs) == {"XRP/ETH"}
+
+
+def test_datahandler_trades_data_min_max(testdatadir):
+    dh = FeatherDataHandler(testdatadir)
+    min_max = dh.trades_data_min_max("XRP/ETH", TradingMode.SPOT)
+    assert len(min_max) == 3
+
+    # Empty pair
+    min_max = dh.trades_data_min_max("NADA/ETH", TradingMode.SPOT)
+    assert len(min_max) == 3
+    assert min_max[0] == datetime.fromtimestamp(0, tz=timezone.utc)
+    assert min_max[0] == min_max[1]
+
+    # Existing pair ...
+    min_max = dh.trades_data_min_max("XRP/ETH", TradingMode.SPOT)
+    assert len(min_max) == 3
+    assert min_max[0] == datetime(2019, 10, 11, 0, 0, 11, 620000, tzinfo=timezone.utc)
+    assert min_max[1] == datetime(2019, 10, 13, 11, 19, 28, 844000, tzinfo=timezone.utc)
+
+
 def test_gethandlerclass():
     cl = get_datahandlerclass("json")
     assert cl == JsonDataHandler