feat: Implement rich Progress to download-data

freqtrade/data/history/history_utils.py

@@ -5,6 +5,14 @@ from pathlib import Path
 from typing import Dict, List, Optional, Tuple
 
 from pandas import DataFrame, concat
+from rich.progress import (
+    BarColumn,
+    MofNCompleteColumn,
+    TaskProgressColumn,
+    TextColumn,
+    TimeElapsedColumn,
+    TimeRemainingColumn,
+)
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import (
@@ -29,6 +37,7 @@ from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
 from freqtrade.util import dt_ts, format_ms_time
 from freqtrade.util.datetime_helpers import dt_now
 from freqtrade.util.migrations import migrate_data
+from freqtrade.util.rich_progress import CustomProgress
 
 
 logger = logging.getLogger(__name__)
@@ -155,11 +164,9 @@ def refresh_data(
     :param candle_type: Any of the enum CandleType (must match trading mode!)
     """
     data_handler = get_datahandler(datadir, data_format)
-    for idx, pair in enumerate(pairs):
+    for pair in pairs:
-        process = f"{idx}/{len(pairs)}"
         _download_pair_history(
             pair=pair,
-            process=process,
             timeframe=timeframe,
             datadir=datadir,
             timerange=timerange,
@@ -223,7 +230,6 @@ def _download_pair_history(
     datadir: Path,
     exchange: Exchange,
     timeframe: str = "5m",
-    process: str = "",
     new_pairs_days: int = 30,
     data_handler: Optional[IDataHandler] = None,
     timerange: Optional[TimeRange] = None,
@@ -261,7 +267,7 @@ def _download_pair_history(
         )
 
         logger.info(
-            f'({process}) - Download history data for "{pair}", {timeframe}, '
+            f'Download history data for "{pair}", {timeframe}, '
             f"{candle_type} and store in {datadir}. "
             f'From {format_ms_time(since_ms) if since_ms else "start"} to '
             f'{format_ms_time(until_ms) if until_ms else "now"}'
@@ -345,53 +351,75 @@ def refresh_backtest_ohlcv_data(
     pairs_not_available = []
     data_handler = get_datahandler(datadir, data_format)
     candle_type = CandleType.get_default(trading_mode)
-    process = ""
+    with CustomProgress(
-    for idx, pair in enumerate(pairs, start=1):
+        TextColumn("[progress.description]{task.description}"),
-        if pair not in exchange.markets:
+        BarColumn(bar_width=None),
-            pairs_not_available.append(pair)
+        MofNCompleteColumn(),
-            logger.info(f"Skipping pair {pair}...")
+        TaskProgressColumn(),
-            continue
+        "•",
-        for timeframe in timeframes:
+        TimeElapsedColumn(),
-            logger.debug(f"Downloading pair {pair}, {candle_type}, interval {timeframe}.")
+        "•",
-            process = f"{idx}/{len(pairs)}"
+        TimeRemainingColumn(),
-            _download_pair_history(
+        expand=True,
-                pair=pair,
+    ) as progress:
-                process=process,
+        tf_length = len(timeframes) if trading_mode != "futures" else len(timeframes) + 2
-                datadir=datadir,
+        timeframe_task = progress.add_task("Timeframe", total=tf_length)
-                exchange=exchange,
+        pair_task = progress.add_task("Downloading data...", total=len(pairs))
-                timerange=timerange,
-                data_handler=data_handler,
-                timeframe=str(timeframe),
-                new_pairs_days=new_pairs_days,
-                candle_type=candle_type,
-                erase=erase,
-                prepend=prepend,
-            )
-        if trading_mode == "futures":
-            # Predefined candletype (and timeframe) depending on exchange
-            # Downloads what is necessary to backtest based on futures data.
-            tf_mark = exchange.get_option("mark_ohlcv_timeframe")
-            tf_funding_rate = exchange.get_option("funding_fee_timeframe")
 
-            fr_candle_type = CandleType.from_string(exchange.get_option("mark_ohlcv_price"))
+        for pair in pairs:
-            # All exchanges need FundingRate for futures trading.
+            progress.update(pair_task, description=f"Downloading {pair}")
-            # The timeframe is aligned to the mark-price timeframe.
+            progress.update(timeframe_task, completed=0)
-            combs = ((CandleType.FUNDING_RATE, tf_funding_rate), (fr_candle_type, tf_mark))
+
-            for candle_type_f, tf in combs:
+            if pair not in exchange.markets:
-                logger.debug(f"Downloading pair {pair}, {candle_type_f}, interval {tf}.")
+                pairs_not_available.append(pair)
+                logger.info(f"Skipping pair {pair}...")
+                continue
+            for timeframe in timeframes:
+                progress.update(timeframe_task, description=f"Timeframe {timeframe}")
+                logger.debug(f"Downloading pair {pair}, {candle_type}, interval {timeframe}.")
                 _download_pair_history(
                     pair=pair,
-                    process=process,
                     datadir=datadir,
                     exchange=exchange,
                     timerange=timerange,
                     data_handler=data_handler,
-                    timeframe=str(tf),
+                    timeframe=str(timeframe),
                     new_pairs_days=new_pairs_days,
-                    candle_type=candle_type_f,
+                    candle_type=candle_type,
                     erase=erase,
                     prepend=prepend,
                 )
+                progress.update(timeframe_task, advance=1)
+            if trading_mode == "futures":
+                # Predefined candletype (and timeframe) depending on exchange
+                # Downloads what is necessary to backtest based on futures data.
+                tf_mark = exchange.get_option("mark_ohlcv_timeframe")
+                tf_funding_rate = exchange.get_option("funding_fee_timeframe")
+
+                fr_candle_type = CandleType.from_string(exchange.get_option("mark_ohlcv_price"))
+                # All exchanges need FundingRate for futures trading.
+                # The timeframe is aligned to the mark-price timeframe.
+                combs = ((CandleType.FUNDING_RATE, tf_funding_rate), (fr_candle_type, tf_mark))
+                for candle_type_f, tf in combs:
+                    logger.debug(f"Downloading pair {pair}, {candle_type_f}, interval {tf}.")
+                    _download_pair_history(
+                        pair=pair,
+                        datadir=datadir,
+                        exchange=exchange,
+                        timerange=timerange,
+                        data_handler=data_handler,
+                        timeframe=str(tf),
+                        new_pairs_days=new_pairs_days,
+                        candle_type=candle_type_f,
+                        erase=erase,
+                        prepend=prepend,
+                    )
+                    progress.update(
+                        timeframe_task, advance=1, description=f"Timeframe {candle_type_f}, {tf}"
+                    )
+
+            progress.update(pair_task, advance=1)
+            progress.update(timeframe_task, description="Timeframe")
 
     return pairs_not_available
 

tests/data/test_history.py

@@ -151,9 +151,7 @@ def test_load_data_with_new_pair_1min(
     )
     load_pair_history(datadir=tmp_path, timeframe="1m", pair="MEME/BTC", candle_type=candle_type)
     assert file.is_file()
-    assert log_has_re(
+    assert log_has_re(r'Download history data for "MEME/BTC", 1m, ' r"spot and store in .*", caplog)
-        r'\(0/1\) - Download history data for "MEME/BTC", 1m, ' r"spot and store in .*", caplog
-    )
 
 
 def test_testdata_path(testdatadir) -> None: