Merge pull request #10420 from freqtrade/feat/dl_data_progress

Download Data - progressbar

freqtrade/data/history/history_utils.py

@@ -26,8 +26,7 @@ from freqtrade.enums import CandleType, TradingMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import Exchange
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
-from freqtrade.util import dt_ts, format_ms_time
+from freqtrade.util import dt_now, dt_ts, format_ms_time, get_progress_tracker
-from freqtrade.util.datetime_helpers import dt_now
 from freqtrade.util.migrations import migrate_data
 
 
@@ -155,11 +154,9 @@ def refresh_data(
     :param candle_type: Any of the enum CandleType (must match trading mode!)
     """
     data_handler = get_datahandler(datadir, data_format)
-    for idx, pair in enumerate(pairs):
+    for pair in pairs:
-        process = f"{idx}/{len(pairs)}"
         _download_pair_history(
             pair=pair,
-            process=process,
             timeframe=timeframe,
             datadir=datadir,
             timerange=timerange,
@@ -223,7 +220,6 @@ def _download_pair_history(
     datadir: Path,
     exchange: Exchange,
     timeframe: str = "5m",
-    process: str = "",
     new_pairs_days: int = 30,
     data_handler: Optional[IDataHandler] = None,
     timerange: Optional[TimeRange] = None,
@@ -261,7 +257,7 @@ def _download_pair_history(
         )
 
         logger.info(
-            f'({process}) - Download history data for "{pair}", {timeframe}, '
+            f'Download history data for "{pair}", {timeframe}, '
             f"{candle_type} and store in {datadir}. "
             f'From {format_ms_time(since_ms) if since_ms else "start"} to '
             f'{format_ms_time(until_ms) if until_ms else "now"}'
@@ -345,18 +341,24 @@ def refresh_backtest_ohlcv_data(
     pairs_not_available = []
     data_handler = get_datahandler(datadir, data_format)
     candle_type = CandleType.get_default(trading_mode)
-    process = ""
+    with get_progress_tracker() as progress:
-    for idx, pair in enumerate(pairs, start=1):
+        tf_length = len(timeframes) if trading_mode != "futures" else len(timeframes) + 2
+        timeframe_task = progress.add_task("Timeframe", total=tf_length)
+        pair_task = progress.add_task("Downloading data...", total=len(pairs))
+
+        for pair in pairs:
+            progress.update(pair_task, description=f"Downloading {pair}")
+            progress.update(timeframe_task, completed=0)
+
             if pair not in exchange.markets:
                 pairs_not_available.append(pair)
                 logger.info(f"Skipping pair {pair}...")
                 continue
             for timeframe in timeframes:
+                progress.update(timeframe_task, description=f"Timeframe {timeframe}")
                 logger.debug(f"Downloading pair {pair}, {candle_type}, interval {timeframe}.")
-            process = f"{idx}/{len(pairs)}"
                 _download_pair_history(
                     pair=pair,
-                process=process,
                     datadir=datadir,
                     exchange=exchange,
                     timerange=timerange,
@@ -367,6 +369,7 @@ def refresh_backtest_ohlcv_data(
                     erase=erase,
                     prepend=prepend,
                 )
+                progress.update(timeframe_task, advance=1)
             if trading_mode == "futures":
                 # Predefined candletype (and timeframe) depending on exchange
                 # Downloads what is necessary to backtest based on futures data.
@@ -381,7 +384,6 @@ def refresh_backtest_ohlcv_data(
                     logger.debug(f"Downloading pair {pair}, {candle_type_f}, interval {tf}.")
                     _download_pair_history(
                         pair=pair,
-                    process=process,
                         datadir=datadir,
                         exchange=exchange,
                         timerange=timerange,
@@ -392,6 +394,12 @@ def refresh_backtest_ohlcv_data(
                         erase=erase,
                         prepend=prepend,
                     )
+                    progress.update(
+                        timeframe_task, advance=1, description=f"Timeframe {candle_type_f}, {tf}"
+                    )
+
+            progress.update(pair_task, advance=1)
+            progress.update(timeframe_task, description="Timeframe")
 
     return pairs_not_available
 
@@ -501,7 +509,10 @@ def refresh_backtest_trades_data(
     """
     pairs_not_available = []
     data_handler = get_datahandler(datadir, data_format=data_format)
+    with get_progress_tracker() as progress:
+        pair_task = progress.add_task("Downloading data...", total=len(pairs))
         for pair in pairs:
+            progress.update(pair_task, description=f"Downloading trades [{pair}]")
             if pair not in exchange.markets:
                 pairs_not_available.append(pair)
                 logger.info(f"Skipping pair {pair}...")
@@ -520,6 +531,8 @@ def refresh_backtest_trades_data(
                 data_handler=data_handler,
                 trading_mode=trading_mode,
             )
+            progress.update(pair_task, advance=1)
+
     return pairs_not_available
 
 

freqtrade/optimize/hyperopt.py

@@ -19,14 +19,6 @@ from joblib.externals import cloudpickle
 from pandas import DataFrame
 from rich.align import Align
 from rich.console import Console
-from rich.progress import (
-    BarColumn,
-    MofNCompleteColumn,
-    TaskProgressColumn,
-    TextColumn,
-    TimeElapsedColumn,
-    TimeRemainingColumn,
-)
 
 from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN, Config
 from freqtrade.data.converter import trim_dataframes
@@ -48,7 +40,7 @@ from freqtrade.optimize.hyperopt_tools import (
 )
 from freqtrade.optimize.optimize_reports import generate_strategy_stats
 from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver
-from freqtrade.util import CustomProgress
+from freqtrade.util import get_progress_tracker
 
 
 # Suppress scikit-learn FutureWarnings from skopt
@@ -634,16 +626,7 @@ class Hyperopt:
                 )
 
                 # Define progressbar
-                with CustomProgress(
+                with get_progress_tracker(
-                    TextColumn("[progress.description]{task.description}"),
-                    BarColumn(bar_width=None),
-                    MofNCompleteColumn(),
-                    TaskProgressColumn(),
-                    "•",
-                    TimeElapsedColumn(),
-                    "•",
-                    TimeRemainingColumn(),
-                    expand=True,
                     console=console,
                     cust_objs=[Align.center(self._hyper_out.table)],
                 ) as pbar:

freqtrade/util/__init__.py

@@ -15,6 +15,7 @@ from freqtrade.util.formatters import decimals_per_coin, fmt_coin, round_value
 from freqtrade.util.ft_precise import FtPrecise
 from freqtrade.util.measure_time import MeasureTime
 from freqtrade.util.periodic_cache import PeriodicCache
+from freqtrade.util.progress_tracker import get_progress_tracker  # noqa F401
 from freqtrade.util.rich_progress import CustomProgress
 from freqtrade.util.rich_tables import print_df_rich_table, print_rich_table
 from freqtrade.util.template_renderer import render_template, render_template_with_fallback  # noqa

freqtrade/util/progress_tracker.py

@@ -0,0 +1,28 @@
+from rich.progress import (
+    BarColumn,
+    MofNCompleteColumn,
+    TaskProgressColumn,
+    TextColumn,
+    TimeElapsedColumn,
+    TimeRemainingColumn,
+)
+
+from freqtrade.util.rich_progress import CustomProgress
+
+
+def get_progress_tracker(**kwargs):
+    """
+    Get progress Bar with custom columns.
+    """
+    return CustomProgress(
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(bar_width=None),
+        MofNCompleteColumn(),
+        TaskProgressColumn(),
+        "•",
+        TimeElapsedColumn(),
+        "•",
+        TimeRemainingColumn(),
+        expand=True,
+        **kwargs,
+    )

freqtrade/util/rich_progress.py

@@ -5,7 +5,7 @@ from rich.progress import Progress
 
 
 class CustomProgress(Progress):
-    def __init__(self, *args, cust_objs, **kwargs) -> None:
+    def __init__(self, *args, cust_objs=[], **kwargs) -> None:
         self._cust_objs = cust_objs
         super().__init__(*args, **kwargs)
 

tests/data/test_history.py

@@ -151,9 +151,7 @@ def test_load_data_with_new_pair_1min(
     )
     load_pair_history(datadir=tmp_path, timeframe="1m", pair="MEME/BTC", candle_type=candle_type)
     assert file.is_file()
-    assert log_has_re(
+    assert log_has_re(r'Download history data for "MEME/BTC", 1m, ' r"spot and store in .*", caplog)
-        r'\(0/1\) - Download history data for "MEME/BTC", 1m, ' r"spot and store in .*", caplog
-    )
 
 
 def test_testdata_path(testdatadir) -> None: