Merge pull request #10420 from freqtrade/feat/dl_data_progress

Download Data - progressbar

freqtrade/data/history/history_utils.py

@@ -26,8 +26,7 @@ from freqtrade.enums import CandleType, TradingMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import Exchange
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
-from freqtrade.util import dt_ts, format_ms_time
+from freqtrade.util import dt_now, dt_ts, format_ms_time, get_progress_tracker
-from freqtrade.util.datetime_helpers import dt_now
 from freqtrade.util.migrations import migrate_data
 
 
@@ -155,11 +154,9 @@ def refresh_data(
     :param candle_type: Any of the enum CandleType (must match trading mode!)
     """
     data_handler = get_datahandler(datadir, data_format)
-    for idx, pair in enumerate(pairs):
+    for pair in pairs:
-        process = f"{idx}/{len(pairs)}"
         _download_pair_history(
             pair=pair,
-            process=process,
             timeframe=timeframe,
             datadir=datadir,
             timerange=timerange,
@@ -223,7 +220,6 @@ def _download_pair_history(
     datadir: Path,
     exchange: Exchange,
     timeframe: str = "5m",
-    process: str = "",
     new_pairs_days: int = 30,
     data_handler: Optional[IDataHandler] = None,
     timerange: Optional[TimeRange] = None,
@@ -261,7 +257,7 @@ def _download_pair_history(
         )
 
         logger.info(
-            f'({process}) - Download history data for "{pair}", {timeframe}, '
+            f'Download history data for "{pair}", {timeframe}, '
             f"{candle_type} and store in {datadir}. "
             f'From {format_ms_time(since_ms) if since_ms else "start"} to '
             f'{format_ms_time(until_ms) if until_ms else "now"}'
@@ -345,53 +341,65 @@ def refresh_backtest_ohlcv_data(
     pairs_not_available = []
     data_handler = get_datahandler(datadir, data_format)
     candle_type = CandleType.get_default(trading_mode)
-    process = ""
+    with get_progress_tracker() as progress:
-    for idx, pair in enumerate(pairs, start=1):
+        tf_length = len(timeframes) if trading_mode != "futures" else len(timeframes) + 2
-        if pair not in exchange.markets:
+        timeframe_task = progress.add_task("Timeframe", total=tf_length)
-            pairs_not_available.append(pair)
+        pair_task = progress.add_task("Downloading data...", total=len(pairs))
-            logger.info(f"Skipping pair {pair}...")
-            continue
-        for timeframe in timeframes:
-            logger.debug(f"Downloading pair {pair}, {candle_type}, interval {timeframe}.")
-            process = f"{idx}/{len(pairs)}"
-            _download_pair_history(
-                pair=pair,
-                process=process,
-                datadir=datadir,
-                exchange=exchange,
-                timerange=timerange,
-                data_handler=data_handler,
-                timeframe=str(timeframe),
-                new_pairs_days=new_pairs_days,
-                candle_type=candle_type,
-                erase=erase,
-                prepend=prepend,
-            )
-        if trading_mode == "futures":
-            # Predefined candletype (and timeframe) depending on exchange
-            # Downloads what is necessary to backtest based on futures data.
-            tf_mark = exchange.get_option("mark_ohlcv_timeframe")
-            tf_funding_rate = exchange.get_option("funding_fee_timeframe")
 
-            fr_candle_type = CandleType.from_string(exchange.get_option("mark_ohlcv_price"))
+        for pair in pairs:
-            # All exchanges need FundingRate for futures trading.
+            progress.update(pair_task, description=f"Downloading {pair}")
-            # The timeframe is aligned to the mark-price timeframe.
+            progress.update(timeframe_task, completed=0)
-            combs = ((CandleType.FUNDING_RATE, tf_funding_rate), (fr_candle_type, tf_mark))
+
-            for candle_type_f, tf in combs:
+            if pair not in exchange.markets:
-                logger.debug(f"Downloading pair {pair}, {candle_type_f}, interval {tf}.")
+                pairs_not_available.append(pair)
+                logger.info(f"Skipping pair {pair}...")
+                continue
+            for timeframe in timeframes:
+                progress.update(timeframe_task, description=f"Timeframe {timeframe}")
+                logger.debug(f"Downloading pair {pair}, {candle_type}, interval {timeframe}.")
                 _download_pair_history(
                     pair=pair,
-                    process=process,
                     datadir=datadir,
                     exchange=exchange,
                     timerange=timerange,
                     data_handler=data_handler,
-                    timeframe=str(tf),
+                    timeframe=str(timeframe),
                     new_pairs_days=new_pairs_days,
-                    candle_type=candle_type_f,
+                    candle_type=candle_type,
                     erase=erase,
                     prepend=prepend,
                 )
+                progress.update(timeframe_task, advance=1)
+            if trading_mode == "futures":
+                # Predefined candletype (and timeframe) depending on exchange
+                # Downloads what is necessary to backtest based on futures data.
+                tf_mark = exchange.get_option("mark_ohlcv_timeframe")
+                tf_funding_rate = exchange.get_option("funding_fee_timeframe")
+
+                fr_candle_type = CandleType.from_string(exchange.get_option("mark_ohlcv_price"))
+                # All exchanges need FundingRate for futures trading.
+                # The timeframe is aligned to the mark-price timeframe.
+                combs = ((CandleType.FUNDING_RATE, tf_funding_rate), (fr_candle_type, tf_mark))
+                for candle_type_f, tf in combs:
+                    logger.debug(f"Downloading pair {pair}, {candle_type_f}, interval {tf}.")
+                    _download_pair_history(
+                        pair=pair,
+                        datadir=datadir,
+                        exchange=exchange,
+                        timerange=timerange,
+                        data_handler=data_handler,
+                        timeframe=str(tf),
+                        new_pairs_days=new_pairs_days,
+                        candle_type=candle_type_f,
+                        erase=erase,
+                        prepend=prepend,
+                    )
+                    progress.update(
+                        timeframe_task, advance=1, description=f"Timeframe {candle_type_f}, {tf}"
+                    )
+
+            progress.update(pair_task, advance=1)
+            progress.update(timeframe_task, description="Timeframe")
 
     return pairs_not_available
 
@@ -501,25 +509,30 @@ def refresh_backtest_trades_data(
     """
     pairs_not_available = []
     data_handler = get_datahandler(datadir, data_format=data_format)
-    for pair in pairs:
+    with get_progress_tracker() as progress:
-        if pair not in exchange.markets:
+        pair_task = progress.add_task("Downloading data...", total=len(pairs))
-            pairs_not_available.append(pair)
+        for pair in pairs:
-            logger.info(f"Skipping pair {pair}...")
+            progress.update(pair_task, description=f"Downloading trades [{pair}]")
-            continue
+            if pair not in exchange.markets:
+                pairs_not_available.append(pair)
+                logger.info(f"Skipping pair {pair}...")
+                continue
 
-        if erase:
+            if erase:
-            if data_handler.trades_purge(pair, trading_mode):
+                if data_handler.trades_purge(pair, trading_mode):
-                logger.info(f"Deleting existing data for pair {pair}.")
+                    logger.info(f"Deleting existing data for pair {pair}.")
+
+            logger.info(f"Downloading trades for pair {pair}.")
+            _download_trades_history(
+                exchange=exchange,
+                pair=pair,
+                new_pairs_days=new_pairs_days,
+                timerange=timerange,
+                data_handler=data_handler,
+                trading_mode=trading_mode,
+            )
+            progress.update(pair_task, advance=1)
 
-        logger.info(f"Downloading trades for pair {pair}.")
-        _download_trades_history(
-            exchange=exchange,
-            pair=pair,
-            new_pairs_days=new_pairs_days,
-            timerange=timerange,
-            data_handler=data_handler,
-            trading_mode=trading_mode,
-        )
     return pairs_not_available
 
 

freqtrade/optimize/hyperopt.py

@@ -19,14 +19,6 @@ from joblib.externals import cloudpickle
 from pandas import DataFrame
 from rich.align import Align
 from rich.console import Console
-from rich.progress import (
-    BarColumn,
-    MofNCompleteColumn,
-    TaskProgressColumn,
-    TextColumn,
-    TimeElapsedColumn,
-    TimeRemainingColumn,
-)
 
 from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN, Config
 from freqtrade.data.converter import trim_dataframes
@@ -48,7 +40,7 @@ from freqtrade.optimize.hyperopt_tools import (
 )
 from freqtrade.optimize.optimize_reports import generate_strategy_stats
 from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver
-from freqtrade.util import CustomProgress
+from freqtrade.util import get_progress_tracker
 
 
 # Suppress scikit-learn FutureWarnings from skopt
@@ -634,16 +626,7 @@ class Hyperopt:
                 )
 
                 # Define progressbar
-                with CustomProgress(
+                with get_progress_tracker(
-                    TextColumn("[progress.description]{task.description}"),
-                    BarColumn(bar_width=None),
-                    MofNCompleteColumn(),
-                    TaskProgressColumn(),
-                    "•",
-                    TimeElapsedColumn(),
-                    "•",
-                    TimeRemainingColumn(),
-                    expand=True,
                     console=console,
                     cust_objs=[Align.center(self._hyper_out.table)],
                 ) as pbar:

freqtrade/util/__init__.py

@@ -15,6 +15,7 @@ from freqtrade.util.formatters import decimals_per_coin, fmt_coin, round_value
 from freqtrade.util.ft_precise import FtPrecise
 from freqtrade.util.measure_time import MeasureTime
 from freqtrade.util.periodic_cache import PeriodicCache
+from freqtrade.util.progress_tracker import get_progress_tracker  # noqa F401
 from freqtrade.util.rich_progress import CustomProgress
 from freqtrade.util.rich_tables import print_df_rich_table, print_rich_table
 from freqtrade.util.template_renderer import render_template, render_template_with_fallback  # noqa

freqtrade/util/progress_tracker.py

@@ -0,0 +1,28 @@
+from rich.progress import (
+    BarColumn,
+    MofNCompleteColumn,
+    TaskProgressColumn,
+    TextColumn,
+    TimeElapsedColumn,
+    TimeRemainingColumn,
+)
+
+from freqtrade.util.rich_progress import CustomProgress
+
+
+def get_progress_tracker(**kwargs):
+    """
+    Get progress Bar with custom columns.
+    """
+    return CustomProgress(
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(bar_width=None),
+        MofNCompleteColumn(),
+        TaskProgressColumn(),
+        "•",
+        TimeElapsedColumn(),
+        "•",
+        TimeRemainingColumn(),
+        expand=True,
+        **kwargs,
+    )

freqtrade/util/rich_progress.py

@@ -5,7 +5,7 @@ from rich.progress import Progress
 
 
 class CustomProgress(Progress):
-    def __init__(self, *args, cust_objs, **kwargs) -> None:
+    def __init__(self, *args, cust_objs=[], **kwargs) -> None:
         self._cust_objs = cust_objs
         super().__init__(*args, **kwargs)
 

tests/data/test_history.py

@@ -151,9 +151,7 @@ def test_load_data_with_new_pair_1min(
     )
     load_pair_history(datadir=tmp_path, timeframe="1m", pair="MEME/BTC", candle_type=candle_type)
     assert file.is_file()
-    assert log_has_re(
+    assert log_has_re(r'Download history data for "MEME/BTC", 1m, ' r"spot and store in .*", caplog)
-        r'\(0/1\) - Download history data for "MEME/BTC", 1m, ' r"spot and store in .*", caplog
-    )
 
 
 def test_testdata_path(testdatadir) -> None: