Migrate download-data out of commands section

freqtrade/commands/data_commands.py

@@ -1,18 +1,16 @@
 import logging
 import sys
 from collections import defaultdict
-from datetime import datetime, timedelta
+from typing import Any, Dict
-from typing import Any, Dict, List
 
 from freqtrade.configuration import TimeRange, setup_utils_configuration
 from freqtrade.constants import DATETIME_PRINT_FORMAT, Config
 from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format
-from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data,
+from freqtrade.data.history import convert_trades_to_ohlcv, download_data_main
-                                    refresh_backtest_trades_data)
 from freqtrade.enums import CandleType, RunMode, TradingMode
 from freqtrade.exceptions import OperationalException
-from freqtrade.exchange import market_is_active, timeframe_to_minutes
+from freqtrade.exchange import timeframe_to_minutes
-from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist
+from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
 from freqtrade.util.binance_mig import migrate_binance_futures_data
 
@@ -38,77 +36,13 @@ def start_download_data(args: Dict[str, Any]) -> None:
     config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
 
     _check_data_config_download_sanity(config)
-    timerange = TimeRange()
-    if 'days' in config:
-        time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d")
-        timerange = TimeRange.parse_timerange(f'{time_since}-')
-
-    if 'timerange' in config:
-        timerange = timerange.parse_timerange(config['timerange'])
-
-    # Remove stake-currency to skip checks which are not relevant for datadownload
-    config['stake_currency'] = ''
-
-    pairs_not_available: List[str] = []
-
-    # Init exchange
-    exchange = ExchangeResolver.load_exchange(config, validate=False)
-    markets = [p for p, m in exchange.markets.items() if market_is_active(m)
-               or config.get('include_inactive')]
-
-    expanded_pairs = dynamic_expand_pairlist(config, markets)
-
-    # Manual validations of relevant settings
-    if not config['exchange'].get('skip_pair_validation', False):
-        exchange.validate_pairs(expanded_pairs)
-    logger.info(f"About to download pairs: {expanded_pairs}, "
-                f"intervals: {config['timeframes']} to {config['datadir']}")
-
-    for timeframe in config['timeframes']:
-        exchange.validate_timeframes(timeframe)
 
     try:
-
+        download_data_main(config)
-        if config.get('download_trades'):
-            if config.get('trading_mode') == 'futures':
-                raise OperationalException("Trade download not supported for futures.")
-            pairs_not_available = refresh_backtest_trades_data(
-                exchange, pairs=expanded_pairs, datadir=config['datadir'],
-                timerange=timerange, new_pairs_days=config['new_pairs_days'],
-                erase=bool(config.get('erase')), data_format=config['dataformat_trades'])
-
-            # Convert downloaded trade data to different timeframes
-            convert_trades_to_ohlcv(
-                pairs=expanded_pairs, timeframes=config['timeframes'],
-                datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
-                data_format_ohlcv=config['dataformat_ohlcv'],
-                data_format_trades=config['dataformat_trades'],
-            )
-        else:
-            if not exchange.get_option('ohlcv_has_history', True):
-                raise OperationalException(
-                    f"Historic klines not available for {exchange.name}. "
-                    "Please use `--dl-trades` instead for this exchange "
-                    "(will unfortunately take a long time)."
-                    )
-            migrate_binance_futures_data(config)
-            pairs_not_available = refresh_backtest_ohlcv_data(
-                exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
-                datadir=config['datadir'], timerange=timerange,
-                new_pairs_days=config['new_pairs_days'],
-                erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'],
-                trading_mode=config.get('trading_mode', 'spot'),
-                prepend=config.get('prepend_data', False)
-            )
 
     except KeyboardInterrupt:
         sys.exit("SIGINT received, aborting ...")
 
-    finally:
-        if pairs_not_available:
-            logger.info(f"Pairs [{','.join(pairs_not_available)}] not available "
-                        f"on exchange {exchange.name}.")
-
 
 def start_convert_trades(args: Dict[str, Any]) -> None:
 

freqtrade/data/history/__init__.py

@@ -6,7 +6,7 @@ Includes:
 * download data from exchange and store to disk
 """
 # flake8: noqa: F401
-from .history_utils import (convert_trades_to_ohlcv, get_timerange, load_data, load_pair_history,
+from .history_utils import (convert_trades_to_ohlcv, download_data_main, get_timerange, load_data,
-                            refresh_backtest_ohlcv_data, refresh_backtest_trades_data, refresh_data,
+                            load_pair_history, refresh_backtest_ohlcv_data,
-                            validate_backtest_data)
+                            refresh_backtest_trades_data, refresh_data, validate_backtest_data)
 from .idatahandler import get_datahandler

freqtrade/data/history/history_utils.py

@@ -7,14 +7,16 @@ from typing import Dict, List, Optional, Tuple
 from pandas import DataFrame, concat
 
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import DATETIME_PRINT_FORMAT, DEFAULT_DATAFRAME_COLUMNS
+from freqtrade.constants import DATETIME_PRINT_FORMAT, DEFAULT_DATAFRAME_COLUMNS, Config
 from freqtrade.data.converter import (clean_ohlcv_dataframe, ohlcv_to_dataframe,
                                       trades_remove_duplicates, trades_to_ohlcv)
 from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
 from freqtrade.enums import CandleType
 from freqtrade.exceptions import OperationalException
-from freqtrade.exchange import Exchange
+from freqtrade.exchange import Exchange, market_is_active
 from freqtrade.misc import format_ms_time
+from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
+from freqtrade.util.binance_mig import migrate_binance_futures_data
 
 
 logger = logging.getLogger(__name__)
@@ -483,3 +485,74 @@ def validate_backtest_data(data: DataFrame, pair: str, min_date: datetime,
         logger.warning("%s has missing frames: expected %s, got %s, that's %s missing values",
                        pair, expected_frames, dflen, expected_frames - dflen)
     return found_missing
+
+
+def download_data_main(config: Config) -> None:
+
+    timerange = TimeRange()
+    if 'days' in config:
+        time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d")
+        timerange = TimeRange.parse_timerange(f'{time_since}-')
+
+    if 'timerange' in config:
+        timerange = timerange.parse_timerange(config['timerange'])
+
+    # Remove stake-currency to skip checks which are not relevant for datadownload
+    config['stake_currency'] = ''
+
+    pairs_not_available: List[str] = []
+
+    # Init exchange
+    from freqtrade.resolvers.exchange_resolver import ExchangeResolver
+    exchange = ExchangeResolver.load_exchange(config, validate=False)
+    markets = [p for p, m in exchange.markets.items() if market_is_active(m)
+               or config.get('include_inactive')]
+
+    expanded_pairs = dynamic_expand_pairlist(config, markets)
+
+    # Manual validations of relevant settings
+    if not config['exchange'].get('skip_pair_validation', False):
+        exchange.validate_pairs(expanded_pairs)
+    logger.info(f"About to download pairs: {expanded_pairs}, "
+                f"intervals: {config['timeframes']} to {config['datadir']}")
+
+    for timeframe in config['timeframes']:
+        exchange.validate_timeframes(timeframe)
+
+    # Start downloading
+    try:
+        if config.get('download_trades'):
+            if config.get('trading_mode') == 'futures':
+                raise OperationalException("Trade download not supported for futures.")
+            pairs_not_available = refresh_backtest_trades_data(
+                exchange, pairs=expanded_pairs, datadir=config['datadir'],
+                timerange=timerange, new_pairs_days=config['new_pairs_days'],
+                erase=bool(config.get('erase')), data_format=config['dataformat_trades'])
+
+            # Convert downloaded trade data to different timeframes
+            convert_trades_to_ohlcv(
+                pairs=expanded_pairs, timeframes=config['timeframes'],
+                datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
+                data_format_ohlcv=config['dataformat_ohlcv'],
+                data_format_trades=config['dataformat_trades'],
+            )
+        else:
+            if not exchange.get_option('ohlcv_has_history', True):
+                raise OperationalException(
+                    f"Historic klines not available for {exchange.name}. "
+                    "Please use `--dl-trades` instead for this exchange "
+                    "(will unfortunately take a long time)."
+                    )
+            migrate_binance_futures_data(config)
+            pairs_not_available = refresh_backtest_ohlcv_data(
+                exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
+                datadir=config['datadir'], timerange=timerange,
+                new_pairs_days=config['new_pairs_days'],
+                erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'],
+                trading_mode=config.get('trading_mode', 'spot'),
+                prepend=config.get('prepend_data', False)
+            )
+    finally:
+        if pairs_not_available:
+            logger.info(f"Pairs [{','.join(pairs_not_available)}] not available "
+                        f"on exchange {exchange.name}.")