diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index d4fd050eb..6d26b3ba9 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -1,18 +1,16 @@ import logging import sys from collections import defaultdict -from datetime import datetime, timedelta -from typing import Any, Dict, List +from typing import Any, Dict from freqtrade.configuration import TimeRange, setup_utils_configuration from freqtrade.constants import DATETIME_PRINT_FORMAT, Config from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format -from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data, - refresh_backtest_trades_data) +from freqtrade.data.history import convert_trades_to_ohlcv, download_data_main from freqtrade.enums import CandleType, RunMode, TradingMode from freqtrade.exceptions import OperationalException -from freqtrade.exchange import market_is_active, timeframe_to_minutes -from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist +from freqtrade.exchange import timeframe_to_minutes +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import ExchangeResolver from freqtrade.util.binance_mig import migrate_binance_futures_data @@ -38,77 +36,13 @@ def start_download_data(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) _check_data_config_download_sanity(config) - timerange = TimeRange() - if 'days' in config: - time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d") - timerange = TimeRange.parse_timerange(f'{time_since}-') - - if 'timerange' in config: - timerange = timerange.parse_timerange(config['timerange']) - - # Remove stake-currency to skip checks which are not relevant for datadownload - config['stake_currency'] = '' - - pairs_not_available: List[str] = [] - - # Init exchange - exchange = ExchangeResolver.load_exchange(config, validate=False) - markets = [p for p, m in exchange.markets.items() if market_is_active(m) - or config.get('include_inactive')] - - expanded_pairs = dynamic_expand_pairlist(config, markets) - - # Manual validations of relevant settings - if not config['exchange'].get('skip_pair_validation', False): - exchange.validate_pairs(expanded_pairs) - logger.info(f"About to download pairs: {expanded_pairs}, " - f"intervals: {config['timeframes']} to {config['datadir']}") - - for timeframe in config['timeframes']: - exchange.validate_timeframes(timeframe) try: - - if config.get('download_trades'): - if config.get('trading_mode') == 'futures': - raise OperationalException("Trade download not supported for futures.") - pairs_not_available = refresh_backtest_trades_data( - exchange, pairs=expanded_pairs, datadir=config['datadir'], - timerange=timerange, new_pairs_days=config['new_pairs_days'], - erase=bool(config.get('erase')), data_format=config['dataformat_trades']) - - # Convert downloaded trade data to different timeframes - convert_trades_to_ohlcv( - pairs=expanded_pairs, timeframes=config['timeframes'], - datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), - data_format_ohlcv=config['dataformat_ohlcv'], - data_format_trades=config['dataformat_trades'], - ) - else: - if not exchange.get_option('ohlcv_has_history', True): - raise OperationalException( - f"Historic klines not available for {exchange.name}. " - "Please use `--dl-trades` instead for this exchange " - "(will unfortunately take a long time)." - ) - migrate_binance_futures_data(config) - pairs_not_available = refresh_backtest_ohlcv_data( - exchange, pairs=expanded_pairs, timeframes=config['timeframes'], - datadir=config['datadir'], timerange=timerange, - new_pairs_days=config['new_pairs_days'], - erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'], - trading_mode=config.get('trading_mode', 'spot'), - prepend=config.get('prepend_data', False) - ) + download_data_main(config) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") - finally: - if pairs_not_available: - logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " - f"on exchange {exchange.name}.") - def start_convert_trades(args: Dict[str, Any]) -> None: diff --git a/freqtrade/data/history/__init__.py b/freqtrade/data/history/__init__.py index 107f9c401..414848c22 100644 --- a/freqtrade/data/history/__init__.py +++ b/freqtrade/data/history/__init__.py @@ -6,7 +6,7 @@ Includes: * download data from exchange and store to disk """ # flake8: noqa: F401 -from .history_utils import (convert_trades_to_ohlcv, get_timerange, load_data, load_pair_history, - refresh_backtest_ohlcv_data, refresh_backtest_trades_data, refresh_data, - validate_backtest_data) +from .history_utils import (convert_trades_to_ohlcv, download_data_main, get_timerange, load_data, + load_pair_history, refresh_backtest_ohlcv_data, + refresh_backtest_trades_data, refresh_data, validate_backtest_data) from .idatahandler import get_datahandler diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 7881130e2..b2b59337d 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -7,14 +7,16 @@ from typing import Dict, List, Optional, Tuple from pandas import DataFrame, concat from freqtrade.configuration import TimeRange -from freqtrade.constants import DATETIME_PRINT_FORMAT, DEFAULT_DATAFRAME_COLUMNS +from freqtrade.constants import DATETIME_PRINT_FORMAT, DEFAULT_DATAFRAME_COLUMNS, Config from freqtrade.data.converter import (clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_remove_duplicates, trades_to_ohlcv) from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException -from freqtrade.exchange import Exchange +from freqtrade.exchange import Exchange, market_is_active from freqtrade.misc import format_ms_time +from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist +from freqtrade.util.binance_mig import migrate_binance_futures_data logger = logging.getLogger(__name__) @@ -483,3 +485,74 @@ def validate_backtest_data(data: DataFrame, pair: str, min_date: datetime, logger.warning("%s has missing frames: expected %s, got %s, that's %s missing values", pair, expected_frames, dflen, expected_frames - dflen) return found_missing + + +def download_data_main(config: Config) -> None: + + timerange = TimeRange() + if 'days' in config: + time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d") + timerange = TimeRange.parse_timerange(f'{time_since}-') + + if 'timerange' in config: + timerange = timerange.parse_timerange(config['timerange']) + + # Remove stake-currency to skip checks which are not relevant for datadownload + config['stake_currency'] = '' + + pairs_not_available: List[str] = [] + + # Init exchange + from freqtrade.resolvers.exchange_resolver import ExchangeResolver + exchange = ExchangeResolver.load_exchange(config, validate=False) + markets = [p for p, m in exchange.markets.items() if market_is_active(m) + or config.get('include_inactive')] + + expanded_pairs = dynamic_expand_pairlist(config, markets) + + # Manual validations of relevant settings + if not config['exchange'].get('skip_pair_validation', False): + exchange.validate_pairs(expanded_pairs) + logger.info(f"About to download pairs: {expanded_pairs}, " + f"intervals: {config['timeframes']} to {config['datadir']}") + + for timeframe in config['timeframes']: + exchange.validate_timeframes(timeframe) + + # Start downloading + try: + if config.get('download_trades'): + if config.get('trading_mode') == 'futures': + raise OperationalException("Trade download not supported for futures.") + pairs_not_available = refresh_backtest_trades_data( + exchange, pairs=expanded_pairs, datadir=config['datadir'], + timerange=timerange, new_pairs_days=config['new_pairs_days'], + erase=bool(config.get('erase')), data_format=config['dataformat_trades']) + + # Convert downloaded trade data to different timeframes + convert_trades_to_ohlcv( + pairs=expanded_pairs, timeframes=config['timeframes'], + datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), + data_format_ohlcv=config['dataformat_ohlcv'], + data_format_trades=config['dataformat_trades'], + ) + else: + if not exchange.get_option('ohlcv_has_history', True): + raise OperationalException( + f"Historic klines not available for {exchange.name}. " + "Please use `--dl-trades` instead for this exchange " + "(will unfortunately take a long time)." + ) + migrate_binance_futures_data(config) + pairs_not_available = refresh_backtest_ohlcv_data( + exchange, pairs=expanded_pairs, timeframes=config['timeframes'], + datadir=config['datadir'], timerange=timerange, + new_pairs_days=config['new_pairs_days'], + erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'], + trading_mode=config.get('trading_mode', 'spot'), + prepend=config.get('prepend_data', False) + ) + finally: + if pairs_not_available: + logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " + f"on exchange {exchange.name}.")