diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 5341009f2..fdfef6316 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -7,7 +7,7 @@ from freqtrade.configuration import TimeRange, setup_utils_configuration from freqtrade.constants import DATETIME_PRINT_FORMAT, DL_DATA_TIMEFRAMES, Config from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format from freqtrade.data.history import convert_trades_to_ohlcv, download_data_main -from freqtrade.enums import CandleType, RunMode, TradingMode +from freqtrade.enums import RunMode, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist @@ -88,11 +88,10 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) if ohlcv: migrate_binance_futures_data(config) - candle_types = [CandleType.from_string(ct) for ct in config.get('candle_types', ['spot'])] - for candle_type in candle_types: - convert_ohlcv_format(config, - convert_from=args['format_from'], convert_to=args['format_to'], - erase=args['erase'], candle_type=candle_type) + convert_ohlcv_format(config, + convert_from=args['format_from'], + convert_to=args['format_to'], + erase=args['erase']) else: convert_trades_format(config, convert_from=args['format_from'], convert_to=args['format_to'], diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 2d3855d87..4b4c89d08 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -11,7 +11,7 @@ import pandas as pd from pandas import DataFrame, to_datetime from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, Config, TradeList -from freqtrade.enums import CandleType +from freqtrade.enums import CandleType, TradingMode logger = logging.getLogger(__name__) @@ -264,7 +264,6 @@ def convert_ohlcv_format( convert_from: str, convert_to: str, erase: bool, - candle_type: CandleType ): """ Convert OHLCV from one format to another @@ -272,7 +271,6 @@ def convert_ohlcv_format( :param convert_from: Source format :param convert_to: Target format :param erase: Erase source data (does not apply if source and target format are identical) - :param candle_type: Any of the enum CandleType (must match trading mode!) """ from freqtrade.data.history.idatahandler import get_datahandler src = get_datahandler(config['datadir'], convert_from) @@ -280,37 +278,45 @@ def convert_ohlcv_format( timeframes = config.get('timeframes', [config.get('timeframe')]) logger.info(f"Converting candle (OHLCV) for timeframe {timeframes}") - if 'pairs' not in config: - config['pairs'] = [] - # Check timeframes or fall back to timeframe. - for timeframe in timeframes: - config['pairs'].extend(src.ohlcv_get_pairs( - config['datadir'], - timeframe, - candle_type=candle_type - )) - config['pairs'] = sorted(set(config['pairs'])) - logger.info(f"Converting candle (OHLCV) data for {config['pairs']}") + candle_types = [CandleType.from_string(ct) for ct in config.get('candle_types', [ + c.value for c in CandleType])] + logger.info(candle_types) + paircombs = src.ohlcv_get_available_data(config['datadir'], TradingMode.SPOT) + paircombs.extend(src.ohlcv_get_available_data(config['datadir'], TradingMode.FUTURES)) - for timeframe in timeframes: - for pair in config['pairs']: - data = src.ohlcv_load(pair=pair, timeframe=timeframe, - timerange=None, - fill_missing=False, - drop_incomplete=False, - startup_candles=0, - candle_type=candle_type) - logger.info(f"Converting {len(data)} {timeframe} {candle_type} candles for {pair}") - if len(data) > 0: - trg.ohlcv_store( - pair=pair, - timeframe=timeframe, - data=data, - candle_type=candle_type - ) - if erase and convert_from != convert_to: - logger.info(f"Deleting source data for {pair} / {timeframe}") - src.ohlcv_purge(pair=pair, timeframe=timeframe, candle_type=candle_type) + if 'pairs' in config: + # Filter pairs + paircombs = [comb for comb in paircombs if comb[0] in config['pairs']] + + if 'timeframes' in config: + paircombs = [comb for comb in paircombs if comb[1] in config['timeframes']] + paircombs = [comb for comb in paircombs if comb[2] in candle_types] + + paircombs = sorted(paircombs, key=lambda x: (x[0], x[1], x[2].value)) + + formatted_paircombs = '\n'.join([f"{pair}, {timeframe}, {candle_type}" + for pair, timeframe, candle_type in paircombs]) + + logger.info(f"Converting candle (OHLCV) data for the following pair combinations:\n" + f"{formatted_paircombs}") + for pair, timeframe, candle_type in paircombs: + data = src.ohlcv_load(pair=pair, timeframe=timeframe, + timerange=None, + fill_missing=False, + drop_incomplete=False, + startup_candles=0, + candle_type=candle_type) + logger.info(f"Converting {len(data)} {timeframe} {candle_type} candles for {pair}") + if len(data) > 0: + trg.ohlcv_store( + pair=pair, + timeframe=timeframe, + data=data, + candle_type=candle_type + ) + if erase and convert_from != convert_to: + logger.info(f"Deleting source data for {pair} / {timeframe}") + src.ohlcv_purge(pair=pair, timeframe=timeframe, candle_type=candle_type) def reduce_dataframe_footprint(df: DataFrame) -> DataFrame: diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index b37a2100d..da1459ef7 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -315,6 +315,8 @@ def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base, cand files_new.append(file_new) default_conf['datadir'] = tmpdir1 + default_conf['candle_types'] = [candletype] + if candletype == CandleType.SPOT: default_conf['pairs'] = ['XRP/ETH', 'XRP/USDT', 'UNITTEST/USDT'] else: @@ -328,7 +330,6 @@ def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base, cand convert_from='json', convert_to='jsongz', erase=False, - candle_type=candletype ) for file in (files_temp + files_new): assert file.exists() @@ -342,7 +343,6 @@ def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base, cand convert_from='jsongz', convert_to='json', erase=True, - candle_type=candletype ) for file in (files_temp): assert file.exists()