Merge pull request #8874 from freqtrade/improve/convert_data

Improve/convert data

freqtrade/commands/data_commands.py

@@ -7,7 +7,7 @@ from freqtrade.configuration import TimeRange, setup_utils_configuration
 from freqtrade.constants import DATETIME_PRINT_FORMAT, DL_DATA_TIMEFRAMES, Config
 from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format
 from freqtrade.data.history import convert_trades_to_ohlcv, download_data_main
-from freqtrade.enums import CandleType, RunMode, TradingMode
+from freqtrade.enums import RunMode, TradingMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
@@ -88,11 +88,10 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
     if ohlcv:
         migrate_binance_futures_data(config)
-        candle_types = [CandleType.from_string(ct) for ct in config.get('candle_types', ['spot'])]
+        convert_ohlcv_format(config,
-        for candle_type in candle_types:
+                             convert_from=args['format_from'],
-            convert_ohlcv_format(config,
+                             convert_to=args['format_to'],
-                                 convert_from=args['format_from'], convert_to=args['format_to'],
+                             erase=args['erase'])
-                                 erase=args['erase'], candle_type=candle_type)
     else:
         convert_trades_format(config,
                               convert_from=args['format_from'], convert_to=args['format_to'],

freqtrade/data/converter.py

@@ -11,7 +11,7 @@ import pandas as pd
 from pandas import DataFrame, to_datetime
 
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, Config, TradeList
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 
 
 logger = logging.getLogger(__name__)
@@ -264,7 +264,6 @@ def convert_ohlcv_format(
     convert_from: str,
     convert_to: str,
     erase: bool,
-    candle_type: CandleType
 ):
     """
     Convert OHLCV from one format to another
@@ -272,7 +271,6 @@ def convert_ohlcv_format(
     :param convert_from: Source format
     :param convert_to: Target format
     :param erase: Erase source data (does not apply if source and target format are identical)
-    :param candle_type: Any of the enum CandleType (must match trading mode!)
     """
     from freqtrade.data.history.idatahandler import get_datahandler
     src = get_datahandler(config['datadir'], convert_from)
@@ -280,37 +278,45 @@ def convert_ohlcv_format(
     timeframes = config.get('timeframes', [config.get('timeframe')])
     logger.info(f"Converting candle (OHLCV) for timeframe {timeframes}")
 
-    if 'pairs' not in config:
+    candle_types = [CandleType.from_string(ct) for ct in config.get('candle_types', [
-        config['pairs'] = []
+        c.value for c in CandleType])]
-        # Check timeframes or fall back to timeframe.
+    logger.info(candle_types)
-        for timeframe in timeframes:
+    paircombs = src.ohlcv_get_available_data(config['datadir'], TradingMode.SPOT)
-            config['pairs'].extend(src.ohlcv_get_pairs(
+    paircombs.extend(src.ohlcv_get_available_data(config['datadir'], TradingMode.FUTURES))
-                config['datadir'],
-                timeframe,
-                candle_type=candle_type
-            ))
-        config['pairs'] = sorted(set(config['pairs']))
-    logger.info(f"Converting candle (OHLCV) data for {config['pairs']}")
 
-    for timeframe in timeframes:
+    if 'pairs' in config:
-        for pair in config['pairs']:
+        # Filter pairs
-            data = src.ohlcv_load(pair=pair, timeframe=timeframe,
+        paircombs = [comb for comb in paircombs if comb[0] in config['pairs']]
-                                  timerange=None,
+
-                                  fill_missing=False,
+    if 'timeframes' in config:
-                                  drop_incomplete=False,
+        paircombs = [comb for comb in paircombs if comb[1] in config['timeframes']]
-                                  startup_candles=0,
+    paircombs = [comb for comb in paircombs if comb[2] in candle_types]
-                                  candle_type=candle_type)
+
-            logger.info(f"Converting {len(data)} {timeframe} {candle_type} candles for {pair}")
+    paircombs = sorted(paircombs, key=lambda x: (x[0], x[1], x[2].value))
-            if len(data) > 0:
+
-                trg.ohlcv_store(
+    formatted_paircombs = '\n'.join([f"{pair}, {timeframe}, {candle_type}"
-                    pair=pair,
+                                    for pair, timeframe, candle_type in paircombs])
-                    timeframe=timeframe,
+
-                    data=data,
+    logger.info(f"Converting candle (OHLCV) data for the following pair combinations:\n"
-                    candle_type=candle_type
+                f"{formatted_paircombs}")
-                )
+    for pair, timeframe, candle_type in paircombs:
-                if erase and convert_from != convert_to:
+        data = src.ohlcv_load(pair=pair, timeframe=timeframe,
-                    logger.info(f"Deleting source data for {pair} / {timeframe}")
+                              timerange=None,
-                    src.ohlcv_purge(pair=pair, timeframe=timeframe, candle_type=candle_type)
+                              fill_missing=False,
+                              drop_incomplete=False,
+                              startup_candles=0,
+                              candle_type=candle_type)
+        logger.info(f"Converting {len(data)} {timeframe} {candle_type} candles for {pair}")
+        if len(data) > 0:
+            trg.ohlcv_store(
+                pair=pair,
+                timeframe=timeframe,
+                data=data,
+                candle_type=candle_type
+            )
+            if erase and convert_from != convert_to:
+                logger.info(f"Deleting source data for {pair} / {timeframe}")
+                src.ohlcv_purge(pair=pair, timeframe=timeframe, candle_type=candle_type)
 
 
 def reduce_dataframe_footprint(df: DataFrame) -> DataFrame:

tests/data/test_converter.py

@@ -315,6 +315,8 @@ def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base, cand
         files_new.append(file_new)
 
     default_conf['datadir'] = tmpdir1
+    default_conf['candle_types'] = [candletype]
+
     if candletype == CandleType.SPOT:
         default_conf['pairs'] = ['XRP/ETH', 'XRP/USDT', 'UNITTEST/USDT']
     else:
@@ -328,7 +330,6 @@ def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base, cand
         convert_from='json',
         convert_to='jsongz',
         erase=False,
-        candle_type=candletype
     )
     for file in (files_temp + files_new):
         assert file.exists()
@@ -342,7 +343,6 @@ def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base, cand
         convert_from='jsongz',
         convert_to='json',
         erase=True,
-        candle_type=candletype
     )
     for file in (files_temp):
         assert file.exists()