Merge pull request #9890 from freqtrade/feat/dl-trades_futures

Support for --dl-trades for futures

freqtrade/commands/arguments.py

@@ -69,7 +69,8 @@ ARGS_CONVERT_DATA_TRADES = ["pairs", "format_from_trades", "format_to", "erase",
 ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase", "exchange"]
 ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes", "trading_mode", "candle_types"]
 
-ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"]
+ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades",
+                       "trading_mode"]
 
 ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode", "show_timerange"]
 

freqtrade/commands/data_commands.py

@@ -8,9 +8,10 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, DL_DATA_TIMEFRAMES, Confi
 from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_format,
                                       convert_trades_to_ohlcv)
 from freqtrade.data.history import download_data_main
-from freqtrade.enums import RunMode, TradingMode
+from freqtrade.enums import CandleType, RunMode, TradingMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes
+from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
 from freqtrade.util.migrations import migrate_data
 
@@ -62,13 +63,21 @@ def start_convert_trades(args: Dict[str, Any]) -> None:
 
     for timeframe in config['timeframes']:
         exchange.validate_timeframes(timeframe)
+    available_pairs = [
+        p for p in exchange.get_markets(
+            tradable_only=True, active_only=not config.get('include_inactive')
+            ).keys()
+    ]
+
+    expanded_pairs = dynamic_expand_pairlist(config, available_pairs)
 
     # Convert downloaded trade data to different timeframes
     convert_trades_to_ohlcv(
-        pairs=config.get('pairs', []), timeframes=config['timeframes'],
+        pairs=expanded_pairs, timeframes=config['timeframes'],
         datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
         data_format_ohlcv=config['dataformat_ohlcv'],
         data_format_trades=config['dataformat_trades'],
+        candle_type=config.get('candle_type_def', CandleType.SPOT)
     )
 
 

freqtrade/data/converter/trade_converter.py

@@ -11,7 +11,7 @@ from pandas import DataFrame, to_datetime
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TRADES_DTYPES,
                                  Config, TradeList)
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 from freqtrade.exceptions import OperationalException
 
 
@@ -88,10 +88,10 @@ def convert_trades_to_ohlcv(
     timeframes: List[str],
     datadir: Path,
     timerange: TimeRange,
-    erase: bool = False,
+    erase: bool,
-    data_format_ohlcv: str = 'feather',
+    data_format_ohlcv: str,
-    data_format_trades: str = 'feather',
+    data_format_trades: str,
-    candle_type: CandleType = CandleType.SPOT
+    candle_type: CandleType,
 ) -> None:
     """
     Convert stored trades data to ohlcv data
@@ -99,14 +99,12 @@ def convert_trades_to_ohlcv(
     from freqtrade.data.history.idatahandler import get_datahandler
     data_handler_trades = get_datahandler(datadir, data_format=data_format_trades)
     data_handler_ohlcv = get_datahandler(datadir, data_format=data_format_ohlcv)
-    if not pairs:
-        pairs = data_handler_trades.trades_get_pairs(datadir)
 
     logger.info(f"About to convert pairs: '{', '.join(pairs)}', "
                 f"intervals: '{', '.join(timeframes)}' to {datadir}")
-
+    trading_mode = TradingMode.FUTURES if candle_type != CandleType.SPOT else TradingMode.SPOT
     for pair in pairs:
-        trades = data_handler_trades.trades_load(pair)
+        trades = data_handler_trades.trades_load(pair, trading_mode)
         for timeframe in timeframes:
             if erase:
                 if data_handler_ohlcv.ohlcv_purge(pair, timeframe, candle_type=candle_type):
@@ -116,7 +114,7 @@ def convert_trades_to_ohlcv(
                 # Store ohlcv
                 data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv, candle_type=candle_type)
             except ValueError:
-                logger.exception(f'Could not convert {pair} to OHLCV.')
+                logger.warning(f'Could not convert {pair} to OHLCV.')
 
 
 def convert_trades_format(config: Config, convert_from: str, convert_to: str, erase: bool):
@@ -144,11 +142,12 @@ def convert_trades_format(config: Config, convert_from: str, convert_to: str, er
     if 'pairs' not in config:
         config['pairs'] = src.trades_get_pairs(config['datadir'])
     logger.info(f"Converting trades for {config['pairs']}")
-
+    trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT)
     for pair in config['pairs']:
-        data = src.trades_load(pair=pair)
+        data = src.trades_load(pair, trading_mode)
         logger.info(f"Converting {len(data)} trades for {pair}")
-        trg.trades_store(pair, data)
+        trg.trades_store(pair, data, trading_mode)
+
         if erase and convert_from != convert_to:
             logger.info(f"Deleting source Trade data for {pair}.")
-            src.trades_purge(pair=pair)
+            src.trades_purge(pair, trading_mode)

freqtrade/data/converter/trade_converter_kraken.py

@@ -7,6 +7,7 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, DEFAULT_TRADES_COLUMNS, C
 from freqtrade.data.converter.trade_converter import (trades_convert_types,
                                                       trades_df_remove_duplicates)
 from freqtrade.data.history.idatahandler import get_datahandler
+from freqtrade.enums import TradingMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
@@ -79,4 +80,4 @@ def import_kraken_trades_from_csv(config: Config, convert_to: str):
                     f"{trades_df['date'].min():{DATETIME_PRINT_FORMAT}} to "
                     f"{trades_df['date'].max():{DATETIME_PRINT_FORMAT}}")
 
-        data_handler.trades_store(pair, trades_df)
+        data_handler.trades_store(pair, trades_df, TradingMode.SPOT)

freqtrade/data/history/featherdatahandler.py

@@ -5,7 +5,7 @@ from pandas import DataFrame, read_feather, to_datetime
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 
 from .idatahandler import IDataHandler
 
@@ -82,14 +82,15 @@ class FeatherDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_store(self, pair: str, data: DataFrame) -> None:
+    def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
         :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
+        :param trading_mode: Trading mode to use (used to determine the filename)
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         self.create_dir_if_needed(filename)
         data.reset_index(drop=True).to_feather(filename, compression_level=9, compression='lz4')
 
@@ -102,15 +103,18 @@ class FeatherDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
+    def _trades_load(
+            self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
+    ) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         # TODO: respect timerange ...
         :param pair: Load trades for this pair
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :param timerange: Timerange to load trades for - currently not implemented
         :return: Dataframe containing trades
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         if not filename.exists():
             return DataFrame(columns=DEFAULT_TRADES_COLUMNS)
 

freqtrade/data/history/hdf5datahandler.py

@@ -6,7 +6,7 @@ import pandas as pd
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 
 from .idatahandler import IDataHandler
 
@@ -100,17 +100,18 @@ class HDF5DataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_store(self, pair: str, data: pd.DataFrame) -> None:
+    def _trades_store(self, pair: str, data: pd.DataFrame, trading_mode: TradingMode) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
         :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
+        :param trading_mode: Trading mode to use (used to determine the filename)
         """
         key = self._pair_trades_key(pair)
 
         data.to_hdf(
-            self._pair_trades_filename(self._datadir, pair), key=key,
+            self._pair_trades_filename(self._datadir, pair, trading_mode), key=key,
             mode='a', complevel=9, complib='blosc',
             format='table', data_columns=['timestamp']
         )
@@ -124,15 +125,18 @@ class HDF5DataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> pd.DataFrame:
+    def _trades_load(
+        self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
+    ) -> pd.DataFrame:
         """
         Load a pair from h5 file.
         :param pair: Load trades for this pair
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :param timerange: Timerange to load trades for - currently not implemented
         :return: Dataframe containing trades
         """
         key = self._pair_trades_key(pair)
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
 
         if not filename.exists():
             return pd.DataFrame(columns=DEFAULT_TRADES_COLUMNS)

freqtrade/data/history/history_utils.py

@@ -13,7 +13,7 @@ from freqtrade.data.converter import (clean_ohlcv_dataframe, convert_trades_to_o
                                       ohlcv_to_dataframe, trades_df_remove_duplicates,
                                       trades_list_to_df)
 from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import Exchange
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
@@ -333,7 +333,8 @@ def _download_trades_history(exchange: Exchange,
                              pair: str, *,
                              new_pairs_days: int = 30,
                              timerange: Optional[TimeRange] = None,
-                             data_handler: IDataHandler
+                             data_handler: IDataHandler,
+                             trading_mode: TradingMode,
                              ) -> bool:
     """
     Download trade history from the exchange.
@@ -349,7 +350,7 @@ def _download_trades_history(exchange: Exchange,
             if timerange.stoptype == 'date':
                 until = timerange.stopts * 1000
 
-        trades = data_handler.trades_load(pair)
+        trades = data_handler.trades_load(pair, trading_mode)
 
         # TradesList columns are defined in constants.DEFAULT_TRADES_COLUMNS
         # DEFAULT_TRADES_COLUMNS: 0 -> timestamp
@@ -388,7 +389,7 @@ def _download_trades_history(exchange: Exchange,
         trades = concat([trades, new_trades_df], axis=0)
         # Remove duplicates to make sure we're not storing data we don't need
         trades = trades_df_remove_duplicates(trades)
-        data_handler.trades_store(pair, data=trades)
+        data_handler.trades_store(pair, trades, trading_mode)
 
         logger.debug("New Start: %s", 'None' if trades.empty else
                      f"{trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}")
@@ -405,8 +406,10 @@ def _download_trades_history(exchange: Exchange,
 
 
 def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
-                                 timerange: TimeRange, new_pairs_days: int = 30,
+                                 timerange: TimeRange, trading_mode: TradingMode,
-                                 erase: bool = False, data_format: str = 'feather') -> List[str]:
+                                 new_pairs_days: int = 30,
+                                 erase: bool = False, data_format: str = 'feather',
+                                 ) -> List[str]:
     """
     Refresh stored trades data for backtesting and hyperopt operations.
     Used by freqtrade download-data subcommand.
@@ -421,7 +424,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
             continue
 
         if erase:
-            if data_handler.trades_purge(pair):
+            if data_handler.trades_purge(pair, trading_mode):
                 logger.info(f'Deleting existing data for pair {pair}.')
 
         logger.info(f'Downloading trades for pair {pair}.')
@@ -429,7 +432,8 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
                                  pair=pair,
                                  new_pairs_days=new_pairs_days,
                                  timerange=timerange,
-                                 data_handler=data_handler)
+                                 data_handler=data_handler,
+                                 trading_mode=trading_mode)
     return pairs_not_available
 
 
@@ -516,12 +520,12 @@ def download_data_main(config: Config) -> None:
     # Start downloading
     try:
         if config.get('download_trades'):
-            if config.get('trading_mode') == 'futures':
-                raise OperationalException("Trade download not supported for futures.")
             pairs_not_available = refresh_backtest_trades_data(
                 exchange, pairs=expanded_pairs, datadir=config['datadir'],
                 timerange=timerange, new_pairs_days=config['new_pairs_days'],
-                erase=bool(config.get('erase')), data_format=config['dataformat_trades'])
+                erase=bool(config.get('erase')), data_format=config['dataformat_trades'],
+                trading_mode=config.get('trading_mode', TradingMode.SPOT),
+                )
 
             # Convert downloaded trade data to different timeframes
             convert_trades_to_ohlcv(
@@ -529,6 +533,7 @@ def download_data_main(config: Config) -> None:
                 datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
                 data_format_ohlcv=config['dataformat_ohlcv'],
                 data_format_trades=config['dataformat_trades'],
+                candle_type=config.get('candle_type_def', CandleType.SPOT),
             )
         else:
             if not exchange.get_option('ohlcv_has_history', True):

freqtrade/data/history/idatahandler.py

@@ -172,12 +172,13 @@ class IDataHandler(ABC):
         return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match]
 
     @abstractmethod
-    def _trades_store(self, pair: str, data: DataFrame) -> None:
+    def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
         :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
+        :param trading_mode: Trading mode to use (used to determine the filename)
         """
 
     @abstractmethod
@@ -190,45 +191,55 @@ class IDataHandler(ABC):
         """
 
     @abstractmethod
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
+    def _trades_load(
+        self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
+    ) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         :param pair: Load trades for this pair
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :param timerange: Timerange to load trades for - currently not implemented
         :return: Dataframe containing trades
         """
 
-    def trades_store(self, pair: str, data: DataFrame) -> None:
+    def trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
         :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
+        :param trading_mode: Trading mode to use (used to determine the filename)
         """
         # Filter on expected columns (will remove the actual date column).
-        self._trades_store(pair, data[DEFAULT_TRADES_COLUMNS])
+        self._trades_store(pair, data[DEFAULT_TRADES_COLUMNS], trading_mode)
 
-    def trades_purge(self, pair: str) -> bool:
+    def trades_purge(self, pair: str, trading_mode: TradingMode) -> bool:
         """
         Remove data for this pair
         :param pair: Delete data for this pair.
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :return: True when deleted, false if file did not exist.
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         if filename.exists():
             filename.unlink()
             return True
         return False
 
-    def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
+    def trades_load(
+            self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
+    ) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         Removes duplicates in the process.
         :param pair: Load trades for this pair
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :param timerange: Timerange to load trades for - currently not implemented
         :return: List of trades
         """
-        trades = trades_df_remove_duplicates(self._trades_load(pair, timerange=timerange))
+        trades = trades_df_remove_duplicates(
+            self._trades_load(pair, trading_mode, timerange=timerange)
+        )
 
         trades = trades_convert_types(trades)
         return trades
@@ -264,8 +275,12 @@ class IDataHandler(ABC):
         return filename
 
     @classmethod
-    def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path:
+    def _pair_trades_filename(cls, datadir: Path, pair: str, trading_mode: TradingMode) -> Path:
         pair_s = misc.pair_to_filename(pair)
+        if trading_mode == TradingMode.FUTURES:
+            # Futures pair ...
+            datadir = datadir.joinpath('futures')
+
         filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}')
         return filename
 

freqtrade/data/history/jsondatahandler.py

@@ -8,7 +8,7 @@ from freqtrade import misc
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
 from freqtrade.data.converter import trades_dict_to_list, trades_list_to_df
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 
 from .idatahandler import IDataHandler
 
@@ -94,14 +94,15 @@ class JsonDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_store(self, pair: str, data: DataFrame) -> None:
+    def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
         :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
+        :param trading_mode: Trading mode to use (used to determine the filename)
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         trades = data.values.tolist()
         misc.file_dump_json(filename, trades, is_zip=self._use_zip)
 
@@ -114,15 +115,18 @@ class JsonDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
+    def _trades_load(
+        self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
+    ) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         # TODO: respect timerange ...
         :param pair: Load trades for this pair
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :param timerange: Timerange to load trades for - currently not implemented
         :return: Dataframe containing trades
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         tradesdata = misc.file_load_json(filename)
 
         if not tradesdata:

freqtrade/data/history/parquetdatahandler.py

@@ -4,8 +4,8 @@ from typing import Optional
 from pandas import DataFrame, read_parquet, to_datetime
 
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList
+from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 
 from .idatahandler import IDataHandler
 
@@ -81,14 +81,15 @@ class ParquetDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_store(self, pair: str, data: DataFrame) -> None:
+    def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
         :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
+        :param trading_mode: Trading mode to use (used to determine the filename)
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         self.create_dir_if_needed(filename)
         data.reset_index(drop=True).to_parquet(filename)
 
@@ -101,15 +102,18 @@ class ParquetDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+    def _trades_load(
+        self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
+    ) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         # TODO: respect timerange ...
         :param pair: Load trades for this pair
+        :param trading_mode: Trading mode to use (used to determine the filename)
         :param timerange: Timerange to load trades for - currently not implemented
         :return: List of trades
         """
-        filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         if not filename.exists():
             return DataFrame(columns=DEFAULT_TRADES_COLUMNS)
 

tests/commands/test_commands.py

@@ -820,11 +820,6 @@ def test_download_data_trades(mocker):
         "--trading-mode", "futures",
         "--dl-trades"
     ]
-    with pytest.raises(OperationalException,
-                       match="Trade download not supported for futures."):
-        pargs = get_args(args)
-        pargs['config'] = None
-        start_download_data(pargs)
 
 
 def test_download_data_data_invalid(mocker):
@@ -842,10 +837,11 @@ def test_download_data_data_invalid(mocker):
         start_download_data(pargs)
 
 
-def test_start_convert_trades(mocker, caplog):
+def test_start_convert_trades(mocker):
     convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv',
                                 MagicMock(return_value=[]))
     patch_exchange(mocker)
+    mocker.patch(f'{EXMS}.get_markets')
     mocker.patch(f'{EXMS}.markets', PropertyMock(return_value={}))
     args = [
         "trades-to-ohlcv",

tests/data/test_converter.py

@@ -542,7 +542,9 @@ def test_convert_trades_to_ohlcv(testdatadir, tmp_path, caplog):
 
     convert_trades_to_ohlcv([pair], timeframes=['1m', '5m'],
                             data_format_trades='jsongz',
-                            datadir=tmp_path, timerange=tr, erase=True)
+                            datadir=tmp_path, timerange=tr, erase=True,
+                            data_format_ohlcv='feather',
+                            candle_type=CandleType.SPOT)
 
     assert log_has("Deleting existing data for pair XRP/ETH, interval 1m.", caplog)
     # Load new data
@@ -556,5 +558,7 @@ def test_convert_trades_to_ohlcv(testdatadir, tmp_path, caplog):
 
     convert_trades_to_ohlcv(['NoDatapair'], timeframes=['1m', '5m'],
                             data_format_trades='jsongz',
-                            datadir=tmp_path, timerange=tr, erase=True)
+                            datadir=tmp_path, timerange=tr, erase=True,
+                            data_format_ohlcv='feather',
+                            candle_type=CandleType.SPOT)
     assert log_has(msg, caplog)

tests/data/test_datahandler.py

@@ -261,11 +261,11 @@ def test_datahandler_trades_not_supported(datahandler, testdatadir, ):
 def test_jsondatahandler_trades_load(testdatadir, caplog):
     dh = JsonGzDataHandler(testdatadir)
     logmsg = "Old trades format detected - converting"
-    dh.trades_load('XRP/ETH')
+    dh.trades_load('XRP/ETH', TradingMode.SPOT)
     assert not log_has(logmsg, caplog)
 
     # Test conversation is happening
-    dh.trades_load('XRP/OLD')
+    dh.trades_load('XRP/OLD', TradingMode.SPOT)
     assert log_has(logmsg, caplog)
 
 
@@ -300,16 +300,16 @@ def test_datahandler_trades_get_pairs(testdatadir, datahandler, expected):
 
 def test_hdf5datahandler_trades_load(testdatadir):
     dh = get_datahandler(testdatadir, 'hdf5')
-    trades = dh.trades_load('XRP/ETH')
+    trades = dh.trades_load('XRP/ETH', TradingMode.SPOT)
     assert isinstance(trades, DataFrame)
 
-    trades1 = dh.trades_load('UNITTEST/NONEXIST')
+    trades1 = dh.trades_load('UNITTEST/NONEXIST', TradingMode.SPOT)
     assert isinstance(trades1, DataFrame)
     assert trades1.empty
     # data goes from 2019-10-11 - 2019-10-13
     timerange = TimeRange.parse_timerange('20191011-20191012')
 
-    trades2 = dh._trades_load('XRP/ETH', timerange)
+    trades2 = dh._trades_load('XRP/ETH', TradingMode.SPOT, timerange)
     assert len(trades) > len(trades2)
     # Check that ID is None (If it's nan, it's wrong)
     assert trades2.iloc[0]['type'] is None
@@ -451,13 +451,13 @@ def test_hdf5datahandler_ohlcv_purge(mocker, testdatadir):
 @pytest.mark.parametrize('datahandler', ['jsongz', 'hdf5', 'feather', 'parquet'])
 def test_datahandler_trades_load(testdatadir, datahandler):
     dh = get_datahandler(testdatadir, datahandler)
-    trades = dh.trades_load('XRP/ETH')
+    trades = dh.trades_load('XRP/ETH', TradingMode.SPOT)
     assert isinstance(trades, DataFrame)
     assert trades.iloc[0]['timestamp'] == 1570752011620
     assert trades.iloc[0]['date'] == Timestamp('2019-10-11 00:00:11.620000+0000')
     assert trades.iloc[-1]['cost'] == 0.1986231
 
-    trades1 = dh.trades_load('UNITTEST/NONEXIST')
+    trades1 = dh.trades_load('UNITTEST/NONEXIST', TradingMode.SPOT)
     assert isinstance(trades, DataFrame)
     assert trades1.empty
 
@@ -465,15 +465,15 @@ def test_datahandler_trades_load(testdatadir, datahandler):
 @pytest.mark.parametrize('datahandler', ['jsongz', 'hdf5', 'feather', 'parquet'])
 def test_datahandler_trades_store(testdatadir, tmp_path, datahandler):
     dh = get_datahandler(testdatadir, datahandler)
-    trades = dh.trades_load('XRP/ETH')
+    trades = dh.trades_load('XRP/ETH', TradingMode.SPOT)
 
     dh1 = get_datahandler(tmp_path, datahandler)
-    dh1.trades_store('XRP/NEW', trades)
+    dh1.trades_store('XRP/NEW', trades, TradingMode.SPOT)
 
     file = tmp_path / f'XRP_NEW-trades.{dh1._get_file_extension()}'
     assert file.is_file()
     # Load trades back
-    trades_new = dh1.trades_load('XRP/NEW')
+    trades_new = dh1.trades_load('XRP/NEW', TradingMode.SPOT)
     assert_frame_equal(trades, trades_new, check_exact=True)
     assert len(trades_new) == len(trades)
 
@@ -483,11 +483,11 @@ def test_datahandler_trades_purge(mocker, testdatadir, datahandler):
     mocker.patch.object(Path, "exists", MagicMock(return_value=False))
     unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
     dh = get_datahandler(testdatadir, datahandler)
-    assert not dh.trades_purge('UNITTEST/NONEXIST')
+    assert not dh.trades_purge('UNITTEST/NONEXIST', TradingMode.SPOT)
     assert unlinkmock.call_count == 0
 
     mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    assert dh.trades_purge('UNITTEST/NONEXIST')
+    assert dh.trades_purge('UNITTEST/NONEXIST', TradingMode.SPOT)
     assert unlinkmock.call_count == 1
 
 

tests/data/test_download_data.py

@@ -78,10 +78,6 @@ def test_download_data_main_trades(mocker):
         "trading_mode": "futures",
     })
 
-    with pytest.raises(OperationalException,
-                       match="Trade download not supported for futures."):
-        download_data_main(config)
-
 
 def test_download_data_main_data_invalid(mocker):
     patch_exchange(mocker, id="kraken")

tests/data/test_history.py

@@ -23,7 +23,7 @@ from freqtrade.data.history.history_utils import (_download_pair_history, _downl
                                                   validate_backtest_data)
 from freqtrade.data.history.idatahandler import get_datahandler
 from freqtrade.data.history.jsondatahandler import JsonDataHandler, JsonGzDataHandler
-from freqtrade.enums import CandleType
+from freqtrade.enums import CandleType, TradingMode
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.misc import file_dump_json
 from freqtrade.resolvers import StrategyResolver
@@ -168,20 +168,21 @@ def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type):
     assert fn == Path(expected_result + '.gz')
 
 
-@pytest.mark.parametrize("pair,expected_result", [
+@pytest.mark.parametrize("pair,trading_mode,expected_result", [
-    ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-trades.json'),
+    ("ETH/BTC", '', 'freqtrade/hello/world/ETH_BTC-trades.json'),
-    ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-trades.json'),
+    ("ETH/USDT:USDT", 'futures', 'freqtrade/hello/world/futures/ETH_USDT_USDT-trades.json'),
-    ("ETHH20", 'freqtrade/hello/world/ETHH20-trades.json'),
+    ("Fabric Token/ETH", '', 'freqtrade/hello/world/Fabric_Token_ETH-trades.json'),
-    (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-trades.json'),
+    ("ETHH20", '', 'freqtrade/hello/world/ETHH20-trades.json'),
-    ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-trades.json'),
+    (".XBTBON2H", '', 'freqtrade/hello/world/_XBTBON2H-trades.json'),
-    ("ACC_OLD_BTC", 'freqtrade/hello/world/ACC_OLD_BTC-trades.json'),
+    ("ETHUSD.d", '', 'freqtrade/hello/world/ETHUSD_d-trades.json'),
+    ("ACC_OLD_BTC", '', 'freqtrade/hello/world/ACC_OLD_BTC-trades.json'),
 ])
-def test_json_pair_trades_filename(pair, expected_result):
+def test_json_pair_trades_filename(pair, trading_mode, expected_result):
-    fn = JsonDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair)
+    fn = JsonDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair, trading_mode)
     assert isinstance(fn, Path)
     assert fn == Path(expected_result)
 
-    fn = JsonGzDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair)
+    fn = JsonGzDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair, trading_mode)
     assert isinstance(fn, Path)
     assert fn == Path(expected_result + '.gz')
 
@@ -559,7 +560,8 @@ def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, tes
     unavailable_pairs = refresh_backtest_trades_data(exchange=ex,
                                                      pairs=["ETH/BTC", "XRP/BTC", "XRP/ETH"],
                                                      datadir=testdatadir,
-                                                     timerange=timerange, erase=True
+                                                     timerange=timerange, erase=True,
+                                                     trading_mode=TradingMode.SPOT,
                                                      )
 
     assert dl_mock.call_count == 2
@@ -584,7 +586,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
     assert not file1.is_file()
 
     assert _download_trades_history(data_handler=data_handler, exchange=exchange,
-                                    pair='ETH/BTC')
+                                    pair='ETH/BTC', trading_mode=TradingMode.SPOT)
     assert log_has("Current Amount of trades: 0", caplog)
     assert log_has("New Amount of trades: 6", caplog)
     assert ght_mock.call_count == 1
@@ -597,8 +599,9 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
     since_time = int(trades_history[-3][0] // 1000)
     since_time2 = int(trades_history[-1][0] // 1000)
     timerange = TimeRange('date', None, since_time, 0)
-    assert _download_trades_history(data_handler=data_handler, exchange=exchange,
+    assert _download_trades_history(
-                                    pair='ETH/BTC', timerange=timerange)
+        data_handler=data_handler, exchange=exchange, pair='ETH/BTC',
+        timerange=timerange, trading_mode=TradingMode.SPOT)
 
     assert ght_mock.call_count == 1
     # Check this in seconds - since we had to convert to seconds above too.
@@ -611,7 +614,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
     caplog.clear()
 
     assert not _download_trades_history(data_handler=data_handler, exchange=exchange,
-                                        pair='ETH/BTC')
+                                        pair='ETH/BTC', trading_mode=TradingMode.SPOT)
     assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog)
 
     file2 = tmp_path / 'XRP_ETH-trades.json.gz'
@@ -623,8 +626,9 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
     since_time = int(trades_history[0][0] // 1000) - 500
     timerange = TimeRange('date', None, since_time, 0)
 
-    assert _download_trades_history(data_handler=data_handler, exchange=exchange,
+    assert _download_trades_history(
-                                    pair='XRP/ETH', timerange=timerange)
+        data_handler=data_handler, exchange=exchange, pair='XRP/ETH',
+        timerange=timerange, trading_mode=TradingMode.SPOT)
 
     assert ght_mock.call_count == 1
 

tests/data/test_trade_converter_kraken.py

@@ -6,6 +6,7 @@ import pytest
 
 from freqtrade.data.converter.trade_converter_kraken import import_kraken_trades_from_csv
 from freqtrade.data.history.idatahandler import get_datahandler
+from freqtrade.enums import TradingMode
 from freqtrade.exceptions import OperationalException
 from tests.conftest import EXMS, log_has, log_has_re, patch_exchange
 
@@ -40,7 +41,7 @@ def test_import_kraken_trades_from_csv(testdatadir, tmp_path, caplog, default_co
     assert dstfile.is_file()
 
     dh = get_datahandler(tmp_path, 'feather')
-    trades = dh.trades_load('BCH_EUR')
+    trades = dh.trades_load('BCH_EUR', TradingMode.SPOT)
     assert len(trades) == 340
 
     assert trades['date'].min().to_pydatetime() == datetime(2023, 1, 1, 0, 3, 56,