Add support for trades data in Parquet format

freqtrade/commands/cli_options.py

@@ -441,7 +441,7 @@ AVAILABLE_CLI_OPTIONS = {
     "dataformat_trades": Arg(
         '--data-format-trades',
         help='Storage format for downloaded trades data. (default: `feather`).',
-        choices=constants.AVAILABLE_DATAHANDLERS_TRADES,
+        choices=constants.AVAILABLE_DATAHANDLERS,
     ),
     "show_timerange": Arg(
         '--show-timerange',

freqtrade/constants.py

@@ -38,8 +38,7 @@ AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', '
                        'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
 AVAILABLE_PROTECTIONS = ['CooldownPeriod',
                          'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard']
-AVAILABLE_DATAHANDLERS_TRADES = ['json', 'jsongz', 'hdf5', 'feather']
+AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5', 'feather', 'parquet']
-AVAILABLE_DATAHANDLERS = AVAILABLE_DATAHANDLERS_TRADES + ['parquet']
 BACKTEST_BREAKDOWNS = ['day', 'week', 'month']
 BACKTEST_CACHE_AGE = ['none', 'day', 'week', 'month']
 BACKTEST_CACHE_DEFAULT = 'day'
@@ -459,7 +458,7 @@ CONF_SCHEMA = {
         },
         'dataformat_trades': {
             'type': 'string',
-            'enum': AVAILABLE_DATAHANDLERS_TRADES,
+            'enum': AVAILABLE_DATAHANDLERS,
             'default': 'feather'
         },
         'position_adjustment_enable': {'type': 'boolean'},

freqtrade/data/history/parquetdatahandler.py

@@ -4,7 +4,7 @@ from typing import Optional
 from pandas import DataFrame, read_parquet, to_datetime
 
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, TradeList
+from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList
 from freqtrade.enums import CandleType
 
 from .idatahandler import IDataHandler
@@ -88,12 +88,9 @@ class ParquetDataHandler(IDataHandler):
         :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
         """
-        # filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair)
-
+        self.create_dir_if_needed(filename)
-        raise NotImplementedError()
+        data.reset_index(drop=True).to_parquet(filename)
-        # array = pa.array(data)
-        # array
-        # feather.write_feather(data, filename)
 
     def trades_append(self, pair: str, data: DataFrame):
         """
@@ -112,14 +109,13 @@ class ParquetDataHandler(IDataHandler):
         :param timerange: Timerange to load trades for - currently not implemented
         :return: List of trades
         """
-        raise NotImplementedError()
+        filename = self._pair_trades_filename(self._datadir, pair)
-        # filename = self._pair_trades_filename(self._datadir, pair)
+        if not filename.exists():
-        # tradesdata = misc.file_load_json(filename)
+            return DataFrame(columns=DEFAULT_TRADES_COLUMNS)
 
-        # if not tradesdata:
+        tradesdata = read_parquet(filename)
-        #     return []
 
-        # return tradesdata
+        return tradesdata
 
     @classmethod
     def _get_file_extension(cls):

tests/data/test_datahandler.py

@@ -247,8 +247,10 @@ def test_datahandler__check_empty_df(testdatadir, caplog):
     assert log_has_re(expected_text, caplog)
 
 
-@pytest.mark.parametrize('datahandler', ['parquet'])
+# @pytest.mark.parametrize('datahandler', [])
+@pytest.mark.skip("All datahandlers currently support trades data.")
 def test_datahandler_trades_not_supported(datahandler, testdatadir, ):
+    # Currently disabled. Reenable should a new provider not support trades data.
     dh = get_datahandler(testdatadir, datahandler)
     with pytest.raises(NotImplementedError):
         dh.trades_load('UNITTEST/ETH')