Update Datahandlers to work with trades data as dataframes

freqtrade/data/history/featherdatahandler.py

@@ -104,13 +104,13 @@ class FeatherDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         # TODO: respect timerange ...
         :param pair: Load trades for this pair
         :param timerange: Timerange to load trades for - currently not implemented
-        :return: List of trades
+        :return: Dataframe containing trades
         """
         filename = self._pair_trades_filename(self._datadir, pair)
         if not filename.exists():
@@ -118,7 +118,7 @@ class FeatherDataHandler(IDataHandler):
 
         tradesdata = read_feather(filename)
 
-        return tradesdata.values.tolist()
+        return tradesdata
 
     @classmethod
     def _get_file_extension(cls):

freqtrade/data/history/hdf5datahandler.py

@@ -124,18 +124,18 @@ class HDF5DataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> pd.DataFrame:
         """
         Load a pair from h5 file.
         :param pair: Load trades for this pair
         :param timerange: Timerange to load trades for - currently not implemented
-        :return: List of trades
+        :return: Dataframe containing trades
         """
         key = self._pair_trades_key(pair)
         filename = self._pair_trades_filename(self._datadir, pair)
 
         if not filename.exists():
-            return []
+            return pd.DataFrame(columns=DEFAULT_TRADES_COLUMNS)
         where = []
         if timerange:
             if timerange.starttype == 'date':
@@ -145,7 +145,7 @@ class HDF5DataHandler(IDataHandler):
 
         trades: pd.DataFrame = pd.read_hdf(filename, key=key, mode="r", where=where)
         trades[['id', 'type']] = trades[['id', 'type']].replace({np.nan: None})
-        return trades.values.tolist()
+        return trades
 
     @classmethod
     def _get_file_extension(cls):

freqtrade/data/history/history_utils.py

@@ -343,7 +343,7 @@ def _download_trades_history(exchange: Exchange,
             if timerange.stoptype == 'date':
                 until = timerange.stopts * 1000
 
-        trades = data_handler.trades_load(pair)
+        trades = data_handler.trades_load_aslist(pair)
 
         # TradesList columns are defined in constants.DEFAULT_TRADES_COLUMNS
         # DEFAULT_TRADES_COLUMNS: 0 -> timestamp

freqtrade/data/history/idatahandler.py

@@ -11,7 +11,7 @@ from datetime import datetime, timezone
 from pathlib import Path
 from typing import List, Optional, Tuple, Type
 
-from pandas import DataFrame
+from pandas import DataFrame, to_datetime
 
 from freqtrade import misc
 from freqtrade.configuration import TimeRange
@@ -188,12 +188,12 @@ class IDataHandler(ABC):
         """
 
     @abstractmethod
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         :param pair: Load trades for this pair
         :param timerange: Timerange to load trades for - currently not implemented
-        :return: List of trades
+        :return: Dataframe containing trades
         """
 
     def trades_purge(self, pair: str) -> bool:
@@ -208,7 +208,7 @@ class IDataHandler(ABC):
             return True
         return False
 
-    def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+    def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         Removes duplicates in the process.
@@ -216,7 +216,13 @@ class IDataHandler(ABC):
         :param timerange: Timerange to load trades for - currently not implemented
         :return: List of trades
         """
-        return trades_remove_duplicates(self._trades_load(pair, timerange=timerange))
+        trades = trades_remove_duplicates(self._trades_load(pair, timerange=timerange))
+        trades['timestamp'] = to_datetime(trades['timestamp'], unit='ms', utc=True)
+        return trades
+
+    def trades_load_aslist(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+        trades = self.trades_load(pair, timerange)
+        return trades.values.tolist()
 
     @classmethod
     def create_dir_if_needed(cls, datadir: Path):

freqtrade/data/history/jsondatahandler.py

@@ -6,7 +6,7 @@ from pandas import DataFrame, read_json, to_datetime
 
 from freqtrade import misc
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, TradeList
+from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList
 from freqtrade.data.converter import trades_dict_to_list
 from freqtrade.enums import CandleType
 
@@ -113,13 +113,13 @@ class JsonDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         # TODO: respect timerange ...
         :param pair: Load trades for this pair
         :param timerange: Timerange to load trades for - currently not implemented
-        :return: List of trades
+        :return: Dataframe containing trades
         """
         filename = self._pair_trades_filename(self._datadir, pair)
         tradesdata = misc.file_load_json(filename)
@@ -132,7 +132,8 @@ class JsonDataHandler(IDataHandler):
             logger.info("Old trades format detected - converting")
             tradesdata = trades_dict_to_list(tradesdata)
             pass
-        return tradesdata
+        trades = DataFrame(tradesdata, columns=DEFAULT_TRADES_COLUMNS)
+        return trades
 
     @classmethod
     def _get_file_extension(cls):