mohammadian7/freqtrade
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: add better error-handling for dataloading

Browse Source 
closes #10978
This commit is contained in:
Matthias
2024-11-24 10:36:02 +01:00
parent 5841b56f2d
commit 3b542ea008
3 changed files with 66 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
freqtrade/data/history/datahandlers/featherdatahandler.py
+19 -14
View File
@@ -58,20 +58,25 @@ class FeatherDataHandler(IDataHandler):
) )
if not filename.exists(): if not filename.exists():
return DataFrame(columns=self._columns) return DataFrame(columns=self._columns)
try:
pairdata = read_feather(filename) pairdata = read_feather(filename)
pairdata.columns = self._columns pairdata.columns = self._columns
pairdata = pairdata.astype( pairdata = pairdata.astype(
dtype={ dtype={
"open": "float", "open": "float",
"high": "float", "high": "float",
"low": "float", "low": "float",
"close": "float", "close": "float",
"volume": "float", "volume": "float",
} }
) )
pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True) pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True)
return pairdata return pairdata
except Exception as e:
logger.exception(
f"Error loading data from {filename}. Exception: {e}. Returning empty dataframe."
)
return DataFrame(columns=self._columns)
def ohlcv_append( def ohlcv_append(
self, pair: str, timeframe: str, data: DataFrame, candle_type: CandleType self, pair: str, timeframe: str, data: DataFrame, candle_type: CandleType
freqtrade/data/history/datahandlers/hdf5datahandler.py
+28 -20
View File
@@ -68,28 +68,36 @@ class HDF5DataHandler(IDataHandler):
) )
if not filename.exists(): if not filename.exists():
return pd.DataFrame(columns=self._columns) return pd.DataFrame(columns=self._columns)
where = [] try:
if timerange: where = []
if timerange.starttype == "date": if timerange:
where.append(f"date >= Timestamp({timerange.startts * 1e9})") if timerange.starttype == "date":
if timerange.stoptype == "date": where.append(f"date >= Timestamp({timerange.startts * 1e9})")
where.append(f"date <= Timestamp({timerange.stopts * 1e9})") if timerange.stoptype == "date":
where.append(f"date <= Timestamp({timerange.stopts * 1e9})")
pairdata = pd.read_hdf(filename, key=key, mode="r", where=where) pairdata = pd.read_hdf(filename, key=key, mode="r", where=where)
if list(pairdata.columns) != self._columns: if list(pairdata.columns) != self._columns:
raise ValueError("Wrong dataframe format") raise ValueError("Wrong dataframe format")
pairdata = pairdata.astype( pairdata = pairdata.astype(
dtype={ dtype={
"open": "float", "open": "float",
"high": "float", "high": "float",
"low": "float", "low": "float",
"close": "float", "close": "float",
"volume": "float", "volume": "float",
} }
) )
pairdata = pairdata.reset_index(drop=True) pairdata = pairdata.reset_index(drop=True)
return pairdata return pairdata
except ValueError as e:
raise
except Exception as e:
logger.exception(
f"Error loading data from {filename}. Exception: {e}. Returning empty dataframe."
)
return pd.DataFrame(columns=self._columns)
def ohlcv_append( def ohlcv_append(
self, pair: str, timeframe: str, data: pd.DataFrame, candle_type: CandleType self, pair: str, timeframe: str, data: pd.DataFrame, candle_type: CandleType
freqtrade/data/history/datahandlers/parquetdatahandler.py
+19 -14
View File
@@ -56,20 +56,25 @@ class ParquetDataHandler(IDataHandler):
) )
if not filename.exists(): if not filename.exists():
return DataFrame(columns=self._columns) return DataFrame(columns=self._columns)
try:
pairdata = read_parquet(filename) pairdata = read_parquet(filename)
pairdata.columns = self._columns pairdata.columns = self._columns
pairdata = pairdata.astype( pairdata = pairdata.astype(
dtype={ dtype={
"open": "float", "open": "float",
"high": "float", "high": "float",
"low": "float", "low": "float",
"close": "float", "close": "float",
"volume": "float", "volume": "float",
} }
) )
pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True) pairdata["date"] = to_datetime(pairdata["date"], unit="ms", utc=True)
return pairdata return pairdata
except Exception as e:
logger.exception(
f"Error loading data from {filename}. Exception: {e}. Returning empty dataframe."
)
return DataFrame(columns=self._columns)
def ohlcv_append( def ohlcv_append(
self, pair: str, timeframe: str, data: DataFrame, candle_type: CandleType self, pair: str, timeframe: str, data: DataFrame, candle_type: CandleType