mohammadian7/freqtrade
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: don't auto-delete trades data due to invalid timerange

Browse Source 
trades-data is already slow enough to download
This commit is contained in:
Matthias
2024-11-16 10:34:24 +01:00
parent 927f5e2c8b
commit 14d384a30a
1 changed files with 81 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
freqtrade/data/history/history_utils.py
+81 -78
View File
@@ -345,7 +345,7 @@ def refresh_backtest_ohlcv_data(
progress.update(timeframe_task, completed=0) progress.update(timeframe_task, completed=0)
if pair not in exchange.markets: if pair not in exchange.markets:
pairs_not_available.append(pair) pairs_not_available.append(f"{pair}: Pair not available on exchange.")
logger.info(f"Skipping pair {pair}...") logger.info(f"Skipping pair {pair}...")
continue continue
for timeframe in timeframes: for timeframe in timeframes:
@@ -411,79 +411,74 @@ def _download_trades_history(
Download trade history from the exchange. Download trade history from the exchange.
Appends to previously downloaded trades data. Appends to previously downloaded trades data.
""" """
try: until = None
until = None since = 0
since = 0 if timerange:
if timerange: if timerange.starttype == "date":
if timerange.starttype == "date": since = timerange.startts * 1000
since = timerange.startts * 1000 if timerange.stoptype == "date":
if timerange.stoptype == "date": until = timerange.stopts * 1000
until = timerange.stopts * 1000
trades = data_handler.trades_load(pair, trading_mode) trades = data_handler.trades_load(pair, trading_mode)
# TradesList columns are defined in constants.DEFAULT_TRADES_COLUMNS # TradesList columns are defined in constants.DEFAULT_TRADES_COLUMNS
# DEFAULT_TRADES_COLUMNS: 0 -> timestamp # DEFAULT_TRADES_COLUMNS: 0 -> timestamp
# DEFAULT_TRADES_COLUMNS: 1 -> id # DEFAULT_TRADES_COLUMNS: 1 -> id
if not trades.empty and since > 0 and since < trades.iloc[0]["timestamp"]: if not trades.empty and since > 0 and since < trades.iloc[0]["timestamp"]:
# since is before the first trade # since is before the first trade
logger.info( raise ValueError(
f"Start ({trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}) earlier than " f"Start {format_ms_time(since)} earlier than "
f"available data. Redownloading trades for {pair}..." f"available data ({trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}). "
) f"Please use `--erase` if you'd like to redownload {pair}."
trades = trades_list_to_df([])
from_id = trades.iloc[-1]["id"] if not trades.empty else None
if not trades.empty and since < trades.iloc[-1]["timestamp"]:
# Reset since to the last available point
# - 5 seconds (to ensure we're getting all trades)
since = trades.iloc[-1]["timestamp"] - (5 * 1000)
logger.info(
f"Using last trade date -5s - Downloading trades for {pair} "
f"since: {format_ms_time(since)}."
)
if not since:
since = dt_ts(dt_now() - timedelta(days=new_pairs_days))
logger.debug(
"Current Start: %s",
"None" if trades.empty else f"{trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}",
) )
logger.debug(
"Current End: %s",
"None" if trades.empty else f"{trades.iloc[-1]['date']:{DATETIME_PRINT_FORMAT}}",
)
logger.info(f"Current Amount of trades: {len(trades)}")
# Default since_ms to 30 days if nothing is given from_id = trades.iloc[-1]["id"] if not trades.empty else None
new_trades = exchange.get_historic_trades( if not trades.empty and since < trades.iloc[-1]["timestamp"]:
pair=pair, # Reset since to the last available point
since=since, # - 5 seconds (to ensure we're getting all trades)
until=until, since = trades.iloc[-1]["timestamp"] - (5 * 1000)
from_id=from_id, logger.info(
f"Using last trade date -5s - Downloading trades for {pair} "
f"since: {format_ms_time(since)}."
) )
new_trades_df = trades_list_to_df(new_trades[1])
trades = concat([trades, new_trades_df], axis=0)
# Remove duplicates to make sure we're not storing data we don't need
trades = trades_df_remove_duplicates(trades)
data_handler.trades_store(pair, trades, trading_mode)
logger.debug( if not since:
"New Start: %s", since = dt_ts(dt_now() - timedelta(days=new_pairs_days))
"None" if trades.empty else f"{trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}",
)
logger.debug(
"New End: %s",
"None" if trades.empty else f"{trades.iloc[-1]['date']:{DATETIME_PRINT_FORMAT}}",
)
logger.info(f"New Amount of trades: {len(trades)}")
return True
except Exception: logger.debug(
logger.exception(f'Failed to download and store historic trades for pair: "{pair}". ') "Current Start: %s",
return False "None" if trades.empty else f"{trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}",
)
logger.debug(
"Current End: %s",
"None" if trades.empty else f"{trades.iloc[-1]['date']:{DATETIME_PRINT_FORMAT}}",
)
logger.info(f"Current Amount of trades: {len(trades)}")
# Default since_ms to 30 days if nothing is given
new_trades = exchange.get_historic_trades(
pair=pair,
since=since,
until=until,
from_id=from_id,
)
new_trades_df = trades_list_to_df(new_trades[1])
trades = concat([trades, new_trades_df], axis=0)
# Remove duplicates to make sure we're not storing data we don't need
trades = trades_df_remove_duplicates(trades)
data_handler.trades_store(pair, trades, trading_mode)
logger.debug(
"New Start: %s",
"None" if trades.empty else f"{trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}",
)
logger.debug(
"New End: %s",
"None" if trades.empty else f"{trades.iloc[-1]['date']:{DATETIME_PRINT_FORMAT}}",
)
logger.info(f"New Amount of trades: {len(trades)}")
return True
def refresh_backtest_trades_data( def refresh_backtest_trades_data(
@@ -508,7 +503,7 @@ def refresh_backtest_trades_data(
for pair in pairs: for pair in pairs:
progress.update(pair_task, description=f"Downloading trades [{pair}]") progress.update(pair_task, description=f"Downloading trades [{pair}]")
if pair not in exchange.markets: if pair not in exchange.markets:
pairs_not_available.append(pair) pairs_not_available.append(f"{pair}: Pair not available on exchange.")
logger.info(f"Skipping pair {pair}...") logger.info(f"Skipping pair {pair}...")
continue continue
@@ -517,14 +512,22 @@ def refresh_backtest_trades_data(
logger.info(f"Deleting existing data for pair {pair}.") logger.info(f"Deleting existing data for pair {pair}.")
logger.info(f"Downloading trades for pair {pair}.") logger.info(f"Downloading trades for pair {pair}.")
_download_trades_history( try:
exchange=exchange, _download_trades_history(
pair=pair, exchange=exchange,
new_pairs_days=new_pairs_days, pair=pair,
timerange=timerange, new_pairs_days=new_pairs_days,
data_handler=data_handler, timerange=timerange,
trading_mode=trading_mode, data_handler=data_handler,
) trading_mode=trading_mode,
)
except ValueError as e:
pairs_not_available.append(f"{pair}: {str(e)}")
except Exception:
logger.exception(
f'Failed to download and store historic trades for pair: "{pair}". '
)
progress.update(pair_task, advance=1) progress.update(pair_task, advance=1)
return pairs_not_available return pairs_not_available
@@ -674,7 +677,7 @@ def download_data_main(config: Config) -> None:
) )
finally: finally:
if pairs_not_available: if pairs_not_available:
logger.info( logger.warning(
f"Pairs [{','.join(pairs_not_available)}] not available " f"Encountered a problem downloading the following pairs from {exchange.name}: \n"
f"on exchange {exchange.name}." f"{'\n'.join(pairs_not_available)}"
) )