mohammadian7/freqtrade
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix imports and handling of trade regarding changes from PR#9065

Browse Source
This commit is contained in:
Joe Schr
2023-10-09 11:26:07 +02:00
parent 9f507e0146
commit a81a3be8d6
3 changed files with 42 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
freqtrade/data/converter/__init__.py
+4 -3
View File
@@ -1,7 +1,8 @@
from freqtrade.data.converter.converter import (clean_ohlcv_dataframe, convert_ohlcv_format, from freqtrade.data.converter.converter import (clean_duplicate_trades, clean_ohlcv_dataframe, convert_ohlcv_format,
ohlcv_fill_up_missing_data, ohlcv_to_dataframe, ohlcv_fill_up_missing_data, ohlcv_to_dataframe,
order_book_to_dataframe, reduce_dataframe_footprint, order_book_to_dataframe,
trim_dataframe, trim_dataframes) populate_dataframe_with_trades, public_trades_to_dataframe,
reduce_dataframe_footprint, trim_dataframe, trim_dataframes)
from freqtrade.data.converter.trade_converter import (convert_trades_format, from freqtrade.data.converter.trade_converter import (convert_trades_format,
convert_trades_to_ohlcv, trades_convert_types, convert_trades_to_ohlcv, trades_convert_types,
trades_df_remove_duplicates, trades_df_remove_duplicates,
freqtrade/data/converter/converter.py
+20 -9
View File
@@ -7,9 +7,11 @@ from typing import Dict
import numpy as np import numpy as np
import pandas as pd import pandas as pd
from pandas import DataFrame, to_datetime from pandas import DataFrame, to_datetime
import itertools
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, Config from freqtrade.constants import DEFAULT_ORDERFLOW_COLUMNS, DEFAULT_DATAFRAME_COLUMNS, Config
from freqtrade.enums import CandleType, TradingMode from freqtrade.enums import CandleType, TradingMode
from freqtrade.data.converter.trade_converter import trades_df_remove_duplicates
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -74,6 +76,7 @@ def _calculate_ohlcv_candle_start_and_end(df: DataFrame, timeframe: str):
timeframe_frequency, timeframe_minutes = _convert_timeframe_to_pandas_frequency( timeframe_frequency, timeframe_minutes = _convert_timeframe_to_pandas_frequency(
timeframe) timeframe)
# calculate ohlcv candle start and end # calculate ohlcv candle start and end
if df is not None and not df.empty:
df['datetime'] = pd.to_datetime(df['date'], unit='ms') df['datetime'] = pd.to_datetime(df['date'], unit='ms')
df['candle_start'] = df['datetime'].dt.floor(timeframe_frequency) df['candle_start'] = df['datetime'].dt.floor(timeframe_frequency)
df['candle_end'] = df['candle_start'] + pd.Timedelta(timeframe_minutes) df['candle_end'] = df['candle_start'] + pd.Timedelta(timeframe_minutes)
@@ -168,7 +171,8 @@ def populate_dataframe_with_trades(config: Config, dataframe: DataFrame, trades:
max_delta = np.max(deltas_per_trade) max_delta = np.max(deltas_per_trade)
df.loc[is_between, 'total_trades'] = len(trades_grouped_df) df.loc[is_between, 'total_trades'] = len(trades_grouped_df)
dataframe.loc[is_between] = df.loc[is_between].copy() # copy to avoid memory leaks # copy to avoid memory leaks
dataframe.loc[is_between] = df.loc[is_between].copy()
else: else:
logger.debug( logger.debug(
f"Found NO candles for trades starting with {candle_start}") f"Found NO candles for trades starting with {candle_start}")
@@ -456,7 +460,8 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str)
df.reset_index(inplace=True) df.reset_index(inplace=True)
len_before = len(dataframe) len_before = len(dataframe)
len_after = len(df) len_after = len(df)
pct_missing = (len_after - len_before) / len_before if len_before > 0 else 0 pct_missing = (len_after - len_before) / \
len_before if len_before > 0 else 0
if len_before != len_after: if len_before != len_after:
message = (f"Missing data fillup for {pair}: before: {len_before} - after: {len_after}" message = (f"Missing data fillup for {pair}: before: {len_before} - after: {len_after}"
f" - {pct_missing:.2%}") f" - {pct_missing:.2%}")
@@ -501,7 +506,8 @@ def trim_dataframes(preprocessed: Dict[str, DataFrame], timerange,
processed: Dict[str, DataFrame] = {} processed: Dict[str, DataFrame] = {}
for pair, df in preprocessed.items(): for pair, df in preprocessed.items():
trimed_df = trim_dataframe(df, timerange, startup_candles=startup_candles) trimed_df = trim_dataframe(
df, timerange, startup_candles=startup_candles)
if not trimed_df.empty: if not trimed_df.empty:
processed[pair] = trimed_df processed[pair] = trimed_df
else: else:
@@ -557,15 +563,18 @@ def convert_ohlcv_format(
candle_types = [CandleType.from_string(ct) for ct in config.get('candle_types', [ candle_types = [CandleType.from_string(ct) for ct in config.get('candle_types', [
c.value for c in CandleType])] c.value for c in CandleType])]
logger.info(candle_types) logger.info(candle_types)
paircombs = src.ohlcv_get_available_data(config['datadir'], TradingMode.SPOT) paircombs = src.ohlcv_get_available_data(
paircombs.extend(src.ohlcv_get_available_data(config['datadir'], TradingMode.FUTURES)) config['datadir'], TradingMode.SPOT)
paircombs.extend(src.ohlcv_get_available_data(
config['datadir'], TradingMode.FUTURES))
if 'pairs' in config: if 'pairs' in config:
# Filter pairs # Filter pairs
paircombs = [comb for comb in paircombs if comb[0] in config['pairs']] paircombs = [comb for comb in paircombs if comb[0] in config['pairs']]
if 'timeframes' in config: if 'timeframes' in config:
paircombs = [comb for comb in paircombs if comb[1] in config['timeframes']] paircombs = [comb for comb in paircombs if comb[1]
in config['timeframes']]
paircombs = [comb for comb in paircombs if comb[2] in candle_types] paircombs = [comb for comb in paircombs if comb[2] in candle_types]
paircombs = sorted(paircombs, key=lambda x: (x[0], x[1], x[2].value)) paircombs = sorted(paircombs, key=lambda x: (x[0], x[1], x[2].value))
@@ -582,7 +591,8 @@ def convert_ohlcv_format(
drop_incomplete=False, drop_incomplete=False,
startup_candles=0, startup_candles=0,
candle_type=candle_type) candle_type=candle_type)
logger.info(f"Converting {len(data)} {timeframe} {candle_type} candles for {pair}") logger.info(
f"Converting {len(data)} {timeframe} {candle_type} candles for {pair}")
if len(data) > 0: if len(data) > 0:
trg.ohlcv_store( trg.ohlcv_store(
pair=pair, pair=pair,
@@ -592,7 +602,8 @@ def convert_ohlcv_format(
) )
if erase and convert_from != convert_to: if erase and convert_from != convert_to:
logger.info(f"Deleting source data for {pair} / {timeframe}") logger.info(f"Deleting source data for {pair} / {timeframe}")
src.ohlcv_purge(pair=pair, timeframe=timeframe, candle_type=candle_type) src.ohlcv_purge(pair=pair, timeframe=timeframe,
candle_type=candle_type)
def reduce_dataframe_footprint(df: DataFrame) -> DataFrame: def reduce_dataframe_footprint(df: DataFrame) -> DataFrame:
freqtrade/exchange/exchange.py
+14 -10
View File
@@ -8,6 +8,7 @@ import logging
import signal import signal
from copy import deepcopy from copy import deepcopy
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
import arrow
from math import floor from math import floor
from threading import Lock from threading import Lock
from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union, Callable from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union, Callable
@@ -2319,7 +2320,7 @@ class Exchange:
results_df = {} results_df = {}
for pair, timeframe, candle_type in set(pair_list): for pair, timeframe, candle_type in set(pair_list):
new_ticks = [] new_ticks = []
all_stored_ticks = [] all_stored_ticks_df = DataFrame(columns=DEFAULT_TRADES_COLUMNS + ['date'])
first_candle_ms = self.needed_candle_ms(timeframe, candle_type) first_candle_ms = self.needed_candle_ms(timeframe, candle_type)
# refresh, if # refresh, if
# a. not in _trades # a. not in _trades
@@ -2340,19 +2341,20 @@ class Exchange:
else: else:
until = int(timeframe_to_prev_date(timeframe).timestamp()) * 1000 until = int(timeframe_to_prev_date(timeframe).timestamp()) * 1000
all_stored_ticks = data_handler.trades_load(f"{pair}-cached") all_stored_ticks_df = data_handler.trades_load(f"{pair}-cached")
if all_stored_ticks:
if all_stored_ticks[0][0] <= first_candle_ms: if not all_stored_ticks_df.empty:
last_cached_ms = all_stored_ticks[-1][0] if all_stored_ticks_df.iloc[0]['timestamp'] <= first_candle_ms:
last_cached_ms = all_stored_ticks_df.iloc[-1]['timestamp']
# only use cached if it's closer than first_candle_ms # only use cached if it's closer than first_candle_ms
since_ms = last_cached_ms if last_cached_ms > first_candle_ms else first_candle_ms since_ms = last_cached_ms if last_cached_ms > first_candle_ms else first_candle_ms
# doesn't go far enough # doesn't go far enough
else: else:
all_stored_ticks = [] all_stored_ticks_df = DataFrame(columns=DEFAULT_TRADES_COLUMNS + ['date'])
# from_id overrules with exchange set to id paginate # from_id overrules with exchange set to id paginate
# TODO: DEBUG: # TODO: DEBUG:
# since_ms = 1682609520000 # since_ms = 1695832200000
# from_id = None # from_id = None
# TODO: /DEBUG # TODO: /DEBUG
[ticks_pair, new_ticks]=self._download_trades_history(pair, [ticks_pair, new_ticks]=self._download_trades_history(pair,
@@ -2367,12 +2369,14 @@ class Exchange:
if new_ticks: if new_ticks:
drop_incomplete = False # TODO: remove, no incomplete trades drop_incomplete = False # TODO: remove, no incomplete trades
all_stored_ticks.extend(new_ticks) # drop 'date' column from stored ticks
all_stored_ticks_list = all_stored_ticks_df[DEFAULT_TRADES_COLUMNS].values.tolist()
all_stored_ticks_list.extend(new_ticks)
# NOTE: only process new trades # NOTE: only process new trades
# self._trades = until_first_candle(stored_trades) + fetch_trades # self._trades = until_first_candle(stored_trades) + fetch_trades
trades_df = self._process_trades_df(pair, timeframe, candle_type, all_stored_ticks, cache, drop_incomplete, first_candle_ms) trades_df = self._process_trades_df(pair, timeframe, candle_type, all_stored_ticks_list, cache, drop_incomplete, first_candle_ms)
results_df[(pair, timeframe, candle_type)] = trades_df results_df[(pair, timeframe, candle_type)] = trades_df
data_handler.trades_store(f"{pair}-cached", trades_df[DEFAULT_TRADES_COLUMNS].values.tolist()) data_handler.trades_store(f"{pair}-cached", trades_df[DEFAULT_TRADES_COLUMNS])
return results_df return results_df