mohammadian7/freqtrade
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ruff format: orderflow / public trades

Browse Source
This commit is contained in:
Joe Schr
2024-05-15 17:09:32 +02:00
parent 6c696e14f0
commit a56faf503b
13 changed files with 420 additions and 314 deletions
Download Patch File Download Diff File Expand all files Collapse all files
freqtrade/configuration/config_validation.py
+2 -2
View File
@@ -423,8 +423,8 @@ def _validate_consumers(conf: Dict[str, Any]) -> None:
def _validate_orderflow(conf: Dict[str, Any]) -> None: def _validate_orderflow(conf: Dict[str, Any]) -> None:
if conf.get('exchange', {}).get('use_public_trades'): if conf.get("exchange", {}).get("use_public_trades"):
if 'orderflow' not in conf: if "orderflow" not in conf:
raise ConfigurationError( raise ConfigurationError(
"Orderflow is a required configuration key when using public trades." "Orderflow is a required configuration key when using public trades."
) )
freqtrade/constants.py
+2 -1
View File
@@ -3,6 +3,7 @@
""" """
bot constants bot constants
""" """
from typing import Any, Dict, List, Literal, Optional, Tuple from typing import Any, Dict, List, Literal, Optional, Tuple
from freqtrade.enums import CandleType, PriceType, RPCMessageType from freqtrade.enums import CandleType, PriceType, RPCMessageType
@@ -531,7 +532,7 @@ CONF_SCHEMA = {
"stacked_imbalance_range": {"type": "number"}, "stacked_imbalance_range": {"type": "number"},
"imbalance_volume": {"type": "number"}, "imbalance_volume": {"type": "number"},
"imbalance_ratio": {"type": "number"}, "imbalance_ratio": {"type": "number"},
} },
}, },
}, },
"definitions": { "definitions": {
freqtrade/data/converter/__init__.py
+16 -16
View File
@@ -21,20 +21,20 @@ from freqtrade.data.converter.trade_converter import (
__all__ = [ __all__ = [
'clean_ohlcv_dataframe', "clean_ohlcv_dataframe",
'convert_ohlcv_format', "convert_ohlcv_format",
'ohlcv_fill_up_missing_data', "ohlcv_fill_up_missing_data",
'ohlcv_to_dataframe', "ohlcv_to_dataframe",
'order_book_to_dataframe', "order_book_to_dataframe",
'reduce_dataframe_footprint', "reduce_dataframe_footprint",
'trim_dataframe', "trim_dataframe",
'trim_dataframes', "trim_dataframes",
'convert_trades_format', "convert_trades_format",
'convert_trades_to_ohlcv', "convert_trades_to_ohlcv",
'populate_dataframe_with_trades', "populate_dataframe_with_trades",
'trades_convert_types', "trades_convert_types",
'trades_df_remove_duplicates', "trades_df_remove_duplicates",
'trades_dict_to_list', "trades_dict_to_list",
'trades_list_to_df', "trades_list_to_df",
'trades_to_ohlcv', "trades_to_ohlcv",
] ]
freqtrade/data/converter/orderflow.py
+105 -100
View File
@@ -1,6 +1,7 @@
""" """
Functions to convert orderflow data from public_trades Functions to convert orderflow data from public_trades
""" """
import logging import logging
import time import time
@@ -18,23 +19,24 @@ def _init_dataframe_with_trades_columns(dataframe: pd.DataFrame):
Populates a dataframe with trades columns Populates a dataframe with trades columns
:param dataframe: Dataframe to populate :param dataframe: Dataframe to populate
""" """
dataframe['trades'] = dataframe.apply(lambda _: [], axis=1) dataframe["trades"] = dataframe.apply(lambda _: [], axis=1)
dataframe['orderflow'] = dataframe.apply(lambda _: {}, axis=1) dataframe["orderflow"] = dataframe.apply(lambda _: {}, axis=1)
dataframe['bid'] = np.nan dataframe["bid"] = np.nan
dataframe['ask'] = np.nan dataframe["ask"] = np.nan
dataframe['delta'] = np.nan dataframe["delta"] = np.nan
dataframe['min_delta'] = np.nan dataframe["min_delta"] = np.nan
dataframe['max_delta'] = np.nan dataframe["max_delta"] = np.nan
dataframe['total_trades'] = np.nan dataframe["total_trades"] = np.nan
dataframe['stacked_imbalances_bid'] = np.nan dataframe["stacked_imbalances_bid"] = np.nan
dataframe['stacked_imbalances_ask'] = np.nan dataframe["stacked_imbalances_ask"] = np.nan
def _convert_timeframe_to_pandas_frequency(timeframe: str): def _convert_timeframe_to_pandas_frequency(timeframe: str):
# convert timeframe to format usable by pandas # convert timeframe to format usable by pandas
from freqtrade.exchange import timeframe_to_minutes from freqtrade.exchange import timeframe_to_minutes
timeframe_minutes = timeframe_to_minutes(timeframe) timeframe_minutes = timeframe_to_minutes(timeframe)
timeframe_frequency = f'{timeframe_minutes}min' timeframe_frequency = f"{timeframe_minutes}min"
return (timeframe_frequency, timeframe_minutes) return (timeframe_frequency, timeframe_minutes)
@@ -44,26 +46,26 @@ def _calculate_ohlcv_candle_start_and_end(df: pd.DataFrame, timeframe: str):
timeframe_frequency = timeframe_to_resample_freq(timeframe) timeframe_frequency = timeframe_to_resample_freq(timeframe)
# calculate ohlcv candle start and end # calculate ohlcv candle start and end
if df is not None and not df.empty: if df is not None and not df.empty:
df['datetime'] = pd.to_datetime(df['date'], unit='ms') df["datetime"] = pd.to_datetime(df["date"], unit="ms")
df['candle_start'] = df['datetime'].dt.floor(timeframe_frequency) df["candle_start"] = df["datetime"].dt.floor(timeframe_frequency)
# used in _now_is_time_to_refresh_trades # used in _now_is_time_to_refresh_trades
df['candle_end'] = df['candle_start'].apply( df["candle_end"] = df["candle_start"].apply(
lambda candle_start: timeframe_to_next_date(timeframe, candle_start) lambda candle_start: timeframe_to_next_date(timeframe, candle_start)
) )
df.drop(columns=["datetime"], inplace=True) df.drop(columns=["datetime"], inplace=True)
def populate_dataframe_with_trades(config: Config, def populate_dataframe_with_trades(
dataframe: pd.DataFrame, config: Config, dataframe: pd.DataFrame, trades: pd.DataFrame
trades: pd.DataFrame) -> pd.DataFrame: ) -> pd.DataFrame:
""" """
Populates a dataframe with trades Populates a dataframe with trades
:param dataframe: Dataframe to populate :param dataframe: Dataframe to populate
:param trades: Trades to populate with :param trades: Trades to populate with
:return: Dataframe with trades populated :return: Dataframe with trades populated
""" """
config_orderflow = config['orderflow'] config_orderflow = config["orderflow"]
timeframe = config['timeframe'] timeframe = config["timeframe"]
# create columns for trades # create columns for trades
_init_dataframe_with_trades_columns(dataframe) _init_dataframe_with_trades_columns(dataframe)
@@ -79,8 +81,7 @@ def populate_dataframe_with_trades(config: Config,
trades.reset_index(inplace=True, drop=True) trades.reset_index(inplace=True, drop=True)
# group trades by candle start # group trades by candle start
trades_grouped_by_candle_start = trades.groupby( trades_grouped_by_candle_start = trades.groupby("candle_start", group_keys=False)
'candle_start', group_keys=False)
for candle_start in trades_grouped_by_candle_start.groups: for candle_start in trades_grouped_by_candle_start.groups:
trades_grouped_df = trades[candle_start == trades["candle_start"]] trades_grouped_df = trades[candle_start == trades["candle_start"]]
@@ -94,36 +95,50 @@ def populate_dataframe_with_trades(config: Config,
if candle_next not in trades_grouped_by_candle_start.groups: if candle_next not in trades_grouped_by_candle_start.groups:
logger.warning( logger.warning(
f"candle at {candle_start} with {len(trades_grouped_df)} trades " f"candle at {candle_start} with {len(trades_grouped_df)} trades "
f"might be unfinished, because no finished trades at {candle_next}") f"might be unfinished, because no finished trades at {candle_next}"
)
# add trades to each candle # add trades to each candle
df.loc[is_between, 'trades'] = df.loc[is_between, df.loc[is_between, "trades"] = df.loc[is_between, "trades"].apply(
'trades'].apply(lambda _: trades_grouped_df) lambda _: trades_grouped_df
)
# calculate orderflow for each candle # calculate orderflow for each candle
df.loc[is_between, 'orderflow'] = df.loc[is_between, 'orderflow'].apply( df.loc[is_between, "orderflow"] = df.loc[is_between, "orderflow"].apply(
lambda _: trades_to_volumeprofile_with_total_delta_bid_ask( lambda _: trades_to_volumeprofile_with_total_delta_bid_ask(
pd.DataFrame(trades_grouped_df), pd.DataFrame(trades_grouped_df), scale=config_orderflow["scale"]
scale=config_orderflow['scale'])) )
)
# calculate imbalances for each candle's orderflow # calculate imbalances for each candle's orderflow
df.loc[is_between, 'imbalances'] = df.loc[is_between, 'orderflow'].apply( df.loc[is_between, "imbalances"] = df.loc[is_between, "orderflow"].apply(
lambda x: trades_orderflow_to_imbalances( lambda x: trades_orderflow_to_imbalances(
x, x,
imbalance_ratio=config_orderflow['imbalance_ratio'], imbalance_ratio=config_orderflow["imbalance_ratio"],
imbalance_volume=config_orderflow['imbalance_volume'])) imbalance_volume=config_orderflow["imbalance_volume"],
)
)
_stacked_imb = config_orderflow['stacked_imbalance_range'] _stacked_imb = config_orderflow["stacked_imbalance_range"]
df.loc[is_between, 'stacked_imbalances_bid'] = df.loc[ df.loc[is_between, "stacked_imbalances_bid"] = df.loc[
is_between, 'imbalances'].apply(lambda x: stacked_imbalance_bid( is_between, "imbalances"
x, ].apply(lambda x: stacked_imbalance_bid(x, stacked_imbalance_range=_stacked_imb))
stacked_imbalance_range=_stacked_imb)) df.loc[is_between, "stacked_imbalances_ask"] = df.loc[
df.loc[is_between, 'stacked_imbalances_ask'] = df.loc[ is_between, "imbalances"
is_between, 'imbalances'].apply( ].apply(lambda x: stacked_imbalance_ask(x, stacked_imbalance_range=_stacked_imb))
lambda x: stacked_imbalance_ask(x, stacked_imbalance_range=_stacked_imb))
buy = df.loc[is_between, 'bid'].apply(lambda _: np.where( buy = df.loc[is_between, "bid"].apply(
trades_grouped_df['side'].str.contains('buy'), 0, trades_grouped_df['amount'])) lambda _: np.where(
sell = df.loc[is_between, 'ask'].apply(lambda _: np.where( trades_grouped_df["side"].str.contains("buy"),
trades_grouped_df['side'].str.contains('sell'), 0, trades_grouped_df['amount'])) 0,
trades_grouped_df["amount"],
)
)
sell = df.loc[is_between, "ask"].apply(
lambda _: np.where(
trades_grouped_df["side"].str.contains("sell"),
0,
trades_grouped_df["amount"],
)
)
deltas_per_trade = sell - buy deltas_per_trade = sell - buy
min_delta = 0 min_delta = 0
max_delta = 0 max_delta = 0
@@ -135,29 +150,27 @@ def populate_dataframe_with_trades(config: Config,
max_delta = delta max_delta = delta
if delta < min_delta: if delta < min_delta:
min_delta = delta min_delta = delta
df.loc[is_between, 'max_delta'] = max_delta df.loc[is_between, "max_delta"] = max_delta
df.loc[is_between, 'min_delta'] = min_delta df.loc[is_between, "min_delta"] = min_delta
df.loc[is_between, 'bid'] = np.where(trades_grouped_df['side'].str.contains( df.loc[is_between, "bid"] = np.where(
'buy'), 0, trades_grouped_df['amount']).sum() trades_grouped_df["side"].str.contains("buy"), 0, trades_grouped_df["amount"]
df.loc[is_between, 'ask'] = np.where(trades_grouped_df['side'].str.contains( ).sum()
'sell'), 0, trades_grouped_df['amount']).sum() df.loc[is_between, "ask"] = np.where(
df.loc[is_between, 'delta'] = df.loc[is_between, trades_grouped_df["side"].str.contains("sell"), 0, trades_grouped_df["amount"]
'ask'] - df.loc[is_between, 'bid'] ).sum()
df.loc[is_between, "delta"] = df.loc[is_between, "ask"] - df.loc[is_between, "bid"]
min_delta = np.min(deltas_per_trade) min_delta = np.min(deltas_per_trade)
max_delta = np.max(deltas_per_trade) max_delta = np.max(deltas_per_trade)
df.loc[is_between, 'total_trades'] = len(trades_grouped_df) df.loc[is_between, "total_trades"] = len(trades_grouped_df)
# copy to avoid memory leaks # copy to avoid memory leaks
dataframe.loc[is_between] = df.loc[is_between].copy() dataframe.loc[is_between] = df.loc[is_between].copy()
else: else:
logger.debug( logger.debug(f"Found NO candles for trades starting with {candle_start}")
f"Found NO candles for trades starting with {candle_start}") logger.debug(f"trades.groups_keys in {time.time() - start_time} seconds")
logger.debug(
f"trades.groups_keys in {time.time() - start_time} seconds")
logger.debug( logger.debug(f"trades.singleton_iterate in {time.time() - start_time} seconds")
f"trades.singleton_iterate in {time.time() - start_time} seconds")
except Exception as e: except Exception as e:
logger.exception("Error populating dataframe with trades:", e) logger.exception("Error populating dataframe with trades:", e)
@@ -174,27 +187,24 @@ def trades_to_volumeprofile_with_total_delta_bid_ask(trades: pd.DataFrame, scale
""" """
df = pd.DataFrame([], columns=DEFAULT_ORDERFLOW_COLUMNS) df = pd.DataFrame([], columns=DEFAULT_ORDERFLOW_COLUMNS)
# create bid, ask where side is sell or buy # create bid, ask where side is sell or buy
df['bid_amount'] = np.where( df["bid_amount"] = np.where(trades["side"].str.contains("buy"), 0, trades["amount"])
trades['side'].str.contains('buy'), 0, trades['amount']) df["ask_amount"] = np.where(trades["side"].str.contains("sell"), 0, trades["amount"])
df['ask_amount'] = np.where( df["bid"] = np.where(trades["side"].str.contains("buy"), 0, 1)
trades['side'].str.contains('sell'), 0, trades['amount']) df["ask"] = np.where(trades["side"].str.contains("sell"), 0, 1)
df['bid'] = np.where(trades['side'].str.contains('buy'), 0, 1)
df['ask'] = np.where(trades['side'].str.contains('sell'), 0, 1)
# round the prices to the nearest multiple of the scale # round the prices to the nearest multiple of the scale
df['price'] = ((trades['price'] / scale).round() df["price"] = ((trades["price"] / scale).round() * scale).astype("float64").values
* scale).astype('float64').values
if df.empty: if df.empty:
df['total'] = np.nan df["total"] = np.nan
df['delta'] = np.nan df["delta"] = np.nan
return df return df
df['delta'] = df['ask_amount'] - df['bid_amount'] df["delta"] = df["ask_amount"] - df["bid_amount"]
df['total_volume'] = df['ask_amount'] + df['bid_amount'] df["total_volume"] = df["ask_amount"] + df["bid_amount"]
df['total_trades'] = df['ask'] + df['bid'] df["total_trades"] = df["ask"] + df["bid"]
# group to bins aka apply scale # group to bins aka apply scale
df = df.groupby('price').sum(numeric_only=True) df = df.groupby("price").sum(numeric_only=True)
return df return df
@@ -209,54 +219,49 @@ def trades_orderflow_to_imbalances(df: pd.DataFrame, imbalance_ratio: int, imbal
ask = df.ask.shift(-1) ask = df.ask.shift(-1)
bid_imbalance = (bid / ask) > (imbalance_ratio / 100) bid_imbalance = (bid / ask) > (imbalance_ratio / 100)
# overwrite bid_imbalance with False if volume is not big enough # overwrite bid_imbalance with False if volume is not big enough
bid_imbalance_filtered = np.where( bid_imbalance_filtered = np.where(df.total_volume < imbalance_volume, False, bid_imbalance)
df.total_volume < imbalance_volume, False, bid_imbalance)
ask_imbalance = (ask / bid) > (imbalance_ratio / 100) ask_imbalance = (ask / bid) > (imbalance_ratio / 100)
# overwrite ask_imbalance with False if volume is not big enough # overwrite ask_imbalance with False if volume is not big enough
ask_imbalance_filtered = np.where( ask_imbalance_filtered = np.where(df.total_volume < imbalance_volume, False, ask_imbalance)
df.total_volume < imbalance_volume, False, ask_imbalance) dataframe = pd.DataFrame(
dataframe = pd.DataFrame({ {"bid_imbalance": bid_imbalance_filtered, "ask_imbalance": ask_imbalance_filtered},
"bid_imbalance": bid_imbalance_filtered, index=df.index,
"ask_imbalance": ask_imbalance_filtered
}, index=df.index,
) )
return dataframe return dataframe
def stacked_imbalance(df: pd.DataFrame, def stacked_imbalance(
label: str, df: pd.DataFrame, label: str, stacked_imbalance_range: int, should_reverse: bool
stacked_imbalance_range: int, ):
should_reverse: bool):
""" """
y * (y.groupby((y != y.shift()).cumsum()).cumcount() + 1) y * (y.groupby((y != y.shift()).cumsum()).cumcount() + 1)
https://stackoverflow.com/questions/27626542/counting-consecutive-positive-values-in-python-pandas-array https://stackoverflow.com/questions/27626542/counting-consecutive-positive-values-in-python-pandas-array
""" """
imbalance = df[f'{label}_imbalance'] imbalance = df[f"{label}_imbalance"]
int_series = pd.Series(np.where(imbalance, 1, 0)) int_series = pd.Series(np.where(imbalance, 1, 0))
stacked = ( stacked = int_series * (
int_series * ( int_series.groupby((int_series != int_series.shift()).cumsum()).cumcount() + 1
int_series.groupby(
(int_series != int_series.shift()).cumsum()).cumcount() + 1
)
) )
max_stacked_imbalance_idx = stacked.index[stacked >= max_stacked_imbalance_idx = stacked.index[stacked >= stacked_imbalance_range]
stacked_imbalance_range]
stacked_imbalance_price = np.nan stacked_imbalance_price = np.nan
if not max_stacked_imbalance_idx.empty: if not max_stacked_imbalance_idx.empty:
idx = max_stacked_imbalance_idx[0] if not should_reverse else np.flipud( idx = (
max_stacked_imbalance_idx)[0] max_stacked_imbalance_idx[0]
if not should_reverse
else np.flipud(max_stacked_imbalance_idx)[0]
)
stacked_imbalance_price = imbalance.index[idx] stacked_imbalance_price = imbalance.index[idx]
return stacked_imbalance_price return stacked_imbalance_price
def stacked_imbalance_bid(df: pd.DataFrame, stacked_imbalance_range: int): def stacked_imbalance_bid(df: pd.DataFrame, stacked_imbalance_range: int):
return stacked_imbalance(df, 'bid', stacked_imbalance_range, should_reverse=False) return stacked_imbalance(df, "bid", stacked_imbalance_range, should_reverse=False)
def stacked_imbalance_ask(df: pd.DataFrame, stacked_imbalance_range: int): def stacked_imbalance_ask(df: pd.DataFrame, stacked_imbalance_range: int):
return stacked_imbalance(df, 'ask', stacked_imbalance_range, should_reverse=True) return stacked_imbalance(df, "ask", stacked_imbalance_range, should_reverse=True)
def orderflow_to_volume_profile(df: pd.DataFrame): def orderflow_to_volume_profile(df: pd.DataFrame):
@@ -264,9 +269,9 @@ def orderflow_to_volume_profile(df: pd.DataFrame):
:param orderflow: dataframe :param orderflow: dataframe
:return: volume profile dataframe :return: volume profile dataframe
""" """
bid = df.groupby('level').bid.sum() bid = df.groupby("level").bid.sum()
ask = df.groupby('level').ask.sum() ask = df.groupby("level").ask.sum()
df.groupby('level')['level'].sum() df.groupby("level")["level"].sum()
delta = df.groupby('level').ask.sum() - df.groupby('level').bid.sum() delta = df.groupby("level").ask.sum() - df.groupby("level").bid.sum()
df = pd.DataFrame({'bid': bid, 'ask': ask, 'delta': delta}) df = pd.DataFrame({"bid": bid, "ask": ask, "delta": delta})
return df return df
freqtrade/data/dataprovider.py
+15 -13
View File
@@ -455,7 +455,7 @@ class DataProvider:
Refresh latest trades data (if enabled in config) Refresh latest trades data (if enabled in config)
""" """
use_public_trades = self._config.get('exchange', {}).get('use_public_trades', False) use_public_trades = self._config.get("exchange", {}).get("use_public_trades", False)
if use_public_trades: if use_public_trades:
if self._exchange: if self._exchange:
self._exchange.refresh_latest_trades(pairlist) self._exchange.refresh_latest_trades(pairlist)
@@ -497,11 +497,7 @@ class DataProvider:
return DataFrame() return DataFrame()
def trades( def trades(
self, self, pair: str, timeframe: Optional[str] = None, copy: bool = True, candle_type: str = ""
pair: str,
timeframe: Optional[str] = None,
copy: bool = True,
candle_type: str = ''
) -> DataFrame: ) -> DataFrame:
""" """
Get candle (TRADES) data for the given pair as DataFrame Get candle (TRADES) data for the given pair as DataFrame
@@ -515,17 +511,23 @@ class DataProvider:
if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
if self._exchange is None: if self._exchange is None:
raise OperationalException(NO_EXCHANGE_EXCEPTION) raise OperationalException(NO_EXCHANGE_EXCEPTION)
_candle_type = CandleType.from_string( _candle_type = (
candle_type) if candle_type != '' else self._config['candle_type_def'] CandleType.from_string(candle_type)
if candle_type != ""
else self._config["candle_type_def"]
)
return self._exchange.trades( return self._exchange.trades(
(pair, timeframe or self._config['timeframe'], _candle_type), (pair, timeframe or self._config["timeframe"], _candle_type), copy=copy
copy=copy
) )
elif self.runmode in (RunMode.BACKTEST, RunMode.HYPEROPT): elif self.runmode in (RunMode.BACKTEST, RunMode.HYPEROPT):
_candle_type = CandleType.from_string( _candle_type = (
candle_type) if candle_type != '' else self._config['candle_type_def'] CandleType.from_string(candle_type)
if candle_type != ""
else self._config["candle_type_def"]
)
data_handler = get_datahandler( data_handler = get_datahandler(
self._config['datadir'], data_format=self._config['dataformat_trades']) self._config["datadir"], data_format=self._config["dataformat_trades"]
)
trades_df = data_handler.trades_load(pair, TradingMode.FUTURES) trades_df = data_handler.trades_load(pair, TradingMode.FUTURES)
return trades_df return trades_df
freqtrade/data/history/history_utils.py
+1 -3
View File
@@ -480,9 +480,7 @@ def _download_trades_history(
return True return True
except Exception: except Exception:
logger.exception( logger.exception(f'Failed to download and store historic trades for pair: "{pair}". ')
f'Failed to download and store historic trades for pair: "{pair}". '
)
return False return False
freqtrade/exchange/exchange.py
+48 -36
View File
@@ -2541,19 +2541,20 @@ class Exchange:
# fetch Trade data stuff # fetch Trade data stuff
def needed_candle_ms(self, timeframe: str, candle_type: CandleType): def needed_candle_ms(self, timeframe: str, candle_type: CandleType):
one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit( one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe, candle_type)
timeframe, candle_type)
move_to = one_call * self.required_candle_call_count move_to = one_call * self.required_candle_call_count
now = timeframe_to_next_date(timeframe) now = timeframe_to_next_date(timeframe)
return int((now - timedelta(seconds=move_to // 1000)).timestamp() * 1000) return int((now - timedelta(seconds=move_to // 1000)).timestamp() * 1000)
def _process_trades_df(self, def _process_trades_df(
pair: str, self,
timeframe: str, pair: str,
c_type: CandleType, timeframe: str,
ticks: List[List], c_type: CandleType,
cache: bool, ticks: List[List],
first_required_candle_date: Optional[int]) -> DataFrame: cache: bool,
first_required_candle_date: Optional[int],
) -> DataFrame:
# keeping parsed dataframe in cache # keeping parsed dataframe in cache
trades_df = trades_list_to_df(ticks, True) trades_df = trades_list_to_df(ticks, True)
@@ -2563,22 +2564,23 @@ class Exchange:
# Reassign so we return the updated, combined df # Reassign so we return the updated, combined df
combined_df = concat([old, trades_df], axis=0) combined_df = concat([old, trades_df], axis=0)
logger.debug(f"Clean duplicated ticks from Trades data {pair}") logger.debug(f"Clean duplicated ticks from Trades data {pair}")
trades_df = DataFrame(trades_df_remove_duplicates(combined_df), trades_df = DataFrame(
columns=combined_df.columns) trades_df_remove_duplicates(combined_df), columns=combined_df.columns
)
# Age out old candles # Age out old candles
if first_required_candle_date: if first_required_candle_date:
# slice of older dates # slice of older dates
trades_df = trades_df[ trades_df = trades_df[first_required_candle_date < trades_df["timestamp"]]
first_required_candle_date < trades_df['timestamp']]
trades_df = trades_df.reset_index(drop=True) trades_df = trades_df.reset_index(drop=True)
self._trades[(pair, timeframe, c_type)] = trades_df self._trades[(pair, timeframe, c_type)] = trades_df
return trades_df return trades_df
def refresh_latest_trades(self, def refresh_latest_trades(
pair_list: ListPairsWithTimeframes, self,
*, pair_list: ListPairsWithTimeframes,
cache: bool = True, *,
) -> Dict[PairWithTimeframe, DataFrame]: cache: bool = True,
) -> Dict[PairWithTimeframe, DataFrame]:
""" """
Refresh in-memory TRADES asynchronously and set `_trades` with the result Refresh in-memory TRADES asynchronously and set `_trades` with the result
Loops asynchronously over pair_list and downloads all pairs async (semi-parallel). Loops asynchronously over pair_list and downloads all pairs async (semi-parallel).
@@ -2588,24 +2590,27 @@ class Exchange:
:return: Dict of [{(pair, timeframe): Dataframe}] :return: Dict of [{(pair, timeframe): Dataframe}]
""" """
from freqtrade.data.history import get_datahandler from freqtrade.data.history import get_datahandler
data_handler = get_datahandler( data_handler = get_datahandler(
self._config['datadir'], data_format=self._config['dataformat_trades'] self._config["datadir"], data_format=self._config["dataformat_trades"]
) )
logger.debug("Refreshing TRADES data for %d pairs", len(pair_list)) logger.debug("Refreshing TRADES data for %d pairs", len(pair_list))
since_ms = None since_ms = None
results_df = {} results_df = {}
for pair, timeframe, candle_type in set(pair_list): for pair, timeframe, candle_type in set(pair_list):
new_ticks: List = [] new_ticks: List = []
all_stored_ticks_df = DataFrame(columns=DEFAULT_TRADES_COLUMNS + ['date']) all_stored_ticks_df = DataFrame(columns=DEFAULT_TRADES_COLUMNS + ["date"])
first_candle_ms = self.needed_candle_ms(timeframe, candle_type) first_candle_ms = self.needed_candle_ms(timeframe, candle_type)
# refresh, if # refresh, if
# a. not in _trades # a. not in _trades
# b. no cache used # b. no cache used
# c. need new data # c. need new data
is_in_cache = (pair, timeframe, candle_type) in self._trades is_in_cache = (pair, timeframe, candle_type) in self._trades
if (not is_in_cache if (
or not cache not is_in_cache
or self._now_is_time_to_refresh_trades(pair, timeframe, candle_type)): or not cache
or self._now_is_time_to_refresh_trades(pair, timeframe, candle_type)
):
logger.debug(f"Refreshing TRADES data for {pair}") logger.debug(f"Refreshing TRADES data for {pair}")
# fetch trades since latest _trades and # fetch trades since latest _trades and
# store together with existing trades # store together with existing trades
@@ -2613,26 +2618,29 @@ class Exchange:
until = None until = None
from_id = None from_id = None
if is_in_cache: if is_in_cache:
from_id = self._trades[(pair, timeframe, candle_type)].iloc[-1]['id'] from_id = self._trades[(pair, timeframe, candle_type)].iloc[-1]["id"]
until = dt_ts() # now until = dt_ts() # now
else: else:
until = int(timeframe_to_prev_date(timeframe).timestamp()) * 1000 until = int(timeframe_to_prev_date(timeframe).timestamp()) * 1000
all_stored_ticks_df = data_handler.trades_load( all_stored_ticks_df = data_handler.trades_load(
f"{pair}-cached", self.trading_mode) f"{pair}-cached", self.trading_mode
)
if not all_stored_ticks_df.empty: if not all_stored_ticks_df.empty:
if all_stored_ticks_df.iloc[0]['timestamp'] <= first_candle_ms: if all_stored_ticks_df.iloc[0]["timestamp"] <= first_candle_ms:
last_cached_ms = all_stored_ticks_df.iloc[-1]['timestamp'] last_cached_ms = all_stored_ticks_df.iloc[-1]["timestamp"]
# only use cached if it's closer than first_candle_ms # only use cached if it's closer than first_candle_ms
since_ms = ( since_ms = (
last_cached_ms if last_cached_ms > first_candle_ms last_cached_ms
if last_cached_ms > first_candle_ms
else first_candle_ms else first_candle_ms
) )
# doesn't go far enough # doesn't go far enough
else: else:
all_stored_ticks_df = DataFrame( all_stored_ticks_df = DataFrame(
columns=DEFAULT_TRADES_COLUMNS + ['date']) columns=DEFAULT_TRADES_COLUMNS + ["date"]
)
# from_id overrules with exchange set to id paginate # from_id overrules with exchange set to id paginate
# TODO: DEBUG: # TODO: DEBUG:
@@ -2643,7 +2651,7 @@ class Exchange:
pair, pair,
since=since_ms if since_ms else first_candle_ms, since=since_ms if since_ms else first_candle_ms,
until=until, until=until,
from_id=from_id from_id=from_id,
) )
except Exception as e: except Exception as e:
@@ -2652,8 +2660,9 @@ class Exchange:
raise e raise e
if new_ticks: if new_ticks:
all_stored_ticks_list = all_stored_ticks_df[DEFAULT_TRADES_COLUMNS all_stored_ticks_list = all_stored_ticks_df[
].values.tolist() DEFAULT_TRADES_COLUMNS
].values.tolist()
all_stored_ticks_list.extend(new_ticks) all_stored_ticks_list.extend(new_ticks)
trades_df = self._process_trades_df( trades_df = self._process_trades_df(
pair, pair,
@@ -2661,11 +2670,12 @@ class Exchange:
candle_type, candle_type,
all_stored_ticks_list, all_stored_ticks_list,
cache, cache,
first_required_candle_date=first_candle_ms first_required_candle_date=first_candle_ms,
) )
results_df[(pair, timeframe, candle_type)] = trades_df results_df[(pair, timeframe, candle_type)] = trades_df
data_handler.trades_store( data_handler.trades_store(
f"{pair}-cached", trades_df[DEFAULT_TRADES_COLUMNS], self.trading_mode) f"{pair}-cached", trades_df[DEFAULT_TRADES_COLUMNS], self.trading_mode
)
else: else:
raise OperationalException("no new ticks") raise OperationalException("no new ticks")
@@ -2677,8 +2687,10 @@ class Exchange:
) -> bool: # Timeframe in seconds ) -> bool: # Timeframe in seconds
trades = self.trades((pair, timeframe, candle_type), False) trades = self.trades((pair, timeframe, candle_type), False)
pair_last_refreshed = int(trades.iloc[-1]["timestamp"]) pair_last_refreshed = int(trades.iloc[-1]["timestamp"])
full_candle = int(timeframe_to_next_date( full_candle = (
timeframe, dt_from_ts(pair_last_refreshed)).timestamp()) * 1000 int(timeframe_to_next_date(timeframe, dt_from_ts(pair_last_refreshed)).timestamp())
* 1000
)
now = dt_ts() now = dt_ts()
return full_candle <= now return full_candle <= now
freqtrade/strategy/interface.py
+4 -7
View File
@@ -1594,17 +1594,14 @@ class IStrategy(ABC, HyperStrategyMixin):
return dataframe return dataframe
def _if_enabled_populate_trades(self, dataframe: DataFrame, metadata: dict): def _if_enabled_populate_trades(self, dataframe: DataFrame, metadata: dict):
use_public_trades = self.config.get('exchange', {}).get('use_public_trades', False) use_public_trades = self.config.get("exchange", {}).get("use_public_trades", False)
if use_public_trades: if use_public_trades:
trades = self.dp.trades(pair=metadata['pair'], copy=False) trades = self.dp.trades(pair=metadata["pair"], copy=False)
config = self.config config = self.config
config['timeframe'] = self.timeframe config["timeframe"] = self.timeframe
# TODO: slice trades to size of dataframe for faster backtesting # TODO: slice trades to size of dataframe for faster backtesting
dataframe = populate_dataframe_with_trades( dataframe = populate_dataframe_with_trades(config, dataframe, trades)
config,
dataframe,
trades)
logger.debug("Populated dataframe with trades.") logger.debug("Populated dataframe with trades.")
tests/data/test_converter_public_trades.py
+200 -109
View File
@@ -11,49 +11,68 @@ from freqtrade.data.converter.trade_converter import trades_list_to_df
BIN_SIZE_SCALE = 0.5 BIN_SIZE_SCALE = 0.5
def read_csv(filename, converter_columns: list = ['side', 'type']): def read_csv(filename, converter_columns: list = ["side", "type"]):
return pd.read_csv(filename, skipinitialspace=True, infer_datetime_format=True, index_col=0, return pd.read_csv(
parse_dates=True, converters={col: str.strip for col in converter_columns}) filename,
skipinitialspace=True,
infer_datetime_format=True,
index_col=0,
parse_dates=True,
converters={col: str.strip for col in converter_columns},
)
@pytest.fixture @pytest.fixture
def populate_dataframe_with_trades_dataframe(testdatadir): def populate_dataframe_with_trades_dataframe(testdatadir):
return pd.read_feather(testdatadir / 'orderflow/populate_dataframe_with_trades_DF.feather') return pd.read_feather(testdatadir / "orderflow/populate_dataframe_with_trades_DF.feather")
@pytest.fixture @pytest.fixture
def populate_dataframe_with_trades_trades(testdatadir): def populate_dataframe_with_trades_trades(testdatadir):
return pd.read_feather(testdatadir / 'orderflow/populate_dataframe_with_trades_TRADES.feather') return pd.read_feather(testdatadir / "orderflow/populate_dataframe_with_trades_TRADES.feather")
@pytest.fixture @pytest.fixture
def candles(testdatadir): def candles(testdatadir):
return pd.read_json(testdatadir / 'orderflow/candles.json').copy() return pd.read_json(testdatadir / "orderflow/candles.json").copy()
@pytest.fixture @pytest.fixture
def public_trades_list(testdatadir): def public_trades_list(testdatadir):
return read_csv(testdatadir / 'orderflow/public_trades_list.csv').copy() return read_csv(testdatadir / "orderflow/public_trades_list.csv").copy()
@pytest.fixture @pytest.fixture
def public_trades_list_simple(testdatadir): def public_trades_list_simple(testdatadir):
return read_csv(testdatadir / 'orderflow/public_trades_list_simple_example.csv').copy() return read_csv(testdatadir / "orderflow/public_trades_list_simple_example.csv").copy()
def test_public_trades_columns_before_change( def test_public_trades_columns_before_change(
populate_dataframe_with_trades_dataframe, populate_dataframe_with_trades_dataframe, populate_dataframe_with_trades_trades
populate_dataframe_with_trades_trades): ):
assert populate_dataframe_with_trades_dataframe.columns.tolist() == [ assert populate_dataframe_with_trades_dataframe.columns.tolist() == [
'date', 'open', 'high', 'low', 'close', 'volume'] "date",
"open",
"high",
"low",
"close",
"volume",
]
assert populate_dataframe_with_trades_trades.columns.tolist() == [ assert populate_dataframe_with_trades_trades.columns.tolist() == [
'timestamp', 'id', 'type', 'side', 'price', "timestamp",
'amount', 'cost', 'date'] "id",
"type",
"side",
"price",
"amount",
"cost",
"date",
]
def test_public_trades_mock_populate_dataframe_with_trades__check_orderflow( def test_public_trades_mock_populate_dataframe_with_trades__check_orderflow(
populate_dataframe_with_trades_dataframe, populate_dataframe_with_trades_dataframe, populate_dataframe_with_trades_trades
populate_dataframe_with_trades_trades): ):
""" """
Tests the `populate_dataframe_with_trades` function's order flow calculation. Tests the `populate_dataframe_with_trades` function's order flow calculation.
@@ -64,24 +83,25 @@ def test_public_trades_mock_populate_dataframe_with_trades__check_orderflow(
dataframe = populate_dataframe_with_trades_dataframe.copy() dataframe = populate_dataframe_with_trades_dataframe.copy()
trades = populate_dataframe_with_trades_trades.copy() trades = populate_dataframe_with_trades_trades.copy()
# Convert the 'date' column to datetime format with milliseconds # Convert the 'date' column to datetime format with milliseconds
dataframe['date'] = pd.to_datetime( dataframe["date"] = pd.to_datetime(dataframe["date"], unit="ms")
dataframe['date'], unit='ms')
# Select the last rows and reset the index (optional, depends on usage) # Select the last rows and reset the index (optional, depends on usage)
dataframe = dataframe.copy().tail().reset_index(drop=True) dataframe = dataframe.copy().tail().reset_index(drop=True)
# Define the configuration for order flow calculation # Define the configuration for order flow calculation
config = {'timeframe': '5m', config = {
'orderflow': { "timeframe": "5m",
'scale': 0.005, "orderflow": {
'imbalance_volume': 0, "scale": 0.005,
'imbalance_ratio': 300, "imbalance_volume": 0,
'stacked_imbalance_range': 3 "imbalance_ratio": 300,
}} "stacked_imbalance_range": 3,
},
}
# Apply the function to populate the data frame with order flow data # Apply the function to populate the data frame with order flow data
df = populate_dataframe_with_trades(config, dataframe, trades) df = populate_dataframe_with_trades(config, dataframe, trades)
# Extract results from the first row of the DataFrame # Extract results from the first row of the DataFrame
results = df.iloc[0] results = df.iloc[0]
t = results['trades'] t = results["trades"]
of = results['orderflow'] of = results["orderflow"]
# Assert basic properties of the results # Assert basic properties of the results
assert 0 != len(results) assert 0 != len(results)
@@ -98,7 +118,7 @@ def test_public_trades_mock_populate_dataframe_with_trades__check_orderflow(
assert [0.0, 1.0, 0.103, 0.0, 0.103, 0.103, 1.0] == of.iloc[-1].values.tolist() assert [0.0, 1.0, 0.103, 0.0, 0.103, 0.103, 1.0] == of.iloc[-1].values.tolist()
# Extract order flow from the last row of the DataFrame # Extract order flow from the last row of the DataFrame
of = df.iloc[-1]['orderflow'] of = df.iloc[-1]["orderflow"]
# Assert number of order flow data points in the last row # Assert number of order flow data points in the last row
assert 19 == len(of) # Assert expected number of data points assert 19 == len(of) # Assert expected number of data points
@@ -107,42 +127,49 @@ def test_public_trades_mock_populate_dataframe_with_trades__check_orderflow(
assert [1.0, 0.0, -12.536, 12.536, 0.0, 12.536, 1.0] == of.iloc[0].values.tolist() assert [1.0, 0.0, -12.536, 12.536, 0.0, 12.536, 1.0] == of.iloc[0].values.tolist()
# Assert specific order flow values at the end of the last row # Assert specific order flow values at the end of the last row
assert [4.0, 3.0, -40.94800000000001, 59.18200000000001, assert [
18.233999999999998, 77.41600000000001, 7.0] == of.iloc[-1].values.tolist() 4.0,
3.0,
-40.94800000000001,
59.18200000000001,
18.233999999999998,
77.41600000000001,
7.0,
] == of.iloc[-1].values.tolist()
# --- Delta and Other Results --- # --- Delta and Other Results ---
# Assert delta value from the first row # Assert delta value from the first row
assert -50.519000000000005 == results['delta'] assert -50.519000000000005 == results["delta"]
# Assert min and max delta values from the first row # Assert min and max delta values from the first row
assert -79.469 == results['min_delta'] assert -79.469 == results["min_delta"]
assert 17.298 == results['max_delta'] assert 17.298 == results["max_delta"]
# Assert that stacked imbalances are NaN (not applicable in this test) # Assert that stacked imbalances are NaN (not applicable in this test)
assert np.isnan(results['stacked_imbalances_bid']) assert np.isnan(results["stacked_imbalances_bid"])
assert np.isnan(results['stacked_imbalances_ask']) assert np.isnan(results["stacked_imbalances_ask"])
# Repeat assertions for the third from last row # Repeat assertions for the third from last row
results = df.iloc[-2] results = df.iloc[-2]
assert -20.86200000000008 == results['delta'] assert -20.86200000000008 == results["delta"]
assert -54.55999999999999 == results['min_delta'] assert -54.55999999999999 == results["min_delta"]
assert 82.842 == results['max_delta'] assert 82.842 == results["max_delta"]
assert 234.99 == results['stacked_imbalances_bid'] assert 234.99 == results["stacked_imbalances_bid"]
assert 234.96 == results['stacked_imbalances_ask'] assert 234.96 == results["stacked_imbalances_ask"]
# Repeat assertions for the last row # Repeat assertions for the last row
results = df.iloc[-1] results = df.iloc[-1]
assert -49.30200000000002 == results['delta'] assert -49.30200000000002 == results["delta"]
assert -70.222 == results['min_delta'] assert -70.222 == results["min_delta"]
assert 11.213000000000003 == results['max_delta'] assert 11.213000000000003 == results["max_delta"]
assert np.isnan(results['stacked_imbalances_bid']) assert np.isnan(results["stacked_imbalances_bid"])
assert np.isnan(results['stacked_imbalances_ask']) assert np.isnan(results["stacked_imbalances_ask"])
def test_public_trades_trades_mock_populate_dataframe_with_trades__check_trades( def test_public_trades_trades_mock_populate_dataframe_with_trades__check_trades(
populate_dataframe_with_trades_dataframe, populate_dataframe_with_trades_dataframe, populate_dataframe_with_trades_trades
populate_dataframe_with_trades_trades): ):
""" """
Tests the `populate_dataframe_with_trades` function's handling of trades, Tests the `populate_dataframe_with_trades` function's handling of trades,
ensuring correct integration of trades data into the generated DataFrame. ensuring correct integration of trades data into the generated DataFrame.
@@ -155,7 +182,7 @@ def test_public_trades_trades_mock_populate_dataframe_with_trades__check_trades(
# --- Data Preparation --- # --- Data Preparation ---
# Convert the 'date' column to datetime format with milliseconds # Convert the 'date' column to datetime format with milliseconds
dataframe['date'] = pd.to_datetime(dataframe['date'], unit='ms') dataframe["date"] = pd.to_datetime(dataframe["date"], unit="ms")
# Select the final row of the DataFrame # Select the final row of the DataFrame
dataframe = dataframe.tail().reset_index(drop=True) dataframe = dataframe.tail().reset_index(drop=True)
@@ -165,19 +192,19 @@ def test_public_trades_trades_mock_populate_dataframe_with_trades__check_trades(
trades.reset_index(inplace=True, drop=True) # Reset index for clarity trades.reset_index(inplace=True, drop=True) # Reset index for clarity
# Assert the first trade ID to ensure filtering worked correctly # Assert the first trade ID to ensure filtering worked correctly
assert trades['id'][0] == '313881442' assert trades["id"][0] == "313881442"
# --- Configuration and Function Call --- # --- Configuration and Function Call ---
# Define configuration for order flow calculation (used for context) # Define configuration for order flow calculation (used for context)
config = { config = {
'timeframe': '5m', "timeframe": "5m",
'orderflow': { "orderflow": {
'scale': 0.5, "scale": 0.5,
'imbalance_volume': 0, "imbalance_volume": 0,
'imbalance_ratio': 300, "imbalance_ratio": 300,
'stacked_imbalance_range': 3 "stacked_imbalance_range": 3,
} },
} }
# Populate the DataFrame with trades and order flow data # Populate the DataFrame with trades and order flow data
@@ -190,28 +217,38 @@ def test_public_trades_trades_mock_populate_dataframe_with_trades__check_trades(
# Assert DataFrame structure # Assert DataFrame structure
assert list(df.columns) == [ assert list(df.columns) == [
# ... (list of expected column names) # ... (list of expected column names)
'date', 'open', 'high', 'low', "date",
'close', 'volume', 'trades', 'orderflow', "open",
'bid', 'ask', 'delta', 'min_delta', "high",
'max_delta', 'total_trades', "low",
'stacked_imbalances_bid', "close",
'stacked_imbalances_ask' "volume",
"trades",
"orderflow",
"bid",
"ask",
"delta",
"min_delta",
"max_delta",
"total_trades",
"stacked_imbalances_bid",
"stacked_imbalances_ask",
] ]
# Assert delta, bid, and ask values # Assert delta, bid, and ask values
assert -50.519 == pytest.approx(row['delta']) assert -50.519 == pytest.approx(row["delta"])
assert 219.961 == row['bid'] assert 219.961 == row["bid"]
assert 169.442 == row['ask'] assert 169.442 == row["ask"]
# Assert the number of trades # Assert the number of trades
assert 151 == len(row.trades) assert 151 == len(row.trades)
# Assert specific details of the first trade # Assert specific details of the first trade
t = row['trades'].iloc[0] t = row["trades"].iloc[0]
assert trades['id'][0] == t["id"] assert trades["id"][0] == t["id"]
assert int(trades['timestamp'][0]) == int(t['timestamp']) assert int(trades["timestamp"][0]) == int(t["timestamp"])
assert 'sell' == t['side'] assert "sell" == t["side"]
assert '313881442' == t['id'] assert "313881442" == t["id"]
assert 234.72 == t['price'] assert 234.72 == t["price"]
def test_public_trades_put_volume_profile_into_ohlcv_candles(public_trades_list_simple, candles): def test_public_trades_put_volume_profile_into_ohlcv_candles(public_trades_list_simple, candles):
@@ -232,18 +269,19 @@ def test_public_trades_put_volume_profile_into_ohlcv_candles(public_trades_list_
df = trades_to_volumeprofile_with_total_delta_bid_ask(df, scale=BIN_SIZE_SCALE) df = trades_to_volumeprofile_with_total_delta_bid_ask(df, scale=BIN_SIZE_SCALE)
# Initialize the 'vp' column in the candles DataFrame with NaNs # Initialize the 'vp' column in the candles DataFrame with NaNs
candles['vp'] = np.nan candles["vp"] = np.nan
# Select the second candle (index 1) and attempt to assign the volume profile data # Select the second candle (index 1) and attempt to assign the volume profile data
# (as a DataFrame) to the 'vp' element. # (as a DataFrame) to the 'vp' element.
candles.loc[candles.index == 1, ['vp']] = candles.loc[candles.index == 1, [ candles.loc[candles.index == 1, ["vp"]] = candles.loc[candles.index == 1, ["vp"]].applymap(
'vp']].applymap(lambda x: pd.DataFrame(df.to_dict())) lambda x: pd.DataFrame(df.to_dict())
)
# Assert the delta value in the 'vp' element of the second candle # Assert the delta value in the 'vp' element of the second candle
assert 0.14 == candles['vp'][1].values.tolist()[1][2] assert 0.14 == candles["vp"][1].values.tolist()[1][2]
# Alternative assertion using `.iat` accessor (assuming correct assignment logic) # Alternative assertion using `.iat` accessor (assuming correct assignment logic)
assert 0.14 == candles['vp'][1]['delta'].iat[1] assert 0.14 == candles["vp"][1]["delta"].iat[1]
def test_public_trades_binned_big_sample_list(public_trades_list): def test_public_trades_binned_big_sample_list(public_trades_list):
@@ -262,8 +300,15 @@ def test_public_trades_binned_big_sample_list(public_trades_list):
df = trades_to_volumeprofile_with_total_delta_bid_ask(trades, scale=BIN_SIZE_SCALE) df = trades_to_volumeprofile_with_total_delta_bid_ask(trades, scale=BIN_SIZE_SCALE)
# Assert that the DataFrame has the expected columns # Assert that the DataFrame has the expected columns
assert df.columns.tolist() == ['bid', 'ask', 'delta', 'bid_amount', assert df.columns.tolist() == [
'ask_amount', 'total_volume', 'total_trades'] "bid",
"ask",
"delta",
"bid_amount",
"ask_amount",
"total_volume",
"total_trades",
]
# Assert the number of rows in the DataFrame (expected 23 for this bin size) # Assert the number of rows in the DataFrame (expected 23 for this bin size)
assert len(df) == 23 assert len(df) == 23
@@ -271,22 +316,22 @@ def test_public_trades_binned_big_sample_list(public_trades_list):
# Assert that the index values are in ascending order and spaced correctly # Assert that the index values are in ascending order and spaced correctly
assert all(df.index[i] < df.index[i + 1] for i in range(len(df) - 1)) assert all(df.index[i] < df.index[i + 1] for i in range(len(df) - 1))
assert df.index[0] + BIN_SIZE_SCALE == df.index[1] assert df.index[0] + BIN_SIZE_SCALE == df.index[1]
assert (trades['price'].min() - BIN_SIZE_SCALE) < df.index[0] < trades['price'].max() assert (trades["price"].min() - BIN_SIZE_SCALE) < df.index[0] < trades["price"].max()
assert (df.index[0] + BIN_SIZE_SCALE) >= df.index[1] assert (df.index[0] + BIN_SIZE_SCALE) >= df.index[1]
assert (trades['price'].max() - BIN_SIZE_SCALE) < df.index[-1] < trades['price'].max() assert (trades["price"].max() - BIN_SIZE_SCALE) < df.index[-1] < trades["price"].max()
# Assert specific values in the first and last rows of the DataFrame # Assert specific values in the first and last rows of the DataFrame
assert 32 == df['bid'].iloc[0] # bid price assert 32 == df["bid"].iloc[0] # bid price
assert 197.512 == df['bid_amount'].iloc[0] # total bid amount assert 197.512 == df["bid_amount"].iloc[0] # total bid amount
assert 88.98 == df['ask_amount'].iloc[0] # total ask amount assert 88.98 == df["ask_amount"].iloc[0] # total ask amount
assert 26 == df['ask'].iloc[0] # ask price assert 26 == df["ask"].iloc[0] # ask price
assert -108.532 == pytest.approx(df['delta'].iloc[0]) # delta (bid amount - ask amount) assert -108.532 == pytest.approx(df["delta"].iloc[0]) # delta (bid amount - ask amount)
assert 3 == df['bid'].iloc[-1] # bid price assert 3 == df["bid"].iloc[-1] # bid price
assert 50.659 == df['bid_amount'].iloc[-1] # total bid amount assert 50.659 == df["bid_amount"].iloc[-1] # total bid amount
assert 108.21 == df['ask_amount'].iloc[-1] # total ask amount assert 108.21 == df["ask_amount"].iloc[-1] # total ask amount
assert 44 == df['ask'].iloc[-1] # ask price assert 44 == df["ask"].iloc[-1] # ask price
assert 57.551 == df['delta'].iloc[-1] # delta (bid amount - ask amount) assert 57.551 == df["delta"].iloc[-1] # delta (bid amount - ask amount)
# Repeat the process with a larger bin size # Repeat the process with a larger bin size
BIN_SIZE_SCALE = 1 BIN_SIZE_SCALE = 1
@@ -299,43 +344,89 @@ def test_public_trades_binned_big_sample_list(public_trades_list):
# Repeat similar assertions for index ordering and spacing # Repeat similar assertions for index ordering and spacing
assert all(df.index[i] < df.index[i + 1] for i in range(len(df) - 1)) assert all(df.index[i] < df.index[i + 1] for i in range(len(df) - 1))
assert (trades['price'].min() - BIN_SIZE_SCALE) < df.index[0] < trades['price'].max() assert (trades["price"].min() - BIN_SIZE_SCALE) < df.index[0] < trades["price"].max()
assert (df.index[0] + BIN_SIZE_SCALE) >= df.index[1] assert (df.index[0] + BIN_SIZE_SCALE) >= df.index[1]
assert (trades['price'].max() - BIN_SIZE_SCALE) < df.index[-1] < trades['price'].max() assert (trades["price"].max() - BIN_SIZE_SCALE) < df.index[-1] < trades["price"].max()
# Assert the value in the last row of the DataFrame with the larger bin size # Assert the value in the last row of the DataFrame with the larger bin size
assert 1667.0 == df.index[-1] assert 1667.0 == df.index[-1]
assert 710.98 == df['bid_amount'].iat[0] assert 710.98 == df["bid_amount"].iat[0]
assert 111 == df['bid'].iat[0] assert 111 == df["bid"].iat[0]
assert 52.7199999 == pytest.approx(df['delta'].iat[0]) # delta assert 52.7199999 == pytest.approx(df["delta"].iat[0]) # delta
def test_public_trades_testdata_sanity( def test_public_trades_testdata_sanity(
candles, candles,
public_trades_list, public_trades_list,
public_trades_list_simple, public_trades_list_simple,
populate_dataframe_with_trades_dataframe, populate_dataframe_with_trades_dataframe,
populate_dataframe_with_trades_trades): populate_dataframe_with_trades_trades,
):
assert 10999 == len(candles) assert 10999 == len(candles)
assert 1000 == len(public_trades_list) assert 1000 == len(public_trades_list)
assert 999 == len(populate_dataframe_with_trades_dataframe) assert 999 == len(populate_dataframe_with_trades_dataframe)
assert 293532 == len(populate_dataframe_with_trades_trades) assert 293532 == len(populate_dataframe_with_trades_trades)
assert 7 == len(public_trades_list_simple) assert 7 == len(public_trades_list_simple)
assert 5 == public_trades_list_simple.loc[ assert (
public_trades_list_simple['side'].str.contains('sell'), 'id'].count() 5
assert 2 == public_trades_list_simple.loc[ == public_trades_list_simple.loc[
public_trades_list_simple['side'].str.contains('buy'), 'id'].count() public_trades_list_simple["side"].str.contains("sell"), "id"
].count()
)
assert (
2
== public_trades_list_simple.loc[
public_trades_list_simple["side"].str.contains("buy"), "id"
].count()
)
assert public_trades_list.columns.tolist() == [ assert public_trades_list.columns.tolist() == [
'timestamp', 'id', 'type', 'side', 'price', "timestamp",
'amount', 'cost', 'date'] "id",
"type",
"side",
"price",
"amount",
"cost",
"date",
]
assert public_trades_list.columns.tolist() == [ assert public_trades_list.columns.tolist() == [
'timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost', 'date'] "timestamp",
"id",
"type",
"side",
"price",
"amount",
"cost",
"date",
]
assert public_trades_list_simple.columns.tolist() == [ assert public_trades_list_simple.columns.tolist() == [
'timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost', 'date'] "timestamp",
"id",
"type",
"side",
"price",
"amount",
"cost",
"date",
]
assert populate_dataframe_with_trades_dataframe.columns.tolist() == [ assert populate_dataframe_with_trades_dataframe.columns.tolist() == [
'date', 'open', 'high', 'low', 'close', 'volume'] "date",
"open",
"high",
"low",
"close",
"volume",
]
assert populate_dataframe_with_trades_trades.columns.tolist() == [ assert populate_dataframe_with_trades_trades.columns.tolist() == [
'timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost', 'date'] "timestamp",
"id",
"type",
"side",
"price",
"amount",
"cost",
"date",
]
tests/data/test_dataprovider.py
+5 -5
View File
@@ -66,7 +66,7 @@ def test_historic_trades(mocker, default_conf, trades_history_df):
historymock = MagicMock(return_value=trades_history_df) historymock = MagicMock(return_value=trades_history_df)
mocker.patch( mocker.patch(
"freqtrade.data.history.datahandlers.featherdatahandler.FeatherDataHandler._trades_load", "freqtrade.data.history.datahandlers.featherdatahandler.FeatherDataHandler._trades_load",
historymock historymock,
) )
dp = DataProvider(default_conf, None) dp = DataProvider(default_conf, None)
@@ -82,8 +82,8 @@ def test_historic_trades(mocker, default_conf, trades_history_df):
assert len(data) == 0 assert len(data) == 0
# Switch to backtest mode # Switch to backtest mode
default_conf['runmode'] = RunMode.BACKTEST default_conf["runmode"] = RunMode.BACKTEST
default_conf['dataformat_trades'] = 'feather' default_conf["dataformat_trades"] = "feather"
exchange = get_patched_exchange(mocker, default_conf) exchange = get_patched_exchange(mocker, default_conf)
dp = DataProvider(default_conf, exchange) dp = DataProvider(default_conf, exchange)
data = dp.trades("UNITTEST/BTC", "5m") data = dp.trades("UNITTEST/BTC", "5m")
@@ -91,7 +91,7 @@ def test_historic_trades(mocker, default_conf, trades_history_df):
assert len(data) == len(trades_history_df) assert len(data) == len(trades_history_df)
# Random other runmode # Random other runmode
default_conf['runmode'] = RunMode.UTIL_EXCHANGE default_conf["runmode"] = RunMode.UTIL_EXCHANGE
dp = DataProvider(default_conf, None) dp = DataProvider(default_conf, None)
data = dp.trades("UNITTEST/BTC", "5m") data = dp.trades("UNITTEST/BTC", "5m")
assert isinstance(data, DataFrame) assert isinstance(data, DataFrame)
@@ -311,7 +311,7 @@ def test_refresh(mocker, default_conf):
# Test with public trades # Test with public trades
refresh_mock.reset_mock() refresh_mock.reset_mock()
refresh_mock.reset_mock() refresh_mock.reset_mock()
default_conf['exchange']['use_public_trades'] = True default_conf["exchange"]["use_public_trades"] = True
dp.refresh(pairs, pairs_non_trad) dp.refresh(pairs, pairs_non_trad)
assert mock_refresh_trades.call_count == 1 assert mock_refresh_trades.call_count == 1
assert refresh_mock.call_count == 1 assert refresh_mock.call_count == 1
tests/data/test_history.py
+3 -2
View File
@@ -674,8 +674,9 @@ def test_download_trades_history(
mocker.patch(f"{EXMS}.get_historic_trades", MagicMock(side_effect=ValueError)) mocker.patch(f"{EXMS}.get_historic_trades", MagicMock(side_effect=ValueError))
caplog.clear() caplog.clear()
assert not _download_trades_history(data_handler=data_handler, exchange=exchange, assert not _download_trades_history(
pair='ETH/BTC', trading_mode=TradingMode.SPOT) data_handler=data_handler, exchange=exchange, pair="ETH/BTC", trading_mode=TradingMode.SPOT
)
assert log_has_re('Failed to download and store historic trades for pair: "ETH/BTC".*', caplog) assert log_has_re('Failed to download and store historic trades for pair: "ETH/BTC".*', caplog)
file2 = tmp_path / "XRP_ETH-trades.json.gz" file2 = tmp_path / "XRP_ETH-trades.json.gz"
tests/exchange/test_exchange.py
+7 -14
View File
@@ -2414,8 +2414,7 @@ def test_refresh_latest_trades(mocker, default_conf, caplog, candle_type, tmp_pa
exchange._api_async.fetch_trades = get_mock_coro(trades) exchange._api_async.fetch_trades = get_mock_coro(trades)
exchange._ft_has["exchange_has_overrides"]["fetchTrades"] = True exchange._ft_has["exchange_has_overrides"]["fetchTrades"] = True
pairs = [("IOTA/USDT:USDT", "5m", candle_type), pairs = [("IOTA/USDT:USDT", "5m", candle_type), ("XRP/USDT:USDT", "5m", candle_type)]
("XRP/USDT:USDT", "5m", candle_type)]
# empty dicts # empty dicts
assert not exchange._trades assert not exchange._trades
res = exchange.refresh_latest_trades(pairs, cache=False) res = exchange.refresh_latest_trades(pairs, cache=False)
@@ -2442,10 +2441,8 @@ def test_refresh_latest_trades(mocker, default_conf, caplog, candle_type, tmp_pa
# if copy is "True" # if copy is "True"
assert exchange.trades(pair) is not exchange.trades(pair) assert exchange.trades(pair) is not exchange.trades(pair)
assert exchange.trades(pair) is not exchange.trades(pair, copy=True) assert exchange.trades(pair) is not exchange.trades(pair, copy=True)
assert exchange.trades( assert exchange.trades(pair, copy=True) is not exchange.trades(pair, copy=True)
pair, copy=True) is not exchange.trades(pair, copy=True) assert exchange.trades(pair, copy=False) is exchange.trades(pair, copy=False)
assert exchange.trades(
pair, copy=False) is exchange.trades(pair, copy=False)
# test caching # test caching
ohlcv = [ ohlcv = [
@@ -2470,12 +2467,10 @@ def test_refresh_latest_trades(mocker, default_conf, caplog, candle_type, tmp_pa
trades_df = DataFrame(ohlcv, columns=cols) trades_df = DataFrame(ohlcv, columns=cols)
trades_df["date"] = to_datetime(trades_df["date"], unit="ms", utc=True) trades_df["date"] = to_datetime(trades_df["date"], unit="ms", utc=True)
trades_df["date"] = trades_df["date"].apply( trades_df["date"] = trades_df["date"].apply(lambda date: timeframe_to_prev_date("5m", date))
lambda date: timeframe_to_prev_date("5m", date))
exchange._klines[pair] = trades_df exchange._klines[pair] = trades_df
res = exchange.refresh_latest_trades( res = exchange.refresh_latest_trades(
[("IOTA/USDT:USDT", "5m", candle_type), [("IOTA/USDT:USDT", "5m", candle_type), ("XRP/USDT:USDT", "5m", candle_type)]
("XRP/USDT:USDT", "5m", candle_type)]
) )
assert len(res) == 0 assert len(res) == 0
assert exchange._api_async.fetch_trades.call_count == 0 assert exchange._api_async.fetch_trades.call_count == 0
@@ -2501,12 +2496,10 @@ def test_refresh_latest_trades(mocker, default_conf, caplog, candle_type, tmp_pa
} }
] ]
trades_df = DataFrame(trades) trades_df = DataFrame(trades)
trades_df["date"] = to_datetime( trades_df["date"] = to_datetime(trades_df["timestamp"], unit="ms", utc=True)
trades_df["timestamp"], unit="ms", utc=True)
exchange._trades[pair] = trades_df exchange._trades[pair] = trades_df
res = exchange.refresh_latest_trades( res = exchange.refresh_latest_trades(
[("IOTA/USDT:USDT", "5m", candle_type), [("IOTA/USDT:USDT", "5m", candle_type), ("XRP/USDT:USDT", "5m", candle_type)]
("XRP/USDT:USDT", "5m", candle_type)]
) )
assert len(res) == len(pairs) assert len(res) == len(pairs)
tests/test_configuration.py
+12 -6
View File
@@ -1076,14 +1076,20 @@ def test__validate_consumers(default_conf, caplog) -> None:
def test__validate_orderflow(default_conf) -> None: def test__validate_orderflow(default_conf) -> None:
conf = deepcopy(default_conf) conf = deepcopy(default_conf)
conf['exchange']['use_public_trades'] = True conf["exchange"]["use_public_trades"] = True
with pytest.raises(ConfigurationError, with pytest.raises(
match="Orderflow is a required configuration key when using public trades."): ConfigurationError,
match="Orderflow is a required configuration key when using public trades.",
):
validate_config_consistency(conf) validate_config_consistency(conf)
conf.update({'orderflow': { conf.update(
"scale": 0.5, {
}}) "orderflow": {
"scale": 0.5,
}
}
)
# Should pass. # Should pass.
validate_config_consistency(conf) validate_config_consistency(conf)