From 4b1177e07e48007da7721984a9eb25c482f83a93 Mon Sep 17 00:00:00 2001 From: jainanuj94 Date: Wed, 24 Jul 2024 19:09:45 +0530 Subject: [PATCH 01/13] 10348 | Create new pair list to dynamically fetch pairs based on percent volume change --- .../pairlist/PercentVolumeChangePairList.py | 316 +++++++++++++++ .../test_percentvolumechangepairlist.py | 376 ++++++++++++++++++ 2 files changed, 692 insertions(+) create mode 100644 freqtrade/plugins/pairlist/PercentVolumeChangePairList.py create mode 100644 tests/plugins/test_percentvolumechangepairlist.py diff --git a/freqtrade/plugins/pairlist/PercentVolumeChangePairList.py b/freqtrade/plugins/pairlist/PercentVolumeChangePairList.py new file mode 100644 index 000000000..0aee37959 --- /dev/null +++ b/freqtrade/plugins/pairlist/PercentVolumeChangePairList.py @@ -0,0 +1,316 @@ +""" +Change PairList provider + +Provides dynamic pair list based on trade change +sorted based on percentage change in volume over a +defined period +""" +import logging +from datetime import timedelta +from typing import Any, Dict, List, Literal + +from cachetools import TTLCache + +from freqtrade.constants import ListPairsWithTimeframes +from freqtrade.exceptions import OperationalException +from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date +from freqtrade.exchange.types import Tickers +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting +from freqtrade.util import dt_now, format_ms_time + + +logger = logging.getLogger(__name__) + +SORT_VALUES = ["rolling_volume_change"] + + +class PercentVolumeChangePairList(IPairList): + is_pairlist_generator = True + supports_backtesting = SupportsBacktesting.NO + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + if "number_assets" not in self._pairlistconfig: + raise OperationalException( + "`number_assets` not specified. Please check your configuration " + 'for "pairlist.config.number_assets"' + ) + + self._stake_currency = self._config["stake_currency"] + self._number_pairs = self._pairlistconfig["number_assets"] + self._sort_key: Literal["rolling_volume_change"] = self._pairlistconfig.get( + "sort_key", "rolling_volume_change" + ) + self._min_value = self._pairlistconfig.get("min_value", 0) + self._max_value = self._pairlistconfig.get("max_value", None) + self._refresh_period = self._pairlistconfig.get("refresh_period", 1800) + self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) + self._lookback_days = self._pairlistconfig.get("lookback_days", 0) + self._lookback_timeframe = self._pairlistconfig.get("lookback_timeframe", "1d") + self._lookback_period = self._pairlistconfig.get("lookback_period", 0) + self._def_candletype = self._config["candle_type_def"] + + if (self._lookback_days > 0) & (self._lookback_period > 0): + raise OperationalException( + "Ambiguous configuration: lookback_days and lookback_period both set in pairlist " + "config. Please set lookback_days only or lookback_period and lookback_timeframe " + "and restart the bot." + ) + + # overwrite lookback timeframe and days when lookback_days is set + if self._lookback_days > 0: + self._lookback_timeframe = "1d" + self._lookback_period = self._lookback_days + + # get timeframe in minutes and seconds + self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe) + _tf_in_sec = self._tf_in_min * 60 + + # whether to use range lookback or not + self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0) + + if self._use_range & (self._refresh_period < _tf_in_sec): + raise OperationalException( + f"Refresh period of {self._refresh_period} seconds is smaller than one " + f"timeframe of {self._lookback_timeframe}. Please adjust refresh_period " + f"to at least {_tf_in_sec} and restart the bot." + ) + + if not self._use_range and not ( + self._exchange.exchange_has("fetchTickers") + and self._exchange.get_option("tickers_have_change") + ): + raise OperationalException( + "Exchange does not support dynamic whitelist in this configuration. " + "Please edit your config and either remove PercentVolumeChangePairList, " + "or switch to using candles. and restart the bot." + ) + + if not self._validate_keys(self._sort_key): + raise OperationalException(f"key {self._sort_key} not in {SORT_VALUES}") + + candle_limit = self._exchange.ohlcv_candle_limit( + self._lookback_timeframe, self._config["candle_type_def"] + ) + if self._lookback_period < 4: + raise OperationalException("ChangeFilter requires lookback_period to be >= 4") + self.log_once(f"Candle limit is {candle_limit}", logger.info) + if self._lookback_period > candle_limit: + raise OperationalException( + "ChangeFilter requires lookback_period to not " + f"exceed exchange max request size ({candle_limit})" + ) + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty Dict is passed + as tickers argument to filter_pairlist + """ + return not self._use_range + + def _validate_keys(self, key): + return key in SORT_VALUES + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return (f"{self.name} - top {self._pairlistconfig['number_assets']} percent " + f"volume change pairs.") + + @staticmethod + def description() -> str: + return "Provides dynamic pair list based on percentage volume change." + + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "number_assets": { + "type": "number", + "default": 30, + "description": "Number of assets", + "help": "Number of assets to use from the pairlist", + }, + "sort_key": { + "type": "option", + "default": "rolling_volume_change", + "options": SORT_VALUES, + "description": "Sort key", + "help": "Sort key to use for sorting the pairlist.", + }, + "min_value": { + "type": "number", + "default": 0, + "description": "Minimum value", + "help": "Minimum value to use for filtering the pairlist.", + }, + "max_value": { + "type": "number", + "default": None, + "description": "Maximum value", + "help": "Maximum value to use for filtering the pairlist.", + }, + "refresh_period": { + "type": "number", + "default": 1800, + "description": "Refresh period", + "help": "Refresh period in seconds", + }, + "lookback_days": { + "type": "number", + "default": 0, + "description": "Lookback Days", + "help": "Number of days to look back at.", + }, + "lookback_timeframe": { + "type": "string", + "default": "1d", + "description": "Lookback Timeframe", + "help": "Timeframe to use for lookback.", + }, + "lookback_period": { + "type": "number", + "default": 0, + "description": "Lookback Period", + "help": "Number of periods to look back at.", + }, + } + + def gen_pairlist(self, tickers: Tickers) -> List[str]: + """ + Generate the pairlist + :param tickers: Tickers (from exchange.get_tickers). May be cached. + :return: List of pairs + """ + # Generate dynamic whitelist + # Must always run if this pairlist is not the first in the list. + pairlist = self._pair_cache.get("pairlist") + if pairlist: + # Item found - no refresh necessary + return pairlist.copy() + else: + # Use fresh pairlist + # Check if pair quote currency equals to the stake currency. + _pairlist = [ + k + for k in self._exchange.get_markets( + quote_currencies=[self._stake_currency], tradable_only=True, active_only=True + ).keys() + ] + + # No point in testing for blacklisted pairs... + _pairlist = self.verify_blacklist(_pairlist, logger.info) + if not self._use_range: + filtered_tickers = [ + v + for k, v in tickers.items() + if ( + self._exchange.get_pair_quote_currency(k) == self._stake_currency + and (self._use_range or v.get(self._sort_key) is not None) + and v["symbol"] in _pairlist + ) + ] + pairlist = [s["symbol"] for s in filtered_tickers] + else: + pairlist = _pairlist + + pairlist = self.filter_pairlist(pairlist, tickers) + self._pair_cache["pairlist"] = pairlist.copy() + + return pairlist + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers). May be cached. + :return: new whitelist + """ + self.log_once(f"Filter ticker is self use range {pairlist}", logger.warning) + if self._use_range: + filtered_tickers: List[Dict[str, Any]] = [{"symbol": k} for k in pairlist] + + # get lookback period in ms, for exchange ohlcv fetch + since_ms = ( + int( + timeframe_to_prev_date( + self._lookback_timeframe, + dt_now() + + timedelta( + minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min + ), + ).timestamp() + ) + * 1000 + ) + + to_ms = ( + int( + timeframe_to_prev_date( + self._lookback_timeframe, dt_now() - timedelta(minutes=self._tf_in_min) + ).timestamp() + ) + * 1000 + ) + + # todo: utc date output for starting date + self.log_once( + f"Using change range of {self._lookback_period} candles, timeframe: " + f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} " + f"till {format_ms_time(to_ms)}", + logger.info, + ) + needed_pairs: ListPairsWithTimeframes = [ + (p, self._lookback_timeframe, self._def_candletype) + for p in [s["symbol"] for s in filtered_tickers] + if p not in self._pair_cache + ] + + candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms) + + for i, p in enumerate(filtered_tickers): + pair_candles = ( + candles[(p["symbol"], self._lookback_timeframe, self._def_candletype)] + if (p["symbol"], self._lookback_timeframe, self._def_candletype) in candles + else None + ) + + # in case of candle data calculate typical price and change for candle + if pair_candles is not None and not pair_candles.empty: + pair_candles["rolling_volume_sum"] = ( + pair_candles["volume"].rolling(window=self._lookback_period).sum() + ) + pair_candles["rolling_volume_change"] = ( + pair_candles["rolling_volume_sum"].pct_change() * 100 + ) + + # ensure that a rolling sum over the lookback_period is built + # if pair_candles contains more candles than lookback_period + rolling_volume_change = pair_candles["rolling_volume_change"].fillna(0).iloc[-1] + + # replace change with a range change sum calculated above + filtered_tickers[i]["rolling_volume_change"] = rolling_volume_change + self.log_once(f"ticker {filtered_tickers[i]}", logger.info) + else: + filtered_tickers[i]["rolling_volume_change"] = 0 + else: + filtered_tickers = [v for k, v in tickers.items() if k in pairlist] + + filtered_tickers = [v for v in filtered_tickers if v[self._sort_key] > self._min_value] + if self._max_value is not None: + filtered_tickers = [v for v in filtered_tickers if v[self._sort_key] < self._max_value] + + sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[self._sort_key]) + + self.log_once(f"Sorted Tickers {sorted_tickers}", logger.info) + # Validate whitelist to only have active market pairs + pairs = self._whitelist_for_active_markets([s["symbol"] for s in sorted_tickers]) + pairs = self.verify_blacklist(pairs, logmethod=logger.info) + # Limit pairlist to the requested number of pairs + pairs = pairs[: self._number_pairs] + + return pairs diff --git a/tests/plugins/test_percentvolumechangepairlist.py b/tests/plugins/test_percentvolumechangepairlist.py new file mode 100644 index 000000000..b09307151 --- /dev/null +++ b/tests/plugins/test_percentvolumechangepairlist.py @@ -0,0 +1,376 @@ +from datetime import datetime, timezone +from unittest.mock import MagicMock + +import pandas as pd +import pytest + +from freqtrade.data.converter import ohlcv_to_dataframe +from freqtrade.enums import CandleType +from freqtrade.exceptions import OperationalException +from freqtrade.plugins.pairlist.PercentVolumeChangePairList import PercentVolumeChangePairList +from freqtrade.plugins.pairlistmanager import PairListManager +from tests.conftest import ( + EXMS, + generate_test_data_raw, + get_patched_exchange, + get_patched_freqtradebot, +) + + +@pytest.fixture(scope="function") +def rpl_config(default_conf): + default_conf["stake_currency"] = "USDT" + + default_conf["exchange"]["pair_whitelist"] = [ + "ETH/USDT", + "XRP/USDT", + ] + default_conf["exchange"]["pair_blacklist"] = ["BLK/USDT"] + + return default_conf + + +def test_volume_change_pair_list_init_exchange_support(mocker, rpl_config): + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "number_assets": 2, + "sort_key": "rolling_volume_change", + "min_value": 0, + "refresh_period": 86400, + } + ] + + with pytest.raises( + OperationalException, + match=r"Exchange does not support dynamic whitelist in this configuration. " + r"Please edit your config and either remove PercentVolumeChangePairList, " + r"or switch to using candles. and restart the bot.", + ): + get_patched_freqtradebot(mocker, rpl_config) + + +def test_volume_change_pair_list_init_wrong_refresh_period(mocker, rpl_config): + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "number_assets": 2, + "sort_key": "rolling_volume_change", + "min_value": 0, + "refresh_period": 1800, + "lookback_days": 4, + } + ] + + with pytest.raises( + OperationalException, + match=r"Refresh period of 1800 seconds is smaller than one " + r"timeframe of 1d. Please adjust refresh_period " + r"to at least 86400 and restart the bot.", + ): + get_patched_freqtradebot(mocker, rpl_config) + + +def test_volume_change_pair_list_init_wrong_lookback_period(mocker, rpl_config): + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "number_assets": 2, + "sort_key": "rolling_volume_change", + "min_value": 0, + "refresh_period": 86400, + "lookback_days": 3, + "lookback_period": 3, + } + ] + + with pytest.raises( + OperationalException, + match=r"Ambiguous configuration: lookback_days " + r"and lookback_period both set in pairlist config. " + r"Please set lookback_days only or lookback_period " + r"and lookback_timeframe and restart the bot.", + ): + get_patched_freqtradebot(mocker, rpl_config) + + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "number_assets": 2, + "sort_key": "rolling_volume_change", + "min_value": 0, + "refresh_period": 86400, + "lookback_days": 3, + } + ] + + with pytest.raises( + OperationalException, match=r"ChangeFilter requires lookback_period to be >= 4" + ): + get_patched_freqtradebot(mocker, rpl_config) + + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "number_assets": 2, + "sort_key": "rolling_volume_change", + "min_value": 0, + "refresh_period": 86400, + "lookback_period": 3, + } + ] + + with pytest.raises( + OperationalException, match=r"ChangeFilter requires lookback_period to be >= 4" + ): + get_patched_freqtradebot(mocker, rpl_config) + + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "number_assets": 2, + "sort_key": "rolling_volume_change", + "min_value": 0, + "refresh_period": 86400, + "lookback_days": 1001, + } + ] + + with pytest.raises( + OperationalException, + match=r"ChangeFilter requires lookback_period to not exceed" + r" exchange max request size \(1000\)", + ): + get_patched_freqtradebot(mocker, rpl_config) + + +def test_volume_change_pair_list_init_wrong_config(mocker, rpl_config): + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "sort_key": "rolling_volume_change", + "min_value": 0, + "refresh_period": 86400, + } + ] + + with pytest.raises( + OperationalException, + match=r"`number_assets` not specified. Please check your configuration " + r'for "pairlist.config.number_assets"', + ): + get_patched_freqtradebot(mocker, rpl_config) + + +def test_gen_pairlist_with_valid_change_pair_list_config(mocker, rpl_config, tickers, time_machine): + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "number_assets": 2, + "sort_key": "rolling_volume_change", + "min_value": 0, + "refresh_period": 86400, + "lookback_days": 4, + } + ] + start = datetime(2024, 8, 1, 0, 0, 0, 0, tzinfo=timezone.utc) + time_machine.move_to(start, tick=False) + + mock_ohlcv_data = { + ("ETH/USDT", "1d", CandleType.SPOT): pd.DataFrame( + ohlcv_to_dataframe( + generate_test_data_raw("1d", 100, start.strftime("%Y-%m-%d"), random_seed=12), + "1d", + pair="ETH/USDT", + fill_missing=True, + ) + ), + ("BTC/USDT", "1d", CandleType.SPOT): pd.DataFrame( + ohlcv_to_dataframe( + generate_test_data_raw("1d", 100, start.strftime("%Y-%m-%d"), random_seed=13), + "1d", + pair="BTC/USDT", + fill_missing=True, + ) + ), + ("XRP/USDT", "1d", CandleType.SPOT): pd.DataFrame( + ohlcv_to_dataframe( + generate_test_data_raw("1d", 100, start.strftime("%Y-%m-%d"), random_seed=14), + "1d", + pair="XRP/USDT", + fill_missing=True, + ) + ), + ("NEO/USDT", "1d", CandleType.SPOT): pd.DataFrame( + ohlcv_to_dataframe( + generate_test_data_raw("1d", 100, start.strftime("%Y-%m-%d"), random_seed=15), + "1d", + pair="NEO/USDT", + fill_missing=True, + ) + ), + ("TKN/USDT", "1d", CandleType.SPOT): pd.DataFrame( + # Make sure always have highest rolling_volume_change + { + "timestamp": [ + "2024-07-01 00:00:00", + "2024-07-01 01:00:00", + "2024-07-01 02:00:00", + "2024-07-01 03:00:00", + "2024-07-01 04:00:00", + "2024-07-01 05:00:00", + ], + "open": [100, 102, 101, 103, 104, 105], + "high": [102, 103, 102, 104, 105, 106], + "low": [99, 101, 100, 102, 103, 104], + "close": [101, 102, 103, 104, 105, 106], + "volume": [1000, 1500, 2000, 2500, 3000, 3500], + } + ), + } + + mocker.patch(f"{EXMS}.refresh_latest_ohlcv", MagicMock(return_value=mock_ohlcv_data)) + + exchange = get_patched_exchange(mocker, rpl_config, exchange="binance") + pairlistmanager = PairListManager(exchange, rpl_config) + + remote_pairlist = PercentVolumeChangePairList( + exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0 + ) + + result = remote_pairlist.gen_pairlist(tickers) + + assert len(result) == 2 + assert result == ["TKN/USDT", "BTC/USDT"] + + +def test_filter_pairlist_with_empty_ticker(mocker, rpl_config, tickers, time_machine): + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "number_assets": 2, + "sort_key": "rolling_volume_change", + "min_value": 0, + "refresh_period": 86400, + "lookback_days": 4, + } + ] + start = datetime(2024, 8, 1, 0, 0, 0, 0, tzinfo=timezone.utc) + time_machine.move_to(start, tick=False) + + mock_ohlcv_data = { + ("ETH/USDT", "1d", CandleType.SPOT): pd.DataFrame( + { + "timestamp": [ + "2024-07-01 00:00:00", + "2024-07-01 01:00:00", + "2024-07-01 02:00:00", + "2024-07-01 03:00:00", + "2024-07-01 04:00:00", + "2024-07-01 05:00:00", + ], + "open": [100, 102, 101, 103, 104, 105], + "high": [102, 103, 102, 104, 105, 106], + "low": [99, 101, 100, 102, 103, 104], + "close": [101, 102, 103, 104, 105, 106], + "volume": [1000, 1500, 2000, 2500, 3000, 3500], + } + ), + ("XRP/USDT", "1d", CandleType.SPOT): pd.DataFrame( + { + "timestamp": [ + "2024-07-01 00:00:00", + "2024-07-01 01:00:00", + "2024-07-01 02:00:00", + "2024-07-01 03:00:00", + "2024-07-01 04:00:00", + "2024-07-01 05:00:00", + ], + "open": [100, 102, 101, 103, 104, 105], + "high": [102, 103, 102, 104, 105, 106], + "low": [99, 101, 100, 102, 103, 104], + "close": [101, 102, 103, 104, 105, 106], + "volume": [1000, 1500, 2000, 2500, 3000, 3500], + } + ), + } + + mocker.patch(f"{EXMS}.refresh_latest_ohlcv", MagicMock(return_value=mock_ohlcv_data)) + exchange = get_patched_exchange(mocker, rpl_config, exchange="binance") + pairlistmanager = PairListManager(exchange, rpl_config) + + remote_pairlist = PercentVolumeChangePairList( + exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0 + ) + + result = remote_pairlist.filter_pairlist(rpl_config["exchange"]["pair_whitelist"], {}) + + assert len(result) == 2 + assert result == ["ETH/USDT", "XRP/USDT"] + + +def test_filter_pairlist_with_max_value_set(mocker, rpl_config, tickers, time_machine): + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "number_assets": 2, + "sort_key": "rolling_volume_change", + "min_value": 0, + "max_value": 15, + "refresh_period": 86400, + "lookback_days": 4, + } + ] + + start = datetime(2024, 8, 1, 0, 0, 0, 0, tzinfo=timezone.utc) + time_machine.move_to(start, tick=False) + + mock_ohlcv_data = { + ("ETH/USDT", "1d", CandleType.SPOT): pd.DataFrame( + { + "timestamp": [ + "2024-07-01 00:00:00", + "2024-07-01 01:00:00", + "2024-07-01 02:00:00", + "2024-07-01 03:00:00", + "2024-07-01 04:00:00", + "2024-07-01 05:00:00", + ], + "open": [100, 102, 101, 103, 104, 105], + "high": [102, 103, 102, 104, 105, 106], + "low": [99, 101, 100, 102, 103, 104], + "close": [101, 102, 103, 104, 105, 106], + "volume": [1000, 1500, 2000, 1800, 2400, 2500], + } + ), + ("XRP/USDT", "1d", CandleType.SPOT): pd.DataFrame( + { + "timestamp": [ + "2024-07-01 00:00:00", + "2024-07-01 01:00:00", + "2024-07-01 02:00:00", + "2024-07-01 03:00:00", + "2024-07-01 04:00:00", + "2024-07-01 05:00:00", + ], + "open": [100, 102, 101, 103, 104, 105], + "high": [102, 103, 102, 104, 105, 106], + "low": [99, 101, 100, 102, 103, 104], + "close": [101, 102, 103, 104, 105, 106], + "volume": [1000, 1500, 2000, 2500, 3000, 3500], + } + ), + } + + mocker.patch(f"{EXMS}.refresh_latest_ohlcv", MagicMock(return_value=mock_ohlcv_data)) + exchange = get_patched_exchange(mocker, rpl_config, exchange="binance") + pairlistmanager = PairListManager(exchange, rpl_config) + + remote_pairlist = PercentVolumeChangePairList( + exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0 + ) + + result = remote_pairlist.filter_pairlist(rpl_config["exchange"]["pair_whitelist"], {}) + + assert len(result) == 1 + assert result == ["ETH/USDT"] From b09f9e8c12fa638042336957181ccd732277881f Mon Sep 17 00:00:00 2001 From: jainanuj94 Date: Wed, 24 Jul 2024 19:12:11 +0530 Subject: [PATCH 02/13] 10348 | Update tests and add pairlist constants --- freqtrade/constants.py | 1 + tests/plugins/test_pairlist.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 86c1d71cd..4d8894c26 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -42,6 +42,7 @@ HYPEROPT_LOSS_BUILTIN = [ AVAILABLE_PAIRLISTS = [ "StaticPairList", "VolumePairList", + "PercentVolumeChangePairList", "ProducerPairList", "RemotePairList", "MarketCapPairList", diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 37ebdc58b..e5a19ea74 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -31,9 +31,11 @@ from tests.conftest import ( ) -# Exclude RemotePairList from tests. -# It has a mandatory parameter, and requires special handling, which happens in test_remotepairlist. -TESTABLE_PAIRLISTS = [p for p in AVAILABLE_PAIRLISTS if p not in ["RemotePairList"]] +# Exclude RemotePairList and PercentVolumeChangePairList from tests. +# They have mandatory parameters, and requires special handling, +# which happens in test_remotepairlist and test_percentvolumechangepairlist. +TESTABLE_PAIRLISTS = [p for p in AVAILABLE_PAIRLISTS + if p not in ["RemotePairList", "PercentVolumeChangePairList"]] @pytest.fixture(scope="function") From 1b81de01b48689cb805d20d88cea4b1d0b356901 Mon Sep 17 00:00:00 2001 From: jainanuj94 Date: Thu, 25 Jul 2024 00:04:06 +0530 Subject: [PATCH 03/13] 10348 | run ruff formatter --- freqtrade/plugins/pairlist/PercentVolumeChangePairList.py | 7 +++++-- tests/plugins/test_pairlist.py | 5 +++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/freqtrade/plugins/pairlist/PercentVolumeChangePairList.py b/freqtrade/plugins/pairlist/PercentVolumeChangePairList.py index 0aee37959..777acda60 100644 --- a/freqtrade/plugins/pairlist/PercentVolumeChangePairList.py +++ b/freqtrade/plugins/pairlist/PercentVolumeChangePairList.py @@ -5,6 +5,7 @@ Provides dynamic pair list based on trade change sorted based on percentage change in volume over a defined period """ + import logging from datetime import timedelta from typing import Any, Dict, List, Literal @@ -118,8 +119,10 @@ class PercentVolumeChangePairList(IPairList): """ Short whitelist method description - used for startup-messages """ - return (f"{self.name} - top {self._pairlistconfig['number_assets']} percent " - f"volume change pairs.") + return ( + f"{self.name} - top {self._pairlistconfig['number_assets']} percent " + f"volume change pairs." + ) @staticmethod def description() -> str: diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index e5a19ea74..7e5fd4c12 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -34,8 +34,9 @@ from tests.conftest import ( # Exclude RemotePairList and PercentVolumeChangePairList from tests. # They have mandatory parameters, and requires special handling, # which happens in test_remotepairlist and test_percentvolumechangepairlist. -TESTABLE_PAIRLISTS = [p for p in AVAILABLE_PAIRLISTS - if p not in ["RemotePairList", "PercentVolumeChangePairList"]] +TESTABLE_PAIRLISTS = [ + p for p in AVAILABLE_PAIRLISTS if p not in ["RemotePairList", "PercentVolumeChangePairList"] +] @pytest.fixture(scope="function") From dad4f30597b693acab57075565d1d6455b535b82 Mon Sep 17 00:00:00 2001 From: jainanuj94 Date: Thu, 25 Jul 2024 23:33:28 +0530 Subject: [PATCH 04/13] Correct calculation for percent calculation and use tickers --- freqtrade/constants.py | 2 +- freqtrade/exchange/exchange.py | 1 + freqtrade/exchange/types.py | 1 + ...gePairList.py => PercentChangePairList.py} | 203 ++++++++++-------- tests/conftest.py | 2 +- tests/plugins/test_pairlist.py | 6 +- ...rlist.py => test_percentchangepairlist.py} | 133 ++++++------ 7 files changed, 195 insertions(+), 153 deletions(-) rename freqtrade/plugins/pairlist/{PercentVolumeChangePairList.py => PercentChangePairList.py} (64%) rename tests/plugins/{test_percentvolumechangepairlist.py => test_percentchangepairlist.py} (82%) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 4d8894c26..17a829955 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -42,7 +42,7 @@ HYPEROPT_LOSS_BUILTIN = [ AVAILABLE_PAIRLISTS = [ "StaticPairList", "VolumePairList", - "PercentVolumeChangePairList", + "PercentChangePairList", "ProducerPairList", "RemotePairList", "MarketCapPairList", diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 62f0ca4de..34ca8a3e8 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -121,6 +121,7 @@ class Exchange: # Check https://github.com/ccxt/ccxt/issues/10767 for removal of ohlcv_volume_currency "ohlcv_volume_currency": "base", # "base" or "quote" "tickers_have_quoteVolume": True, + "tickers_have_percentage": True, "tickers_have_bid_ask": True, # bid / ask empty for fetch_tickers "tickers_have_price": True, "trades_pagination": "time", # Possible are "time" or "id" diff --git a/freqtrade/exchange/types.py b/freqtrade/exchange/types.py index 5568e4336..a0d315c78 100644 --- a/freqtrade/exchange/types.py +++ b/freqtrade/exchange/types.py @@ -12,6 +12,7 @@ class Ticker(TypedDict): last: Optional[float] quoteVolume: Optional[float] baseVolume: Optional[float] + percentage: Optional[float] # Several more - only listing required. diff --git a/freqtrade/plugins/pairlist/PercentVolumeChangePairList.py b/freqtrade/plugins/pairlist/PercentChangePairList.py similarity index 64% rename from freqtrade/plugins/pairlist/PercentVolumeChangePairList.py rename to freqtrade/plugins/pairlist/PercentChangePairList.py index 777acda60..a1be6729b 100644 --- a/freqtrade/plugins/pairlist/PercentVolumeChangePairList.py +++ b/freqtrade/plugins/pairlist/PercentChangePairList.py @@ -8,24 +8,24 @@ defined period import logging from datetime import timedelta -from typing import Any, Dict, List, Literal +from typing import Any, Dict, List, Literal, Optional from cachetools import TTLCache from freqtrade.constants import ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date -from freqtrade.exchange.types import Tickers +from freqtrade.exchange.types import Ticker, Tickers from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting from freqtrade.util import dt_now, format_ms_time logger = logging.getLogger(__name__) -SORT_VALUES = ["rolling_volume_change"] +SORT_VALUES = ["percentage"] -class PercentVolumeChangePairList(IPairList): +class PercentChangePairList(IPairList): is_pairlist_generator = True supports_backtesting = SupportsBacktesting.NO @@ -50,6 +50,7 @@ class PercentVolumeChangePairList(IPairList): self._lookback_days = self._pairlistconfig.get("lookback_days", 0) self._lookback_timeframe = self._pairlistconfig.get("lookback_timeframe", "1d") self._lookback_period = self._pairlistconfig.get("lookback_period", 0) + self._sort_direction: Optional[str] = self._pairlistconfig.get("sort_direction", "desc") self._def_candletype = self._config["candle_type_def"] if (self._lookback_days > 0) & (self._lookback_period > 0): @@ -80,11 +81,11 @@ class PercentVolumeChangePairList(IPairList): if not self._use_range and not ( self._exchange.exchange_has("fetchTickers") - and self._exchange.get_option("tickers_have_change") + and self._exchange.get_option("tickers_have_percentage") ): raise OperationalException( "Exchange does not support dynamic whitelist in this configuration. " - "Please edit your config and either remove PercentVolumeChangePairList, " + "Please edit your config and either remove PercentChangePairList, " "or switch to using candles. and restart the bot." ) @@ -94,9 +95,7 @@ class PercentVolumeChangePairList(IPairList): candle_limit = self._exchange.ohlcv_candle_limit( self._lookback_timeframe, self._config["candle_type_def"] ) - if self._lookback_period < 4: - raise OperationalException("ChangeFilter requires lookback_period to be >= 4") - self.log_once(f"Candle limit is {candle_limit}", logger.info) + if self._lookback_period > candle_limit: raise OperationalException( "ChangeFilter requires lookback_period to not " @@ -119,14 +118,11 @@ class PercentVolumeChangePairList(IPairList): """ Short whitelist method description - used for startup-messages """ - return ( - f"{self.name} - top {self._pairlistconfig['number_assets']} percent " - f"volume change pairs." - ) + return f"{self.name} - top {self._pairlistconfig['number_assets']} percent change pairs." @staticmethod def description() -> str: - return "Provides dynamic pair list based on percentage volume change." + return "Provides dynamic pair list based on percentage change." @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: @@ -156,12 +152,14 @@ class PercentVolumeChangePairList(IPairList): "description": "Maximum value", "help": "Maximum value to use for filtering the pairlist.", }, - "refresh_period": { - "type": "number", - "default": 1800, - "description": "Refresh period", - "help": "Refresh period in seconds", + "sort_direction": { + "type": "option", + "default": "desc", + "options": ["", "asc", "desc"], + "description": "Sort pairlist", + "help": "Sort Pairlist ascending or descending by rate of change.", }, + **IPairList.refresh_period_parameter(), "lookback_days": { "type": "number", "default": 0, @@ -233,83 +231,24 @@ class PercentVolumeChangePairList(IPairList): :param tickers: Tickers (from exchange.get_tickers). May be cached. :return: new whitelist """ - self.log_once(f"Filter ticker is self use range {pairlist}", logger.warning) + filtered_tickers: List[Dict[str, Any]] = [{"symbol": k} for k in pairlist] if self._use_range: - filtered_tickers: List[Dict[str, Any]] = [{"symbol": k} for k in pairlist] - - # get lookback period in ms, for exchange ohlcv fetch - since_ms = ( - int( - timeframe_to_prev_date( - self._lookback_timeframe, - dt_now() - + timedelta( - minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min - ), - ).timestamp() - ) - * 1000 - ) - - to_ms = ( - int( - timeframe_to_prev_date( - self._lookback_timeframe, dt_now() - timedelta(minutes=self._tf_in_min) - ).timestamp() - ) - * 1000 - ) - - # todo: utc date output for starting date - self.log_once( - f"Using change range of {self._lookback_period} candles, timeframe: " - f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} " - f"till {format_ms_time(to_ms)}", - logger.info, - ) - needed_pairs: ListPairsWithTimeframes = [ - (p, self._lookback_timeframe, self._def_candletype) - for p in [s["symbol"] for s in filtered_tickers] - if p not in self._pair_cache - ] - - candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms) - - for i, p in enumerate(filtered_tickers): - pair_candles = ( - candles[(p["symbol"], self._lookback_timeframe, self._def_candletype)] - if (p["symbol"], self._lookback_timeframe, self._def_candletype) in candles - else None - ) - - # in case of candle data calculate typical price and change for candle - if pair_candles is not None and not pair_candles.empty: - pair_candles["rolling_volume_sum"] = ( - pair_candles["volume"].rolling(window=self._lookback_period).sum() - ) - pair_candles["rolling_volume_change"] = ( - pair_candles["rolling_volume_sum"].pct_change() * 100 - ) - - # ensure that a rolling sum over the lookback_period is built - # if pair_candles contains more candles than lookback_period - rolling_volume_change = pair_candles["rolling_volume_change"].fillna(0).iloc[-1] - - # replace change with a range change sum calculated above - filtered_tickers[i]["rolling_volume_change"] = rolling_volume_change - self.log_once(f"ticker {filtered_tickers[i]}", logger.info) - else: - filtered_tickers[i]["rolling_volume_change"] = 0 + # calculating using lookback_period + self.fetch_percent_change_from_lookback_period(filtered_tickers) else: - filtered_tickers = [v for k, v in tickers.items() if k in pairlist] + # Fetching 24h change by default from supported exchange tickers + self.fetch_percent_change_from_tickers(filtered_tickers, tickers) filtered_tickers = [v for v in filtered_tickers if v[self._sort_key] > self._min_value] if self._max_value is not None: filtered_tickers = [v for v in filtered_tickers if v[self._sort_key] < self._max_value] - sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[self._sort_key]) + sorted_tickers = sorted( + filtered_tickers, + reverse=self._sort_direction == "desc", + key=lambda t: t[self._sort_key], + ) - self.log_once(f"Sorted Tickers {sorted_tickers}", logger.info) # Validate whitelist to only have active market pairs pairs = self._whitelist_for_active_markets([s["symbol"] for s in sorted_tickers]) pairs = self.verify_blacklist(pairs, logmethod=logger.info) @@ -317,3 +256,91 @@ class PercentVolumeChangePairList(IPairList): pairs = pairs[: self._number_pairs] return pairs + + def fetch_candles_for_lookback_period(self, filtered_tickers): + since_ms = ( + int( + timeframe_to_prev_date( + self._lookback_timeframe, + dt_now() + + timedelta( + minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min + ), + ).timestamp() + ) + * 1000 + ) + to_ms = ( + int( + timeframe_to_prev_date( + self._lookback_timeframe, dt_now() - timedelta(minutes=self._tf_in_min) + ).timestamp() + ) + * 1000 + ) + # todo: utc date output for starting date + self.log_once( + f"Using change range of {self._lookback_period} candles, timeframe: " + f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} " + f"till {format_ms_time(to_ms)}", + logger.info, + ) + needed_pairs: ListPairsWithTimeframes = [ + (p, self._lookback_timeframe, self._def_candletype) + for p in [s["symbol"] for s in filtered_tickers] + if p not in self._pair_cache + ] + candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms) + return candles + + def fetch_percent_change_from_lookback_period(self, filtered_tickers): + # get lookback period in ms, for exchange ohlcv fetch + candles = self.fetch_candles_for_lookback_period(filtered_tickers) + + for i, p in enumerate(filtered_tickers): + pair_candles = ( + candles[(p["symbol"], self._lookback_timeframe, self._def_candletype)] + if (p["symbol"], self._lookback_timeframe, self._def_candletype) in candles + else None + ) + + # in case of candle data calculate typical price and change for candle + if pair_candles is not None and not pair_candles.empty: + current_close = pair_candles["close"].iloc[-1] + previous_close = pair_candles["close"].shift(self._lookback_period).iloc[-1] + pct_change = ( + ((current_close - previous_close) / previous_close) if previous_close > 0 else 0 + ) + + # replace change with a range change sum calculated above + filtered_tickers[i]["percentage"] = pct_change + self.log_once(f"Tickers: {filtered_tickers}", logger.info) + else: + filtered_tickers[i]["percentage"] = 0 + + def fetch_percent_change_from_tickers(self, filtered_tickers, tickers): + for i, p in enumerate(filtered_tickers): + # Filter out assets + if not self._validate_pair( + p["symbol"], tickers[p["symbol"]] if p["symbol"] in tickers else None + ): + filtered_tickers.remove(p) + else: + filtered_tickers[i]["percentage"] = tickers[p["symbol"]]["percentage"] + + def _validate_pair(self, pair: str, ticker: Optional[Ticker]) -> bool: + """ + Check if one price-step (pip) is > than a certain barrier. + :param pair: Pair that's currently validated + :param ticker: ticker dict as returned from ccxt.fetch_ticker + :return: True if the pair can stay, false if it should be removed + """ + if not ticker or "percentage" not in ticker or ticker["percentage"] is None: + self.log_once( + f"Removed {pair} from whitelist, because " + "ticker['percentage'] is empty (Usually no trade in the last 24h).", + logger.info, + ) + return False + + return True diff --git a/tests/conftest.py b/tests/conftest.py index fee8cab72..22bc2556e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2187,7 +2187,7 @@ def tickers(): "first": None, "last": 530.21, "change": 0.558, - "percentage": None, + "percentage": 2.349, "average": None, "baseVolume": 72300.0659, "quoteVolume": 37670097.3022171, diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 7e5fd4c12..31e746d5c 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -31,11 +31,11 @@ from tests.conftest import ( ) -# Exclude RemotePairList and PercentVolumeChangePairList from tests. +# Exclude RemotePairList and PercentVolumePairList from tests. # They have mandatory parameters, and requires special handling, -# which happens in test_remotepairlist and test_percentvolumechangepairlist. +# which happens in test_remotepairlist and test_percentchangepairlist. TESTABLE_PAIRLISTS = [ - p for p in AVAILABLE_PAIRLISTS if p not in ["RemotePairList", "PercentVolumeChangePairList"] + p for p in AVAILABLE_PAIRLISTS if p not in ["RemotePairList", "PercentChangePairList"] ] diff --git a/tests/plugins/test_percentvolumechangepairlist.py b/tests/plugins/test_percentchangepairlist.py similarity index 82% rename from tests/plugins/test_percentvolumechangepairlist.py rename to tests/plugins/test_percentchangepairlist.py index b09307151..0a7960d22 100644 --- a/tests/plugins/test_percentvolumechangepairlist.py +++ b/tests/plugins/test_percentchangepairlist.py @@ -7,7 +7,7 @@ import pytest from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException -from freqtrade.plugins.pairlist.PercentVolumeChangePairList import PercentVolumeChangePairList +from freqtrade.plugins.pairlist.PercentChangePairList import PercentChangePairList from freqtrade.plugins.pairlistmanager import PairListManager from tests.conftest import ( EXMS, @@ -33,9 +33,9 @@ def rpl_config(default_conf): def test_volume_change_pair_list_init_exchange_support(mocker, rpl_config): rpl_config["pairlists"] = [ { - "method": "PercentVolumeChangePairList", + "method": "PercentChangePairList", "number_assets": 2, - "sort_key": "rolling_volume_change", + "sort_key": "percentage", "min_value": 0, "refresh_period": 86400, } @@ -44,7 +44,7 @@ def test_volume_change_pair_list_init_exchange_support(mocker, rpl_config): with pytest.raises( OperationalException, match=r"Exchange does not support dynamic whitelist in this configuration. " - r"Please edit your config and either remove PercentVolumeChangePairList, " + r"Please edit your config and either remove PercentChangePairList, " r"or switch to using candles. and restart the bot.", ): get_patched_freqtradebot(mocker, rpl_config) @@ -53,9 +53,9 @@ def test_volume_change_pair_list_init_exchange_support(mocker, rpl_config): def test_volume_change_pair_list_init_wrong_refresh_period(mocker, rpl_config): rpl_config["pairlists"] = [ { - "method": "PercentVolumeChangePairList", + "method": "PercentChangePairList", "number_assets": 2, - "sort_key": "rolling_volume_change", + "sort_key": "percentage", "min_value": 0, "refresh_period": 1800, "lookback_days": 4, @@ -71,12 +71,31 @@ def test_volume_change_pair_list_init_wrong_refresh_period(mocker, rpl_config): get_patched_freqtradebot(mocker, rpl_config) +def test_volume_change_pair_list_init_invalid_sort_key(mocker, rpl_config): + rpl_config["pairlists"] = [ + { + "method": "PercentChangePairList", + "number_assets": 2, + "sort_key": "wrong_key", + "min_value": 0, + "refresh_period": 86400, + "lookback_days": 1, + } + ] + + with pytest.raises( + OperationalException, + match=r"key wrong_key not in \['percentage'\]", + ): + get_patched_freqtradebot(mocker, rpl_config) + + def test_volume_change_pair_list_init_wrong_lookback_period(mocker, rpl_config): rpl_config["pairlists"] = [ { - "method": "PercentVolumeChangePairList", + "method": "PercentChangePairList", "number_assets": 2, - "sort_key": "rolling_volume_change", + "sort_key": "percentage", "min_value": 0, "refresh_period": 86400, "lookback_days": 3, @@ -95,41 +114,9 @@ def test_volume_change_pair_list_init_wrong_lookback_period(mocker, rpl_config): rpl_config["pairlists"] = [ { - "method": "PercentVolumeChangePairList", + "method": "PercentChangePairList", "number_assets": 2, - "sort_key": "rolling_volume_change", - "min_value": 0, - "refresh_period": 86400, - "lookback_days": 3, - } - ] - - with pytest.raises( - OperationalException, match=r"ChangeFilter requires lookback_period to be >= 4" - ): - get_patched_freqtradebot(mocker, rpl_config) - - rpl_config["pairlists"] = [ - { - "method": "PercentVolumeChangePairList", - "number_assets": 2, - "sort_key": "rolling_volume_change", - "min_value": 0, - "refresh_period": 86400, - "lookback_period": 3, - } - ] - - with pytest.raises( - OperationalException, match=r"ChangeFilter requires lookback_period to be >= 4" - ): - get_patched_freqtradebot(mocker, rpl_config) - - rpl_config["pairlists"] = [ - { - "method": "PercentVolumeChangePairList", - "number_assets": 2, - "sort_key": "rolling_volume_change", + "sort_key": "percentage", "min_value": 0, "refresh_period": 86400, "lookback_days": 1001, @@ -147,8 +134,8 @@ def test_volume_change_pair_list_init_wrong_lookback_period(mocker, rpl_config): def test_volume_change_pair_list_init_wrong_config(mocker, rpl_config): rpl_config["pairlists"] = [ { - "method": "PercentVolumeChangePairList", - "sort_key": "rolling_volume_change", + "method": "PercentChangePairList", + "sort_key": "percentage", "min_value": 0, "refresh_period": 86400, } @@ -165,9 +152,9 @@ def test_volume_change_pair_list_init_wrong_config(mocker, rpl_config): def test_gen_pairlist_with_valid_change_pair_list_config(mocker, rpl_config, tickers, time_machine): rpl_config["pairlists"] = [ { - "method": "PercentVolumeChangePairList", + "method": "PercentChangePairList", "number_assets": 2, - "sort_key": "rolling_volume_change", + "sort_key": "percentage", "min_value": 0, "refresh_period": 86400, "lookback_days": 4, @@ -210,7 +197,7 @@ def test_gen_pairlist_with_valid_change_pair_list_config(mocker, rpl_config, tic ) ), ("TKN/USDT", "1d", CandleType.SPOT): pd.DataFrame( - # Make sure always have highest rolling_volume_change + # Make sure always have highest percentage { "timestamp": [ "2024-07-01 00:00:00", @@ -234,24 +221,25 @@ def test_gen_pairlist_with_valid_change_pair_list_config(mocker, rpl_config, tic exchange = get_patched_exchange(mocker, rpl_config, exchange="binance") pairlistmanager = PairListManager(exchange, rpl_config) - remote_pairlist = PercentVolumeChangePairList( + remote_pairlist = PercentChangePairList( exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0 ) result = remote_pairlist.gen_pairlist(tickers) assert len(result) == 2 - assert result == ["TKN/USDT", "BTC/USDT"] + assert result == ["NEO/USDT", "TKN/USDT"] def test_filter_pairlist_with_empty_ticker(mocker, rpl_config, tickers, time_machine): rpl_config["pairlists"] = [ { - "method": "PercentVolumeChangePairList", + "method": "PercentChangePairList", "number_assets": 2, - "sort_key": "rolling_volume_change", + "sort_key": "percentage", "min_value": 0, "refresh_period": 86400, + "sort_direction": "asc", "lookback_days": 4, } ] @@ -272,7 +260,7 @@ def test_filter_pairlist_with_empty_ticker(mocker, rpl_config, tickers, time_mac "open": [100, 102, 101, 103, 104, 105], "high": [102, 103, 102, 104, 105, 106], "low": [99, 101, 100, 102, 103, 104], - "close": [101, 102, 103, 104, 105, 106], + "close": [101, 102, 103, 104, 105, 105], "volume": [1000, 1500, 2000, 2500, 3000, 3500], } ), @@ -289,8 +277,8 @@ def test_filter_pairlist_with_empty_ticker(mocker, rpl_config, tickers, time_mac "open": [100, 102, 101, 103, 104, 105], "high": [102, 103, 102, 104, 105, 106], "low": [99, 101, 100, 102, 103, 104], - "close": [101, 102, 103, 104, 105, 106], - "volume": [1000, 1500, 2000, 2500, 3000, 3500], + "close": [101, 102, 103, 104, 105, 104], + "volume": [1000, 1500, 2000, 2500, 3000, 3400], } ), } @@ -299,22 +287,22 @@ def test_filter_pairlist_with_empty_ticker(mocker, rpl_config, tickers, time_mac exchange = get_patched_exchange(mocker, rpl_config, exchange="binance") pairlistmanager = PairListManager(exchange, rpl_config) - remote_pairlist = PercentVolumeChangePairList( + remote_pairlist = PercentChangePairList( exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0 ) result = remote_pairlist.filter_pairlist(rpl_config["exchange"]["pair_whitelist"], {}) assert len(result) == 2 - assert result == ["ETH/USDT", "XRP/USDT"] + assert result == ["XRP/USDT", "ETH/USDT"] def test_filter_pairlist_with_max_value_set(mocker, rpl_config, tickers, time_machine): rpl_config["pairlists"] = [ { - "method": "PercentVolumeChangePairList", + "method": "PercentChangePairList", "number_assets": 2, - "sort_key": "rolling_volume_change", + "sort_key": "percentage", "min_value": 0, "max_value": 15, "refresh_period": 86400, @@ -356,7 +344,7 @@ def test_filter_pairlist_with_max_value_set(mocker, rpl_config, tickers, time_ma "open": [100, 102, 101, 103, 104, 105], "high": [102, 103, 102, 104, 105, 106], "low": [99, 101, 100, 102, 103, 104], - "close": [101, 102, 103, 104, 105, 106], + "close": [101, 102, 103, 104, 105, 101], "volume": [1000, 1500, 2000, 2500, 3000, 3500], } ), @@ -366,7 +354,7 @@ def test_filter_pairlist_with_max_value_set(mocker, rpl_config, tickers, time_ma exchange = get_patched_exchange(mocker, rpl_config, exchange="binance") pairlistmanager = PairListManager(exchange, rpl_config) - remote_pairlist = PercentVolumeChangePairList( + remote_pairlist = PercentChangePairList( exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0 ) @@ -374,3 +362,28 @@ def test_filter_pairlist_with_max_value_set(mocker, rpl_config, tickers, time_ma assert len(result) == 1 assert result == ["ETH/USDT"] + + +def test_gen_pairlist_from_tickers(mocker, rpl_config, tickers): + rpl_config["pairlists"] = [ + { + "method": "PercentChangePairList", + "number_assets": 2, + "sort_key": "percentage", + "min_value": 0, + } + ] + + mocker.patch(f"{EXMS}.exchange_has", MagicMock(return_value=True)) + + exchange = get_patched_exchange(mocker, rpl_config, exchange="binance") + pairlistmanager = PairListManager(exchange, rpl_config) + + remote_pairlist = PercentChangePairList( + exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0 + ) + + result = remote_pairlist.gen_pairlist(tickers.return_value) + + assert len(result) == 1 + assert result == ["ETH/USDT"] From 4a768682ea8681e67a13e83bcb2389effcff11a8 Mon Sep 17 00:00:00 2001 From: jainanuj94 Date: Fri, 26 Jul 2024 13:13:26 +0530 Subject: [PATCH 05/13] Remove unnecessary logs and up description --- freqtrade/plugins/pairlist/PercentChangePairList.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/freqtrade/plugins/pairlist/PercentChangePairList.py b/freqtrade/plugins/pairlist/PercentChangePairList.py index a1be6729b..b9ffa7e45 100644 --- a/freqtrade/plugins/pairlist/PercentChangePairList.py +++ b/freqtrade/plugins/pairlist/PercentChangePairList.py @@ -1,9 +1,9 @@ """ -Change PairList provider +Percent Change PairList provider Provides dynamic pair list based on trade change -sorted based on percentage change in volume over a -defined period +sorted based on percentage change in price over a +defined period or as coming from ticker """ import logging @@ -314,7 +314,6 @@ class PercentChangePairList(IPairList): # replace change with a range change sum calculated above filtered_tickers[i]["percentage"] = pct_change - self.log_once(f"Tickers: {filtered_tickers}", logger.info) else: filtered_tickers[i]["percentage"] = 0 From 8637f4a70d835abd3fdf9f3beda7784d3ddc5fbe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Jul 2024 16:04:51 +0200 Subject: [PATCH 06/13] Remove SortKey dynamics and setting --- .../plugins/pairlist/PercentChangePairList.py | 28 ++++--------------- tests/plugins/test_percentchangepairlist.py | 19 ------------- 2 files changed, 5 insertions(+), 42 deletions(-) diff --git a/freqtrade/plugins/pairlist/PercentChangePairList.py b/freqtrade/plugins/pairlist/PercentChangePairList.py index b9ffa7e45..14b08b2b7 100644 --- a/freqtrade/plugins/pairlist/PercentChangePairList.py +++ b/freqtrade/plugins/pairlist/PercentChangePairList.py @@ -8,7 +8,7 @@ defined period or as coming from ticker import logging from datetime import timedelta -from typing import Any, Dict, List, Literal, Optional +from typing import Any, Dict, List, Optional from cachetools import TTLCache @@ -22,8 +22,6 @@ from freqtrade.util import dt_now, format_ms_time logger = logging.getLogger(__name__) -SORT_VALUES = ["percentage"] - class PercentChangePairList(IPairList): is_pairlist_generator = True @@ -40,9 +38,6 @@ class PercentChangePairList(IPairList): self._stake_currency = self._config["stake_currency"] self._number_pairs = self._pairlistconfig["number_assets"] - self._sort_key: Literal["rolling_volume_change"] = self._pairlistconfig.get( - "sort_key", "rolling_volume_change" - ) self._min_value = self._pairlistconfig.get("min_value", 0) self._max_value = self._pairlistconfig.get("max_value", None) self._refresh_period = self._pairlistconfig.get("refresh_period", 1800) @@ -89,9 +84,6 @@ class PercentChangePairList(IPairList): "or switch to using candles. and restart the bot." ) - if not self._validate_keys(self._sort_key): - raise OperationalException(f"key {self._sort_key} not in {SORT_VALUES}") - candle_limit = self._exchange.ohlcv_candle_limit( self._lookback_timeframe, self._config["candle_type_def"] ) @@ -111,9 +103,6 @@ class PercentChangePairList(IPairList): """ return not self._use_range - def _validate_keys(self, key): - return key in SORT_VALUES - def short_desc(self) -> str: """ Short whitelist method description - used for startup-messages @@ -133,13 +122,6 @@ class PercentChangePairList(IPairList): "description": "Number of assets", "help": "Number of assets to use from the pairlist", }, - "sort_key": { - "type": "option", - "default": "rolling_volume_change", - "options": SORT_VALUES, - "description": "Sort key", - "help": "Sort key to use for sorting the pairlist.", - }, "min_value": { "type": "number", "default": 0, @@ -210,7 +192,7 @@ class PercentChangePairList(IPairList): for k, v in tickers.items() if ( self._exchange.get_pair_quote_currency(k) == self._stake_currency - and (self._use_range or v.get(self._sort_key) is not None) + and (self._use_range or v.get("percentage") is not None) and v["symbol"] in _pairlist ) ] @@ -239,14 +221,14 @@ class PercentChangePairList(IPairList): # Fetching 24h change by default from supported exchange tickers self.fetch_percent_change_from_tickers(filtered_tickers, tickers) - filtered_tickers = [v for v in filtered_tickers if v[self._sort_key] > self._min_value] + filtered_tickers = [v for v in filtered_tickers if v["percentage"] > self._min_value] if self._max_value is not None: - filtered_tickers = [v for v in filtered_tickers if v[self._sort_key] < self._max_value] + filtered_tickers = [v for v in filtered_tickers if v["percentage"] < self._max_value] sorted_tickers = sorted( filtered_tickers, reverse=self._sort_direction == "desc", - key=lambda t: t[self._sort_key], + key=lambda t: t["percentage"], ) # Validate whitelist to only have active market pairs diff --git a/tests/plugins/test_percentchangepairlist.py b/tests/plugins/test_percentchangepairlist.py index 0a7960d22..df165cf98 100644 --- a/tests/plugins/test_percentchangepairlist.py +++ b/tests/plugins/test_percentchangepairlist.py @@ -71,25 +71,6 @@ def test_volume_change_pair_list_init_wrong_refresh_period(mocker, rpl_config): get_patched_freqtradebot(mocker, rpl_config) -def test_volume_change_pair_list_init_invalid_sort_key(mocker, rpl_config): - rpl_config["pairlists"] = [ - { - "method": "PercentChangePairList", - "number_assets": 2, - "sort_key": "wrong_key", - "min_value": 0, - "refresh_period": 86400, - "lookback_days": 1, - } - ] - - with pytest.raises( - OperationalException, - match=r"key wrong_key not in \['percentage'\]", - ): - get_patched_freqtradebot(mocker, rpl_config) - - def test_volume_change_pair_list_init_wrong_lookback_period(mocker, rpl_config): rpl_config["pairlists"] = [ { From 283e8045d82902bd5d7214b25ece714e9855d457 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Jul 2024 16:05:59 +0200 Subject: [PATCH 07/13] PercentChangePairlist should partecipate in regular tests --- tests/plugins/test_pairlist.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 31e746d5c..37ebdc58b 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -31,12 +31,9 @@ from tests.conftest import ( ) -# Exclude RemotePairList and PercentVolumePairList from tests. -# They have mandatory parameters, and requires special handling, -# which happens in test_remotepairlist and test_percentchangepairlist. -TESTABLE_PAIRLISTS = [ - p for p in AVAILABLE_PAIRLISTS if p not in ["RemotePairList", "PercentChangePairList"] -] +# Exclude RemotePairList from tests. +# It has a mandatory parameter, and requires special handling, which happens in test_remotepairlist. +TESTABLE_PAIRLISTS = [p for p in AVAILABLE_PAIRLISTS if p not in ["RemotePairList"]] @pytest.fixture(scope="function") From 4ac7a4fdab27c7ad341e77b1d4dc50da22fd4f8d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Jul 2024 16:07:51 +0200 Subject: [PATCH 08/13] Allow empty min_Value setting... --- freqtrade/plugins/pairlist/PercentChangePairList.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/PercentChangePairList.py b/freqtrade/plugins/pairlist/PercentChangePairList.py index 14b08b2b7..457c29275 100644 --- a/freqtrade/plugins/pairlist/PercentChangePairList.py +++ b/freqtrade/plugins/pairlist/PercentChangePairList.py @@ -38,7 +38,7 @@ class PercentChangePairList(IPairList): self._stake_currency = self._config["stake_currency"] self._number_pairs = self._pairlistconfig["number_assets"] - self._min_value = self._pairlistconfig.get("min_value", 0) + self._min_value = self._pairlistconfig.get("min_value", None) self._max_value = self._pairlistconfig.get("max_value", None) self._refresh_period = self._pairlistconfig.get("refresh_period", 1800) self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) @@ -221,7 +221,8 @@ class PercentChangePairList(IPairList): # Fetching 24h change by default from supported exchange tickers self.fetch_percent_change_from_tickers(filtered_tickers, tickers) - filtered_tickers = [v for v in filtered_tickers if v["percentage"] > self._min_value] + if self._min_value is not None: + filtered_tickers = [v for v in filtered_tickers if v["percentage"] > self._min_value] if self._max_value is not None: filtered_tickers = [v for v in filtered_tickers if v["percentage"] < self._max_value] From 206baf7d80c49382b37355cf7691f1003c7c98b3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Jul 2024 16:13:17 +0200 Subject: [PATCH 09/13] chore: add a bit of typehinting --- freqtrade/plugins/pairlist/PercentChangePairList.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/freqtrade/plugins/pairlist/PercentChangePairList.py b/freqtrade/plugins/pairlist/PercentChangePairList.py index 457c29275..eded33d1d 100644 --- a/freqtrade/plugins/pairlist/PercentChangePairList.py +++ b/freqtrade/plugins/pairlist/PercentChangePairList.py @@ -11,8 +11,9 @@ from datetime import timedelta from typing import Any, Dict, List, Optional from cachetools import TTLCache +from pandas import DataFrame -from freqtrade.constants import ListPairsWithTimeframes +from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date from freqtrade.exchange.types import Ticker, Tickers @@ -168,8 +169,6 @@ class PercentChangePairList(IPairList): :param tickers: Tickers (from exchange.get_tickers). May be cached. :return: List of pairs """ - # Generate dynamic whitelist - # Must always run if this pairlist is not the first in the list. pairlist = self._pair_cache.get("pairlist") if pairlist: # Item found - no refresh necessary @@ -240,7 +239,9 @@ class PercentChangePairList(IPairList): return pairs - def fetch_candles_for_lookback_period(self, filtered_tickers): + def fetch_candles_for_lookback_period( + self, filtered_tickers: List[Dict[str, str]] + ) -> Dict[PairWithTimeframe, DataFrame]: since_ms = ( int( timeframe_to_prev_date( @@ -276,7 +277,7 @@ class PercentChangePairList(IPairList): candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms) return candles - def fetch_percent_change_from_lookback_period(self, filtered_tickers): + def fetch_percent_change_from_lookback_period(self, filtered_tickers: List[Dict[str, Any]]): # get lookback period in ms, for exchange ohlcv fetch candles = self.fetch_candles_for_lookback_period(filtered_tickers) @@ -300,7 +301,7 @@ class PercentChangePairList(IPairList): else: filtered_tickers[i]["percentage"] = 0 - def fetch_percent_change_from_tickers(self, filtered_tickers, tickers): + def fetch_percent_change_from_tickers(self, filtered_tickers: List[Dict[str, Any]], tickers): for i, p in enumerate(filtered_tickers): # Filter out assets if not self._validate_pair( From 4932473b3f49c7d3d2e1258d504a66d765ca1a8f Mon Sep 17 00:00:00 2001 From: jainanuj94 Date: Sat, 27 Jul 2024 23:41:32 +0530 Subject: [PATCH 10/13] Add documentation --- docs/includes/pairlists.md | 81 +++++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index fbf8f4be0..15f1dbc85 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -2,11 +2,11 @@ Pairlist Handlers define the list of pairs (pairlist) that the bot should trade. They are configured in the `pairlists` section of the configuration settings. -In your configuration, you can use Static Pairlist (defined by the [`StaticPairList`](#static-pair-list) Pairlist Handler) and Dynamic Pairlist (defined by the [`VolumePairList`](#volume-pair-list) Pairlist Handler). +In your configuration, you can use Static Pairlist (defined by the [`StaticPairList`](#static-pair-list) Pairlist Handler) and Dynamic Pairlist (defined by the [`VolumePairList`](#volume-pair-list) and [`PercentChangePairList`](#percent-change-pair-list) Pairlist Handlers). Additionally, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter), [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist. -If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You can define either `StaticPairList`, `VolumePairList`, `ProducerPairList`, `RemotePairList` or `MarketCapPairList` as the starting Pairlist Handler. +If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You can define either `StaticPairList`, `VolumePairList`, `ProducerPairList`, `RemotePairList`, `MarketCapPairList` or `PercentChangePairList` as the starting Pairlist Handler. Inactive markets are always removed from the resulting pairlist. Explicitly blacklisted pairs (those in the `pair_blacklist` configuration setting) are also always removed from the resulting pairlist. @@ -22,6 +22,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged * [`StaticPairList`](#static-pair-list) (default, if not configured differently) * [`VolumePairList`](#volume-pair-list) +* [`PercentChangePairList`](#percent-change-pair-list) * [`ProducerPairList`](#producerpairlist) * [`RemotePairList`](#remotepairlist) * [`MarketCapPairList`](#marketcappairlist) @@ -152,6 +153,82 @@ More sophisticated approach can be used, by using `lookback_timeframe` for candl !!! Note `VolumePairList` does not support backtesting mode. +#### Percent Change Pair List + +`PercentChangePairList` filters and sorts pairs based on the percentage change in their price over the last 24 hours or any defined timeframe as part of advanced options. This allows traders to focus on assets that have experienced significant price movements, either positive or negative. + +**Configuration Options** + +- `number_assets`: Specifies the number of top pairs to select based on the 24-hour percentage change. +- `min_value`: Sets a minimum percentage change threshold. Pairs with a percentage change below this value will be filtered out. +- `max_value`: Sets a maximum percentage change threshold. Pairs with a percentage change above this value will be filtered out. +- `refresh_period`: Defines the interval (in seconds) at which the pairlist will be refreshed. The default is 1800 seconds (30 minutes). +- `lookback_days`: Number of days to look back. When `lookback_days` is selected, the `lookback_timeframe` is defaulted to 1 day. +- `lookback_timeframe`: Timeframe to use for the lookback period. +- `lookback_period`: Number of periods to look back at. + +When PercentChangePairList is used after other Pairlist Handlers, it will operate on the outputs of those handlers. If it is the leading Pairlist Handler, it will select pairs from all available markets with the specified stake currency. + +`PercentChangePairList` uses ticker data from the exchange, provided via the ccxt library: +The percentage change is calculated as the change in price over the last 24 hours, expressed as a percentage, depending on the exchange. + +??? Tip "Unsupported exchanges" + On some exchanges (like HTX), regular PercentChangePairList does not work as the api does not natively provide 24h percent change in price. This can be worked around by using candle data to calculate the percentage change. To roughly simulate 24h percent change, you can use the following configuration. Please note that These pairlists will only refresh once per day. + ```json + "pairlists": [ + { + "method": "PercentChangePairList", + "number_assets": 20, + "sort_key": "percentage", + "min_value": 0, + "refresh_period": 86400, + "lookback_days": 1 + } + ], + ``` + +**Example Configuration to Read from Ticker** +```json +"pairlists": [ + { + "method": "PercentChangePairList", + "number_assets": 15, + "sort_key": "percentage", + "min_value": -10, + "max_value": 50 + } +], +``` +In this configuration: + +1. The top 15 pairs are selected based on the highest percentage change in price over the last 24 hours. +2. Only pairs with a percentage change between -10% and 50% are considered. + +**Example Configuration to Read from Candles** +```json +"pairlists": [ + { + "method": "PercentChangePairList", + "number_assets": 15, + "sort_key": "percentage", + "min_value": 0, + "refresh_period": 3600, + "lookback_timeframe": "1h", + "lookback_period": 72 + } +], +``` +This example builds the percent change pairs based on a rolling period of 3 days of 1-hour candles by using `lookback_timeframe` for candle size and `lookback_period` which specifies the number of candles. + +!!! Warning "Range look back and refresh period" + When used in conjunction with `lookback_days` and `lookback_timeframe` the `refresh_period` can not be smaller than the candle size in seconds. As this will result in unnecessary requests to the exchanges API. + +!!! Warning "Performance implications when using lookback range" + If used in first position in combination with lookback, the computation of the range-based percent change can be time and resource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `PercentChangePairList` to narrow the pairlist down for further range volume calculation. + +!!! Note Backtesting + `PercentChangePairList` does not support backtesting mode. + #### ProducerPairList With `ProducerPairList`, you can reuse the pairlist from a [Producer](producer-consumer.md) without explicitly defining the pairlist on each consumer. From ac1e405c341c0eb5e1ffd470e8f8a3dce6f348fa Mon Sep 17 00:00:00 2001 From: jainanuj94 Date: Sun, 28 Jul 2024 21:10:20 +0530 Subject: [PATCH 11/13] Update documentation and fix doc test --- docs/includes/pairlists.md | 13 ++++++++----- freqtrade/plugins/pairlist/PercentChangePairList.py | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 15f1dbc85..9ea797682 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -162,6 +162,7 @@ More sophisticated approach can be used, by using `lookback_timeframe` for candl - `number_assets`: Specifies the number of top pairs to select based on the 24-hour percentage change. - `min_value`: Sets a minimum percentage change threshold. Pairs with a percentage change below this value will be filtered out. - `max_value`: Sets a maximum percentage change threshold. Pairs with a percentage change above this value will be filtered out. +- `sort_direction`: Specifies the order in which pairs are sorted based on their percentage change. Accepts two values: `asc` for ascending order and `desc` for descending order. - `refresh_period`: Defines the interval (in seconds) at which the pairlist will be refreshed. The default is 1800 seconds (30 minutes). - `lookback_days`: Number of days to look back. When `lookback_days` is selected, the `lookback_timeframe` is defaulted to 1 day. - `lookback_timeframe`: Timeframe to use for the lookback period. @@ -170,7 +171,7 @@ More sophisticated approach can be used, by using `lookback_timeframe` for candl When PercentChangePairList is used after other Pairlist Handlers, it will operate on the outputs of those handlers. If it is the leading Pairlist Handler, it will select pairs from all available markets with the specified stake currency. `PercentChangePairList` uses ticker data from the exchange, provided via the ccxt library: -The percentage change is calculated as the change in price over the last 24 hours, expressed as a percentage, depending on the exchange. +The percentage change is calculated as the change in price over the last 24 hours. ??? Tip "Unsupported exchanges" On some exchanges (like HTX), regular PercentChangePairList does not work as the api does not natively provide 24h percent change in price. This can be worked around by using candle data to calculate the percentage change. To roughly simulate 24h percent change, you can use the following configuration. Please note that These pairlists will only refresh once per day. @@ -179,7 +180,6 @@ The percentage change is calculated as the change in price over the last 24 hour { "method": "PercentChangePairList", "number_assets": 20, - "sort_key": "percentage", "min_value": 0, "refresh_period": 86400, "lookback_days": 1 @@ -193,7 +193,6 @@ The percentage change is calculated as the change in price over the last 24 hour { "method": "PercentChangePairList", "number_assets": 15, - "sort_key": "percentage", "min_value": -10, "max_value": 50 } @@ -220,13 +219,17 @@ In this configuration: ``` This example builds the percent change pairs based on a rolling period of 3 days of 1-hour candles by using `lookback_timeframe` for candle size and `lookback_period` which specifies the number of candles. +The percent change in price is calculated using the following formula, which expresses the percentage difference between the current candle's close price and the previous candle's close price, as defined by the specified timeframe and lookback period: + +$$ Percent Change = (\frac{Current Close - Previous Close}{Previous Close}) * 100 $$ + !!! Warning "Range look back and refresh period" When used in conjunction with `lookback_days` and `lookback_timeframe` the `refresh_period` can not be smaller than the candle size in seconds. As this will result in unnecessary requests to the exchanges API. !!! Warning "Performance implications when using lookback range" - If used in first position in combination with lookback, the computation of the range-based percent change can be time and resource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `PercentChangePairList` to narrow the pairlist down for further range volume calculation. + If used in first position in combination with lookback, the computation of the range-based percent change can be time and resource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `PercentChangePairList` to narrow the pairlist down for further percent-change calculation. -!!! Note Backtesting +!!! Note "Backtesting" `PercentChangePairList` does not support backtesting mode. #### ProducerPairList diff --git a/freqtrade/plugins/pairlist/PercentChangePairList.py b/freqtrade/plugins/pairlist/PercentChangePairList.py index eded33d1d..b22891b98 100644 --- a/freqtrade/plugins/pairlist/PercentChangePairList.py +++ b/freqtrade/plugins/pairlist/PercentChangePairList.py @@ -125,7 +125,7 @@ class PercentChangePairList(IPairList): }, "min_value": { "type": "number", - "default": 0, + "default": None, "description": "Minimum value", "help": "Minimum value to use for filtering the pairlist.", }, From 27aed5cd7ee907cb8a990ee7324d28646994297f Mon Sep 17 00:00:00 2001 From: jainanuj94 Date: Sun, 28 Jul 2024 22:34:34 +0530 Subject: [PATCH 12/13] Update schema.json --- build_helpers/schema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/build_helpers/schema.json b/build_helpers/schema.json index c0933d9f8..c2b930558 100644 --- a/build_helpers/schema.json +++ b/build_helpers/schema.json @@ -562,6 +562,7 @@ "enum": [ "StaticPairList", "VolumePairList", + "PercentChangePairList", "ProducerPairList", "RemotePairList", "MarketCapPairList", From eb0fc0fc807ffae7c891c4e823a166c8726bd1fa Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Jul 2024 20:29:21 +0200 Subject: [PATCH 13/13] docs: Fix minor typo --- docs/includes/pairlists.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 9ea797682..b3b69f996 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -159,22 +159,22 @@ More sophisticated approach can be used, by using `lookback_timeframe` for candl **Configuration Options** -- `number_assets`: Specifies the number of top pairs to select based on the 24-hour percentage change. -- `min_value`: Sets a minimum percentage change threshold. Pairs with a percentage change below this value will be filtered out. -- `max_value`: Sets a maximum percentage change threshold. Pairs with a percentage change above this value will be filtered out. -- `sort_direction`: Specifies the order in which pairs are sorted based on their percentage change. Accepts two values: `asc` for ascending order and `desc` for descending order. -- `refresh_period`: Defines the interval (in seconds) at which the pairlist will be refreshed. The default is 1800 seconds (30 minutes). -- `lookback_days`: Number of days to look back. When `lookback_days` is selected, the `lookback_timeframe` is defaulted to 1 day. -- `lookback_timeframe`: Timeframe to use for the lookback period. -- `lookback_period`: Number of periods to look back at. +* `number_assets`: Specifies the number of top pairs to select based on the 24-hour percentage change. +* `min_value`: Sets a minimum percentage change threshold. Pairs with a percentage change below this value will be filtered out. +* `max_value`: Sets a maximum percentage change threshold. Pairs with a percentage change above this value will be filtered out. +* `sort_direction`: Specifies the order in which pairs are sorted based on their percentage change. Accepts two values: `asc` for ascending order and `desc` for descending order. +* `refresh_period`: Defines the interval (in seconds) at which the pairlist will be refreshed. The default is 1800 seconds (30 minutes). +* `lookback_days`: Number of days to look back. When `lookback_days` is selected, the `lookback_timeframe` is defaulted to 1 day. +* `lookback_timeframe`: Timeframe to use for the lookback period. +* `lookback_period`: Number of periods to look back at. When PercentChangePairList is used after other Pairlist Handlers, it will operate on the outputs of those handlers. If it is the leading Pairlist Handler, it will select pairs from all available markets with the specified stake currency. `PercentChangePairList` uses ticker data from the exchange, provided via the ccxt library: The percentage change is calculated as the change in price over the last 24 hours. -??? Tip "Unsupported exchanges" - On some exchanges (like HTX), regular PercentChangePairList does not work as the api does not natively provide 24h percent change in price. This can be worked around by using candle data to calculate the percentage change. To roughly simulate 24h percent change, you can use the following configuration. Please note that These pairlists will only refresh once per day. +??? Note "Unsupported exchanges" + On some exchanges (like HTX), regular PercentChangePairList does not work as the api does not natively provide 24h percent change in price. This can be worked around by using candle data to calculate the percentage change. To roughly simulate 24h percent change, you can use the following configuration. Please note that these pairlists will only refresh once per day. ```json "pairlists": [ { @@ -188,6 +188,7 @@ The percentage change is calculated as the change in price over the last 24 hour ``` **Example Configuration to Read from Ticker** + ```json "pairlists": [ { @@ -198,12 +199,14 @@ The percentage change is calculated as the change in price over the last 24 hour } ], ``` + In this configuration: 1. The top 15 pairs are selected based on the highest percentage change in price over the last 24 hours. 2. Only pairs with a percentage change between -10% and 50% are considered. **Example Configuration to Read from Candles** + ```json "pairlists": [ { @@ -217,6 +220,7 @@ In this configuration: } ], ``` + This example builds the percent change pairs based on a rolling period of 3 days of 1-hour candles by using `lookback_timeframe` for candle size and `lookback_period` which specifies the number of candles. The percent change in price is calculated using the following formula, which expresses the percentage difference between the current candle's close price and the previous candle's close price, as defined by the specified timeframe and lookback period: