From 142690c93068be58937aaef08dd9004871f237fb Mon Sep 17 00:00:00 2001 From: gbojen Date: Tue, 6 Apr 2021 10:05:03 +0200 Subject: [PATCH 01/12] resolves freqtrade/freqtrade#4650 --- config_binance.json.example | 99 --------------- docker-compose.yml | 2 +- freqtrade/constants.py | 4 +- .../plugins/pairlist/VolatilityFilter.py | 120 ++++++++++++++++++ 4 files changed, 123 insertions(+), 102 deletions(-) delete mode 100644 config_binance.json.example create mode 100644 freqtrade/plugins/pairlist/VolatilityFilter.py diff --git a/config_binance.json.example b/config_binance.json.example deleted file mode 100644 index 4fa615d6d..000000000 --- a/config_binance.json.example +++ /dev/null @@ -1,99 +0,0 @@ -{ - "max_open_trades": 3, - "stake_currency": "BTC", - "stake_amount": 0.05, - "tradable_balance_ratio": 0.99, - "fiat_display_currency": "USD", - "timeframe": "5m", - "dry_run": true, - "cancel_open_orders_on_exit": false, - "unfilledtimeout": { - "buy": 10, - "sell": 30 - }, - "bid_strategy": { - "ask_last_balance": 0.0, - "use_order_book": false, - "order_book_top": 1, - "check_depth_of_market": { - "enabled": false, - "bids_to_ask_delta": 1 - } - }, - "ask_strategy": { - "use_order_book": false, - "order_book_min": 1, - "order_book_max": 1, - "use_sell_signal": true, - "sell_profit_only": false, - "ignore_roi_if_buy_signal": false - }, - "exchange": { - "name": "binance", - "key": "your_exchange_key", - "secret": "your_exchange_secret", - "ccxt_config": {"enableRateLimit": true}, - "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 200 - }, - "pair_whitelist": [ - "ALGO/BTC", - "ATOM/BTC", - "BAT/BTC", - "BCH/BTC", - "BRD/BTC", - "EOS/BTC", - "ETH/BTC", - "IOTA/BTC", - "LINK/BTC", - "LTC/BTC", - "NEO/BTC", - "NXS/BTC", - "XMR/BTC", - "XRP/BTC", - "XTZ/BTC" - ], - "pair_blacklist": [ - "BNB/BTC" - ] - }, - "pairlists": [ - {"method": "StaticPairList"} - ], - "edge": { - "enabled": false, - "process_throttle_secs": 3600, - "calculate_since_number_of_days": 7, - "allowed_risk": 0.01, - "stoploss_range_min": -0.01, - "stoploss_range_max": -0.1, - "stoploss_range_step": -0.01, - "minimum_winrate": 0.60, - "minimum_expectancy": 0.20, - "min_trade_number": 10, - "max_trade_duration_minute": 1440, - "remove_pumps": false - }, - "telegram": { - "enabled": false, - "token": "your_telegram_token", - "chat_id": "your_telegram_chat_id" - }, - "api_server": { - "enabled": false, - "listen_ip_address": "127.0.0.1", - "listen_port": 8080, - "verbosity": "error", - "jwt_secret_key": "somethingrandom", - "CORS_origins": [], - "username": "freqtrader", - "password": "SuperSecurePassword" - }, - "bot_name": "freqtrade", - "initial_state": "running", - "forcebuy_enable": false, - "internals": { - "process_throttle_secs": 5 - } -} diff --git a/docker-compose.yml b/docker-compose.yml index 80e194ab2..71572140c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,4 +25,4 @@ services: --logfile /freqtrade/user_data/logs/freqtrade.log --db-url sqlite:////freqtrade/user_data/tradesv3.sqlite --config /freqtrade/user_data/config.json - --strategy SampleStrategy + --strategy BinHV45.py diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 3a2ed98e9..c4a360d18 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -26,7 +26,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'AgeFilter', 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', - 'SpreadFilter'] + 'SpreadFilter', 'VolatilityFilter'] AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] DRY_RUN_WALLET = 1000 @@ -416,4 +416,4 @@ PairWithTimeframe = Tuple[str, str] ListPairsWithTimeframes = List[PairWithTimeframe] # Type for trades list -TradeList = List[List] +TradeList = List[List] \ No newline at end of file diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py new file mode 100644 index 000000000..ea1ebeb29 --- /dev/null +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -0,0 +1,120 @@ +""" +Rate of change pairlist filter +""" +import logging +from copy import deepcopy +from typing import Any, Dict, List, Optional + +import sys +import arrow +from cachetools.ttl import TTLCache +from pandas import DataFrame +import numpy as np + +from freqtrade.exceptions import OperationalException +from freqtrade.misc import plural +from freqtrade.plugins.pairlist.IPairList import IPairList + + + +logger = logging.getLogger(__name__) + + +class VolatilityFilter(IPairList): + ''' + Filters pairs by volatility + ''' + + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._days = pairlistconfig.get('lookback_days', 10) + self._min_volatility = pairlistconfig.get('min_volatility', 0) + self._max_volatility = pairlistconfig.get('max_volatility', sys.maxsize) + self._refresh_period = pairlistconfig.get('refresh_period', 1440) + + self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period) + + if self._days < 1: + raise OperationalException("VolatilityFilter requires lookback_days to be >= 1") + if self._days > exchange.ohlcv_candle_limit('1d'): + raise OperationalException("VolatilityFilter requires lookback_days to not " + "exceed exchange max request size " + f"({exchange.ohlcv_candle_limit('1d')})") + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return False + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return (f"{self.name} - Filtering pairs with volatility range " + f"{self._min_volatility}-{self._max_volatility} the last {self._days} {plural(self._days, 'day')}.") + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Validate trading range + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new allowlist + """ + needed_pairs = [(p, '1h') for p in pairlist if p not in self._pair_cache] + + since_ms = int(arrow.utcnow() + .floor('day') + .shift(days=-self._days - 1) + .float_timestamp) * 1000 + # Get all candles + candles = {} + if needed_pairs: + candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, + cache=False) + + if self._enabled: + for p in deepcopy(pairlist): + daily_candles = candles[(p, '1h')] if (p, '1h') in candles else None + if not self._validate_pair_loc(p, daily_candles): + pairlist.remove(p) + return pairlist + + def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool: + """ + Validate trading range + :param pair: Pair that's currently validated + :param ticker: ticker dict as returned from ccxt.load_markets() + :return: True if the pair can stay, false if it should be removed + """ + # Check symbol in cache + if pair in self._pair_cache: + return self._pair_cache[pair] + + result = False + if daily_candles is not None and not daily_candles.empty: + returns = (np.log(daily_candles.close / daily_candles.close.shift(-1))) + returns.fillna(0, inplace=True) + + volatility_series = returns.rolling(window=self._days*24).std()*np.sqrt(self._days*24) + volatility_avg = volatility_series.mean() + + if self._min_volatility <= volatility_avg <= self._max_volatility: + result = True + else: + self.log_once(f"Removed {pair} from whitelist, because volatility " + f"over {self._days} {plural(self._days, 'day')} " + f"is: {volatility_avg:.3f} " + f"which is not in the configured range of " + f"{self._min_volatility}-{self._max_volatility}.", + logger.info) + result = False + self._pair_cache[pair] = result + + return result From 6f02acdbbd8d1f524e552ae9e24a775fbe7071b1 Mon Sep 17 00:00:00 2001 From: gbojen Date: Tue, 6 Apr 2021 10:39:27 +0200 Subject: [PATCH 02/12] Revert "resolves freqtrade/freqtrade#4650" This reverts commit 142690c93068be58937aaef08dd9004871f237fb. --- config_binance.json.example | 99 +++++++++++++++ docker-compose.yml | 2 +- freqtrade/constants.py | 4 +- .../plugins/pairlist/VolatilityFilter.py | 120 ------------------ 4 files changed, 102 insertions(+), 123 deletions(-) create mode 100644 config_binance.json.example delete mode 100644 freqtrade/plugins/pairlist/VolatilityFilter.py diff --git a/config_binance.json.example b/config_binance.json.example new file mode 100644 index 000000000..4fa615d6d --- /dev/null +++ b/config_binance.json.example @@ -0,0 +1,99 @@ +{ + "max_open_trades": 3, + "stake_currency": "BTC", + "stake_amount": 0.05, + "tradable_balance_ratio": 0.99, + "fiat_display_currency": "USD", + "timeframe": "5m", + "dry_run": true, + "cancel_open_orders_on_exit": false, + "unfilledtimeout": { + "buy": 10, + "sell": 30 + }, + "bid_strategy": { + "ask_last_balance": 0.0, + "use_order_book": false, + "order_book_top": 1, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + } + }, + "ask_strategy": { + "use_order_book": false, + "order_book_min": 1, + "order_book_max": 1, + "use_sell_signal": true, + "sell_profit_only": false, + "ignore_roi_if_buy_signal": false + }, + "exchange": { + "name": "binance", + "key": "your_exchange_key", + "secret": "your_exchange_secret", + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": true, + "rateLimit": 200 + }, + "pair_whitelist": [ + "ALGO/BTC", + "ATOM/BTC", + "BAT/BTC", + "BCH/BTC", + "BRD/BTC", + "EOS/BTC", + "ETH/BTC", + "IOTA/BTC", + "LINK/BTC", + "LTC/BTC", + "NEO/BTC", + "NXS/BTC", + "XMR/BTC", + "XRP/BTC", + "XTZ/BTC" + ], + "pair_blacklist": [ + "BNB/BTC" + ] + }, + "pairlists": [ + {"method": "StaticPairList"} + ], + "edge": { + "enabled": false, + "process_throttle_secs": 3600, + "calculate_since_number_of_days": 7, + "allowed_risk": 0.01, + "stoploss_range_min": -0.01, + "stoploss_range_max": -0.1, + "stoploss_range_step": -0.01, + "minimum_winrate": 0.60, + "minimum_expectancy": 0.20, + "min_trade_number": 10, + "max_trade_duration_minute": 1440, + "remove_pumps": false + }, + "telegram": { + "enabled": false, + "token": "your_telegram_token", + "chat_id": "your_telegram_chat_id" + }, + "api_server": { + "enabled": false, + "listen_ip_address": "127.0.0.1", + "listen_port": 8080, + "verbosity": "error", + "jwt_secret_key": "somethingrandom", + "CORS_origins": [], + "username": "freqtrader", + "password": "SuperSecurePassword" + }, + "bot_name": "freqtrade", + "initial_state": "running", + "forcebuy_enable": false, + "internals": { + "process_throttle_secs": 5 + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 71572140c..80e194ab2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,4 +25,4 @@ services: --logfile /freqtrade/user_data/logs/freqtrade.log --db-url sqlite:////freqtrade/user_data/tradesv3.sqlite --config /freqtrade/user_data/config.json - --strategy BinHV45.py + --strategy SampleStrategy diff --git a/freqtrade/constants.py b/freqtrade/constants.py index c4a360d18..3a2ed98e9 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -26,7 +26,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'AgeFilter', 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', - 'SpreadFilter', 'VolatilityFilter'] + 'SpreadFilter'] AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] DRY_RUN_WALLET = 1000 @@ -416,4 +416,4 @@ PairWithTimeframe = Tuple[str, str] ListPairsWithTimeframes = List[PairWithTimeframe] # Type for trades list -TradeList = List[List] \ No newline at end of file +TradeList = List[List] diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py deleted file mode 100644 index ea1ebeb29..000000000 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ /dev/null @@ -1,120 +0,0 @@ -""" -Rate of change pairlist filter -""" -import logging -from copy import deepcopy -from typing import Any, Dict, List, Optional - -import sys -import arrow -from cachetools.ttl import TTLCache -from pandas import DataFrame -import numpy as np - -from freqtrade.exceptions import OperationalException -from freqtrade.misc import plural -from freqtrade.plugins.pairlist.IPairList import IPairList - - - -logger = logging.getLogger(__name__) - - -class VolatilityFilter(IPairList): - ''' - Filters pairs by volatility - ''' - - def __init__(self, exchange, pairlistmanager, - config: Dict[str, Any], pairlistconfig: Dict[str, Any], - pairlist_pos: int) -> None: - super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) - - self._days = pairlistconfig.get('lookback_days', 10) - self._min_volatility = pairlistconfig.get('min_volatility', 0) - self._max_volatility = pairlistconfig.get('max_volatility', sys.maxsize) - self._refresh_period = pairlistconfig.get('refresh_period', 1440) - - self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period) - - if self._days < 1: - raise OperationalException("VolatilityFilter requires lookback_days to be >= 1") - if self._days > exchange.ohlcv_candle_limit('1d'): - raise OperationalException("VolatilityFilter requires lookback_days to not " - "exceed exchange max request size " - f"({exchange.ohlcv_candle_limit('1d')})") - - @property - def needstickers(self) -> bool: - """ - Boolean property defining if tickers are necessary. - If no Pairlist requires tickers, an empty List is passed - as tickers argument to filter_pairlist - """ - return False - - def short_desc(self) -> str: - """ - Short whitelist method description - used for startup-messages - """ - return (f"{self.name} - Filtering pairs with volatility range " - f"{self._min_volatility}-{self._max_volatility} the last {self._days} {plural(self._days, 'day')}.") - - def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: - """ - Validate trading range - :param pairlist: pairlist to filter or sort - :param tickers: Tickers (from exchange.get_tickers()). May be cached. - :return: new allowlist - """ - needed_pairs = [(p, '1h') for p in pairlist if p not in self._pair_cache] - - since_ms = int(arrow.utcnow() - .floor('day') - .shift(days=-self._days - 1) - .float_timestamp) * 1000 - # Get all candles - candles = {} - if needed_pairs: - candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, - cache=False) - - if self._enabled: - for p in deepcopy(pairlist): - daily_candles = candles[(p, '1h')] if (p, '1h') in candles else None - if not self._validate_pair_loc(p, daily_candles): - pairlist.remove(p) - return pairlist - - def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool: - """ - Validate trading range - :param pair: Pair that's currently validated - :param ticker: ticker dict as returned from ccxt.load_markets() - :return: True if the pair can stay, false if it should be removed - """ - # Check symbol in cache - if pair in self._pair_cache: - return self._pair_cache[pair] - - result = False - if daily_candles is not None and not daily_candles.empty: - returns = (np.log(daily_candles.close / daily_candles.close.shift(-1))) - returns.fillna(0, inplace=True) - - volatility_series = returns.rolling(window=self._days*24).std()*np.sqrt(self._days*24) - volatility_avg = volatility_series.mean() - - if self._min_volatility <= volatility_avg <= self._max_volatility: - result = True - else: - self.log_once(f"Removed {pair} from whitelist, because volatility " - f"over {self._days} {plural(self._days, 'day')} " - f"is: {volatility_avg:.3f} " - f"which is not in the configured range of " - f"{self._min_volatility}-{self._max_volatility}.", - logger.info) - result = False - self._pair_cache[pair] = result - - return result From be770a89417b90a82d537ac5f26d2d76ee726a76 Mon Sep 17 00:00:00 2001 From: gbojen Date: Tue, 6 Apr 2021 10:42:53 +0200 Subject: [PATCH 03/12] added VolatilityFilter resolves freqtrade#4650 --- freqtrade/constants.py | 2 +- .../plugins/pairlist/VolatilityFilter.py | 120 ++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 freqtrade/plugins/pairlist/VolatilityFilter.py diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 3a2ed98e9..b98161ff2 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -26,7 +26,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'AgeFilter', 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', - 'SpreadFilter'] + 'SpreadFilter', 'VolatilityFilter'] AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] DRY_RUN_WALLET = 1000 diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py new file mode 100644 index 000000000..97e86bab6 --- /dev/null +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -0,0 +1,120 @@ +""" +Rate of change pairlist filter +""" +import logging +from copy import deepcopy +from typing import Any, Dict, List, Optional + +import sys +import arrow +from cachetools.ttl import TTLCache +from pandas import DataFrame +import numpy as np + +from freqtrade.exceptions import OperationalException +from freqtrade.misc import plural +from freqtrade.plugins.pairlist.IPairList import IPairList + + + +logger = logging.getLogger(__name__) + + +class VolatilityFilter(IPairList): + ''' + Filters pairs by volatility + ''' + + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._days = pairlistconfig.get('lookback_days', 10) + self._min_volatility = pairlistconfig.get('min_volatility', 0) + self._max_volatility = pairlistconfig.get('max_volatility', sys.maxsize) + self._refresh_period = pairlistconfig.get('refresh_period', 1440) + + self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period) + + if self._days < 1: + raise OperationalException("VolatilityFilter requires lookback_days to be >= 1") + if self._days > exchange.ohlcv_candle_limit('1d'): + raise OperationalException("VolatilityFilter requires lookback_days to not " + "exceed exchange max request size " + f"({exchange.ohlcv_candle_limit('1d')})") + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return False + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return (f"{self.name} - Filtering pairs with volatility range " + f"{self._min_volatility}-{self._max_volatility} the last {self._days} {plural(self._days, 'day')}.") + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Validate trading range + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new allowlist + """ + needed_pairs = [(p, '1h') for p in pairlist if p not in self._pair_cache] + + since_ms = int(arrow.utcnow() + .floor('day') + .shift(days=-self._days - 1) + .float_timestamp) * 1000 + # Get all candles + candles = {} + if needed_pairs: + candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, + cache=False) + + if self._enabled: + for p in deepcopy(pairlist): + daily_candles = candles[(p, '1h')] if (p, '1h') in candles else None + if not self._validate_pair_loc(p, daily_candles): + pairlist.remove(p) + return pairlist + + def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool: + """ + Validate trading range + :param pair: Pair that's currently validated + :param ticker: ticker dict as returned from ccxt.load_markets() + :return: True if the pair can stay, false if it should be removed + """ + # Check symbol in cache + if pair in self._pair_cache: + return self._pair_cache[pair] + + result = False + if daily_candles is not None and not daily_candles.empty: + returns = (np.log(daily_candles.close / daily_candles.close.shift(-1))) + returns.fillna(0, inplace=True) + + volatility_series = returns.rolling(window=self._days*24).std()*np.sqrt(self._days*24) + volatility_avg = volatility_series.mean() + + if self._min_volatility <= volatility_avg <= self._max_volatility: + result = True + else: + self.log_once(f"Removed {pair} from whitelist, because volatility " + f"over {self._days} {plural(self._days, 'day')} " + f"is: {volatility_avg:.3f} " + f"which is not in the configured range of " + f"{self._min_volatility}-{self._max_volatility}.", + logger.info) + result = False + self._pair_cache[pair] = result + + return result \ No newline at end of file From 1733e24062339363084200e12f1bbda26f41dde0 Mon Sep 17 00:00:00 2001 From: gbojen Date: Tue, 6 Apr 2021 10:44:13 +0200 Subject: [PATCH 04/12] pyLint adjustment resolves freqtrade#4650 --- freqtrade/plugins/pairlist/VolatilityFilter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 97e86bab6..1913bfcc1 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -117,4 +117,5 @@ class VolatilityFilter(IPairList): result = False self._pair_cache[pair] = result - return result \ No newline at end of file + return result + \ No newline at end of file From 187cf6dcd5217914cea52c3296afb9f58728d017 Mon Sep 17 00:00:00 2001 From: gbojen Date: Tue, 6 Apr 2021 22:41:15 +0200 Subject: [PATCH 05/12] VolatilityFilter resolves freqtrade/freqtrade#4650 --- freqtrade/plugins/pairlist/VolatilityFilter.py | 6 +++--- tests/plugins/test_pairlist.py | 15 +++++++++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 1913bfcc1..1bb836e76 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -67,7 +67,7 @@ class VolatilityFilter(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new allowlist """ - needed_pairs = [(p, '1h') for p in pairlist if p not in self._pair_cache] + needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache] since_ms = int(arrow.utcnow() .floor('day') @@ -81,7 +81,7 @@ class VolatilityFilter(IPairList): if self._enabled: for p in deepcopy(pairlist): - daily_candles = candles[(p, '1h')] if (p, '1h') in candles else None + daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None if not self._validate_pair_loc(p, daily_candles): pairlist.remove(p) return pairlist @@ -102,7 +102,7 @@ class VolatilityFilter(IPairList): returns = (np.log(daily_candles.close / daily_candles.close.shift(-1))) returns.fillna(0, inplace=True) - volatility_series = returns.rolling(window=self._days*24).std()*np.sqrt(self._days*24) + volatility_series = returns.rolling(window=self._days).std()*np.sqrt(self._days) volatility_avg = volatility_series.mean() if self._min_volatility <= volatility_avg <= self._max_volatility: diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 67cd96f5b..7d39014f1 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -407,6 +407,10 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): {"method": "RangeStabilityFilter", "lookback_days": 10, "min_rate_of_change": 0.01, "refresh_period": 1440}], "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']), + ([{"method": "StaticPairList"}, + {"method": "VolatilityFilter", "lookback_days": 3, + "min_volatility": 0.002, "max_volatility": 0.004, "refresh_period": 1440}], + "BTC", ['ETH/BTC', 'TKN/BTC']) ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history, pairlists, base_currency, @@ -414,13 +418,19 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t whitelist_conf['pairlists'] = pairlists whitelist_conf['stake_currency'] = base_currency + ohlcv_history_high_vola = ohlcv_history.copy() + ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index==1, 'close'] = 0.00090 + ohlcv_data = { ('ETH/BTC', '1d'): ohlcv_history, ('TKN/BTC', '1d'): ohlcv_history, ('LTC/BTC', '1d'): ohlcv_history, ('XRP/BTC', '1d'): ohlcv_history, - ('HOT/BTC', '1d'): ohlcv_history, + ('HOT/BTC', '1d'): ohlcv_history_high_vola, } + + + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) @@ -487,7 +497,8 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t assert log_has(logmsg, caplog) else: assert not log_has(logmsg, caplog) - + if pairlist["method"] == 'VolatilityFilter': + assert log_has_re(r'^Removed .* from whitelist, because volatility.*$', caplog) def test_PrecisionFilter_error(mocker, whitelist_conf) -> None: whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PrecisionFilter"}] From 9089323d266980c1d896c35341e3fd3e68dd8362 Mon Sep 17 00:00:00 2001 From: gbojen Date: Tue, 6 Apr 2021 22:46:36 +0200 Subject: [PATCH 06/12] resolves freqtrade/freqtrade#4650 --- freqtrade/plugins/pairlist/VolatilityFilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 1bb836e76..5b50e04e4 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -1,5 +1,5 @@ """ -Rate of change pairlist filter +Volatility pairlist filter """ import logging from copy import deepcopy From 9772a93634a1c60be321cbb78a51cec52cd59a5c Mon Sep 17 00:00:00 2001 From: gbojen Date: Tue, 6 Apr 2021 23:11:40 +0200 Subject: [PATCH 07/12] resolves freqtrade/freqtrade#4650 --- freqtrade/plugins/pairlist/VolatilityFilter.py | 5 ++--- tests/plugins/test_pairlist.py | 8 +++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 5b50e04e4..6ef3841f5 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -16,7 +16,6 @@ from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList - logger = logging.getLogger(__name__) @@ -58,7 +57,8 @@ class VolatilityFilter(IPairList): Short whitelist method description - used for startup-messages """ return (f"{self.name} - Filtering pairs with volatility range " - f"{self._min_volatility}-{self._max_volatility} the last {self._days} {plural(self._days, 'day')}.") + f"{self._min_volatility}-{self._max_volatility} " + f" the last {self._days} {plural(self._days, 'day')}.") def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ @@ -118,4 +118,3 @@ class VolatilityFilter(IPairList): self._pair_cache[pair] = result return result - \ No newline at end of file diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 7d39014f1..4db0b7098 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -419,7 +419,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t whitelist_conf['stake_currency'] = base_currency ohlcv_history_high_vola = ohlcv_history.copy() - ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index==1, 'close'] = 0.00090 + ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index == 1, 'close'] = 0.00090 ohlcv_data = { ('ETH/BTC', '1d'): ohlcv_history, @@ -428,15 +428,12 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t ('XRP/BTC', '1d'): ohlcv_history, ('HOT/BTC', '1d'): ohlcv_history_high_vola, } - - - mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) if whitelist_result == 'static_in_the_middle': with pytest.raises(OperationalException, - match=r"StaticPairList can only be used in the first position " + match=r"StaticPairList only in the first position " r"in the list of Pairlist Handlers."): freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) return @@ -500,6 +497,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t if pairlist["method"] == 'VolatilityFilter': assert log_has_re(r'^Removed .* from whitelist, because volatility.*$', caplog) + def test_PrecisionFilter_error(mocker, whitelist_conf) -> None: whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PrecisionFilter"}] del whitelist_conf['stoploss'] From 17508efbbc88e89a7380cb6d14f12c5350e0ed5c Mon Sep 17 00:00:00 2001 From: gbojen Date: Wed, 7 Apr 2021 08:59:44 +0200 Subject: [PATCH 08/12] resolves freqtrade/freqtrade#4650 --- docs/includes/pairlists.md | 33 ++++++++++++++++++++++++++++++++- tests/plugins/test_pairlist.py | 2 +- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 2653406e7..ad1ac6efc 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -4,7 +4,7 @@ Pairlist Handlers define the list of pairs (pairlist) that the bot should trade. In your configuration, you can use Static Pairlist (defined by the [`StaticPairList`](#static-pair-list) Pairlist Handler) and Dynamic Pairlist (defined by the [`VolumePairList`](#volume-pair-list) Pairlist Handler). -Additionally, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter) and [`SpreadFilter`](#spreadfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist. +Additionally, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter), [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist. If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You should always configure either `StaticPairList` or `VolumePairList` as the starting Pairlist Handler. @@ -29,6 +29,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged * [`ShuffleFilter`](#shufflefilter) * [`SpreadFilter`](#spreadfilter) * [`RangeStabilityFilter`](#rangestabilityfilter) +* [`VolatilityFilter`](#volatilityfilter) !!! Tip "Testing pairlists" Pairlist configurations can be quite tricky to get right. Best use the [`test-pairlist`](utils.md#test-pairlist) utility sub-command to test your configuration quickly. @@ -164,6 +165,29 @@ If the trading range over the last 10 days is <1%, remove the pair from the whit !!! Tip This Filter can be used to automatically remove stable coin pairs, which have a very low trading range, and are therefore extremely difficult to trade with profit. +#### VolatilityFilter + +Volatily is the degree of historical variation of a pairs over time, is is measured by the standard deviation of logarithmic daily returns. + +Removes pairs where the average volatility over a `lookback_days` days is below `min_volatility` and above `max_volatility`. Since this is a filter that requires additional data, the results are cached for `refresh_period`. + +This filter can be used to narrow down your pairs to a certain volatilty or avoid very volatile pairs. + +In the below example: +If the volatilty over the last 10 days is not in the range of 0.20-0.30, remove the pair from the whitelist. The filter is applied every 24h. + +```json +"pairlists": [ + { + "method": "VolatilityFilter", + "lookback_days": 10, + "min_volatilty": 0.20, + "max_volatilty": 0.30, + "refresh_period": 86400 + } +] +``` + ### Full example of Pairlist Handlers The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies both [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#price-filter), filtering all assets where 1 price unit is > 1%. Then the `SpreadFilter` is applied and pairs are finally shuffled with the random seed set to some predefined value. @@ -189,6 +213,13 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, "min_rate_of_change": 0.01, "refresh_period": 1440 }, + { + "method": "VolatilityFilter", + "lookback_days": 10, + "min_volatilty": 0.20, + "max_volatilty": 0.30, + "refresh_period": 86400 + }, {"method": "ShuffleFilter", "seed": 42} ], ``` diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 4db0b7098..bf225271f 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -433,7 +433,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t if whitelist_result == 'static_in_the_middle': with pytest.raises(OperationalException, - match=r"StaticPairList only in the first position " + match=r"StaticPairList can only be used in the first position " r"in the list of Pairlist Handlers."): freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) return From 5ee879a747a5bbb916601c90af1e0cb25de56515 Mon Sep 17 00:00:00 2001 From: gbojen Date: Wed, 7 Apr 2021 10:15:51 +0200 Subject: [PATCH 09/12] isort resolves freqtrade/freqtrade#4650 --- freqtrade/plugins/pairlist/VolatilityFilter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 6ef3841f5..f8e380f56 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -2,14 +2,14 @@ Volatility pairlist filter """ import logging +import sys from copy import deepcopy from typing import Any, Dict, List, Optional -import sys import arrow +import numpy as np from cachetools.ttl import TTLCache from pandas import DataFrame -import numpy as np from freqtrade.exceptions import OperationalException from freqtrade.misc import plural From f8244d9d76610c96e0995754a934272bf94a536f Mon Sep 17 00:00:00 2001 From: gbojen Date: Wed, 7 Apr 2021 22:25:54 +0200 Subject: [PATCH 10/12] resolves freqtrade/freqtrade#4650 --- docs/includes/pairlists.md | 6 +++--- freqtrade/plugins/pairlist/VolatilityFilter.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index ad1ac6efc..8c65753b6 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -167,7 +167,7 @@ If the trading range over the last 10 days is <1%, remove the pair from the whit #### VolatilityFilter -Volatily is the degree of historical variation of a pairs over time, is is measured by the standard deviation of logarithmic daily returns. +Volatily is the degree of historical variation of a pairs over time, is is measured by the standard deviation of logarithmic daily returns. Returns are assumed to be normally distributed, although actual distribution might be different. In a normal distribution, 68% of observations fall within one standard deviation and 95% of observations fall within two standard deviations. Assuming a volatilty of 0.05 means that the expected returns for 20 out of 30 days is expected to be less than 5% (one standard deviation). Volatility is a positive ratio of the expected deviation of return and can be greater than 1.00. Please refer to the wikipedia definition of [`volatility`](https://en.wikipedia.org/wiki/Volatility_(finance)). Removes pairs where the average volatility over a `lookback_days` days is below `min_volatility` and above `max_volatility`. Since this is a filter that requires additional data, the results are cached for `refresh_period`. @@ -181,8 +181,8 @@ If the volatilty over the last 10 days is not in the range of 0.20-0.30, remove { "method": "VolatilityFilter", "lookback_days": 10, - "min_volatilty": 0.20, - "max_volatilty": 0.30, + "min_volatility": 0.05, + "max_volatility": 0.50, "refresh_period": 86400 } ] diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index f8e380f56..400b1577d 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -94,8 +94,9 @@ class VolatilityFilter(IPairList): :return: True if the pair can stay, false if it should be removed """ # Check symbol in cache - if pair in self._pair_cache: - return self._pair_cache[pair] + cached_res = self._pair_cache.get(pair, None) + if cached_res is not None: + return cached_res result = False if daily_candles is not None and not daily_candles.empty: From 862f69f895c4dd1774ba78ca9a221692ad9bb930 Mon Sep 17 00:00:00 2001 From: gbojen Date: Thu, 8 Apr 2021 16:43:38 +0200 Subject: [PATCH 11/12] removed typos --- docs/includes/pairlists.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 8c65753b6..3aa2c63f4 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -167,14 +167,14 @@ If the trading range over the last 10 days is <1%, remove the pair from the whit #### VolatilityFilter -Volatily is the degree of historical variation of a pairs over time, is is measured by the standard deviation of logarithmic daily returns. Returns are assumed to be normally distributed, although actual distribution might be different. In a normal distribution, 68% of observations fall within one standard deviation and 95% of observations fall within two standard deviations. Assuming a volatilty of 0.05 means that the expected returns for 20 out of 30 days is expected to be less than 5% (one standard deviation). Volatility is a positive ratio of the expected deviation of return and can be greater than 1.00. Please refer to the wikipedia definition of [`volatility`](https://en.wikipedia.org/wiki/Volatility_(finance)). +Volatility is the degree of historical variation of a pairs over time, is is measured by the standard deviation of logarithmic daily returns. Returns are assumed to be normally distributed, although actual distribution might be different. In a normal distribution, 68% of observations fall within one standard deviation and 95% of observations fall within two standard deviations. Assuming a volatilty of 0.05 means that the expected returns for 20 out of 30 days is expected to be less than 5% (one standard deviation). Volatility is a positive ratio of the expected deviation of return and can be greater than 1.00. Please refer to the wikipedia definition of [`volatility`](https://en.wikipedia.org/wiki/Volatility_(finance)). -Removes pairs where the average volatility over a `lookback_days` days is below `min_volatility` and above `max_volatility`. Since this is a filter that requires additional data, the results are cached for `refresh_period`. +This filter removes pairs if the average volatility over a `lookback_days` days is below `min_volatility` or above `max_volatility`. Since this is a filter that requires additional data, the results are cached for `refresh_period`. -This filter can be used to narrow down your pairs to a certain volatilty or avoid very volatile pairs. +This filter can be used to narrow down your pairs to a certain volatility or avoid very volatile pairs. In the below example: -If the volatilty over the last 10 days is not in the range of 0.20-0.30, remove the pair from the whitelist. The filter is applied every 24h. +If the volatility over the last 10 days is not in the range of 0.20-0.30, remove the pair from the whitelist. The filter is applied every 24h. ```json "pairlists": [ From 74bf0b6399adf9766f8af6b2c68140723d0c3d51 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 8 Apr 2021 19:29:51 +0200 Subject: [PATCH 12/12] Fix typo in documentation --- docs/includes/pairlists.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 3aa2c63f4..d57757bbd 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -167,7 +167,7 @@ If the trading range over the last 10 days is <1%, remove the pair from the whit #### VolatilityFilter -Volatility is the degree of historical variation of a pairs over time, is is measured by the standard deviation of logarithmic daily returns. Returns are assumed to be normally distributed, although actual distribution might be different. In a normal distribution, 68% of observations fall within one standard deviation and 95% of observations fall within two standard deviations. Assuming a volatilty of 0.05 means that the expected returns for 20 out of 30 days is expected to be less than 5% (one standard deviation). Volatility is a positive ratio of the expected deviation of return and can be greater than 1.00. Please refer to the wikipedia definition of [`volatility`](https://en.wikipedia.org/wiki/Volatility_(finance)). +Volatility is the degree of historical variation of a pairs over time, is is measured by the standard deviation of logarithmic daily returns. Returns are assumed to be normally distributed, although actual distribution might be different. In a normal distribution, 68% of observations fall within one standard deviation and 95% of observations fall within two standard deviations. Assuming a volatility of 0.05 means that the expected returns for 20 out of 30 days is expected to be less than 5% (one standard deviation). Volatility is a positive ratio of the expected deviation of return and can be greater than 1.00. Please refer to the wikipedia definition of [`volatility`](https://en.wikipedia.org/wiki/Volatility_(finance)). This filter removes pairs if the average volatility over a `lookback_days` days is below `min_volatility` or above `max_volatility`. Since this is a filter that requires additional data, the results are cached for `refresh_period`. @@ -216,8 +216,8 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, { "method": "VolatilityFilter", "lookback_days": 10, - "min_volatilty": 0.20, - "max_volatilty": 0.30, + "min_volatility": 0.05, + "max_volatility": 0.50, "refresh_period": 86400 }, {"method": "ShuffleFilter", "seed": 42}