From 466a3b87fcda0001ea4325ba0aef2c0457d59679 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 28 Oct 2019 16:13:31 +0100 Subject: [PATCH 01/39] Enhance tests to cover precision_filter correctly --- tests/conftest.py | 60 +++++++++++++++++++++++++++++++++ tests/pairlist/test_pairlist.py | 33 +++++++++++------- 2 files changed, 81 insertions(+), 12 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4feae6a60..a291a6676 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -572,6 +572,44 @@ def get_markets(): } +@pytest.fixture +def shitcoinmarkets(markets): + """ + Fixture with shitcoin markets - used to test filters in pairlists + """ + shitmarkets = deepcopy(markets) + shitmarkets.update({'HOT/BTC': { + 'id': 'HOTBTC', + 'symbol': 'HOT/BTC', + 'base': 'HOT', + 'quote': 'BTC', + 'active': True, + 'precision': { + 'base': 8, + 'quote': 8, + 'amount': 0, + 'price': 8 + }, + 'limits': { + 'amount': { + 'min': 1.0, + 'max': 90000000.0 + }, + 'price': { + 'min': None, + 'max': None + }, + 'cost': { + 'min': 0.001, + 'max': None + } + }, + 'info': {}, + }, + }) + return shitmarkets + + @pytest.fixture def markets_empty(): return MagicMock(return_value=[]) @@ -866,6 +904,28 @@ def tickers(): 'quoteVolume': 1215.14489611, 'info': {} }, + 'HOT/BTC': { + 'symbol': 'HOT/BTC', + 'timestamp': 1572273518661, + 'datetime': '2019-10-28T14:38:38.661Z', + 'high': 0.00000011, + 'low': 0.00000009, + 'bid': 0.0000001, + 'bidVolume': 1476027288.0, + 'ask': 0.00000011, + 'askVolume': 820153831.0, + 'vwap': 0.0000001, + 'open': 0.00000009, + 'close': 0.00000011, + 'last': 0.00000011, + 'previousClose': 0.00000009, + 'change': 0.00000002, + 'percentage': 22.222, + 'average': None, + 'baseVolume': 1442290324.0, + 'quoteVolume': 143.78311994, + 'info': {} + }, 'ETH/USDT': { 'symbol': 'ETH/USDT', 'timestamp': 1522014804118, diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 929fc0ba0..6f050a77d 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -2,11 +2,12 @@ from unittest.mock import MagicMock, PropertyMock +import pytest + from freqtrade import OperationalException from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.resolvers import PairListResolver -from tests.conftest import get_patched_freqtradebot -import pytest +from tests.conftest import get_patched_freqtradebot, log_has_re # whitelist, blacklist @@ -67,20 +68,25 @@ def test_refresh_pairlists(mocker, markets, whitelist_conf): assert whitelist_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist -def test_refresh_pairlist_dynamic(mocker, markets, tickers, whitelist_conf): +def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_conf): whitelist_conf['pairlist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 5} + 'config': {'number_assets': 5, + 'precision_filter': False} } + mocker.patch.multiple( 'freqtrade.exchange.Exchange', - markets=PropertyMock(return_value=markets), get_tickers=tickers, exchange_has=MagicMock(return_value=True) ) freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) - + # Remock markets with shitcoinmarkets since get_patched_freqtradebot uses the markets fixture + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=shitcoinmarkets), + ) # argument: use the whitelist dynamically by exchange-volume - whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'] + whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC'] freqtradebot.pairlists.refresh_pairlist() assert whitelist == freqtradebot.pairlists.whitelist @@ -108,19 +114,20 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): @pytest.mark.parametrize("precision_filter,base_currency,key,whitelist_result", [ - (False, "BTC", "quoteVolume", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), - (False, "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC']), + (False, "BTC", "quoteVolume", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC']), + (False, "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'HOT/BTC']), (False, "USDT", "quoteVolume", ['ETH/USDT']), (False, "ETH", "quoteVolume", []), # this replaces tests that were removed from test_exchange (True, "BTC", "quoteVolume", ["LTC/BTC", "ETH/BTC", "TKN/BTC"]), (True, "BTC", "bidVolume", ["LTC/BTC", "TKN/BTC", "ETH/BTC"]) ]) -def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers, base_currency, key, - whitelist_result, precision_filter) -> None: +def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, + precision_filter, base_currency, key, whitelist_result, + caplog) -> None: whitelist_conf['pairlist']['method'] = 'VolumePairList' mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) - mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) + mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=shitcoinmarkets)) mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, p, r: round(r, 8)) @@ -128,6 +135,8 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers, freqtrade.config['stake_currency'] = base_currency whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency=base_currency, key=key) assert sorted(whitelist) == sorted(whitelist_result) + if precision_filter: + assert log_has_re(r'^Removed .* from whitelist, because stop price .* would be <= stop limit.*', caplog) def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: From 4ff035537b01cf454a3cb34b60f232ad48972a2f Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 28 Oct 2019 16:21:00 +0100 Subject: [PATCH 02/39] Simplify precision_filter code --- config_full.json.example | 2 +- docs/configuration.md | 5 ++--- freqtrade/pairlist/VolumePairList.py | 21 +++++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 5789e49ac..a5af0f7a6 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -55,7 +55,7 @@ "config": { "number_assets": 20, "sort_key": "quoteVolume", - "precision_filter": false + "precision_filter": true } }, "exchange": { diff --git a/docs/configuration.md b/docs/configuration.md index 1ad13c87a..03f15e07d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -423,8 +423,7 @@ section of the configuration. * `VolumePairList` * It selects `number_assets` top pairs based on `sort_key`, which can be one of `askVolume`, `bidVolume` and `quoteVolume`, defaults to `quoteVolume`. - * There is a possibility to filter low-value coins that would not allow setting a stop loss -(set `precision_filter` parameter to `true` for this). + * By default, low-value coins that would not allow setting a stop loss are filtered out. (set `precision_filter` parameter to `false` to disable this behaviour). * `VolumePairList` does not consider `pair_whitelist`, but builds this automatically based the pairlist configuration. * Pairs in `pair_blacklist` are not considered for VolumePairList, even if all other filters would match. @@ -440,7 +439,7 @@ Example: "config": { "number_assets": 20, "sort_key": "quoteVolume", - "precision_filter": false + "precision_filter": true } }, ``` diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 5f53cd17b..5e20f0fb1 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -26,7 +26,7 @@ class VolumePairList(IPairList): 'for "pairlist.config.number_assets"') self._number_pairs = self._whitelistconf['number_assets'] self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume') - self._precision_filter = self._whitelistconf.get('precision_filter', False) + self._precision_filter = self._whitelistconf.get('precision_filter', True) if not self._freqtrade.exchange.exchange_has('fetchTickers'): raise OperationalException( @@ -76,18 +76,19 @@ class VolumePairList(IPairList): valid_tickers = [t for t in sorted_tickers if t["symbol"] in valid_pairs] if self._freqtrade.strategy.stoploss is not None and self._precision_filter: + # Precalculate correct stoploss value + stoploss = 1 - abs(self._freqtrade.strategy.stoploss) - stop_prices = [self._freqtrade.get_target_bid(t["symbol"], t) - * (1 - abs(self._freqtrade.strategy.stoploss)) for t in valid_tickers] - rates = [sp * 0.99 for sp in stop_prices] - logger.debug("\n".join([f"{sp} : {r}" for sp, r in zip(stop_prices[:10], rates[:10])])) for i, t in enumerate(valid_tickers): - sp = self._freqtrade.exchange.symbol_price_prec(t["symbol"], stop_prices[i]) - r = self._freqtrade.exchange.symbol_price_prec(t["symbol"], rates[i]) - logger.debug(f"{t['symbol']} - {sp} : {r}") - if sp <= r: + stop_price = (self._freqtrade.get_target_bid(t["symbol"], t) * stoploss) + # Adjust stop-prices to precision + sp = self._freqtrade.exchange.symbol_price_prec(t["symbol"], stop_price) + stop_gap_price = self._freqtrade.exchange.symbol_price_prec(t["symbol"], + stop_price * 0.99) + logger.debug(f"{t['symbol']} - {sp} : {stop_gap_price}") + if sp <= stop_gap_price: logger.info(f"Removed {t['symbol']} from whitelist, " - f"because stop price {sp} would be <= stop limit {r}") + f"because stop price {sp} would be <= stop limit {stop_gap_price}") valid_tickers.remove(t) pairs = [s['symbol'] for s in valid_tickers] From d706571e6f9d29b6f843377bb7a2c0a62e778e6c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 28 Oct 2019 16:25:01 +0100 Subject: [PATCH 03/39] Extract precision_filter to seperate function --- freqtrade/pairlist/VolumePairList.py | 42 +++++++++++++++++++--------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 5e20f0fb1..44dbd0ecf 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -56,6 +56,27 @@ class VolumePairList(IPairList): self._whitelist = self._gen_pair_whitelist( self._config['stake_currency'], self._sort_key) + def _validate_precision_filter(self, ticker: dict, stoploss: float) -> bool: + """ + Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very + low value pairs. + :param ticker: ticker dict as returned from ccxt.load_markets() + :param stoploss: stoploss value as set in the configuration + (already cleaned to be guaranteed negative) + :return: True if the pair can stay, false if it should be removed + """ + stop_price = (self._freqtrade.get_target_bid(ticker["symbol"], ticker) * stoploss) + # Adjust stop-prices to precision + sp = self._freqtrade.exchange.symbol_price_prec(ticker["symbol"], stop_price) + stop_gap_price = self._freqtrade.exchange.symbol_price_prec(ticker["symbol"], + stop_price * 0.99) + logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}") + if sp <= stop_gap_price: + logger.info(f"Removed {ticker['symbol']} from whitelist, " + f"because stop price {sp} would be <= stop limit {stop_gap_price}") + return False + return True + @cached(TTLCache(maxsize=1, ttl=1800)) def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]: """ @@ -75,21 +96,16 @@ class VolumePairList(IPairList): valid_pairs = self._validate_whitelist([s['symbol'] for s in sorted_tickers]) valid_tickers = [t for t in sorted_tickers if t["symbol"] in valid_pairs] - if self._freqtrade.strategy.stoploss is not None and self._precision_filter: - # Precalculate correct stoploss value + stoploss = None + if self._freqtrade.strategy.stoploss is not None: + # Precalculate sanitized stoploss value to avoid recalculation for every pair stoploss = 1 - abs(self._freqtrade.strategy.stoploss) - for i, t in enumerate(valid_tickers): - stop_price = (self._freqtrade.get_target_bid(t["symbol"], t) * stoploss) - # Adjust stop-prices to precision - sp = self._freqtrade.exchange.symbol_price_prec(t["symbol"], stop_price) - stop_gap_price = self._freqtrade.exchange.symbol_price_prec(t["symbol"], - stop_price * 0.99) - logger.debug(f"{t['symbol']} - {sp} : {stop_gap_price}") - if sp <= stop_gap_price: - logger.info(f"Removed {t['symbol']} from whitelist, " - f"because stop price {sp} would be <= stop limit {stop_gap_price}") - valid_tickers.remove(t) + for t in valid_tickers: + # Filter out assets which would not allow setting a stoploss + if (stoploss and self._precision_filter + and not self._validate_precision_filter(t, stoploss)): + valid_tickers.remove(t) pairs = [s['symbol'] for s in valid_tickers] logger.info(f"Searching pairs: {pairs[:self._number_pairs]}") From d803d86f4d846ba8e67c2bde84b0de5efaea1ee6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2019 06:53:26 +0100 Subject: [PATCH 04/39] Add low_price_percent_filter --- config_full.json.example | 3 ++- docs/configuration.md | 5 ++++- freqtrade/pairlist/VolumePairList.py | 23 +++++++++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index a5af0f7a6..5ae8021d5 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -55,7 +55,8 @@ "config": { "number_assets": 20, "sort_key": "quoteVolume", - "precision_filter": true + "precision_filter": true, + "low_price_percent_filter": null } }, "exchange": { diff --git a/docs/configuration.md b/docs/configuration.md index 03f15e07d..c7e0dac31 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -426,6 +426,8 @@ section of the configuration. * By default, low-value coins that would not allow setting a stop loss are filtered out. (set `precision_filter` parameter to `false` to disable this behaviour). * `VolumePairList` does not consider `pair_whitelist`, but builds this automatically based the pairlist configuration. * Pairs in `pair_blacklist` are not considered for VolumePairList, even if all other filters would match. + * `low_price_percent_filter` allows filtering of pairs where a raise of 1 price unit is below the `low_price_percent_filter` ratio. + Calculation example: Min price precision is 8 decimals. If price is 0.00000011 - one step would be 0.00000012 - which is almost 10% higher than the previous value. Example: @@ -439,7 +441,8 @@ Example: "config": { "number_assets": 20, "sort_key": "quoteVolume", - "precision_filter": true + "precision_filter": true, + "low_price_percent_filter": 0.03 } }, ``` diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 44dbd0ecf..4a6768efa 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -27,6 +27,8 @@ class VolumePairList(IPairList): self._number_pairs = self._whitelistconf['number_assets'] self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume') self._precision_filter = self._whitelistconf.get('precision_filter', True) + self._low_price_percent_filter = self._whitelistconf.get('low_price_percent_filter', None) + print(self._whitelistconf) if not self._freqtrade.exchange.exchange_has('fetchTickers'): raise OperationalException( @@ -77,6 +79,23 @@ class VolumePairList(IPairList): return False return True + def _validate_precision_filter_lowprice(self, ticker) -> bool: + """ + Check if if one price-step is > than a certain barrier. + :param ticker: ticker dict as returned from ccxt.load_markets() + :param precision: Precision + :return: True if the pair can stay, false if it should be removed + """ + precision = self._freqtrade.exchange.markets[ticker['symbol']]['precision']['price'] + + compare = ticker['last'] + 1 / pow(10, precision) + changeperc = (compare - ticker['last']) / ticker['last'] + if changeperc > self._low_price_percent_filter: + logger.info(f"Removed {ticker['symbol']} from whitelist, " + f"because 1 unit is {changeperc * 100:.3f}%") + return False + return True + @cached(TTLCache(maxsize=1, ttl=1800)) def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]: """ @@ -106,6 +125,10 @@ class VolumePairList(IPairList): if (stoploss and self._precision_filter and not self._validate_precision_filter(t, stoploss)): valid_tickers.remove(t) + continue + if self._low_price_percent_filter and not self._validate_precision_filter_lowprice(t,): + valid_tickers.remove(t) + continue pairs = [s['symbol'] for s in valid_tickers] logger.info(f"Searching pairs: {pairs[:self._number_pairs]}") From de2cc58b0cf0024256bd0c2c924d080838b5b86f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2019 10:39:27 +0100 Subject: [PATCH 05/39] Final cleanups and added tests --- config_full.json.example | 2 +- docs/configuration.md | 4 ++- freqtrade/pairlist/VolumePairList.py | 15 +++++---- tests/conftest.py | 50 ++++++++++++++++++++++++++++ tests/pairlist/test_pairlist.py | 36 ++++++++++++-------- 5 files changed, 85 insertions(+), 22 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 5ae8021d5..5935de392 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -56,7 +56,7 @@ "number_assets": 20, "sort_key": "quoteVolume", "precision_filter": true, - "low_price_percent_filter": null + "low_price_percent_filter": 0 } }, "exchange": { diff --git a/docs/configuration.md b/docs/configuration.md index c7e0dac31..7ccb6840a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -427,7 +427,9 @@ section of the configuration. * `VolumePairList` does not consider `pair_whitelist`, but builds this automatically based the pairlist configuration. * Pairs in `pair_blacklist` are not considered for VolumePairList, even if all other filters would match. * `low_price_percent_filter` allows filtering of pairs where a raise of 1 price unit is below the `low_price_percent_filter` ratio. + This option is disabled by default, and will only apply if set to <> 0. Calculation example: Min price precision is 8 decimals. If price is 0.00000011 - one step would be 0.00000012 - which is almost 10% higher than the previous value. + Example: @@ -442,7 +444,7 @@ Example: "number_assets": 20, "sort_key": "quoteVolume", "precision_filter": true, - "low_price_percent_filter": 0.03 + "low_price_percent_filter": 0.05 } }, ``` diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 4a6768efa..7c17c6d57 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -5,11 +5,14 @@ Provides lists as configured in config.json """ import logging +from copy import deepcopy from typing import List + from cachetools import TTLCache, cached -from freqtrade.pairlist.IPairList import IPairList from freqtrade import OperationalException +from freqtrade.pairlist.IPairList import IPairList + logger = logging.getLogger(__name__) SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] @@ -28,7 +31,6 @@ class VolumePairList(IPairList): self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume') self._precision_filter = self._whitelistconf.get('precision_filter', True) self._low_price_percent_filter = self._whitelistconf.get('low_price_percent_filter', None) - print(self._whitelistconf) if not self._freqtrade.exchange.exchange_has('fetchTickers'): raise OperationalException( @@ -64,10 +66,10 @@ class VolumePairList(IPairList): low value pairs. :param ticker: ticker dict as returned from ccxt.load_markets() :param stoploss: stoploss value as set in the configuration - (already cleaned to be guaranteed negative) + (already cleaned to be 1 - stoploss) :return: True if the pair can stay, false if it should be removed """ - stop_price = (self._freqtrade.get_target_bid(ticker["symbol"], ticker) * stoploss) + stop_price = self._freqtrade.get_target_bid(ticker["symbol"], ticker) * stoploss # Adjust stop-prices to precision sp = self._freqtrade.exchange.symbol_price_prec(ticker["symbol"], stop_price) stop_gap_price = self._freqtrade.exchange.symbol_price_prec(ticker["symbol"], @@ -120,10 +122,11 @@ class VolumePairList(IPairList): # Precalculate sanitized stoploss value to avoid recalculation for every pair stoploss = 1 - abs(self._freqtrade.strategy.stoploss) - for t in valid_tickers: + # Copy list since we're modifying this list + for t in deepcopy(valid_tickers): # Filter out assets which would not allow setting a stoploss if (stoploss and self._precision_filter - and not self._validate_precision_filter(t, stoploss)): + and not self._validate_precision_filter(t, stoploss)): valid_tickers.remove(t) continue if self._low_price_percent_filter and not self._validate_precision_filter_lowprice(t,): diff --git a/tests/conftest.py b/tests/conftest.py index a291a6676..d551596f0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -606,6 +606,34 @@ def shitcoinmarkets(markets): }, 'info': {}, }, + 'FUEL/BTC': { + 'id': 'FUELBTC', + 'symbol': 'FUEL/BTC', + 'base': 'FUEL', + 'quote': 'BTC', + 'active': True, + 'precision': { + 'base': 8, + 'quote': 8, + 'amount': 0, + 'price': 8 + }, + 'limits': { + 'amount': { + 'min': 1.0, + 'max': 90000000.0 + }, + 'price': { + 'min': 1e-08, + 'max': 1000.0 + }, + 'cost': { + 'min': 0.001, + 'max': None + } + }, + 'info': {}, + }, }) return shitmarkets @@ -926,6 +954,28 @@ def tickers(): 'quoteVolume': 143.78311994, 'info': {} }, + 'FUEL/BTC': { + 'symbol': 'FUEL/BTC', + 'timestamp': 1572340250771, + 'datetime': '2019-10-29T09:10:50.771Z', + 'high': 0.00000040, + 'low': 0.00000035, + 'bid': 0.00000036, + 'bidVolume': 8932318.0, + 'ask': 0.00000037, + 'askVolume': 10140774.0, + 'vwap': 0.00000037, + 'open': 0.00000039, + 'close': 0.00000037, + 'last': 0.00000037, + 'previousClose': 0.00000038, + 'change': -0.00000002, + 'percentage': -5.128, + 'average': None, + 'baseVolume': 168927742.0, + 'quoteVolume': 62.68220262, + 'info': {} + }, 'ETH/USDT': { 'symbol': 'ETH/USDT', 'timestamp': 1522014804118, diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 6f050a77d..ac27d1c8e 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -26,7 +26,7 @@ def whitelist_conf(default_conf): 'BLK/BTC' ] default_conf['pairlist'] = {'method': 'StaticPairList', - 'config': {'number_assets': 3} + 'config': {'number_assets': 5} } return default_conf @@ -86,7 +86,7 @@ def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_co markets=PropertyMock(return_value=shitcoinmarkets), ) # argument: use the whitelist dynamically by exchange-volume - whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC'] + whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC'] freqtradebot.pairlists.refresh_pairlist() assert whitelist == freqtradebot.pairlists.whitelist @@ -113,30 +113,38 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): assert set(whitelist) == set(pairslist) -@pytest.mark.parametrize("precision_filter,base_currency,key,whitelist_result", [ - (False, "BTC", "quoteVolume", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC']), - (False, "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'HOT/BTC']), - (False, "USDT", "quoteVolume", ['ETH/USDT']), - (False, "ETH", "quoteVolume", []), # this replaces tests that were removed from test_exchange - (True, "BTC", "quoteVolume", ["LTC/BTC", "ETH/BTC", "TKN/BTC"]), - (True, "BTC", "bidVolume", ["LTC/BTC", "TKN/BTC", "ETH/BTC"]) +@pytest.mark.parametrize("precision_filter,low_price_filter,base_currency,key,whitelist_result", [ + (False, 0, "BTC", "quoteVolume", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']), + (False, 0, "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'HOT/BTC', 'FUEL/BTC']), + (False, 0, "USDT", "quoteVolume", ['ETH/USDT']), + (False, 0, "ETH", "quoteVolume", []), + (True, 0, "BTC", "quoteVolume", ["LTC/BTC", "ETH/BTC", "TKN/BTC", 'FUEL/BTC']), + (True, 0, "BTC", "bidVolume", ["LTC/BTC", "TKN/BTC", "ETH/BTC", 'FUEL/BTC']), + (False, 0.03, "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'FUEL/BTC']), + # Hot is removed by precision_filter, Fuel by low_price_filter. + (True, 0.02, "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC']), ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, - precision_filter, base_currency, key, whitelist_result, - caplog) -> None: + precision_filter, low_price_filter, base_currency, key, + whitelist_result, caplog) -> None: whitelist_conf['pairlist']['method'] = 'VolumePairList' + whitelist_conf['pairlist']['config']['precision_filter'] = precision_filter + whitelist_conf['pairlist']['config']['low_price_percent_filter'] = low_price_filter + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=shitcoinmarkets)) mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) - mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, p, r: round(r, 8)) - freqtrade.pairlists._precision_filter = precision_filter freqtrade.config['stake_currency'] = base_currency whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency=base_currency, key=key) assert sorted(whitelist) == sorted(whitelist_result) if precision_filter: - assert log_has_re(r'^Removed .* from whitelist, because stop price .* would be <= stop limit.*', caplog) + assert log_has_re(r'^Removed .* from whitelist, because stop price .* ' + r'would be <= stop limit.*', caplog) + + if low_price_filter: + assert log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: From 44289e4c586324e1026ad0de03e1ef3c746128bd Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Oct 2019 15:55:35 +0100 Subject: [PATCH 06/39] Allow not using files from user_dir --- freqtrade/resolvers/iresolver.py | 10 +++++----- freqtrade/resolvers/pairlist_resolver.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 51c4f7dba..3bad42fd9 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -17,13 +17,13 @@ class IResolver: This class contains all the logic to load custom classes """ - def build_search_paths(self, config, current_path: Path, user_subdir: str, + def build_search_paths(self, config, current_path: Path, user_subdir: Optional[str] = None, extra_dir: Optional[str] = None) -> List[Path]: - abs_paths = [ - config['user_data_dir'].joinpath(user_subdir), - current_path, - ] + abs_paths: List[Path] = [current_path] + + if user_subdir: + abs_paths.insert(0, config['user_data_dir'].joinpath(user_subdir)) if extra_dir: # Add extra directory to the top of the search paths diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index c2782a219..0f23bb3fd 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -40,7 +40,7 @@ class PairListResolver(IResolver): current_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() abs_paths = self.build_search_paths(config, current_path=current_path, - user_subdir='pairlist', extra_dir=None) + user_subdir=None, extra_dir=None) pairlist = self._load_object(paths=abs_paths, object_type=IPairList, object_name=pairlist_name, kwargs=kwargs) From fd9c02603cfea41876dd562e224e15025c0637a9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Oct 2019 15:59:52 +0100 Subject: [PATCH 07/39] Introduce chainable PairlistFilters --- freqtrade/pairlist/IPairList.py | 36 +++++++++-- freqtrade/pairlist/IPairListFilter.py | 18 ++++++ freqtrade/pairlist/LowPriceFilter.py | 48 ++++++++++++++ freqtrade/pairlist/PrecisionFilter.py | 51 +++++++++++++++ freqtrade/pairlist/StaticPairList.py | 2 +- freqtrade/pairlist/VolumePairList.py | 62 +------------------ .../resolvers/pairlistfilter_resolver.py | 53 ++++++++++++++++ 7 files changed, 203 insertions(+), 67 deletions(-) create mode 100644 freqtrade/pairlist/IPairListFilter.py create mode 100644 freqtrade/pairlist/LowPriceFilter.py create mode 100644 freqtrade/pairlist/PrecisionFilter.py create mode 100644 freqtrade/resolvers/pairlistfilter_resolver.py diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 5afb0c4c2..eb6af9d52 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -6,10 +6,11 @@ Provides lists as configured in config.json """ import logging from abc import ABC, abstractmethod -from typing import List +from typing import Dict, List from freqtrade.exchange import market_is_active - +from freqtrade.pairlist.IPairListFilter import IPairListFilter +from freqtrade.resolvers.pairlistfilter_resolver import PairListFilterResolver logger = logging.getLogger(__name__) @@ -21,6 +22,12 @@ class IPairList(ABC): self._config = config self._whitelist = self._config['exchange']['pair_whitelist'] self._blacklist = self._config['exchange'].get('pair_blacklist', []) + self._filters = self._config.get('pairlist', {}).get('filters', {}) + self._pairlistfilters: List[IPairListFilter] = [] + for pl_filter in self._filters.keys(): + self._pairlistfilters.append( + PairListFilterResolver(pl_filter, freqtrade, self._config).pairlistfilter + ) @property def name(self) -> str: @@ -60,7 +67,23 @@ class IPairList(ABC): -> Please overwrite in subclasses """ - def _validate_whitelist(self, whitelist: List[str]) -> List[str]: + def validate_whitelist(self, pairlist: List[str], + tickers: List[Dict] = []) -> List[str]: + """ + Validate pairlist against active markets and blacklist. + Run PairlistFilters if these are configured. + """ + pairlist = self._whitelist_for_active_markets(pairlist) + + if not tickers: + # Refresh tickers if they are not used by the parent Pairlist + tickers = self._freqtrade.exchange.get_tickers() + + for pl_filter in self._pairlistfilters: + pairlist = pl_filter.filter_pairlist(pairlist, tickers) + return pairlist + + def _whitelist_for_active_markets(self, whitelist: List[str]) -> List[str]: """ Check available markets and remove pair from whitelist if necessary :param whitelist: the sorted list of pairs the user might want to trade @@ -69,7 +92,7 @@ class IPairList(ABC): """ markets = self._freqtrade.exchange.markets - sanitized_whitelist = set() + sanitized_whitelist: List[str] = [] for pair in whitelist: # pair is not in the generated dynamic market, or in the blacklist ... ignore it if (pair in self.blacklist or pair not in markets @@ -83,7 +106,8 @@ class IPairList(ABC): if not market_is_active(market): logger.info(f"Ignoring {pair} from whitelist. Market is not active.") continue - sanitized_whitelist.add(pair) + if pair not in sanitized_whitelist: + sanitized_whitelist.append(pair) # We need to remove pairs that are unknown - return list(sanitized_whitelist) + return sanitized_whitelist diff --git a/freqtrade/pairlist/IPairListFilter.py b/freqtrade/pairlist/IPairListFilter.py new file mode 100644 index 000000000..4b43f0e9f --- /dev/null +++ b/freqtrade/pairlist/IPairListFilter.py @@ -0,0 +1,18 @@ +import logging +from abc import ABC, abstractmethod +from typing import Dict, List + +logger = logging.getLogger(__name__) + + +class IPairListFilter(ABC): + + def __init__(self, freqtrade, config: dict) -> None: + self._freqtrade = freqtrade + self._config = config + + @abstractmethod + def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: + """ + Method doing the filtering + """ diff --git a/freqtrade/pairlist/LowPriceFilter.py b/freqtrade/pairlist/LowPriceFilter.py new file mode 100644 index 000000000..499dd0c15 --- /dev/null +++ b/freqtrade/pairlist/LowPriceFilter.py @@ -0,0 +1,48 @@ +import logging +from copy import deepcopy +from typing import Dict, List + +from freqtrade.pairlist.IPairListFilter import IPairListFilter + +logger = logging.getLogger(__name__) + + +class LowPriceFilter(IPairListFilter): + + def __init__(self, freqtrade, config: dict) -> None: + super().__init__(freqtrade, config) + + self._low_price_percent = config['pairlist']['filters']['LowPriceFilter'].get( + 'low_price_percent', 0) + + def _validate_precision_filter_lowprice(self, ticker) -> bool: + """ + Check if if one price-step is > than a certain barrier. + :param ticker: ticker dict as returned from ccxt.load_markets() + :param precision: Precision + :return: True if the pair can stay, false if it should be removed + """ + precision = self._freqtrade.exchange.markets[ticker['symbol']]['precision']['price'] + + compare = ticker['last'] + 1 / pow(10, precision) + changeperc = (compare - ticker['last']) / ticker['last'] + if changeperc > self._low_price_percent: + logger.info(f"Removed {ticker['symbol']} from whitelist, " + f"because 1 unit is {changeperc * 100:.3f}%") + return False + return True + + def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: + """ + Method doing the filtering + """ + + # Copy list since we're modifying this list + for p in deepcopy(pairlist): + ticker = [t for t in tickers if t['symbol'] == p][0] + + # Filter out assets which would not allow setting a stoploss + if self._low_price_percent and not self._validate_precision_filter_lowprice(ticker): + pairlist.remove(p) + + return pairlist diff --git a/freqtrade/pairlist/PrecisionFilter.py b/freqtrade/pairlist/PrecisionFilter.py new file mode 100644 index 000000000..c720b8e61 --- /dev/null +++ b/freqtrade/pairlist/PrecisionFilter.py @@ -0,0 +1,51 @@ +import logging +from copy import deepcopy +from typing import Dict, List + +from freqtrade.pairlist.IPairListFilter import IPairListFilter + +logger = logging.getLogger(__name__) + + +class PrecisionFilter(IPairListFilter): + + def __init__(self, freqtrade, config: dict) -> None: + super().__init__(freqtrade, config) + + def _validate_precision_filter(self, ticker: dict, stoploss: float) -> bool: + """ + Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very + low value pairs. + :param ticker: ticker dict as returned from ccxt.load_markets() + :param stoploss: stoploss value as set in the configuration + (already cleaned to be 1 - stoploss) + :return: True if the pair can stay, false if it should be removed + """ + stop_price = self._freqtrade.get_target_bid(ticker["symbol"], ticker) * stoploss + # Adjust stop-prices to precision + sp = self._freqtrade.exchange.symbol_price_prec(ticker["symbol"], stop_price) + stop_gap_price = self._freqtrade.exchange.symbol_price_prec(ticker["symbol"], + stop_price * 0.99) + logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}") + if sp <= stop_gap_price: + logger.info(f"Removed {ticker['symbol']} from whitelist, " + f"because stop price {sp} would be <= stop limit {stop_gap_price}") + return False + return True + + def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: + """ + Method doing the filtering + """ + if self._freqtrade.strategy.stoploss is not None: + # Precalculate sanitized stoploss value to avoid recalculation for every pair + stoploss = 1 - abs(self._freqtrade.strategy.stoploss) + # Copy list since we're modifying this list + for p in deepcopy(pairlist): + ticker = [t for t in tickers if t['symbol'] == p][0] + # Filter out assets which would not allow setting a stoploss + if (stoploss and not self._validate_precision_filter(ticker, stoploss)): + pairlist.remove(p) + continue + + return pairlist diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 5896e814a..074652b25 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -27,4 +27,4 @@ class StaticPairList(IPairList): """ Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively """ - self._whitelist = self._validate_whitelist(self._config['exchange']['pair_whitelist']) + self._whitelist = self.validate_whitelist(self._config['exchange']['pair_whitelist']) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 7c17c6d57..911bb3bda 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -5,7 +5,6 @@ Provides lists as configured in config.json """ import logging -from copy import deepcopy from typing import List from cachetools import TTLCache, cached @@ -30,7 +29,6 @@ class VolumePairList(IPairList): self._number_pairs = self._whitelistconf['number_assets'] self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume') self._precision_filter = self._whitelistconf.get('precision_filter', True) - self._low_price_percent_filter = self._whitelistconf.get('low_price_percent_filter', None) if not self._freqtrade.exchange.exchange_has('fetchTickers'): raise OperationalException( @@ -60,44 +58,6 @@ class VolumePairList(IPairList): self._whitelist = self._gen_pair_whitelist( self._config['stake_currency'], self._sort_key) - def _validate_precision_filter(self, ticker: dict, stoploss: float) -> bool: - """ - Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very - low value pairs. - :param ticker: ticker dict as returned from ccxt.load_markets() - :param stoploss: stoploss value as set in the configuration - (already cleaned to be 1 - stoploss) - :return: True if the pair can stay, false if it should be removed - """ - stop_price = self._freqtrade.get_target_bid(ticker["symbol"], ticker) * stoploss - # Adjust stop-prices to precision - sp = self._freqtrade.exchange.symbol_price_prec(ticker["symbol"], stop_price) - stop_gap_price = self._freqtrade.exchange.symbol_price_prec(ticker["symbol"], - stop_price * 0.99) - logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}") - if sp <= stop_gap_price: - logger.info(f"Removed {ticker['symbol']} from whitelist, " - f"because stop price {sp} would be <= stop limit {stop_gap_price}") - return False - return True - - def _validate_precision_filter_lowprice(self, ticker) -> bool: - """ - Check if if one price-step is > than a certain barrier. - :param ticker: ticker dict as returned from ccxt.load_markets() - :param precision: Precision - :return: True if the pair can stay, false if it should be removed - """ - precision = self._freqtrade.exchange.markets[ticker['symbol']]['precision']['price'] - - compare = ticker['last'] + 1 / pow(10, precision) - changeperc = (compare - ticker['last']) / ticker['last'] - if changeperc > self._low_price_percent_filter: - logger.info(f"Removed {ticker['symbol']} from whitelist, " - f"because 1 unit is {changeperc * 100:.3f}%") - return False - return True - @cached(TTLCache(maxsize=1, ttl=1800)) def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]: """ @@ -114,26 +74,8 @@ class VolumePairList(IPairList): and v[key] is not None)] sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) # Validate whitelist to only have active market pairs - valid_pairs = self._validate_whitelist([s['symbol'] for s in sorted_tickers]) - valid_tickers = [t for t in sorted_tickers if t["symbol"] in valid_pairs] + pairs = self.validate_whitelist([s['symbol'] for s in sorted_tickers], tickers) - stoploss = None - if self._freqtrade.strategy.stoploss is not None: - # Precalculate sanitized stoploss value to avoid recalculation for every pair - stoploss = 1 - abs(self._freqtrade.strategy.stoploss) - - # Copy list since we're modifying this list - for t in deepcopy(valid_tickers): - # Filter out assets which would not allow setting a stoploss - if (stoploss and self._precision_filter - and not self._validate_precision_filter(t, stoploss)): - valid_tickers.remove(t) - continue - if self._low_price_percent_filter and not self._validate_precision_filter_lowprice(t,): - valid_tickers.remove(t) - continue - - pairs = [s['symbol'] for s in valid_tickers] - logger.info(f"Searching pairs: {pairs[:self._number_pairs]}") + logger.info(f"Searching {self._number_pairs} pairs: {pairs[:self._number_pairs]}") return pairs diff --git a/freqtrade/resolvers/pairlistfilter_resolver.py b/freqtrade/resolvers/pairlistfilter_resolver.py new file mode 100644 index 000000000..bf86d1c6c --- /dev/null +++ b/freqtrade/resolvers/pairlistfilter_resolver.py @@ -0,0 +1,53 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load custom pairlists +""" +import logging +from pathlib import Path + +from freqtrade import OperationalException +from freqtrade.pairlist.IPairListFilter import IPairListFilter +from freqtrade.resolvers import IResolver + +logger = logging.getLogger(__name__) + + +class PairListFilterResolver(IResolver): + """ + This class contains all the logic to load custom PairList class + """ + + __slots__ = ['pairlistfilter'] + + def __init__(self, pairlist_name: str, freqtrade, config: dict) -> None: + """ + Load the custom class from config parameter + :param config: configuration dictionary or None + """ + self.pairlistfilter = self._load_pairlist(pairlist_name, config, + kwargs={'freqtrade': freqtrade, + 'config': config}) + + def _load_pairlist( + self, pairlistfilter_name: str, config: dict, kwargs: dict) -> IPairListFilter: + """ + Search and loads the specified pairlist. + :param pairlistfilter_name: name of the module to import + :param config: configuration dictionary + :param extra_dir: additional directory to search for the given pairlist + :return: PairList instance or None + """ + current_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() + + abs_paths = self.build_search_paths(config, current_path=current_path, + user_subdir=None, extra_dir=None) + + pairlist = self._load_object(paths=abs_paths, object_type=IPairListFilter, + object_name=pairlistfilter_name, kwargs=kwargs) + if pairlist: + return pairlist + raise OperationalException( + f"Impossible to load PairlistFilter '{pairlistfilter_name}'. This class does not exist " + "or contains Python code errors." + ) From 640423c362195224aa62a7287e35f6bfee1a6508 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Oct 2019 16:00:16 +0100 Subject: [PATCH 08/39] Add config samples for chainable pairlist filters --- config_full.json.example | 6 +++-- tests/pairlist/test_pairlist.py | 40 +++++++++++++++++---------------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 5935de392..366b075fa 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -55,8 +55,10 @@ "config": { "number_assets": 20, "sort_key": "quoteVolume", - "precision_filter": true, - "low_price_percent_filter": 0 + }, + "filters":{ + "PrecisionFilter": {}, + "LowPriceFilter": {"low_price_percent": 0.01} } }, "exchange": { diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index ac27d1c8e..23c48545e 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -26,7 +26,8 @@ def whitelist_conf(default_conf): 'BLK/BTC' ] default_conf['pairlist'] = {'method': 'StaticPairList', - 'config': {'number_assets': 5} + 'config': {'number_assets': 5}, + 'filters': {}, } return default_conf @@ -113,23 +114,24 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): assert set(whitelist) == set(pairslist) -@pytest.mark.parametrize("precision_filter,low_price_filter,base_currency,key,whitelist_result", [ - (False, 0, "BTC", "quoteVolume", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']), - (False, 0, "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'HOT/BTC', 'FUEL/BTC']), - (False, 0, "USDT", "quoteVolume", ['ETH/USDT']), - (False, 0, "ETH", "quoteVolume", []), - (True, 0, "BTC", "quoteVolume", ["LTC/BTC", "ETH/BTC", "TKN/BTC", 'FUEL/BTC']), - (True, 0, "BTC", "bidVolume", ["LTC/BTC", "TKN/BTC", "ETH/BTC", 'FUEL/BTC']), - (False, 0.03, "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'FUEL/BTC']), +@pytest.mark.parametrize("filters,base_currency,key,whitelist_result", [ + ({}, "BTC", "quoteVolume", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']), + ({}, "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'HOT/BTC', 'FUEL/BTC']), + ({}, "USDT", "quoteVolume", ['ETH/USDT']), + ({}, "ETH", "quoteVolume", []), + ({"PrecisionFilter": {}}, "BTC", "quoteVolume", ["LTC/BTC", "ETH/BTC", "TKN/BTC", 'FUEL/BTC']), + ({"PrecisionFilter": {}}, "BTC", "bidVolume", ["LTC/BTC", "TKN/BTC", "ETH/BTC", 'FUEL/BTC']), + ({"LowPriceFilter": {"low_price_percent": 0.03}}, "BTC", + "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'FUEL/BTC']), # Hot is removed by precision_filter, Fuel by low_price_filter. - (True, 0.02, "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC']), + ({"PrecisionFilter": {}, "LowPriceFilter": {"low_price_percent": 0.02}}, + "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC']), ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, - precision_filter, low_price_filter, base_currency, key, - whitelist_result, caplog) -> None: + filters, base_currency, key, whitelist_result, + caplog) -> None: whitelist_conf['pairlist']['method'] = 'VolumePairList' - whitelist_conf['pairlist']['config']['precision_filter'] = precision_filter - whitelist_conf['pairlist']['config']['low_price_percent_filter'] = low_price_filter + whitelist_conf['pairlist']['filters'] = filters mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) @@ -139,11 +141,11 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t freqtrade.config['stake_currency'] = base_currency whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency=base_currency, key=key) assert sorted(whitelist) == sorted(whitelist_result) - if precision_filter: + if 'PrecisionFilter' in filters: assert log_has_re(r'^Removed .* from whitelist, because stop price .* ' r'would be <= stop limit.*', caplog) - if low_price_filter: + if 'LowPriceFilter' in filters: assert log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) @@ -179,15 +181,15 @@ def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): (['ETH/BTC', 'TKN/BTC', 'BLK/BTC'], "is not compatible with exchange"), # BLK/BTC in blacklist (['ETH/BTC', 'TKN/BTC', 'BTT/BTC'], "Market is not active") # BTT/BTC is inactive ]) -def test_validate_whitelist(mocker, whitelist_conf, markets, pairlist, whitelist, caplog, - log_message): +def test__whitelist_for_active_markets(mocker, whitelist_conf, markets, pairlist, whitelist, caplog, + log_message): whitelist_conf['pairlist']['method'] = pairlist mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) caplog.clear() - new_whitelist = freqtrade.pairlists._validate_whitelist(whitelist) + new_whitelist = freqtrade.pairlists._whitelist_for_active_markets(whitelist) assert set(new_whitelist) == set(['ETH/BTC', 'TKN/BTC']) assert log_message in caplog.text From d89a7d523502ed680e50387316cccca28ae5004d Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Oct 2019 16:30:47 +0100 Subject: [PATCH 09/39] Document new method to configure filters --- docs/configuration.md | 126 ++++++++++++++++++++++++++---------------- 1 file changed, 79 insertions(+), 47 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 7ccb6840a..39fb4b8ac 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -377,6 +377,82 @@ The valid values are: "BTC", "ETH", "XRP", "LTC", "BCH", "USDT" ``` +## Pairlists + +Pairlists define the list of pairs that the bot should trade. +There are [`StaticPairList`](#static-pair-list) and dynamic Whitelists available. + +In addition to pairlists, [pairlist filters](#available-pairlist-filters) can be configured, which remove certain assets. +These Filters work with all Pairlist providers and are applied in the sequence they occur. + +### Available Pairlists + +* [`StaticPairList`](#static-pair-list) (default, if not configured differently) +* [`VolumePairList`](#volume-pair-list) + +#### Static Pair List + +By default, the `StaticPairList` method is used, which uses a statically defined pair whitelist from the configuration. Inactive markets and blacklisted pairs are removed from the pair_whitelist. + +It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklist`. + +#### Volume Pair List + +`VolumePairList` selects `number_assets` top pairs based on `sort_key`, which can be one of `askVolume`, `bidVolume` and `quoteVolume` and defaults to `quoteVolume`. + +`VolumePairList` does not consider `pair_whitelist`, but selects the top assets from all available markets (with matching stake-currency) on the exchange. +Pairs in `pair_blacklist` are not considered for `VolumePairList`, even if all other filters would match. + +```json +"pairlist": { + "method": "VolumePairList", + "config": { + "number_assets": 20, + "sort_key": "quoteVolume", + }, +``` + +### Available Pairlist Filters + +* [`PrecisionFilter`](#precision-filter) +* [`LowPriceFilter`](#low-price-pair-filter) + +#### Precision Filter + +Filters low-value coins which would not allow setting a stoploss. + +#### Low Price Pair Filter + +The `LowPriceFilter` allows filtering of pairs where a raise of 1 price unit is below the `low_price_percent` ratio. +This option is disabled by default, and will only apply if set to <> 0. + +Calculation example: +Min price precision is 8 decimals. If price is 0.00000011 - one step would be 0.00000012 - which is almost 10% higher than the previous value. + +These pairs are dangerous since it may be impossible to place the desired stoploss - and often result in high losses. + +### Full Pairlist example + +The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting by `quoteVolume` and applies both [`PrecisionFilter`](#precision-filter) and [`LowPriceFilter`](#low-price-pair-filter), filtering all assets where 1 priceunit is > 1%. + +```json +"exchange": { + "pair_whitelist": [], + "pair_blacklist": ["BNB/BTC"] +}, +"pairlist": { + "method": "VolumePairList", + "config": { + "number_assets": 20, + "sort_key": "quoteVolume", + }, + "filters":{ + "PrecisionFilter": {}, + "LowPriceFilter": {"low_price_percent": 0.01} + } + }, +``` + ## Switch to Dry-run mode We recommend starting the bot in the Dry-run mode to see how your bot will @@ -406,49 +482,6 @@ creating trades on the exchange. Once you will be happy with your bot performance running in the Dry-run mode, you can switch it to production mode. -### Dynamic Pairlists - -Dynamic pairlists select pairs for you based on the logic configured. -The bot runs against all pairs (with that stake) on the exchange, and a number of assets -(`number_assets`) is selected based on the selected criteria. - -By default, the `StaticPairList` method is used. -The Pairlist method is configured as `pair_whitelist` parameter under the `exchange` -section of the configuration. - -**Available Pairlist methods:** - -* `StaticPairList` - * It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklist`. -* `VolumePairList` - * It selects `number_assets` top pairs based on `sort_key`, which can be one of -`askVolume`, `bidVolume` and `quoteVolume`, defaults to `quoteVolume`. - * By default, low-value coins that would not allow setting a stop loss are filtered out. (set `precision_filter` parameter to `false` to disable this behaviour). - * `VolumePairList` does not consider `pair_whitelist`, but builds this automatically based the pairlist configuration. - * Pairs in `pair_blacklist` are not considered for VolumePairList, even if all other filters would match. - * `low_price_percent_filter` allows filtering of pairs where a raise of 1 price unit is below the `low_price_percent_filter` ratio. - This option is disabled by default, and will only apply if set to <> 0. - Calculation example: Min price precision is 8 decimals. If price is 0.00000011 - one step would be 0.00000012 - which is almost 10% higher than the previous value. - - -Example: - -```json -"exchange": { - "pair_whitelist": [], - "pair_blacklist": ["BNB/BTC"] -}, -"pairlist": { - "method": "VolumePairList", - "config": { - "number_assets": 20, - "sort_key": "quoteVolume", - "precision_filter": true, - "low_price_percent_filter": 0.05 - } - }, -``` - ## Switch to production mode In production mode, the bot will engage your money. Be careful, since a wrong @@ -479,7 +512,7 @@ you run it in production mode. !!! Note If you have an exchange API key yet, [see our tutorial](/pre-requisite). -### Using proxy with FreqTrade +## Using proxy with FreqTrade To use a proxy with freqtrade, add the kwarg `"aiohttp_trust_env"=true` to the `"ccxt_async_kwargs"` dict in the exchange section of the configuration. @@ -499,14 +532,13 @@ export HTTPS_PROXY="http://addr:port" freqtrade ``` - -### Embedding Strategies +## Embedding Strategies FreqTrade provides you with with an easy way to embed the strategy into your configuration file. This is done by utilizing BASE64 encoding and providing this string at the strategy configuration field, in your chosen config file. -#### Encoding a string as BASE64 +### Encoding a string as BASE64 This is a quick example, how to generate the BASE64 string in python From 14758dbe107d4a460444719c78b7f9fd5477b5b1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Oct 2019 16:32:22 +0100 Subject: [PATCH 10/39] Some small cleanups --- freqtrade/configuration/deprecated_settings.py | 7 +++++++ freqtrade/pairlist/LowPriceFilter.py | 4 ++-- tests/test_configuration.py | 10 ++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/freqtrade/configuration/deprecated_settings.py b/freqtrade/configuration/deprecated_settings.py index f00b23894..8471028aa 100644 --- a/freqtrade/configuration/deprecated_settings.py +++ b/freqtrade/configuration/deprecated_settings.py @@ -57,3 +57,10 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None: 'experimental', 'sell_profit_only') process_deprecated_setting(config, 'ask_strategy', 'ignore_roi_if_buy_signal', 'experimental', 'ignore_roi_if_buy_signal') + + if config.get('pairlist', {}).get('config', {}).get('precision_filter'): + logger.warning( + "DEPRECATED: " + f"Using precision_filter setting is deprecated and has been replaced by" + "PrecisionFilter. Please refer to the docs on configuration details") + config['pairlist'].update({'filters': {'PrecisionFilter': {}}}) diff --git a/freqtrade/pairlist/LowPriceFilter.py b/freqtrade/pairlist/LowPriceFilter.py index 499dd0c15..778c9b4e0 100644 --- a/freqtrade/pairlist/LowPriceFilter.py +++ b/freqtrade/pairlist/LowPriceFilter.py @@ -15,7 +15,7 @@ class LowPriceFilter(IPairListFilter): self._low_price_percent = config['pairlist']['filters']['LowPriceFilter'].get( 'low_price_percent', 0) - def _validate_precision_filter_lowprice(self, ticker) -> bool: + def _validate_ticker_lowprice(self, ticker) -> bool: """ Check if if one price-step is > than a certain barrier. :param ticker: ticker dict as returned from ccxt.load_markets() @@ -42,7 +42,7 @@ class LowPriceFilter(IPairListFilter): ticker = [t for t in tickers if t['symbol'] == p][0] # Filter out assets which would not allow setting a stoploss - if self._low_price_percent and not self._validate_precision_filter_lowprice(ticker): + if self._low_price_percent and not self._validate_ticker_lowprice(ticker): pairlist.remove(p) return pairlist diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 545dd5df4..258088925 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -963,6 +963,16 @@ def test_process_temporary_deprecated_settings(mocker, default_conf, setting, ca assert default_conf[setting[0]][setting[1]] == setting[5] +def test_process_deprecated_setting_precision_filter(mocker, default_conf, caplog): + patched_configuration_load_config_file(mocker, default_conf) + default_conf.update({'pairlist': { + 'config': {'precision_filter': True} + }}) + + process_temporary_deprecated_settings(default_conf) + assert log_has_re(r'DEPRECATED.*precision_filter.*', caplog) + + def test_check_conflicting_settings(mocker, default_conf, caplog): patched_configuration_load_config_file(mocker, default_conf) From ad98d61939f9808b0adffb0b06099fdef241eee2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Oct 2019 16:39:45 +0100 Subject: [PATCH 11/39] Update developer docs --- docs/developer.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/developer.md b/docs/developer.md index 391493b09..346578c2e 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -127,7 +127,7 @@ This is called with each iteration of the bot - so consider implementing caching Assign the resulting whiteslist to `self._whitelist` and `self._blacklist` respectively. These will then be used to run the bot in this iteration. Pairs with open trades will be added to the whitelist to have the sell-methods run correctly. -Please also run `self._validate_whitelist(pairs)` and to check and remove pairs with inactive markets. This function is available in the Parent class (`StaticPairList`) and should ideally not be overwritten. +Please also run `self.validate_whitelist(pairs, tickers)` (tickers is optional, but should be passed when you're using tickers anyway) and to check and remove pairs with inactive markets. This function is available in the Parent class (`StaticPairList`) and should ideally not be overwritten. ##### sample @@ -136,7 +136,7 @@ Please also run `self._validate_whitelist(pairs)` and to check and remove pairs # Generate dynamic whitelist pairs = self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key) # Validate whitelist to only have active market pairs - self._whitelist = self._validate_whitelist(pairs)[:self._number_pairs] + self._whitelist = self.validate_whitelist(pairs)[:self._number_pairs] ``` #### _gen_pair_whitelist From e632720c02bf982f2a4627d2ce8ce002b460ab97 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 06:55:16 +0100 Subject: [PATCH 12/39] Allow chaining of pairlists --- freqtrade/freqtradebot.py | 7 +- freqtrade/pairlist/IPairList.py | 65 ++++-------------- freqtrade/pairlist/IPairListFilter.py | 18 ----- freqtrade/pairlist/LowPriceFilter.py | 29 +++++--- freqtrade/pairlist/PrecisionFilter.py | 13 ++-- freqtrade/pairlist/StaticPairList.py | 15 ++-- freqtrade/pairlist/VolumePairList.py | 32 +++++---- freqtrade/pairlist/pairlistmanager.py | 68 +++++++++++++++++++ freqtrade/resolvers/pairlist_resolver.py | 8 ++- .../resolvers/pairlistfilter_resolver.py | 53 --------------- 10 files changed, 143 insertions(+), 165 deletions(-) delete mode 100644 freqtrade/pairlist/IPairListFilter.py create mode 100644 freqtrade/pairlist/pairlistmanager.py delete mode 100644 freqtrade/resolvers/pairlistfilter_resolver.py diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a8fc6bc7e..9871ffb89 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -20,9 +20,9 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date from freqtrade.persistence import Trade -from freqtrade.resolvers import (ExchangeResolver, PairListResolver, - StrategyResolver) +from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.rpc import RPCManager, RPCMessageType +from freqtrade.pairlist.pairlistmanager import PairListManager from freqtrade.state import State from freqtrade.strategy.interface import IStrategy, SellType from freqtrade.wallets import Wallets @@ -70,8 +70,7 @@ class FreqtradeBot: # Attach Wallets to Strategy baseclass IStrategy.wallets = self.wallets - pairlistname = self.config.get('pairlist', {}).get('method', 'StaticPairList') - self.pairlists = PairListResolver(pairlistname, self, self.config).pairlist + self.pairlists = PairListManager(self.exchange, self.config) # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index eb6af9d52..845c5d01f 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -9,25 +9,16 @@ from abc import ABC, abstractmethod from typing import Dict, List from freqtrade.exchange import market_is_active -from freqtrade.pairlist.IPairListFilter import IPairListFilter -from freqtrade.resolvers.pairlistfilter_resolver import PairListFilterResolver logger = logging.getLogger(__name__) class IPairList(ABC): - def __init__(self, freqtrade, config: dict) -> None: - self._freqtrade = freqtrade + def __init__(self, exchange, config, pairlistconfig: dict) -> None: + self._exchange = exchange self._config = config - self._whitelist = self._config['exchange']['pair_whitelist'] - self._blacklist = self._config['exchange'].get('pair_blacklist', []) - self._filters = self._config.get('pairlist', {}).get('filters', {}) - self._pairlistfilters: List[IPairListFilter] = [] - for pl_filter in self._filters.keys(): - self._pairlistfilters.append( - PairListFilterResolver(pl_filter, freqtrade, self._config).pairlistfilter - ) + self._pairlistconfig = pairlistconfig @property def name(self) -> str: @@ -37,22 +28,6 @@ class IPairList(ABC): """ return self.__class__.__name__ - @property - def whitelist(self) -> List[str]: - """ - Has the current whitelist - -> no need to overwrite in subclasses - """ - return self._whitelist - - @property - def blacklist(self) -> List[str]: - """ - Has the current blacklist - -> no need to overwrite in subclasses - """ - return self._blacklist - @abstractmethod def short_desc(self) -> str: """ @@ -61,28 +36,16 @@ class IPairList(ABC): """ @abstractmethod - def refresh_pairlist(self) -> None: + def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: """ - Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary -> Please overwrite in subclasses + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist """ - def validate_whitelist(self, pairlist: List[str], - tickers: List[Dict] = []) -> List[str]: - """ - Validate pairlist against active markets and blacklist. - Run PairlistFilters if these are configured. - """ - pairlist = self._whitelist_for_active_markets(pairlist) - - if not tickers: - # Refresh tickers if they are not used by the parent Pairlist - tickers = self._freqtrade.exchange.get_tickers() - - for pl_filter in self._pairlistfilters: - pairlist = pl_filter.filter_pairlist(pairlist, tickers) - return pairlist - def _whitelist_for_active_markets(self, whitelist: List[str]) -> List[str]: """ Check available markets and remove pair from whitelist if necessary @@ -90,16 +53,14 @@ class IPairList(ABC): :return: the list of pairs the user wants to trade without those unavailable or black_listed """ - markets = self._freqtrade.exchange.markets + markets = self._exchange.markets sanitized_whitelist: List[str] = [] for pair in whitelist: - # pair is not in the generated dynamic market, or in the blacklist ... ignore it - if (pair in self.blacklist or pair not in markets - or not pair.endswith(self._config['stake_currency'])): + # pair is not in the generated dynamic market or has the wrong stake currency + if (pair not in markets or not pair.endswith(self._config['stake_currency'])): logger.warning(f"Pair {pair} is not compatible with exchange " - f"{self._freqtrade.exchange.name} or contained in " - f"your blacklist. Removing it from whitelist..") + f"{self._exchange.name}. Removing it from whitelist..") continue # Check if market is active market = markets[pair] diff --git a/freqtrade/pairlist/IPairListFilter.py b/freqtrade/pairlist/IPairListFilter.py deleted file mode 100644 index 4b43f0e9f..000000000 --- a/freqtrade/pairlist/IPairListFilter.py +++ /dev/null @@ -1,18 +0,0 @@ -import logging -from abc import ABC, abstractmethod -from typing import Dict, List - -logger = logging.getLogger(__name__) - - -class IPairListFilter(ABC): - - def __init__(self, freqtrade, config: dict) -> None: - self._freqtrade = freqtrade - self._config = config - - @abstractmethod - def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: - """ - Method doing the filtering - """ diff --git a/freqtrade/pairlist/LowPriceFilter.py b/freqtrade/pairlist/LowPriceFilter.py index 778c9b4e0..2f4a3be75 100644 --- a/freqtrade/pairlist/LowPriceFilter.py +++ b/freqtrade/pairlist/LowPriceFilter.py @@ -2,18 +2,23 @@ import logging from copy import deepcopy from typing import Dict, List -from freqtrade.pairlist.IPairListFilter import IPairListFilter +from freqtrade.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) -class LowPriceFilter(IPairListFilter): +class LowPriceFilter(IPairList): - def __init__(self, freqtrade, config: dict) -> None: - super().__init__(freqtrade, config) + def __init__(self, exchange, config, pairlistconfig: dict) -> None: + super().__init__(exchange, config, pairlistconfig) - self._low_price_percent = config['pairlist']['filters']['LowPriceFilter'].get( - 'low_price_percent', 0) + self._low_price_percent = pairlistconfig.get('low_price_percent', 0) + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return f"{self.name} - Filtering pairs priced below {self._low_price_percent * 100}%." def _validate_ticker_lowprice(self, ticker) -> bool: """ @@ -22,7 +27,7 @@ class LowPriceFilter(IPairListFilter): :param precision: Precision :return: True if the pair can stay, false if it should be removed """ - precision = self._freqtrade.exchange.markets[ticker['symbol']]['precision']['price'] + precision = self._exchange.markets[ticker['symbol']]['precision']['price'] compare = ticker['last'] + 1 / pow(10, precision) changeperc = (compare - ticker['last']) / ticker['last'] @@ -33,10 +38,14 @@ class LowPriceFilter(IPairListFilter): return True def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: - """ - Method doing the filtering - """ + """ + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist + """ # Copy list since we're modifying this list for p in deepcopy(pairlist): ticker = [t for t in tickers if t['symbol'] == p][0] diff --git a/freqtrade/pairlist/PrecisionFilter.py b/freqtrade/pairlist/PrecisionFilter.py index c720b8e61..0a590bec6 100644 --- a/freqtrade/pairlist/PrecisionFilter.py +++ b/freqtrade/pairlist/PrecisionFilter.py @@ -2,15 +2,18 @@ import logging from copy import deepcopy from typing import Dict, List -from freqtrade.pairlist.IPairListFilter import IPairListFilter +from freqtrade.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) -class PrecisionFilter(IPairListFilter): +class PrecisionFilter(IPairList): - def __init__(self, freqtrade, config: dict) -> None: - super().__init__(freqtrade, config) + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return f"{self.name} - Filtering untradable pairs." def _validate_precision_filter(self, ticker: dict, stoploss: float) -> bool: """ @@ -35,7 +38,7 @@ class PrecisionFilter(IPairListFilter): def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: """ - Method doing the filtering + Filters and sorts pairlists and assigns and returns them again. """ if self._freqtrade.strategy.stoploss is not None: # Precalculate sanitized stoploss value to avoid recalculation for every pair diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 074652b25..c10dde5a6 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -5,6 +5,7 @@ Provides lists as configured in config.json """ import logging +from typing import List, Dict from freqtrade.pairlist.IPairList import IPairList @@ -13,8 +14,8 @@ logger = logging.getLogger(__name__) class StaticPairList(IPairList): - def __init__(self, freqtrade, config: dict) -> None: - super().__init__(freqtrade, config) + def __init__(self, exchange, config, pairlistconfig: dict) -> None: + super().__init__(exchange, config, pairlistconfig) def short_desc(self) -> str: """ @@ -23,8 +24,12 @@ class StaticPairList(IPairList): """ return f"{self.name}: {self.whitelist}" - def refresh_pairlist(self) -> None: + def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: """ - Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist """ - self._whitelist = self.validate_whitelist(self._config['exchange']['pair_whitelist']) + return self.validate_whitelist(self._config['exchange']['pair_whitelist']) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 911bb3bda..ff601bc44 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -5,7 +5,7 @@ Provides lists as configured in config.json """ import logging -from typing import List +from typing import List, Dict from cachetools import TTLCache, cached @@ -19,18 +19,17 @@ SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] class VolumePairList(IPairList): - def __init__(self, freqtrade, config: dict) -> None: - super().__init__(freqtrade, config) - self._whitelistconf = self._config.get('pairlist', {}).get('config') - if 'number_assets' not in self._whitelistconf: + def __init__(self, exchange, config, pairlistconfig: dict) -> None: + super().__init__(exchange, config, pairlistconfig) + + if 'number_assets' not in self._pairlistconfig: raise OperationalException( f'`number_assets` not specified. Please check your configuration ' 'for "pairlist.config.number_assets"') - self._number_pairs = self._whitelistconf['number_assets'] - self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume') - self._precision_filter = self._whitelistconf.get('precision_filter', True) + self._number_pairs = self._pairlistconfig['number_assets'] + self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume') - if not self._freqtrade.exchange.exchange_has('fetchTickers'): + if not self._exchange.exchange_has('fetchTickers'): raise OperationalException( 'Exchange does not support dynamic whitelist.' 'Please edit your config and restart the bot' @@ -45,14 +44,16 @@ class VolumePairList(IPairList): def short_desc(self) -> str: """ Short whitelist method description - used for startup-messages - -> Please overwrite in subclasses """ return f"{self.name} - top {self._whitelistconf['number_assets']} volume pairs." - def refresh_pairlist(self) -> None: + def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: """ - Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively - -> Please overwrite in subclasses + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist """ # Generate dynamic whitelist self._whitelist = self._gen_pair_whitelist( @@ -64,17 +65,18 @@ class VolumePairList(IPairList): Updates the whitelist with with a dynamically generated list :param base_currency: base currency as str :param key: sort key (defaults to 'quoteVolume') + :param tickers: Tickers (from exchange.get_tickers()). :return: List of pairs """ - tickers = self._freqtrade.exchange.get_tickers() + tickers = self._exchange.get_tickers() # check length so that we make sure that '/' is actually in the string tickers = [v for k, v in tickers.items() if (len(k.split('/')) == 2 and k.split('/')[1] == base_currency and v[key] is not None)] sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) # Validate whitelist to only have active market pairs - pairs = self.validate_whitelist([s['symbol'] for s in sorted_tickers], tickers) + pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers]) logger.info(f"Searching {self._number_pairs} pairs: {pairs[:self._number_pairs]}") diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py new file mode 100644 index 000000000..9e92fdb6a --- /dev/null +++ b/freqtrade/pairlist/pairlistmanager.py @@ -0,0 +1,68 @@ +""" +Static List provider + +Provides lists as configured in config.json + + """ +import logging +from copy import deepcopy +from typing import List + +from freqtrade.pairlist.IPairList import IPairList +from freqtrade.resolvers import PairListResolver + +logger = logging.getLogger(__name__) + + +class PairListManager(): + + def __init__(self, exchange, config: dict) -> None: + self._exchange = exchange + self._config = config + self._whitelist = self._config['exchange'].get('pair_whitelist') + self._blacklist = self._config['exchange'].get('pair_blacklist', []) + self._pairlists: List[IPairList] = [] + + for pl in self._config.get('pairlists', [{'method': "StaticPairList"}]): + pairl = PairListResolver(pl.get('method'), + exchange, config, + pl.get('config')).pairlist + self._pairlists.append(pairl) + + @property + def whitelist(self) -> List[str]: + """ + Has the current whitelist + """ + return self._whitelist + + @property + def blacklist(self) -> List[str]: + """ + Has the current blacklist + -> no need to overwrite in subclasses + """ + return self._blacklist + + def refresh_pairlist(self) -> None: + """ + Run pairlist through all pairlists. + """ + + pairlist = self._whitelist.copy() + + # tickers should be cached to avoid calling the exchange on each call. + tickers = self._exchange.get_tickers() + for pl in self._pairlists: + pl.filter_pairlist(pairlist, tickers) + + pairlist = self._verify_blacklist(pairlist) + self._whitelist = pairlist + + def _verify_blacklist(self, pairlist: List[str]) -> List[str]: + + for pair in deepcopy(pairlist): + if pair in self.blacklist: + logger.warning(f"Pair {pair} in your blacklist. Removing it from whitelist...") + pairlist.remove(pair) + return pairlist diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index 0f23bb3fd..db00f6515 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -20,13 +20,15 @@ class PairListResolver(IResolver): __slots__ = ['pairlist'] - def __init__(self, pairlist_name: str, freqtrade, config: dict) -> None: + def __init__(self, pairlist_name: str, exchange, config: dict, pairlistconfig) -> None: """ Load the custom class from config parameter :param config: configuration dictionary or None """ - self.pairlist = self._load_pairlist(pairlist_name, config, kwargs={'freqtrade': freqtrade, - 'config': config}) + self.pairlist = self._load_pairlist(pairlist_name, config, + kwargs={'exchange': exchange, + 'config': config, + 'pairlistconfig': pairlistconfig}) def _load_pairlist( self, pairlist_name: str, config: dict, kwargs: dict) -> IPairList: diff --git a/freqtrade/resolvers/pairlistfilter_resolver.py b/freqtrade/resolvers/pairlistfilter_resolver.py deleted file mode 100644 index bf86d1c6c..000000000 --- a/freqtrade/resolvers/pairlistfilter_resolver.py +++ /dev/null @@ -1,53 +0,0 @@ -# pragma pylint: disable=attribute-defined-outside-init - -""" -This module load custom pairlists -""" -import logging -from pathlib import Path - -from freqtrade import OperationalException -from freqtrade.pairlist.IPairListFilter import IPairListFilter -from freqtrade.resolvers import IResolver - -logger = logging.getLogger(__name__) - - -class PairListFilterResolver(IResolver): - """ - This class contains all the logic to load custom PairList class - """ - - __slots__ = ['pairlistfilter'] - - def __init__(self, pairlist_name: str, freqtrade, config: dict) -> None: - """ - Load the custom class from config parameter - :param config: configuration dictionary or None - """ - self.pairlistfilter = self._load_pairlist(pairlist_name, config, - kwargs={'freqtrade': freqtrade, - 'config': config}) - - def _load_pairlist( - self, pairlistfilter_name: str, config: dict, kwargs: dict) -> IPairListFilter: - """ - Search and loads the specified pairlist. - :param pairlistfilter_name: name of the module to import - :param config: configuration dictionary - :param extra_dir: additional directory to search for the given pairlist - :return: PairList instance or None - """ - current_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() - - abs_paths = self.build_search_paths(config, current_path=current_path, - user_subdir=None, extra_dir=None) - - pairlist = self._load_object(paths=abs_paths, object_type=IPairListFilter, - object_name=pairlistfilter_name, kwargs=kwargs) - if pairlist: - return pairlist - raise OperationalException( - f"Impossible to load PairlistFilter '{pairlistfilter_name}'. This class does not exist " - "or contains Python code errors." - ) From b610e8c7e6b9a2d5f6c007fd446541f773fc688b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 07:05:17 +0100 Subject: [PATCH 13/39] Don't refresh tickers if they are not needed --- freqtrade/pairlist/IPairList.py | 10 +++++++++- freqtrade/pairlist/LowPriceFilter.py | 9 +++++++++ freqtrade/pairlist/PrecisionFilter.py | 9 +++++++++ freqtrade/pairlist/StaticPairList.py | 11 ++++++++++- freqtrade/pairlist/VolumePairList.py | 9 +++++++++ freqtrade/pairlist/pairlistmanager.py | 16 +++++++++++----- 6 files changed, 57 insertions(+), 7 deletions(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 845c5d01f..366a49bae 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -5,7 +5,7 @@ Provides lists as configured in config.json """ import logging -from abc import ABC, abstractmethod +from abc import ABC, abstractmethod, abstractproperty from typing import Dict, List from freqtrade.exchange import market_is_active @@ -28,6 +28,14 @@ class IPairList(ABC): """ return self.__class__.__name__ + @abstractproperty + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requries tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + @abstractmethod def short_desc(self) -> str: """ diff --git a/freqtrade/pairlist/LowPriceFilter.py b/freqtrade/pairlist/LowPriceFilter.py index 2f4a3be75..4d4a92676 100644 --- a/freqtrade/pairlist/LowPriceFilter.py +++ b/freqtrade/pairlist/LowPriceFilter.py @@ -14,6 +14,15 @@ class LowPriceFilter(IPairList): self._low_price_percent = pairlistconfig.get('low_price_percent', 0) + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requries tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return True + def short_desc(self) -> str: """ Short whitelist method description - used for startup-messages diff --git a/freqtrade/pairlist/PrecisionFilter.py b/freqtrade/pairlist/PrecisionFilter.py index 0a590bec6..3e68ec607 100644 --- a/freqtrade/pairlist/PrecisionFilter.py +++ b/freqtrade/pairlist/PrecisionFilter.py @@ -9,6 +9,15 @@ logger = logging.getLogger(__name__) class PrecisionFilter(IPairList): + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requries tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return True + def short_desc(self) -> str: """ Short whitelist method description - used for startup-messages diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index c10dde5a6..54cd21d13 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -17,6 +17,15 @@ class StaticPairList(IPairList): def __init__(self, exchange, config, pairlistconfig: dict) -> None: super().__init__(exchange, config, pairlistconfig) + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requries tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return False + def short_desc(self) -> str: """ Short whitelist method description - used for startup-messages @@ -32,4 +41,4 @@ class StaticPairList(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new whitelist """ - return self.validate_whitelist(self._config['exchange']['pair_whitelist']) + return self._config['exchange']['pair_whitelist'] diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index ff601bc44..27b99ade8 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -38,6 +38,15 @@ class VolumePairList(IPairList): raise OperationalException( f'key {self._sort_key} not in {SORT_VALUES}') + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requries tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return True + def _validate_keys(self, key): return key in SORT_VALUES diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index 9e92fdb6a..d3532ee54 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -22,11 +22,12 @@ class PairListManager(): self._whitelist = self._config['exchange'].get('pair_whitelist') self._blacklist = self._config['exchange'].get('pair_blacklist', []) self._pairlists: List[IPairList] = [] - + self._tickers_needed = False for pl in self._config.get('pairlists', [{'method': "StaticPairList"}]): pairl = PairListResolver(pl.get('method'), exchange, config, pl.get('config')).pairlist + self._tickers_needed = pairl.needstickers or self._tickers_needed self._pairlists.append(pairl) @property @@ -52,17 +53,22 @@ class PairListManager(): pairlist = self._whitelist.copy() # tickers should be cached to avoid calling the exchange on each call. - tickers = self._exchange.get_tickers() + tickers = [] + if self._tickers_needed: + tickers = self._exchange.get_tickers() + for pl in self._pairlists: pl.filter_pairlist(pairlist, tickers) - pairlist = self._verify_blacklist(pairlist) + # Validation against blacklist happens after the pairlists to ensure blacklist is respected. + pairlist = self.verify_blacklist(pairlist, self.blacklist) self._whitelist = pairlist - def _verify_blacklist(self, pairlist: List[str]) -> List[str]: + @staticmethod + def verify_blacklist(pairlist: List[str], blacklist: List[str]) -> List[str]: for pair in deepcopy(pairlist): - if pair in self.blacklist: + if pair in blacklist: logger.warning(f"Pair {pair} in your blacklist. Removing it from whitelist...") pairlist.remove(pair) return pairlist From 10595862268288e7cafca1c6ed85996f906b2588 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 07:07:33 +0100 Subject: [PATCH 14/39] Small adjustments --- freqtrade/pairlist/StaticPairList.py | 4 ++-- freqtrade/pairlist/VolumePairList.py | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 54cd21d13..e70bd671c 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -5,7 +5,7 @@ Provides lists as configured in config.json """ import logging -from typing import List, Dict +from typing import Dict, List from freqtrade.pairlist.IPairList import IPairList @@ -31,7 +31,7 @@ class StaticPairList(IPairList): Short whitelist method description - used for startup-messages -> Please overwrite in subclasses """ - return f"{self.name}: {self.whitelist}" + return f"{self.name}" def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: """ diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 27b99ade8..ba32c8681 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -5,7 +5,7 @@ Provides lists as configured in config.json """ import logging -from typing import List, Dict +from typing import Dict, List from cachetools import TTLCache, cached @@ -54,7 +54,7 @@ class VolumePairList(IPairList): """ Short whitelist method description - used for startup-messages """ - return f"{self.name} - top {self._whitelistconf['number_assets']} volume pairs." + return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs." def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: """ @@ -65,8 +65,7 @@ class VolumePairList(IPairList): :return: new whitelist """ # Generate dynamic whitelist - self._whitelist = self._gen_pair_whitelist( - self._config['stake_currency'], self._sort_key) + return self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key) @cached(TTLCache(maxsize=1, ttl=1800)) def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]: From eaf3fd80c591889d30f882ed110cd2a032b13bef Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 07:19:46 +0100 Subject: [PATCH 15/39] Allow blacklist-verification from all pairlists --- freqtrade/pairlist/IPairList.py | 13 ++++++++++++- freqtrade/pairlist/VolumePairList.py | 6 ++++-- freqtrade/pairlist/pairlistmanager.py | 15 +++------------ freqtrade/resolvers/pairlist_resolver.py | 3 ++- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 366a49bae..b666b12f2 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -6,6 +6,7 @@ Provides lists as configured in config.json """ import logging from abc import ABC, abstractmethod, abstractproperty +from copy import deepcopy from typing import Dict, List from freqtrade.exchange import market_is_active @@ -15,8 +16,9 @@ logger = logging.getLogger(__name__) class IPairList(ABC): - def __init__(self, exchange, config, pairlistconfig: dict) -> None: + def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict) -> None: self._exchange = exchange + self._pairlistmanager = pairlistmanager self._config = config self._pairlistconfig = pairlistconfig @@ -54,6 +56,15 @@ class IPairList(ABC): :return: new whitelist """ + @staticmethod + def _verify_blacklist(self, pairlist: List[str]) -> List[str]: + + for pair in deepcopy(pairlist): + if pair in self._pairlistmanager.blacklist: + logger.warning(f"Pair {pair} in your blacklist. Removing it from whitelist...") + pairlist.remove(pair) + return pairlist + def _whitelist_for_active_markets(self, whitelist: List[str]) -> List[str]: """ Check available markets and remove pair from whitelist if necessary diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index ba32c8681..77bdf472d 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -85,7 +85,9 @@ class VolumePairList(IPairList): sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) # Validate whitelist to only have active market pairs pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers]) - - logger.info(f"Searching {self._number_pairs} pairs: {pairs[:self._number_pairs]}") + pairs = self._verify_blacklist(pairs) + # Limit to X number of pairs + pairs = pairs[:self._number_pairs] + logger.info(f"Searching {self._number_pairs} pairs: {pairs}") return pairs diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index d3532ee54..b1681afef 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -5,7 +5,6 @@ Provides lists as configured in config.json """ import logging -from copy import deepcopy from typing import List from freqtrade.pairlist.IPairList import IPairList @@ -25,7 +24,7 @@ class PairListManager(): self._tickers_needed = False for pl in self._config.get('pairlists', [{'method': "StaticPairList"}]): pairl = PairListResolver(pl.get('method'), - exchange, config, + exchange, self, config, pl.get('config')).pairlist self._tickers_needed = pairl.needstickers or self._tickers_needed self._pairlists.append(pairl) @@ -57,18 +56,10 @@ class PairListManager(): if self._tickers_needed: tickers = self._exchange.get_tickers() + # Process all pairlists in chain for pl in self._pairlists: - pl.filter_pairlist(pairlist, tickers) + pairlist = pl.filter_pairlist(pairlist, tickers) # Validation against blacklist happens after the pairlists to ensure blacklist is respected. pairlist = self.verify_blacklist(pairlist, self.blacklist) self._whitelist = pairlist - - @staticmethod - def verify_blacklist(pairlist: List[str], blacklist: List[str]) -> List[str]: - - for pair in deepcopy(pairlist): - if pair in blacklist: - logger.warning(f"Pair {pair} in your blacklist. Removing it from whitelist...") - pairlist.remove(pair) - return pairlist diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index db00f6515..a37d7dc05 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -20,13 +20,14 @@ class PairListResolver(IResolver): __slots__ = ['pairlist'] - def __init__(self, pairlist_name: str, exchange, config: dict, pairlistconfig) -> None: + def __init__(self, pairlist_name: str, exchange, pairlistmanager, config: dict, pairlistconfig) -> None: """ Load the custom class from config parameter :param config: configuration dictionary or None """ self.pairlist = self._load_pairlist(pairlist_name, config, kwargs={'exchange': exchange, + 'pairlistmanager': pairlistmanager, 'config': config, 'pairlistconfig': pairlistconfig}) From 31c7189b8b3c2cac88670e7a2204e9771bb24117 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 07:23:34 +0100 Subject: [PATCH 16/39] Verify blacklist correctly --- freqtrade/pairlist/IPairList.py | 14 +++++++++++--- freqtrade/pairlist/pairlistmanager.py | 3 ++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index b666b12f2..e14ae4187 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -57,14 +57,22 @@ class IPairList(ABC): """ @staticmethod - def _verify_blacklist(self, pairlist: List[str]) -> List[str]: - + def verify_blacklist(self, pairlist: List[str], blacklist: List[str]) -> List[str]: + """ + Verify and remove items from pairlist - returning a filtered pairlist. + """ for pair in deepcopy(pairlist): - if pair in self._pairlistmanager.blacklist: + if pair in blacklist: logger.warning(f"Pair {pair} in your blacklist. Removing it from whitelist...") pairlist.remove(pair) return pairlist + def _verify_blacklist(self, pairlist: List[str]) -> List[str]: + """ + Proxy method to verify_blacklist for easy access for child classes. + """ + return IPairList.verify_blacklist(pairlist, self._pairlistmanager.blacklist) + def _whitelist_for_active_markets(self, whitelist: List[str]) -> List[str]: """ Check available markets and remove pair from whitelist if necessary diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index b1681afef..ee6b3e37b 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -61,5 +61,6 @@ class PairListManager(): pairlist = pl.filter_pairlist(pairlist, tickers) # Validation against blacklist happens after the pairlists to ensure blacklist is respected. - pairlist = self.verify_blacklist(pairlist, self.blacklist) + pairlist = IPairList.verify_blacklist(pairlist, self.blacklist) + self._whitelist = pairlist From bf69b055ebc8b648095e0dfdbf3b461d80577ae7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 09:07:46 +0100 Subject: [PATCH 17/39] Add name getting --- freqtrade/pairlist/IPairList.py | 2 +- freqtrade/pairlist/LowPriceFilter.py | 4 ++-- freqtrade/pairlist/PrecisionFilter.py | 11 +++++------ freqtrade/pairlist/StaticPairList.py | 4 ++-- freqtrade/pairlist/VolumePairList.py | 4 ++-- freqtrade/pairlist/pairlistmanager.py | 16 ++++++++++++++-- freqtrade/resolvers/pairlist_resolver.py | 3 ++- 7 files changed, 28 insertions(+), 16 deletions(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index e14ae4187..ef9ea1d04 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -57,7 +57,7 @@ class IPairList(ABC): """ @staticmethod - def verify_blacklist(self, pairlist: List[str], blacklist: List[str]) -> List[str]: + def verify_blacklist(pairlist: List[str], blacklist: List[str]) -> List[str]: """ Verify and remove items from pairlist - returning a filtered pairlist. """ diff --git a/freqtrade/pairlist/LowPriceFilter.py b/freqtrade/pairlist/LowPriceFilter.py index 4d4a92676..9552b56b8 100644 --- a/freqtrade/pairlist/LowPriceFilter.py +++ b/freqtrade/pairlist/LowPriceFilter.py @@ -9,8 +9,8 @@ logger = logging.getLogger(__name__) class LowPriceFilter(IPairList): - def __init__(self, exchange, config, pairlistconfig: dict) -> None: - super().__init__(exchange, config, pairlistconfig) + def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig) self._low_price_percent = pairlistconfig.get('low_price_percent', 0) diff --git a/freqtrade/pairlist/PrecisionFilter.py b/freqtrade/pairlist/PrecisionFilter.py index 3e68ec607..b18237b00 100644 --- a/freqtrade/pairlist/PrecisionFilter.py +++ b/freqtrade/pairlist/PrecisionFilter.py @@ -33,11 +33,10 @@ class PrecisionFilter(IPairList): (already cleaned to be 1 - stoploss) :return: True if the pair can stay, false if it should be removed """ - stop_price = self._freqtrade.get_target_bid(ticker["symbol"], ticker) * stoploss + stop_price = ticker['ask'] * stoploss # Adjust stop-prices to precision - sp = self._freqtrade.exchange.symbol_price_prec(ticker["symbol"], stop_price) - stop_gap_price = self._freqtrade.exchange.symbol_price_prec(ticker["symbol"], - stop_price * 0.99) + sp = self._exchange.symbol_price_prec(ticker["symbol"], stop_price) + stop_gap_price = self._exchange.symbol_price_prec(ticker["symbol"], stop_price * 0.99) logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}") if sp <= stop_gap_price: logger.info(f"Removed {ticker['symbol']} from whitelist, " @@ -49,9 +48,9 @@ class PrecisionFilter(IPairList): """ Filters and sorts pairlists and assigns and returns them again. """ - if self._freqtrade.strategy.stoploss is not None: + if self._config.get('stoploss') is not None: # Precalculate sanitized stoploss value to avoid recalculation for every pair - stoploss = 1 - abs(self._freqtrade.strategy.stoploss) + stoploss = 1 - abs(self._config.get('stoploss')) # Copy list since we're modifying this list for p in deepcopy(pairlist): ticker = [t for t in tickers if t['symbol'] == p][0] diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index e70bd671c..f663f8b02 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -14,8 +14,8 @@ logger = logging.getLogger(__name__) class StaticPairList(IPairList): - def __init__(self, exchange, config, pairlistconfig: dict) -> None: - super().__init__(exchange, config, pairlistconfig) + def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig) @property def needstickers(self) -> bool: diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 77bdf472d..610986a72 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -19,8 +19,8 @@ SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] class VolumePairList(IPairList): - def __init__(self, exchange, config, pairlistconfig: dict) -> None: - super().__init__(exchange, config, pairlistconfig) + def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig) if 'number_assets' not in self._pairlistconfig: raise OperationalException( diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index ee6b3e37b..e6b2b6103 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -5,7 +5,7 @@ Provides lists as configured in config.json """ import logging -from typing import List +from typing import Dict, List from freqtrade.pairlist.IPairList import IPairList from freqtrade.resolvers import PairListResolver @@ -44,6 +44,18 @@ class PairListManager(): """ return self._blacklist + @property + def name(self) -> str: + """ + """ + return str([p.name for p in self._pairlists]) + + def short_desc(self) -> List[Dict]: + """ + List of short_desc for each pairlist + """ + return [{p.name: p.short_desc()} for p in self._pairlists] + def refresh_pairlist(self) -> None: """ Run pairlist through all pairlists. @@ -52,7 +64,7 @@ class PairListManager(): pairlist = self._whitelist.copy() # tickers should be cached to avoid calling the exchange on each call. - tickers = [] + tickers: List[Dict] = [] if self._tickers_needed: tickers = self._exchange.get_tickers() diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index a37d7dc05..9e051fa8f 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -20,7 +20,8 @@ class PairListResolver(IResolver): __slots__ = ['pairlist'] - def __init__(self, pairlist_name: str, exchange, pairlistmanager, config: dict, pairlistconfig) -> None: + def __init__(self, pairlist_name: str, exchange, pairlistmanager, + config: dict, pairlistconfig) -> None: """ Load the custom class from config parameter :param config: configuration dictionary or None From 85beb3b6a98dc83c5d6cd405c6e260b7743cd20d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 09:31:17 +0100 Subject: [PATCH 18/39] Fix test --- tests/pairlist/test_pairlist.py | 69 ++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 26 deletions(-) diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 23c48545e..5c718dfdf 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -7,6 +7,7 @@ import pytest from freqtrade import OperationalException from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.resolvers import PairListResolver +from freqtrade.pairlist.pairlistmanager import PairListManager from tests.conftest import get_patched_freqtradebot, log_has_re # whitelist, blacklist @@ -25,26 +26,41 @@ def whitelist_conf(default_conf): default_conf['exchange']['pair_blacklist'] = [ 'BLK/BTC' ] - default_conf['pairlist'] = {'method': 'StaticPairList', - 'config': {'number_assets': 5}, - 'filters': {}, - } + default_conf['pairlists'] = [ + { + "method": "VolumePairList", + "config": { + "number_assets": 5, + "sort_key": "quoteVolume", + } + }, + ] + return default_conf + +@pytest.fixture(scope="function") +def static_pl_conf(default_conf): + default_conf['pairlists'] = [ + { + "method": "StaticPairList", + }, + ] return default_conf def test_load_pairlist_noexist(mocker, markets, default_conf): - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + bot = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) + plm = PairListManager(bot.exchange, default_conf) with pytest.raises(OperationalException, match=r"Impossible to load Pairlist 'NonexistingPairList'. " r"This class does not exist or contains Python code errors."): - PairListResolver('NonexistingPairList', freqtradebot, default_conf).pairlist + PairListResolver('NonexistingPairList', bot.exchange, plm, default_conf, {}) -def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf): +def test_refresh_market_pair_not_in_whitelist(mocker, markets, static_pl_conf): - freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) + freqtradebot = get_patched_freqtradebot(mocker, static_pl_conf) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) freqtradebot.pairlists.refresh_pairlist() @@ -53,34 +69,33 @@ def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf): # Ensure all except those in whitelist are removed assert set(whitelist) == set(freqtradebot.pairlists.whitelist) # Ensure config dict hasn't been changed - assert (whitelist_conf['exchange']['pair_whitelist'] == + assert (static_pl_conf['exchange']['pair_whitelist'] == freqtradebot.config['exchange']['pair_whitelist']) -def test_refresh_pairlists(mocker, markets, whitelist_conf): - freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) - - mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) +def test_refresh_pairlists(mocker, markets, static_pl_conf): + freqtradebot = get_patched_freqtradebot(mocker, static_pl_conf) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + markets=PropertyMock(return_value=markets), + ) freqtradebot.pairlists.refresh_pairlist() # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed assert set(whitelist) == set(freqtradebot.pairlists.whitelist) - assert whitelist_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist + assert static_pl_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_conf): - whitelist_conf['pairlist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 5, - 'precision_filter': False} - } mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_tickers=tickers, - exchange_has=MagicMock(return_value=True) + exchange_has=MagicMock(return_value=True), ) - freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) + bot = get_patched_freqtradebot(mocker, whitelist_conf) # Remock markets with shitcoinmarkets since get_patched_freqtradebot uses the markets fixture mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -88,17 +103,19 @@ def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_co ) # argument: use the whitelist dynamically by exchange-volume whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC'] - freqtradebot.pairlists.refresh_pairlist() + bot.pairlists.refresh_pairlist() - assert whitelist == freqtradebot.pairlists.whitelist + assert whitelist == bot.pairlists.whitelist + + whitelist_conf['pairlists'] = [{'method': 'VolumePairList', + 'config': {} + } + ] - whitelist_conf['pairlist'] = {'method': 'VolumePairList', - 'config': {} - } with pytest.raises(OperationalException, match=r'`number_assets` not specified. Please check your configuration ' r'for "pairlist.config.number_assets"'): - PairListResolver('VolumePairList', freqtradebot, whitelist_conf).pairlist + PairListManager(bot.exchange, whitelist_conf) def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): From 870966dcd0fc8e1e67b2d7b60e22e765ba8c9898 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 09:42:34 +0100 Subject: [PATCH 19/39] Fix more tests --- freqtrade/pairlist/StaticPairList.py | 2 +- tests/pairlist/test_pairlist.py | 25 ++++++++++++++++--------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index f663f8b02..83450c4bc 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -41,4 +41,4 @@ class StaticPairList(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new whitelist """ - return self._config['exchange']['pair_whitelist'] + return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist']) diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 5c718dfdf..ffebaf60f 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -16,6 +16,7 @@ from tests.conftest import get_patched_freqtradebot, log_has_re @pytest.fixture(scope="function") def whitelist_conf(default_conf): default_conf['stake_currency'] = 'BTC' + default_conf['exchange']['name'] = 'binance' default_conf['exchange']['pair_whitelist'] = [ 'ETH/BTC', 'TKN/BTC', @@ -39,13 +40,13 @@ def whitelist_conf(default_conf): @pytest.fixture(scope="function") -def static_pl_conf(default_conf): - default_conf['pairlists'] = [ +def static_pl_conf(whitelist_conf): + whitelist_conf['pairlists'] = [ { "method": "StaticPairList", }, ] - return default_conf + return whitelist_conf def test_load_pairlist_noexist(mocker, markets, default_conf): @@ -73,7 +74,7 @@ def test_refresh_market_pair_not_in_whitelist(mocker, markets, static_pl_conf): freqtradebot.config['exchange']['pair_whitelist']) -def test_refresh_pairlists(mocker, markets, static_pl_conf): +def test_refresh_static_pairlist(mocker, markets, static_pl_conf): freqtradebot = get_patched_freqtradebot(mocker, static_pl_conf) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -119,6 +120,10 @@ def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_co def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + ) freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets_empty)) @@ -179,13 +184,15 @@ def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None @pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS) def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): - whitelist_conf['pairlist']['method'] = pairlist - mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) - mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + whitelist_conf['pairlists'][0]['method'] = pairlist + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True) + ) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) - assert freqtrade.pairlists.name == pairlist - assert pairlist in freqtrade.pairlists.short_desc() + assert freqtrade.pairlists.name == str([pairlist]) + assert pairlist in str(freqtrade.pairlists.short_desc()) assert isinstance(freqtrade.pairlists.whitelist, list) assert isinstance(freqtrade.pairlists.blacklist, list) From d7262c0b4e7e1b2b02605da117ccd56c0bcc9da3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 13:40:36 +0100 Subject: [PATCH 20/39] Fix correct ticker type --- freqtrade/pairlist/IPairList.py | 3 +- freqtrade/pairlist/LowPriceFilter.py | 6 ++- freqtrade/pairlist/PrecisionFilter.py | 6 +-- freqtrade/pairlist/StaticPairList.py | 2 +- freqtrade/pairlist/VolumePairList.py | 21 ++++++--- freqtrade/pairlist/pairlistmanager.py | 2 +- tests/pairlist/test_pairlist.py | 68 ++++++++++++++++----------- 7 files changed, 65 insertions(+), 43 deletions(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index ef9ea1d04..4a83bc939 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -46,7 +46,7 @@ class IPairList(ABC): """ @abstractmethod - def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. Called on each bot iteration - please use internal caching if necessary @@ -97,5 +97,6 @@ class IPairList(ABC): if pair not in sanitized_whitelist: sanitized_whitelist.append(pair) + sanitized_whitelist = self._verify_blacklist(sanitized_whitelist) # We need to remove pairs that are unknown return sanitized_whitelist diff --git a/freqtrade/pairlist/LowPriceFilter.py b/freqtrade/pairlist/LowPriceFilter.py index 9552b56b8..4e1ba52c8 100644 --- a/freqtrade/pairlist/LowPriceFilter.py +++ b/freqtrade/pairlist/LowPriceFilter.py @@ -46,7 +46,7 @@ class LowPriceFilter(IPairList): return False return True - def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. @@ -57,7 +57,9 @@ class LowPriceFilter(IPairList): """ # Copy list since we're modifying this list for p in deepcopy(pairlist): - ticker = [t for t in tickers if t['symbol'] == p][0] + ticker = tickers.get(p) + if not ticker: + pairlist.remove(p) # Filter out assets which would not allow setting a stoploss if self._low_price_percent and not self._validate_ticker_lowprice(ticker): diff --git a/freqtrade/pairlist/PrecisionFilter.py b/freqtrade/pairlist/PrecisionFilter.py index b18237b00..d7b2c96ae 100644 --- a/freqtrade/pairlist/PrecisionFilter.py +++ b/freqtrade/pairlist/PrecisionFilter.py @@ -44,7 +44,7 @@ class PrecisionFilter(IPairList): return False return True - def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlists and assigns and returns them again. """ @@ -53,9 +53,9 @@ class PrecisionFilter(IPairList): stoploss = 1 - abs(self._config.get('stoploss')) # Copy list since we're modifying this list for p in deepcopy(pairlist): - ticker = [t for t in tickers if t['symbol'] == p][0] + ticker = tickers.get(p) # Filter out assets which would not allow setting a stoploss - if (stoploss and not self._validate_precision_filter(ticker, stoploss)): + if not ticker or (stoploss and not self._validate_precision_filter(ticker, stoploss)): pairlist.remove(p) continue diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 83450c4bc..a7b71875c 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -33,7 +33,7 @@ class StaticPairList(IPairList): """ return f"{self.name}" - def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. Called on each bot iteration - please use internal caching if necessary diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 610986a72..e6ff69daf 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -5,10 +5,9 @@ Provides lists as configured in config.json """ import logging +from datetime import datetime from typing import Dict, List -from cachetools import TTLCache, cached - from freqtrade import OperationalException from freqtrade.pairlist.IPairList import IPairList @@ -28,6 +27,7 @@ class VolumePairList(IPairList): 'for "pairlist.config.number_assets"') self._number_pairs = self._pairlistconfig['number_assets'] self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume') + self._ttl = self._pairlistconfig.get('ttl', 1800) if not self._exchange.exchange_has('fetchTickers'): raise OperationalException( @@ -37,6 +37,7 @@ class VolumePairList(IPairList): if not self._validate_keys(self._sort_key): raise OperationalException( f'key {self._sort_key} not in {SORT_VALUES}') + self._last_refresh = 0 @property def needstickers(self) -> bool: @@ -56,7 +57,7 @@ class VolumePairList(IPairList): """ return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs." - def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. Called on each bot iteration - please use internal caching if necessary @@ -65,10 +66,17 @@ class VolumePairList(IPairList): :return: new whitelist """ # Generate dynamic whitelist - return self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key) + if self._last_refresh + self._ttl < datetime.now().timestamp(): + self._last_refresh = datetime.now().timestamp() + return self._gen_pair_whitelist(pairlist, + tickers, + self._config['stake_currency'], + self._sort_key, + ) + else: + return pairlist - @cached(TTLCache(maxsize=1, ttl=1800)) - def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]: + def _gen_pair_whitelist(self, pairlist, tickers, base_currency: str, key: str) -> List[str]: """ Updates the whitelist with with a dynamically generated list :param base_currency: base currency as str @@ -77,7 +85,6 @@ class VolumePairList(IPairList): :return: List of pairs """ - tickers = self._exchange.get_tickers() # check length so that we make sure that '/' is actually in the string tickers = [v for k, v in tickers.items() if (len(k.split('/')) == 2 and k.split('/')[1] == base_currency diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index e6b2b6103..03451e725 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -64,7 +64,7 @@ class PairListManager(): pairlist = self._whitelist.copy() # tickers should be cached to avoid calling the exchange on each call. - tickers: List[Dict] = [] + tickers: Dict = {} if self._tickers_needed: tickers = self._exchange.get_tickers() diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index ffebaf60f..5ad7fdf5a 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -16,7 +16,6 @@ from tests.conftest import get_patched_freqtradebot, log_has_re @pytest.fixture(scope="function") def whitelist_conf(default_conf): default_conf['stake_currency'] = 'BTC' - default_conf['exchange']['name'] = 'binance' default_conf['exchange']['pair_whitelist'] = [ 'ETH/BTC', 'TKN/BTC', @@ -137,31 +136,37 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): @pytest.mark.parametrize("filters,base_currency,key,whitelist_result", [ - ({}, "BTC", "quoteVolume", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']), - ({}, "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'HOT/BTC', 'FUEL/BTC']), - ({}, "USDT", "quoteVolume", ['ETH/USDT']), - ({}, "ETH", "quoteVolume", []), - ({"PrecisionFilter": {}}, "BTC", "quoteVolume", ["LTC/BTC", "ETH/BTC", "TKN/BTC", 'FUEL/BTC']), - ({"PrecisionFilter": {}}, "BTC", "bidVolume", ["LTC/BTC", "TKN/BTC", "ETH/BTC", 'FUEL/BTC']), - ({"LowPriceFilter": {"low_price_percent": 0.03}}, "BTC", - "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'FUEL/BTC']), + ([], "BTC", "quoteVolume", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']), + ([], "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'HOT/BTC', 'FUEL/BTC']), + ([], "USDT", "quoteVolume", ['ETH/USDT']), + ([], "ETH", "quoteVolume", []), + ([{"method": "PrecisionFilter"}], "BTC", + "quoteVolume", ["LTC/BTC", "ETH/BTC", "TKN/BTC", 'FUEL/BTC']), + ([{"method": "PrecisionFilter"}], + "BTC", "bidVolume", ["LTC/BTC", "TKN/BTC", "ETH/BTC", 'FUEL/BTC']), + ([{"method": "LowPriceFilter", "config": {"low_price_percent": 0.03}}], + "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'FUEL/BTC']), # Hot is removed by precision_filter, Fuel by low_price_filter. - ({"PrecisionFilter": {}, "LowPriceFilter": {"low_price_percent": 0.02}}, - "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC']), -]) + ([{"method": "PrecisionFilter"}, + {"method": "LowPriceFilter", "config": {"low_price_percent": 0.02}} + ], "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC'])]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, filters, base_currency, key, whitelist_result, caplog) -> None: - whitelist_conf['pairlist']['method'] = 'VolumePairList' - whitelist_conf['pairlist']['filters'] = filters + whitelist_conf['pairlists'].extend(filters) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) - mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=shitcoinmarkets)) - mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) + + mocker.patch.multiple('freqtrade.exchange.Exchange', + get_tickers=tickers, + markets=PropertyMock(return_value=shitcoinmarkets), + ) freqtrade.config['stake_currency'] = base_currency - whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency=base_currency, key=key) + freqtrade.pairlists.refresh_pairlist() + whitelist = freqtrade.pairlists.whitelist + assert sorted(whitelist) == sorted(whitelist_result) if 'PrecisionFilter' in filters: assert log_has_re(r'^Removed .* from whitelist, because stop price .* ' @@ -172,11 +177,14 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: - default_conf['pairlist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 10} - } - mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) - mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) + default_conf['pairlists'] = [{'method': 'VolumePairList', + 'config': {'number_assets': 10} + }] + + mocker.patch.multiple('freqtrade.exchange.Exchange', + get_tickers=tickers, + exchange_has=MagicMock(return_value=False), + ) with pytest.raises(OperationalException): get_patched_freqtradebot(mocker, default_conf) @@ -202,18 +210,22 @@ def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): (['ETH/BTC', 'TKN/BTC'], ""), (['ETH/BTC', 'TKN/BTC', 'TRX/ETH'], "is not compatible with exchange"), # TRX/ETH wrong stake (['ETH/BTC', 'TKN/BTC', 'BCH/BTC'], "is not compatible with exchange"), # BCH/BTC not available - (['ETH/BTC', 'TKN/BTC', 'BLK/BTC'], "is not compatible with exchange"), # BLK/BTC in blacklist + (['ETH/BTC', 'TKN/BTC', 'BLK/BTC'], "in your blacklist. Removing "), # BLK/BTC in blacklist (['ETH/BTC', 'TKN/BTC', 'BTT/BTC'], "Market is not active") # BTT/BTC is inactive ]) def test__whitelist_for_active_markets(mocker, whitelist_conf, markets, pairlist, whitelist, caplog, - log_message): - whitelist_conf['pairlist']['method'] = pairlist - mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) - mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + log_message, tickers): + whitelist_conf['pairlists'][0]['method'] = pairlist + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) caplog.clear() - new_whitelist = freqtrade.pairlists._whitelist_for_active_markets(whitelist) + # Assign starting whitelist + new_whitelist = freqtrade.pairlists._pairlists[0]._whitelist_for_active_markets(whitelist) assert set(new_whitelist) == set(['ETH/BTC', 'TKN/BTC']) assert log_message in caplog.text From c3b4a4dde1e2421bbd3db783b463ab6b1e9fbee9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 13:59:19 +0100 Subject: [PATCH 21/39] Update sample configurations --- config.json.example | 5 ++++- config_binance.json.example | 5 +++++ config_full.json.example | 23 ++++++++++++++--------- config_kraken.json.example | 5 +++++ 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/config.json.example b/config.json.example index 9a6dafd04..a2add358f 100644 --- a/config.json.example +++ b/config.json.example @@ -52,6 +52,9 @@ "DOGE/BTC" ] }, + "pairlists": [ + {"method": "StaticPairList"} + ], "edge": { "enabled": false, "process_throttle_secs": 3600, @@ -68,7 +71,7 @@ "remove_pumps": false }, "telegram": { - "enabled": true, + "enabled": false, "token": "your_telegram_token", "chat_id": "your_telegram_chat_id" }, diff --git a/config_binance.json.example b/config_binance.json.example index 58817a78e..aa36ed035 100644 --- a/config_binance.json.example +++ b/config_binance.json.example @@ -54,6 +54,11 @@ "BNB/BTC" ] }, + "pairlists": [ + { + "method": "StaticPairList" + } + ], "edge": { "enabled": false, "process_throttle_secs": 3600, diff --git a/config_full.json.example b/config_full.json.example index 9cec8fcb8..2bc2cde9e 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -50,17 +50,22 @@ "buy": "gtc", "sell": "gtc" }, - "pairlist": { - "method": "VolumePairList", - "config": { - "number_assets": 20, - "sort_key": "quoteVolume", + "pairlists": [ + {"method": "StaticPairList"}, + { + "method": "VolumePairList", + "config": { + "number_assets": 20, + "sort_key": "quoteVolume" + } }, - "filters":{ - "PrecisionFilter": {}, - "LowPriceFilter": {"low_price_percent": 0.01} + {"method": "PrecisionFilter"}, + {"method": "LowPriceFilter", + "config": { + "low_price_percent": 0.01 + } } - }, + ], "exchange": { "name": "bittrex", "sandbox": false, diff --git a/config_kraken.json.example b/config_kraken.json.example index 5a36941d8..9ca2ca065 100644 --- a/config_kraken.json.example +++ b/config_kraken.json.example @@ -46,6 +46,11 @@ ] }, + "pairlists": [ + { + "method": "StaticPairList" + } + ], "edge": { "enabled": false, "process_throttle_secs": 3600, From 37985310d5a748302ec0e4d974f66b7efbaddab6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 13:59:35 +0100 Subject: [PATCH 22/39] remove cachetools dependency --- requirements-common.txt | 1 - setup.py | 1 - 2 files changed, 2 deletions(-) diff --git a/requirements-common.txt b/requirements-common.txt index 64a43ee62..4793d25f6 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -4,7 +4,6 @@ ccxt==1.18.1346 SQLAlchemy==1.3.10 python-telegram-bot==12.2.0 arrow==0.15.2 -cachetools==3.1.1 requests==2.22.0 urllib3==1.25.6 wrapt==1.11.2 diff --git a/setup.py b/setup.py index 50b8eee9c..781a5d138 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,6 @@ setup(name='freqtrade', 'SQLAlchemy', 'python-telegram-bot', 'arrow', - 'cachetools', 'requests', 'urllib3', 'wrapt', From c74d76627546fd2ea84149abc629bde8161e7414 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 14:00:32 +0100 Subject: [PATCH 23/39] move from name to name_list --- freqtrade/pairlist/VolumePairList.py | 2 +- freqtrade/pairlist/pairlistmanager.py | 13 ++++++++++--- freqtrade/rpc/rpc.py | 4 ++-- tests/pairlist/test_pairlist.py | 2 +- tests/rpc/test_rpc.py | 17 ++++++++++------- tests/rpc/test_rpc_apiserver.py | 6 +++--- 6 files changed, 27 insertions(+), 17 deletions(-) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index e6ff69daf..902f7abd4 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -67,7 +67,7 @@ class VolumePairList(IPairList): """ # Generate dynamic whitelist if self._last_refresh + self._ttl < datetime.now().timestamp(): - self._last_refresh = datetime.now().timestamp() + self._last_refresh = int(datetime.now().timestamp()) return self._gen_pair_whitelist(pairlist, tickers, self._config['stake_currency'], diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index 03451e725..09e024497 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -7,6 +7,7 @@ Provides lists as configured in config.json import logging from typing import Dict, List +from freqtrade import OperationalException from freqtrade.pairlist.IPairList import IPairList from freqtrade.resolvers import PairListResolver @@ -22,12 +23,17 @@ class PairListManager(): self._blacklist = self._config['exchange'].get('pair_blacklist', []) self._pairlists: List[IPairList] = [] self._tickers_needed = False - for pl in self._config.get('pairlists', [{'method': "StaticPairList"}]): + pairlists = self._config.get('pairlists', None) + if not pairlists: + pairlists = [{'method': "StaticPairList"}] + for pl in pairlists: pairl = PairListResolver(pl.get('method'), exchange, self, config, pl.get('config')).pairlist self._tickers_needed = pairl.needstickers or self._tickers_needed self._pairlists.append(pairl) + if not self._pairlists: + raise OperationalException("No Pairlist defined!!") @property def whitelist(self) -> List[str]: @@ -45,10 +51,11 @@ class PairListManager(): return self._blacklist @property - def name(self) -> str: + def name_list(self) -> List[str]: """ + Get list of loaded pairlists names """ - return str([p.name for p in self._pairlists]) + return [p.name for p in self._pairlists] def short_desc(self) -> List[Dict]: """ diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index f994ac006..2de73ac25 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -469,7 +469,7 @@ class RPC: def _rpc_whitelist(self) -> Dict: """ Returns the currently active whitelist""" - res = {'method': self._freqtrade.pairlists.name, + res = {'method': self._freqtrade.pairlists.name_list, 'length': len(self._freqtrade.active_pair_whitelist), 'whitelist': self._freqtrade.active_pair_whitelist } @@ -484,7 +484,7 @@ class RPC: and pair not in self._freqtrade.pairlists.blacklist): self._freqtrade.pairlists.blacklist.append(pair) - res = {'method': self._freqtrade.pairlists.name, + res = {'method': self._freqtrade.pairlists.name_list, 'length': len(self._freqtrade.pairlists.blacklist), 'blacklist': self._freqtrade.pairlists.blacklist, } diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 5ad7fdf5a..bca3f4aea 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -199,7 +199,7 @@ def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): ) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) - assert freqtrade.pairlists.name == str([pairlist]) + assert freqtrade.pairlists.name_list == [pairlist] assert pairlist in str(freqtrade.pairlists.short_desc()) assert isinstance(freqtrade.pairlists.whitelist, list) assert isinstance(freqtrade.pairlists.blacklist, list) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index df2261c1f..8747fe6ff 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -719,21 +719,23 @@ def test_rpc_whitelist(mocker, default_conf) -> None: freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(freqtradebot) ret = rpc._rpc_whitelist() - assert ret['method'] == 'StaticPairList' + assert len(ret['method']) == 1 + assert 'StaticPairList' in ret['method'] assert ret['whitelist'] == default_conf['exchange']['pair_whitelist'] def test_rpc_whitelist_dynamic(mocker, default_conf) -> None: - default_conf['pairlist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 4} - } + default_conf['pairlists'] = [{'method': 'VolumePairList', + 'config': {'number_assets': 4} + }] mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(freqtradebot) ret = rpc._rpc_whitelist() - assert ret['method'] == 'VolumePairList' + assert len(ret['method']) == 1 + assert 'VolumePairList' in ret['method'] assert ret['length'] == 4 assert ret['whitelist'] == default_conf['exchange']['pair_whitelist'] @@ -744,13 +746,14 @@ def test_rpc_blacklist(mocker, default_conf) -> None: freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(freqtradebot) ret = rpc._rpc_blacklist(None) - assert ret['method'] == 'StaticPairList' + assert len(ret['method']) == 1 + assert 'StaticPairList' in ret['method'] assert len(ret['blacklist']) == 2 assert ret['blacklist'] == default_conf['exchange']['pair_blacklist'] assert ret['blacklist'] == ['DOGE/BTC', 'HOT/BTC'] ret = rpc._rpc_blacklist(["ETH/BTC"]) - assert ret['method'] == 'StaticPairList' + assert 'StaticPairList' in ret['method'] assert len(ret['blacklist']) == 3 assert ret['blacklist'] == default_conf['exchange']['pair_blacklist'] assert ret['blacklist'] == ['DOGE/BTC', 'HOT/BTC', 'ETH/BTC'] diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index b572a0514..aa5054314 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -456,7 +456,7 @@ def test_api_blacklist(botclient, mocker): assert_response(rc) assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC"], "length": 2, - "method": "StaticPairList"} + "method": ["StaticPairList"]} # Add ETH/BTC to blacklist rc = client_post(client, f"{BASE_URI}/blacklist", @@ -464,7 +464,7 @@ def test_api_blacklist(botclient, mocker): assert_response(rc) assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC"], "length": 3, - "method": "StaticPairList"} + "method": ["StaticPairList"]} def test_api_whitelist(botclient): @@ -474,7 +474,7 @@ def test_api_whitelist(botclient): assert_response(rc) assert rc.json == {"whitelist": ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC'], "length": 4, - "method": "StaticPairList"} + "method": ["StaticPairList"]} def test_api_forcebuy(botclient, mocker, fee): From 25cb935eeedce1e19afc2383953301b6fe79ab6e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 14:15:47 +0100 Subject: [PATCH 24/39] Some more adjustments for new pairlist --- freqtrade/configuration/config_validation.py | 7 ++++--- freqtrade/configuration/configuration.py | 3 +++ freqtrade/configuration/deprecated_settings.py | 11 ++++++++++- tests/conftest.py | 3 +++ tests/rpc/test_rpc_telegram.py | 14 +++++++------- tests/test_configuration.py | 8 +++++--- 6 files changed, 32 insertions(+), 14 deletions(-) diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 93d93263f..21086c913 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -121,6 +121,7 @@ def _validate_whitelist(conf: Dict[str, Any]) -> None: if conf.get('runmode', RunMode.OTHER) in [RunMode.OTHER, RunMode.PLOT]: return - if (conf.get('pairlist', {}).get('method', 'StaticPairList') == 'StaticPairList' - and not conf.get('exchange', {}).get('pair_whitelist')): - raise OperationalException("StaticPairList requires pair_whitelist to be set.") + for pl in conf.get('pairlists', [{'method': 'StaticPairList'}]): + if (pl.get('method') == 'StaticPairList' + and not conf.get('exchange', {}).get('pair_whitelist')): + raise OperationalException("StaticPairList requires pair_whitelist to be set.") diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index be1c7ab4e..b8995edf9 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -81,6 +81,9 @@ class Configuration: if 'ask_strategy' not in config: config['ask_strategy'] = {} + if 'pairlists' not in config: + config['pairlists'] = [] + # validate configuration before returning logger.info('Validating configuration ...') validate_config_schema(config) diff --git a/freqtrade/configuration/deprecated_settings.py b/freqtrade/configuration/deprecated_settings.py index 8471028aa..3aec85ae2 100644 --- a/freqtrade/configuration/deprecated_settings.py +++ b/freqtrade/configuration/deprecated_settings.py @@ -58,9 +58,18 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None: process_deprecated_setting(config, 'ask_strategy', 'ignore_roi_if_buy_signal', 'experimental', 'ignore_roi_if_buy_signal') + if config.get('pairlist', {}).get("method") == 'VolumePairList': + logger.warning( + "DEPRECATED: " + f"Using VolumePairList in pairlist is deprecated and must be moved to pairlists. " + "Please refer to the docs on configuration details") + config['pairlists'].append({'method': 'VolumePairList', + 'config': config.get('pairlist', {}).get('config') + }) + if config.get('pairlist', {}).get('config', {}).get('precision_filter'): logger.warning( "DEPRECATED: " f"Using precision_filter setting is deprecated and has been replaced by" "PrecisionFilter. Please refer to the docs on configuration details") - config['pairlist'].update({'filters': {'PrecisionFilter': {}}}) + config['pairlists'].append({'method': 'PrecisionFilter'}) diff --git a/tests/conftest.py b/tests/conftest.py index d551596f0..b2c76baec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -242,6 +242,9 @@ def default_conf(testdatadir): "HOT/BTC", ] }, + "pairlists": [ + {"method": "StaticPairList"} + ], "telegram": { "enabled": True, "token": "token", diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 766511d2d..bb9d88658 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1050,8 +1050,8 @@ def test_whitelist_static(default_conf, update, mocker) -> None: telegram._whitelist(update=update, context=MagicMock()) assert msg_mock.call_count == 1 - assert ('Using whitelist `StaticPairList` with 4 pairs\n`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`' - in msg_mock.call_args_list[0][0][0]) + assert ("Using whitelist `['StaticPairList']` with 4 pairs\n" + "`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`" in msg_mock.call_args_list[0][0][0]) def test_whitelist_dynamic(default_conf, update, mocker) -> None: @@ -1062,17 +1062,17 @@ def test_whitelist_dynamic(default_conf, update, mocker) -> None: _send_msg=msg_mock ) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) - default_conf['pairlist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 4} - } + default_conf['pairlists'] = [{'method': 'VolumePairList', + 'config': {'number_assets': 4} + }] freqtradebot = get_patched_freqtradebot(mocker, default_conf) telegram = Telegram(freqtradebot) telegram._whitelist(update=update, context=MagicMock()) assert msg_mock.call_count == 1 - assert ('Using whitelist `VolumePairList` with 4 pairs\n`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`' - in msg_mock.call_args_list[0][0][0]) + assert ("Using whitelist `['VolumePairList']` with 4 pairs\n" + "`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`" in msg_mock.call_args_list[0][0][0]) def test_blacklist_static(default_conf, update, mocker) -> None: diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 258088925..6cfae01c6 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -743,9 +743,9 @@ def test_validate_whitelist(default_conf): conf = deepcopy(default_conf) - conf.update({"pairlist": { + conf.update({"pairlists": [{ "method": "VolumePairList", - }}) + }]}) # Dynamic whitelist should not care about pair_whitelist validate_config_consistency(conf) del conf['exchange']['pair_whitelist'] @@ -963,14 +963,16 @@ def test_process_temporary_deprecated_settings(mocker, default_conf, setting, ca assert default_conf[setting[0]][setting[1]] == setting[5] -def test_process_deprecated_setting_precision_filter(mocker, default_conf, caplog): +def test_process_deprecated_setting_pairlists(mocker, default_conf, caplog): patched_configuration_load_config_file(mocker, default_conf) default_conf.update({'pairlist': { + 'method': 'VolumePairList', 'config': {'precision_filter': True} }}) process_temporary_deprecated_settings(default_conf) assert log_has_re(r'DEPRECATED.*precision_filter.*', caplog) + assert log_has_re(r'DEPRECATED.*in pairlist is deprecated and must be moved*', caplog) def test_check_conflicting_settings(mocker, default_conf, caplog): From ed0c7a6aaf80c4c92df7480577bcbe6d10a66d02 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 14:16:11 +0100 Subject: [PATCH 25/39] Update configschema to fit new pairlists approach --- freqtrade/constants.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 5fdd45916..c98c32d4c 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -20,7 +20,7 @@ REQUIRED_ORDERTIF = ['buy', 'sell'] REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange'] ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] -AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList'] +AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'PrecisionFilter', 'LowPriceFilter'] DRY_RUN_WALLET = 999.9 MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons @@ -151,13 +151,16 @@ CONF_SCHEMA = { 'block_bad_exchanges': {'type': 'boolean'} } }, - 'pairlist': { - 'type': 'object', - 'properties': { - 'method': {'type': 'string', 'enum': AVAILABLE_PAIRLISTS}, - 'config': {'type': 'object'} - }, - 'required': ['method'] + 'pairlists': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'method': {'type': 'string', 'enum': AVAILABLE_PAIRLISTS}, + 'config': {'type': 'object'} + }, + 'required': ['method'], + } }, 'telegram': { 'type': 'object', From 02b9da8aba530bc0f9115401f634108d2bdb947f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 14:39:28 +0100 Subject: [PATCH 26/39] Update documentation --- config_full.json.example | 3 ++- docs/configuration.md | 46 ++++++++++++++++++++++++---------------- docs/developer.md | 29 ++++++++++++++----------- 3 files changed, 47 insertions(+), 31 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 2bc2cde9e..ba53f47d6 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -56,7 +56,8 @@ "method": "VolumePairList", "config": { "number_assets": 20, - "sort_key": "quoteVolume" + "sort_key": "quoteVolume", + "ttl": 1800 } }, {"method": "PrecisionFilter"}, diff --git a/docs/configuration.md b/docs/configuration.md index 39fb4b8ac..8f62708d1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -82,8 +82,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exchange.markets_refresh_interval` | 60 | The interval in minutes in which markets are reloaded. | `edge` | false | Please refer to [edge configuration document](edge.md) for detailed explanation. | `experimental.block_bad_exchanges` | true | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now. -| `pairlist.method` | StaticPairList | Use static or dynamic volume-based pairlist. [More information below](#dynamic-pairlists). -| `pairlist.config` | None | Additional configuration for dynamic pairlists. [More information below](#dynamic-pairlists). +| `pairlists` | StaticPairList | Define one or more pairlists to be used. [More information below](#dynamic-pairlists). | `telegram.enabled` | true | **Required.** Enable or not the usage of Telegram. | `telegram.token` | token | Your Telegram bot token. Only required if `telegram.enabled` is `true`. ***Keep it in secrete, do not disclose publicly.*** | `telegram.chat_id` | chat_id | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. ***Keep it in secrete, do not disclose publicly.*** @@ -382,41 +381,50 @@ The valid values are: Pairlists define the list of pairs that the bot should trade. There are [`StaticPairList`](#static-pair-list) and dynamic Whitelists available. -In addition to pairlists, [pairlist filters](#available-pairlist-filters) can be configured, which remove certain assets. -These Filters work with all Pairlist providers and are applied in the sequence they occur. +[`PrecisionFilter`](#precision-filter) and [`LowPriceFilter`](#low-price-pair-filter) act as filters, removing low-value pairs. + +All pairlists can be chained, and a combination of all pairlists will become your new whitelist. + +Inactive markets and blacklisted pairs are always removed from the resulting `pair_whitelist`. ### Available Pairlists * [`StaticPairList`](#static-pair-list) (default, if not configured differently) * [`VolumePairList`](#volume-pair-list) +* [`PrecisionFilter`](#precision-filter) +* [`LowPriceFilter`](#low-price-pair-filter) #### Static Pair List -By default, the `StaticPairList` method is used, which uses a statically defined pair whitelist from the configuration. Inactive markets and blacklisted pairs are removed from the pair_whitelist. +By default, the `StaticPairList` method is used, which uses a statically defined pair whitelist from the configuration. It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklist`. +```json +"pairlists": [ + {"method": "StaticPairList"} + ], +``` + #### Volume Pair List `VolumePairList` selects `number_assets` top pairs based on `sort_key`, which can be one of `askVolume`, `bidVolume` and `quoteVolume` and defaults to `quoteVolume`. `VolumePairList` does not consider `pair_whitelist`, but selects the top assets from all available markets (with matching stake-currency) on the exchange. -Pairs in `pair_blacklist` are not considered for `VolumePairList`, even if all other filters would match. + +`ttl` allows setting the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes). ```json -"pairlist": { +"pairlists": [{ "method": "VolumePairList", "config": { "number_assets": 20, "sort_key": "quoteVolume", - }, + "ttl": 1800, + } +], ``` -### Available Pairlist Filters - -* [`PrecisionFilter`](#precision-filter) -* [`LowPriceFilter`](#low-price-pair-filter) - #### Precision Filter Filters low-value coins which would not allow setting a stoploss. @@ -440,17 +448,19 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, "pair_whitelist": [], "pair_blacklist": ["BNB/BTC"] }, -"pairlist": { +"pairlists": [ + { "method": "VolumePairList", "config": { "number_assets": 20, "sort_key": "quoteVolume", }, - "filters":{ - "PrecisionFilter": {}, - "LowPriceFilter": {"low_price_percent": 0.01} - } }, + {"method": "PrecisionFilter"}, + {"method": "LowPriceFilter", + "config": {"low_price_percent": 0.01} + } + }], ``` ## Switch to Dry-run mode diff --git a/docs/developer.md b/docs/developer.md index 346578c2e..e63d970e2 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -46,15 +46,18 @@ def test_method_to_test(caplog): The fastest and easiest way to start up is to use docker-compose.develop which gives developers the ability to start the bot up with all the required dependencies, *without* needing to install any freqtrade specific dependencies on your local machine. #### Install + * [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) * [docker](https://docs.docker.com/install/) * [docker-compose](https://docs.docker.com/compose/install/) #### Starting the bot ##### Use the develop dockerfile + ``` bash rm docker-compose.yml && mv docker-compose.develop.yml docker-compose.yml ``` + #### Docker Compose ##### Starting @@ -62,9 +65,11 @@ rm docker-compose.yml && mv docker-compose.develop.yml docker-compose.yml ``` bash docker-compose up ``` + ![Docker compose up](https://user-images.githubusercontent.com/419355/65456322-47f63a80-de06-11e9-90c6-3c74d1bad0b8.png) ##### Rebuilding + ``` bash docker-compose build ``` @@ -77,8 +82,8 @@ that can be effected by `docker-compose up` or `docker-compose run freqtrade_dev ``` bash docker-compose exec freqtrade_develop /bin/bash ``` -![image](https://user-images.githubusercontent.com/419355/65456522-ba671a80-de06-11e9-9598-df9ca0d8dcac.png) +![image](https://user-images.githubusercontent.com/419355/65456522-ba671a80-de06-11e9-9598-df9ca0d8dcac.png) ## Modules @@ -95,7 +100,7 @@ This is a simple provider, which however serves as a good example on how to star Next, modify the classname of the provider (ideally align this with the Filename). -The base-class provides the an instance of the bot (`self._freqtrade`), as well as the configuration (`self._config`), and initiates both `_blacklist` and `_whitelist`. +The base-class provides an instance of the exchange (`self._exchange`) the pairlist manager (`self._pairlistmanager`), as well as the main configuration (`self._config`) and the pairlist dedicated configuration (`self._pairlistconfig`). ```python self._freqtrade = freqtrade @@ -104,10 +109,9 @@ The base-class provides the an instance of the bot (`self._freqtrade`), as well self._blacklist = self._config['exchange'].get('pair_blacklist', []) ``` - Now, let's step through the methods which require actions: -#### configuration +#### Pairlist configuration Configuration for PairListProvider is done in the bot configuration file in the element `"pairlist"`. This Pairlist-object may contain a `"config"` dict with additional configurations for the configured pairlist. @@ -120,29 +124,30 @@ Additional elements can be configured as needed. `VolumePairList` uses `"sort_ke Returns a description used for Telegram messages. This should contain the name of the Provider, as well as a short description containing the number of assets. Please follow the format `"PairlistName - top/bottom X pairs"`. -#### refresh_pairlist +#### filter_pairlist Override this method and run all calculations needed in this method. This is called with each iteration of the bot - so consider implementing caching for compute/network heavy calculations. -Assign the resulting whiteslist to `self._whitelist` and `self._blacklist` respectively. These will then be used to run the bot in this iteration. Pairs with open trades will be added to the whitelist to have the sell-methods run correctly. +It get's passed a pairlist (which can be the result of previous pairlists) as well as `tickers`, a pre-fetched version of `get_tickers()`. -Please also run `self.validate_whitelist(pairs, tickers)` (tickers is optional, but should be passed when you're using tickers anyway) and to check and remove pairs with inactive markets. This function is available in the Parent class (`StaticPairList`) and should ideally not be overwritten. +It must return the resulting pairlist (which may then be passed into the next pairlist filter). + +Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filters. Use this if you limit your result to a certain number of pairs - so the endresult is not shorter than expected. ##### sample ``` python - def refresh_pairlist(self) -> None: + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: # Generate dynamic whitelist - pairs = self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key) - # Validate whitelist to only have active market pairs - self._whitelist = self.validate_whitelist(pairs)[:self._number_pairs] + pairs = self._calculate_pairlist(pairlist, tickers) + return pairs ``` #### _gen_pair_whitelist This is a simple method used by `VolumePairList` - however serves as a good example. -It implements caching (`@cached(TTLCache(maxsize=1, ttl=1800))`) as well as a configuration option to allow different (but similar) strategies to work with the same PairListProvider. +In VolumePairList, this implements different methods of sorting, does early validation so only the expected number of pairs is returned. ## Implement a new Exchange (WIP) From a01b34a004baa585cf3a9045d8aeb0374e076a83 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 14:44:39 +0100 Subject: [PATCH 27/39] tests --- freqtrade/pairlist/IPairList.py | 4 ++-- tests/pairlist/test_pairlist.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 4a83bc939..fc4187856 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -73,7 +73,7 @@ class IPairList(ABC): """ return IPairList.verify_blacklist(pairlist, self._pairlistmanager.blacklist) - def _whitelist_for_active_markets(self, whitelist: List[str]) -> List[str]: + def _whitelist_for_active_markets(self, pairlist: List[str]) -> List[str]: """ Check available markets and remove pair from whitelist if necessary :param whitelist: the sorted list of pairs the user might want to trade @@ -83,7 +83,7 @@ class IPairList(ABC): markets = self._exchange.markets sanitized_whitelist: List[str] = [] - for pair in whitelist: + for pair in pairlist: # pair is not in the generated dynamic market or has the wrong stake currency if (pair not in markets or not pair.endswith(self._config['stake_currency'])): logger.warning(f"Pair {pair} is not compatible with exchange " diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index bca3f4aea..d19c18715 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -229,3 +229,31 @@ def test__whitelist_for_active_markets(mocker, whitelist_conf, markets, pairlist assert set(new_whitelist) == set(['ETH/BTC', 'TKN/BTC']) assert log_message in caplog.text + + +def test_volumepairlist_invalid_sortvalue(mocker, markets, whitelist_conf): + whitelist_conf['pairlists'][0]['config'].update({"sort_key": "asdf"}) + + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + with pytest.raises(OperationalException, + match=r"key asdf not in .*"): + get_patched_freqtradebot(mocker, whitelist_conf) + + +def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers): + + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + bot = get_patched_freqtradebot(mocker, whitelist_conf) + assert bot.pairlists._pairlists[0]._last_refresh == 0 + + bot.pairlists.refresh_pairlist() + + assert bot.pairlists._pairlists[0]._last_refresh != 0 + lrf = bot.pairlists._pairlists[0]._last_refresh + bot.pairlists.refresh_pairlist() + # Time should not be updated. + assert bot.pairlists._pairlists[0]._last_refresh == lrf From ae356493665270b557e93099970e78377a8b185d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 14:49:25 +0100 Subject: [PATCH 28/39] improve pairlistmanager errorhandling --- freqtrade/pairlist/pairlistmanager.py | 12 +++++++----- tests/pairlist/test_pairlist.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index 09e024497..f29f89abf 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -23,17 +23,19 @@ class PairListManager(): self._blacklist = self._config['exchange'].get('pair_blacklist', []) self._pairlists: List[IPairList] = [] self._tickers_needed = False - pairlists = self._config.get('pairlists', None) - if not pairlists: - pairlists = [{'method': "StaticPairList"}] - for pl in pairlists: + + for pl in self._config.get('pairlists', None): + if 'method' not in pl: + logger.warning(f"No method in {pl}") + continue pairl = PairListResolver(pl.get('method'), exchange, self, config, pl.get('config')).pairlist self._tickers_needed = pairl.needstickers or self._tickers_needed self._pairlists.append(pairl) + if not self._pairlists: - raise OperationalException("No Pairlist defined!!") + raise OperationalException("No Pairlist defined!") @property def whitelist(self) -> List[str]: diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index d19c18715..017f78561 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -257,3 +257,19 @@ def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers): bot.pairlists.refresh_pairlist() # Time should not be updated. assert bot.pairlists._pairlists[0]._last_refresh == lrf + + +def test_pairlistmanager_no_pairlist(mocker, markets, whitelist_conf, caplog): + del whitelist_conf['pairlists'][0]['method'] + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + with pytest.raises(OperationalException, + match=r"No Pairlist defined!"): + get_patched_freqtradebot(mocker, whitelist_conf) + assert log_has_re("No method in .*", caplog) + + whitelist_conf['pairlists'] = [] + + with pytest.raises(OperationalException, + match=r"No Pairlist defined!"): + get_patched_freqtradebot(mocker, whitelist_conf) + From 7ff61f12e99ac67fa61f33e87e06d76e6d303008 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 15:04:04 +0100 Subject: [PATCH 29/39] pass pairlist position into the pairlists --- freqtrade/pairlist/IPairList.py | 11 ++++++++++- freqtrade/pairlist/LowPriceFilter.py | 5 +++-- freqtrade/pairlist/StaticPairList.py | 3 --- freqtrade/pairlist/VolumePairList.py | 22 +++++++++++++++------- freqtrade/pairlist/pairlistmanager.py | 11 +++++++---- freqtrade/resolvers/pairlist_resolver.py | 5 +++-- tests/pairlist/test_pairlist.py | 2 +- 7 files changed, 39 insertions(+), 20 deletions(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index fc4187856..231755cb0 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -16,11 +16,20 @@ logger = logging.getLogger(__name__) class IPairList(ABC): - def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict) -> None: + def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict, + pairlist_pos: int) -> None: + """ + :param exchange: Exchange instance + :param pairlistmanager: Instanciating Pairlist manager + :param config: Global bot configuration + :param pairlistconfig: Configuration for this pairlist - can be empty. + :param pairlist_pos: Position of the filter in the pairlist-filter-list + """ self._exchange = exchange self._pairlistmanager = pairlistmanager self._config = config self._pairlistconfig = pairlistconfig + self._pairlist_pos = pairlist_pos @property def name(self) -> str: diff --git a/freqtrade/pairlist/LowPriceFilter.py b/freqtrade/pairlist/LowPriceFilter.py index 4e1ba52c8..83b6a85e6 100644 --- a/freqtrade/pairlist/LowPriceFilter.py +++ b/freqtrade/pairlist/LowPriceFilter.py @@ -9,8 +9,9 @@ logger = logging.getLogger(__name__) class LowPriceFilter(IPairList): - def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict) -> None: - super().__init__(exchange, pairlistmanager, config, pairlistconfig) + def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict, + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) self._low_price_percent = pairlistconfig.get('low_price_percent', 0) diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index a7b71875c..0050fbd5c 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -14,9 +14,6 @@ logger = logging.getLogger(__name__) class StaticPairList(IPairList): - def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict) -> None: - super().__init__(exchange, pairlistmanager, config, pairlistconfig) - @property def needstickers(self) -> bool: """ diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 902f7abd4..708c8d7c2 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -18,8 +18,9 @@ SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] class VolumePairList(IPairList): - def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict) -> None: - super().__init__(exchange, pairlistmanager, config, pairlistconfig) + def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict, + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) if 'number_assets' not in self._pairlistconfig: raise OperationalException( @@ -85,11 +86,18 @@ class VolumePairList(IPairList): :return: List of pairs """ - # check length so that we make sure that '/' is actually in the string - tickers = [v for k, v in tickers.items() - if (len(k.split('/')) == 2 and k.split('/')[1] == base_currency - and v[key] is not None)] - sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) + if self._pairlist_pos == 0: + # If VolumePairList is the first in the list, use fresh pairlist + # check length so that we make sure that '/' is actually in the string + filtered_tickers = [v for k, v in tickers.items() + if (len(k.split('/')) == 2 and k.split('/')[1] == base_currency + and v[key] is not None)] + else: + # If other pairlist is in front, use the incomming pairlist. + filtered_tickers = [v for k, v in tickers.items() if k in pairlist] + + sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[key]) + # Validate whitelist to only have active market pairs pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers]) pairs = self._verify_blacklist(pairs) diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index f29f89abf..309ada094 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -23,14 +23,17 @@ class PairListManager(): self._blacklist = self._config['exchange'].get('pair_blacklist', []) self._pairlists: List[IPairList] = [] self._tickers_needed = False - for pl in self._config.get('pairlists', None): if 'method' not in pl: logger.warning(f"No method in {pl}") continue pairl = PairListResolver(pl.get('method'), - exchange, self, config, - pl.get('config')).pairlist + exchange=exchange, + pairlistmanager=self, + config=config, + pairlistconfig=pl.get('config'), + pairlist_pos=len(self._pairlists) + ).pairlist self._tickers_needed = pairl.needstickers or self._tickers_needed self._pairlists.append(pairl) @@ -67,7 +70,7 @@ class PairListManager(): def refresh_pairlist(self) -> None: """ - Run pairlist through all pairlists. + Run pairlist through all configured pairlists. """ pairlist = self._whitelist.copy() diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index 9e051fa8f..d849f4ffb 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -21,7 +21,7 @@ class PairListResolver(IResolver): __slots__ = ['pairlist'] def __init__(self, pairlist_name: str, exchange, pairlistmanager, - config: dict, pairlistconfig) -> None: + config: dict, pairlistconfig: dict, pairlist_pos: int) -> None: """ Load the custom class from config parameter :param config: configuration dictionary or None @@ -30,7 +30,8 @@ class PairListResolver(IResolver): kwargs={'exchange': exchange, 'pairlistmanager': pairlistmanager, 'config': config, - 'pairlistconfig': pairlistconfig}) + 'pairlistconfig': pairlistconfig, + 'pairlist_pos': pairlist_pos}) def _load_pairlist( self, pairlist_name: str, config: dict, kwargs: dict) -> IPairList: diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 017f78561..c322dd8c4 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -55,7 +55,7 @@ def test_load_pairlist_noexist(mocker, markets, default_conf): with pytest.raises(OperationalException, match=r"Impossible to load Pairlist 'NonexistingPairList'. " r"This class does not exist or contains Python code errors."): - PairListResolver('NonexistingPairList', bot.exchange, plm, default_conf, {}) + PairListResolver('NonexistingPairList', bot.exchange, plm, default_conf, {}, 1) def test_refresh_market_pair_not_in_whitelist(mocker, markets, static_pl_conf): From 5caeca75096d2e2fd963bcb1645475c818864f96 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 15:23:36 +0100 Subject: [PATCH 30/39] Improve tests for pairlist-sequence behaviour --- docs/developer.md | 9 ++--- tests/pairlist/test_pairlist.py | 63 +++++++++++++++++++++------------ 2 files changed, 46 insertions(+), 26 deletions(-) diff --git a/docs/developer.md b/docs/developer.md index e63d970e2..c65f429e9 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -100,13 +100,14 @@ This is a simple provider, which however serves as a good example on how to star Next, modify the classname of the provider (ideally align this with the Filename). -The base-class provides an instance of the exchange (`self._exchange`) the pairlist manager (`self._pairlistmanager`), as well as the main configuration (`self._config`) and the pairlist dedicated configuration (`self._pairlistconfig`). +The base-class provides an instance of the exchange (`self._exchange`) the pairlist manager (`self._pairlistmanager`), as well as the main configuration (`self._config`), the pairlist dedicated configuration (`self._pairlistconfig`) and the absolute position within the list of pairlists. ```python - self._freqtrade = freqtrade + self._exchange = exchange + self._pairlistmanager = pairlistmanager self._config = config - self._whitelist = self._config['exchange']['pair_whitelist'] - self._blacklist = self._config['exchange'].get('pair_blacklist', []) + self._pairlistconfig = pairlistconfig + self._pairlist_pos = pairlist_pos ``` Now, let's step through the methods which require actions: diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index c322dd8c4..c40187cc9 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -135,25 +135,44 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): assert set(whitelist) == set(pairslist) -@pytest.mark.parametrize("filters,base_currency,key,whitelist_result", [ - ([], "BTC", "quoteVolume", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']), - ([], "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'HOT/BTC', 'FUEL/BTC']), - ([], "USDT", "quoteVolume", ['ETH/USDT']), - ([], "ETH", "quoteVolume", []), - ([{"method": "PrecisionFilter"}], "BTC", - "quoteVolume", ["LTC/BTC", "ETH/BTC", "TKN/BTC", 'FUEL/BTC']), - ([{"method": "PrecisionFilter"}], - "BTC", "bidVolume", ["LTC/BTC", "TKN/BTC", "ETH/BTC", 'FUEL/BTC']), - ([{"method": "LowPriceFilter", "config": {"low_price_percent": 0.03}}], - "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'FUEL/BTC']), +@pytest.mark.parametrize("pairlists,base_currency,whitelist_result", [ + ([{"method": "VolumePairList", "config": {"number_assets": 5, "sort_key": "quoteVolume"}}], + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']), + # Different sorting depending on quote or bid volume + ([{"method": "VolumePairList", "config": {"number_assets": 5, "sort_key": "bidVolume"}}], + "BTC", ['HOT/BTC', 'FUEL/BTC', 'LTC/BTC', 'TKN/BTC', 'ETH/BTC']), + ([{"method": "VolumePairList", "config": {"number_assets": 5, "sort_key": "quoteVolume"}}], + "USDT", ['ETH/USDT']), + # No pair for ETH ... + ([{"method": "VolumePairList", "config": {"number_assets": 5, "sort_key": "quoteVolume"}}], + "ETH", []), + # Precisionfilter and quote volume + ([{"method": "VolumePairList", "config": {"number_assets": 5, "sort_key": "quoteVolume"}}, + {"method": "PrecisionFilter"}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'FUEL/BTC']), + # Precisionfilter bid + ([{"method": "VolumePairList", "config": {"number_assets": 5, "sort_key": "bidVolume"}}, + {"method": "PrecisionFilter"}], "BTC", ['FUEL/BTC', 'LTC/BTC', 'TKN/BTC', 'ETH/BTC']), + # Lowpricefilter and VolumePairList + ([{"method": "VolumePairList", "config": {"number_assets": 5, "sort_key": "quoteVolume"}}, + {"method": "LowPriceFilter", "config": {"low_price_percent": 0.03}}], + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'FUEL/BTC']), # Hot is removed by precision_filter, Fuel by low_price_filter. - ([{"method": "PrecisionFilter"}, + ([{"method": "VolumePairList", "config": {"number_assets": 5, "sort_key": "quoteVolume"}}, + {"method": "PrecisionFilter"}, {"method": "LowPriceFilter", "config": {"low_price_percent": 0.02}} - ], "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC'])]) + ], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), + # StaticPairlist Only + ([{"method": "StaticPairList"}, + ], "BTC", ['ETH/BTC', 'TKN/BTC']), + # Static Pairlist before VolumePairList - sorting changes + ([{"method": "StaticPairList"}, + {"method": "VolumePairList", "config": {"number_assets": 5, "sort_key": "bidVolume"}}, + ], "BTC", ['TKN/BTC', 'ETH/BTC']), +]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, - filters, base_currency, key, whitelist_result, + pairlists, base_currency, whitelist_result, caplog) -> None: - whitelist_conf['pairlists'].extend(filters) + whitelist_conf['pairlists'] = pairlists mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) @@ -167,13 +186,13 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t freqtrade.pairlists.refresh_pairlist() whitelist = freqtrade.pairlists.whitelist - assert sorted(whitelist) == sorted(whitelist_result) - if 'PrecisionFilter' in filters: - assert log_has_re(r'^Removed .* from whitelist, because stop price .* ' - r'would be <= stop limit.*', caplog) - - if 'LowPriceFilter' in filters: - assert log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) + assert whitelist == whitelist_result + for pairlist in pairlists: + if pairlist['method'] == 'PrecisionFilter': + assert log_has_re(r'^Removed .* from whitelist, because stop price .* ' + r'would be <= stop limit.*', caplog) + if pairlist['method'] == 'LowPriceFilter': + assert log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: From 0b4800835c9bfa7f08d2eac5fa228e35bdc06d9e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 15:26:30 +0100 Subject: [PATCH 31/39] update documentation --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 8f62708d1..a8bfae6f9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -410,7 +410,7 @@ It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklis `VolumePairList` selects `number_assets` top pairs based on `sort_key`, which can be one of `askVolume`, `bidVolume` and `quoteVolume` and defaults to `quoteVolume`. -`VolumePairList` does not consider `pair_whitelist`, but selects the top assets from all available markets (with matching stake-currency) on the exchange. +`VolumePairList` considers outputs of previous pairlists unless it's the first configured pairlist, it does not consider `pair_whitelist`, but selects the top assets from all available markets (with matching stake-currency) on the exchange. `ttl` allows setting the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes). From 86a5dfa62e3d72b4d37cf4589dcae68f88a5e1b2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 15:28:36 +0100 Subject: [PATCH 32/39] Update documentation --- docs/configuration.md | 2 +- freqtrade/configuration/config_validation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index a8bfae6f9..86acab8d6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -383,7 +383,7 @@ There are [`StaticPairList`](#static-pair-list) and dynamic Whitelists available [`PrecisionFilter`](#precision-filter) and [`LowPriceFilter`](#low-price-pair-filter) act as filters, removing low-value pairs. -All pairlists can be chained, and a combination of all pairlists will become your new whitelist. +All pairlists can be chained, and a combination of all pairlists will become your new whitelist. Pairlists are executed in the sequence they are configured. You should always configure either `StaticPairList` or `DynamicPairList` as starting pairlists. Inactive markets and blacklisted pairs are always removed from the resulting `pair_whitelist`. diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 21086c913..83d58c9e4 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -123,5 +123,5 @@ def _validate_whitelist(conf: Dict[str, Any]) -> None: for pl in conf.get('pairlists', [{'method': 'StaticPairList'}]): if (pl.get('method') == 'StaticPairList' - and not conf.get('exchange', {}).get('pair_whitelist')): + and not conf.get('exchange', {}).get('pair_whitelist')): raise OperationalException("StaticPairList requires pair_whitelist to be set.") From 4b15873ee151bc5a651bce0a0d39d3920f6aeb29 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 15:36:11 +0100 Subject: [PATCH 33/39] Simplify examples --- config_binance.json.example | 4 +--- config_kraken.json.example | 4 +--- tests/pairlist/test_pairlist.py | 1 - 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/config_binance.json.example b/config_binance.json.example index aa36ed035..7d616fe91 100644 --- a/config_binance.json.example +++ b/config_binance.json.example @@ -55,9 +55,7 @@ ] }, "pairlists": [ - { - "method": "StaticPairList" - } + {"method": "StaticPairList"} ], "edge": { "enabled": false, diff --git a/config_kraken.json.example b/config_kraken.json.example index 9ca2ca065..854aeef71 100644 --- a/config_kraken.json.example +++ b/config_kraken.json.example @@ -47,9 +47,7 @@ ] }, "pairlists": [ - { - "method": "StaticPairList" - } + {"method": "StaticPairList"} ], "edge": { "enabled": false, diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index c40187cc9..0ec6766c2 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -291,4 +291,3 @@ def test_pairlistmanager_no_pairlist(mocker, markets, whitelist_conf, caplog): with pytest.raises(OperationalException, match=r"No Pairlist defined!"): get_patched_freqtradebot(mocker, whitelist_conf) - From 085aa3084ebc07a4bc5dac098577e38dbb415217 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Nov 2019 19:45:09 +0100 Subject: [PATCH 34/39] Implement ticker caching --- freqtrade/pairlist/pairlistmanager.py | 7 ++++++- requirements-common.txt | 1 + setup.py | 1 + tests/pairlist/test_pairlist.py | 4 +++- 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index 309ada094..0734d7f8f 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -4,6 +4,7 @@ Static List provider Provides lists as configured in config.json """ +from cachetools import TTLCache, cached import logging from typing import Dict, List @@ -68,6 +69,10 @@ class PairListManager(): """ return [{p.name: p.short_desc()} for p in self._pairlists] + @cached(TTLCache(maxsize=1, ttl=1800)) + def _get_cached_tickers(self): + return self._exchange.get_tickers() + def refresh_pairlist(self) -> None: """ Run pairlist through all configured pairlists. @@ -78,7 +83,7 @@ class PairListManager(): # tickers should be cached to avoid calling the exchange on each call. tickers: Dict = {} if self._tickers_needed: - tickers = self._exchange.get_tickers() + tickers = self._get_cached_tickers() # Process all pairlists in chain for pl in self._pairlists: diff --git a/requirements-common.txt b/requirements-common.txt index 52b80d501..c11179fbb 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -4,6 +4,7 @@ ccxt==1.19.14 SQLAlchemy==1.3.10 python-telegram-bot==12.2.0 arrow==0.15.4 +cachetools==3.1.1 requests==2.22.0 urllib3==1.25.6 wrapt==1.11.2 diff --git a/setup.py b/setup.py index 781a5d138..50b8eee9c 100644 --- a/setup.py +++ b/setup.py @@ -66,6 +66,7 @@ setup(name='freqtrade', 'SQLAlchemy', 'python-telegram-bot', 'arrow', + 'cachetools', 'requests', 'urllib3', 'wrapt', diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 0ec6766c2..94b2147f5 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -268,12 +268,14 @@ def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers): ) bot = get_patched_freqtradebot(mocker, whitelist_conf) assert bot.pairlists._pairlists[0]._last_refresh == 0 - + assert tickers.call_count == 0 bot.pairlists.refresh_pairlist() + assert tickers.call_count == 1 assert bot.pairlists._pairlists[0]._last_refresh != 0 lrf = bot.pairlists._pairlists[0]._last_refresh bot.pairlists.refresh_pairlist() + assert tickers.call_count == 1 # Time should not be updated. assert bot.pairlists._pairlists[0]._last_refresh == lrf From 52e24c3a257c5ddd53c255516d578205a5b1277a Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 12 Nov 2019 09:27:53 +0100 Subject: [PATCH 35/39] Split error-messsage between incompatible and wrong stake amount --- freqtrade/pairlist/IPairList.py | 7 ++++++- tests/pairlist/test_pairlist.py | 14 ++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 231755cb0..d722e70f5 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -94,10 +94,15 @@ class IPairList(ABC): sanitized_whitelist: List[str] = [] for pair in pairlist: # pair is not in the generated dynamic market or has the wrong stake currency - if (pair not in markets or not pair.endswith(self._config['stake_currency'])): + if pair not in markets: logger.warning(f"Pair {pair} is not compatible with exchange " f"{self._exchange.name}. Removing it from whitelist..") continue + if not pair.endswith(self._config['stake_currency']): + logger.warning(f"Pair {pair} is not compatible with your stake currency " + f"{self._config['stake_currency']}. Removing it from whitelist..") + continue + # Check if market is active market = markets[pair] if not market_is_active(market): diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 94b2147f5..13f868c7a 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -227,10 +227,16 @@ def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): @pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS) @pytest.mark.parametrize("whitelist,log_message", [ (['ETH/BTC', 'TKN/BTC'], ""), - (['ETH/BTC', 'TKN/BTC', 'TRX/ETH'], "is not compatible with exchange"), # TRX/ETH wrong stake - (['ETH/BTC', 'TKN/BTC', 'BCH/BTC'], "is not compatible with exchange"), # BCH/BTC not available - (['ETH/BTC', 'TKN/BTC', 'BLK/BTC'], "in your blacklist. Removing "), # BLK/BTC in blacklist - (['ETH/BTC', 'TKN/BTC', 'BTT/BTC'], "Market is not active") # BTT/BTC is inactive + # TRX/ETH not in markets + (['ETH/BTC', 'TKN/BTC', 'TRX/ETH'], "is not compatible with exchange"), + # wrong stake + (['ETH/BTC', 'TKN/BTC', 'ETH/USDT'], "is not compatible with your stake currency"), + # BCH/BTC not available + (['ETH/BTC', 'TKN/BTC', 'BCH/BTC'], "is not compatible with exchange"), + # BLK/BTC in blacklist + (['ETH/BTC', 'TKN/BTC', 'BLK/BTC'], "in your blacklist. Removing "), + # BTT/BTC is inactive + (['ETH/BTC', 'TKN/BTC', 'BTT/BTC'], "Market is not active") ]) def test__whitelist_for_active_markets(mocker, whitelist_conf, markets, pairlist, whitelist, caplog, log_message, tickers): From c22b00b30303ba44f3dcf140b4fd8b70fb2e2f80 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 19 Nov 2019 06:34:54 +0100 Subject: [PATCH 36/39] move pairlist filters out of config[] --- config_full.json.example | 5 +---- docs/configuration.md | 22 +++++++------------- freqtrade/pairlist/LowPriceFilter.py | 8 +++---- freqtrade/pairlist/pairlistmanager.py | 2 +- tests/pairlist/test_pairlist.py | 30 +++++++++++++-------------- tests/rpc/test_rpc.py | 2 +- tests/rpc/test_rpc_telegram.py | 2 +- 7 files changed, 30 insertions(+), 41 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index ba53f47d6..270938237 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -61,10 +61,7 @@ } }, {"method": "PrecisionFilter"}, - {"method": "LowPriceFilter", - "config": { - "low_price_percent": 0.01 - } + {"method": "LowPriceFilter", "low_price_ratio": 0.01 } ], "exchange": { diff --git a/docs/configuration.md b/docs/configuration.md index 8a2b74ed5..120842cdb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -416,11 +416,9 @@ It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklis ```json "pairlists": [{ "method": "VolumePairList", - "config": { - "number_assets": 20, - "sort_key": "quoteVolume", - "ttl": 1800, - } + "number_assets": 20, + "sort_key": "quoteVolume", + "ttl": 1800, ], ``` @@ -430,7 +428,7 @@ Filters low-value coins which would not allow setting a stoploss. #### Low Price Pair Filter -The `LowPriceFilter` allows filtering of pairs where a raise of 1 price unit is below the `low_price_percent` ratio. +The `LowPriceFilter` allows filtering of pairs where a raise of 1 price unit is below the `low_price_ratio` ratio. This option is disabled by default, and will only apply if set to <> 0. Calculation example: @@ -450,16 +448,12 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, "pairlists": [ { "method": "VolumePairList", - "config": { - "number_assets": 20, - "sort_key": "quoteVolume", - }, + "number_assets": 20, + "sort_key": "quoteVolume", }, {"method": "PrecisionFilter"}, - {"method": "LowPriceFilter", - "config": {"low_price_percent": 0.01} - } - }], + {"method": "LowPriceFilter", "low_price_ratio": 0.01} + ], ``` ## Switch to Dry-run mode diff --git a/freqtrade/pairlist/LowPriceFilter.py b/freqtrade/pairlist/LowPriceFilter.py index 83b6a85e6..ff18b97c8 100644 --- a/freqtrade/pairlist/LowPriceFilter.py +++ b/freqtrade/pairlist/LowPriceFilter.py @@ -13,7 +13,7 @@ class LowPriceFilter(IPairList): pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) - self._low_price_percent = pairlistconfig.get('low_price_percent', 0) + self._low_price_ratio = pairlistconfig.get('low_price_ratio', 0) @property def needstickers(self) -> bool: @@ -28,7 +28,7 @@ class LowPriceFilter(IPairList): """ Short whitelist method description - used for startup-messages """ - return f"{self.name} - Filtering pairs priced below {self._low_price_percent * 100}%." + return f"{self.name} - Filtering pairs priced below {self._low_price_ratio * 100}%." def _validate_ticker_lowprice(self, ticker) -> bool: """ @@ -41,7 +41,7 @@ class LowPriceFilter(IPairList): compare = ticker['last'] + 1 / pow(10, precision) changeperc = (compare - ticker['last']) / ticker['last'] - if changeperc > self._low_price_percent: + if changeperc > self._low_price_ratio: logger.info(f"Removed {ticker['symbol']} from whitelist, " f"because 1 unit is {changeperc * 100:.3f}%") return False @@ -63,7 +63,7 @@ class LowPriceFilter(IPairList): pairlist.remove(p) # Filter out assets which would not allow setting a stoploss - if self._low_price_percent and not self._validate_ticker_lowprice(ticker): + if self._low_price_ratio and not self._validate_ticker_lowprice(ticker): pairlist.remove(p) return pairlist diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index 0734d7f8f..fa5382c37 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -32,7 +32,7 @@ class PairListManager(): exchange=exchange, pairlistmanager=self, config=config, - pairlistconfig=pl.get('config'), + pairlistconfig=pl, pairlist_pos=len(self._pairlists) ).pairlist self._tickers_needed = pairl.needstickers or self._tickers_needed diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 13f868c7a..9daaa5353 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -29,10 +29,8 @@ def whitelist_conf(default_conf): default_conf['pairlists'] = [ { "method": "VolumePairList", - "config": { - "number_assets": 5, - "sort_key": "quoteVolume", - } + "number_assets": 5, + "sort_key": "quoteVolume", }, ] return default_conf @@ -136,37 +134,37 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): @pytest.mark.parametrize("pairlists,base_currency,whitelist_result", [ - ([{"method": "VolumePairList", "config": {"number_assets": 5, "sort_key": "quoteVolume"}}], + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']), # Different sorting depending on quote or bid volume - ([{"method": "VolumePairList", "config": {"number_assets": 5, "sort_key": "bidVolume"}}], + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}], "BTC", ['HOT/BTC', 'FUEL/BTC', 'LTC/BTC', 'TKN/BTC', 'ETH/BTC']), - ([{"method": "VolumePairList", "config": {"number_assets": 5, "sort_key": "quoteVolume"}}], + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], "USDT", ['ETH/USDT']), # No pair for ETH ... - ([{"method": "VolumePairList", "config": {"number_assets": 5, "sort_key": "quoteVolume"}}], + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], "ETH", []), # Precisionfilter and quote volume - ([{"method": "VolumePairList", "config": {"number_assets": 5, "sort_key": "quoteVolume"}}, + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PrecisionFilter"}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'FUEL/BTC']), # Precisionfilter bid - ([{"method": "VolumePairList", "config": {"number_assets": 5, "sort_key": "bidVolume"}}, + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}, {"method": "PrecisionFilter"}], "BTC", ['FUEL/BTC', 'LTC/BTC', 'TKN/BTC', 'ETH/BTC']), # Lowpricefilter and VolumePairList - ([{"method": "VolumePairList", "config": {"number_assets": 5, "sort_key": "quoteVolume"}}, - {"method": "LowPriceFilter", "config": {"low_price_percent": 0.03}}], + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "LowPriceFilter", "low_price_ratio": 0.03}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'FUEL/BTC']), # Hot is removed by precision_filter, Fuel by low_price_filter. - ([{"method": "VolumePairList", "config": {"number_assets": 5, "sort_key": "quoteVolume"}}, + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PrecisionFilter"}, - {"method": "LowPriceFilter", "config": {"low_price_percent": 0.02}} + {"method": "LowPriceFilter", "low_price_ratio": 0.02} ], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), # StaticPairlist Only ([{"method": "StaticPairList"}, ], "BTC", ['ETH/BTC', 'TKN/BTC']), # Static Pairlist before VolumePairList - sorting changes ([{"method": "StaticPairList"}, - {"method": "VolumePairList", "config": {"number_assets": 5, "sort_key": "bidVolume"}}, + {"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}, ], "BTC", ['TKN/BTC', 'ETH/BTC']), ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, @@ -257,7 +255,7 @@ def test__whitelist_for_active_markets(mocker, whitelist_conf, markets, pairlist def test_volumepairlist_invalid_sortvalue(mocker, markets, whitelist_conf): - whitelist_conf['pairlists'][0]['config'].update({"sort_key": "asdf"}) + whitelist_conf['pairlists'][0].update({"sort_key": "asdf"}) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) with pytest.raises(OperationalException, diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 8747fe6ff..edc0fdb8a 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -726,7 +726,7 @@ def test_rpc_whitelist(mocker, default_conf) -> None: def test_rpc_whitelist_dynamic(mocker, default_conf) -> None: default_conf['pairlists'] = [{'method': 'VolumePairList', - 'config': {'number_assets': 4} + 'number_assets': 4, }] mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index bb9d88658..52d58e887 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1063,7 +1063,7 @@ def test_whitelist_dynamic(default_conf, update, mocker) -> None: ) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) default_conf['pairlists'] = [{'method': 'VolumePairList', - 'config': {'number_assets': 4} + 'number_assets': 4 }] freqtradebot = get_patched_freqtradebot(mocker, default_conf) From a8855bf7955eed07a107e31ca53b0a418426c373 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 19 Nov 2019 06:41:05 +0100 Subject: [PATCH 37/39] rename LowPriceFilter to PrieFilter --- config_full.json.example | 2 +- docs/configuration.md | 13 +++++++------ freqtrade/constants.py | 2 +- .../pairlist/{LowPriceFilter.py => PriceFilter.py} | 4 ++-- tests/pairlist/test_pairlist.py | 8 ++++---- 5 files changed, 15 insertions(+), 14 deletions(-) rename freqtrade/pairlist/{LowPriceFilter.py => PriceFilter.py} (96%) diff --git a/config_full.json.example b/config_full.json.example index 270938237..5ceeaff09 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -61,7 +61,7 @@ } }, {"method": "PrecisionFilter"}, - {"method": "LowPriceFilter", "low_price_ratio": 0.01 + {"method": "PriceFilter", "low_price_ratio": 0.01 } ], "exchange": { diff --git a/docs/configuration.md b/docs/configuration.md index 120842cdb..ed50b521e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -380,7 +380,7 @@ The valid values are: Pairlists define the list of pairs that the bot should trade. There are [`StaticPairList`](#static-pair-list) and dynamic Whitelists available. -[`PrecisionFilter`](#precision-filter) and [`LowPriceFilter`](#low-price-pair-filter) act as filters, removing low-value pairs. +[`PrecisionFilter`](#precision-filter) and [`PriceFilter`](#price-pair-filter) act as filters, removing low-value pairs. All pairlists can be chained, and a combination of all pairlists will become your new whitelist. Pairlists are executed in the sequence they are configured. You should always configure either `StaticPairList` or `DynamicPairList` as starting pairlists. @@ -391,7 +391,7 @@ Inactive markets and blacklisted pairs are always removed from the resulting `pa * [`StaticPairList`](#static-pair-list) (default, if not configured differently) * [`VolumePairList`](#volume-pair-list) * [`PrecisionFilter`](#precision-filter) -* [`LowPriceFilter`](#low-price-pair-filter) +* [`PriceFilter`](#price-pair-filter) #### Static Pair List @@ -426,9 +426,10 @@ It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklis Filters low-value coins which would not allow setting a stoploss. -#### Low Price Pair Filter +#### Price Pair Filter -The `LowPriceFilter` allows filtering of pairs where a raise of 1 price unit is below the `low_price_ratio` ratio. +The `PriceFilter` allows filtering of pairs by price. +Currently, only `low_price_ratio` is implemented, where a raise of 1 price unit (pip) is below the `low_price_ratio` ratio. This option is disabled by default, and will only apply if set to <> 0. Calculation example: @@ -438,7 +439,7 @@ These pairs are dangerous since it may be impossible to place the desired stoplo ### Full Pairlist example -The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting by `quoteVolume` and applies both [`PrecisionFilter`](#precision-filter) and [`LowPriceFilter`](#low-price-pair-filter), filtering all assets where 1 priceunit is > 1%. +The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting by `quoteVolume` and applies both [`PrecisionFilter`](#precision-filter) and [`PriceFilter`](#price-pair-filter), filtering all assets where 1 priceunit is > 1%. ```json "exchange": { @@ -452,7 +453,7 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, "sort_key": "quoteVolume", }, {"method": "PrecisionFilter"}, - {"method": "LowPriceFilter", "low_price_ratio": 0.01} + {"method": "PriceFilter", "low_price_ratio": 0.01} ], ``` diff --git a/freqtrade/constants.py b/freqtrade/constants.py index c98c32d4c..24e3589fd 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -20,7 +20,7 @@ REQUIRED_ORDERTIF = ['buy', 'sell'] REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange'] ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] -AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'PrecisionFilter', 'LowPriceFilter'] +AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'PrecisionFilter', 'PriceFilter'] DRY_RUN_WALLET = 999.9 MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons diff --git a/freqtrade/pairlist/LowPriceFilter.py b/freqtrade/pairlist/PriceFilter.py similarity index 96% rename from freqtrade/pairlist/LowPriceFilter.py rename to freqtrade/pairlist/PriceFilter.py index ff18b97c8..b3546ebd9 100644 --- a/freqtrade/pairlist/LowPriceFilter.py +++ b/freqtrade/pairlist/PriceFilter.py @@ -7,7 +7,7 @@ from freqtrade.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) -class LowPriceFilter(IPairList): +class PriceFilter(IPairList): def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict, pairlist_pos: int) -> None: @@ -32,7 +32,7 @@ class LowPriceFilter(IPairList): def _validate_ticker_lowprice(self, ticker) -> bool: """ - Check if if one price-step is > than a certain barrier. + Check if if one price-step (pip) is > than a certain barrier. :param ticker: ticker dict as returned from ccxt.load_markets() :param precision: Precision :return: True if the pair can stay, false if it should be removed diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 9daaa5353..76537880c 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -150,14 +150,14 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): # Precisionfilter bid ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}, {"method": "PrecisionFilter"}], "BTC", ['FUEL/BTC', 'LTC/BTC', 'TKN/BTC', 'ETH/BTC']), - # Lowpricefilter and VolumePairList + # PriceFilter and VolumePairList ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, - {"method": "LowPriceFilter", "low_price_ratio": 0.03}], + {"method": "PriceFilter", "low_price_ratio": 0.03}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'FUEL/BTC']), # Hot is removed by precision_filter, Fuel by low_price_filter. ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PrecisionFilter"}, - {"method": "LowPriceFilter", "low_price_ratio": 0.02} + {"method": "PriceFilter", "low_price_ratio": 0.02} ], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), # StaticPairlist Only ([{"method": "StaticPairList"}, @@ -189,7 +189,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t if pairlist['method'] == 'PrecisionFilter': assert log_has_re(r'^Removed .* from whitelist, because stop price .* ' r'would be <= stop limit.*', caplog) - if pairlist['method'] == 'LowPriceFilter': + if pairlist['method'] == 'PriceFilter': assert log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) From 5f62a9e4d8eed5d3c7a15eca92c041c84bf12b57 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 19 Nov 2019 06:48:56 +0100 Subject: [PATCH 38/39] rename ttl to refresh_period --- config_full.json.example | 8 +++----- docs/configuration.md | 4 ++-- docs/developer.md | 2 +- freqtrade/pairlist/VolumePairList.py | 4 ++-- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 5ceeaff09..b9631f63d 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -54,11 +54,9 @@ {"method": "StaticPairList"}, { "method": "VolumePairList", - "config": { - "number_assets": 20, - "sort_key": "quoteVolume", - "ttl": 1800 - } + "number_assets": 20, + "sort_key": "quoteVolume", + "refresh_period": 1800 }, {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.01 diff --git a/docs/configuration.md b/docs/configuration.md index ed50b521e..946ee3bf7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -411,14 +411,14 @@ It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklis `VolumePairList` considers outputs of previous pairlists unless it's the first configured pairlist, it does not consider `pair_whitelist`, but selects the top assets from all available markets (with matching stake-currency) on the exchange. -`ttl` allows setting the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes). +`refresh_period` allows setting the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes). ```json "pairlists": [{ "method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume", - "ttl": 1800, + "refresh_period": 1800, ], ``` diff --git a/docs/developer.md b/docs/developer.md index 0eba6fd59..2d39893bc 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -115,7 +115,7 @@ Now, let's step through the methods which require actions: #### Pairlist configuration Configuration for PairListProvider is done in the bot configuration file in the element `"pairlist"`. -This Pairlist-object may contain a `"config"` dict with additional configurations for the configured pairlist. +This Pairlist-object may contain configurations with additional configurations for the configured pairlist. By convention, `"number_assets"` is used to specify the maximum number of pairs to keep in the whitelist. Please follow this to ensure a consistent user experience. Additional elements can be configured as needed. `VolumePairList` uses `"sort_key"` to specify the sorting value - however feel free to specify whatever is necessary for your great algorithm to be successfull and dynamic. diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 708c8d7c2..2df9ba691 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -28,7 +28,7 @@ class VolumePairList(IPairList): 'for "pairlist.config.number_assets"') self._number_pairs = self._pairlistconfig['number_assets'] self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume') - self._ttl = self._pairlistconfig.get('ttl', 1800) + self.refresh_period = self._pairlistconfig.get('refresh_period', 1800) if not self._exchange.exchange_has('fetchTickers'): raise OperationalException( @@ -67,7 +67,7 @@ class VolumePairList(IPairList): :return: new whitelist """ # Generate dynamic whitelist - if self._last_refresh + self._ttl < datetime.now().timestamp(): + if self._last_refresh + self.refresh_period < datetime.now().timestamp(): self._last_refresh = int(datetime.now().timestamp()) return self._gen_pair_whitelist(pairlist, tickers, From c92f233c1565174b4b8aa6f0ef2bbd57e1b1db2e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 19 Nov 2019 19:33:04 +0100 Subject: [PATCH 39/39] Move settings to correct location --- freqtrade/configuration/deprecated_settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/configuration/deprecated_settings.py b/freqtrade/configuration/deprecated_settings.py index 3aec85ae2..8f3dbd675 100644 --- a/freqtrade/configuration/deprecated_settings.py +++ b/freqtrade/configuration/deprecated_settings.py @@ -63,9 +63,9 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None: "DEPRECATED: " f"Using VolumePairList in pairlist is deprecated and must be moved to pairlists. " "Please refer to the docs on configuration details") - config['pairlists'].append({'method': 'VolumePairList', - 'config': config.get('pairlist', {}).get('config') - }) + pl = {'method': 'VolumePairList'} + pl.update(config.get('pairlist', {}).get('config')) + config['pairlists'].append(pl) if config.get('pairlist', {}).get('config', {}).get('precision_filter'): logger.warning(