diff --git a/config.json.example b/config.json.example index 9a6dafd04..a2add358f 100644 --- a/config.json.example +++ b/config.json.example @@ -52,6 +52,9 @@ "DOGE/BTC" ] }, + "pairlists": [ + {"method": "StaticPairList"} + ], "edge": { "enabled": false, "process_throttle_secs": 3600, @@ -68,7 +71,7 @@ "remove_pumps": false }, "telegram": { - "enabled": true, + "enabled": false, "token": "your_telegram_token", "chat_id": "your_telegram_chat_id" }, diff --git a/config_binance.json.example b/config_binance.json.example index 58817a78e..7d616fe91 100644 --- a/config_binance.json.example +++ b/config_binance.json.example @@ -54,6 +54,9 @@ "BNB/BTC" ] }, + "pairlists": [ + {"method": "StaticPairList"} + ], "edge": { "enabled": false, "process_throttle_secs": 3600, diff --git a/config_full.json.example b/config_full.json.example index 5edc19419..b9631f63d 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -50,14 +50,18 @@ "buy": "gtc", "sell": "gtc" }, - "pairlist": { - "method": "VolumePairList", - "config": { + "pairlists": [ + {"method": "StaticPairList"}, + { + "method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume", - "precision_filter": false + "refresh_period": 1800 + }, + {"method": "PrecisionFilter"}, + {"method": "PriceFilter", "low_price_ratio": 0.01 } - }, + ], "exchange": { "name": "bittrex", "sandbox": false, diff --git a/config_kraken.json.example b/config_kraken.json.example index 5a36941d8..854aeef71 100644 --- a/config_kraken.json.example +++ b/config_kraken.json.example @@ -46,6 +46,9 @@ ] }, + "pairlists": [ + {"method": "StaticPairList"} + ], "edge": { "enabled": false, "process_throttle_secs": 3600, diff --git a/docs/configuration.md b/docs/configuration.md index b7607e619..f913d7dbb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -82,8 +82,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exchange.markets_refresh_interval` | 60 | The interval in minutes in which markets are reloaded. | `edge` | false | Please refer to [edge configuration document](edge.md) for detailed explanation. | `experimental.block_bad_exchanges` | true | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now. -| `pairlist.method` | StaticPairList | Use static or dynamic volume-based pairlist. [More information below](#dynamic-pairlists). -| `pairlist.config` | None | Additional configuration for dynamic pairlists. [More information below](#dynamic-pairlists). +| `pairlists` | StaticPairList | Define one or more pairlists to be used. [More information below](#dynamic-pairlists). | `telegram.enabled` | true | **Required.** Enable or not the usage of Telegram. | `telegram.token` | token | Your Telegram bot token. Only required if `telegram.enabled` is `true`. ***Keep it in secrete, do not disclose publicly.*** | `telegram.chat_id` | chat_id | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. ***Keep it in secrete, do not disclose publicly.*** @@ -362,7 +361,7 @@ For example, to test the order type `FOK` with Kraken, and modify candle_limit t The `fiat_display_currency` configuration parameter sets the base currency to use for the conversion from coin to fiat in the bot Telegram reports. -The valid values are:p +The valid values are: ```json "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD" @@ -376,6 +375,88 @@ The valid values are: "BTC", "ETH", "XRP", "LTC", "BCH", "USDT" ``` +## Pairlists + +Pairlists define the list of pairs that the bot should trade. +There are [`StaticPairList`](#static-pair-list) and dynamic Whitelists available. + +[`PrecisionFilter`](#precision-filter) and [`PriceFilter`](#price-pair-filter) act as filters, removing low-value pairs. + +All pairlists can be chained, and a combination of all pairlists will become your new whitelist. Pairlists are executed in the sequence they are configured. You should always configure either `StaticPairList` or `DynamicPairList` as starting pairlists. + +Inactive markets and blacklisted pairs are always removed from the resulting `pair_whitelist`. + +### Available Pairlists + +* [`StaticPairList`](#static-pair-list) (default, if not configured differently) +* [`VolumePairList`](#volume-pair-list) +* [`PrecisionFilter`](#precision-filter) +* [`PriceFilter`](#price-pair-filter) + +#### Static Pair List + +By default, the `StaticPairList` method is used, which uses a statically defined pair whitelist from the configuration. + +It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklist`. + +```json +"pairlists": [ + {"method": "StaticPairList"} + ], +``` + +#### Volume Pair List + +`VolumePairList` selects `number_assets` top pairs based on `sort_key`, which can be one of `askVolume`, `bidVolume` and `quoteVolume` and defaults to `quoteVolume`. + +`VolumePairList` considers outputs of previous pairlists unless it's the first configured pairlist, it does not consider `pair_whitelist`, but selects the top assets from all available markets (with matching stake-currency) on the exchange. + +`refresh_period` allows setting the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes). + +```json +"pairlists": [{ + "method": "VolumePairList", + "number_assets": 20, + "sort_key": "quoteVolume", + "refresh_period": 1800, +], +``` + +#### Precision Filter + +Filters low-value coins which would not allow setting a stoploss. + +#### Price Pair Filter + +The `PriceFilter` allows filtering of pairs by price. +Currently, only `low_price_ratio` is implemented, where a raise of 1 price unit (pip) is below the `low_price_ratio` ratio. +This option is disabled by default, and will only apply if set to <> 0. + +Calculation example: +Min price precision is 8 decimals. If price is 0.00000011 - one step would be 0.00000012 - which is almost 10% higher than the previous value. + +These pairs are dangerous since it may be impossible to place the desired stoploss - and often result in high losses. + +### Full Pairlist example + +The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting by `quoteVolume` and applies both [`PrecisionFilter`](#precision-filter) and [`PriceFilter`](#price-pair-filter), filtering all assets where 1 priceunit is > 1%. + +```json +"exchange": { + "pair_whitelist": [], + "pair_blacklist": ["BNB/BTC"] +}, +"pairlists": [ + { + "method": "VolumePairList", + "number_assets": 20, + "sort_key": "quoteVolume", + }, + {"method": "PrecisionFilter"}, + {"method": "PriceFilter", "low_price_ratio": 0.01} + ], +``` + ## Switch to Dry-run mode We recommend starting the bot in the Dry-run mode to see how your bot will @@ -405,45 +486,6 @@ creating trades on the exchange. Once you will be happy with your bot performance running in the Dry-run mode, you can switch it to production mode. -### Dynamic Pairlists - -Dynamic pairlists select pairs for you based on the logic configured. -The bot runs against all pairs (with that stake) on the exchange, and a number of assets -(`number_assets`) is selected based on the selected criteria. - -By default, the `StaticPairList` method is used. -The Pairlist method is configured as `pair_whitelist` parameter under the `exchange` -section of the configuration. - -**Available Pairlist methods:** - -* `StaticPairList` - * It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklist`. -* `VolumePairList` - * It selects `number_assets` top pairs based on `sort_key`, which can be one of -`askVolume`, `bidVolume` and `quoteVolume`, defaults to `quoteVolume`. - * There is a possibility to filter low-value coins that would not allow setting a stop loss -(set `precision_filter` parameter to `true` for this). - * `VolumePairList` does not consider `pair_whitelist`, but builds this automatically based the pairlist configuration. - * Pairs in `pair_blacklist` are not considered for VolumePairList, even if all other filters would match. - -Example: - -```json -"exchange": { - "pair_whitelist": [], - "pair_blacklist": ["BNB/BTC"] -}, -"pairlist": { - "method": "VolumePairList", - "config": { - "number_assets": 20, - "sort_key": "quoteVolume", - "precision_filter": false - } - }, -``` - ## Switch to production mode In production mode, the bot will engage your money. Be careful, since a wrong @@ -476,7 +518,7 @@ you run it in production mode. You should also make sure to read the [Exchanges](exchanges.md) section of the documentation to be aware of potential configuration details specific to your exchange. -### Using proxy with FreqTrade +### Using proxy with Freqtrade To use a proxy with freqtrade, add the kwarg `"aiohttp_trust_env"=true` to the `"ccxt_async_kwargs"` dict in the exchange section of the configuration. @@ -496,14 +538,13 @@ export HTTPS_PROXY="http://addr:port" freqtrade ``` - -### Embedding Strategies +## Embedding Strategies FreqTrade provides you with with an easy way to embed the strategy into your configuration file. This is done by utilizing BASE64 encoding and providing this string at the strategy configuration field, in your chosen config file. -#### Encoding a string as BASE64 +### Encoding a string as BASE64 This is a quick example, how to generate the BASE64 string in python diff --git a/docs/developer.md b/docs/developer.md index 41c66ce37..ab647726c 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -46,15 +46,18 @@ def test_method_to_test(caplog): The fastest and easiest way to start up is to use docker-compose.develop which gives developers the ability to start the bot up with all the required dependencies, *without* needing to install any freqtrade specific dependencies on your local machine. #### Install + * [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) * [docker](https://docs.docker.com/install/) * [docker-compose](https://docs.docker.com/compose/install/) #### Starting the bot ##### Use the develop dockerfile + ``` bash rm docker-compose.yml && mv docker-compose.develop.yml docker-compose.yml ``` + #### Docker Compose ##### Starting @@ -62,9 +65,11 @@ rm docker-compose.yml && mv docker-compose.develop.yml docker-compose.yml ``` bash docker-compose up ``` + ![Docker compose up](https://user-images.githubusercontent.com/419355/65456322-47f63a80-de06-11e9-90c6-3c74d1bad0b8.png) ##### Rebuilding + ``` bash docker-compose build ``` @@ -77,8 +82,8 @@ that can be effected by `docker-compose up` or `docker-compose run freqtrade_dev ``` bash docker-compose exec freqtrade_develop /bin/bash ``` -![image](https://user-images.githubusercontent.com/419355/65456522-ba671a80-de06-11e9-9598-df9ca0d8dcac.png) +![image](https://user-images.githubusercontent.com/419355/65456522-ba671a80-de06-11e9-9598-df9ca0d8dcac.png) ## Modules @@ -95,22 +100,22 @@ This is a simple provider, which however serves as a good example on how to star Next, modify the classname of the provider (ideally align this with the Filename). -The base-class provides the an instance of the bot (`self._freqtrade`), as well as the configuration (`self._config`), and initiates both `_blacklist` and `_whitelist`. +The base-class provides an instance of the exchange (`self._exchange`) the pairlist manager (`self._pairlistmanager`), as well as the main configuration (`self._config`), the pairlist dedicated configuration (`self._pairlistconfig`) and the absolute position within the list of pairlists. ```python - self._freqtrade = freqtrade + self._exchange = exchange + self._pairlistmanager = pairlistmanager self._config = config - self._whitelist = self._config['exchange']['pair_whitelist'] - self._blacklist = self._config['exchange'].get('pair_blacklist', []) + self._pairlistconfig = pairlistconfig + self._pairlist_pos = pairlist_pos ``` - Now, let's step through the methods which require actions: -#### configuration +#### Pairlist configuration Configuration for PairListProvider is done in the bot configuration file in the element `"pairlist"`. -This Pairlist-object may contain a `"config"` dict with additional configurations for the configured pairlist. +This Pairlist-object may contain configurations with additional configurations for the configured pairlist. By convention, `"number_assets"` is used to specify the maximum number of pairs to keep in the whitelist. Please follow this to ensure a consistent user experience. Additional elements can be configured as needed. `VolumePairList` uses `"sort_key"` to specify the sorting value - however feel free to specify whatever is necessary for your great algorithm to be successfull and dynamic. @@ -120,29 +125,30 @@ Additional elements can be configured as needed. `VolumePairList` uses `"sort_ke Returns a description used for Telegram messages. This should contain the name of the Provider, as well as a short description containing the number of assets. Please follow the format `"PairlistName - top/bottom X pairs"`. -#### refresh_pairlist +#### filter_pairlist Override this method and run all calculations needed in this method. This is called with each iteration of the bot - so consider implementing caching for compute/network heavy calculations. -Assign the resulting whiteslist to `self._whitelist` and `self._blacklist` respectively. These will then be used to run the bot in this iteration. Pairs with open trades will be added to the whitelist to have the sell-methods run correctly. +It get's passed a pairlist (which can be the result of previous pairlists) as well as `tickers`, a pre-fetched version of `get_tickers()`. -Please also run `self._validate_whitelist(pairs)` and to check and remove pairs with inactive markets. This function is available in the Parent class (`StaticPairList`) and should ideally not be overwritten. +It must return the resulting pairlist (which may then be passed into the next pairlist filter). + +Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filters. Use this if you limit your result to a certain number of pairs - so the endresult is not shorter than expected. ##### sample ``` python - def refresh_pairlist(self) -> None: + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: # Generate dynamic whitelist - pairs = self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key) - # Validate whitelist to only have active market pairs - self._whitelist = self._validate_whitelist(pairs)[:self._number_pairs] + pairs = self._calculate_pairlist(pairlist, tickers) + return pairs ``` #### _gen_pair_whitelist This is a simple method used by `VolumePairList` - however serves as a good example. -It implements caching (`@cached(TTLCache(maxsize=1, ttl=1800))`) as well as a configuration option to allow different (but similar) strategies to work with the same PairListProvider. +In VolumePairList, this implements different methods of sorting, does early validation so only the expected number of pairs is returned. ## Implement a new Exchange (WIP) diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 8083264e2..8a7641a08 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -122,6 +122,7 @@ def _validate_whitelist(conf: Dict[str, Any]) -> None: RunMode.UTIL_NO_EXCHANGE, RunMode.UTIL_EXCHANGE]: return - if (conf.get('pairlist', {}).get('method', 'StaticPairList') == 'StaticPairList' - and not conf.get('exchange', {}).get('pair_whitelist')): - raise OperationalException("StaticPairList requires pair_whitelist to be set.") + for pl in conf.get('pairlists', [{'method': 'StaticPairList'}]): + if (pl.get('method') == 'StaticPairList' + and not conf.get('exchange', {}).get('pair_whitelist')): + raise OperationalException("StaticPairList requires pair_whitelist to be set.") diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 3bf5e63a6..93eee3912 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -81,6 +81,9 @@ class Configuration: if 'ask_strategy' not in config: config['ask_strategy'] = {} + if 'pairlists' not in config: + config['pairlists'] = [] + # validate configuration before returning logger.info('Validating configuration ...') validate_config_schema(config) diff --git a/freqtrade/configuration/deprecated_settings.py b/freqtrade/configuration/deprecated_settings.py index f00b23894..8f3dbd675 100644 --- a/freqtrade/configuration/deprecated_settings.py +++ b/freqtrade/configuration/deprecated_settings.py @@ -57,3 +57,19 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None: 'experimental', 'sell_profit_only') process_deprecated_setting(config, 'ask_strategy', 'ignore_roi_if_buy_signal', 'experimental', 'ignore_roi_if_buy_signal') + + if config.get('pairlist', {}).get("method") == 'VolumePairList': + logger.warning( + "DEPRECATED: " + f"Using VolumePairList in pairlist is deprecated and must be moved to pairlists. " + "Please refer to the docs on configuration details") + pl = {'method': 'VolumePairList'} + pl.update(config.get('pairlist', {}).get('config')) + config['pairlists'].append(pl) + + if config.get('pairlist', {}).get('config', {}).get('precision_filter'): + logger.warning( + "DEPRECATED: " + f"Using precision_filter setting is deprecated and has been replaced by" + "PrecisionFilter. Please refer to the docs on configuration details") + config['pairlists'].append({'method': 'PrecisionFilter'}) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index dabda910c..a92371bc3 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -18,7 +18,7 @@ REQUIRED_ORDERTIF = ['buy', 'sell'] REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange'] ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] -AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList'] +AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'PrecisionFilter', 'PriceFilter'] DRY_RUN_WALLET = 999.9 MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons @@ -149,13 +149,16 @@ CONF_SCHEMA = { 'block_bad_exchanges': {'type': 'boolean'} } }, - 'pairlist': { - 'type': 'object', - 'properties': { - 'method': {'type': 'string', 'enum': AVAILABLE_PAIRLISTS}, - 'config': {'type': 'object'} - }, - 'required': ['method'] + 'pairlists': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'method': {'type': 'string', 'enum': AVAILABLE_PAIRLISTS}, + 'config': {'type': 'object'} + }, + 'required': ['method'], + } }, 'telegram': { 'type': 'object', diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f7cec080b..58f676aef 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -20,9 +20,9 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date from freqtrade.persistence import Trade -from freqtrade.resolvers import (ExchangeResolver, PairListResolver, - StrategyResolver) +from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.rpc import RPCManager, RPCMessageType +from freqtrade.pairlist.pairlistmanager import PairListManager from freqtrade.state import State from freqtrade.strategy.interface import IStrategy, SellType from freqtrade.wallets import Wallets @@ -70,8 +70,7 @@ class FreqtradeBot: # Attach Wallets to Strategy baseclass IStrategy.wallets = self.wallets - pairlistname = self.config.get('pairlist', {}).get('method', 'StaticPairList') - self.pairlists = PairListResolver(pairlistname, self, self.config).pairlist + self.pairlists = PairListManager(self.exchange, self.config) # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 5afb0c4c2..d722e70f5 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -5,22 +5,31 @@ Provides lists as configured in config.json """ import logging -from abc import ABC, abstractmethod -from typing import List +from abc import ABC, abstractmethod, abstractproperty +from copy import deepcopy +from typing import Dict, List from freqtrade.exchange import market_is_active - logger = logging.getLogger(__name__) class IPairList(ABC): - def __init__(self, freqtrade, config: dict) -> None: - self._freqtrade = freqtrade + def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict, + pairlist_pos: int) -> None: + """ + :param exchange: Exchange instance + :param pairlistmanager: Instanciating Pairlist manager + :param config: Global bot configuration + :param pairlistconfig: Configuration for this pairlist - can be empty. + :param pairlist_pos: Position of the filter in the pairlist-filter-list + """ + self._exchange = exchange + self._pairlistmanager = pairlistmanager self._config = config - self._whitelist = self._config['exchange']['pair_whitelist'] - self._blacklist = self._config['exchange'].get('pair_blacklist', []) + self._pairlistconfig = pairlistconfig + self._pairlist_pos = pairlist_pos @property def name(self) -> str: @@ -30,21 +39,13 @@ class IPairList(ABC): """ return self.__class__.__name__ - @property - def whitelist(self) -> List[str]: + @abstractproperty + def needstickers(self) -> bool: """ - Has the current whitelist - -> no need to overwrite in subclasses + Boolean property defining if tickers are necessary. + If no Pairlist requries tickers, an empty List is passed + as tickers argument to filter_pairlist """ - return self._whitelist - - @property - def blacklist(self) -> List[str]: - """ - Has the current blacklist - -> no need to overwrite in subclasses - """ - return self._blacklist @abstractmethod def short_desc(self) -> str: @@ -54,36 +55,62 @@ class IPairList(ABC): """ @abstractmethod - def refresh_pairlist(self) -> None: + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ - Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary -> Please overwrite in subclasses + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist """ - def _validate_whitelist(self, whitelist: List[str]) -> List[str]: + @staticmethod + def verify_blacklist(pairlist: List[str], blacklist: List[str]) -> List[str]: + """ + Verify and remove items from pairlist - returning a filtered pairlist. + """ + for pair in deepcopy(pairlist): + if pair in blacklist: + logger.warning(f"Pair {pair} in your blacklist. Removing it from whitelist...") + pairlist.remove(pair) + return pairlist + + def _verify_blacklist(self, pairlist: List[str]) -> List[str]: + """ + Proxy method to verify_blacklist for easy access for child classes. + """ + return IPairList.verify_blacklist(pairlist, self._pairlistmanager.blacklist) + + def _whitelist_for_active_markets(self, pairlist: List[str]) -> List[str]: """ Check available markets and remove pair from whitelist if necessary :param whitelist: the sorted list of pairs the user might want to trade :return: the list of pairs the user wants to trade without those unavailable or black_listed """ - markets = self._freqtrade.exchange.markets + markets = self._exchange.markets - sanitized_whitelist = set() - for pair in whitelist: - # pair is not in the generated dynamic market, or in the blacklist ... ignore it - if (pair in self.blacklist or pair not in markets - or not pair.endswith(self._config['stake_currency'])): + sanitized_whitelist: List[str] = [] + for pair in pairlist: + # pair is not in the generated dynamic market or has the wrong stake currency + if pair not in markets: logger.warning(f"Pair {pair} is not compatible with exchange " - f"{self._freqtrade.exchange.name} or contained in " - f"your blacklist. Removing it from whitelist..") + f"{self._exchange.name}. Removing it from whitelist..") continue + if not pair.endswith(self._config['stake_currency']): + logger.warning(f"Pair {pair} is not compatible with your stake currency " + f"{self._config['stake_currency']}. Removing it from whitelist..") + continue + # Check if market is active market = markets[pair] if not market_is_active(market): logger.info(f"Ignoring {pair} from whitelist. Market is not active.") continue - sanitized_whitelist.add(pair) + if pair not in sanitized_whitelist: + sanitized_whitelist.append(pair) + sanitized_whitelist = self._verify_blacklist(sanitized_whitelist) # We need to remove pairs that are unknown - return list(sanitized_whitelist) + return sanitized_whitelist diff --git a/freqtrade/pairlist/PrecisionFilter.py b/freqtrade/pairlist/PrecisionFilter.py new file mode 100644 index 000000000..d7b2c96ae --- /dev/null +++ b/freqtrade/pairlist/PrecisionFilter.py @@ -0,0 +1,62 @@ +import logging +from copy import deepcopy +from typing import Dict, List + +from freqtrade.pairlist.IPairList import IPairList + +logger = logging.getLogger(__name__) + + +class PrecisionFilter(IPairList): + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requries tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return True + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return f"{self.name} - Filtering untradable pairs." + + def _validate_precision_filter(self, ticker: dict, stoploss: float) -> bool: + """ + Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very + low value pairs. + :param ticker: ticker dict as returned from ccxt.load_markets() + :param stoploss: stoploss value as set in the configuration + (already cleaned to be 1 - stoploss) + :return: True if the pair can stay, false if it should be removed + """ + stop_price = ticker['ask'] * stoploss + # Adjust stop-prices to precision + sp = self._exchange.symbol_price_prec(ticker["symbol"], stop_price) + stop_gap_price = self._exchange.symbol_price_prec(ticker["symbol"], stop_price * 0.99) + logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}") + if sp <= stop_gap_price: + logger.info(f"Removed {ticker['symbol']} from whitelist, " + f"because stop price {sp} would be <= stop limit {stop_gap_price}") + return False + return True + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Filters and sorts pairlists and assigns and returns them again. + """ + if self._config.get('stoploss') is not None: + # Precalculate sanitized stoploss value to avoid recalculation for every pair + stoploss = 1 - abs(self._config.get('stoploss')) + # Copy list since we're modifying this list + for p in deepcopy(pairlist): + ticker = tickers.get(p) + # Filter out assets which would not allow setting a stoploss + if not ticker or (stoploss and not self._validate_precision_filter(ticker, stoploss)): + pairlist.remove(p) + continue + + return pairlist diff --git a/freqtrade/pairlist/PriceFilter.py b/freqtrade/pairlist/PriceFilter.py new file mode 100644 index 000000000..b3546ebd9 --- /dev/null +++ b/freqtrade/pairlist/PriceFilter.py @@ -0,0 +1,69 @@ +import logging +from copy import deepcopy +from typing import Dict, List + +from freqtrade.pairlist.IPairList import IPairList + +logger = logging.getLogger(__name__) + + +class PriceFilter(IPairList): + + def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict, + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._low_price_ratio = pairlistconfig.get('low_price_ratio', 0) + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requries tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return True + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return f"{self.name} - Filtering pairs priced below {self._low_price_ratio * 100}%." + + def _validate_ticker_lowprice(self, ticker) -> bool: + """ + Check if if one price-step (pip) is > than a certain barrier. + :param ticker: ticker dict as returned from ccxt.load_markets() + :param precision: Precision + :return: True if the pair can stay, false if it should be removed + """ + precision = self._exchange.markets[ticker['symbol']]['precision']['price'] + + compare = ticker['last'] + 1 / pow(10, precision) + changeperc = (compare - ticker['last']) / ticker['last'] + if changeperc > self._low_price_ratio: + logger.info(f"Removed {ticker['symbol']} from whitelist, " + f"because 1 unit is {changeperc * 100:.3f}%") + return False + return True + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + + """ + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist + """ + # Copy list since we're modifying this list + for p in deepcopy(pairlist): + ticker = tickers.get(p) + if not ticker: + pairlist.remove(p) + + # Filter out assets which would not allow setting a stoploss + if self._low_price_ratio and not self._validate_ticker_lowprice(ticker): + pairlist.remove(p) + + return pairlist diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 5896e814a..0050fbd5c 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -5,6 +5,7 @@ Provides lists as configured in config.json """ import logging +from typing import Dict, List from freqtrade.pairlist.IPairList import IPairList @@ -13,18 +14,28 @@ logger = logging.getLogger(__name__) class StaticPairList(IPairList): - def __init__(self, freqtrade, config: dict) -> None: - super().__init__(freqtrade, config) + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requries tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return False def short_desc(self) -> str: """ Short whitelist method description - used for startup-messages -> Please overwrite in subclasses """ - return f"{self.name}: {self.whitelist}" + return f"{self.name}" - def refresh_pairlist(self) -> None: + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ - Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist """ - self._whitelist = self._validate_whitelist(self._config['exchange']['pair_whitelist']) + return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist']) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 5f53cd17b..2df9ba691 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -5,11 +5,12 @@ Provides lists as configured in config.json """ import logging -from typing import List -from cachetools import TTLCache, cached +from datetime import datetime +from typing import Dict, List -from freqtrade.pairlist.IPairList import IPairList from freqtrade import OperationalException +from freqtrade.pairlist.IPairList import IPairList + logger = logging.getLogger(__name__) SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] @@ -17,18 +18,19 @@ SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] class VolumePairList(IPairList): - def __init__(self, freqtrade, config: dict) -> None: - super().__init__(freqtrade, config) - self._whitelistconf = self._config.get('pairlist', {}).get('config') - if 'number_assets' not in self._whitelistconf: + def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict, + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + if 'number_assets' not in self._pairlistconfig: raise OperationalException( f'`number_assets` not specified. Please check your configuration ' 'for "pairlist.config.number_assets"') - self._number_pairs = self._whitelistconf['number_assets'] - self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume') - self._precision_filter = self._whitelistconf.get('precision_filter', False) + self._number_pairs = self._pairlistconfig['number_assets'] + self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume') + self.refresh_period = self._pairlistconfig.get('refresh_period', 1800) - if not self._freqtrade.exchange.exchange_has('fetchTickers'): + if not self._exchange.exchange_has('fetchTickers'): raise OperationalException( 'Exchange does not support dynamic whitelist.' 'Please edit your config and restart the bot' @@ -36,6 +38,16 @@ class VolumePairList(IPairList): if not self._validate_keys(self._sort_key): raise OperationalException( f'key {self._sort_key} not in {SORT_VALUES}') + self._last_refresh = 0 + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requries tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return True def _validate_keys(self, key): return key in SORT_VALUES @@ -43,54 +55,54 @@ class VolumePairList(IPairList): def short_desc(self) -> str: """ Short whitelist method description - used for startup-messages - -> Please overwrite in subclasses """ - return f"{self.name} - top {self._whitelistconf['number_assets']} volume pairs." + return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs." - def refresh_pairlist(self) -> None: + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ - Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively - -> Please overwrite in subclasses + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist """ # Generate dynamic whitelist - self._whitelist = self._gen_pair_whitelist( - self._config['stake_currency'], self._sort_key) + if self._last_refresh + self.refresh_period < datetime.now().timestamp(): + self._last_refresh = int(datetime.now().timestamp()) + return self._gen_pair_whitelist(pairlist, + tickers, + self._config['stake_currency'], + self._sort_key, + ) + else: + return pairlist - @cached(TTLCache(maxsize=1, ttl=1800)) - def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]: + def _gen_pair_whitelist(self, pairlist, tickers, base_currency: str, key: str) -> List[str]: """ Updates the whitelist with with a dynamically generated list :param base_currency: base currency as str :param key: sort key (defaults to 'quoteVolume') + :param tickers: Tickers (from exchange.get_tickers()). :return: List of pairs """ - tickers = self._freqtrade.exchange.get_tickers() - # check length so that we make sure that '/' is actually in the string - tickers = [v for k, v in tickers.items() - if (len(k.split('/')) == 2 and k.split('/')[1] == base_currency - and v[key] is not None)] - sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) + if self._pairlist_pos == 0: + # If VolumePairList is the first in the list, use fresh pairlist + # check length so that we make sure that '/' is actually in the string + filtered_tickers = [v for k, v in tickers.items() + if (len(k.split('/')) == 2 and k.split('/')[1] == base_currency + and v[key] is not None)] + else: + # If other pairlist is in front, use the incomming pairlist. + filtered_tickers = [v for k, v in tickers.items() if k in pairlist] + + sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[key]) + # Validate whitelist to only have active market pairs - valid_pairs = self._validate_whitelist([s['symbol'] for s in sorted_tickers]) - valid_tickers = [t for t in sorted_tickers if t["symbol"] in valid_pairs] - - if self._freqtrade.strategy.stoploss is not None and self._precision_filter: - - stop_prices = [self._freqtrade.get_target_bid(t["symbol"], t) - * (1 - abs(self._freqtrade.strategy.stoploss)) for t in valid_tickers] - rates = [sp * 0.99 for sp in stop_prices] - logger.debug("\n".join([f"{sp} : {r}" for sp, r in zip(stop_prices[:10], rates[:10])])) - for i, t in enumerate(valid_tickers): - sp = self._freqtrade.exchange.symbol_price_prec(t["symbol"], stop_prices[i]) - r = self._freqtrade.exchange.symbol_price_prec(t["symbol"], rates[i]) - logger.debug(f"{t['symbol']} - {sp} : {r}") - if sp <= r: - logger.info(f"Removed {t['symbol']} from whitelist, " - f"because stop price {sp} would be <= stop limit {r}") - valid_tickers.remove(t) - - pairs = [s['symbol'] for s in valid_tickers] - logger.info(f"Searching pairs: {pairs[:self._number_pairs]}") + pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers]) + pairs = self._verify_blacklist(pairs) + # Limit to X number of pairs + pairs = pairs[:self._number_pairs] + logger.info(f"Searching {self._number_pairs} pairs: {pairs}") return pairs diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py new file mode 100644 index 000000000..fa5382c37 --- /dev/null +++ b/freqtrade/pairlist/pairlistmanager.py @@ -0,0 +1,95 @@ +""" +Static List provider + +Provides lists as configured in config.json + + """ +from cachetools import TTLCache, cached +import logging +from typing import Dict, List + +from freqtrade import OperationalException +from freqtrade.pairlist.IPairList import IPairList +from freqtrade.resolvers import PairListResolver + +logger = logging.getLogger(__name__) + + +class PairListManager(): + + def __init__(self, exchange, config: dict) -> None: + self._exchange = exchange + self._config = config + self._whitelist = self._config['exchange'].get('pair_whitelist') + self._blacklist = self._config['exchange'].get('pair_blacklist', []) + self._pairlists: List[IPairList] = [] + self._tickers_needed = False + for pl in self._config.get('pairlists', None): + if 'method' not in pl: + logger.warning(f"No method in {pl}") + continue + pairl = PairListResolver(pl.get('method'), + exchange=exchange, + pairlistmanager=self, + config=config, + pairlistconfig=pl, + pairlist_pos=len(self._pairlists) + ).pairlist + self._tickers_needed = pairl.needstickers or self._tickers_needed + self._pairlists.append(pairl) + + if not self._pairlists: + raise OperationalException("No Pairlist defined!") + + @property + def whitelist(self) -> List[str]: + """ + Has the current whitelist + """ + return self._whitelist + + @property + def blacklist(self) -> List[str]: + """ + Has the current blacklist + -> no need to overwrite in subclasses + """ + return self._blacklist + + @property + def name_list(self) -> List[str]: + """ + Get list of loaded pairlists names + """ + return [p.name for p in self._pairlists] + + def short_desc(self) -> List[Dict]: + """ + List of short_desc for each pairlist + """ + return [{p.name: p.short_desc()} for p in self._pairlists] + + @cached(TTLCache(maxsize=1, ttl=1800)) + def _get_cached_tickers(self): + return self._exchange.get_tickers() + + def refresh_pairlist(self) -> None: + """ + Run pairlist through all configured pairlists. + """ + + pairlist = self._whitelist.copy() + + # tickers should be cached to avoid calling the exchange on each call. + tickers: Dict = {} + if self._tickers_needed: + tickers = self._get_cached_tickers() + + # Process all pairlists in chain + for pl in self._pairlists: + pairlist = pl.filter_pairlist(pairlist, tickers) + + # Validation against blacklist happens after the pairlists to ensure blacklist is respected. + pairlist = IPairList.verify_blacklist(pairlist, self.blacklist) + + self._whitelist = pairlist diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 51c4f7dba..3bad42fd9 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -17,13 +17,13 @@ class IResolver: This class contains all the logic to load custom classes """ - def build_search_paths(self, config, current_path: Path, user_subdir: str, + def build_search_paths(self, config, current_path: Path, user_subdir: Optional[str] = None, extra_dir: Optional[str] = None) -> List[Path]: - abs_paths = [ - config['user_data_dir'].joinpath(user_subdir), - current_path, - ] + abs_paths: List[Path] = [current_path] + + if user_subdir: + abs_paths.insert(0, config['user_data_dir'].joinpath(user_subdir)) if extra_dir: # Add extra directory to the top of the search paths diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index c2782a219..d849f4ffb 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -20,13 +20,18 @@ class PairListResolver(IResolver): __slots__ = ['pairlist'] - def __init__(self, pairlist_name: str, freqtrade, config: dict) -> None: + def __init__(self, pairlist_name: str, exchange, pairlistmanager, + config: dict, pairlistconfig: dict, pairlist_pos: int) -> None: """ Load the custom class from config parameter :param config: configuration dictionary or None """ - self.pairlist = self._load_pairlist(pairlist_name, config, kwargs={'freqtrade': freqtrade, - 'config': config}) + self.pairlist = self._load_pairlist(pairlist_name, config, + kwargs={'exchange': exchange, + 'pairlistmanager': pairlistmanager, + 'config': config, + 'pairlistconfig': pairlistconfig, + 'pairlist_pos': pairlist_pos}) def _load_pairlist( self, pairlist_name: str, config: dict, kwargs: dict) -> IPairList: @@ -40,7 +45,7 @@ class PairListResolver(IResolver): current_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() abs_paths = self.build_search_paths(config, current_path=current_path, - user_subdir='pairlist', extra_dir=None) + user_subdir=None, extra_dir=None) pairlist = self._load_object(paths=abs_paths, object_type=IPairList, object_name=pairlist_name, kwargs=kwargs) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 5ab92cf33..19bc47eb0 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -462,7 +462,7 @@ class RPC: def _rpc_whitelist(self) -> Dict: """ Returns the currently active whitelist""" - res = {'method': self._freqtrade.pairlists.name, + res = {'method': self._freqtrade.pairlists.name_list, 'length': len(self._freqtrade.active_pair_whitelist), 'whitelist': self._freqtrade.active_pair_whitelist } @@ -477,7 +477,7 @@ class RPC: and pair not in self._freqtrade.pairlists.blacklist): self._freqtrade.pairlists.blacklist.append(pair) - res = {'method': self._freqtrade.pairlists.name, + res = {'method': self._freqtrade.pairlists.name_list, 'length': len(self._freqtrade.pairlists.blacklist), 'blacklist': self._freqtrade.pairlists.blacklist, } diff --git a/tests/conftest.py b/tests/conftest.py index c5cfd702b..fbd23a0dc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -242,6 +242,9 @@ def default_conf(testdatadir): "HOT/BTC", ] }, + "pairlists": [ + {"method": "StaticPairList"} + ], "telegram": { "enabled": True, "token": "token", @@ -573,6 +576,72 @@ def get_markets(): } +@pytest.fixture +def shitcoinmarkets(markets): + """ + Fixture with shitcoin markets - used to test filters in pairlists + """ + shitmarkets = deepcopy(markets) + shitmarkets.update({'HOT/BTC': { + 'id': 'HOTBTC', + 'symbol': 'HOT/BTC', + 'base': 'HOT', + 'quote': 'BTC', + 'active': True, + 'precision': { + 'base': 8, + 'quote': 8, + 'amount': 0, + 'price': 8 + }, + 'limits': { + 'amount': { + 'min': 1.0, + 'max': 90000000.0 + }, + 'price': { + 'min': None, + 'max': None + }, + 'cost': { + 'min': 0.001, + 'max': None + } + }, + 'info': {}, + }, + 'FUEL/BTC': { + 'id': 'FUELBTC', + 'symbol': 'FUEL/BTC', + 'base': 'FUEL', + 'quote': 'BTC', + 'active': True, + 'precision': { + 'base': 8, + 'quote': 8, + 'amount': 0, + 'price': 8 + }, + 'limits': { + 'amount': { + 'min': 1.0, + 'max': 90000000.0 + }, + 'price': { + 'min': 1e-08, + 'max': 1000.0 + }, + 'cost': { + 'min': 0.001, + 'max': None + } + }, + 'info': {}, + }, + }) + return shitmarkets + + @pytest.fixture def markets_empty(): return MagicMock(return_value=[]) @@ -867,6 +936,50 @@ def tickers(): 'quoteVolume': 1215.14489611, 'info': {} }, + 'HOT/BTC': { + 'symbol': 'HOT/BTC', + 'timestamp': 1572273518661, + 'datetime': '2019-10-28T14:38:38.661Z', + 'high': 0.00000011, + 'low': 0.00000009, + 'bid': 0.0000001, + 'bidVolume': 1476027288.0, + 'ask': 0.00000011, + 'askVolume': 820153831.0, + 'vwap': 0.0000001, + 'open': 0.00000009, + 'close': 0.00000011, + 'last': 0.00000011, + 'previousClose': 0.00000009, + 'change': 0.00000002, + 'percentage': 22.222, + 'average': None, + 'baseVolume': 1442290324.0, + 'quoteVolume': 143.78311994, + 'info': {} + }, + 'FUEL/BTC': { + 'symbol': 'FUEL/BTC', + 'timestamp': 1572340250771, + 'datetime': '2019-10-29T09:10:50.771Z', + 'high': 0.00000040, + 'low': 0.00000035, + 'bid': 0.00000036, + 'bidVolume': 8932318.0, + 'ask': 0.00000037, + 'askVolume': 10140774.0, + 'vwap': 0.00000037, + 'open': 0.00000039, + 'close': 0.00000037, + 'last': 0.00000037, + 'previousClose': 0.00000038, + 'change': -0.00000002, + 'percentage': -5.128, + 'average': None, + 'baseVolume': 168927742.0, + 'quoteVolume': 62.68220262, + 'info': {} + }, 'ETH/USDT': { 'symbol': 'ETH/USDT', 'timestamp': 1522014804118, diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 929fc0ba0..76537880c 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -2,11 +2,13 @@ from unittest.mock import MagicMock, PropertyMock +import pytest + from freqtrade import OperationalException from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.resolvers import PairListResolver -from tests.conftest import get_patched_freqtradebot -import pytest +from freqtrade.pairlist.pairlistmanager import PairListManager +from tests.conftest import get_patched_freqtradebot, log_has_re # whitelist, blacklist @@ -24,25 +26,39 @@ def whitelist_conf(default_conf): default_conf['exchange']['pair_blacklist'] = [ 'BLK/BTC' ] - default_conf['pairlist'] = {'method': 'StaticPairList', - 'config': {'number_assets': 3} - } - + default_conf['pairlists'] = [ + { + "method": "VolumePairList", + "number_assets": 5, + "sort_key": "quoteVolume", + }, + ] return default_conf +@pytest.fixture(scope="function") +def static_pl_conf(whitelist_conf): + whitelist_conf['pairlists'] = [ + { + "method": "StaticPairList", + }, + ] + return whitelist_conf + + def test_load_pairlist_noexist(mocker, markets, default_conf): - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + bot = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) + plm = PairListManager(bot.exchange, default_conf) with pytest.raises(OperationalException, match=r"Impossible to load Pairlist 'NonexistingPairList'. " r"This class does not exist or contains Python code errors."): - PairListResolver('NonexistingPairList', freqtradebot, default_conf).pairlist + PairListResolver('NonexistingPairList', bot.exchange, plm, default_conf, {}, 1) -def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf): +def test_refresh_market_pair_not_in_whitelist(mocker, markets, static_pl_conf): - freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) + freqtradebot = get_patched_freqtradebot(mocker, static_pl_conf) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) freqtradebot.pairlists.refresh_pairlist() @@ -51,50 +67,60 @@ def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf): # Ensure all except those in whitelist are removed assert set(whitelist) == set(freqtradebot.pairlists.whitelist) # Ensure config dict hasn't been changed - assert (whitelist_conf['exchange']['pair_whitelist'] == + assert (static_pl_conf['exchange']['pair_whitelist'] == freqtradebot.config['exchange']['pair_whitelist']) -def test_refresh_pairlists(mocker, markets, whitelist_conf): - freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) - - mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) +def test_refresh_static_pairlist(mocker, markets, static_pl_conf): + freqtradebot = get_patched_freqtradebot(mocker, static_pl_conf) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + markets=PropertyMock(return_value=markets), + ) freqtradebot.pairlists.refresh_pairlist() # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed assert set(whitelist) == set(freqtradebot.pairlists.whitelist) - assert whitelist_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist + assert static_pl_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist -def test_refresh_pairlist_dynamic(mocker, markets, tickers, whitelist_conf): - whitelist_conf['pairlist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 5} - } +def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_conf): + mocker.patch.multiple( 'freqtrade.exchange.Exchange', - markets=PropertyMock(return_value=markets), get_tickers=tickers, - exchange_has=MagicMock(return_value=True) + exchange_has=MagicMock(return_value=True), ) - freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) - + bot = get_patched_freqtradebot(mocker, whitelist_conf) + # Remock markets with shitcoinmarkets since get_patched_freqtradebot uses the markets fixture + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=shitcoinmarkets), + ) # argument: use the whitelist dynamically by exchange-volume - whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'] - freqtradebot.pairlists.refresh_pairlist() + whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC'] + bot.pairlists.refresh_pairlist() - assert whitelist == freqtradebot.pairlists.whitelist + assert whitelist == bot.pairlists.whitelist + + whitelist_conf['pairlists'] = [{'method': 'VolumePairList', + 'config': {} + } + ] - whitelist_conf['pairlist'] = {'method': 'VolumePairList', - 'config': {} - } with pytest.raises(OperationalException, match=r'`number_assets` not specified. Please check your configuration ' r'for "pairlist.config.number_assets"'): - PairListResolver('VolumePairList', freqtradebot, whitelist_conf).pairlist + PairListManager(bot.exchange, whitelist_conf) def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + ) freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets_empty)) @@ -107,35 +133,75 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): assert set(whitelist) == set(pairslist) -@pytest.mark.parametrize("precision_filter,base_currency,key,whitelist_result", [ - (False, "BTC", "quoteVolume", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), - (False, "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC']), - (False, "USDT", "quoteVolume", ['ETH/USDT']), - (False, "ETH", "quoteVolume", []), # this replaces tests that were removed from test_exchange - (True, "BTC", "quoteVolume", ["LTC/BTC", "ETH/BTC", "TKN/BTC"]), - (True, "BTC", "bidVolume", ["LTC/BTC", "TKN/BTC", "ETH/BTC"]) +@pytest.mark.parametrize("pairlists,base_currency,whitelist_result", [ + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']), + # Different sorting depending on quote or bid volume + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}], + "BTC", ['HOT/BTC', 'FUEL/BTC', 'LTC/BTC', 'TKN/BTC', 'ETH/BTC']), + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], + "USDT", ['ETH/USDT']), + # No pair for ETH ... + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], + "ETH", []), + # Precisionfilter and quote volume + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "PrecisionFilter"}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'FUEL/BTC']), + # Precisionfilter bid + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}, + {"method": "PrecisionFilter"}], "BTC", ['FUEL/BTC', 'LTC/BTC', 'TKN/BTC', 'ETH/BTC']), + # PriceFilter and VolumePairList + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "PriceFilter", "low_price_ratio": 0.03}], + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'FUEL/BTC']), + # Hot is removed by precision_filter, Fuel by low_price_filter. + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "PrecisionFilter"}, + {"method": "PriceFilter", "low_price_ratio": 0.02} + ], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), + # StaticPairlist Only + ([{"method": "StaticPairList"}, + ], "BTC", ['ETH/BTC', 'TKN/BTC']), + # Static Pairlist before VolumePairList - sorting changes + ([{"method": "StaticPairList"}, + {"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}, + ], "BTC", ['TKN/BTC', 'ETH/BTC']), ]) -def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers, base_currency, key, - whitelist_result, precision_filter) -> None: - whitelist_conf['pairlist']['method'] = 'VolumePairList' +def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, + pairlists, base_currency, whitelist_result, + caplog) -> None: + whitelist_conf['pairlists'] = pairlists + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) - mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) - mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) - mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, p, r: round(r, 8)) - freqtrade.pairlists._precision_filter = precision_filter + mocker.patch.multiple('freqtrade.exchange.Exchange', + get_tickers=tickers, + markets=PropertyMock(return_value=shitcoinmarkets), + ) + freqtrade.config['stake_currency'] = base_currency - whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency=base_currency, key=key) - assert sorted(whitelist) == sorted(whitelist_result) + freqtrade.pairlists.refresh_pairlist() + whitelist = freqtrade.pairlists.whitelist + + assert whitelist == whitelist_result + for pairlist in pairlists: + if pairlist['method'] == 'PrecisionFilter': + assert log_has_re(r'^Removed .* from whitelist, because stop price .* ' + r'would be <= stop limit.*', caplog) + if pairlist['method'] == 'PriceFilter': + assert log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: - default_conf['pairlist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 10} - } - mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) - mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) + default_conf['pairlists'] = [{'method': 'VolumePairList', + 'config': {'number_assets': 10} + }] + + mocker.patch.multiple('freqtrade.exchange.Exchange', + get_tickers=tickers, + exchange_has=MagicMock(return_value=False), + ) with pytest.raises(OperationalException): get_patched_freqtradebot(mocker, default_conf) @@ -143,13 +209,15 @@ def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None @pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS) def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): - whitelist_conf['pairlist']['method'] = pairlist - mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) - mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + whitelist_conf['pairlists'][0]['method'] = pairlist + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True) + ) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) - assert freqtrade.pairlists.name == pairlist - assert pairlist in freqtrade.pairlists.short_desc() + assert freqtrade.pairlists.name_list == [pairlist] + assert pairlist in str(freqtrade.pairlists.short_desc()) assert isinstance(freqtrade.pairlists.whitelist, list) assert isinstance(freqtrade.pairlists.blacklist, list) @@ -157,20 +225,75 @@ def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): @pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS) @pytest.mark.parametrize("whitelist,log_message", [ (['ETH/BTC', 'TKN/BTC'], ""), - (['ETH/BTC', 'TKN/BTC', 'TRX/ETH'], "is not compatible with exchange"), # TRX/ETH wrong stake - (['ETH/BTC', 'TKN/BTC', 'BCH/BTC'], "is not compatible with exchange"), # BCH/BTC not available - (['ETH/BTC', 'TKN/BTC', 'BLK/BTC'], "is not compatible with exchange"), # BLK/BTC in blacklist - (['ETH/BTC', 'TKN/BTC', 'BTT/BTC'], "Market is not active") # BTT/BTC is inactive + # TRX/ETH not in markets + (['ETH/BTC', 'TKN/BTC', 'TRX/ETH'], "is not compatible with exchange"), + # wrong stake + (['ETH/BTC', 'TKN/BTC', 'ETH/USDT'], "is not compatible with your stake currency"), + # BCH/BTC not available + (['ETH/BTC', 'TKN/BTC', 'BCH/BTC'], "is not compatible with exchange"), + # BLK/BTC in blacklist + (['ETH/BTC', 'TKN/BTC', 'BLK/BTC'], "in your blacklist. Removing "), + # BTT/BTC is inactive + (['ETH/BTC', 'TKN/BTC', 'BTT/BTC'], "Market is not active") ]) -def test_validate_whitelist(mocker, whitelist_conf, markets, pairlist, whitelist, caplog, - log_message): - whitelist_conf['pairlist']['method'] = pairlist - mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) - mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) +def test__whitelist_for_active_markets(mocker, whitelist_conf, markets, pairlist, whitelist, caplog, + log_message, tickers): + whitelist_conf['pairlists'][0]['method'] = pairlist + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) caplog.clear() - new_whitelist = freqtrade.pairlists._validate_whitelist(whitelist) + # Assign starting whitelist + new_whitelist = freqtrade.pairlists._pairlists[0]._whitelist_for_active_markets(whitelist) assert set(new_whitelist) == set(['ETH/BTC', 'TKN/BTC']) assert log_message in caplog.text + + +def test_volumepairlist_invalid_sortvalue(mocker, markets, whitelist_conf): + whitelist_conf['pairlists'][0].update({"sort_key": "asdf"}) + + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + with pytest.raises(OperationalException, + match=r"key asdf not in .*"): + get_patched_freqtradebot(mocker, whitelist_conf) + + +def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers): + + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + bot = get_patched_freqtradebot(mocker, whitelist_conf) + assert bot.pairlists._pairlists[0]._last_refresh == 0 + assert tickers.call_count == 0 + bot.pairlists.refresh_pairlist() + assert tickers.call_count == 1 + + assert bot.pairlists._pairlists[0]._last_refresh != 0 + lrf = bot.pairlists._pairlists[0]._last_refresh + bot.pairlists.refresh_pairlist() + assert tickers.call_count == 1 + # Time should not be updated. + assert bot.pairlists._pairlists[0]._last_refresh == lrf + + +def test_pairlistmanager_no_pairlist(mocker, markets, whitelist_conf, caplog): + del whitelist_conf['pairlists'][0]['method'] + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + with pytest.raises(OperationalException, + match=r"No Pairlist defined!"): + get_patched_freqtradebot(mocker, whitelist_conf) + assert log_has_re("No method in .*", caplog) + + whitelist_conf['pairlists'] = [] + + with pytest.raises(OperationalException, + match=r"No Pairlist defined!"): + get_patched_freqtradebot(mocker, whitelist_conf) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index e7f41f6e7..fb7a5276a 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -736,21 +736,23 @@ def test_rpc_whitelist(mocker, default_conf) -> None: freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(freqtradebot) ret = rpc._rpc_whitelist() - assert ret['method'] == 'StaticPairList' + assert len(ret['method']) == 1 + assert 'StaticPairList' in ret['method'] assert ret['whitelist'] == default_conf['exchange']['pair_whitelist'] def test_rpc_whitelist_dynamic(mocker, default_conf) -> None: - default_conf['pairlist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 4} - } + default_conf['pairlists'] = [{'method': 'VolumePairList', + 'number_assets': 4, + }] mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(freqtradebot) ret = rpc._rpc_whitelist() - assert ret['method'] == 'VolumePairList' + assert len(ret['method']) == 1 + assert 'VolumePairList' in ret['method'] assert ret['length'] == 4 assert ret['whitelist'] == default_conf['exchange']['pair_whitelist'] @@ -761,13 +763,14 @@ def test_rpc_blacklist(mocker, default_conf) -> None: freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(freqtradebot) ret = rpc._rpc_blacklist(None) - assert ret['method'] == 'StaticPairList' + assert len(ret['method']) == 1 + assert 'StaticPairList' in ret['method'] assert len(ret['blacklist']) == 2 assert ret['blacklist'] == default_conf['exchange']['pair_blacklist'] assert ret['blacklist'] == ['DOGE/BTC', 'HOT/BTC'] ret = rpc._rpc_blacklist(["ETH/BTC"]) - assert ret['method'] == 'StaticPairList' + assert 'StaticPairList' in ret['method'] assert len(ret['blacklist']) == 3 assert ret['blacklist'] == default_conf['exchange']['pair_blacklist'] assert ret['blacklist'] == ['DOGE/BTC', 'HOT/BTC', 'ETH/BTC'] diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 6e65cf934..94b57a09f 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -460,7 +460,7 @@ def test_api_blacklist(botclient, mocker): assert_response(rc) assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC"], "length": 2, - "method": "StaticPairList"} + "method": ["StaticPairList"]} # Add ETH/BTC to blacklist rc = client_post(client, f"{BASE_URI}/blacklist", @@ -468,7 +468,7 @@ def test_api_blacklist(botclient, mocker): assert_response(rc) assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC"], "length": 3, - "method": "StaticPairList"} + "method": ["StaticPairList"]} def test_api_whitelist(botclient): @@ -478,7 +478,7 @@ def test_api_whitelist(botclient): assert_response(rc) assert rc.json == {"whitelist": ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC'], "length": 4, - "method": "StaticPairList"} + "method": ["StaticPairList"]} def test_api_forcebuy(botclient, mocker, fee): diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 766511d2d..52d58e887 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1050,8 +1050,8 @@ def test_whitelist_static(default_conf, update, mocker) -> None: telegram._whitelist(update=update, context=MagicMock()) assert msg_mock.call_count == 1 - assert ('Using whitelist `StaticPairList` with 4 pairs\n`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`' - in msg_mock.call_args_list[0][0][0]) + assert ("Using whitelist `['StaticPairList']` with 4 pairs\n" + "`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`" in msg_mock.call_args_list[0][0][0]) def test_whitelist_dynamic(default_conf, update, mocker) -> None: @@ -1062,17 +1062,17 @@ def test_whitelist_dynamic(default_conf, update, mocker) -> None: _send_msg=msg_mock ) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) - default_conf['pairlist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 4} - } + default_conf['pairlists'] = [{'method': 'VolumePairList', + 'number_assets': 4 + }] freqtradebot = get_patched_freqtradebot(mocker, default_conf) telegram = Telegram(freqtradebot) telegram._whitelist(update=update, context=MagicMock()) assert msg_mock.call_count == 1 - assert ('Using whitelist `VolumePairList` with 4 pairs\n`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`' - in msg_mock.call_args_list[0][0][0]) + assert ("Using whitelist `['VolumePairList']` with 4 pairs\n" + "`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`" in msg_mock.call_args_list[0][0][0]) def test_blacklist_static(default_conf, update, mocker) -> None: diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 7959f28d1..240b7c784 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -777,9 +777,9 @@ def test_validate_whitelist(default_conf): conf = deepcopy(default_conf) - conf.update({"pairlist": { + conf.update({"pairlists": [{ "method": "VolumePairList", - }}) + }]}) # Dynamic whitelist should not care about pair_whitelist validate_config_consistency(conf) del conf['exchange']['pair_whitelist'] @@ -997,6 +997,18 @@ def test_process_temporary_deprecated_settings(mocker, default_conf, setting, ca assert default_conf[setting[0]][setting[1]] == setting[5] +def test_process_deprecated_setting_pairlists(mocker, default_conf, caplog): + patched_configuration_load_config_file(mocker, default_conf) + default_conf.update({'pairlist': { + 'method': 'VolumePairList', + 'config': {'precision_filter': True} + }}) + + process_temporary_deprecated_settings(default_conf) + assert log_has_re(r'DEPRECATED.*precision_filter.*', caplog) + assert log_has_re(r'DEPRECATED.*in pairlist is deprecated and must be moved*', caplog) + + def test_check_conflicting_settings(mocker, default_conf, caplog): patched_configuration_load_config_file(mocker, default_conf)