From a93592c46738c02da2399f719dbd001aa1954880 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Wed, 13 Sep 2023 14:17:43 +0900 Subject: [PATCH 01/12] add fulltradespairlist --- freqtrade/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 318c414e1..897358520 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -33,7 +33,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss', 'ProfitDrawDownHyperOptLoss'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', 'RemotePairList', - 'AgeFilter', 'OffsetFilter', 'PerformanceFilter', + 'AgeFilter', "FullTradesFilter", 'OffsetFilter', 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter'] AVAILABLE_PROTECTIONS = ['CooldownPeriod', From e0a06ca454dcb84ecbd8dbec77ef131168536451 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Wed, 13 Sep 2023 14:18:07 +0900 Subject: [PATCH 02/12] add fulltradesfilter --- .../plugins/pairlist/FullTradesFilter.py | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 freqtrade/plugins/pairlist/FullTradesFilter.py diff --git a/freqtrade/plugins/pairlist/FullTradesFilter.py b/freqtrade/plugins/pairlist/FullTradesFilter.py new file mode 100644 index 000000000..9365abf02 --- /dev/null +++ b/freqtrade/plugins/pairlist/FullTradesFilter.py @@ -0,0 +1,78 @@ +""" +Performance pair list filter +""" +import logging +from typing import Any, Dict, List + +import pandas as pd + +from freqtrade.constants import Config +from freqtrade.exchange.types import Tickers +from freqtrade.persistence import Trade +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter + + +logger = logging.getLogger(__name__) + + +class FullTradesFilter(IPairList): + + def __init__(self, exchange, pairlistmanager, + config: Config, pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._minutes = pairlistconfig.get('minutes', 0) + self._min_profit = pairlistconfig.get('min_profit') + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return False + + def short_desc(self) -> str: + """ + Short allowlist method description - used for startup-messages + """ + return f"{self.name} - Emptying whitelist when trade slots are full." + + @staticmethod + def description() -> str: + return "Emptying whitelist when trade slots are full." + + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + + } + + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: + """ + Filters and sorts pairlist and returns the allowlist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers). May be cached. + :return: new allowlist + """ + # Get the trading performance for pairs from database + try: + trades = Trade.get_trades(Trade.is_open.is_(True)).all() + except AttributeError: + # Performancefilter does not work in backtesting. + self.log_once("PerformanceFilter is not available in this mode.", logger.warning) + return pairlist + + # Skip performance-based sorting if no performance data is available + num_open = len(trades) + if num_open == 0: + return pairlist + + max_trades = self._config['max_open_trades'] + + self.log_once(f"Max open trades: {max_trades}, current open trades: {num_open}") + + return pairlist From dd01ae880f78b5301ff2971fb8e9f5b324d89034 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Wed, 13 Sep 2023 14:38:16 +0900 Subject: [PATCH 03/12] fix error --- freqtrade/plugins/pairlist/FullTradesFilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/FullTradesFilter.py b/freqtrade/plugins/pairlist/FullTradesFilter.py index 9365abf02..f8233dcba 100644 --- a/freqtrade/plugins/pairlist/FullTradesFilter.py +++ b/freqtrade/plugins/pairlist/FullTradesFilter.py @@ -73,6 +73,6 @@ class FullTradesFilter(IPairList): max_trades = self._config['max_open_trades'] - self.log_once(f"Max open trades: {max_trades}, current open trades: {num_open}") + self.log_once(f"Max open trades: {max_trades}, current open trades: {num_open}", logger.info) return pairlist From 25ae25248c7286d26bec2a225abb8912af29cea2 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Wed, 13 Sep 2023 14:42:43 +0900 Subject: [PATCH 04/12] add logger --- freqtrade/plugins/pairlist/FullTradesFilter.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/plugins/pairlist/FullTradesFilter.py b/freqtrade/plugins/pairlist/FullTradesFilter.py index f8233dcba..7f3a79f02 100644 --- a/freqtrade/plugins/pairlist/FullTradesFilter.py +++ b/freqtrade/plugins/pairlist/FullTradesFilter.py @@ -75,4 +75,8 @@ class FullTradesFilter(IPairList): self.log_once(f"Max open trades: {max_trades}, current open trades: {num_open}", logger.info) + if (num_open >= max_trades): + logger.info('Slots full. Emptying pairlist!!') + return []; + return pairlist From 7655bf6ea7bd0ee28b55ae5bbda97879202e66d6 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Wed, 13 Sep 2023 14:53:33 +0900 Subject: [PATCH 05/12] turn off logger for now --- freqtrade/plugins/pairlist/FullTradesFilter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/plugins/pairlist/FullTradesFilter.py b/freqtrade/plugins/pairlist/FullTradesFilter.py index 7f3a79f02..ccc43c577 100644 --- a/freqtrade/plugins/pairlist/FullTradesFilter.py +++ b/freqtrade/plugins/pairlist/FullTradesFilter.py @@ -63,7 +63,7 @@ class FullTradesFilter(IPairList): trades = Trade.get_trades(Trade.is_open.is_(True)).all() except AttributeError: # Performancefilter does not work in backtesting. - self.log_once("PerformanceFilter is not available in this mode.", logger.warning) + self.log_once("FullTradesFilter is not available in this mode.", logger.warning) return pairlist # Skip performance-based sorting if no performance data is available @@ -73,10 +73,10 @@ class FullTradesFilter(IPairList): max_trades = self._config['max_open_trades'] - self.log_once(f"Max open trades: {max_trades}, current open trades: {num_open}", logger.info) + # self.log_once(f"Max open trades: {max_trades}, current open trades: {num_open}", logger.info) if (num_open >= max_trades): - logger.info('Slots full. Emptying pairlist!!') + # logger.info('Slots full. Emptying pairlist!!') return []; return pairlist From 552f947248c2d23e2504391b8ff85eb55cfdac1b Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Wed, 13 Sep 2023 15:09:13 +0900 Subject: [PATCH 06/12] remove unnecessary check --- freqtrade/plugins/pairlist/FullTradesFilter.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/freqtrade/plugins/pairlist/FullTradesFilter.py b/freqtrade/plugins/pairlist/FullTradesFilter.py index ccc43c577..479213db4 100644 --- a/freqtrade/plugins/pairlist/FullTradesFilter.py +++ b/freqtrade/plugins/pairlist/FullTradesFilter.py @@ -66,11 +66,7 @@ class FullTradesFilter(IPairList): self.log_once("FullTradesFilter is not available in this mode.", logger.warning) return pairlist - # Skip performance-based sorting if no performance data is available num_open = len(trades) - if num_open == 0: - return pairlist - max_trades = self._config['max_open_trades'] # self.log_once(f"Max open trades: {max_trades}, current open trades: {num_open}", logger.info) From 44ca6f1c4695d6a42717ea1f6574ec84ec6f0740 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Wed, 13 Sep 2023 17:07:45 +0900 Subject: [PATCH 07/12] remove unused vars and change wording --- freqtrade/plugins/pairlist/FullTradesFilter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/plugins/pairlist/FullTradesFilter.py b/freqtrade/plugins/pairlist/FullTradesFilter.py index 479213db4..77b755009 100644 --- a/freqtrade/plugins/pairlist/FullTradesFilter.py +++ b/freqtrade/plugins/pairlist/FullTradesFilter.py @@ -22,8 +22,8 @@ class FullTradesFilter(IPairList): pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) - self._minutes = pairlistconfig.get('minutes', 0) - self._min_profit = pairlistconfig.get('min_profit') + # self._minutes = pairlistconfig.get('minutes', 0) + # self._min_profit = pairlistconfig.get('min_profit') @property def needstickers(self) -> bool: @@ -38,11 +38,11 @@ class FullTradesFilter(IPairList): """ Short allowlist method description - used for startup-messages """ - return f"{self.name} - Emptying whitelist when trade slots are full." + return f"{self.name} - Shrink whitelist when trade slots are full." @staticmethod def description() -> str: - return "Emptying whitelist when trade slots are full." + return "Shrink whitelist when trade slots are full." @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: From 5b073854144a3376cdf25eeeb8ce5c50025fd2d8 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Thu, 14 Sep 2023 08:13:47 +0900 Subject: [PATCH 08/12] simplify the filter --- freqtrade/plugins/pairlist/FullTradesFilter.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/freqtrade/plugins/pairlist/FullTradesFilter.py b/freqtrade/plugins/pairlist/FullTradesFilter.py index 77b755009..f1578733a 100644 --- a/freqtrade/plugins/pairlist/FullTradesFilter.py +++ b/freqtrade/plugins/pairlist/FullTradesFilter.py @@ -59,14 +59,7 @@ class FullTradesFilter(IPairList): :return: new allowlist """ # Get the trading performance for pairs from database - try: - trades = Trade.get_trades(Trade.is_open.is_(True)).all() - except AttributeError: - # Performancefilter does not work in backtesting. - self.log_once("FullTradesFilter is not available in this mode.", logger.warning) - return pairlist - - num_open = len(trades) + num_open = Trade.get_open_trade_count() max_trades = self._config['max_open_trades'] # self.log_once(f"Max open trades: {max_trades}, current open trades: {num_open}", logger.info) From 383bdb7d56769e14aa92979aba36a22bfa62ba14 Mon Sep 17 00:00:00 2001 From: Stefano Date: Fri, 15 Sep 2023 10:15:19 +0900 Subject: [PATCH 09/12] flake8 fix --- freqtrade/plugins/pairlist/FullTradesFilter.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/freqtrade/plugins/pairlist/FullTradesFilter.py b/freqtrade/plugins/pairlist/FullTradesFilter.py index f1578733a..44a6074f2 100644 --- a/freqtrade/plugins/pairlist/FullTradesFilter.py +++ b/freqtrade/plugins/pairlist/FullTradesFilter.py @@ -4,8 +4,6 @@ Performance pair list filter import logging from typing import Any, Dict, List -import pandas as pd - from freqtrade.constants import Config from freqtrade.exchange.types import Tickers from freqtrade.persistence import Trade @@ -47,7 +45,7 @@ class FullTradesFilter(IPairList): @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { - + } def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: @@ -62,10 +60,7 @@ class FullTradesFilter(IPairList): num_open = Trade.get_open_trade_count() max_trades = self._config['max_open_trades'] - # self.log_once(f"Max open trades: {max_trades}, current open trades: {num_open}", logger.info) - if (num_open >= max_trades): - # logger.info('Slots full. Emptying pairlist!!') - return []; + return [] return pairlist From c19fe95d3914b9a6d28999be704f083811763cba Mon Sep 17 00:00:00 2001 From: Stefano Date: Fri, 15 Sep 2023 11:34:56 +0900 Subject: [PATCH 10/12] add the filter to docs, tyding up the py file --- config_examples/config_full.example.json | 1 + docs/includes/pairlists.md | 12 ++++++++++++ freqtrade/plugins/pairlist/FullTradesFilter.py | 13 ++----------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/config_examples/config_full.example.json b/config_examples/config_full.example.json index 4681ec7df..cb2d4797e 100644 --- a/config_examples/config_full.example.json +++ b/config_examples/config_full.example.json @@ -70,6 +70,7 @@ }, "pairlists": [ {"method": "StaticPairList"}, + {"method": "FullTradesFilter"}, { "method": "VolumePairList", "number_assets": 20, diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 9cdcc9bca..83c913ab4 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -25,6 +25,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged * [`ProducerPairList`](#producerpairlist) * [`RemotePairList`](#remotepairlist) * [`AgeFilter`](#agefilter) +* [`FullTradesFilter`](#fulltradesfilter) * [`OffsetFilter`](#offsetfilter) * [`PerformanceFilter`](#performancefilter) * [`PrecisionFilter`](#precisionfilter) @@ -236,6 +237,17 @@ be caught out buying before the pair has finished dropping in price. This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days and listed before `max_days_listed`. +#### FullTradesFilter + +Shrink whitelist to consist only in-trade pairs when the trade slots are full (when `max_open_trades` isn't being set to `-1` in the config). + +When the trade slots are full, there is no need to calculate indicators of the rest of the pairs (except informative pairs) since no new trade can be opened. By shrinking the whitelist to just the in-trade pairs, you can improve calculation speeds and reduce CPU usage. When a trade slot is free (either a trade is closed or `max_open_trades` value in config is increased), then the whitelist will return to normal state. + +When multiple pairlist filters are being used, it's recommended to put this filter at second position directly below the primary pairlist, so when the trade slots are full, the bot don't have to download data for the rest of the filters. + +!!! Warning "Backtesting" + `FullTradesFilter` does not support backtesting mode. + #### OffsetFilter Offsets an incoming pairlist by a given `offset` value. diff --git a/freqtrade/plugins/pairlist/FullTradesFilter.py b/freqtrade/plugins/pairlist/FullTradesFilter.py index 44a6074f2..376d0540e 100644 --- a/freqtrade/plugins/pairlist/FullTradesFilter.py +++ b/freqtrade/plugins/pairlist/FullTradesFilter.py @@ -1,5 +1,5 @@ """ -Performance pair list filter +Full trade slots pair list filter """ import logging from typing import Any, Dict, List @@ -20,9 +20,6 @@ class FullTradesFilter(IPairList): pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) - # self._minutes = pairlistconfig.get('minutes', 0) - # self._min_profit = pairlistconfig.get('min_profit') - @property def needstickers(self) -> bool: """ @@ -42,12 +39,6 @@ class FullTradesFilter(IPairList): def description() -> str: return "Shrink whitelist when trade slots are full." - @staticmethod - def available_parameters() -> Dict[str, PairlistParameter]: - return { - - } - def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Filters and sorts pairlist and returns the allowlist again. @@ -56,7 +47,7 @@ class FullTradesFilter(IPairList): :param tickers: Tickers (from exchange.get_tickers). May be cached. :return: new allowlist """ - # Get the trading performance for pairs from database + # Get the number of open trades and max open trades config num_open = Trade.get_open_trade_count() max_trades = self._config['max_open_trades'] From 9814cf53604971099bfaf8e6bd786713a9fe1794 Mon Sep 17 00:00:00 2001 From: Stefano Date: Fri, 15 Sep 2023 12:35:56 +0900 Subject: [PATCH 11/12] flake8 fix --- freqtrade/plugins/pairlist/FullTradesFilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/FullTradesFilter.py b/freqtrade/plugins/pairlist/FullTradesFilter.py index 376d0540e..68a58252a 100644 --- a/freqtrade/plugins/pairlist/FullTradesFilter.py +++ b/freqtrade/plugins/pairlist/FullTradesFilter.py @@ -7,7 +7,7 @@ from typing import Any, Dict, List from freqtrade.constants import Config from freqtrade.exchange.types import Tickers from freqtrade.persistence import Trade -from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter +from freqtrade.plugins.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) From eb7df300618cbf97c90745296dafab15834c44ee Mon Sep 17 00:00:00 2001 From: Stefano Date: Fri, 22 Sep 2023 09:43:03 +0900 Subject: [PATCH 12/12] add test --- .../plugins/pairlist/FullTradesFilter.py | 2 +- tests/plugins/test_pairlist.py | 52 ++++++++++++++++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/FullTradesFilter.py b/freqtrade/plugins/pairlist/FullTradesFilter.py index 68a58252a..69779d896 100644 --- a/freqtrade/plugins/pairlist/FullTradesFilter.py +++ b/freqtrade/plugins/pairlist/FullTradesFilter.py @@ -51,7 +51,7 @@ class FullTradesFilter(IPairList): num_open = Trade.get_open_trade_count() max_trades = self._config['max_open_trades'] - if (num_open >= max_trades): + if (num_open >= max_trades) and (max_trades > 0): return [] return pairlist diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 8319ef4c4..d66a47aa6 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -14,7 +14,7 @@ from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import CandleType, RunMode from freqtrade.exceptions import OperationalException -from freqtrade.persistence import Trade +from freqtrade.persistence import LocalTrade, Trade from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.resolvers import PairListResolver @@ -1463,3 +1463,53 @@ def test_ProducerPairlist(mocker, whitelist_conf, markets): pm.refresh_pairlist() assert len(pm.whitelist) == 4 assert pm.whitelist == ['TKN/BTC'] + pairs + + +@pytest.mark.usefixtures("init_persistence") +def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None: + default_conf_usdt['exchange']['pair_whitelist'].extend(['ADA/USDT', 'XRP/USDT', 'ETC/USDT']) + default_conf_usdt['pairlists'] = [ + {"method": "StaticPairList"}, + {"method": "FullTradesFilter"} + ] + default_conf_usdt['max_open_trades'] = -1 + mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True)) + exchange = get_patched_exchange(mocker, default_conf_usdt) + pm = PairListManager(exchange, default_conf_usdt) + pm.refresh_pairlist() + + assert pm.whitelist == ['ETH/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT'] + + with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: + create_mock_trades_usdt(fee) + pm.refresh_pairlist() + + # Unlimited max open trades, so no change to whitelist + pm.refresh_pairlist() + assert pm.whitelist == ['ETH/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT'] + + # Set max_open_trades to 4, the filter should empty the whitelist + default_conf_usdt['max_open_trades'] = 4 + pm.refresh_pairlist() + assert pm.whitelist == [] + assert log_has_re(r'Whitelist with 0 pairs: \[]', caplog) + + list_trades = LocalTrade.get_open_trades() + assert len(list_trades) == 4 + + # Move to 1 hour later, close a trade, so original sorting is restored. + t.move_to("2021-09-01 07:00:00 +00:00") + list_trades[2].close(12) + Trade.commit() + + # open trades count below max_open_trades, whitelist restored + list_trades = LocalTrade.get_open_trades() + assert len(list_trades) == 3 + pm.refresh_pairlist() + assert pm.whitelist == ['ETH/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT'] + + # Set max_open_trades to 3, the filter should empty the whitelist + default_conf_usdt['max_open_trades'] = 3 + pm.refresh_pairlist() + assert pm.whitelist == [] + assert log_has_re(r'Whitelist with 0 pairs: \[]', caplog)