Merge pull request #8819 from Bloodhunter4rc/remotepairlist

Remotepairlist - add blacklist mode

docs/includes/pairlists.md

@@ -184,6 +184,8 @@ The RemotePairList is defined in the pairlists section of the configuration sett
 "pairlists": [
     {
         "method": "RemotePairList",
+        "mode": "whitelist",
+        "processing_mode": "filter",
         "pairlist_url": "https://example.com/pairlist",
         "number_assets": 10,
         "refresh_period": 1800,
@@ -194,6 +196,14 @@ The RemotePairList is defined in the pairlists section of the configuration sett
 ]
 ```
 
+The optional `mode` option specifies if the pairlist should be used as a `blacklist` or as a `whitelist`. The default value is "whitelist".
+
+The optional `processing_mode` option in the RemotePairList configuration determines how the retrieved pairlist is processed. It can have two values: "filter" or "append".
+
+In "filter" mode, the retrieved pairlist is used as a filter. Only the pairs present in both the original pairlist and the retrieved pairlist are included in the final pairlist. Other pairs are filtered out.
+
+In "append" mode, the retrieved pairlist is added to the original pairlist. All pairs from both lists are included in the final pairlist without any filtering.
+
 The `pairlist_url` option specifies the URL of the remote server where the pairlist is located, or the path to a local file (if file:/// is prepended). This allows the user to use either a remote server or a local file as the source for the pairlist.
 
 The user is responsible for providing a server or local file that returns a JSON object with the following structure:

freqtrade/plugins/pairlist/RemotePairList.py

@@ -16,6 +16,7 @@ from freqtrade.constants import Config
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 
 
 logger = logging.getLogger(__name__)
@@ -40,6 +41,8 @@ class RemotePairList(IPairList):
                 '`pairlist_url` not specified. Please check your configuration '
                 'for "pairlist.config.pairlist_url"')
 
+        self._mode = self._pairlistconfig.get('mode', 'whitelist')
+        self._processing_mode = self._pairlistconfig.get('processing_mode', 'filter')
         self._number_pairs = self._pairlistconfig['number_assets']
         self._refresh_period: int = self._pairlistconfig.get('refresh_period', 1800)
         self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True)
@@ -50,6 +53,21 @@ class RemotePairList(IPairList):
         self._init_done = False
         self._last_pairlist: List[Any] = list()
 
+        if self._mode not in ['whitelist', 'blacklist']:
+            raise OperationalException(
+                '`mode` not configured correctly. Supported Modes '
+                'are "whitelist","blacklist"')
+
+        if self._processing_mode not in ['filter', 'append']:
+            raise OperationalException(
+                '`processing_mode` not configured correctly. Supported Modes '
+                'are "filter","append"')
+
+        if self._pairlist_pos == 0 and self._mode == 'blacklist':
+            raise OperationalException(
+                'A `blacklist` mode RemotePairList can not be on the first '
+                'position of your pairlist.')
+
     @property
     def needstickers(self) -> bool:
         """
@@ -67,23 +85,37 @@ class RemotePairList(IPairList):
 
     @staticmethod
     def description() -> str:
-        return "Retrieve pairs from a remote API."
+        return "Retrieve pairs from a remote API or local file."
 
     @staticmethod
     def available_parameters() -> Dict[str, PairlistParameter]:
         return {
-            "number_assets": {
-                "type": "number",
-                "default": 0,
-                "description": "Number of assets",
-                "help": "Number of assets to use from the pairlist.",
-            },
             "pairlist_url": {
                 "type": "string",
                 "default": "",
                 "description": "URL to fetch pairlist from",
                 "help": "URL to fetch pairlist from",
             },
+            "number_assets": {
+                "type": "number",
+                "default": 30,
+                "description": "Number of assets",
+                "help": "Number of assets to use from the pairlist.",
+            },
+            "mode": {
+                "type": "option",
+                "default": "whitelist",
+                "options": ["whitelist", "blacklist"],
+                "description": "Pairlist mode",
+                "help": "Should this pairlist operate as a whitelist or blacklist?",
+            },
+            "processing_mode": {
+                "type": "option",
+                "default": "filter",
+                "options": ["filter", "append"],
+                "description": "Processing mode",
+                "help": "Append pairs to incomming pairlist or filter them?",
+            },
             **IPairList.refresh_period_parameter(),
             "keep_pairlist_on_failure": {
                 "type": "boolean",
@@ -223,6 +255,7 @@ class RemotePairList(IPairList):
 
         self.log_once(f"Fetched pairs: {pairlist}", logger.debug)
 
+        pairlist = expand_pairlist(pairlist, list(self._exchange.get_markets().keys()))
         pairlist = self._whitelist_for_active_markets(pairlist)
         pairlist = pairlist[:self._number_pairs]
 
@@ -250,6 +283,23 @@ class RemotePairList(IPairList):
         :return: new whitelist
         """
         rpl_pairlist = self.gen_pairlist(tickers)
-        merged_list = pairlist + rpl_pairlist
+        merged_list = []
-        merged_list = sorted(set(merged_list), key=merged_list.index)
+        filtered = []
+
+        if self._mode == "whitelist":
+            if self._processing_mode == "filter":
+                merged_list = [pair for pair in pairlist if pair in rpl_pairlist]
+            elif self._processing_mode == "append":
+                merged_list = pairlist + rpl_pairlist
+            merged_list = sorted(set(merged_list), key=merged_list.index)
+        else:
+            for pair in pairlist:
+                if pair not in rpl_pairlist:
+                    merged_list.append(pair)
+                else:
+                    filtered.append(pair)
+            if filtered:
+                self.log_once(f"Blacklist - Filtered out pairs: {filtered}", logger.info)
+
+        merged_list = merged_list[:self._number_pairs]
         return merged_list

tests/plugins/test_pairlist.py

@@ -1200,6 +1200,10 @@ def test_spreadfilter_invalid_data(mocker, default_conf, markets, tickers, caplo
      "[{'ProducerPairList': 'ProducerPairList - default'}]",
      None
      ),
+    ({"method": "RemotePairList", "number_assets": 10, "pairlist_url": "https://example.com"},
+     "[{'RemotePairList': 'RemotePairList - 10 pairs from RemotePairlist.'}]",
+     None
+     ),
 ])
 def test_pricefilter_desc(mocker, whitelist_conf, markets, pairlistconfig,
                           desc_expected, exception_expected):

tests/plugins/test_remotepairlist.py

@@ -1,5 +1,5 @@
 import json
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, PropertyMock
 
 import pytest
 import requests
@@ -7,7 +7,7 @@ import requests
 from freqtrade.exceptions import OperationalException
 from freqtrade.plugins.pairlist.RemotePairList import RemotePairList
 from freqtrade.plugins.pairlistmanager import PairListManager
-from tests.conftest import get_patched_exchange, get_patched_freqtradebot, log_has
+from tests.conftest import EXMS, get_patched_exchange, get_patched_freqtradebot, log_has
 
 
 @pytest.fixture(scope="function")
@@ -16,11 +16,12 @@ def rpl_config(default_conf):
 
     default_conf['exchange']['pair_whitelist'] = [
         'ETH/USDT',
-        'BTC/USDT',
+        'XRP/USDT',
     ]
     default_conf['exchange']['pair_blacklist'] = [
         'BLK/USDT'
     ]
+
     return default_conf
 
 
@@ -183,3 +184,159 @@ def test_fetch_pairlist_mock_response_valid(mocker, rpl_config):
     assert pairs == ["ETH/USDT", "XRP/USDT", "LTC/USDT", "EOS/USDT"]
     assert time_elapsed == 0.4
     assert remote_pairlist._refresh_period == 60
+
+
+def test_remote_pairlist_init_wrong_mode(mocker, rpl_config):
+    rpl_config['pairlists'] = [
+        {
+            "method": "RemotePairList",
+            "mode": "blacklis",
+            "number_assets": 20,
+            "pairlist_url": "http://example.com/pairlist",
+            "keep_pairlist_on_failure": True,
+        }
+    ]
+
+    with pytest.raises(
+        OperationalException,
+        match=r'`mode` not configured correctly. Supported Modes are "whitelist","blacklist"'
+    ):
+        get_patched_freqtradebot(mocker, rpl_config)
+
+    rpl_config['pairlists'] = [
+        {
+            "method": "RemotePairList",
+            "mode": "blacklist",
+            "number_assets": 20,
+            "pairlist_url": "http://example.com/pairlist",
+            "keep_pairlist_on_failure": True,
+        }
+    ]
+
+    with pytest.raises(
+            OperationalException,
+            match=r'A `blacklist` mode RemotePairList can not be.*first.*'
+    ):
+        get_patched_freqtradebot(mocker, rpl_config)
+
+
+def test_remote_pairlist_init_wrong_proc_mode(mocker, rpl_config):
+    rpl_config['pairlists'] = [
+        {
+            "method": "RemotePairList",
+            "processing_mode": "filler",
+            "mode": "whitelist",
+            "number_assets": 20,
+            "pairlist_url": "http://example.com/pairlist",
+            "keep_pairlist_on_failure": True,
+        }
+    ]
+
+    get_patched_exchange(mocker, rpl_config)
+    with pytest.raises(
+        OperationalException,
+        match=r'`processing_mode` not configured correctly. Supported Modes are "filter","append"'
+    ):
+        get_patched_freqtradebot(mocker, rpl_config)
+
+
+def test_remote_pairlist_blacklist(mocker, rpl_config, caplog, markets, tickers):
+
+    mock_response = MagicMock()
+
+    mock_response.json.return_value = {
+        "pairs": ["XRP/USDT"],
+        "refresh_period": 60
+    }
+
+    mock_response.headers = {
+        "content-type": "application/json"
+    }
+
+    rpl_config['pairlists'] = [
+        {
+            "method": "StaticPairList",
+        },
+        {
+            "method": "RemotePairList",
+            "mode": "blacklist",
+            "pairlist_url": "http://example.com/pairlist",
+            "number_assets": 3
+        }
+    ]
+
+    mocker.patch.multiple(EXMS,
+                          markets=PropertyMock(return_value=markets),
+                          exchange_has=MagicMock(return_value=True),
+                          get_tickers=tickers
+                          )
+
+    mocker.patch("freqtrade.plugins.pairlist.RemotePairList.requests.get",
+                 return_value=mock_response)
+
+    exchange = get_patched_exchange(mocker, rpl_config)
+
+    pairlistmanager = PairListManager(exchange, rpl_config)
+
+    remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config,
+                                     rpl_config["pairlists"][1], 1)
+
+    pairs, time_elapsed = remote_pairlist.fetch_pairlist()
+
+    assert pairs == ["XRP/USDT"]
+
+    whitelist = remote_pairlist.filter_pairlist(rpl_config['exchange']['pair_whitelist'], {})
+    assert whitelist == ["ETH/USDT"]
+
+    assert log_has(f"Blacklist - Filtered out pairs: {pairs}", caplog)
+
+
+@pytest.mark.parametrize("processing_mode", ["filter", "append"])
+def test_remote_pairlist_whitelist(mocker, rpl_config, processing_mode, markets, tickers):
+
+    mock_response = MagicMock()
+
+    mock_response.json.return_value = {
+        "pairs": ["XRP/USDT"],
+        "refresh_period": 60
+    }
+
+    mock_response.headers = {
+        "content-type": "application/json"
+    }
+
+    rpl_config['pairlists'] = [
+        {
+            "method": "StaticPairList",
+        },
+        {
+            "method": "RemotePairList",
+            "mode": "whitelist",
+            "processing_mode": processing_mode,
+            "pairlist_url": "http://example.com/pairlist",
+            "number_assets": 3
+        }
+    ]
+
+    mocker.patch.multiple(EXMS,
+                          markets=PropertyMock(return_value=markets),
+                          exchange_has=MagicMock(return_value=True),
+                          get_tickers=tickers
+                          )
+
+    mocker.patch("freqtrade.plugins.pairlist.RemotePairList.requests.get",
+                 return_value=mock_response)
+
+    exchange = get_patched_exchange(mocker, rpl_config)
+
+    pairlistmanager = PairListManager(exchange, rpl_config)
+
+    remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config,
+                                     rpl_config["pairlists"][1], 1)
+
+    pairs, time_elapsed = remote_pairlist.fetch_pairlist()
+
+    assert pairs == ["XRP/USDT"]
+
+    whitelist = remote_pairlist.filter_pairlist(rpl_config['exchange']['pair_whitelist'], {})
+    assert whitelist == (["XRP/USDT"] if processing_mode == "filter" else ['ETH/USDT', 'XRP/USDT'])