Merge branch 'develop' into feature/fetch-public-trades

tests/conftest.py

@@ -587,6 +587,7 @@ def get_default_conf(testdatadir):
         "exchange": {
             "name": "binance",
             "key": "key",
+            "enable_ws": False,
             "secret": "secret",
             "pair_whitelist": ["ETH/BTC", "LTC/BTC", "XRP/BTC", "NEO/BTC"],
             "pair_blacklist": [
@@ -629,6 +630,7 @@ def get_default_conf_usdt(testdatadir):
                 "name": "binance",
                 "enabled": True,
                 "key": "key",
+                "enable_ws": False,
                 "secret": "secret",
                 "pair_whitelist": [
                     "ETH/USDT",

tests/exchange/test_exchange_ws.py

@@ -0,0 +1,69 @@
+import asyncio
+import threading
+from time import sleep
+from unittest.mock import AsyncMock, MagicMock
+
+from freqtrade.enums import CandleType
+from freqtrade.exchange.exchange_ws import ExchangeWS
+
+
+def test_exchangews_init(mocker):
+    config = MagicMock()
+    ccxt_object = MagicMock()
+    mocker.patch("freqtrade.exchange.exchange_ws.ExchangeWS._start_forever", MagicMock())
+
+    exchange_ws = ExchangeWS(config, ccxt_object)
+    sleep(0.1)
+
+    assert exchange_ws.config == config
+    assert exchange_ws.ccxt_object == ccxt_object
+    assert exchange_ws._thread.name == "ccxt_ws"
+    assert exchange_ws._background_tasks == set()
+    assert exchange_ws._klines_watching == set()
+    assert exchange_ws._klines_scheduled == set()
+    assert exchange_ws.klines_last_refresh == {}
+    assert exchange_ws.klines_last_request == {}
+    # Cleanup
+    exchange_ws.cleanup()
+
+
+def patch_eventloop_threading(exchange):
+    is_init = False
+
+    def thread_fuck():
+        nonlocal is_init
+        exchange._loop = asyncio.new_event_loop()
+        is_init = True
+        exchange._loop.run_forever()
+
+    x = threading.Thread(target=thread_fuck, daemon=True)
+    x.start()
+    while not is_init:
+        pass
+
+
+async def test_exchangews_ohlcv(mocker):
+    config = MagicMock()
+    ccxt_object = MagicMock()
+    ccxt_object.watch_ohlcv = AsyncMock()
+    ccxt_object.close = AsyncMock()
+    mocker.patch("freqtrade.exchange.exchange_ws.ExchangeWS._start_forever", MagicMock())
+
+    exchange_ws = ExchangeWS(config, ccxt_object)
+    patch_eventloop_threading(exchange_ws)
+    try:
+        assert exchange_ws._klines_watching == set()
+        assert exchange_ws._klines_scheduled == set()
+
+        exchange_ws.schedule_ohlcv("ETH/BTC", "1m", CandleType.SPOT)
+        asyncio.sleep(0.5)
+
+        assert exchange_ws._klines_watching == {("ETH/BTC", "1m", CandleType.SPOT)}
+        assert exchange_ws._klines_scheduled == {("ETH/BTC", "1m", CandleType.SPOT)}
+        asyncio.sleep(0.1)
+        assert ccxt_object.watch_ohlcv.call_count == 1
+    except Exception as e:
+        print(e)
+    finally:
+        # Cleanup
+        exchange_ws.cleanup()

tests/exchange_online/conftest.py

@@ -11,6 +11,8 @@ from tests.conftest import EXMS, get_default_conf_usdt
 
 
 EXCHANGE_FIXTURE_TYPE = Tuple[Exchange, str]
+EXCHANGE_WS_FIXTURE_TYPE = Tuple[Exchange, str, str]
+
 
 # Exchanges that should be tested online
 EXCHANGES = {
@@ -360,6 +362,7 @@ def set_test_proxy(config: Config, use_proxy: bool) -> Config:
         config1 = deepcopy(config)
         config1["exchange"]["ccxt_config"] = {
             "httpsProxy": proxy,
+            "wsProxy": proxy,
         }
         return config1
 
@@ -376,7 +379,7 @@ def get_exchange(exchange_name, exchange_conf):
         exchange_conf, validate=True, load_leverage_tiers=True
     )
 
-    yield exchange, exchange_name
+    return exchange, exchange_name
 
 
 def get_futures_exchange(exchange_name, exchange_conf, class_mocker):
@@ -398,15 +401,41 @@ def get_futures_exchange(exchange_name, exchange_conf, class_mocker):
         class_mocker.patch(f"{EXMS}.load_cached_leverage_tiers", return_value=None)
         class_mocker.patch(f"{EXMS}.cache_leverage_tiers")
 
-        yield from get_exchange(exchange_name, exchange_conf)
+        return get_exchange(exchange_name, exchange_conf)
 
 
 @pytest.fixture(params=EXCHANGES, scope="class")
 def exchange(request, exchange_conf, class_mocker):
     class_mocker.patch("freqtrade.exchange.bybit.Bybit.additional_exchange_init")
-    yield from get_exchange(request.param, exchange_conf)
+    return get_exchange(request.param, exchange_conf)
 
 
 @pytest.fixture(params=EXCHANGES, scope="class")
 def exchange_futures(request, exchange_conf, class_mocker):
-    yield from get_futures_exchange(request.param, exchange_conf, class_mocker)
+    return get_futures_exchange(request.param, exchange_conf, class_mocker)
+
+
+@pytest.fixture(params=["spot", "futures"], scope="class")
+def exchange_mode(request):
+    return request.param
+
+
+@pytest.fixture(params=EXCHANGES, scope="class")
+def exchange_ws(request, exchange_conf, exchange_mode, class_mocker):
+    class_mocker.patch("freqtrade.exchange.bybit.Bybit.additional_exchange_init")
+    exchange_conf["exchange"]["enable_ws"] = True
+    if exchange_mode == "spot":
+        exchange, name = get_exchange(request.param, exchange_conf)
+        pair = EXCHANGES[request.param]["pair"]
+    elif EXCHANGES[request.param].get("futures"):
+        exchange, name = get_futures_exchange(
+            request.param, exchange_conf, class_mocker=class_mocker
+        )
+        pair = EXCHANGES[request.param]["futures_pair"]
+    else:
+        pytest.skip("Exchange does not support futures.")
+
+    if not exchange._has_watch_ohlcv:
+        pytest.skip("Exchange does not support watch_ohlcv.")
+    yield exchange, name, pair
+    exchange.close()

tests/exchange_online/test_ccxt_ws_compat.py

@@ -0,0 +1,64 @@
+"""
+Tests in this file do NOT mock network calls, so they are expected to be fluky at times.
+
+However, these tests aim to test ccxt compatibility, specifically regarding websockets.
+"""
+
+import logging
+from datetime import timedelta
+from time import sleep
+
+import pytest
+
+from freqtrade.enums import CandleType
+from freqtrade.exchange.exchange_utils import timeframe_to_prev_date
+from freqtrade.loggers.set_log_levels import set_loggers
+from freqtrade.util.datetime_helpers import dt_now
+from tests.conftest import log_has_re
+from tests.exchange_online.conftest import EXCHANGE_WS_FIXTURE_TYPE
+
+
+@pytest.mark.longrun
+@pytest.mark.timeout(3 * 60)
+class TestCCXTExchangeWs:
+    def test_ccxt_ohlcv(self, exchange_ws: EXCHANGE_WS_FIXTURE_TYPE, caplog, mocker):
+        exch, exchangename, pair = exchange_ws
+
+        assert exch._ws_async is not None
+        timeframe = "1m"
+        pair_tf = (pair, timeframe, CandleType.SPOT)
+        m_hist = mocker.spy(exch, "_async_get_historic_ohlcv")
+        m_cand = mocker.spy(exch, "_async_get_candle_history")
+
+        res = exch.refresh_latest_ohlcv([pair_tf])
+        assert m_cand.call_count == 1
+
+        # Currently open candle
+        next_candle = timeframe_to_prev_date(timeframe, dt_now())
+        now = next_candle - timedelta(seconds=1)
+        # Currently closed candle
+        curr_candle = timeframe_to_prev_date(timeframe, now)
+
+        assert pair_tf in exch._exchange_ws._klines_watching
+        assert pair_tf in exch._exchange_ws._klines_scheduled
+        assert res[pair_tf] is not None
+        df1 = res[pair_tf]
+        caplog.set_level(logging.DEBUG)
+        set_loggers(1)
+        assert df1.iloc[-1]["date"] == curr_candle
+
+        # Wait until the next candle (might be up to 1 minute).
+        while True:
+            caplog.clear()
+            res = exch.refresh_latest_ohlcv([pair_tf])
+            df2 = res[pair_tf]
+            assert df2 is not None
+            if df2.iloc[-1]["date"] == next_candle:
+                break
+            assert df2.iloc[-1]["date"] == curr_candle
+            sleep(1)
+
+        assert m_hist.call_count == 0
+        # shouldn't have tried fetch_ohlcv a second time.
+        assert m_cand.call_count == 1
+        assert log_has_re(r"watch result.*", caplog)

tests/rpc/test_rpc_apiserver.py

@@ -2176,7 +2176,7 @@ def test_api_exchanges(botclient):
         "valid": True,
         "supported": False,
         "dex": True,
-        "comment": "",
+        "comment": ANY,
         "trade_modes": [{"trading_mode": "spot", "margin_mode": ""}],
     }