diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 6f125aaa9..e6b8db322 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -197,7 +197,8 @@ def _download_pair_history(pair: str, *, timeframe=timeframe, since_ms=since_ms if since_ms else arrow.utcnow().shift( - days=-new_pairs_days).int_timestamp * 1000 + days=-new_pairs_days).int_timestamp * 1000, + is_new_pair=data.empty ) # TODO: Maybe move parsing to exchange class (?) new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 189f5f481..8dced3894 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -1,7 +1,8 @@ """ Binance exchange subclass """ import logging -from typing import Dict +from typing import Dict, List +import arrow import ccxt from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException, @@ -90,3 +91,20 @@ class Binance(Exchange): f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e except ccxt.BaseError as e: raise OperationalException(e) from e + + async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, + since_ms: int, is_new_pair: bool + ) -> List: + """ + Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date + Does not work for other exchanges, which don't return the earliest data when called with "0" + """ + if is_new_pair: + x = await self._async_get_candle_history(pair, timeframe, 0) + if x and x[2] and x[2][0] and x[2][0][0] > since_ms: + # Set starting date to first available candle. + since_ms = x[2][0][0] + logger.info(f"Candle-data for {pair} available starting with " + f"{arrow.get(since_ms // 1000).isoformat()}.") + return await super()._async_get_historic_ohlcv( + pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 80f20b17e..79fd33dfe 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1194,7 +1194,7 @@ class Exchange: # Historic data def get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int) -> List: + since_ms: int, is_new_pair: bool = False) -> List: """ Get candle history using asyncio and returns the list of candles. Handles all async work for this. @@ -1206,7 +1206,7 @@ class Exchange: """ return asyncio.get_event_loop().run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, - since_ms=since_ms)) + since_ms=since_ms, is_new_pair=is_new_pair)) def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, since_ms: int) -> DataFrame: @@ -1221,11 +1221,12 @@ class Exchange: return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, drop_incomplete=self._ohlcv_partial_candle) - async def _async_get_historic_ohlcv(self, pair: str, - timeframe: str, - since_ms: int) -> List: + async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, + since_ms: int, is_new_pair: bool + ) -> List: """ Download historic ohlcv + :param is_new_pair: used by binance subclass to allow "fast" new pair downloading """ one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index f2b508761..dd85c3abe 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -1,3 +1,4 @@ +from datetime import datetime, timezone from random import randint from unittest.mock import MagicMock @@ -5,7 +6,7 @@ import ccxt import pytest from freqtrade.exceptions import DependencyException, InvalidOrderException, OperationalException -from tests.conftest import get_patched_exchange +from tests.conftest import get_mock_coro, get_patched_exchange, log_has_re from tests.exchange.test_exchange import ccxt_exceptionhandlers @@ -105,3 +106,35 @@ def test_stoploss_adjust_binance(mocker, default_conf): # Test with invalid order case order['type'] = 'stop_loss' assert not exchange.stoploss_adjust(1501, order) + + +@pytest.mark.asyncio +async def test__async_get_historic_ohlcv_binance(default_conf, mocker, caplog): + ohlcv = [ + [ + int((datetime.now(timezone.utc).timestamp() - 1000) * 1000), + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + + exchange = get_patched_exchange(mocker, default_conf, id='binance') + # Monkey-patch async function + exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) + + pair = 'ETH/BTC' + res = await exchange._async_get_historic_ohlcv(pair, "5m", + 1500000000000, is_new_pair=False) + # Call with very old timestamp - causes tons of requests + assert exchange._api_async.fetch_ohlcv.call_count > 400 + # assert res == ohlcv + exchange._api_async.fetch_ohlcv.reset_mock() + res = await exchange._async_get_historic_ohlcv(pair, "5m", 1500000000000, is_new_pair=True) + + # Called twice - one "init" call - and one to get the actual data. + assert exchange._api_async.fetch_ohlcv.call_count == 2 + assert res == ohlcv + assert log_has_re(r"Candle-data for ETH/BTC available starting with .*", caplog)