From 71ff214adfe844a1ff57624094169f48dbd202e5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Sep 2021 07:14:40 +0200 Subject: [PATCH 1/3] Support "initial_call" for download-data of new pairs --- freqtrade/data/history/history_utils.py | 3 ++- freqtrade/exchange/binance.py | 1 + freqtrade/exchange/exchange.py | 17 ++++++++++++----- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 6f125aaa9..e6b8db322 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -197,7 +197,8 @@ def _download_pair_history(pair: str, *, timeframe=timeframe, since_ms=since_ms if since_ms else arrow.utcnow().shift( - days=-new_pairs_days).int_timestamp * 1000 + days=-new_pairs_days).int_timestamp * 1000, + is_new_pair=data.empty ) # TODO: Maybe move parsing to exchange class (?) new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 189f5f481..7a3111f85 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -20,6 +20,7 @@ class Binance(Exchange): "order_time_in_force": ['gtc', 'fok', 'ioc'], "time_in_force_parameter": "timeInForce", "ohlcv_candle_limit": 1000, + "ohlcv_initial_call": True, "trades_pagination": "id", "trades_pagination_arg": "fromId", "l2_limit_range": [5, 10, 20, 50, 100, 500, 1000], diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 80f20b17e..425b89f88 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1194,7 +1194,7 @@ class Exchange: # Historic data def get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int) -> List: + since_ms: int, is_new_pair: bool = False) -> List: """ Get candle history using asyncio and returns the list of candles. Handles all async work for this. @@ -1206,7 +1206,7 @@ class Exchange: """ return asyncio.get_event_loop().run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, - since_ms=since_ms)) + since_ms=since_ms, is_new_pair=is_new_pair)) def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, since_ms: int) -> DataFrame: @@ -1221,9 +1221,9 @@ class Exchange: return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, drop_incomplete=self._ohlcv_partial_candle) - async def _async_get_historic_ohlcv(self, pair: str, - timeframe: str, - since_ms: int) -> List: + async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, + since_ms: int, is_new_pair: bool + ) -> List: """ Download historic ohlcv """ @@ -1234,6 +1234,13 @@ class Exchange: one_call, arrow.utcnow().shift(seconds=one_call // 1000).humanize(only_distance=True) ) + if self._ft_has.get('ohlcv_initial_call', False) and is_new_pair: + x = await self._async_get_candle_history(pair, timeframe, 0) + if x and x[2] and x[2][0] and x[2][0][0] > since_ms: + # Set starting date to first available candle. + since_ms = x[2][0][0] + logger.info(f"Candle-data available starting with {since_ms}.") + input_coroutines = [self._async_get_candle_history( pair, timeframe, since) for since in range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] From 8c83c258a52c9ea4d91ed8f0d1a87e084109f828 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Sep 2021 19:29:10 +0200 Subject: [PATCH 2/3] Move "first-pair_getting" to binance subclass --- freqtrade/exchange/binance.py | 21 +++++++++++++++++++-- freqtrade/exchange/exchange.py | 8 +------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 7a3111f85..284f9cdad 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -1,7 +1,8 @@ """ Binance exchange subclass """ import logging -from typing import Dict +from typing import Dict, List +import arrow import ccxt from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException, @@ -20,7 +21,6 @@ class Binance(Exchange): "order_time_in_force": ['gtc', 'fok', 'ioc'], "time_in_force_parameter": "timeInForce", "ohlcv_candle_limit": 1000, - "ohlcv_initial_call": True, "trades_pagination": "id", "trades_pagination_arg": "fromId", "l2_limit_range": [5, 10, 20, 50, 100, 500, 1000], @@ -91,3 +91,20 @@ class Binance(Exchange): f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e except ccxt.BaseError as e: raise OperationalException(e) from e + + async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, + since_ms: int, is_new_pair: bool + ) -> List: + """ + Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date + Does not work for other exchanges, which don't return the earliest data when called with "0" + """ + if is_new_pair: + x = await self._async_get_candle_history(pair, timeframe, 0) + if x and x[2] and x[2][0] and x[2][0][0] > since_ms: + # Set starting date to first available candle. + since_ms = x[2][0][0] + logger.info(f"Candle-data available starting with " + f"{arrow.get(since_ms // 1000).isoformat()}.") + return await super()._async_get_historic_ohlcv( + pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 425b89f88..79fd33dfe 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1226,6 +1226,7 @@ class Exchange: ) -> List: """ Download historic ohlcv + :param is_new_pair: used by binance subclass to allow "fast" new pair downloading """ one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) @@ -1234,13 +1235,6 @@ class Exchange: one_call, arrow.utcnow().shift(seconds=one_call // 1000).humanize(only_distance=True) ) - if self._ft_has.get('ohlcv_initial_call', False) and is_new_pair: - x = await self._async_get_candle_history(pair, timeframe, 0) - if x and x[2] and x[2][0] and x[2][0][0] > since_ms: - # Set starting date to first available candle. - since_ms = x[2][0][0] - logger.info(f"Candle-data available starting with {since_ms}.") - input_coroutines = [self._async_get_candle_history( pair, timeframe, since) for since in range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] From 2d66987ac7ed38961c2110680b03893693f86231 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Sep 2021 19:51:07 +0200 Subject: [PATCH 3/3] Add test for "pair-startdate" detection --- freqtrade/exchange/binance.py | 2 +- tests/exchange/test_binance.py | 35 +++++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 284f9cdad..8dced3894 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -104,7 +104,7 @@ class Binance(Exchange): if x and x[2] and x[2][0] and x[2][0][0] > since_ms: # Set starting date to first available candle. since_ms = x[2][0][0] - logger.info(f"Candle-data available starting with " + logger.info(f"Candle-data for {pair} available starting with " f"{arrow.get(since_ms // 1000).isoformat()}.") return await super()._async_get_historic_ohlcv( pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair) diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index f2b508761..dd85c3abe 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -1,3 +1,4 @@ +from datetime import datetime, timezone from random import randint from unittest.mock import MagicMock @@ -5,7 +6,7 @@ import ccxt import pytest from freqtrade.exceptions import DependencyException, InvalidOrderException, OperationalException -from tests.conftest import get_patched_exchange +from tests.conftest import get_mock_coro, get_patched_exchange, log_has_re from tests.exchange.test_exchange import ccxt_exceptionhandlers @@ -105,3 +106,35 @@ def test_stoploss_adjust_binance(mocker, default_conf): # Test with invalid order case order['type'] = 'stop_loss' assert not exchange.stoploss_adjust(1501, order) + + +@pytest.mark.asyncio +async def test__async_get_historic_ohlcv_binance(default_conf, mocker, caplog): + ohlcv = [ + [ + int((datetime.now(timezone.utc).timestamp() - 1000) * 1000), + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + + exchange = get_patched_exchange(mocker, default_conf, id='binance') + # Monkey-patch async function + exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) + + pair = 'ETH/BTC' + res = await exchange._async_get_historic_ohlcv(pair, "5m", + 1500000000000, is_new_pair=False) + # Call with very old timestamp - causes tons of requests + assert exchange._api_async.fetch_ohlcv.call_count > 400 + # assert res == ohlcv + exchange._api_async.fetch_ohlcv.reset_mock() + res = await exchange._async_get_historic_ohlcv(pair, "5m", 1500000000000, is_new_pair=True) + + # Called twice - one "init" call - and one to get the actual data. + assert exchange._api_async.fetch_ohlcv.call_count == 2 + assert res == ohlcv + assert log_has_re(r"Candle-data for ETH/BTC available starting with .*", caplog)