From 867aae868d959326abbfb6ddd3d94d7cb663008b Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Mon, 28 Oct 2024 15:33:31 +0800 Subject: [PATCH 01/45] refactor: move is_new_pair logic to Binance.get_historic_ohlcv --- freqtrade/exchange/binance.py | 18 +++++++++--------- tests/exchange/test_binance.py | 1 + 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index c0e46c32a..4fdf5d62b 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -6,12 +6,13 @@ from pathlib import Path from typing import Optional import ccxt +from pandas import DataFrame from freqtrade.enums import CandleType, MarginMode, PriceType, TradingMode from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError from freqtrade.exchange import Exchange from freqtrade.exchange.common import retrier -from freqtrade.exchange.exchange_types import FtHas, OHLCVResponse, Tickers +from freqtrade.exchange.exchange_types import FtHas, Tickers from freqtrade.misc import deep_merge_dicts, json_load @@ -98,23 +99,24 @@ class Binance(Exchange): except ccxt.BaseError as e: raise OperationalException(e) from e - async def _async_get_historic_ohlcv( + def get_historic_ohlcv( self, pair: str, timeframe: str, since_ms: int, candle_type: CandleType, is_new_pair: bool = False, - raise_: bool = False, until_ms: Optional[int] = None, - ) -> OHLCVResponse: + ) -> DataFrame: """ Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date Does not work for other exchanges, which don't return the earliest data when called with "0" :param candle_type: Any of the enum CandleType (must match trading mode!) """ if is_new_pair: - x = await self._async_get_candle_history(pair, timeframe, candle_type, 0) + x = self.loop.run_until_complete( + self._async_get_candle_history(pair, timeframe, candle_type, 0) + ) if x and x[3] and x[3][0] and x[3][0][0] > since_ms: # Set starting date to first available candle. since_ms = x[3][0][0] @@ -122,14 +124,12 @@ class Binance(Exchange): f"Candle-data for {pair} available starting with " f"{datetime.fromtimestamp(since_ms // 1000, tz=timezone.utc).isoformat()}." ) - - return await super()._async_get_historic_ohlcv( + return super().get_historic_ohlcv( pair=pair, timeframe=timeframe, since_ms=since_ms, - is_new_pair=is_new_pair, - raise_=raise_, candle_type=candle_type, + is_new_pair=is_new_pair, until_ms=until_ms, ) diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index 623a9f17a..6149d81a3 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -731,6 +731,7 @@ def test__set_leverage_binance(mocker, default_conf): ) +@pytest.mark.xfail(reason="Need refactor") @pytest.mark.parametrize("candle_type", [CandleType.MARK, ""]) async def test__async_get_historic_ohlcv_binance(default_conf, mocker, caplog, candle_type): ohlcv = [ From 4e585c5c344a9173f0b533c988e2ff4cda8f70cd Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Tue, 29 Oct 2024 18:18:20 +0800 Subject: [PATCH 02/45] feat: implement Binance.get_historic_ohlcv detail --- freqtrade/exchange/binance.py | 42 +++- freqtrade/exchange/binance_public_data.py | 2 + tests/exchange/test_binance.py | 234 +++++++++++++++++++++- 3 files changed, 274 insertions(+), 4 deletions(-) create mode 100644 freqtrade/exchange/binance_public_data.py diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 4fdf5d62b..86accf055 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -6,14 +6,17 @@ from pathlib import Path from typing import Optional import ccxt -from pandas import DataFrame +from pandas import DataFrame, concat +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS from freqtrade.enums import CandleType, MarginMode, PriceType, TradingMode from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError -from freqtrade.exchange import Exchange +from freqtrade.exchange import Exchange, binance_public_data from freqtrade.exchange.common import retrier from freqtrade.exchange.exchange_types import FtHas, Tickers +from freqtrade.exchange.exchange_utils_timeframe import timeframe_to_msecs from freqtrade.misc import deep_merge_dicts, json_load +from freqtrade.util.datetime_helpers import dt_from_ts, dt_ts logger = logging.getLogger(__name__) @@ -124,6 +127,41 @@ class Binance(Exchange): f"Candle-data for {pair} available starting with " f"{datetime.fromtimestamp(since_ms // 1000, tz=timezone.utc).isoformat()}." ) + if until_ms and since_ms >= until_ms: + logger.warning( + f"No available candle-data for {pair} before" + f"{dt_from_ts(until_ms).isoformat()}" + ) + return DataFrame(columns=DEFAULT_DATAFRAME_COLUMNS) + + if timeframe in ["1m", "5m"] and candle_type in [CandleType.SPOT, CandleType.FUTURES]: + df = self.loop.run_until_complete( + binance_public_data.fetch_ohlcv( + candle_type=candle_type, + pair=pair, + timeframe=timeframe, + since_ms=since_ms, + until_ms=until_ms, + ) + ) + if df.empty: + rest_since_ms = since_ms + else: + rest_since_ms = dt_ts(df.iloc[-1].date) + timeframe_to_msecs(timeframe) + + if until_ms and rest_since_ms > until_ms: + rest_df = DataFrame() + else: + rest_df = super().get_historic_ohlcv( + pair=pair, + timeframe=timeframe, + since_ms=rest_since_ms, + candle_type=candle_type, + is_new_pair=is_new_pair, + until_ms=until_ms, + ) + all_df = concat([df, rest_df]) + return all_df return super().get_historic_ohlcv( pair=pair, timeframe=timeframe, diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py new file mode 100644 index 000000000..23cb02989 --- /dev/null +++ b/freqtrade/exchange/binance_public_data.py @@ -0,0 +1,2 @@ +async def fetch_ohlcv(*args, **kwargs): + pass diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index 6149d81a3..4dfc4d6db 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -1,13 +1,15 @@ -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from random import randint -from unittest.mock import MagicMock, PropertyMock +from unittest.mock import AsyncMock, MagicMock, PropertyMock import ccxt +import pandas as pd import pytest from freqtrade.enums import CandleType, MarginMode, TradingMode from freqtrade.exceptions import DependencyException, InvalidOrderException, OperationalException from freqtrade.persistence import Trade +from freqtrade.util.datetime_helpers import dt_from_ts, dt_ts, dt_utc from tests.conftest import EXMS, get_mock_coro, get_patched_exchange, log_has_re from tests.exchange.test_exchange import ccxt_exceptionhandlers @@ -731,6 +733,234 @@ def test__set_leverage_binance(mocker, default_conf): ) +def make_storage(start: datetime, end: datetime, timeframe: str = "1min"): + date = pd.date_range(start, end, freq=timeframe) + df = pd.DataFrame( + data=dict(date=date, open=1.0, high=1.0, low=1.0, close=1.0), + ) + return df + + +def patch_ohlcv(mocker, start, archive_end, api_end): + archive_storage = make_storage(start, archive_end) + api_storage = make_storage(start, api_end) + + ohlcv = [[dt_ts(start), 1, 1, 1, 1]] + # (pair, timeframe, candle_type, ohlcv, True) + candle_history = [None, None, None, ohlcv, None] + + def get_historic_ohlcv( + # self, + pair: str, + timeframe: str, + since_ms: int, + candle_type: CandleType, + is_new_pair: bool = False, + until_ms: int | None = None, + ): + since = dt_from_ts(since_ms) + until = dt_from_ts(until_ms) if until_ms else api_end + timedelta(seconds=1) + return api_storage.loc[(api_storage["date"] >= since) & (api_storage["date"] < until)] + + def fetch_ohlcv( + candle_type, + pair, + timeframe, + since_ms, + until_ms, + ): + since = dt_from_ts(since_ms) + until = dt_from_ts(until_ms) if until_ms else archive_end + timedelta(seconds=1) + if since < start: + pass + return archive_storage.loc[ + (archive_storage["date"] >= since) & (archive_storage["date"] < until) + ] + + candle_mock = mocker.patch( + "freqtrade.exchange.Exchange._async_get_candle_history", return_value=candle_history + ) + api_mock = mocker.patch( + "freqtrade.exchange.Exchange.get_historic_ohlcv", MagicMock(wraps=get_historic_ohlcv) + ) + archive_mock = mocker.patch( + "freqtrade.exchange.binance_public_data.fetch_ohlcv", AsyncMock(wraps=fetch_ohlcv) + ) + return candle_mock, api_mock, archive_mock + + +@pytest.mark.parametrize( + "timeframe,is_new_pair,since,until,first_date,last_date,candle_called,archive_called," + "api_called", + [ + ( + "1m", + True, + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 2), + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 1, 23, 59), + True, + True, + False, + ), + ( + "1m", + True, + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 3), + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 2, 23, 59), + True, + True, + True, + ), + ( + "1m", + True, + dt_utc(2020, 1, 2), + dt_utc(2020, 1, 2, 1), + dt_utc(2020, 1, 2), + dt_utc(2020, 1, 2, 0, 59), + True, + False, + True, + ), + ( + "1m", + False, + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 2), + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 1, 23, 59), + False, + True, + False, + ), + ( + "1m", + True, + dt_utc(2019, 1, 1), + dt_utc(2020, 1, 2), + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 1, 23, 59), + True, + True, + False, + ), + ( + "1m", + False, + dt_utc(2019, 1, 1), + dt_utc(2020, 1, 2), + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 1, 23, 59), + False, + True, + False, + ), + ( + "1m", + False, + dt_utc(2019, 1, 1), + dt_utc(2019, 1, 2), + None, + None, + False, + True, + True, + ), + ( + "1m", + True, + dt_utc(2019, 1, 1), + dt_utc(2019, 1, 2), + None, + None, + True, + False, + False, + ), + ( + "1m", + False, + dt_utc(2021, 1, 1), + dt_utc(2021, 1, 2), + None, + None, + False, + False, + False, + ), + ( + "1m", + True, + dt_utc(2021, 1, 1), + dt_utc(2021, 1, 2), + None, + None, + True, + False, + False, + ), + ( + "1h", + False, + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 2), + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 1, 23, 59), + False, + False, + True, + ), + ], +) +def test_get_historic_ohlcv_binance( + mocker, + default_conf, + timeframe, + is_new_pair, + since, + until, + first_date, + last_date, + candle_called, + archive_called, + api_called, +): + exchange = get_patched_exchange(mocker, default_conf, exchange="binance") + + start = dt_utc(2020, 1, 1) + archive_end = dt_utc(2020, 1, 2) + api_end = dt_utc(2020, 1, 3) + candle_mock, api_mock, archive_mock = patch_ohlcv( + mocker, start=start, archive_end=archive_end, api_end=api_end + ) + + candle_type = CandleType.SPOT + pair = "BTC/USDT" + + since_ms = dt_ts(since) + until_ms = dt_ts(until) + + df = exchange.get_historic_ohlcv(pair, timeframe, since_ms, candle_type, is_new_pair, until_ms) + + if df.empty: + assert first_date is None + assert last_date is None + else: + assert df["date"].iloc[0] == first_date + assert df["date"].iloc[-1] == last_date + + if candle_called: + candle_mock.assert_called_once() + if archive_called: + archive_mock.assert_called_once() + if api_called: + api_mock.assert_called_once() + + @pytest.mark.xfail(reason="Need refactor") @pytest.mark.parametrize("candle_type", [CandleType.MARK, ""]) async def test__async_get_historic_ohlcv_binance(default_conf, mocker, caplog, candle_type): From 04d3633545ad2fa430c996a13a4ef78ee9041efc Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Wed, 30 Oct 2024 04:03:41 +0800 Subject: [PATCH 03/45] feat: implement fetch data from data.binance.vision --- freqtrade/exchange/binance_public_data.py | 179 +++++++++++++++- tests/exchange/test_binance_public_data.py | 192 ++++++++++++++++++ ...utures-um-klines-BTCUSDT-1h-2024-10-28.zip | Bin 0 -> 1533 bytes .../spot-klines-BTCUSDT-1h-2024-10-28.zip | Bin 0 -> 1578 bytes 4 files changed, 370 insertions(+), 1 deletion(-) create mode 100644 tests/exchange/test_binance_public_data.py create mode 100644 tests/testdata/binance/binance_public_data/futures-um-klines-BTCUSDT-1h-2024-10-28.zip create mode 100644 tests/testdata/binance/binance_public_data/spot-klines-BTCUSDT-1h-2024-10-28.zip diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index 23cb02989..adc1ea511 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -1,2 +1,179 @@ -async def fetch_ohlcv(*args, **kwargs): +""" +Fetch daily-archived OHLCV data from https://data.binance.vision/ +""" + +import asyncio +import datetime +import io +import itertools +import logging +import zipfile + +import aiohttp +import pandas as pd +from pandas import DataFrame + +from freqtrade.enums import CandleType +from freqtrade.util.datetime_helpers import dt_from_ts, dt_now + + +logger = logging.getLogger(__name__) + + +class BadHttpStatus(Exception): + """Not 200/404""" + pass + + +async def fetch_ohlcv( + candle_type: CandleType, pair: str, timeframe: str, since_ms: int, until_ms: int | None +) -> DataFrame: + """ + Fetch OHLCV data from https://data.binance.vision/ + :candle_type: Currently only spot and futures are supported + :param until_ms: `None` indicates the timestamp of the latest available data + :return: None if no data available in the time range + """ + if candle_type == CandleType.SPOT: + asset_type = "spot" + elif candle_type == CandleType.FUTURES: + asset_type = "futures/um" + else: + raise ValueError(f"Unsupported CandleType: {candle_type}") + symbol = symbol_ccxt_to_binance(pair) + start = dt_from_ts(since_ms) + end = dt_from_ts(until_ms) if until_ms else dt_now() + + # We use two days ago as the last available day because the daily archives are daily uploaded + # and have several hours delay + last_available_date = dt_now() - datetime.timedelta(days=2) + end = min(end, last_available_date) + if start >= end: + return DataFrame() + return await _fetch_ohlcv(asset_type, symbol, timeframe, start, end) + + +def symbol_ccxt_to_binance(symbol: str) -> str: + """ + Convert ccxt symbol notation to binance notation + e.g. BTC/USDT -> BTCUSDT, BTC/USDT:USDT -> BTCUSDT + """ + if ":" in symbol: + parts = symbol.split() + if len(parts) != 2: + raise ValueError(f"Cannot recognize symbol: {symbol}") + return parts[0].replace("/", "") + else: + return symbol.replace("/", "") + + +def concat(dfs) -> DataFrame: + if all(df is None for df in dfs): + return DataFrame() + else: + return pd.concat(dfs) + + +async def _fetch_ohlcv(asset_type, symbol, timeframe, start, end) -> DataFrame: + dfs: list[DataFrame | None] = [] + + connector = aiohttp.TCPConnector(limit=100) + async with aiohttp.ClientSession(connector=connector) as session: + coroutines = [ + get_daily_ohlcv(asset_type, symbol, timeframe, date, session) + for date in date_range(start, end) + ] + # the HTTP connections has been throttled by TCPConnector + for batch in itertools.batched(coroutines, 1000): + results = await asyncio.gather(*batch) + for result in results: + if isinstance(result, BaseException): + logger.warning(f"An exception raised: : {result}") + # Directly return the existing data, do not allow the gap + # between the data + return concat(dfs) + else: + dfs.append(result) + return concat(dfs) + + +def date_range(start: datetime.date, end: datetime.date): + date = start + while date <= end: + yield date + date += datetime.timedelta(days=1) + + +def format_date(date: datetime.date) -> str: + return date.strftime("%Y-%m-%d") + + +def zip_name(symbol: str, timeframe: str, date: datetime.date) -> str: + return f"{symbol}-{timeframe}-{format_date(date)}.zip" + + +async def get_daily_ohlcv( + asset_type: str, + symbol: str, + timeframe: str, + date: datetime.date, + session: aiohttp.ClientSession, + retry_count: int = 3, +) -> DataFrame | None: + """ + Get daily OHLCV from https://data.binance.vision + See https://github.com/binance/binance-public-data + """ + + # example urls: + # https://data.binance.vision/data/spot/daily/klines/BTCUSDT/1s/BTCUSDT-1s-2023-10-27.zip + # https://data.binance.vision/data/futures/um/daily/klines/BTCUSDT/1h/BTCUSDT-1h-2023-10-27.zip + url = ( + f"https://data.binance.vision/data/{asset_type}/daily/klines/{symbol}/{timeframe}/" + f"{zip_name(symbol, timeframe, date)}" + ) + + logger.debug(f"download data from binance: {url}") + + retry = 0 + while True: + if retry > 0: + sleep_secs = retry * 0.5 + logger.debug( + f"[{retry}/{retry_count}] retry to download {url} after {sleep_secs} seconds" + ) + await asyncio.sleep(sleep_secs) + try: + async with session.get(url) as resp: + if resp.status == 200: + content = await resp.read() + logger.debug(f"Successfully downloaded {url}") + with zipfile.ZipFile(io.BytesIO(content)) as zipf: + with zipf.open(zipf.namelist()[0]) as csvf: + # https://github.com/binance/binance-public-data/issues/283 + first_byte = csvf.read(1)[0] + if chr(first_byte).isdigit(): + header = None + else: + header = 0 + csvf.seek(0) + + df = pd.read_csv( + csvf, + usecols=[0, 1, 2, 3, 4, 5], + names=["date", "open", "high", "low", "close", "volume"], + header=header, + ) + df["date"] = pd.to_datetime(df["date"], unit="ms", utc=True) + return df + elif resp.status == 404: + logger.warning(f"No data available for {symbol} in {format_date(date)}") + return None + else: + raise BadHttpStatus(f"{resp.status} - {resp.reason}") + except Exception as e: + retry += 1 + if retry >= retry_count: + logger.warning(f"Failed to get data from {url}: {e}") + raise diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py new file mode 100644 index 000000000..5e0a414f2 --- /dev/null +++ b/tests/exchange/test_binance_public_data.py @@ -0,0 +1,192 @@ +import datetime +import io +import re +import zipfile +from datetime import timedelta + +import aiohttp +import pandas as pd +import pytest + +from freqtrade.enums import CandleType +from freqtrade.exchange.binance_public_data import ( + BadHttpStatus, + fetch_ohlcv, + get_daily_ohlcv, + zip_name, +) +from freqtrade.util.datetime_helpers import dt_ts, dt_utc + + +# spot klines archive csv file format, the futures/um klines don't have the header line +# +# open_time,open,high,low,close,volume,close_time,quote_volume,count,taker_buy_volume,taker_buy_quote_volume,ignore # noqa: E501 +# 1698364800000,34161.6,34182.5,33977.4,34024.2,409953,1698368399999,1202.97118037,15095,192220,564.12041453,0 # noqa: E501 +# 1698368400000,34024.2,34060.1,33776.4,33848.4,740960,1698371999999,2183.75671155,23938,368266,1085.17080793,0 # noqa: E501 +# 1698372000000,33848.5,34150.0,33815.1,34094.2,390376,1698375599999,1147.73267094,13854,231446,680.60405822,0 # noqa: E501 + + +def make_daily_df(date, timeframe): + start = dt_utc(date.year, date.month, date.day) + end = start + timedelta(days=1) + date_col = pd.date_range(start, end, freq=timeframe.replace("m", "min"), inclusive="left") + cols = ( + "open_time,open,high,low,close,volume,close_time,quote_volume,count,taker_buy_volume," + "taker_buy_quote_volume,ignore" + ) + df = pd.DataFrame(columns=cols.split(","), dtype=float) + df["open_time"] = date_col.astype("int64") // 10**6 + df["open"] = df["high"] = df["low"] = df["close"] = df["volume"] = 1.0 + return df + + +def make_daily_zip(asset_type, symbol, timeframe, date) -> bytes: + df = make_daily_df(date, timeframe) + if asset_type == "spot": + header = True + elif asset_type == "futures/um": + header = None + else: + raise ValueError + csv = df.to_csv(index=False, header=header) + zip_buffer = io.BytesIO() + with zipfile.ZipFile(zip_buffer, "w") as zipf: + zipf.writestr(zip_name(symbol, timeframe, date), csv) + return zip_buffer.getvalue() + + +class MockResponse: + def __init__(self, content, status, reason=""): + self._content = content + self.status = status + self.reason = reason + + async def read(self): + return self._content + + async def __aexit__(self, exc_type, exc, tb): + pass + + async def __aenter__(self): + return self + + +def make_response_from_url(start_date, end_date): + def make_response(url): + pattern = ( + r"https://data.binance.vision/data/(?Pspot|futures/um)/daily/klines/" + r"(?P.*?)/(?P.*?)/(?P=symbol)-(?P=timeframe)-" + r"(?P\d{4}-\d{2}-\d{2}).zip" + ) + m = re.match(pattern, url) + if not m: + return MockResponse(content="", status=404) + + date = datetime.datetime.strptime(m["date"], "%Y-%m-%d").date() + if date < start_date or date > end_date: + return MockResponse(content="", status=404) + + zip_file = make_daily_zip(m["asset_type"], m["symbol"], m["timeframe"], date) + return MockResponse(content=zip_file, status=200) + + return make_response + + +@pytest.mark.parametrize( + "since,until,first_date,last_date", + [ + (dt_utc(2020, 1, 1), dt_utc(2020, 1, 2), dt_utc(2020, 1, 1), dt_utc(2020, 1, 2, 23)), + ( + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 1, 23, 59, 59), + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 1, 23), + ), + ( + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 5), + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 3, 23), + ), + ( + dt_utc(2019, 1, 1), + dt_utc(2020, 1, 5), + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 3, 23), + ), + ( + dt_utc(2019, 1, 1), + dt_utc(2019, 1, 5), + None, + None, + ), + ( + dt_utc(2021, 1, 1), + dt_utc(2021, 1, 5), + None, + None, + ), + ( + dt_utc(2020, 1, 2), + None, + dt_utc(2020, 1, 2), + dt_utc(2020, 1, 3, 23), + ), + ], +) +async def test_fetch_ohlcv(mocker, since, until, first_date, last_date): + history_start = dt_utc(2020, 1, 1).date() + history_end = dt_utc(2020, 1, 3).date() + candle_type = CandleType.SPOT + pair = "BTC/USDT" + timeframe = "1h" + + since_ms = dt_ts(since) + until_ms = dt_ts(until) + + mocker.patch( + "aiohttp.ClientSession.get", side_effect=make_response_from_url(history_start, history_end) + ) + df = await fetch_ohlcv(candle_type, pair, timeframe, since_ms, until_ms) + + if df.empty: + assert first_date is None and last_date is None + else: + assert df["date"].iloc[0] == first_date + assert df["date"].iloc[-1] == last_date + + +async def test_get_daily_ohlcv(mocker, testdatadir): + symbol = "BTCUSDT" + timeframe = "1h" + date = dt_utc(2024, 10, 28).date() + first_date = dt_utc(2024, 10, 28) + last_date = dt_utc(2024, 10, 28, 23) + + async with aiohttp.ClientSession() as session: + path = testdatadir / "binance/binance_public_data/spot-klines-BTCUSDT-1h-2024-10-28.zip" + mocker.patch("aiohttp.ClientSession.get", return_value=MockResponse(path.read_bytes(), 200)) + df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) + assert df["date"].iloc[0] == first_date + assert df["date"].iloc[-1] == last_date + + path = ( + testdatadir / "binance/binance_public_data/futures-um-klines-BTCUSDT-1h-2024-10-28.zip" + ) + mocker.patch("aiohttp.ClientSession.get", return_value=MockResponse(path.read_bytes(), 200)) + df = await get_daily_ohlcv("futures/um", symbol, timeframe, date, session) + assert df["date"].iloc[0] == first_date + assert df["date"].iloc[-1] == last_date + + mocker.patch("aiohttp.ClientSession.get", return_value=MockResponse(b"", 404)) + df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) + assert df is None + + mocker.patch("aiohttp.ClientSession.get", return_value=MockResponse(b"", 500)) + mocker.patch("asyncio.sleep") + with pytest.raises(BadHttpStatus): + df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) + + mocker.patch("aiohttp.ClientSession.get", return_value=MockResponse(b"nop", 200)) + with pytest.raises(zipfile.BadZipFile): + df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) diff --git a/tests/testdata/binance/binance_public_data/futures-um-klines-BTCUSDT-1h-2024-10-28.zip b/tests/testdata/binance/binance_public_data/futures-um-klines-BTCUSDT-1h-2024-10-28.zip new file mode 100644 index 0000000000000000000000000000000000000000..5bda1b27101dedd7d639726b70ebb2229076c411 GIT binary patch literal 1533 zcmaLXdpOez7zgm*%q^3V+gx)?ib;!+<}%AAOD$Q&$lYeIKo8Vn;M39+690sqtuAML%RY2K!Q_X01%wq zS&`>@gJ8zuE#C-&bZELvn>GoexhtnDkj&zK3@^JCmVx~o(3wKy)0;3 z%de{s3ljH!*#5E1|Ofq$b%cF8^@2@@cbWB;K9r1sVn@x5rQ?L#DL>et^ceZdy zK08bc8}uO$*_HHo@O%cxMD{u4P7)=^b?F$16=dr3HS-0@zDTnUEDeV8JAV%n;poan zOgY$GflhU9nR+AD>AE0eBb#{6CCBf{sO9E#)<&p$b<8+Ag;sp@L9@zCk} z+N+@_#8og#eoyKhjVm>f0CTqY1iJ<_R6l2x@;Pyi@5=VTz>QaA5os|_J{Qt+pRO2P z!WwvsHB4~~I)`kTcJL15uSSqc^n{wSZTj?pR;eDr3vb>XXP z?AH+xKF1;ai^T`hL$KC#EFbcx6x&8s6c!pbKO|nDQ5WJ-tGmMBTblBb#cTd7w#QUF z1x->BC+G4Qb?&KL=0`m>#3ZM|T9`rtTBfK1BWan|XVJ1<23*&Y%fTvrD>3}bn!&3> zI1TJQs~YlE#U!;woMg&rWN1)2{tM2aB9i&sP|IAK4054C%iKvD!rc@UJK}glUv)OU zQ7j;5i53&SAT0hDW4glMZ}K3R75vqBKoYca1P zO*47ZPcLI7=rW*a21stckSC|u$rpsjcGMpXq*oN%E4I^wc(u4fvCG=1>;2tT_0f-H zkC7?@ua%bI(4R#RS6U>Q333rP;*SaNnfo0BL|8n?8k7qeIxJpE;Y_JNZAK9 zVfE%$ye`NoBn{|9awXg%dg1j9!3W{-Ri=`k7Bk7+Nz73sZVr}rW>2Ar(u+%vjDfC$ zjV@zgLpS>*{G8-|hUllsE>%J>D0tvnt+2Q(x-2+9yMSH5xv{NUhDh`Cg3g-;wFd&H zxeL+CMvN?D?_^}(GZ!s>r&%foCqP=&%KrL9JD{WFS!qJl-M5soktM5M{rupi2wtG3 zrDRL)ypMC-Oz)v#v}~)}SPW~TS@^IZq|IoJ&Er)EaO1v9K zw+dep0!__`v2+?_(ee(g*7$Lc)Efl!{J@?}79<~wN)xXxk1yxdqXJ27r5cI}MXM%q z)%BsCb*cECTf^i$=0&r&iW-|G{XfcxXuUNiM=T2|?R#@Uin;NcqsrtCvW?E2>d?FO zQdtv!cX#dH#DpsCQXR6>zHNQ(F93%H38;eqOJdlL&;S71NeTV;^*=5P{p0e#OvYh_ SgnvJRcE)|D<9Cn;fWHCTt-L1y literal 0 HcmV?d00001 diff --git a/tests/testdata/binance/binance_public_data/spot-klines-BTCUSDT-1h-2024-10-28.zip b/tests/testdata/binance/binance_public_data/spot-klines-BTCUSDT-1h-2024-10-28.zip new file mode 100644 index 0000000000000000000000000000000000000000..b94090741ef3e3fee9aeb72c96313a5a9b351777 GIT binary patch literal 1578 zcmZ{lYdjMQ0L3Q_MP(YgH|CXH*E|=7A|cXLLLS{^+LnY+Uh|m9qlecNn&y!f@>-7> zd9P7h%S5GG#S*sNY$eNLrCr_co%7-Re&=^S|BoM1c`XJQjMb9J!t&Payc|RxUO_JHJ`R&~U zc?jvu{1--xtLuEbaPRh0F6kSjnP3~K=Ik_Zehy?Ft*yA;WGhsrrmeK9z7)5`#iQbc zE!38YH@$JmEneu6Y2b)t4Z{0;ek3j8(sS*9bvVGS+=e8lTBX$Ef#eFobHvVXN5}Yh zL0JR&2$7CoJm6iRT4P;zniJ4361h8}8lKN3pho70(;g^21s5ZxaOpSzXA%^KL@WzB4Xf((y zCP#fGw8R8bO8C)u1s&{iQjUqp=^?okVl@zUK>>9?vyPE9qZ^f{J;YIv?hmH$q#jJr zN`8RnaBciOy*ji(vWmOIGmL!0!S1jQO-hq~J)O4VzcN}k^IB=S4O_T5JnE8$u>Ku7 zs~f;c`x?C8&W(m(z}p5&95aRs&OeQwUU@%V%z}I@FdVr+sN~Y%E?1&kxvQV&1_Wdp;6qRGrOw)kDBs&b(X@ ziLReWU>MK?t5G&(_T+iGq+~<=u_XGN!zS%~NH1 zOhN-S(4;ge+F7Gdf zeQO#Fp18q;KVsyfV+REEAslFHU9!Sl9v-fenNnCMbG~uiQf0AeE;{P%w=>pi=P#ve zQL3#LwYwv!;M|UTyxWYb?4|-LAl~ZWE;WMgIN~O~B*TZ=4^|@)%76x-kW;xPpCC-z zYuC8;(rvsY?%O7} zGvYOD1uSuG=k~n6`fN=@*3aJiD5`fCI$YL+vc<#twoo|@T6Q+Uw~YQ;0%@Pgwr2M? zMW4FLm{xtCb-$3_MbsJA0vmXJ0J4^;a)N6L$eEN{GvS2*8T1D2uREkEC(bOjrck()9YU)?Aj&^YKg^D2VerY z=<2Z>pF@1Kv^VifXOK>~NA$m|KNqEUeH1sqqAP1Gv`~XaT|rFf$8R@3-7?iDrEfTJ!b$HtX?XO$M_)Ke!tFdyn8fBzJ(jVG8BB3&W??B~=)!>! z?oZ#{&~~y-r*m3$2kQc#veJBC+fjh0?0LZbkaT#`gOpnZ#=p;-#Ns4Nx#|JM2I}Hv zRq3`zQ5HYas?aw>3Q6M$BC=SJ_wwmE3bq_Ez>l|?_J5ItJ1GoayNwb}dCZB? z9K1(tFL-;kXPFl|nhW3Qt5<0j8g7*E}Jq7&hHoWtLEOfkhG zODqL%?#8QWgnr2PS9v%uO_}}boKfV8S*GFkTaCT4*Vvr`9ai`umDYfje~ Date: Wed, 30 Oct 2024 07:24:10 +0800 Subject: [PATCH 04/45] tests: binance compare data.binance.vision to rest API --- freqtrade/exchange/binance_public_data.py | 21 +++-- .../test_binance_compare_ohlcv.py | 87 +++++++++++++++++++ 2 files changed, 101 insertions(+), 7 deletions(-) create mode 100644 tests/exchange_online/test_binance_compare_ohlcv.py diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index adc1ea511..b2b449510 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -113,6 +113,19 @@ def zip_name(symbol: str, timeframe: str, date: datetime.date) -> str: return f"{symbol}-{timeframe}-{format_date(date)}.zip" +def zip_url(asset_type: str, symbol: str, timeframe: str, date: datetime.date) -> str: + """ + example urls: + https://data.binance.vision/data/spot/daily/klines/BTCUSDT/1s/BTCUSDT-1s-2023-10-27.zip + https://data.binance.vision/data/futures/um/daily/klines/BTCUSDT/1h/BTCUSDT-1h-2023-10-27.zip + """ + url = ( + f"https://data.binance.vision/data/{asset_type}/daily/klines/{symbol}/{timeframe}/" + f"{zip_name(symbol, timeframe, date)}" + ) + return url + + async def get_daily_ohlcv( asset_type: str, symbol: str, @@ -126,13 +139,7 @@ async def get_daily_ohlcv( See https://github.com/binance/binance-public-data """ - # example urls: - # https://data.binance.vision/data/spot/daily/klines/BTCUSDT/1s/BTCUSDT-1s-2023-10-27.zip - # https://data.binance.vision/data/futures/um/daily/klines/BTCUSDT/1h/BTCUSDT-1h-2023-10-27.zip - url = ( - f"https://data.binance.vision/data/{asset_type}/daily/klines/{symbol}/{timeframe}/" - f"{zip_name(symbol, timeframe, date)}" - ) + url = zip_url(asset_type, symbol, timeframe, date) logger.debug(f"download data from binance: {url}") diff --git a/tests/exchange_online/test_binance_compare_ohlcv.py b/tests/exchange_online/test_binance_compare_ohlcv.py new file mode 100644 index 000000000..7a76e19f2 --- /dev/null +++ b/tests/exchange_online/test_binance_compare_ohlcv.py @@ -0,0 +1,87 @@ +""" +Check if the earliest klines from rest API have its counterpart on https://data.binance.vision + +Not expected to run in CI + +Manually run from shell: +TEST_BINANCE_COMPARE_OHLCV=1 pytest tests/exchange_online/test_binance_compare_ohlcv.py +""" + +import asyncio +import os + +import aiohttp +import pytest + +from freqtrade.exchange.binance_public_data import zip_url +from freqtrade.util.datetime_helpers import dt_from_ts + + +class Check: + def __init__(self, asset_type, timeframe): + self.asset_type = asset_type + self.timeframe = timeframe + self.klines_endpoint = "https://api.binance.com/api/v3/klines" + self.exchange_endpoint = "https://api.binance.com/api/v3/exchangeInfo" + self.mismatch = set() + + if asset_type == "futures/um": + self.klines_endpoint = "https://fapi.binance.com/fapi/v1/klines" + self.exchange_endpoint = "https://fapi.binance.com/fapi/v1/exchangeInfo" + + async def check_one_symbol(self, symbol): + async with self.session.get( + self.klines_endpoint, params=dict(symbol=symbol, interval=self.timeframe, startTime=0) + ) as resp: + resp.raise_for_status() + json = await resp.json() + first_kline = json[0] + first_kline_ts = first_kline[0] + date = dt_from_ts(first_kline_ts).date() + + archive_url = zip_url(self.asset_type, symbol=symbol, timeframe=self.timeframe, date=date) + async with self.session.get( + archive_url, params=dict(symbol=symbol, interval=self.timeframe, startTime=0) + ) as resp: + if resp.status != 200: + self.mismatch.add(symbol) + print( + f"{resp.status} API first kline: {dt_from_ts(first_kline_ts).isoformat()} " + f"{archive_url}" + ) + web_url = archive_url.rsplit("/", 1)[0].replace( + "https://data.binance.vision/", "https://data.binance.vision/?prefix=" + ) + print(f"Check {web_url}") + + async def get_symbols(self): + async with self.session.get(self.exchange_endpoint) as resp: + resp.raise_for_status() + json = await resp.json() + symbols = [ + symbol["symbol"] + for symbol in json["symbols"] + if not symbol["status"] == "PENDING_TRADING" + ] + return symbols + + async def run(self) -> list: + async with aiohttp.ClientSession() as session: + self.session = session + symbols = await self.get_symbols() + await asyncio.gather(*[self.check_one_symbol(symbol) for symbol in symbols]) + return self.mismatch + + +@pytest.mark.skipif( + not bool(os.environ.get("TEST_BINANCE_COMPARE_OHLCV")), + reason="Simply to demonstrate the availabity of the archive endpoint", +) +async def test_binance_compare_ohlcv(): + futures_mismatch = await Check("futures/um", "1m").run() + assert futures_mismatch == set(["BTCUSDT", "ETHUSDT", "BCHUSDT"]) + + spot_mismatch = await Check("spot", "1m").run() + assert not spot_mismatch + + assert 0 From e49b5b03dba5ef46c13c3c63054208762f8ec2f2 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Wed, 30 Oct 2024 07:59:06 +0800 Subject: [PATCH 05/45] feat: stop on 404 to prevent missing data --- freqtrade/exchange/binance.py | 1 + freqtrade/exchange/binance_public_data.py | 28 +++++++++++++++++----- tests/exchange/test_binance_public_data.py | 21 ++++++++++++---- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 86accf055..65baf7c2c 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -142,6 +142,7 @@ class Binance(Exchange): timeframe=timeframe, since_ms=since_ms, until_ms=until_ms, + stop_on_404=True, ) ) if df.empty: diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index b2b449510..ba9d2f2e9 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -27,12 +27,18 @@ class BadHttpStatus(Exception): async def fetch_ohlcv( - candle_type: CandleType, pair: str, timeframe: str, since_ms: int, until_ms: int | None + candle_type: CandleType, + pair: str, + timeframe: str, + since_ms: int, + until_ms: int | None, + stop_on_404: bool = False, ) -> DataFrame: """ Fetch OHLCV data from https://data.binance.vision/ :candle_type: Currently only spot and futures are supported :param until_ms: `None` indicates the timestamp of the latest available data + :param stop_on_404: Stop to download the following data when a 404 returned :return: None if no data available in the time range """ if candle_type == CandleType.SPOT: @@ -51,7 +57,7 @@ async def fetch_ohlcv( end = min(end, last_available_date) if start >= end: return DataFrame() - return await _fetch_ohlcv(asset_type, symbol, timeframe, start, end) + return await _fetch_ohlcv(asset_type, symbol, timeframe, start, end, stop_on_404) def symbol_ccxt_to_binance(symbol: str) -> str: @@ -60,7 +66,7 @@ def symbol_ccxt_to_binance(symbol: str) -> str: e.g. BTC/USDT -> BTCUSDT, BTC/USDT:USDT -> BTCUSDT """ if ":" in symbol: - parts = symbol.split() + parts = symbol.split(":") if len(parts) != 2: raise ValueError(f"Cannot recognize symbol: {symbol}") return parts[0].replace("/", "") @@ -75,7 +81,14 @@ def concat(dfs) -> DataFrame: return pd.concat(dfs) -async def _fetch_ohlcv(asset_type, symbol, timeframe, start, end) -> DataFrame: +async def _fetch_ohlcv( + asset_type: str, + symbol: str, + timeframe: str, + start: datetime.date, + end: datetime.date, + stop_on_404: bool, +) -> DataFrame: dfs: list[DataFrame | None] = [] connector = aiohttp.TCPConnector(limit=100) @@ -93,6 +106,9 @@ async def _fetch_ohlcv(asset_type, symbol, timeframe, start, end) -> DataFrame: # Directly return the existing data, do not allow the gap # between the data return concat(dfs) + elif result is None and stop_on_404: + logger.debug("Abort downloading from data.binance.vision due to 404") + return concat(dfs) else: dfs.append(result) return concat(dfs) @@ -175,12 +191,12 @@ async def get_daily_ohlcv( df["date"] = pd.to_datetime(df["date"], unit="ms", utc=True) return df elif resp.status == 404: - logger.warning(f"No data available for {symbol} in {format_date(date)}") + logger.debug(f"No data available for {symbol} in {format_date(date)}") return None else: raise BadHttpStatus(f"{resp.status} - {resp.reason}") except Exception as e: retry += 1 if retry >= retry_count: - logger.warning(f"Failed to get data from {url}: {e}") + logger.debug(f"Failed to get data from {url}: {e}") raise diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py index 5e0a414f2..995f562a7 100644 --- a/tests/exchange/test_binance_public_data.py +++ b/tests/exchange/test_binance_public_data.py @@ -93,48 +93,61 @@ def make_response_from_url(start_date, end_date): @pytest.mark.parametrize( - "since,until,first_date,last_date", + "since,until,first_date,last_date,stop_on_404", [ - (dt_utc(2020, 1, 1), dt_utc(2020, 1, 2), dt_utc(2020, 1, 1), dt_utc(2020, 1, 2, 23)), + (dt_utc(2020, 1, 1), dt_utc(2020, 1, 2), dt_utc(2020, 1, 1), dt_utc(2020, 1, 2, 23), False), ( dt_utc(2020, 1, 1), dt_utc(2020, 1, 1, 23, 59, 59), dt_utc(2020, 1, 1), dt_utc(2020, 1, 1, 23), + False, ), ( dt_utc(2020, 1, 1), dt_utc(2020, 1, 5), dt_utc(2020, 1, 1), dt_utc(2020, 1, 3, 23), + False, ), ( dt_utc(2019, 1, 1), dt_utc(2020, 1, 5), dt_utc(2020, 1, 1), dt_utc(2020, 1, 3, 23), + False, ), ( dt_utc(2019, 1, 1), dt_utc(2019, 1, 5), None, None, + False, ), ( dt_utc(2021, 1, 1), dt_utc(2021, 1, 5), None, None, + False, ), ( dt_utc(2020, 1, 2), None, dt_utc(2020, 1, 2), dt_utc(2020, 1, 3, 23), + False, + ), + ( + dt_utc(2019, 1, 1), + dt_utc(2020, 1, 5), + None, + None, + True, ), ], ) -async def test_fetch_ohlcv(mocker, since, until, first_date, last_date): +async def test_fetch_ohlcv(mocker, since, until, first_date, last_date, stop_on_404): history_start = dt_utc(2020, 1, 1).date() history_end = dt_utc(2020, 1, 3).date() candle_type = CandleType.SPOT @@ -147,7 +160,7 @@ async def test_fetch_ohlcv(mocker, since, until, first_date, last_date): mocker.patch( "aiohttp.ClientSession.get", side_effect=make_response_from_url(history_start, history_end) ) - df = await fetch_ohlcv(candle_type, pair, timeframe, since_ms, until_ms) + df = await fetch_ohlcv(candle_type, pair, timeframe, since_ms, until_ms, stop_on_404) if df.empty: assert first_date is None and last_date is None From a417698fcd7dea82405c05daa511f165d41be43c Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Wed, 30 Oct 2024 09:59:13 +0800 Subject: [PATCH 06/45] tests: fix test and improve coverage --- freqtrade/exchange/binance.py | 3 +- freqtrade/exchange/binance_public_data.py | 16 +++- freqtrade/exchange/exchange.py | 2 +- tests/exchange/test_binance.py | 73 +++++---------- tests/exchange/test_binance_public_data.py | 92 ++++++++++++++++--- tests/exchange/test_exchange.py | 2 +- .../test_binance_compare_ohlcv.py | 19 +++- 7 files changed, 134 insertions(+), 73 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 65baf7c2c..1b227265a 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -142,14 +142,13 @@ class Binance(Exchange): timeframe=timeframe, since_ms=since_ms, until_ms=until_ms, - stop_on_404=True, + stop_on_404=False, ) ) if df.empty: rest_since_ms = since_ms else: rest_since_ms = dt_ts(df.iloc[-1].date) + timeframe_to_msecs(timeframe) - if until_ms and rest_since_ms > until_ms: rest_df = DataFrame() else: diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index ba9d2f2e9..007886606 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -39,7 +39,8 @@ async def fetch_ohlcv( :candle_type: Currently only spot and futures are supported :param until_ms: `None` indicates the timestamp of the latest available data :param stop_on_404: Stop to download the following data when a 404 returned - :return: None if no data available in the time range + :return: the date range is between [since_ms, until_ms), + return None if no data available in the time range """ if candle_type == CandleType.SPOT: asset_type = "spot" @@ -57,7 +58,14 @@ async def fetch_ohlcv( end = min(end, last_available_date) if start >= end: return DataFrame() - return await _fetch_ohlcv(asset_type, symbol, timeframe, start, end, stop_on_404) + df = await _fetch_ohlcv(asset_type, symbol, timeframe, start, end, stop_on_404) + logger.info( + f"Downloaded data for {pair} from https://data.binance.vision/ with length {len(df)}." + ) + if not df.empty: + return df.loc[(df["date"] >= start) & (df["date"] < end)] + else: + return df def symbol_ccxt_to_binance(symbol: str) -> str: @@ -149,7 +157,7 @@ async def get_daily_ohlcv( date: datetime.date, session: aiohttp.ClientSession, retry_count: int = 3, -) -> DataFrame | None: +) -> DataFrame | None | Exception: """ Get daily OHLCV from https://data.binance.vision See https://github.com/binance/binance-public-data @@ -199,4 +207,4 @@ async def get_daily_ohlcv( retry += 1 if retry >= retry_count: logger.debug(f"Failed to get data from {url}: {e}") - raise + return e diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 48bb84369..5e1e32a54 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2246,7 +2246,7 @@ class Exchange: candle_type=candle_type, ) ) - logger.info(f"Downloaded data for {pair} with length {len(data)}.") + logger.info(f"Downloaded data for {pair} from ccxt with length {len(data)}.") return ohlcv_to_dataframe(data, timeframe, pair, fill_missing=False, drop_incomplete=True) async def _async_get_historic_ohlcv( diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index 4dfc4d6db..749dff663 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta from random import randint from unittest.mock import AsyncMock, MagicMock, PropertyMock @@ -8,9 +8,10 @@ import pytest from freqtrade.enums import CandleType, MarginMode, TradingMode from freqtrade.exceptions import DependencyException, InvalidOrderException, OperationalException +from freqtrade.exchange.exchange_utils_timeframe import timeframe_to_seconds from freqtrade.persistence import Trade from freqtrade.util.datetime_helpers import dt_from_ts, dt_ts, dt_utc -from tests.conftest import EXMS, get_mock_coro, get_patched_exchange, log_has_re +from tests.conftest import EXMS, get_patched_exchange from tests.exchange.test_exchange import ccxt_exceptionhandlers @@ -733,17 +734,17 @@ def test__set_leverage_binance(mocker, default_conf): ) -def make_storage(start: datetime, end: datetime, timeframe: str = "1min"): - date = pd.date_range(start, end, freq=timeframe) +def make_storage(start: datetime, end: datetime, timeframe: str): + date = pd.date_range(start, end, freq=timeframe.replace("m", "min")) df = pd.DataFrame( data=dict(date=date, open=1.0, high=1.0, low=1.0, close=1.0), ) return df -def patch_ohlcv(mocker, start, archive_end, api_end): - archive_storage = make_storage(start, archive_end) - api_storage = make_storage(start, api_end) +def patch_ohlcv(mocker, start, archive_end, api_end, timeframe): + archive_storage = make_storage(start, archive_end, timeframe) + api_storage = make_storage(start, api_end, timeframe) ohlcv = [[dt_ts(start), 1, 1, 1, 1]] # (pair, timeframe, candle_type, ohlcv, True) @@ -768,6 +769,7 @@ def patch_ohlcv(mocker, start, archive_end, api_end): timeframe, since_ms, until_ms, + stop_on_404=False, ): since = dt_from_ts(since_ms) until = dt_from_ts(until_ms) if until_ms else archive_end + timedelta(seconds=1) @@ -909,11 +911,22 @@ def patch_ohlcv(mocker, start, archive_end, api_end): dt_utc(2020, 1, 1), dt_utc(2020, 1, 2), dt_utc(2020, 1, 1), - dt_utc(2020, 1, 1, 23, 59), + dt_utc(2020, 1, 1, 23), False, False, True, ), + ( + "1m", + False, + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 1, 3, 50, 30), + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 1, 3, 50), + False, + True, + False, + ), ], ) def test_get_historic_ohlcv_binance( @@ -935,7 +948,7 @@ def test_get_historic_ohlcv_binance( archive_end = dt_utc(2020, 1, 2) api_end = dt_utc(2020, 1, 3) candle_mock, api_mock, archive_mock = patch_ohlcv( - mocker, start=start, archive_end=archive_end, api_end=api_end + mocker, start=start, archive_end=archive_end, api_end=api_end, timeframe=timeframe ) candle_type = CandleType.SPOT @@ -952,6 +965,9 @@ def test_get_historic_ohlcv_binance( else: assert df["date"].iloc[0] == first_date assert df["date"].iloc[-1] == last_date + assert ( + df["date"].diff().iloc[1:] == timedelta(seconds=timeframe_to_seconds(timeframe)) + ).all() if candle_called: candle_mock.assert_called_once() @@ -961,45 +977,6 @@ def test_get_historic_ohlcv_binance( api_mock.assert_called_once() -@pytest.mark.xfail(reason="Need refactor") -@pytest.mark.parametrize("candle_type", [CandleType.MARK, ""]) -async def test__async_get_historic_ohlcv_binance(default_conf, mocker, caplog, candle_type): - ohlcv = [ - [ - int((datetime.now(timezone.utc).timestamp() - 1000) * 1000), - 1, # open - 2, # high - 3, # low - 4, # close - 5, # volume (in quote currency) - ] - ] - - exchange = get_patched_exchange(mocker, default_conf, exchange="binance") - # Monkey-patch async function - exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) - - pair = "ETH/BTC" - respair, restf, restype, res, _ = await exchange._async_get_historic_ohlcv( - pair, "5m", 1500000000000, is_new_pair=False, candle_type=candle_type - ) - assert respair == pair - assert restf == "5m" - assert restype == candle_type - # Call with very old timestamp - causes tons of requests - assert exchange._api_async.fetch_ohlcv.call_count > 400 - # assert res == ohlcv - exchange._api_async.fetch_ohlcv.reset_mock() - _, _, _, res, _ = await exchange._async_get_historic_ohlcv( - pair, "5m", 1500000000000, is_new_pair=True, candle_type=candle_type - ) - - # Called twice - one "init" call - and one to get the actual data. - assert exchange._api_async.fetch_ohlcv.call_count == 2 - assert res == ohlcv - assert log_has_re(r"Candle-data for ETH/BTC available starting with .*", caplog) - - @pytest.mark.parametrize( "pair,notional_value,mm_ratio,amt", [ diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py index 995f562a7..f598a0bdd 100644 --- a/tests/exchange/test_binance_public_data.py +++ b/tests/exchange/test_binance_public_data.py @@ -13,6 +13,7 @@ from freqtrade.exchange.binance_public_data import ( BadHttpStatus, fetch_ohlcv, get_daily_ohlcv, + symbol_ccxt_to_binance, zip_name, ) from freqtrade.util.datetime_helpers import dt_ts, dt_utc @@ -93,10 +94,18 @@ def make_response_from_url(start_date, end_date): @pytest.mark.parametrize( - "since,until,first_date,last_date,stop_on_404", + "candle_type,since,until,first_date,last_date,stop_on_404", [ - (dt_utc(2020, 1, 1), dt_utc(2020, 1, 2), dt_utc(2020, 1, 1), dt_utc(2020, 1, 2, 23), False), ( + CandleType.SPOT, + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 2), + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 1, 23), + False, + ), + ( + CandleType.SPOT, dt_utc(2020, 1, 1), dt_utc(2020, 1, 1, 23, 59, 59), dt_utc(2020, 1, 1), @@ -104,6 +113,7 @@ def make_response_from_url(start_date, end_date): False, ), ( + CandleType.SPOT, dt_utc(2020, 1, 1), dt_utc(2020, 1, 5), dt_utc(2020, 1, 1), @@ -111,6 +121,7 @@ def make_response_from_url(start_date, end_date): False, ), ( + CandleType.SPOT, dt_utc(2019, 1, 1), dt_utc(2020, 1, 5), dt_utc(2020, 1, 1), @@ -118,6 +129,7 @@ def make_response_from_url(start_date, end_date): False, ), ( + CandleType.SPOT, dt_utc(2019, 1, 1), dt_utc(2019, 1, 5), None, @@ -125,6 +137,7 @@ def make_response_from_url(start_date, end_date): False, ), ( + CandleType.SPOT, dt_utc(2021, 1, 1), dt_utc(2021, 1, 5), None, @@ -132,6 +145,7 @@ def make_response_from_url(start_date, end_date): False, ), ( + CandleType.SPOT, dt_utc(2020, 1, 2), None, dt_utc(2020, 1, 2), @@ -139,20 +153,44 @@ def make_response_from_url(start_date, end_date): False, ), ( + CandleType.SPOT, dt_utc(2019, 1, 1), dt_utc(2020, 1, 5), None, None, True, ), + ( + CandleType.SPOT, + dt_utc(2020, 1, 5), + dt_utc(2020, 1, 1), + None, + None, + False, + ), + ( + CandleType.FUTURES, + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 1, 23, 59, 59), + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 1, 23), + False, + ), + ( + CandleType.INDEX, + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 1, 23, 59, 59), + None, + None, + False, + ), ], ) -async def test_fetch_ohlcv(mocker, since, until, first_date, last_date, stop_on_404): +async def test_fetch_ohlcv(mocker, candle_type, since, until, first_date, last_date, stop_on_404): history_start = dt_utc(2020, 1, 1).date() history_end = dt_utc(2020, 1, 3).date() - candle_type = CandleType.SPOT - pair = "BTC/USDT" timeframe = "1h" + pair = "BTCUSDT" since_ms = dt_ts(since) until_ms = dt_ts(until) @@ -160,13 +198,32 @@ async def test_fetch_ohlcv(mocker, since, until, first_date, last_date, stop_on_ mocker.patch( "aiohttp.ClientSession.get", side_effect=make_response_from_url(history_start, history_end) ) - df = await fetch_ohlcv(candle_type, pair, timeframe, since_ms, until_ms, stop_on_404) - if df.empty: - assert first_date is None and last_date is None + if candle_type in [CandleType.SPOT, CandleType.FUTURES]: + df = await fetch_ohlcv(candle_type, pair, timeframe, since_ms, until_ms, stop_on_404) + + if df.empty: + assert first_date is None and last_date is None + else: + assert df["date"].iloc[0] == first_date + assert df["date"].iloc[-1] == last_date else: - assert df["date"].iloc[0] == first_date - assert df["date"].iloc[-1] == last_date + with pytest.raises(ValueError): + await fetch_ohlcv(candle_type, pair, timeframe, since_ms, until_ms, stop_on_404) + + +async def test_fetch_ohlcv_exc(mocker): + timeframe = "1h" + pair = "BTCUSDT" + + since_ms = dt_ts(dt_utc(2020, 1, 1)) + until_ms = dt_ts(dt_utc(2020, 1, 2)) + + mocker.patch("aiohttp.ClientSession.get", side_effect=RuntimeError) + + df = await fetch_ohlcv(CandleType.SPOT, pair, timeframe, since_ms, until_ms) + + assert df.empty async def test_get_daily_ohlcv(mocker, testdatadir): @@ -197,9 +254,16 @@ async def test_get_daily_ohlcv(mocker, testdatadir): mocker.patch("aiohttp.ClientSession.get", return_value=MockResponse(b"", 500)) mocker.patch("asyncio.sleep") - with pytest.raises(BadHttpStatus): - df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) + df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) + assert isinstance(df, BadHttpStatus) mocker.patch("aiohttp.ClientSession.get", return_value=MockResponse(b"nop", 200)) - with pytest.raises(zipfile.BadZipFile): - df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) + df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) + assert isinstance(df, zipfile.BadZipFile) + + +def test_symbol_ccxt_to_binance(): + assert symbol_ccxt_to_binance("BTC/USDT") == "BTCUSDT" + assert symbol_ccxt_to_binance("BTC/USDT:USDT") == "BTCUSDT" + with pytest.raises(ValueError): + assert symbol_ccxt_to_binance("BTC:USDT:USDT") == "BTCUSDT" diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 07d5927c2..b123229ad 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -2124,7 +2124,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name, candle_ assert exchange._async_get_candle_history.call_count == 2 # Returns twice the above OHLCV data after truncating the open candle. assert len(ret) == 2 - assert log_has_re(r"Downloaded data for .* with length .*\.", caplog) + assert log_has_re(r"Downloaded data for .* from ccxt with length .*\.", caplog) caplog.clear() diff --git a/tests/exchange_online/test_binance_compare_ohlcv.py b/tests/exchange_online/test_binance_compare_ohlcv.py index 7a76e19f2..af94b1bd1 100644 --- a/tests/exchange_online/test_binance_compare_ohlcv.py +++ b/tests/exchange_online/test_binance_compare_ohlcv.py @@ -1,10 +1,23 @@ """ Check if the earliest klines from rest API have its counterpart on https://data.binance.vision +Not expected to run in CI, manually run from shell: -Not expected to run in CI + TEST_BINANCE_COMPARE_OHLCV=1 pytest tests/exchange_online/test_binance_compare_ohlcv.py -Manually run from shell: -TEST_BINANCE_COMPARE_OHLCV=1 pytest tests/exchange_online/test_binance_compare_ohlcv.py +Until 2024-10-30, there are three usdt-m futures symbols "lack" data +All SPOT symbols are good. + +BTCUSDT-1m 113 days +ARCHIVE: 2019-12-31 00:00:00 │ 2024-10-30 02:51:00 │ 2541772 +API: 2019-09-08 17:57:00 │ 2024-10-30 03:11:00 │ 2704874 + +ETHUSDT 34 days +ARCHIVE: 2019-12-31 00:00:00 │ 2020-02-29 23:59:00 │ 87840 +API: 2019-11-27 07:45:00 │ 2020-03-01 11:03:00 │ 136999 + +BCHUSDT 12 days +ARCHIVE: 2019-12-31 00:00:00 │ 2020-02-29 23:59:00 │ 87840 +API: 2019-12-19 08:57:00 │ 2020-03-01 06:55:00 │ 104999 """ import asyncio From 1aa863a92f683eca0ef351b73178907e73aec726 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Wed, 30 Oct 2024 22:11:37 +0800 Subject: [PATCH 07/45] tests: fix --- freqtrade/exchange/binance_public_data.py | 4 ++-- tests/data/test_history.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index 007886606..ea4f4bc58 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -5,7 +5,6 @@ Fetch daily-archived OHLCV data from https://data.binance.vision/ import asyncio import datetime import io -import itertools import logging import zipfile @@ -14,6 +13,7 @@ import pandas as pd from pandas import DataFrame from freqtrade.enums import CandleType +from freqtrade.misc import chunks from freqtrade.util.datetime_helpers import dt_from_ts, dt_now @@ -106,7 +106,7 @@ async def _fetch_ohlcv( for date in date_range(start, end) ] # the HTTP connections has been throttled by TCPConnector - for batch in itertools.batched(coroutines, 1000): + for batch in chunks(coroutines, 1000): results = await asyncio.gather(*batch) for result in results: if isinstance(result, BaseException): diff --git a/tests/data/test_history.py b/tests/data/test_history.py index b505c4fe3..465a0ee1a 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -128,8 +128,8 @@ def test_load_data_with_new_pair_1min( """ Test load_pair_history() with 1 min timeframe """ - mocker.patch(f"{EXMS}.get_historic_ohlcv", return_value=ohlcv_history) exchange = get_patched_exchange(mocker, default_conf) + mocker.patch.object(exchange, "get_historic_ohlcv", return_value=ohlcv_history) file = tmp_path / "MEME_BTC-1m.feather" # do not download a new pair if refresh_pairs isn't set @@ -305,8 +305,8 @@ def test_load_cached_data_for_updating(mocker, testdatadir) -> None: def test_download_pair_history( ohlcv_history, mocker, default_conf, tmp_path, candle_type, subdir, file_tail ) -> None: - mocker.patch(f"{EXMS}.get_historic_ohlcv", return_value=ohlcv_history) exchange = get_patched_exchange(mocker, default_conf) + mocker.patch.object(exchange, "get_historic_ohlcv", return_value=ohlcv_history) file1_1 = tmp_path / f"{subdir}MEME_BTC-1m{file_tail}.feather" file1_5 = tmp_path / f"{subdir}MEME_BTC-5m{file_tail}.feather" file2_1 = tmp_path / f"{subdir}CFI_BTC-1m{file_tail}.feather" @@ -356,8 +356,8 @@ def test_download_pair_history2(mocker, default_conf, testdatadir, ohlcv_history "freqtrade.data.history.datahandlers.featherdatahandler.FeatherDataHandler.ohlcv_store", return_value=None, ) - mocker.patch(f"{EXMS}.get_historic_ohlcv", return_value=ohlcv_history) exchange = get_patched_exchange(mocker, default_conf) + mocker.patch.object(exchange, "get_historic_ohlcv", return_value=ohlcv_history) _download_pair_history( datadir=testdatadir, exchange=exchange, From 2ceda2987c11513811ebcb247e088a3c7aa4c8e9 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Wed, 30 Oct 2024 23:51:25 +0800 Subject: [PATCH 08/45] feat: add binance 1s ohlcv fast download --- freqtrade/exchange/binance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 1b227265a..5e3588b23 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -134,7 +134,7 @@ class Binance(Exchange): ) return DataFrame(columns=DEFAULT_DATAFRAME_COLUMNS) - if timeframe in ["1m", "5m"] and candle_type in [CandleType.SPOT, CandleType.FUTURES]: + if timeframe in ["1s", "1m", "5m"] and candle_type in [CandleType.SPOT, CandleType.FUTURES]: df = self.loop.run_until_complete( binance_public_data.fetch_ohlcv( candle_type=candle_type, From 45bf046645bb20a57451327606b055ca6207c6fc Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Thu, 31 Oct 2024 01:51:54 +0800 Subject: [PATCH 09/45] tests: add performance comparison --- tests/exchange/binance_ohlcv_compare.py | 136 ++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 tests/exchange/binance_ohlcv_compare.py diff --git a/tests/exchange/binance_ohlcv_compare.py b/tests/exchange/binance_ohlcv_compare.py new file mode 100644 index 000000000..d05b2cf50 --- /dev/null +++ b/tests/exchange/binance_ohlcv_compare.py @@ -0,0 +1,136 @@ +""" +This file is meant to test the OHLCV download performance between +from rest API and from data.binance.vision + +using https://data.binance.vision: +{Task(trading_mode='spot', pair='ETH/USDT', timeframe='1s', timerange='20240101-20240501'): 28.26517415046692, + Task(trading_mode='spot', pair='ETH/USDT', timeframe='1m', timerange='20180101-20240101'): 17.766807794570923, + Task(trading_mode='spot', pair='ETH/USDT', timeframe='5m', timerange='20180101-20240101'): 11.347743034362793, + Task(trading_mode='futures', pair='ETH/USDT:USDT', timeframe='1m', timerange='20200101-20240101'): 15.93356990814209, + Task(trading_mode='futures', pair='ETH/USDT:USDT', timeframe='5m', timerange='20200101-20240101'): 10.111769914627075} + +using rest API: +{Task(trading_mode='spot', pair='ETH/USDT', timeframe='1s', timerange='20240101-20240501'): 257.9407958984375, + Task(trading_mode='spot', pair='ETH/USDT', timeframe='1m', timerange='20180101-20240101'): 86.42260813713074, + Task(trading_mode='spot', pair='ETH/USDT', timeframe='5m', timerange='20180101-20240101'): 20.111007928848267, + Task(trading_mode='futures', pair='ETH/USDT:USDT', timeframe='1m', timerange='20200101-20240101'): 537.5525922775269, + Task(trading_mode='futures', pair='ETH/USDT:USDT', timeframe='5m', timerange='20200101-20240101'): 111.13234090805054} + +compare: +spot-1s: 9X +spot-1m: 5X +spot-5m: 2X +futures-1m: 34X +futures-5m: 11X + +Usage: + # first switch to the branch you want to test + python tests/exchange/binance_ohlcv_compare.py +""" +# flake8: noqa: E501 + +import pprint +import subprocess +import time +from collections import namedtuple +from pathlib import Path + + +def rm_feather(data_dir, exchange, trading_mode, pair, timeframe): + data_dir = Path(data_dir) + is_futures = trading_mode == "futures" + file_name = ( + pair.replace("/", "_").replace(":", "_") + + "-" + + timeframe + + ("-futures" if is_futures else "") + + ".feather" + ) + file_dir = data_dir / exchange / ("futures" if is_futures else ".") + file_path = file_dir / file_name + file_path.unlink(missing_ok=True) + + +def download_data(trading_mode, exchange, pair, timeframe, timerange): + cmd = ( + f"freqtrade download-data --trading-mode {trading_mode} --exchange {exchange} " + f"--pairs {pair} --timeframe {timeframe} --timerange {timerange}" + ) + start = time.time() + subprocess.run(cmd, shell=True) # noqa: S602 + end = time.time() + elapsed = end - start + + print("-----") + print(trading_mode, timeframe, timerange, elapsed) + print("-----") + + return elapsed + + +def main(): + Task = namedtuple("Task", "trading_mode, pair, timeframe, timerange") + tasks = [ + Task( + "spot", + "ETH/USDT", + "1s", + "20240101-20240501", + ), + Task( + "spot", + "ETH/USDT", + "1m", + "20180101-20240101", + ), + Task( + "spot", + "ETH/USDT", + "5m", + "20180101-20240101", + ), + Task( + "futures", + "ETH/USDT:USDT", + "1m", + "20200101-20240101", + ), + Task( + "futures", + "ETH/USDT:USDT", + "5m", + "20200101-20240101", + ), + ] + + exchange = "binance" + data_dir = "user_data/data" + + data_dir = Path(data_dir) + if not data_dir.exists(): + raise FileNotFoundError(data_dir) + + result = {} + + for task in tasks: + rm_feather( + data_dir=data_dir, + exchange=exchange, + trading_mode=task.trading_mode, + pair=task.pair, + timeframe=task.timeframe, + ) + elapsed = download_data( + trading_mode=task.trading_mode, + exchange=exchange, + pair=task.pair, + timeframe=task.timeframe, + timerange=task.timerange, + ) + result[task] = elapsed + + pprint.pp(result) + + +if __name__ == "__main__": + main() From ad12a9eb992cf2dda5f088888361f2842a06e3a5 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Thu, 31 Oct 2024 03:29:21 +0800 Subject: [PATCH 10/45] tests: setup windows asyncio loop --- tests/exchange/test_binance_public_data.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py index f598a0bdd..1b7c0640c 100644 --- a/tests/exchange/test_binance_public_data.py +++ b/tests/exchange/test_binance_public_data.py @@ -16,9 +16,15 @@ from freqtrade.exchange.binance_public_data import ( symbol_ccxt_to_binance, zip_name, ) +from freqtrade.system.asyncio_config import asyncio_setup from freqtrade.util.datetime_helpers import dt_ts, dt_utc +@pytest.fixture(autouse=True, scope="module") +def setup_windows_asyncio_loop(): + asyncio_setup() + + # spot klines archive csv file format, the futures/um klines don't have the header line # # open_time,open,high,low,close,volume,close_time,quote_volume,count,taker_buy_volume,taker_buy_quote_volume,ignore # noqa: E501 From 3c76af9dab59973e41f6cbde31356c3dfe4f56b5 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Thu, 31 Oct 2024 04:18:37 +0800 Subject: [PATCH 11/45] tests: fix windows --- tests/exchange/test_binance_public_data.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py index 1b7c0640c..d92dcf51c 100644 --- a/tests/exchange/test_binance_public_data.py +++ b/tests/exchange/test_binance_public_data.py @@ -1,6 +1,8 @@ +import asyncio import datetime import io import re +import sys import zipfile from datetime import timedelta @@ -16,13 +18,15 @@ from freqtrade.exchange.binance_public_data import ( symbol_ccxt_to_binance, zip_name, ) -from freqtrade.system.asyncio_config import asyncio_setup from freqtrade.util.datetime_helpers import dt_ts, dt_utc -@pytest.fixture(autouse=True, scope="module") -def setup_windows_asyncio_loop(): - asyncio_setup() +@pytest.fixture(scope="module") +def event_loop_policy(request): + if sys.platform == "win32": + return asyncio.WindowsSelectorEventLoopPolicy() + else: + return asyncio.DefaultEventLoopPolicy() # spot klines archive csv file format, the futures/um klines don't have the header line From c3bbedbc5657b0d80a60361b00f3c483f9c7f259 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Sat, 2 Nov 2024 03:45:38 +0800 Subject: [PATCH 12/45] refactor: create coroutines on demand to avoid "coroutine was never awaited" warnings --- freqtrade/exchange/binance_public_data.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index ea4f4bc58..bbe652feb 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -101,13 +101,11 @@ async def _fetch_ohlcv( connector = aiohttp.TCPConnector(limit=100) async with aiohttp.ClientSession(connector=connector) as session: - coroutines = [ - get_daily_ohlcv(asset_type, symbol, timeframe, date, session) - for date in date_range(start, end) - ] # the HTTP connections has been throttled by TCPConnector - for batch in chunks(coroutines, 1000): - results = await asyncio.gather(*batch) + for dates in chunks(list(date_range(start, end)), 1000): + results = await asyncio.gather( + *(get_daily_ohlcv(asset_type, symbol, timeframe, date, session) for date in dates) + ) for result in results: if isinstance(result, BaseException): logger.warning(f"An exception raised: : {result}") From 76187d31cf55a23061a1b558dc1696707616a602 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Sat, 2 Nov 2024 04:02:43 +0800 Subject: [PATCH 13/45] feat: more binance fast download timeframes --- freqtrade/exchange/binance.py | 4 +++- freqtrade/exchange/binance_public_data.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 5e3588b23..44906273f 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -134,7 +134,9 @@ class Binance(Exchange): ) return DataFrame(columns=DEFAULT_DATAFRAME_COLUMNS) - if timeframe in ["1s", "1m", "5m"] and candle_type in [CandleType.SPOT, CandleType.FUTURES]: + if (candle_type == CandleType.SPOT and timeframe in ["1s", "1m", "3m", "5m"]) or ( + candle_type == CandleType.FUTURES and timeframe in ["1m", "3m", "5m", "15m", "30m"] + ): df = self.loop.run_until_complete( binance_public_data.fetch_ohlcv( candle_type=candle_type, diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index bbe652feb..1a577c76e 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -40,7 +40,7 @@ async def fetch_ohlcv( :param until_ms: `None` indicates the timestamp of the latest available data :param stop_on_404: Stop to download the following data when a 404 returned :return: the date range is between [since_ms, until_ms), - return None if no data available in the time range + return and empty DataFrame if no data available in the time range """ if candle_type == CandleType.SPOT: asset_type = "spot" @@ -159,6 +159,9 @@ async def get_daily_ohlcv( """ Get daily OHLCV from https://data.binance.vision See https://github.com/binance/binance-public-data + + :return: None indicates a 404 when trying to download the daily archive file + This function won't raise any exception, but catch and return it """ url = zip_url(asset_type, symbol, timeframe, date) From e2ee7f7b2fbd6c5767fb660612bb56b4406a43c6 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Sat, 2 Nov 2024 05:17:16 +0800 Subject: [PATCH 14/45] feat: fall back to rest API by catching all exceptions --- freqtrade/exchange/binance_public_data.py | 43 ++++++++++++---------- tests/exchange/test_binance_public_data.py | 15 +++----- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index 1a577c76e..8290a807b 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -42,26 +42,31 @@ async def fetch_ohlcv( :return: the date range is between [since_ms, until_ms), return and empty DataFrame if no data available in the time range """ - if candle_type == CandleType.SPOT: - asset_type = "spot" - elif candle_type == CandleType.FUTURES: - asset_type = "futures/um" - else: - raise ValueError(f"Unsupported CandleType: {candle_type}") - symbol = symbol_ccxt_to_binance(pair) - start = dt_from_ts(since_ms) - end = dt_from_ts(until_ms) if until_ms else dt_now() + try: + if candle_type == CandleType.SPOT: + asset_type = "spot" + elif candle_type == CandleType.FUTURES: + asset_type = "futures/um" + else: + raise ValueError(f"Unsupported CandleType: {candle_type}") + symbol = symbol_ccxt_to_binance(pair) + start = dt_from_ts(since_ms) + end = dt_from_ts(until_ms) if until_ms else dt_now() + + # We use two days ago as the last available day because the daily archives are daily + # uploaded and have several hours delay + last_available_date = dt_now() - datetime.timedelta(days=2) + end = min(end, last_available_date) + if start >= end: + return DataFrame() + df = await _fetch_ohlcv(asset_type, symbol, timeframe, start, end, stop_on_404) + logger.info( + f"Downloaded data for {pair} from https://data.binance.vision/ with length {len(df)}." + ) + except Exception as e: + logger.debug("An exception occurred", exc_info=e) + df = DataFrame() - # We use two days ago as the last available day because the daily archives are daily uploaded - # and have several hours delay - last_available_date = dt_now() - datetime.timedelta(days=2) - end = min(end, last_available_date) - if start >= end: - return DataFrame() - df = await _fetch_ohlcv(asset_type, symbol, timeframe, start, end, stop_on_404) - logger.info( - f"Downloaded data for {pair} from https://data.binance.vision/ with length {len(df)}." - ) if not df.empty: return df.loc[(df["date"] >= start) & (df["date"] < end)] else: diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py index d92dcf51c..e4f665afd 100644 --- a/tests/exchange/test_binance_public_data.py +++ b/tests/exchange/test_binance_public_data.py @@ -209,17 +209,14 @@ async def test_fetch_ohlcv(mocker, candle_type, since, until, first_date, last_d "aiohttp.ClientSession.get", side_effect=make_response_from_url(history_start, history_end) ) - if candle_type in [CandleType.SPOT, CandleType.FUTURES]: - df = await fetch_ohlcv(candle_type, pair, timeframe, since_ms, until_ms, stop_on_404) + df = await fetch_ohlcv(candle_type, pair, timeframe, since_ms, until_ms, stop_on_404) - if df.empty: - assert first_date is None and last_date is None - else: - assert df["date"].iloc[0] == first_date - assert df["date"].iloc[-1] == last_date + if df.empty: + assert first_date is None and last_date is None else: - with pytest.raises(ValueError): - await fetch_ohlcv(candle_type, pair, timeframe, since_ms, until_ms, stop_on_404) + assert candle_type in [CandleType.SPOT, CandleType.FUTURES] + assert df["date"].iloc[0] == first_date + assert df["date"].iloc[-1] == last_date async def test_fetch_ohlcv_exc(mocker): From cf0f232635bc14be9e977646f8b0744c25cbc7e0 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Sat, 2 Nov 2024 06:00:37 +0800 Subject: [PATCH 15/45] refactor: move download klines count message --- freqtrade/data/history/history_utils.py | 1 + freqtrade/exchange/binance_public_data.py | 4 ++-- freqtrade/exchange/exchange.py | 2 +- tests/exchange/test_exchange.py | 1 + 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 092faa19a..8ef7870c7 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -284,6 +284,7 @@ def _download_pair_history( candle_type=candle_type, until_ms=until_ms if until_ms else None, ) + logger.info(f"Downloaded data for {pair} with length {len(new_dataframe)}.") if data.empty: data = new_dataframe else: diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index 8290a807b..4cd000c77 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -60,8 +60,8 @@ async def fetch_ohlcv( if start >= end: return DataFrame() df = await _fetch_ohlcv(asset_type, symbol, timeframe, start, end, stop_on_404) - logger.info( - f"Downloaded data for {pair} from https://data.binance.vision/ with length {len(df)}." + logger.debug( + f"Downloaded data for {pair} from https://data.binance.vision with length {len(df)}." ) except Exception as e: logger.debug("An exception occurred", exc_info=e) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 5e1e32a54..0dac3beec 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2246,7 +2246,7 @@ class Exchange: candle_type=candle_type, ) ) - logger.info(f"Downloaded data for {pair} from ccxt with length {len(data)}.") + logger.debug(f"Downloaded data for {pair} from ccxt with length {len(data)}.") return ohlcv_to_dataframe(data, timeframe, pair, fill_missing=False, drop_incomplete=True) async def _async_get_historic_ohlcv( diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index b123229ad..b45155fb1 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -2092,6 +2092,7 @@ def test___now_is_time_to_refresh(default_conf, mocker, exchange_name, time_mach @pytest.mark.parametrize("candle_type", ["mark", ""]) @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name, candle_type): + caplog.set_level(logging.DEBUG) exchange = get_patched_exchange(mocker, default_conf, exchange=exchange_name) pair = "ETH/BTC" calls = 0 From acc53065e5fa7ab5197073276306dc9dc3adbfa3 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Wed, 13 Nov 2024 10:16:06 +0800 Subject: [PATCH 16/45] refactor: strip fast download logic into a method --- freqtrade/exchange/binance.py | 52 ++++++++++++++++++++++++++++------ freqtrade/exchange/exchange.py | 8 +++--- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 44906273f..d1fa28aa6 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -110,11 +110,13 @@ class Binance(Exchange): candle_type: CandleType, is_new_pair: bool = False, until_ms: Optional[int] = None, + only_from_ccxt: bool = False, ) -> DataFrame: """ Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date Does not work for other exchanges, which don't return the earliest data when called with "0" :param candle_type: Any of the enum CandleType (must match trading mode!) + :param only_from_ccxt: Only download data using the API provided by CCXT """ if is_new_pair: x = self.loop.run_until_complete( @@ -134,6 +136,37 @@ class Binance(Exchange): ) return DataFrame(columns=DEFAULT_DATAFRAME_COLUMNS) + if only_from_ccxt: + return super().get_historic_ohlcv( + pair=pair, + timeframe=timeframe, + since_ms=since_ms, + candle_type=candle_type, + is_new_pair=is_new_pair, + until_ms=until_ms, + ) + else: + return self.get_historic_ohlcv_fast( + pair=pair, + timeframe=timeframe, + since_ms=since_ms, + candle_type=candle_type, + is_new_pair=is_new_pair, + until_ms=until_ms, + ) + + def get_historic_ohlcv_fast( + self, + pair: str, + timeframe: str, + since_ms: int, + candle_type: CandleType, + is_new_pair: bool = False, + until_ms: Optional[int] = None, + ): + """ + Fetch ohlcv fast by utilizing https://data.binance.vision + """ if (candle_type == CandleType.SPOT and timeframe in ["1s", "1m", "3m", "5m"]) or ( candle_type == CandleType.FUTURES and timeframe in ["1m", "3m", "5m", "15m", "30m"] ): @@ -163,15 +196,16 @@ class Binance(Exchange): until_ms=until_ms, ) all_df = concat([df, rest_df]) - return all_df - return super().get_historic_ohlcv( - pair=pair, - timeframe=timeframe, - since_ms=since_ms, - candle_type=candle_type, - is_new_pair=is_new_pair, - until_ms=until_ms, - ) + else: + return super().get_historic_ohlcv( + pair=pair, + timeframe=timeframe, + since_ms=since_ms, + candle_type=candle_type, + is_new_pair=is_new_pair, + until_ms=until_ms, + ) + return all_df def funding_fee_cutoff(self, open_date: datetime): """ diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 0dac3beec..de1d1eff4 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2224,6 +2224,7 @@ class Exchange: candle_type: CandleType, is_new_pair: bool = False, until_ms: Optional[int] = None, + only_from_ccxt: bool = False, ) -> DataFrame: """ Get candle history using asyncio and returns the list of candles. @@ -2232,8 +2233,10 @@ class Exchange: :param pair: Pair to download :param timeframe: Timeframe to get data for :param since_ms: Timestamp in milliseconds to get history from - :param until_ms: Timestamp in milliseconds to get history up to :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param is_new_pair: used by binance subclass to allow "fast" new pair downloading + :param until_ms: Timestamp in milliseconds to get history up to + :param only_from_ccxt: Only download data using the API provided by CCXT :return: Dataframe with candle (OHLCV) data """ pair, _, _, data, _ = self.loop.run_until_complete( @@ -2242,7 +2245,6 @@ class Exchange: timeframe=timeframe, since_ms=since_ms, until_ms=until_ms, - is_new_pair=is_new_pair, candle_type=candle_type, ) ) @@ -2255,13 +2257,11 @@ class Exchange: timeframe: str, since_ms: int, candle_type: CandleType, - is_new_pair: bool = False, raise_: bool = False, until_ms: Optional[int] = None, ) -> OHLCVResponse: """ Download historic ohlcv - :param is_new_pair: used by binance subclass to allow "fast" new pair downloading :param candle_type: Any of the enum CandleType (must match trading mode!) """ From c4cf582c9d76bb7de02bd64feb2d5a73123c4e51 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Wed, 13 Nov 2024 11:18:32 +0800 Subject: [PATCH 17/45] refacotr: default to stop on 404 --- freqtrade/exchange/binance.py | 1 - freqtrade/exchange/binance_public_data.py | 33 +++++++++++++++++----- tests/exchange/test_binance_public_data.py | 5 ++-- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index d1fa28aa6..f9890270f 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -177,7 +177,6 @@ class Binance(Exchange): timeframe=timeframe, since_ms=since_ms, until_ms=until_ms, - stop_on_404=False, ) ) if df.empty: diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index 4cd000c77..c14aacd9d 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -20,6 +20,10 @@ from freqtrade.util.datetime_helpers import dt_from_ts, dt_now logger = logging.getLogger(__name__) +class Http404(Exception): + pass + + class BadHttpStatus(Exception): """Not 200/404""" @@ -32,7 +36,7 @@ async def fetch_ohlcv( timeframe: str, since_ms: int, until_ms: int | None, - stop_on_404: bool = False, + stop_on_404: bool = True, ) -> DataFrame: """ Fetch OHLCV data from https://data.binance.vision/ @@ -103,6 +107,7 @@ async def _fetch_ohlcv( stop_on_404: bool, ) -> DataFrame: dfs: list[DataFrame | None] = [] + current_day = 0 connector = aiohttp.TCPConnector(limit=100) async with aiohttp.ClientSession(connector=connector) as session: @@ -112,14 +117,27 @@ async def _fetch_ohlcv( *(get_daily_ohlcv(asset_type, symbol, timeframe, date, session) for date in dates) ) for result in results: - if isinstance(result, BaseException): + current_day += 1 + if isinstance(result, Http404): + if stop_on_404: + if current_day == 1: + # https://github.com/freqtrade/freqtrade/blob/acc53065e5fa7ab5197073276306dc9dc3adbfa3/tests/exchange_online/test_binance_compare_ohlcv.py#L7 + logger.warning( + "Failed to use fast download, fall back to rest API download, this " + "can take more time. If you're downloading BTC/USDT:USDT, " + "ETH/USDT:USDT, BCH/USDT:USDT, please first download " + "data before 2020 (using `--timerange yyyymmdd-20200101`), and " + "then download the full data you need." + ) + logger.debug("Abort downloading from data.binance.vision due to 404") + return concat(dfs) + else: + dfs.append(None) + elif isinstance(result, BaseException): logger.warning(f"An exception raised: : {result}") # Directly return the existing data, do not allow the gap # between the data return concat(dfs) - elif result is None and stop_on_404: - logger.debug("Abort downloading from data.binance.vision due to 404") - return concat(dfs) else: dfs.append(result) return concat(dfs) @@ -160,6 +178,7 @@ async def get_daily_ohlcv( date: datetime.date, session: aiohttp.ClientSession, retry_count: int = 3, + retry_delay: float = 0.0, ) -> DataFrame | None | Exception: """ Get daily OHLCV from https://data.binance.vision @@ -176,7 +195,7 @@ async def get_daily_ohlcv( retry = 0 while True: if retry > 0: - sleep_secs = retry * 0.5 + sleep_secs = retry * retry_delay logger.debug( f"[{retry}/{retry_count}] retry to download {url} after {sleep_secs} seconds" ) @@ -206,7 +225,7 @@ async def get_daily_ohlcv( return df elif resp.status == 404: logger.debug(f"No data available for {symbol} in {format_date(date)}") - return None + raise Http404 else: raise BadHttpStatus(f"{resp.status} - {resp.reason}") except Exception as e: diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py index e4f665afd..2257efc9f 100644 --- a/tests/exchange/test_binance_public_data.py +++ b/tests/exchange/test_binance_public_data.py @@ -13,6 +13,7 @@ import pytest from freqtrade.enums import CandleType from freqtrade.exchange.binance_public_data import ( BadHttpStatus, + Http404, fetch_ohlcv, get_daily_ohlcv, symbol_ccxt_to_binance, @@ -256,8 +257,8 @@ async def test_get_daily_ohlcv(mocker, testdatadir): assert df["date"].iloc[-1] == last_date mocker.patch("aiohttp.ClientSession.get", return_value=MockResponse(b"", 404)) - df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) - assert df is None + df = await get_daily_ohlcv("spot", symbol, timeframe, date, session, retry_delay=0) + assert isinstance(df, Http404) mocker.patch("aiohttp.ClientSession.get", return_value=MockResponse(b"", 500)) mocker.patch("asyncio.sleep") From 37726fba58834e922bc8079141aa57d5585f08ec Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Wed, 13 Nov 2024 11:53:58 +0800 Subject: [PATCH 18/45] refactor: use CCXT for pair to symbol conversion --- freqtrade/exchange/binance_public_data.py | 12 +++++++++++- tests/exchange/test_binance_public_data.py | 16 +++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index c14aacd9d..23ea3ba4d 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -7,8 +7,10 @@ import datetime import io import logging import zipfile +from typing import Any import aiohttp +import ccxt import pandas as pd from pandas import DataFrame @@ -36,6 +38,7 @@ async def fetch_ohlcv( timeframe: str, since_ms: int, until_ms: int | None, + markets: dict[str, Any] | None = None, stop_on_404: bool = True, ) -> DataFrame: """ @@ -53,7 +56,14 @@ async def fetch_ohlcv( asset_type = "futures/um" else: raise ValueError(f"Unsupported CandleType: {candle_type}") - symbol = symbol_ccxt_to_binance(pair) + + if markets: + symbol = markets[pair]["id"] + else: + binance = ccxt.binance() + binance.load_markets() + symbol = binance.markets[pair]["id"] + start = dt_from_ts(since_ms) end = dt_from_ts(until_ms) if until_ms else dt_now() diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py index 2257efc9f..05740e6e4 100644 --- a/tests/exchange/test_binance_public_data.py +++ b/tests/exchange/test_binance_public_data.py @@ -133,7 +133,7 @@ def make_response_from_url(start_date, end_date): ), ( CandleType.SPOT, - dt_utc(2019, 1, 1), + dt_utc(2019, 12, 25), dt_utc(2020, 1, 5), dt_utc(2020, 1, 1), dt_utc(2020, 1, 3, 23), @@ -165,7 +165,7 @@ def make_response_from_url(start_date, end_date): ), ( CandleType.SPOT, - dt_utc(2019, 1, 1), + dt_utc(2019, 12, 25), dt_utc(2020, 1, 5), None, None, @@ -201,7 +201,10 @@ async def test_fetch_ohlcv(mocker, candle_type, since, until, first_date, last_d history_start = dt_utc(2020, 1, 1).date() history_end = dt_utc(2020, 1, 3).date() timeframe = "1h" - pair = "BTCUSDT" + if candle_type == CandleType.SPOT: + pair = "BTC/USDT" + else: + pair = "BTC/USDT:USDT" since_ms = dt_ts(since) until_ms = dt_ts(until) @@ -209,8 +212,9 @@ async def test_fetch_ohlcv(mocker, candle_type, since, until, first_date, last_d mocker.patch( "aiohttp.ClientSession.get", side_effect=make_response_from_url(history_start, history_end) ) + markets = {"BTC/USDT": {"id": "BTCUSDT"}, "BTC/USDT:USDT": {"id": "BTCUSDT"}} - df = await fetch_ohlcv(candle_type, pair, timeframe, since_ms, until_ms, stop_on_404) + df = await fetch_ohlcv(candle_type, pair, timeframe, since_ms, until_ms, markets, stop_on_404) if df.empty: assert first_date is None and last_date is None @@ -222,12 +226,14 @@ async def test_fetch_ohlcv(mocker, candle_type, since, until, first_date, last_d async def test_fetch_ohlcv_exc(mocker): timeframe = "1h" - pair = "BTCUSDT" + pair = "BTC/USDT" since_ms = dt_ts(dt_utc(2020, 1, 1)) until_ms = dt_ts(dt_utc(2020, 1, 2)) mocker.patch("aiohttp.ClientSession.get", side_effect=RuntimeError) + mocker.patch("ccxt.binance.binance") + mocker.patch("ccxt.binance.binance.markets", {"BTC/USDT": {"id": "BTCUSDT"}}) df = await fetch_ohlcv(CandleType.SPOT, pair, timeframe, since_ms, until_ms) From d7555e1f2971b6bcbbd4d978bd0a81e436c91f3d Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Wed, 13 Nov 2024 12:03:41 +0800 Subject: [PATCH 19/45] feat: support proxy from environment variables --- freqtrade/exchange/binance_public_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index 23ea3ba4d..70dfb1df3 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -120,7 +120,7 @@ async def _fetch_ohlcv( current_day = 0 connector = aiohttp.TCPConnector(limit=100) - async with aiohttp.ClientSession(connector=connector) as session: + async with aiohttp.ClientSession(connector=connector, trust_env=True) as session: # the HTTP connections has been throttled by TCPConnector for dates in chunks(list(date_range(start, end)), 1000): results = await asyncio.gather( From 03033a0684384a29266043c68619182131505fef Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Wed, 13 Nov 2024 12:36:18 +0800 Subject: [PATCH 20/45] refactor: use exchange.markets to avoid loading ccxt markets --- freqtrade/exchange/binance.py | 1 + tests/exchange/test_binance.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index eb079e04c..d361fd3da 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -176,6 +176,7 @@ class Binance(Exchange): timeframe=timeframe, since_ms=since_ms, until_ms=until_ms, + markets=self.markets, ) ) if df.empty: diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index 749dff663..f506e7e11 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -1,6 +1,6 @@ from datetime import datetime, timedelta from random import randint -from unittest.mock import AsyncMock, MagicMock, PropertyMock +from unittest.mock import MagicMock, PropertyMock import ccxt import pandas as pd @@ -763,12 +763,13 @@ def patch_ohlcv(mocker, start, archive_end, api_end, timeframe): until = dt_from_ts(until_ms) if until_ms else api_end + timedelta(seconds=1) return api_storage.loc[(api_storage["date"] >= since) & (api_storage["date"] < until)] - def fetch_ohlcv( + async def fetch_ohlcv( candle_type, pair, timeframe, since_ms, until_ms, + markets=None, stop_on_404=False, ): since = dt_from_ts(since_ms) @@ -783,10 +784,10 @@ def patch_ohlcv(mocker, start, archive_end, api_end, timeframe): "freqtrade.exchange.Exchange._async_get_candle_history", return_value=candle_history ) api_mock = mocker.patch( - "freqtrade.exchange.Exchange.get_historic_ohlcv", MagicMock(wraps=get_historic_ohlcv) + "freqtrade.exchange.Exchange.get_historic_ohlcv", side_effect=get_historic_ohlcv ) archive_mock = mocker.patch( - "freqtrade.exchange.binance_public_data.fetch_ohlcv", AsyncMock(wraps=fetch_ohlcv) + "freqtrade.exchange.binance_public_data.fetch_ohlcv", side_effect=fetch_ohlcv ) return candle_mock, api_mock, archive_mock @@ -957,6 +958,9 @@ def test_get_historic_ohlcv_binance( since_ms = dt_ts(since) until_ms = dt_ts(until) + mocker.patch("ccxt.binance.binance") + mocker.patch("ccxt.binance.binance.markets", {"BTC/USDT": {"id": "BTCUSDT"}}) + df = exchange.get_historic_ohlcv(pair, timeframe, since_ms, candle_type, is_new_pair, until_ms) if df.empty: From fc307bcf5bf0a5b5e48c7354b63fe0cbf3b11f1d Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Wed, 13 Nov 2024 12:44:27 +0800 Subject: [PATCH 21/45] tests: fix --- tests/exchange/test_exchange.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 86a332dab..290e8c46d 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -2157,7 +2157,7 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ pair = "ETH/USDT" respair, restf, _, res, _ = await exchange._async_get_historic_ohlcv( - pair, "5m", 1500000000000, candle_type=candle_type, is_new_pair=False + pair, "5m", 1500000000000, candle_type=candle_type ) assert respair == pair assert restf == "5m" @@ -2169,7 +2169,7 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ end_ts = 1_500_500_000_000 start_ts = 1_500_000_000_000 respair, restf, _, res, _ = await exchange._async_get_historic_ohlcv( - pair, "5m", since_ms=start_ts, candle_type=candle_type, is_new_pair=False, until_ms=end_ts + pair, "5m", since_ms=start_ts, candle_type=candle_type, until_ms=end_ts ) # Required candles candles = (end_ts - start_ts) / 300_000 From a748d105ed0a805dc4e40e760eee6ebd19f3f6ab Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Wed, 13 Nov 2024 12:45:46 +0800 Subject: [PATCH 22/45] chore: remove performance compare script --- tests/exchange/binance_ohlcv_compare.py | 136 ------------------------ 1 file changed, 136 deletions(-) delete mode 100644 tests/exchange/binance_ohlcv_compare.py diff --git a/tests/exchange/binance_ohlcv_compare.py b/tests/exchange/binance_ohlcv_compare.py deleted file mode 100644 index d05b2cf50..000000000 --- a/tests/exchange/binance_ohlcv_compare.py +++ /dev/null @@ -1,136 +0,0 @@ -""" -This file is meant to test the OHLCV download performance between -from rest API and from data.binance.vision - -using https://data.binance.vision: -{Task(trading_mode='spot', pair='ETH/USDT', timeframe='1s', timerange='20240101-20240501'): 28.26517415046692, - Task(trading_mode='spot', pair='ETH/USDT', timeframe='1m', timerange='20180101-20240101'): 17.766807794570923, - Task(trading_mode='spot', pair='ETH/USDT', timeframe='5m', timerange='20180101-20240101'): 11.347743034362793, - Task(trading_mode='futures', pair='ETH/USDT:USDT', timeframe='1m', timerange='20200101-20240101'): 15.93356990814209, - Task(trading_mode='futures', pair='ETH/USDT:USDT', timeframe='5m', timerange='20200101-20240101'): 10.111769914627075} - -using rest API: -{Task(trading_mode='spot', pair='ETH/USDT', timeframe='1s', timerange='20240101-20240501'): 257.9407958984375, - Task(trading_mode='spot', pair='ETH/USDT', timeframe='1m', timerange='20180101-20240101'): 86.42260813713074, - Task(trading_mode='spot', pair='ETH/USDT', timeframe='5m', timerange='20180101-20240101'): 20.111007928848267, - Task(trading_mode='futures', pair='ETH/USDT:USDT', timeframe='1m', timerange='20200101-20240101'): 537.5525922775269, - Task(trading_mode='futures', pair='ETH/USDT:USDT', timeframe='5m', timerange='20200101-20240101'): 111.13234090805054} - -compare: -spot-1s: 9X -spot-1m: 5X -spot-5m: 2X -futures-1m: 34X -futures-5m: 11X - -Usage: - # first switch to the branch you want to test - python tests/exchange/binance_ohlcv_compare.py -""" -# flake8: noqa: E501 - -import pprint -import subprocess -import time -from collections import namedtuple -from pathlib import Path - - -def rm_feather(data_dir, exchange, trading_mode, pair, timeframe): - data_dir = Path(data_dir) - is_futures = trading_mode == "futures" - file_name = ( - pair.replace("/", "_").replace(":", "_") - + "-" - + timeframe - + ("-futures" if is_futures else "") - + ".feather" - ) - file_dir = data_dir / exchange / ("futures" if is_futures else ".") - file_path = file_dir / file_name - file_path.unlink(missing_ok=True) - - -def download_data(trading_mode, exchange, pair, timeframe, timerange): - cmd = ( - f"freqtrade download-data --trading-mode {trading_mode} --exchange {exchange} " - f"--pairs {pair} --timeframe {timeframe} --timerange {timerange}" - ) - start = time.time() - subprocess.run(cmd, shell=True) # noqa: S602 - end = time.time() - elapsed = end - start - - print("-----") - print(trading_mode, timeframe, timerange, elapsed) - print("-----") - - return elapsed - - -def main(): - Task = namedtuple("Task", "trading_mode, pair, timeframe, timerange") - tasks = [ - Task( - "spot", - "ETH/USDT", - "1s", - "20240101-20240501", - ), - Task( - "spot", - "ETH/USDT", - "1m", - "20180101-20240101", - ), - Task( - "spot", - "ETH/USDT", - "5m", - "20180101-20240101", - ), - Task( - "futures", - "ETH/USDT:USDT", - "1m", - "20200101-20240101", - ), - Task( - "futures", - "ETH/USDT:USDT", - "5m", - "20200101-20240101", - ), - ] - - exchange = "binance" - data_dir = "user_data/data" - - data_dir = Path(data_dir) - if not data_dir.exists(): - raise FileNotFoundError(data_dir) - - result = {} - - for task in tasks: - rm_feather( - data_dir=data_dir, - exchange=exchange, - trading_mode=task.trading_mode, - pair=task.pair, - timeframe=task.timeframe, - ) - elapsed = download_data( - trading_mode=task.trading_mode, - exchange=exchange, - pair=task.pair, - timeframe=task.timeframe, - timerange=task.timerange, - ) - result[task] = elapsed - - pprint.pp(result) - - -if __name__ == "__main__": - main() From 1893ac7ed6cb54b39b312640d6ad65decb51cf26 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Wed, 13 Nov 2024 13:08:29 +0800 Subject: [PATCH 23/45] tests: patch thirty party libs from imported location --- tests/exchange/test_binance.py | 7 ++-- tests/exchange/test_binance_public_data.py | 39 +++++++++++++++++----- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index f506e7e11..bb9b022ef 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -958,8 +958,11 @@ def test_get_historic_ohlcv_binance( since_ms = dt_ts(since) until_ms = dt_ts(until) - mocker.patch("ccxt.binance.binance") - mocker.patch("ccxt.binance.binance.markets", {"BTC/USDT": {"id": "BTCUSDT"}}) + mocker.patch("freqtrade.exchange.binance_public_data.ccxt.binance") + mocker.patch( + "freqtrade.exchange.binance_public_data.ccxt.binance.markets", + {"BTC/USDT": {"id": "BTCUSDT"}}, + ) df = exchange.get_historic_ohlcv(pair, timeframe, since_ms, candle_type, is_new_pair, until_ms) diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py index 05740e6e4..5fb2fcc67 100644 --- a/tests/exchange/test_binance_public_data.py +++ b/tests/exchange/test_binance_public_data.py @@ -210,7 +210,8 @@ async def test_fetch_ohlcv(mocker, candle_type, since, until, first_date, last_d until_ms = dt_ts(until) mocker.patch( - "aiohttp.ClientSession.get", side_effect=make_response_from_url(history_start, history_end) + "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get", + side_effect=make_response_from_url(history_start, history_end), ) markets = {"BTC/USDT": {"id": "BTCUSDT"}, "BTC/USDT:USDT": {"id": "BTCUSDT"}} @@ -231,9 +232,14 @@ async def test_fetch_ohlcv_exc(mocker): since_ms = dt_ts(dt_utc(2020, 1, 1)) until_ms = dt_ts(dt_utc(2020, 1, 2)) - mocker.patch("aiohttp.ClientSession.get", side_effect=RuntimeError) - mocker.patch("ccxt.binance.binance") - mocker.patch("ccxt.binance.binance.markets", {"BTC/USDT": {"id": "BTCUSDT"}}) + mocker.patch( + "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get", side_effect=RuntimeError + ) + mocker.patch("freqtrade.exchange.binance_public_data.ccxt.binance") + mocker.patch( + "freqtrade.exchange.binance_public_data.ccxt.binance.markets", + {"BTC/USDT": {"id": "BTCUSDT"}}, + ) df = await fetch_ohlcv(CandleType.SPOT, pair, timeframe, since_ms, until_ms) @@ -249,7 +255,10 @@ async def test_get_daily_ohlcv(mocker, testdatadir): async with aiohttp.ClientSession() as session: path = testdatadir / "binance/binance_public_data/spot-klines-BTCUSDT-1h-2024-10-28.zip" - mocker.patch("aiohttp.ClientSession.get", return_value=MockResponse(path.read_bytes(), 200)) + mocker.patch( + "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get", + return_value=MockResponse(path.read_bytes(), 200), + ) df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) assert df["date"].iloc[0] == first_date assert df["date"].iloc[-1] == last_date @@ -257,21 +266,33 @@ async def test_get_daily_ohlcv(mocker, testdatadir): path = ( testdatadir / "binance/binance_public_data/futures-um-klines-BTCUSDT-1h-2024-10-28.zip" ) - mocker.patch("aiohttp.ClientSession.get", return_value=MockResponse(path.read_bytes(), 200)) + mocker.patch( + "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get", + return_value=MockResponse(path.read_bytes(), 200), + ) df = await get_daily_ohlcv("futures/um", symbol, timeframe, date, session) assert df["date"].iloc[0] == first_date assert df["date"].iloc[-1] == last_date - mocker.patch("aiohttp.ClientSession.get", return_value=MockResponse(b"", 404)) + mocker.patch( + "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get", + return_value=MockResponse(b"", 404), + ) df = await get_daily_ohlcv("spot", symbol, timeframe, date, session, retry_delay=0) assert isinstance(df, Http404) - mocker.patch("aiohttp.ClientSession.get", return_value=MockResponse(b"", 500)) + mocker.patch( + "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get", + return_value=MockResponse(b"", 500), + ) mocker.patch("asyncio.sleep") df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) assert isinstance(df, BadHttpStatus) - mocker.patch("aiohttp.ClientSession.get", return_value=MockResponse(b"nop", 200)) + mocker.patch( + "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get", + return_value=MockResponse(b"nop", 200), + ) df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) assert isinstance(df, zipfile.BadZipFile) From cf7016b36d37f1969a4352f974ccd7551e800681 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Wed, 13 Nov 2024 16:09:02 +0800 Subject: [PATCH 24/45] chore: remove unused code --- freqtrade/exchange/binance_public_data.py | 14 -------------- tests/exchange/test_binance_public_data.py | 8 -------- 2 files changed, 22 deletions(-) diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index 70dfb1df3..33691f962 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -87,20 +87,6 @@ async def fetch_ohlcv( return df -def symbol_ccxt_to_binance(symbol: str) -> str: - """ - Convert ccxt symbol notation to binance notation - e.g. BTC/USDT -> BTCUSDT, BTC/USDT:USDT -> BTCUSDT - """ - if ":" in symbol: - parts = symbol.split(":") - if len(parts) != 2: - raise ValueError(f"Cannot recognize symbol: {symbol}") - return parts[0].replace("/", "") - else: - return symbol.replace("/", "") - - def concat(dfs) -> DataFrame: if all(df is None for df in dfs): return DataFrame() diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py index 5fb2fcc67..49caf5f06 100644 --- a/tests/exchange/test_binance_public_data.py +++ b/tests/exchange/test_binance_public_data.py @@ -16,7 +16,6 @@ from freqtrade.exchange.binance_public_data import ( Http404, fetch_ohlcv, get_daily_ohlcv, - symbol_ccxt_to_binance, zip_name, ) from freqtrade.util.datetime_helpers import dt_ts, dt_utc @@ -295,10 +294,3 @@ async def test_get_daily_ohlcv(mocker, testdatadir): ) df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) assert isinstance(df, zipfile.BadZipFile) - - -def test_symbol_ccxt_to_binance(): - assert symbol_ccxt_to_binance("BTC/USDT") == "BTCUSDT" - assert symbol_ccxt_to_binance("BTC/USDT:USDT") == "BTCUSDT" - with pytest.raises(ValueError): - assert symbol_ccxt_to_binance("BTC:USDT:USDT") == "BTCUSDT" From 39b4263b8bc4875ad06157ec529a71c56679c59c Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Wed, 13 Nov 2024 17:14:49 +0800 Subject: [PATCH 25/45] chore: improve comments --- freqtrade/exchange/binance.py | 14 +++++--- freqtrade/exchange/binance_public_data.py | 37 ++++++++++++++++------ tests/exchange/test_binance_public_data.py | 15 ++++++--- 3 files changed, 47 insertions(+), 19 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index d361fd3da..8104876f8 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -5,12 +5,13 @@ from datetime import datetime, timezone from pathlib import Path import ccxt -from pandas import DataFrame, concat +from pandas import DataFrame from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS from freqtrade.enums import CandleType, MarginMode, PriceType, TradingMode from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError from freqtrade.exchange import Exchange, binance_public_data +from freqtrade.exchange.binance_public_data import concat from freqtrade.exchange.common import retrier from freqtrade.exchange.exchange_types import FtHas, Tickers from freqtrade.exchange.exchange_utils_timeframe import timeframe_to_msecs @@ -162,10 +163,11 @@ class Binance(Exchange): candle_type: CandleType, is_new_pair: bool = False, until_ms: int | None = None, - ): + ) -> DataFrame: """ - Fetch ohlcv fast by utilizing https://data.binance.vision + Fastly fetch OHLCV data by leveraging https://data.binance.vision. """ + # only download timeframes with significant improvements, otherwise fall back to rest API if (candle_type == CandleType.SPOT and timeframe in ["1s", "1m", "3m", "5m"]) or ( candle_type == CandleType.FUTURES and timeframe in ["1m", "3m", "5m", "15m", "30m"] ): @@ -179,10 +181,14 @@ class Binance(Exchange): markets=self.markets, ) ) + + # download the remaining data from rest API if df.empty: rest_since_ms = since_ms else: rest_since_ms = dt_ts(df.iloc[-1].date) + timeframe_to_msecs(timeframe) + + # make sure since <= until if until_ms and rest_since_ms > until_ms: rest_df = DataFrame() else: @@ -195,6 +201,7 @@ class Binance(Exchange): until_ms=until_ms, ) all_df = concat([df, rest_df]) + return all_df else: return super().get_historic_ohlcv( pair=pair, @@ -204,7 +211,6 @@ class Binance(Exchange): is_new_pair=is_new_pair, until_ms=until_ms, ) - return all_df def funding_fee_cutoff(self, open_date: datetime): """ diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index 33691f962..685c2c42f 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -42,12 +42,22 @@ async def fetch_ohlcv( stop_on_404: bool = True, ) -> DataFrame: """ - Fetch OHLCV data from https://data.binance.vision/ + Fetch OHLCV data from https://data.binance.vision + The function makes its best effort to download data within the time range + [`since_ms`, `until_ms`) -- including `since_ms`, but excluding `until_ms`. + If `stop_one_404` is True, this returned DataFrame is guaranteed to start from `since_ms` + with no gaps in the data. + :candle_type: Currently only spot and futures are supported + :pair: symbol name in CCXT convention + :since_ms: the start timestamp of data, including itself + :until_ms: the end timestamp of data, excluding itself :param until_ms: `None` indicates the timestamp of the latest available data + :markets: the CCXT markets dict, when it's None, the function will load the markets data + from a new `ccxt.binance` instance :param stop_on_404: Stop to download the following data when a 404 returned - :return: the date range is between [since_ms, until_ms), - return and empty DataFrame if no data available in the time range + :return: the date range is between [since_ms, until_ms), return an empty DataFrame if no data + available in the time range """ try: if candle_type == CandleType.SPOT: @@ -82,6 +92,7 @@ async def fetch_ohlcv( df = DataFrame() if not df.empty: + # only return the data within the requested time range return df.loc[(df["date"] >= start) & (df["date"] < end)] else: return df @@ -102,7 +113,9 @@ async def _fetch_ohlcv( end: datetime.date, stop_on_404: bool, ) -> DataFrame: + # daily dataframes dfs: list[DataFrame | None] = [] + # the current day being processing, starting at 1. current_day = 0 connector = aiohttp.TCPConnector(limit=100) @@ -116,8 +129,10 @@ async def _fetch_ohlcv( current_day += 1 if isinstance(result, Http404): if stop_on_404: + # A 404 error on the first day indicates missing data + # on https://data.binance.vision, we provide the warning and the advice. + # https://github.com/freqtrade/freqtrade/blob/acc53065e5fa7ab5197073276306dc9dc3adbfa3/tests/exchange_online/test_binance_compare_ohlcv.py#L7 if current_day == 1: - # https://github.com/freqtrade/freqtrade/blob/acc53065e5fa7ab5197073276306dc9dc3adbfa3/tests/exchange_online/test_binance_compare_ohlcv.py#L7 logger.warning( "Failed to use fast download, fall back to rest API download, this " "can take more time. If you're downloading BTC/USDT:USDT, " @@ -131,8 +146,7 @@ async def _fetch_ohlcv( dfs.append(None) elif isinstance(result, BaseException): logger.warning(f"An exception raised: : {result}") - # Directly return the existing data, do not allow the gap - # between the data + # Directly return the existing data, do not allow the gap within the data return concat(dfs) else: dfs.append(result) @@ -175,7 +189,7 @@ async def get_daily_ohlcv( session: aiohttp.ClientSession, retry_count: int = 3, retry_delay: float = 0.0, -) -> DataFrame | None | Exception: +) -> DataFrame | Exception: """ Get daily OHLCV from https://data.binance.vision See https://github.com/binance/binance-public-data @@ -225,7 +239,10 @@ async def get_daily_ohlcv( else: raise BadHttpStatus(f"{resp.status} - {resp.reason}") except Exception as e: - retry += 1 - if retry >= retry_count: - logger.debug(f"Failed to get data from {url}: {e}") + if isinstance(e, Http404): return e + else: + if retry >= retry_count: + logger.debug(f"Failed to get data from {url}: {e}") + return e + retry += 1 diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py index 49caf5f06..a87804acc 100644 --- a/tests/exchange/test_binance_public_data.py +++ b/tests/exchange/test_binance_public_data.py @@ -254,43 +254,48 @@ async def test_get_daily_ohlcv(mocker, testdatadir): async with aiohttp.ClientSession() as session: path = testdatadir / "binance/binance_public_data/spot-klines-BTCUSDT-1h-2024-10-28.zip" - mocker.patch( + get = mocker.patch( "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get", return_value=MockResponse(path.read_bytes(), 200), ) df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) + assert get.call_count == 1 assert df["date"].iloc[0] == first_date assert df["date"].iloc[-1] == last_date path = ( testdatadir / "binance/binance_public_data/futures-um-klines-BTCUSDT-1h-2024-10-28.zip" ) - mocker.patch( + get = mocker.patch( "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get", return_value=MockResponse(path.read_bytes(), 200), ) df = await get_daily_ohlcv("futures/um", symbol, timeframe, date, session) + assert get.call_count == 1 assert df["date"].iloc[0] == first_date assert df["date"].iloc[-1] == last_date - mocker.patch( + get = mocker.patch( "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get", return_value=MockResponse(b"", 404), ) df = await get_daily_ohlcv("spot", symbol, timeframe, date, session, retry_delay=0) + assert get.call_count == 1 assert isinstance(df, Http404) - mocker.patch( + get = mocker.patch( "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get", return_value=MockResponse(b"", 500), ) mocker.patch("asyncio.sleep") df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) + assert get.call_count == 4 # 1 + 3 default retries assert isinstance(df, BadHttpStatus) - mocker.patch( + get = mocker.patch( "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get", return_value=MockResponse(b"nop", 200), ) df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) + assert get.call_count == 4 # 1 + 3 default retries assert isinstance(df, zipfile.BadZipFile) From c869dfea3f21dc77a3e6c93b8a39c07dcfa3c686 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Wed, 13 Nov 2024 17:32:25 +0800 Subject: [PATCH 26/45] chore: add more docstrings --- freqtrade/exchange/binance_public_data.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index 685c2c42f..01b3a9f19 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -113,7 +113,7 @@ async def _fetch_ohlcv( end: datetime.date, stop_on_404: bool, ) -> DataFrame: - # daily dataframes + # daily dataframes, `None` indicates missing data in that day (when `stop_on_404` is False) dfs: list[DataFrame | None] = [] # the current day being processing, starting at 1. current_day = 0 @@ -194,8 +194,14 @@ async def get_daily_ohlcv( Get daily OHLCV from https://data.binance.vision See https://github.com/binance/binance-public-data - :return: None indicates a 404 when trying to download the daily archive file - This function won't raise any exception, but catch and return it + :asset_type: `spot` or `futures/um` + :symbol: binance symbol name, e.g. BTCUSDT + :timeframe: e.g. 1m, 1h + :date: the returned DataFrame will cover the entire day of `date` in UTC + :session: an aiohttp.ClientSession instance + :retry_count: times to retry before returning the exceptions + :retry_delay: the time to wait before every retry + :return: This function won't raise any exceptions, it will catch and return them """ url = zip_url(asset_type, symbol, timeframe, date) From 40f70a1cc0a76b592432bc5b8fe648e05eb159b5 Mon Sep 17 00:00:00 2001 From: xzmeng Date: Thu, 14 Nov 2024 04:36:24 +0800 Subject: [PATCH 27/45] chore: warning when fall back to rest API --- freqtrade/exchange/binance_public_data.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index 01b3a9f19..734f54723 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -23,7 +23,10 @@ logger = logging.getLogger(__name__) class Http404(Exception): - pass + def __init__(self, msg, date, url): + super().__init__(msg) + self.date = date + self.url = url class BadHttpStatus(Exception): @@ -83,7 +86,7 @@ async def fetch_ohlcv( end = min(end, last_available_date) if start >= end: return DataFrame() - df = await _fetch_ohlcv(asset_type, symbol, timeframe, start, end, stop_on_404) + df = await _fetch_ohlcv(asset_type, symbol, pair, timeframe, start, end, stop_on_404) logger.debug( f"Downloaded data for {pair} from https://data.binance.vision with length {len(df)}." ) @@ -108,6 +111,7 @@ def concat(dfs) -> DataFrame: async def _fetch_ohlcv( asset_type: str, symbol: str, + pair: str, timeframe: str, start: datetime.date, end: datetime.date, @@ -129,6 +133,8 @@ async def _fetch_ohlcv( current_day += 1 if isinstance(result, Http404): if stop_on_404: + logger.debug(f"Failed to download {result.url} due to 404.") + # A 404 error on the first day indicates missing data # on https://data.binance.vision, we provide the warning and the advice. # https://github.com/freqtrade/freqtrade/blob/acc53065e5fa7ab5197073276306dc9dc3adbfa3/tests/exchange_online/test_binance_compare_ohlcv.py#L7 @@ -140,6 +146,13 @@ async def _fetch_ohlcv( "data before 2020 (using `--timerange yyyymmdd-20200101`), and " "then download the full data you need." ) + else: + logger.warning( + f"Binance fast download for {pair} stopped at {result.date} due to" + f"data missing: {result.url}, fall back to rest API for the " + "remaining data download, this can take more time." + ) + logger.debug("Abort downloading from data.binance.vision due to 404") return concat(dfs) else: @@ -241,7 +254,7 @@ async def get_daily_ohlcv( return df elif resp.status == 404: logger.debug(f"No data available for {symbol} in {format_date(date)}") - raise Http404 + raise Http404(f"404: {url}", date, url) else: raise BadHttpStatus(f"{resp.status} - {resp.reason}") except Exception as e: From 6b18c4f24c16213c52ec098863b787298322072e Mon Sep 17 00:00:00 2001 From: xzmeng Date: Thu, 14 Nov 2024 05:24:31 +0800 Subject: [PATCH 28/45] feat: cancel uncompleted tasks before return --- freqtrade/exchange/binance_public_data.py | 27 +++++++++++++++++------ 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index 734f54723..a0197b98b 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -126,10 +126,12 @@ async def _fetch_ohlcv( async with aiohttp.ClientSession(connector=connector, trust_env=True) as session: # the HTTP connections has been throttled by TCPConnector for dates in chunks(list(date_range(start, end)), 1000): - results = await asyncio.gather( - *(get_daily_ohlcv(asset_type, symbol, timeframe, date, session) for date in dates) - ) - for result in results: + tasks = [ + asyncio.create_task(get_daily_ohlcv(asset_type, symbol, timeframe, date, session)) + for date in dates + ] + for task in tasks: + result = await task current_day += 1 if isinstance(result, Http404): if stop_on_404: @@ -152,20 +154,29 @@ async def _fetch_ohlcv( f"data missing: {result.url}, fall back to rest API for the " "remaining data download, this can take more time." ) - - logger.debug("Abort downloading from data.binance.vision due to 404") + await cancel_uncompleted_tasks(tasks) return concat(dfs) else: dfs.append(None) elif isinstance(result, BaseException): logger.warning(f"An exception raised: : {result}") # Directly return the existing data, do not allow the gap within the data + await cancel_uncompleted_tasks(tasks) return concat(dfs) else: dfs.append(result) return concat(dfs) +async def cancel_uncompleted_tasks(tasks): + logger.debug("Try to cancel uncompleted download tasks.") + uncompleted_tasks = [task for task in tasks if not task.done()] + for task in uncompleted_tasks: + task.cancel() + await asyncio.gather(*uncompleted_tasks) + logger.debug("All uncompleted download tasks were successfully cancelled.") + + def date_range(start: datetime.date, end: datetime.date): date = start while date <= end: @@ -253,10 +264,12 @@ async def get_daily_ohlcv( df["date"] = pd.to_datetime(df["date"], unit="ms", utc=True) return df elif resp.status == 404: - logger.debug(f"No data available for {symbol} in {format_date(date)}") + logger.debug(f"Failed to download {url}") raise Http404(f"404: {url}", date, url) else: raise BadHttpStatus(f"{resp.status} - {resp.reason}") + except asyncio.CancelledError as e: + return e except Exception as e: if isinstance(e, Http404): return e From 8baa0f73103d86057195fa1a4dac74b8b30bd6e4 Mon Sep 17 00:00:00 2001 From: xzmeng Date: Thu, 14 Nov 2024 06:06:15 +0800 Subject: [PATCH 29/45] chore: add user friendly warnings --- freqtrade/exchange/binance_public_data.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index a0197b98b..ba6aa106a 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -142,17 +142,21 @@ async def _fetch_ohlcv( # https://github.com/freqtrade/freqtrade/blob/acc53065e5fa7ab5197073276306dc9dc3adbfa3/tests/exchange_online/test_binance_compare_ohlcv.py#L7 if current_day == 1: logger.warning( - "Failed to use fast download, fall back to rest API download, this " - "can take more time. If you're downloading BTC/USDT:USDT, " - "ETH/USDT:USDT, BCH/USDT:USDT, please first download " - "data before 2020 (using `--timerange yyyymmdd-20200101`), and " - "then download the full data you need." + f"Fast download is unavailable due to missing data: " + f"{result.url}. Falling back to the slower REST API, " + "which may take more time." ) + if pair in ["BTC/USDT:USDT", "ETH/USDT:USDT", "BCH/USDT:USDT"]: + logger.warning( + f"To avoid the delay, you can first download {pair} using " + "`--timerange -20200101`, and then download the " + "remaining data with `--timerange 20200101-`." + ) else: logger.warning( - f"Binance fast download for {pair} stopped at {result.date} due to" - f"data missing: {result.url}, fall back to rest API for the " - "remaining data download, this can take more time." + f"Binance fast download for {pair} stopped at {result.date} due to " + f"missing data: {result.url}, falling back to rest API for the " + "remaining data, this can take more time." ) await cancel_uncompleted_tasks(tasks) return concat(dfs) From 660863392bdbb16e868fffa7a3dc39725241fcba Mon Sep 17 00:00:00 2001 From: xzmeng Date: Thu, 14 Nov 2024 07:29:37 +0800 Subject: [PATCH 30/45] refactor: rename fetch_ohlcv to download_archive_ohlcv --- freqtrade/exchange/binance.py | 6 +++--- freqtrade/exchange/binance_public_data.py | 8 +++++--- tests/exchange/test_binance.py | 4 ++-- tests/exchange/test_binance_public_data.py | 14 +++++++++----- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 8104876f8..2509955be 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -10,8 +10,8 @@ from pandas import DataFrame from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS from freqtrade.enums import CandleType, MarginMode, PriceType, TradingMode from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError -from freqtrade.exchange import Exchange, binance_public_data -from freqtrade.exchange.binance_public_data import concat +from freqtrade.exchange import Exchange +from freqtrade.exchange.binance_public_data import concat, download_archive_ohlcv from freqtrade.exchange.common import retrier from freqtrade.exchange.exchange_types import FtHas, Tickers from freqtrade.exchange.exchange_utils_timeframe import timeframe_to_msecs @@ -172,7 +172,7 @@ class Binance(Exchange): candle_type == CandleType.FUTURES and timeframe in ["1m", "3m", "5m", "15m", "30m"] ): df = self.loop.run_until_complete( - binance_public_data.fetch_ohlcv( + download_archive_ohlcv( candle_type=candle_type, pair=pair, timeframe=timeframe, diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index ba6aa106a..cf47fc0c9 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -35,7 +35,7 @@ class BadHttpStatus(Exception): pass -async def fetch_ohlcv( +async def download_archive_ohlcv( candle_type: CandleType, pair: str, timeframe: str, @@ -86,7 +86,9 @@ async def fetch_ohlcv( end = min(end, last_available_date) if start >= end: return DataFrame() - df = await _fetch_ohlcv(asset_type, symbol, pair, timeframe, start, end, stop_on_404) + df = await _download_archive_ohlcv( + asset_type, symbol, pair, timeframe, start, end, stop_on_404 + ) logger.debug( f"Downloaded data for {pair} from https://data.binance.vision with length {len(df)}." ) @@ -108,7 +110,7 @@ def concat(dfs) -> DataFrame: return pd.concat(dfs) -async def _fetch_ohlcv( +async def _download_archive_ohlcv( asset_type: str, symbol: str, pair: str, diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index bb9b022ef..2b9ecd1d1 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -763,7 +763,7 @@ def patch_ohlcv(mocker, start, archive_end, api_end, timeframe): until = dt_from_ts(until_ms) if until_ms else api_end + timedelta(seconds=1) return api_storage.loc[(api_storage["date"] >= since) & (api_storage["date"] < until)] - async def fetch_ohlcv( + async def download_archive_ohlcv( candle_type, pair, timeframe, @@ -787,7 +787,7 @@ def patch_ohlcv(mocker, start, archive_end, api_end, timeframe): "freqtrade.exchange.Exchange.get_historic_ohlcv", side_effect=get_historic_ohlcv ) archive_mock = mocker.patch( - "freqtrade.exchange.binance_public_data.fetch_ohlcv", side_effect=fetch_ohlcv + "freqtrade.exchange.binance.download_archive_ohlcv", side_effect=download_archive_ohlcv ) return candle_mock, api_mock, archive_mock diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py index a87804acc..ef8a938dd 100644 --- a/tests/exchange/test_binance_public_data.py +++ b/tests/exchange/test_binance_public_data.py @@ -14,7 +14,7 @@ from freqtrade.enums import CandleType from freqtrade.exchange.binance_public_data import ( BadHttpStatus, Http404, - fetch_ohlcv, + download_archive_ohlcv, get_daily_ohlcv, zip_name, ) @@ -196,7 +196,9 @@ def make_response_from_url(start_date, end_date): ), ], ) -async def test_fetch_ohlcv(mocker, candle_type, since, until, first_date, last_date, stop_on_404): +async def test_download_archive_ohlcv( + mocker, candle_type, since, until, first_date, last_date, stop_on_404 +): history_start = dt_utc(2020, 1, 1).date() history_end = dt_utc(2020, 1, 3).date() timeframe = "1h" @@ -214,7 +216,9 @@ async def test_fetch_ohlcv(mocker, candle_type, since, until, first_date, last_d ) markets = {"BTC/USDT": {"id": "BTCUSDT"}, "BTC/USDT:USDT": {"id": "BTCUSDT"}} - df = await fetch_ohlcv(candle_type, pair, timeframe, since_ms, until_ms, markets, stop_on_404) + df = await download_archive_ohlcv( + candle_type, pair, timeframe, since_ms, until_ms, markets, stop_on_404 + ) if df.empty: assert first_date is None and last_date is None @@ -224,7 +228,7 @@ async def test_fetch_ohlcv(mocker, candle_type, since, until, first_date, last_d assert df["date"].iloc[-1] == last_date -async def test_fetch_ohlcv_exc(mocker): +async def test_download_archive_ohlcv_exc(mocker): timeframe = "1h" pair = "BTC/USDT" @@ -240,7 +244,7 @@ async def test_fetch_ohlcv_exc(mocker): {"BTC/USDT": {"id": "BTCUSDT"}}, ) - df = await fetch_ohlcv(CandleType.SPOT, pair, timeframe, since_ms, until_ms) + df = await download_archive_ohlcv(CandleType.SPOT, pair, timeframe, since_ms, until_ms) assert df.empty From bfdbf0248c187c9b07b6a308943bda70a94de4e2 Mon Sep 17 00:00:00 2001 From: xzmeng Date: Thu, 14 Nov 2024 07:51:58 +0800 Subject: [PATCH 31/45] refactor: rename asset_type to asset_type_url_segment --- freqtrade/exchange/binance_public_data.py | 24 ++++++++++--------- tests/exchange/test_binance_public_data.py | 12 +++++----- .../test_binance_compare_ohlcv.py | 10 ++++---- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index cf47fc0c9..a8231d2b6 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -64,9 +64,9 @@ async def download_archive_ohlcv( """ try: if candle_type == CandleType.SPOT: - asset_type = "spot" + asset_type_url_segment = "spot" elif candle_type == CandleType.FUTURES: - asset_type = "futures/um" + asset_type_url_segment = "futures/um" else: raise ValueError(f"Unsupported CandleType: {candle_type}") @@ -87,7 +87,7 @@ async def download_archive_ohlcv( if start >= end: return DataFrame() df = await _download_archive_ohlcv( - asset_type, symbol, pair, timeframe, start, end, stop_on_404 + asset_type_url_segment, symbol, pair, timeframe, start, end, stop_on_404 ) logger.debug( f"Downloaded data for {pair} from https://data.binance.vision with length {len(df)}." @@ -111,7 +111,7 @@ def concat(dfs) -> DataFrame: async def _download_archive_ohlcv( - asset_type: str, + asset_type_url_segment: str, symbol: str, pair: str, timeframe: str, @@ -129,7 +129,9 @@ async def _download_archive_ohlcv( # the HTTP connections has been throttled by TCPConnector for dates in chunks(list(date_range(start, end)), 1000): tasks = [ - asyncio.create_task(get_daily_ohlcv(asset_type, symbol, timeframe, date, session)) + asyncio.create_task( + get_daily_ohlcv(asset_type_url_segment, symbol, timeframe, date, session) + ) for date in dates ] for task in tasks: @@ -198,21 +200,21 @@ def zip_name(symbol: str, timeframe: str, date: datetime.date) -> str: return f"{symbol}-{timeframe}-{format_date(date)}.zip" -def zip_url(asset_type: str, symbol: str, timeframe: str, date: datetime.date) -> str: +def zip_url(asset_type_url_segment: str, symbol: str, timeframe: str, date: datetime.date) -> str: """ example urls: https://data.binance.vision/data/spot/daily/klines/BTCUSDT/1s/BTCUSDT-1s-2023-10-27.zip https://data.binance.vision/data/futures/um/daily/klines/BTCUSDT/1h/BTCUSDT-1h-2023-10-27.zip """ url = ( - f"https://data.binance.vision/data/{asset_type}/daily/klines/{symbol}/{timeframe}/" - f"{zip_name(symbol, timeframe, date)}" + f"https://data.binance.vision/data/{asset_type_url_segment}/daily/klines/{symbol}" + f"/{timeframe}/{zip_name(symbol, timeframe, date)}" ) return url async def get_daily_ohlcv( - asset_type: str, + asset_type_url_segment: str, symbol: str, timeframe: str, date: datetime.date, @@ -224,7 +226,7 @@ async def get_daily_ohlcv( Get daily OHLCV from https://data.binance.vision See https://github.com/binance/binance-public-data - :asset_type: `spot` or `futures/um` + :asset_type_url_segment: `spot` or `futures/um` :symbol: binance symbol name, e.g. BTCUSDT :timeframe: e.g. 1m, 1h :date: the returned DataFrame will cover the entire day of `date` in UTC @@ -234,7 +236,7 @@ async def get_daily_ohlcv( :return: This function won't raise any exceptions, it will catch and return them """ - url = zip_url(asset_type, symbol, timeframe, date) + url = zip_url(asset_type_url_segment, symbol, timeframe, date) logger.debug(f"download data from binance: {url}") diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py index ef8a938dd..6b23cb2d6 100644 --- a/tests/exchange/test_binance_public_data.py +++ b/tests/exchange/test_binance_public_data.py @@ -51,11 +51,11 @@ def make_daily_df(date, timeframe): return df -def make_daily_zip(asset_type, symbol, timeframe, date) -> bytes: +def make_daily_zip(asset_type_url_segment, symbol, timeframe, date) -> bytes: df = make_daily_df(date, timeframe) - if asset_type == "spot": + if asset_type_url_segment == "spot": header = True - elif asset_type == "futures/um": + elif asset_type_url_segment == "futures/um": header = None else: raise ValueError @@ -85,8 +85,8 @@ class MockResponse: def make_response_from_url(start_date, end_date): def make_response(url): pattern = ( - r"https://data.binance.vision/data/(?Pspot|futures/um)/daily/klines/" - r"(?P.*?)/(?P.*?)/(?P=symbol)-(?P=timeframe)-" + r"https://data.binance.vision/data/(?Pspot|futures/um)" + r"/daily/klines/(?P.*?)/(?P.*?)/(?P=symbol)-(?P=timeframe)-" r"(?P\d{4}-\d{2}-\d{2}).zip" ) m = re.match(pattern, url) @@ -97,7 +97,7 @@ def make_response_from_url(start_date, end_date): if date < start_date or date > end_date: return MockResponse(content="", status=404) - zip_file = make_daily_zip(m["asset_type"], m["symbol"], m["timeframe"], date) + zip_file = make_daily_zip(m["asset_type_url_segment"], m["symbol"], m["timeframe"], date) return MockResponse(content=zip_file, status=200) return make_response diff --git a/tests/exchange_online/test_binance_compare_ohlcv.py b/tests/exchange_online/test_binance_compare_ohlcv.py index af94b1bd1..2b23824f7 100644 --- a/tests/exchange_online/test_binance_compare_ohlcv.py +++ b/tests/exchange_online/test_binance_compare_ohlcv.py @@ -31,14 +31,14 @@ from freqtrade.util.datetime_helpers import dt_from_ts class Check: - def __init__(self, asset_type, timeframe): - self.asset_type = asset_type + def __init__(self, asset_type_url_segment, timeframe): + self.asset_type_url_segment = asset_type_url_segment self.timeframe = timeframe self.klines_endpoint = "https://api.binance.com/api/v3/klines" self.exchange_endpoint = "https://api.binance.com/api/v3/exchangeInfo" self.mismatch = set() - if asset_type == "futures/um": + if asset_type_url_segment == "futures/um": self.klines_endpoint = "https://fapi.binance.com/fapi/v1/klines" self.exchange_endpoint = "https://fapi.binance.com/fapi/v1/exchangeInfo" @@ -52,7 +52,9 @@ class Check: first_kline_ts = first_kline[0] date = dt_from_ts(first_kline_ts).date() - archive_url = zip_url(self.asset_type, symbol=symbol, timeframe=self.timeframe, date=date) + archive_url = zip_url( + self.asset_type_url_segment, symbol=symbol, timeframe=self.timeframe, date=date + ) async with self.session.get( archive_url, params=dict(symbol=symbol, interval=self.timeframe, startTime=0) ) as resp: From 19f96d60e3bc899818af6833ba941e4df8df8ff7 Mon Sep 17 00:00:00 2001 From: xzmeng Date: Thu, 14 Nov 2024 08:09:59 +0800 Subject: [PATCH 32/45] refactor: streamline error handling by raising instead of returning --- freqtrade/exchange/binance_public_data.py | 48 ++++++++-------- tests/exchange/test_binance_public_data.py | 66 +++++++++++++++------- 2 files changed, 68 insertions(+), 46 deletions(-) diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index a8231d2b6..c27bb74a6 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -135,11 +135,12 @@ async def _download_archive_ohlcv( for date in dates ] for task in tasks: - result = await task current_day += 1 - if isinstance(result, Http404): + try: + df = await task + except Http404 as e: if stop_on_404: - logger.debug(f"Failed to download {result.url} due to 404.") + logger.debug(f"Failed to download {e.url} due to 404.") # A 404 error on the first day indicates missing data # on https://data.binance.vision, we provide the warning and the advice. @@ -147,7 +148,7 @@ async def _download_archive_ohlcv( if current_day == 1: logger.warning( f"Fast download is unavailable due to missing data: " - f"{result.url}. Falling back to the slower REST API, " + f"{e.url}. Falling back to the slower REST API, " "which may take more time." ) if pair in ["BTC/USDT:USDT", "ETH/USDT:USDT", "BCH/USDT:USDT"]: @@ -158,31 +159,31 @@ async def _download_archive_ohlcv( ) else: logger.warning( - f"Binance fast download for {pair} stopped at {result.date} due to " - f"missing data: {result.url}, falling back to rest API for the " + f"Binance fast download for {pair} stopped at {e.date} due to " + f"missing data: {e.url}, falling back to rest API for the " "remaining data, this can take more time." ) - await cancel_uncompleted_tasks(tasks) + await cancel_and_await_tasks(tasks[tasks.index(task) + 1 :]) return concat(dfs) else: dfs.append(None) - elif isinstance(result, BaseException): - logger.warning(f"An exception raised: : {result}") + except BaseException as e: + logger.warning(f"An exception raised: : {e}") # Directly return the existing data, do not allow the gap within the data - await cancel_uncompleted_tasks(tasks) + await cancel_and_await_tasks(tasks[tasks.index(task) + 1 :]) return concat(dfs) else: - dfs.append(result) + dfs.append(df) return concat(dfs) -async def cancel_uncompleted_tasks(tasks): +async def cancel_and_await_tasks(unawaited_tasks): + """Cancel and await the tasks""" logger.debug("Try to cancel uncompleted download tasks.") - uncompleted_tasks = [task for task in tasks if not task.done()] - for task in uncompleted_tasks: + for task in unawaited_tasks: task.cancel() - await asyncio.gather(*uncompleted_tasks) - logger.debug("All uncompleted download tasks were successfully cancelled.") + await asyncio.gather(*unawaited_tasks, return_exceptions=True) + logger.debug("All download tasks were awaited.") def date_range(start: datetime.date, end: datetime.date): @@ -221,7 +222,7 @@ async def get_daily_ohlcv( session: aiohttp.ClientSession, retry_count: int = 3, retry_delay: float = 0.0, -) -> DataFrame | Exception: +) -> DataFrame: """ Get daily OHLCV from https://data.binance.vision See https://github.com/binance/binance-public-data @@ -233,7 +234,7 @@ async def get_daily_ohlcv( :session: an aiohttp.ClientSession instance :retry_count: times to retry before returning the exceptions :retry_delay: the time to wait before every retry - :return: This function won't raise any exceptions, it will catch and return them + :return: A dataframe containing columns date,open,high,low,close,volume """ url = zip_url(asset_type_url_segment, symbol, timeframe, date) @@ -276,13 +277,8 @@ async def get_daily_ohlcv( raise Http404(f"404: {url}", date, url) else: raise BadHttpStatus(f"{resp.status} - {resp.reason}") - except asyncio.CancelledError as e: - return e except Exception as e: - if isinstance(e, Http404): - return e - else: - if retry >= retry_count: - logger.debug(f"Failed to get data from {url}: {e}") - return e retry += 1 + if isinstance(e, Http404) or retry > retry_count: + logger.debug(f"Failed to get data from {url}: {e}") + raise diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py index 6b23cb2d6..f6e0c2e6a 100644 --- a/tests/exchange/test_binance_public_data.py +++ b/tests/exchange/test_binance_public_data.py @@ -104,10 +104,11 @@ def make_response_from_url(start_date, end_date): @pytest.mark.parametrize( - "candle_type,since,until,first_date,last_date,stop_on_404", + "candle_type,pair,since,until,first_date,last_date,stop_on_404", [ ( CandleType.SPOT, + "BTC/USDT", dt_utc(2020, 1, 1), dt_utc(2020, 1, 2), dt_utc(2020, 1, 1), @@ -116,6 +117,7 @@ def make_response_from_url(start_date, end_date): ), ( CandleType.SPOT, + "BTC/USDT", dt_utc(2020, 1, 1), dt_utc(2020, 1, 1, 23, 59, 59), dt_utc(2020, 1, 1), @@ -124,6 +126,7 @@ def make_response_from_url(start_date, end_date): ), ( CandleType.SPOT, + "BTC/USDT", dt_utc(2020, 1, 1), dt_utc(2020, 1, 5), dt_utc(2020, 1, 1), @@ -132,6 +135,7 @@ def make_response_from_url(start_date, end_date): ), ( CandleType.SPOT, + "BTC/USDT", dt_utc(2019, 12, 25), dt_utc(2020, 1, 5), dt_utc(2020, 1, 1), @@ -140,6 +144,7 @@ def make_response_from_url(start_date, end_date): ), ( CandleType.SPOT, + "BTC/USDT", dt_utc(2019, 1, 1), dt_utc(2019, 1, 5), None, @@ -148,6 +153,7 @@ def make_response_from_url(start_date, end_date): ), ( CandleType.SPOT, + "BTC/USDT", dt_utc(2021, 1, 1), dt_utc(2021, 1, 5), None, @@ -156,6 +162,7 @@ def make_response_from_url(start_date, end_date): ), ( CandleType.SPOT, + "BTC/USDT", dt_utc(2020, 1, 2), None, dt_utc(2020, 1, 2), @@ -164,14 +171,7 @@ def make_response_from_url(start_date, end_date): ), ( CandleType.SPOT, - dt_utc(2019, 12, 25), - dt_utc(2020, 1, 5), - None, - None, - True, - ), - ( - CandleType.SPOT, + "BTC/USDT", dt_utc(2020, 1, 5), dt_utc(2020, 1, 1), None, @@ -180,6 +180,7 @@ def make_response_from_url(start_date, end_date): ), ( CandleType.FUTURES, + "BTC/USDT:USDT", dt_utc(2020, 1, 1), dt_utc(2020, 1, 1, 23, 59, 59), dt_utc(2020, 1, 1), @@ -188,24 +189,49 @@ def make_response_from_url(start_date, end_date): ), ( CandleType.INDEX, + "N/A", dt_utc(2020, 1, 1), dt_utc(2020, 1, 1, 23, 59, 59), None, None, False, ), + # stop_on_404 = True + ( + CandleType.SPOT, + "BTC/USDT", + dt_utc(2019, 12, 25), + dt_utc(2020, 1, 5), + None, + None, + True, + ), + ( + CandleType.SPOT, + "BTC/USDT", + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 5), + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 3, 23), + True, + ), + ( + CandleType.FUTURES, + "BTC/USDT:USDT", + dt_utc(2019, 12, 25), + dt_utc(2020, 1, 5), + None, + None, + True, + ), ], ) async def test_download_archive_ohlcv( - mocker, candle_type, since, until, first_date, last_date, stop_on_404 + mocker, candle_type, pair, since, until, first_date, last_date, stop_on_404 ): history_start = dt_utc(2020, 1, 1).date() history_end = dt_utc(2020, 1, 3).date() timeframe = "1h" - if candle_type == CandleType.SPOT: - pair = "BTC/USDT" - else: - pair = "BTC/USDT:USDT" since_ms = dt_ts(since) until_ms = dt_ts(until) @@ -283,23 +309,23 @@ async def test_get_daily_ohlcv(mocker, testdatadir): "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get", return_value=MockResponse(b"", 404), ) - df = await get_daily_ohlcv("spot", symbol, timeframe, date, session, retry_delay=0) + with pytest.raises(Http404): + df = await get_daily_ohlcv("spot", symbol, timeframe, date, session, retry_delay=0) assert get.call_count == 1 - assert isinstance(df, Http404) get = mocker.patch( "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get", return_value=MockResponse(b"", 500), ) mocker.patch("asyncio.sleep") - df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) + with pytest.raises(BadHttpStatus): + df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) assert get.call_count == 4 # 1 + 3 default retries - assert isinstance(df, BadHttpStatus) get = mocker.patch( "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get", return_value=MockResponse(b"nop", 200), ) - df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) + with pytest.raises(zipfile.BadZipFile): + df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) assert get.call_count == 4 # 1 + 3 default retries - assert isinstance(df, zipfile.BadZipFile) From 0f53dc1b7bb804aad6fd790aa23de1f0d7b6809e Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Fri, 15 Nov 2024 12:08:50 +0800 Subject: [PATCH 33/45] chore: improve log level to warning when falling back to API --- freqtrade/exchange/binance_public_data.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index c27bb74a6..dcd53bfae 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -93,7 +93,11 @@ async def download_archive_ohlcv( f"Downloaded data for {pair} from https://data.binance.vision with length {len(df)}." ) except Exception as e: - logger.debug("An exception occurred", exc_info=e) + logger.warning( + "An exception occurred during fast download from Binance, falling back to" + "the slower REST API, this can take more time.", + exc_info=e, + ) df = DataFrame() if not df.empty: From 69c1de7e4ac51326b8c6f2bb2a6e383b004b1661 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 27 Nov 2024 18:20:44 +0100 Subject: [PATCH 34/45] chore: move fallback param to config directly --- docs/configuration.md | 1 + freqtrade/exchange/binance.py | 5 ++--- freqtrade/exchange/exchange.py | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 074c9a577..80ad534d8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -225,6 +225,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exchange.skip_open_order_update` | Skips open order updates on startup should the exchange cause problems. Only relevant in live conditions.
*Defaults to `false`*
**Datatype:** Boolean | `exchange.unknown_fee_rate` | Fallback value to use when calculating trading fees. This can be useful for exchanges which have fees in non-tradable currencies. The value provided here will be multiplied with the "fee cost".
*Defaults to `None`
**Datatype:** float | `exchange.log_responses` | Log relevant exchange responses. For debug mode only - use with care.
*Defaults to `false`*
**Datatype:** Boolean +| `exchange.only_from_ccxt` | Prevent data-download from data.binance.vision. Leaving this as false can greatly speed up downloads, but may be problematic if the site is not available.
*Defaults to `false`*
**Datatype:** Boolean | `experimental.block_bad_exchanges` | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
*Defaults to `true`.*
**Datatype:** Boolean | | **Plugins** | `edge.*` | Please refer to [edge configuration document](edge.md) for detailed explanation of all possible configuration options. diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 2509955be..4c4271570 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -110,13 +110,11 @@ class Binance(Exchange): candle_type: CandleType, is_new_pair: bool = False, until_ms: int | None = None, - only_from_ccxt: bool = False, ) -> DataFrame: """ Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date Does not work for other exchanges, which don't return the earliest data when called with "0" :param candle_type: Any of the enum CandleType (must match trading mode!) - :param only_from_ccxt: Only download data using the API provided by CCXT """ if is_new_pair: x = self.loop.run_until_complete( @@ -136,7 +134,7 @@ class Binance(Exchange): ) return DataFrame(columns=DEFAULT_DATAFRAME_COLUMNS) - if only_from_ccxt: + if self._config["exchange"].get("only_from_ccxt", False): return super().get_historic_ohlcv( pair=pair, timeframe=timeframe, @@ -146,6 +144,7 @@ class Binance(Exchange): until_ms=until_ms, ) else: + # Download from data.binance.vision return self.get_historic_ohlcv_fast( pair=pair, timeframe=timeframe, diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 983ee8f0a..f2515f7eb 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2230,7 +2230,6 @@ class Exchange: candle_type: CandleType, is_new_pair: bool = False, until_ms: int | None = None, - only_from_ccxt: bool = False, ) -> DataFrame: """ Get candle history using asyncio and returns the list of candles. @@ -2242,7 +2241,6 @@ class Exchange: :param candle_type: '', mark, index, premiumIndex, or funding_rate :param is_new_pair: used by binance subclass to allow "fast" new pair downloading :param until_ms: Timestamp in milliseconds to get history up to - :param only_from_ccxt: Only download data using the API provided by CCXT :return: Dataframe with candle (OHLCV) data """ pair, _, _, data, _ = self.loop.run_until_complete( From 0e77c89d790312159aa432951d6b136eb6429d6f Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 28 Nov 2024 06:43:51 +0100 Subject: [PATCH 35/45] chore: simplify code structure by moving conditional to parent method --- freqtrade/exchange/binance.py | 75 +++++++++++++++++------------------ 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 4c4271570..d40a189f9 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -134,7 +134,19 @@ class Binance(Exchange): ) return DataFrame(columns=DEFAULT_DATAFRAME_COLUMNS) - if self._config["exchange"].get("only_from_ccxt", False): + if ( + self._config["exchange"].get("only_from_ccxt", False) + and + # only download timeframes with significant improvements, + # otherwise fall back to rest API + not ( + (candle_type == CandleType.SPOT and timeframe in ["1s", "1m", "3m", "5m"]) + or ( + candle_type == CandleType.FUTURES + and timeframe in ["1m", "3m", "5m", "15m", "30m"] + ) + ) + ): return super().get_historic_ohlcv( pair=pair, timeframe=timeframe, @@ -166,50 +178,37 @@ class Binance(Exchange): """ Fastly fetch OHLCV data by leveraging https://data.binance.vision. """ - # only download timeframes with significant improvements, otherwise fall back to rest API - if (candle_type == CandleType.SPOT and timeframe in ["1s", "1m", "3m", "5m"]) or ( - candle_type == CandleType.FUTURES and timeframe in ["1m", "3m", "5m", "15m", "30m"] - ): - df = self.loop.run_until_complete( - download_archive_ohlcv( - candle_type=candle_type, - pair=pair, - timeframe=timeframe, - since_ms=since_ms, - until_ms=until_ms, - markets=self.markets, - ) - ) - - # download the remaining data from rest API - if df.empty: - rest_since_ms = since_ms - else: - rest_since_ms = dt_ts(df.iloc[-1].date) + timeframe_to_msecs(timeframe) - - # make sure since <= until - if until_ms and rest_since_ms > until_ms: - rest_df = DataFrame() - else: - rest_df = super().get_historic_ohlcv( - pair=pair, - timeframe=timeframe, - since_ms=rest_since_ms, - candle_type=candle_type, - is_new_pair=is_new_pair, - until_ms=until_ms, - ) - all_df = concat([df, rest_df]) - return all_df - else: - return super().get_historic_ohlcv( + df = self.loop.run_until_complete( + download_archive_ohlcv( + candle_type=candle_type, pair=pair, timeframe=timeframe, since_ms=since_ms, + until_ms=until_ms, + markets=self.markets, + ) + ) + + # download the remaining data from rest API + if df.empty: + rest_since_ms = since_ms + else: + rest_since_ms = dt_ts(df.iloc[-1].date) + timeframe_to_msecs(timeframe) + + # make sure since <= until + if until_ms and rest_since_ms > until_ms: + rest_df = DataFrame() + else: + rest_df = super().get_historic_ohlcv( + pair=pair, + timeframe=timeframe, + since_ms=rest_since_ms, candle_type=candle_type, is_new_pair=is_new_pair, until_ms=until_ms, ) + all_df = concat([df, rest_df]) + return all_df def funding_fee_cutoff(self, open_date: datetime): """ From e2a09f272abef1dac94f37e13f65bad0f214ac73 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 28 Nov 2024 06:55:15 +0100 Subject: [PATCH 36/45] chore: Improve naming, remove unnecessary method --- freqtrade/exchange/binance_public_data.py | 16 +++++++--------- tests/exchange/test_binance_public_data.py | 4 ++-- .../test_binance_compare_ohlcv.py | 4 ++-- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index dcd53bfae..423e30006 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -197,15 +197,13 @@ def date_range(start: datetime.date, end: datetime.date): date += datetime.timedelta(days=1) -def format_date(date: datetime.date) -> str: - return date.strftime("%Y-%m-%d") +def binance_vision_zip_name(symbol: str, timeframe: str, date: datetime.date) -> str: + return f"{symbol}-{timeframe}-{date.strftime('%Y-%m-%d')}.zip" -def zip_name(symbol: str, timeframe: str, date: datetime.date) -> str: - return f"{symbol}-{timeframe}-{format_date(date)}.zip" - - -def zip_url(asset_type_url_segment: str, symbol: str, timeframe: str, date: datetime.date) -> str: +def binance_vision_zip_url( + asset_type_url_segment: str, symbol: str, timeframe: str, date: datetime.date +) -> str: """ example urls: https://data.binance.vision/data/spot/daily/klines/BTCUSDT/1s/BTCUSDT-1s-2023-10-27.zip @@ -213,7 +211,7 @@ def zip_url(asset_type_url_segment: str, symbol: str, timeframe: str, date: date """ url = ( f"https://data.binance.vision/data/{asset_type_url_segment}/daily/klines/{symbol}" - f"/{timeframe}/{zip_name(symbol, timeframe, date)}" + f"/{timeframe}/{binance_vision_zip_name(symbol, timeframe, date)}" ) return url @@ -241,7 +239,7 @@ async def get_daily_ohlcv( :return: A dataframe containing columns date,open,high,low,close,volume """ - url = zip_url(asset_type_url_segment, symbol, timeframe, date) + url = binance_vision_zip_url(asset_type_url_segment, symbol, timeframe, date) logger.debug(f"download data from binance: {url}") diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py index f6e0c2e6a..6f6280850 100644 --- a/tests/exchange/test_binance_public_data.py +++ b/tests/exchange/test_binance_public_data.py @@ -14,9 +14,9 @@ from freqtrade.enums import CandleType from freqtrade.exchange.binance_public_data import ( BadHttpStatus, Http404, + binance_vision_zip_name, download_archive_ohlcv, get_daily_ohlcv, - zip_name, ) from freqtrade.util.datetime_helpers import dt_ts, dt_utc @@ -62,7 +62,7 @@ def make_daily_zip(asset_type_url_segment, symbol, timeframe, date) -> bytes: csv = df.to_csv(index=False, header=header) zip_buffer = io.BytesIO() with zipfile.ZipFile(zip_buffer, "w") as zipf: - zipf.writestr(zip_name(symbol, timeframe, date), csv) + zipf.writestr(binance_vision_zip_name(symbol, timeframe, date), csv) return zip_buffer.getvalue() diff --git a/tests/exchange_online/test_binance_compare_ohlcv.py b/tests/exchange_online/test_binance_compare_ohlcv.py index 2b23824f7..a6335ef22 100644 --- a/tests/exchange_online/test_binance_compare_ohlcv.py +++ b/tests/exchange_online/test_binance_compare_ohlcv.py @@ -26,7 +26,7 @@ import os import aiohttp import pytest -from freqtrade.exchange.binance_public_data import zip_url +from freqtrade.exchange.binance_public_data import binance_vision_zip_url from freqtrade.util.datetime_helpers import dt_from_ts @@ -52,7 +52,7 @@ class Check: first_kline_ts = first_kline[0] date = dt_from_ts(first_kline_ts).date() - archive_url = zip_url( + archive_url = binance_vision_zip_url( self.asset_type_url_segment, symbol=symbol, timeframe=self.timeframe, date=date ) async with self.session.get( From b1ca00b037c05b7cbd2c4f26158d5d663fffadaa Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 28 Nov 2024 07:04:08 +0100 Subject: [PATCH 37/45] chore: don't shadow pandas builtin methods --- freqtrade/exchange/binance.py | 4 ++-- freqtrade/exchange/binance_public_data.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index d40a189f9..5264f644f 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -11,7 +11,7 @@ from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS from freqtrade.enums import CandleType, MarginMode, PriceType, TradingMode from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError from freqtrade.exchange import Exchange -from freqtrade.exchange.binance_public_data import concat, download_archive_ohlcv +from freqtrade.exchange.binance_public_data import concat_safe, download_archive_ohlcv from freqtrade.exchange.common import retrier from freqtrade.exchange.exchange_types import FtHas, Tickers from freqtrade.exchange.exchange_utils_timeframe import timeframe_to_msecs @@ -207,7 +207,7 @@ class Binance(Exchange): is_new_pair=is_new_pair, until_ms=until_ms, ) - all_df = concat([df, rest_df]) + all_df = concat_safe([df, rest_df]) return all_df def funding_fee_cutoff(self, open_date: datetime): diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index 423e30006..a64935791 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -47,7 +47,7 @@ async def download_archive_ohlcv( """ Fetch OHLCV data from https://data.binance.vision The function makes its best effort to download data within the time range - [`since_ms`, `until_ms`) -- including `since_ms`, but excluding `until_ms`. + [`since_ms`, `until_ms`] -- including `since_ms`, but excluding `until_ms`. If `stop_one_404` is True, this returned DataFrame is guaranteed to start from `since_ms` with no gaps in the data. @@ -107,7 +107,7 @@ async def download_archive_ohlcv( return df -def concat(dfs) -> DataFrame: +def concat_safe(dfs) -> DataFrame: if all(df is None for df in dfs): return DataFrame() else: @@ -168,17 +168,17 @@ async def _download_archive_ohlcv( "remaining data, this can take more time." ) await cancel_and_await_tasks(tasks[tasks.index(task) + 1 :]) - return concat(dfs) + return concat_safe(dfs) else: dfs.append(None) except BaseException as e: logger.warning(f"An exception raised: : {e}") # Directly return the existing data, do not allow the gap within the data await cancel_and_await_tasks(tasks[tasks.index(task) + 1 :]) - return concat(dfs) + return concat_safe(dfs) else: dfs.append(df) - return concat(dfs) + return concat_safe(dfs) async def cancel_and_await_tasks(unawaited_tasks): From 21777a580431b9bc299ecb4ce8083f4f2f1264c9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 28 Nov 2024 07:09:57 +0100 Subject: [PATCH 38/45] chore: do more pinpointed imports --- freqtrade/exchange/binance_public_data.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index a64935791..b6bce37da 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -3,10 +3,10 @@ Fetch daily-archived OHLCV data from https://data.binance.vision/ """ import asyncio -import datetime -import io import logging import zipfile +from datetime import date, timedelta +from io import BytesIO from typing import Any import aiohttp @@ -82,7 +82,7 @@ async def download_archive_ohlcv( # We use two days ago as the last available day because the daily archives are daily # uploaded and have several hours delay - last_available_date = dt_now() - datetime.timedelta(days=2) + last_available_date = dt_now() - timedelta(days=2) end = min(end, last_available_date) if start >= end: return DataFrame() @@ -119,8 +119,8 @@ async def _download_archive_ohlcv( symbol: str, pair: str, timeframe: str, - start: datetime.date, - end: datetime.date, + start: date, + end: date, stop_on_404: bool, ) -> DataFrame: # daily dataframes, `None` indicates missing data in that day (when `stop_on_404` is False) @@ -190,19 +190,19 @@ async def cancel_and_await_tasks(unawaited_tasks): logger.debug("All download tasks were awaited.") -def date_range(start: datetime.date, end: datetime.date): +def date_range(start: date, end: date): date = start while date <= end: yield date - date += datetime.timedelta(days=1) + date += timedelta(days=1) -def binance_vision_zip_name(symbol: str, timeframe: str, date: datetime.date) -> str: +def binance_vision_zip_name(symbol: str, timeframe: str, date: date) -> str: return f"{symbol}-{timeframe}-{date.strftime('%Y-%m-%d')}.zip" def binance_vision_zip_url( - asset_type_url_segment: str, symbol: str, timeframe: str, date: datetime.date + asset_type_url_segment: str, symbol: str, timeframe: str, date: date ) -> str: """ example urls: @@ -220,7 +220,7 @@ async def get_daily_ohlcv( asset_type_url_segment: str, symbol: str, timeframe: str, - date: datetime.date, + date: date, session: aiohttp.ClientSession, retry_count: int = 3, retry_delay: float = 0.0, @@ -256,7 +256,7 @@ async def get_daily_ohlcv( if resp.status == 200: content = await resp.read() logger.debug(f"Successfully downloaded {url}") - with zipfile.ZipFile(io.BytesIO(content)) as zipf: + with zipfile.ZipFile(BytesIO(content)) as zipf: with zipf.open(zipf.namelist()[0]) as csvf: # https://github.com/binance/binance-public-data/issues/283 first_byte = csvf.read(1)[0] From 675b996f576ba1dc5252ab360fd7f2eeed5ad7e5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 28 Nov 2024 07:19:13 +0100 Subject: [PATCH 39/45] chore: improved naming in test case --- tests/exchange/test_binance.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index 2b9ecd1d1..7a95366a4 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -734,15 +734,14 @@ def test__set_leverage_binance(mocker, default_conf): ) -def make_storage(start: datetime, end: datetime, timeframe: str): - date = pd.date_range(start, end, freq=timeframe.replace("m", "min")) - df = pd.DataFrame( - data=dict(date=date, open=1.0, high=1.0, low=1.0, close=1.0), - ) - return df +def patch_binance_vision_ohlcv(mocker, start, archive_end, api_end, timeframe): + def make_storage(start: datetime, end: datetime, timeframe: str): + date = pd.date_range(start, end, freq=timeframe.replace("m", "min")) + df = pd.DataFrame( + data=dict(date=date, open=1.0, high=1.0, low=1.0, close=1.0), + ) + return df - -def patch_ohlcv(mocker, start, archive_end, api_end, timeframe): archive_storage = make_storage(start, archive_end, timeframe) api_storage = make_storage(start, api_end, timeframe) @@ -780,12 +779,8 @@ def patch_ohlcv(mocker, start, archive_end, api_end, timeframe): (archive_storage["date"] >= since) & (archive_storage["date"] < until) ] - candle_mock = mocker.patch( - "freqtrade.exchange.Exchange._async_get_candle_history", return_value=candle_history - ) - api_mock = mocker.patch( - "freqtrade.exchange.Exchange.get_historic_ohlcv", side_effect=get_historic_ohlcv - ) + candle_mock = mocker.patch(f"{EXMS}._async_get_candle_history", return_value=candle_history) + api_mock = mocker.patch(f"{EXMS}.get_historic_ohlcv", side_effect=get_historic_ohlcv) archive_mock = mocker.patch( "freqtrade.exchange.binance.download_archive_ohlcv", side_effect=download_archive_ohlcv ) @@ -948,7 +943,7 @@ def test_get_historic_ohlcv_binance( start = dt_utc(2020, 1, 1) archive_end = dt_utc(2020, 1, 2) api_end = dt_utc(2020, 1, 3) - candle_mock, api_mock, archive_mock = patch_ohlcv( + candle_mock, api_mock, archive_mock = patch_binance_vision_ohlcv( mocker, start=start, archive_end=archive_end, api_end=api_end, timeframe=timeframe ) From d1710826e627fd3c4d64dc65d475583e78cbb333 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 28 Nov 2024 07:27:29 +0100 Subject: [PATCH 40/45] chore: improved naming --- tests/exchange/test_binance_public_data.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py index 6f6280850..5f066d75e 100644 --- a/tests/exchange/test_binance_public_data.py +++ b/tests/exchange/test_binance_public_data.py @@ -283,22 +283,24 @@ async def test_get_daily_ohlcv(mocker, testdatadir): last_date = dt_utc(2024, 10, 28, 23) async with aiohttp.ClientSession() as session: - path = testdatadir / "binance/binance_public_data/spot-klines-BTCUSDT-1h-2024-10-28.zip" + spot_path = ( + testdatadir / "binance/binance_public_data/spot-klines-BTCUSDT-1h-2024-10-28.zip" + ) get = mocker.patch( "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get", - return_value=MockResponse(path.read_bytes(), 200), + return_value=MockResponse(spot_path.read_bytes(), 200), ) df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) assert get.call_count == 1 assert df["date"].iloc[0] == first_date assert df["date"].iloc[-1] == last_date - path = ( + futures_path = ( testdatadir / "binance/binance_public_data/futures-um-klines-BTCUSDT-1h-2024-10-28.zip" ) get = mocker.patch( "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get", - return_value=MockResponse(path.read_bytes(), 200), + return_value=MockResponse(futures_path.read_bytes(), 200), ) df = await get_daily_ohlcv("futures/um", symbol, timeframe, date, session) assert get.call_count == 1 From fa1e0af19f330ba080d0736a1317f382d1d30dda Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 Nov 2024 06:46:55 +0100 Subject: [PATCH 41/45] chore: remove direct exchange connection --- freqtrade/exchange/binance_public_data.py | 10 +++------- tests/exchange/test_binance_public_data.py | 20 ++++++++++++-------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index b6bce37da..e956300ee 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -39,9 +39,10 @@ async def download_archive_ohlcv( candle_type: CandleType, pair: str, timeframe: str, + *, since_ms: int, until_ms: int | None, - markets: dict[str, Any] | None = None, + markets: dict[str, Any], stop_on_404: bool = True, ) -> DataFrame: """ @@ -70,12 +71,7 @@ async def download_archive_ohlcv( else: raise ValueError(f"Unsupported CandleType: {candle_type}") - if markets: - symbol = markets[pair]["id"] - else: - binance = ccxt.binance() - binance.load_markets() - symbol = binance.markets[pair]["id"] + symbol = markets[pair]["id"] start = dt_from_ts(since_ms) end = dt_from_ts(until_ms) if until_ms else dt_now() diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py index 5f066d75e..16d803176 100644 --- a/tests/exchange/test_binance_public_data.py +++ b/tests/exchange/test_binance_public_data.py @@ -243,7 +243,13 @@ async def test_download_archive_ohlcv( markets = {"BTC/USDT": {"id": "BTCUSDT"}, "BTC/USDT:USDT": {"id": "BTCUSDT"}} df = await download_archive_ohlcv( - candle_type, pair, timeframe, since_ms, until_ms, markets, stop_on_404 + candle_type, + pair, + timeframe, + since_ms=since_ms, + until_ms=until_ms, + markets=markets, + stop_on_404=stop_on_404, ) if df.empty: @@ -254,23 +260,21 @@ async def test_download_archive_ohlcv( assert df["date"].iloc[-1] == last_date -async def test_download_archive_ohlcv_exc(mocker): +async def test_download_archive_ohlcv_exception(mocker): timeframe = "1h" pair = "BTC/USDT" since_ms = dt_ts(dt_utc(2020, 1, 1)) until_ms = dt_ts(dt_utc(2020, 1, 2)) + markets = {"BTC/USDT": {"id": "BTCUSDT"}, "BTC/USDT:USDT": {"id": "BTCUSDT"}} mocker.patch( "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get", side_effect=RuntimeError ) - mocker.patch("freqtrade.exchange.binance_public_data.ccxt.binance") - mocker.patch( - "freqtrade.exchange.binance_public_data.ccxt.binance.markets", - {"BTC/USDT": {"id": "BTCUSDT"}}, - ) - df = await download_archive_ohlcv(CandleType.SPOT, pair, timeframe, since_ms, until_ms) + df = await download_archive_ohlcv( + CandleType.SPOT, pair, timeframe, since_ms=since_ms, until_ms=until_ms, markets=markets + ) assert df.empty From 71cf7aedb5030d83232d1fea93ca8fa3cbd73085 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 Nov 2024 06:48:39 +0100 Subject: [PATCH 42/45] chore: remove unnecessary test-file that test is flaky for new pairs - and wouldn't run in production anyway. --- .../test_binance_compare_ohlcv.py | 102 ------------------ 1 file changed, 102 deletions(-) delete mode 100644 tests/exchange_online/test_binance_compare_ohlcv.py diff --git a/tests/exchange_online/test_binance_compare_ohlcv.py b/tests/exchange_online/test_binance_compare_ohlcv.py deleted file mode 100644 index a6335ef22..000000000 --- a/tests/exchange_online/test_binance_compare_ohlcv.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -Check if the earliest klines from rest API have its counterpart on https://data.binance.vision -Not expected to run in CI, manually run from shell: - - TEST_BINANCE_COMPARE_OHLCV=1 pytest tests/exchange_online/test_binance_compare_ohlcv.py - -Until 2024-10-30, there are three usdt-m futures symbols "lack" data -All SPOT symbols are good. - -BTCUSDT-1m 113 days -ARCHIVE: 2019-12-31 00:00:00 │ 2024-10-30 02:51:00 │ 2541772 -API: 2019-09-08 17:57:00 │ 2024-10-30 03:11:00 │ 2704874 - -ETHUSDT 34 days -ARCHIVE: 2019-12-31 00:00:00 │ 2020-02-29 23:59:00 │ 87840 -API: 2019-11-27 07:45:00 │ 2020-03-01 11:03:00 │ 136999 - -BCHUSDT 12 days -ARCHIVE: 2019-12-31 00:00:00 │ 2020-02-29 23:59:00 │ 87840 -API: 2019-12-19 08:57:00 │ 2020-03-01 06:55:00 │ 104999 -""" - -import asyncio -import os - -import aiohttp -import pytest - -from freqtrade.exchange.binance_public_data import binance_vision_zip_url -from freqtrade.util.datetime_helpers import dt_from_ts - - -class Check: - def __init__(self, asset_type_url_segment, timeframe): - self.asset_type_url_segment = asset_type_url_segment - self.timeframe = timeframe - self.klines_endpoint = "https://api.binance.com/api/v3/klines" - self.exchange_endpoint = "https://api.binance.com/api/v3/exchangeInfo" - self.mismatch = set() - - if asset_type_url_segment == "futures/um": - self.klines_endpoint = "https://fapi.binance.com/fapi/v1/klines" - self.exchange_endpoint = "https://fapi.binance.com/fapi/v1/exchangeInfo" - - async def check_one_symbol(self, symbol): - async with self.session.get( - self.klines_endpoint, params=dict(symbol=symbol, interval=self.timeframe, startTime=0) - ) as resp: - resp.raise_for_status() - json = await resp.json() - first_kline = json[0] - first_kline_ts = first_kline[0] - date = dt_from_ts(first_kline_ts).date() - - archive_url = binance_vision_zip_url( - self.asset_type_url_segment, symbol=symbol, timeframe=self.timeframe, date=date - ) - async with self.session.get( - archive_url, params=dict(symbol=symbol, interval=self.timeframe, startTime=0) - ) as resp: - if resp.status != 200: - self.mismatch.add(symbol) - print( - f"{resp.status} API first kline: {dt_from_ts(first_kline_ts).isoformat()} " - f"{archive_url}" - ) - web_url = archive_url.rsplit("/", 1)[0].replace( - "https://data.binance.vision/", "https://data.binance.vision/?prefix=" - ) - print(f"Check {web_url}") - - async def get_symbols(self): - async with self.session.get(self.exchange_endpoint) as resp: - resp.raise_for_status() - json = await resp.json() - symbols = [ - symbol["symbol"] - for symbol in json["symbols"] - if not symbol["status"] == "PENDING_TRADING" - ] - return symbols - - async def run(self) -> list: - async with aiohttp.ClientSession() as session: - self.session = session - symbols = await self.get_symbols() - await asyncio.gather(*[self.check_one_symbol(symbol) for symbol in symbols]) - return self.mismatch - - -@pytest.mark.skipif( - not bool(os.environ.get("TEST_BINANCE_COMPARE_OHLCV")), - reason="Simply to demonstrate the availabity of the archive endpoint", -) -async def test_binance_compare_ohlcv(): - futures_mismatch = await Check("futures/um", "1m").run() - assert futures_mismatch == set(["BTCUSDT", "ETHUSDT", "BCHUSDT"]) - - spot_mismatch = await Check("spot", "1m").run() - assert not spot_mismatch - - assert 0 From bd5877ad49633374d9242cfb5f7887c54d2d4205 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 Nov 2024 06:55:33 +0100 Subject: [PATCH 43/45] chore: add space in log message --- freqtrade/exchange/binance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 5264f644f..2ec73fa92 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -129,7 +129,7 @@ class Binance(Exchange): ) if until_ms and since_ms >= until_ms: logger.warning( - f"No available candle-data for {pair} before" + f"No available candle-data for {pair} before " f"{dt_from_ts(until_ms).isoformat()}" ) return DataFrame(columns=DEFAULT_DATAFRAME_COLUMNS) From 5f363f5c9d9207a15fb15bd6c732e653acce3ff6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 Nov 2024 07:00:42 +0100 Subject: [PATCH 44/45] chore: properly remove ccxt reference --- freqtrade/exchange/binance_public_data.py | 1 - tests/exchange/test_binance.py | 6 ------ 2 files changed, 7 deletions(-) diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index e956300ee..c7afe38cf 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -10,7 +10,6 @@ from io import BytesIO from typing import Any import aiohttp -import ccxt import pandas as pd from pandas import DataFrame diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index 7a95366a4..0cd4e3bfb 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -953,12 +953,6 @@ def test_get_historic_ohlcv_binance( since_ms = dt_ts(since) until_ms = dt_ts(until) - mocker.patch("freqtrade.exchange.binance_public_data.ccxt.binance") - mocker.patch( - "freqtrade.exchange.binance_public_data.ccxt.binance.markets", - {"BTC/USDT": {"id": "BTCUSDT"}}, - ) - df = exchange.get_historic_ohlcv(pair, timeframe, since_ms, candle_type, is_new_pair, until_ms) if df.empty: From 162c79029fb8369cd21cdaa783fcbbbb99639820 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 Nov 2024 07:12:54 +0100 Subject: [PATCH 45/45] chore: slightly reorganize test file layout --- tests/exchange/test_binance_public_data.py | 74 +++++++++++----------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py index 16d803176..6eb5a00bb 100644 --- a/tests/exchange/test_binance_public_data.py +++ b/tests/exchange/test_binance_public_data.py @@ -29,44 +29,9 @@ def event_loop_policy(request): return asyncio.DefaultEventLoopPolicy() -# spot klines archive csv file format, the futures/um klines don't have the header line -# -# open_time,open,high,low,close,volume,close_time,quote_volume,count,taker_buy_volume,taker_buy_quote_volume,ignore # noqa: E501 -# 1698364800000,34161.6,34182.5,33977.4,34024.2,409953,1698368399999,1202.97118037,15095,192220,564.12041453,0 # noqa: E501 -# 1698368400000,34024.2,34060.1,33776.4,33848.4,740960,1698371999999,2183.75671155,23938,368266,1085.17080793,0 # noqa: E501 -# 1698372000000,33848.5,34150.0,33815.1,34094.2,390376,1698375599999,1147.73267094,13854,231446,680.60405822,0 # noqa: E501 - - -def make_daily_df(date, timeframe): - start = dt_utc(date.year, date.month, date.day) - end = start + timedelta(days=1) - date_col = pd.date_range(start, end, freq=timeframe.replace("m", "min"), inclusive="left") - cols = ( - "open_time,open,high,low,close,volume,close_time,quote_volume,count,taker_buy_volume," - "taker_buy_quote_volume,ignore" - ) - df = pd.DataFrame(columns=cols.split(","), dtype=float) - df["open_time"] = date_col.astype("int64") // 10**6 - df["open"] = df["high"] = df["low"] = df["close"] = df["volume"] = 1.0 - return df - - -def make_daily_zip(asset_type_url_segment, symbol, timeframe, date) -> bytes: - df = make_daily_df(date, timeframe) - if asset_type_url_segment == "spot": - header = True - elif asset_type_url_segment == "futures/um": - header = None - else: - raise ValueError - csv = df.to_csv(index=False, header=header) - zip_buffer = io.BytesIO() - with zipfile.ZipFile(zip_buffer, "w") as zipf: - zipf.writestr(binance_vision_zip_name(symbol, timeframe, date), csv) - return zip_buffer.getvalue() - - class MockResponse: + """AioHTTP response mock""" + def __init__(self, content, status, reason=""): self._content = content self.status = status @@ -82,7 +47,42 @@ class MockResponse: return self +# spot klines archive csv file format, the futures/um klines don't have the header line +# +# open_time,open,high,low,close,volume,close_time,quote_volume,count,taker_buy_volume,taker_buy_quote_volume,ignore # noqa: E501 +# 1698364800000,34161.6,34182.5,33977.4,34024.2,409953,1698368399999,1202.97118037,15095,192220,564.12041453,0 # noqa: E501 +# 1698368400000,34024.2,34060.1,33776.4,33848.4,740960,1698371999999,2183.75671155,23938,368266,1085.17080793,0 # noqa: E501 +# 1698372000000,33848.5,34150.0,33815.1,34094.2,390376,1698375599999,1147.73267094,13854,231446,680.60405822,0 # noqa: E501 + + def make_response_from_url(start_date, end_date): + def make_daily_df(date, timeframe): + start = dt_utc(date.year, date.month, date.day) + end = start + timedelta(days=1) + date_col = pd.date_range(start, end, freq=timeframe.replace("m", "min"), inclusive="left") + cols = ( + "open_time,open,high,low,close,volume,close_time,quote_volume,count,taker_buy_volume," + "taker_buy_quote_volume,ignore" + ) + df = pd.DataFrame(columns=cols.split(","), dtype=float) + df["open_time"] = date_col.astype("int64") // 10**6 + df["open"] = df["high"] = df["low"] = df["close"] = df["volume"] = 1.0 + return df + + def make_daily_zip(asset_type_url_segment, symbol, timeframe, date) -> bytes: + df = make_daily_df(date, timeframe) + if asset_type_url_segment == "spot": + header = True + elif asset_type_url_segment == "futures/um": + header = None + else: + raise ValueError + csv = df.to_csv(index=False, header=header) + zip_buffer = io.BytesIO() + with zipfile.ZipFile(zip_buffer, "w") as zipf: + zipf.writestr(binance_vision_zip_name(symbol, timeframe, date), csv) + return zip_buffer.getvalue() + def make_response(url): pattern = ( r"https://data.binance.vision/data/(?Pspot|futures/um)"