diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 4c88e4c81..a01344364 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -110,12 +110,13 @@ class DataProvider: def orderbook(self, pair: str, maximum: int) -> Dict[str, List]: """ - fetch latest orderbook data + Fetch latest l2 orderbook data + Warning: Does a network request - so use with common sense. :param pair: pair to get the data for :param maximum: Maximum number of orderbook entries to query :return: dict including bids/asks with a total of `maximum` entries. """ - return self._exchange.get_order_book(pair, maximum) + return self._exchange.fetch_l2_order_book(pair, maximum) @property def runmode(self) -> RunMode: diff --git a/freqtrade/exceptions.py b/freqtrade/exceptions.py index 553a691ef..7cfed87e8 100644 --- a/freqtrade/exceptions.py +++ b/freqtrade/exceptions.py @@ -21,6 +21,14 @@ class DependencyException(FreqtradeException): """ +class PricingError(DependencyException): + """ + Subclass of DependencyException. + Indicates that the price could not be determined. + Implicitly a buy / sell operation. + """ + + class InvalidOrderException(FreqtradeException): """ This is returned when the order is not valid. Example: diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 37183dc2c..4279f392c 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -20,7 +20,7 @@ class Binance(Exchange): "trades_pagination_arg": "fromId", } - def get_order_book(self, pair: str, limit: int = 100) -> dict: + def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict: """ get order book level 2 from exchange @@ -30,7 +30,7 @@ class Binance(Exchange): # get next-higher step in the limit_range list limit = min(list(filter(lambda x: limit <= x, limit_range))) - return super().get_order_book(pair, limit) + return super().fetch_l2_order_book(pair, limit) def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool: """ diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 763c226fa..af745e8d0 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -998,7 +998,7 @@ class Exchange: raise OperationalException(e) from e @retrier - def get_order_book(self, pair: str, limit: int = 100) -> dict: + def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict: """ get order book level 2 from exchange diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d5d512e78..d4afa1d60 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -18,7 +18,7 @@ from freqtrade.configuration import validate_config_consistency from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge -from freqtrade.exceptions import DependencyException, InvalidOrderException +from freqtrade.exceptions import DependencyException, InvalidOrderException, PricingError from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date from freqtrade.misc import safe_value_fallback from freqtrade.pairlist.pairlistmanager import PairListManager @@ -260,12 +260,19 @@ class FreqtradeBot: f"Getting price from order book {bid_strategy['price_side'].capitalize()} side." ) order_book_top = bid_strategy.get('order_book_top', 1) - order_book = self.exchange.get_order_book(pair, order_book_top) + order_book = self.exchange.fetch_l2_order_book(pair, order_book_top) logger.debug('order_book %s', order_book) # top 1 = index 0 - order_book_rate = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0] - logger.info(f'...top {order_book_top} order book buy rate {order_book_rate:.8f}') - used_rate = order_book_rate + try: + rate_from_l2 = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0] + except (IndexError, KeyError) as e: + logger.warning( + "Buy Price from orderbook could not be determined." + f"Orderbook: {order_book}" + ) + raise PricingError from e + logger.info(f'...top {order_book_top} order book buy rate {rate_from_l2:.8f}') + used_rate = rate_from_l2 else: logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price") ticker = self.exchange.fetch_ticker(pair) @@ -446,7 +453,7 @@ class FreqtradeBot: """ conf_bids_to_ask_delta = conf.get('bids_to_ask_delta', 0) logger.info(f"Checking depth of market for {pair} ...") - order_book = self.exchange.get_order_book(pair, 1000) + order_book = self.exchange.fetch_l2_order_book(pair, 1000) order_book_data_frame = order_book_to_dataframe(order_book['bids'], order_book['asks']) order_book_bids = order_book_data_frame['b_size'].sum() order_book_asks = order_book_data_frame['a_size'].sum() @@ -635,7 +642,7 @@ class FreqtradeBot: """ Helper generator to query orderbook in loop (used for early sell-order placing) """ - order_book = self.exchange.get_order_book(pair, order_book_max) + order_book = self.exchange.fetch_l2_order_book(pair, order_book_max) for i in range(order_book_min, order_book_max + 1): yield order_book[side][i - 1][0] @@ -662,8 +669,11 @@ class FreqtradeBot: logger.info( f"Getting price from order book {ask_strategy['price_side'].capitalize()} side." ) - rate = next(self._order_book_gen(pair, f"{ask_strategy['price_side']}s")) - + try: + rate = next(self._order_book_gen(pair, f"{ask_strategy['price_side']}s")) + except (IndexError, KeyError) as e: + logger.warning("Sell Price at location from orderbook could not be determined.") + raise PricingError from e else: rate = self.exchange.fetch_ticker(pair)[ask_strategy['price_side']] self._sell_rate_cache[pair] = rate @@ -690,16 +700,23 @@ class FreqtradeBot: self.dataprovider.ohlcv(trade.pair, self.strategy.ticker_interval)) if config_ask_strategy.get('use_order_book', False): - logger.debug(f'Using order book for selling {trade.pair}...') # logger.debug('Order book %s',orderBook) order_book_min = config_ask_strategy.get('order_book_min', 1) order_book_max = config_ask_strategy.get('order_book_max', 1) + logger.info(f'Using order book between {order_book_min} and {order_book_max} ' + f'for selling {trade.pair}...') order_book = self._order_book_gen(trade.pair, f"{config_ask_strategy['price_side']}s", order_book_min=order_book_min, order_book_max=order_book_max) for i in range(order_book_min, order_book_max + 1): - sell_rate = next(order_book) + try: + sell_rate = next(order_book) + except (IndexError, KeyError) as e: + logger.warning( + f"Sell Price at location {i} from orderbook could not be determined." + ) + raise PricingError from e logger.debug(f" order book {config_ask_strategy['price_side']} top {i}: " f"{sell_rate:0.8f}") diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 7b1e9ddaa..e40f691a8 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1413,13 +1413,13 @@ def test_refresh_latest_ohlcv_inv_result(default_conf, mocker, caplog): @pytest.mark.parametrize("exchange_name", EXCHANGES) -def test_get_order_book(default_conf, mocker, order_book_l2, exchange_name): +def test_fetch_l2_order_book(default_conf, mocker, order_book_l2, exchange_name): default_conf['exchange']['name'] = exchange_name api_mock = MagicMock() api_mock.fetch_l2_order_book = order_book_l2 exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - order_book = exchange.get_order_book(pair='ETH/BTC', limit=10) + order_book = exchange.fetch_l2_order_book(pair='ETH/BTC', limit=10) assert 'bids' in order_book assert 'asks' in order_book assert len(order_book['bids']) == 10 @@ -1427,20 +1427,20 @@ def test_get_order_book(default_conf, mocker, order_book_l2, exchange_name): @pytest.mark.parametrize("exchange_name", EXCHANGES) -def test_get_order_book_exception(default_conf, mocker, exchange_name): +def test_fetch_l2_order_book_exception(default_conf, mocker, exchange_name): api_mock = MagicMock() with pytest.raises(OperationalException): api_mock.fetch_l2_order_book = MagicMock(side_effect=ccxt.NotSupported("Not supported")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - exchange.get_order_book(pair='ETH/BTC', limit=50) + exchange.fetch_l2_order_book(pair='ETH/BTC', limit=50) with pytest.raises(TemporaryError): api_mock.fetch_l2_order_book = MagicMock(side_effect=ccxt.NetworkError("DeadBeef")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - exchange.get_order_book(pair='ETH/BTC', limit=50) + exchange.fetch_l2_order_book(pair='ETH/BTC', limit=50) with pytest.raises(OperationalException): api_mock.fetch_l2_order_book = MagicMock(side_effect=ccxt.BaseError("DeadBeef")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - exchange.get_order_book(pair='ETH/BTC', limit=50) + exchange.fetch_l2_order_book(pair='ETH/BTC', limit=50) def make_fetch_ohlcv_mock(data): diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 9d9d133cc..5e951b585 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -11,18 +11,21 @@ import arrow import pytest import requests -from freqtrade.constants import MATH_CLOSE_PREC, UNLIMITED_STAKE_AMOUNT, CANCEL_REASON +from freqtrade.constants import (CANCEL_REASON, MATH_CLOSE_PREC, + UNLIMITED_STAKE_AMOUNT) from freqtrade.exceptions import (DependencyException, InvalidOrderException, - OperationalException, TemporaryError) + OperationalException, PricingError, + TemporaryError) from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc import RPCMessageType from freqtrade.state import RunMode, State from freqtrade.strategy.interface import SellCheckTuple, SellType from freqtrade.worker import Worker -from tests.conftest import (get_patched_freqtradebot, get_patched_worker, - log_has, log_has_re, patch_edge, patch_exchange, - patch_get_signal, patch_wallet, patch_whitelist, create_mock_trades) +from tests.conftest import (create_mock_trades, get_patched_freqtradebot, + get_patched_worker, log_has, log_has_re, + patch_edge, patch_exchange, patch_get_signal, + patch_wallet, patch_whitelist) def patch_RPCManager(mocker) -> MagicMock: @@ -3695,7 +3698,7 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 0.1 patch_RPCManager(mocker) patch_exchange(mocker) - mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) + mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, @@ -3732,7 +3735,7 @@ def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_o default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 100 patch_RPCManager(mocker) patch_exchange(mocker) - mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) + mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, @@ -3757,7 +3760,7 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: ticker_mock = MagicMock(return_value={'ask': 0.045, 'last': 0.046}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_order_book=order_book_l2, + fetch_l2_order_book=order_book_l2, fetch_ticker=ticker_mock, ) @@ -3772,29 +3775,26 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: assert ticker_mock.call_count == 0 -def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2) -> None: - """ - test if function get_buy_rate will return the ask rate (since its value is lower) - instead of the order book rate (even if enabled) - """ +def test_order_book_bid_strategy_exception(mocker, default_conf, caplog) -> None: patch_exchange(mocker) ticker_mock = MagicMock(return_value={'ask': 0.042, 'last': 0.046}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_order_book=order_book_l2, + fetch_l2_order_book=MagicMock(return_value={'bids': [[]], 'asks': [[]]}), fetch_ticker=ticker_mock, ) default_conf['exchange']['name'] = 'binance' default_conf['bid_strategy']['use_order_book'] = True - default_conf['bid_strategy']['order_book_top'] = 2 + default_conf['bid_strategy']['order_book_top'] = 1 default_conf['bid_strategy']['ask_last_balance'] = 0 default_conf['telegram']['enabled'] = False freqtrade = FreqtradeBot(default_conf) # orderbook shall be used even if tickers would be lower. - assert freqtrade.get_buy_rate('ETH/BTC', True) != 0.042 - assert ticker_mock.call_count == 0 + with pytest.raises(PricingError): + freqtrade.get_buy_rate('ETH/BTC', refresh=True) + assert log_has_re(r'Buy Price from orderbook could not be determined.', caplog) def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None: @@ -3804,7 +3804,7 @@ def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None: patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_order_book=order_book_l2 + fetch_l2_order_book=order_book_l2 ) default_conf['telegram']['enabled'] = False default_conf['exchange']['name'] = 'binance' @@ -3818,11 +3818,11 @@ def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None: def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order, - fee, mocker, order_book_l2) -> None: + fee, mocker, order_book_l2, caplog) -> None: """ test order book ask strategy """ - mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) + mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) default_conf['exchange']['name'] = 'binance' default_conf['ask_strategy']['use_order_book'] = True default_conf['ask_strategy']['order_book_min'] = 1 @@ -3856,6 +3856,13 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True + assert trade.close_rate_requested == order_book_l2.return_value['asks'][0][0] + + mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', + return_value={'bids': [[]], 'asks': [[]]}) + with pytest.raises(PricingError): + freqtrade.handle_trade(trade) + assert log_has('Sell Price at location 1 from orderbook could not be determined.', caplog) @pytest.mark.parametrize('side,ask,bid,expected', [ @@ -3896,9 +3903,8 @@ def test_get_sell_rate_orderbook(default_conf, mocker, caplog, side, expected, o default_conf['ask_strategy']['use_order_book'] = True default_conf['ask_strategy']['order_book_min'] = 1 default_conf['ask_strategy']['order_book_max'] = 2 - # TODO: min/max is irrelevant for this test until refactoring pair = "ETH/BTC" - mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) + mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) ft = get_patched_freqtradebot(mocker, default_conf) rate = ft.get_sell_rate(pair, True) assert not log_has("Using cached sell rate for ETH/BTC.", caplog) @@ -3909,6 +3915,22 @@ def test_get_sell_rate_orderbook(default_conf, mocker, caplog, side, expected, o assert log_has("Using cached sell rate for ETH/BTC.", caplog) +def test_get_sell_rate_orderbook_exception(default_conf, mocker, caplog): + # Test orderbook mode + default_conf['ask_strategy']['price_side'] = 'ask' + default_conf['ask_strategy']['use_order_book'] = True + default_conf['ask_strategy']['order_book_min'] = 1 + default_conf['ask_strategy']['order_book_max'] = 2 + pair = "ETH/BTC" + # Test What happens if the exchange returns an empty orderbook. + mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', + return_value={'bids': [[]], 'asks': [[]]}) + ft = get_patched_freqtradebot(mocker, default_conf) + with pytest.raises(PricingError): + ft.get_sell_rate(pair, True) + assert log_has("Sell Price at location from orderbook could not be determined.", caplog) + + def test_startup_state(default_conf, mocker): default_conf['pairlist'] = {'method': 'VolumePairList', 'config': {'number_assets': 20}