diff --git a/.travis.yml b/.travis.yml index 88121945f..981eedcf8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,12 +13,12 @@ addons: install: - ./install_ta-lib.sh - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH -- pip install --upgrade flake8 coveralls pytest-random-order mypy +- pip install --upgrade flake8 coveralls pytest-random-order pytest-asyncio mypy - pip install -r requirements.txt - pip install -e . jobs: include: - - script: + - script: - pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ - coveralls - script: diff --git a/config.json.example b/config.json.example index 8bd3942e6..7a0bb6b9b 100644 --- a/config.json.example +++ b/config.json.example @@ -11,7 +11,18 @@ "sell": 30 }, "bid_strategy": { - "ask_last_balance": 0.0 + "ask_last_balance": 0.0, + "use_order_book": false, + "order_book_top": 1, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + } + }, + "ask_strategy":{ + "use_order_book": false, + "order_book_min": 1, + "order_book_max": 9 }, "exchange": { "name": "bittrex", diff --git a/config_full.json.example b/config_full.json.example index a06eeb7e6..7083bada6 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -20,7 +20,18 @@ "sell": 30 }, "bid_strategy": { - "ask_last_balance": 0.0 + "ask_last_balance": 0.0, + "use_order_book": false, + "order_book_top": 1, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + } + }, + "ask_strategy":{ + "use_order_book": false, + "order_book_min": 1, + "order_book_max": 9 }, "exchange": { "name": "bittrex", diff --git a/docs/configuration.md b/docs/configuration.md index ff5ce118c..010e693d4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -22,20 +22,29 @@ The table below will list all configuration parameters. | `stake_amount` | 0.05 | Yes | Amount of crypto-currency your bot will use for each trade. Per default, the bot will use (0.05 BTC x 3) = 0.15 BTC in total will be always engaged. Set it to 'unlimited' to allow the bot to use all avaliable balance. | `ticker_interval` | [1m, 5m, 30m, 1h, 1d] | No | The ticker interval to use (1min, 5 min, 30 min, 1 hour or 1 day). Default is 5 minutes | `fiat_display_currency` | USD | Yes | Fiat currency used to show your profits. More information below. -| `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. +| `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. +| `process_only_new_candles` | false | No | If set to true indicators are processed only once a new candle arrives. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. Can be set either in Configuration or in the strategy. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. -| `trailing_stoploss` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). -| `trailing_stoploss_positve` | 0 | No | Changes stop-loss once profit has been reached. -| `trailing_stoploss_positve_offset` | 0 | No | Offset on when to apply `trailing_stoploss_positive`. Percentage value which should be positive. +| `trailing_stop` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). +| `trailing_stop_positve` | 0 | No | Changes stop-loss once profit has been reached. +| `trailing_stop_positve_offset` | 0 | No | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. | `unfilledtimeout.buy` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled. | `unfilledtimeout.sell` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled. | `bid_strategy.ask_last_balance` | 0.0 | Yes | Set the bidding price. More information below. +| `bid_strategy.use_order_book` | false | No | Allows buying of pair using the rates in Order Book Bids. +| `bid_strategy.order_book_top` | 0 | No | Bot will use the top N rate in Order Book Bids. Ie. a value of 2 will allow the bot to pick the 2nd bid rate in Order Book Bids. +| `bid_strategy.check_depth_of_market.enabled` | false | No | Does not buy if the % difference of buy orders and sell orders is met in Order Book. +| `bid_strategy.check_depth_of_market.bids_to_ask_delta` | 0 | No | The % difference of buy orders and sell orders found in Order Book. A value lesser than 1 means sell orders is greater, while value greater than 1 means buy orders is higher. +| `ask_strategy.use_order_book` | false | No | Allows selling of open traded pair using the rates in Order Book Asks. +| `ask_strategy.order_book_min` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. +| `ask_strategy.order_book_max` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode. | `exchange.secret` | secret | No | API secret to use for the exchange. Only required when you are in production mode. | `exchange.pair_whitelist` | [] | No | List of currency to use by the bot. Can be overrided with `--dynamic-whitelist` param. | `exchange.pair_blacklist` | [] | No | List of currency the bot must avoid. Useful when using `--dynamic-whitelist` param. +| `exchange.ccxt_rate_limit` | True | No | Have CCXT handle Exchange rate limits. Depending on the exchange, having this to false can lead to temporary bans from the exchange. | `experimental.use_sell_signal` | false | No | Use your sell strategy in addition of the `minimal_roi`. | `experimental.sell_profit_only` | false | No | waits until you have made a positive profit before taking a sell decision. | `experimental.ignore_roi_if_buy_signal` | false | No | Does not sell if the buy-signal is still active. Takes preference over `minimal_roi` and `use_sell_signal` diff --git a/docs/hyperopt.md b/docs/hyperopt.md index f4b69b632..3f568d82e 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -5,6 +5,8 @@ algorithms included in the `scikit-optimize` package to accomplish this. The search will burn all your CPU cores, make your laptop sound like a fighter jet and still take a long time. +*Note:* Hyperopt will crash when used with only 1 CPU Core as found out in [Issue #1133](https://github.com/freqtrade/freqtrade/issues/1133) + ## Table of Contents - [Prepare your Hyperopt](#prepare-hyperopt) - [Configure your Guards and Triggers](#configure-your-guards-and-triggers) diff --git a/docs/installation.md b/docs/installation.md index 4de05c121..0fecfcf78 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -8,7 +8,6 @@ To understand how to set up the bot please read the [Bot Configuration](https:// * [Table of Contents](#table-of-contents) * [Easy Installation - Linux Script](#easy-installation---linux-script) -* [Manual installation](#manual-installation) * [Automatic Installation - Docker](#automatic-installation---docker) * [Custom Linux MacOS Installation](#custom-installation) - [Requirements](#requirements) @@ -56,34 +55,6 @@ Reset parameter will hard reset your branch (only if you are on `master` or `dev Config parameter is a `config.json` configurator. This script will ask you questions to setup your bot and create your `config.json`. -## Manual installation - Linux/MacOS - -The following steps are made for Linux/MacOS environment - -### 1. Clone the repo - -```bash -git clone git@github.com:freqtrade/freqtrade.git -git checkout develop -cd freqtrade -``` - -### 2. Create the config file - -Switch `"dry_run": true,` - -```bash -cp config.json.example config.json -vi config.json -``` - -### 3. Build your docker image and run it - -```bash -docker build -t freqtrade . -docker run --rm -v /etc/localtime:/etc/localtime:ro -v `pwd`/config.json:/freqtrade/config.json -it freqtrade -``` - ------ ## Automatic Installation - Docker @@ -196,7 +167,7 @@ docker run -d \ freqtrade --db-url sqlite:///tradesv3.sqlite ``` -NOTE: db-url defaults to `sqlite:///tradesv3.sqlite` but it defaults to `sqlite://` if `dry_run=True` is being used. +*Note*: db-url defaults to `sqlite:///tradesv3.sqlite` but it defaults to `sqlite://` if `dry_run=True` is being used. To override this behaviour use a custom db-url value: i.e.: `--db-url sqlite:///tradesv3.dryrun.sqlite` ### 6. Monitor your Docker instance @@ -211,14 +182,15 @@ docker stop freqtrade docker start freqtrade ``` -You do not need to rebuild the image for configuration changes, it will suffice to edit `config.json` and restart the container. +For more information on how to operate Docker, please refer to the [official Docker documentation](https://docs.docker.com/). + +*Note*: You do not need to rebuild the image for configuration changes, it will suffice to edit `config.json` and restart the container. ### 7. Backtest with docker The following assumes that the above steps (1-4) have been completed successfully. Also, backtest-data should be available at `~/.freqtrade/user_data/`. - ``` bash docker run -d \ --name freqtrade \ @@ -238,12 +210,13 @@ Head over to the [Backtesting Documentation](https://github.com/freqtrade/freqtr ## Custom Installation We've included/collected install instructions for Ubuntu 16.04, MacOS, and Windows. These are guidelines and your success may vary with other distros. +OS Specific steps are listed first, the [common](#common) section below is necessary for all systems. ### Requirements Click each one for install guide: -* [Python 3.6.x](http://docs.python-guide.org/en/latest/starting/installation/), note the bot was not tested on Python >= 3.7.x +* [Python >= 3.6.x](http://docs.python-guide.org/en/latest/starting/installation/) * [pip](https://pip.pypa.io/en/stable/installing/) * [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) * [virtualenv](https://virtualenv.pypa.io/en/stable/installation/) (Recommended) @@ -251,7 +224,7 @@ Click each one for install guide: ### Linux - Ubuntu 16.04 -#### 1. Install Python 3.6, Git, and wget +#### Install Python 3.6, Git, and wget ```bash sudo add-apt-repository ppa:jonathonf/python-3.6 @@ -259,7 +232,34 @@ sudo apt-get update sudo apt-get install python3.6 python3.6-venv python3.6-dev build-essential autoconf libtool pkg-config make wget git ``` -#### 2. Install TA-Lib +#### Raspberry Pi / Raspbian + +Before installing FreqTrade on a Raspberry Pi running the official Raspbian Image, make sure you have at least Python 3.6 installed. The default image only provides Python 3.5. Probably the easiest way to get a recent version of python is [miniconda](https://repo.continuum.io/miniconda/). + +The following assumes that miniconda3 is installed and available in your environment, and is installed. +It's recommended to use (mini)conda for this as installation/compilation of `scipy` and `pandas` takes a long time. + +``` bash +conda config --add channels rpi +conda install python=3.6 +conda create -n freqtrade python=3.6 +conda install scipy pandas + +pip install -r requirements.txt +pip install -e . +``` + +### MacOS + +#### Install Python 3.6, git, wget and ta-lib + +```bash +brew install python3 git wget +``` + +### common + +#### 1. Install TA-Lib Official webpage: https://mrjbq7.github.io/ta-lib/install.html @@ -275,15 +275,60 @@ cd .. rm -rf ./ta-lib* ``` +*Note*: An already downloaded version of ta-lib is included in the repository, as the sourceforge.net source seems to have problems frequently. + +#### 2. Setup your Python virtual environment (virtualenv) + +*Note*: This step is optional but strongly recommended to keep your system organized + +```bash +python3 -m venv .env +source .env/bin/activate +``` + #### 3. Install FreqTrade Clone the git repository: ```bash git clone https://github.com/freqtrade/freqtrade.git + ``` -#### 4. Configure `freqtrade` as a `systemd` service +Optionally checkout the stable/master branch: + +```bash +git checkout master +``` + +#### 4. Initialize the configuration + +```bash +cd freqtrade +cp config.json.example config.json +``` + +> *To edit the config please refer to [Bot Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md).* + +#### 5. Install python dependencies + +``` bash +pip3 install --upgrade pip +pip3 install -r requirements.txt +pip3 install -e . +``` + +#### 6. Run the Bot + +If this is the first time you run the bot, ensure you are running it in Dry-run `"dry_run": true,` otherwise it will start to buy and sell coins. + +```bash +python3.6 ./freqtrade/main.py -c config.json +``` + +*Note*: If you run the bot on a server, you should consider using [Docker](#automatic-installation---docker) a terminal multiplexer like `screen` or [`tmux`](https://en.wikipedia.org/wiki/Tmux) to avoid that the bot is stopped on logout. + +#### 7. [Optional] Configure `freqtrade` as a `systemd` service From the freqtrade repo... copy `freqtrade.service` to your systemd user directory (usually `~/.config/systemd/user`) and update `WorkingDirectory` and `ExecStart` to match your setup. @@ -299,57 +344,6 @@ For this to be persistent (run when user is logged out) you'll need to enable `l sudo loginctl enable-linger "$USER" ``` -### MacOS - -#### 1. Install Python 3.6, git, wget and ta-lib - -```bash -brew install python3 git wget ta-lib -``` - -#### 2. Install FreqTrade - -Clone the git repository: - -```bash -git clone https://github.com/freqtrade/freqtrade.git -``` - -Optionally checkout the develop branch: - -```bash -git checkout develop -``` - -### Setup Config and virtual env - -#### 1. Initialize the configuration - -```bash -cd freqtrade -cp config.json.example config.json -``` - -> *To edit the config please refer to [Bot Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md).* - -#### 2. Setup your Python virtual environment (virtualenv) - -```bash -python3.6 -m venv .env -source .env/bin/activate -pip3.6 install --upgrade pip -pip3.6 install -r requirements.txt -pip3.6 install -e . -``` - -#### 3. Run the Bot - -If this is the first time you run the bot, ensure you are running it in Dry-run `"dry_run": true,` otherwise it will start to buy and sell coins. - -```bash -python3.6 ./freqtrade/main.py -c config.json -``` - ------ ## Windows @@ -369,7 +363,7 @@ git clone https://github.com/freqtrade/freqtrade.git copy paste `config.json` to ``\path\freqtrade-develop\freqtrade` -#### install ta-lib +#### Install ta-lib Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows). @@ -390,5 +384,17 @@ REM >pip install TA_Lib‑0.4.17‑cp36‑cp36m‑win32.whl > Thanks [Owdr](https://github.com/Owdr) for the commands. Source: [Issue #222](https://github.com/freqtrade/freqtrade/issues/222) +#### Error during installation under Windows + +``` bash +error: Microsoft Visual C++ 14.0 is required. Get it with "Microsoft Visual C++ Build Tools": http://landinghub.visualstudio.com/visual-cpp-build-tools +``` + +Unfortunately, many packages requiring compilation don't provide a pre-build wheel. It is therefore mandatory to have a C/C++ compiler installed and available for your python environment to use. + +The easiest way is to download install Microsoft Visual Studio Community [here](https://visualstudio.microsoft.com/downloads/) and make sure to install "Common Tools for Visual C++" to enable building c code on Windows. Unfortunately, this is a heavy download / dependency (~4Gb) so you might want to consider WSL or docker first. + +--- + Now you have an environment ready, the next step is [Bot Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md)... diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 175d09405..eadfa6eba 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -53,6 +53,7 @@ CONF_SCHEMA = { }, 'fiat_display_currency': {'type': 'string', 'enum': SUPPORTED_FIAT}, 'dry_run': {'type': 'boolean'}, + 'process_only_new_candles': {'type': 'boolean'}, 'minimal_roi': { 'type': 'object', 'patternProperties': { @@ -78,18 +79,35 @@ CONF_SCHEMA = { 'type': 'number', 'minimum': 0, 'maximum': 1, - 'exclusiveMaximum': False + 'exclusiveMaximum': False, + 'use_order_book': {'type': 'boolean'}, + 'order_book_top': {'type': 'number', 'maximum': 20, 'minimum': 1}, + 'check_depth_of_market': { + 'type': 'object', + 'properties': { + 'enabled': {'type': 'boolean'}, + 'bids_to_ask_delta': {'type': 'number', 'minimum': 0}, + } + }, }, }, 'required': ['ask_last_balance'] }, + 'ask_strategy': { + 'type': 'object', + 'properties': { + 'use_order_book': {'type': 'boolean'}, + 'order_book_min': {'type': 'number', 'minimum': 1}, + 'order_book_max': {'type': 'number', 'minimum': 1, 'maximum': 50} + } + }, 'exchange': {'$ref': '#/definitions/exchange'}, 'experimental': { 'type': 'object', 'properties': { 'use_sell_signal': {'type': 'boolean'}, 'sell_profit_only': {'type': 'boolean'}, - "ignore_roi_if_buy_signal_true": {'type': 'boolean'} + 'ignore_roi_if_buy_signal_true': {'type': 'boolean'} } }, 'telegram': { diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 1a38229ad..f663420e0 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -1,12 +1,15 @@ # pragma pylint: disable=W0603 """ Cryptocurrency Exchanges support """ import logging +import inspect from random import randint -from typing import List, Dict, Any, Optional +from typing import List, Dict, Tuple, Any, Optional from datetime import datetime from math import floor, ceil +import asyncio import ccxt +import ccxt.async_support as ccxt_async import arrow from freqtrade import constants, OperationalException, DependencyException, TemporaryError @@ -23,6 +26,24 @@ _EXCHANGE_URLS = { } +def retrier_async(f): + async def wrapper(*args, **kwargs): + count = kwargs.pop('count', API_RETRY_COUNT) + try: + return await f(*args, **kwargs) + except (TemporaryError, DependencyException) as ex: + logger.warning('%s() returned exception: "%s"', f.__name__, ex) + if count > 0: + count -= 1 + kwargs.update({'count': count}) + logger.warning('retrying %s() still for %s times', f.__name__, count) + return await wrapper(*args, **kwargs) + else: + logger.warning('Giving up retrying: %s()', f.__name__) + raise ex + return wrapper + + def retrier(f): def wrapper(*args, **kwargs): count = kwargs.pop('count', API_RETRY_COUNT) @@ -45,8 +66,8 @@ class Exchange(object): # Current selected exchange _api: ccxt.Exchange = None + _api_async: ccxt_async.Exchange = None _conf: Dict = {} - _cached_ticker: Dict[str, Any] = {} # Holds all open sell orders for dry_run _dry_run_open_orders: Dict[str, Any] = {} @@ -60,11 +81,20 @@ class Exchange(object): """ self._conf.update(config) + self._cached_ticker: Dict[str, Any] = {} + + # Holds last candle refreshed time of each pair + self._pairs_last_refresh_time: Dict[str, int] = {} + + # Holds candles + self.klines: Dict[str, Any] = {} + if config['dry_run']: logger.info('Instance is running with dry_run enabled') exchange_config = config['exchange'] self._api = self._init_ccxt(exchange_config) + self._api_async = self._init_ccxt(exchange_config, ccxt_async) logger.info('Using Exchange "%s"', self.name) @@ -75,7 +105,15 @@ class Exchange(object): # Check if timeframe is available self.validate_timeframes(config['ticker_interval']) - def _init_ccxt(self, exchange_config: dict) -> ccxt.Exchange: + def __del__(self): + """ + Destructor - clean up async stuff + """ + logger.debug("Exchange object destroyed, closing async loop") + if self._api_async and inspect.iscoroutinefunction(self._api_async.close): + asyncio.get_event_loop().run_until_complete(self._api_async.close()) + + def _init_ccxt(self, exchange_config: dict, ccxt_module=ccxt) -> ccxt.Exchange: """ Initialize ccxt with given config and return valid ccxt instance. @@ -83,15 +121,15 @@ class Exchange(object): # Find matching class for the given exchange name name = exchange_config['name'] - if name not in ccxt.exchanges: + if name not in ccxt_module.exchanges: raise OperationalException(f'Exchange {name} is not supported') try: - api = getattr(ccxt, name.lower())({ + api = getattr(ccxt_module, name.lower())({ 'apiKey': exchange_config.get('key'), 'secret': exchange_config.get('secret'), 'password': exchange_config.get('password'), 'uid': exchange_config.get('uid', ''), - 'enableRateLimit': exchange_config.get('ccxt_rate_limit', True), + 'enableRateLimit': exchange_config.get('ccxt_rate_limit', True) }) except (KeyError, AttributeError): raise OperationalException(f'Exchange {name} is not supported') @@ -116,10 +154,19 @@ class Exchange(object): api.urls['api'] = api.urls['test'] logger.info("Enabled Sandbox API on %s", name) else: - logger.warning(self._api.name, "No Sandbox URL in CCXT, exiting. " - "Please check your config.json") + logger.warning(name, "No Sandbox URL in CCXT, exiting. " + "Please check your config.json") raise OperationalException(f'Exchange {name} does not provide a sandbox api') + def _load_async_markets(self) -> None: + try: + if self._api_async: + asyncio.get_event_loop().run_until_complete(self._api_async.load_markets()) + + except ccxt.BaseError as e: + logger.warning('Could not load async markets. Reason: %s', e) + return + def validate_pairs(self, pairs: List[str]) -> None: """ Checks if all given pairs are tradable on the current exchange. @@ -130,6 +177,7 @@ class Exchange(object): try: markets = self._api.load_markets() + self._load_async_markets() except ccxt.BaseError as e: logger.warning('Unable to validate pairs (assuming they are correct). Reason: %s', e) return @@ -329,6 +377,102 @@ class Exchange(object): logger.info("returning cached ticker-data for %s", pair) return self._cached_ticker[pair] + def get_history(self, pair: str, tick_interval: str, + since_ms: int) -> List: + """ + Gets candle history using asyncio and returns the list of candles. + Handles all async doing. + """ + return asyncio.get_event_loop().run_until_complete( + self._async_get_history(pair=pair, tick_interval=tick_interval, + since_ms=since_ms)) + + async def _async_get_history(self, pair: str, + tick_interval: str, + since_ms: int) -> List: + # Assume exchange returns 500 candles + _LIMIT = 500 + + one_call = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 * _LIMIT * 1000 + logger.debug("one_call: %s", one_call) + input_coroutines = [self._async_get_candle_history( + pair, tick_interval, since) for since in + range(since_ms, arrow.utcnow().timestamp * 1000, one_call)] + tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) + + # Combine tickers + data: List = [] + for tick in tickers: + if tick[0] == pair: + data.extend(tick[1]) + # Sort data again after extending the result - above calls return in "async order" order + data = sorted(data, key=lambda x: x[0]) + logger.info("downloaded %s with length %s.", pair, len(data)) + return data + + def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> None: + """ + Refresh tickers asyncronously and return the result. + """ + logger.debug("Refreshing klines for %d pairs", len(pair_list)) + asyncio.get_event_loop().run_until_complete( + self.async_get_candles_history(pair_list, ticker_interval)) + + async def async_get_candles_history(self, pairs: List[str], + tick_interval: str) -> List[Tuple[str, List]]: + """Download ohlcv history for pair-list asyncronously """ + input_coroutines = [self._async_get_candle_history( + symbol, tick_interval) for symbol in pairs] + tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) + return tickers + + @retrier_async + async def _async_get_candle_history(self, pair: str, tick_interval: str, + since_ms: Optional[int] = None) -> Tuple[str, List]: + try: + # fetch ohlcv asynchronously + logger.debug("fetching %s since %s ...", pair, since_ms) + + # Calculating ticker interval in second + interval_in_sec = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 + + # If (last update time) + (interval in second) is greater or equal than now + # that means we don't have to hit the API as there is no new candle + # so we fetch it from local cache + if (not since_ms and + self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >= + arrow.utcnow().timestamp): + data = self.klines[pair] + logger.debug("Using cached klines data for %s ...", pair) + else: + data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, + since=since_ms) + + # Because some exchange sort Tickers ASC and other DESC. + # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) + # when GDAX returns a list of tickers DESC (newest first, oldest last) + data = sorted(data, key=lambda x: x[0]) + + # keeping last candle time as last refreshed time of the pair + if data: + self._pairs_last_refresh_time[pair] = data[-1][0] // 1000 + + # keeping candles in cache + self.klines[pair] = data + + logger.debug("done fetching %s ...", pair) + return pair, data + + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching historical candlestick data.' + f'Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not load ticker history due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(f'Could not fetch ticker data. Msg: {e}') + @retrier def get_candle_history(self, pair: str, tick_interval: str, since_ms: Optional[int] = None) -> List[Dict]: @@ -409,6 +553,37 @@ class Exchange(object): except ccxt.BaseError as e: raise OperationalException(e) + @retrier + def get_order_book(self, pair: str, limit: int = 100) -> dict: + """ + get order book level 2 from exchange + + Notes: + 20180619: bittrex doesnt support limits -.- + 20180619: binance support limits but only on specific range + """ + try: + if self._api.name == 'Binance': + limit_range = [5, 10, 20, 50, 100, 500, 1000] + # get next-higher step in the limit_range list + limit = min(list(filter(lambda x: limit <= x, limit_range))) + # above script works like loop below (but with slightly better performance): + # for limitx in limit_range: + # if limit <= limitx: + # limit = limitx + # break + + return self._api.fetch_l2_order_book(pair, limit) + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching order book.' + f'Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get order book due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + @retrier def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: if self._conf['dry_run']: diff --git a/freqtrade/exchange/exchange_helpers.py b/freqtrade/exchange/exchange_helpers.py index 46f04328c..8f4b03daf 100644 --- a/freqtrade/exchange/exchange_helpers.py +++ b/freqtrade/exchange/exchange_helpers.py @@ -2,6 +2,7 @@ Functions to analyze ticker data with indicators and produce buy and sell signals """ import logging +import pandas as pd from pandas import DataFrame, to_datetime logger = logging.getLogger(__name__) @@ -31,3 +32,27 @@ def parse_ticker_dataframe(ticker: list) -> DataFrame: }) frame.drop(frame.tail(1).index, inplace=True) # eliminate partial candle return frame + + +def order_book_to_dataframe(bids: list, asks: list) -> DataFrame: + """ + Gets order book list, returns dataframe with below format per suggested by creslin + ------------------------------------------------------------------- + b_sum b_size bids asks a_size a_sum + ------------------------------------------------------------------- + """ + cols = ['bids', 'b_size'] + + bids_frame = DataFrame(bids, columns=cols) + # add cumulative sum column + bids_frame['b_sum'] = bids_frame['b_size'].cumsum() + cols2 = ['asks', 'a_size'] + asks_frame = DataFrame(asks, columns=cols2) + # add cumulative sum column + asks_frame['a_sum'] = asks_frame['a_size'].cumsum() + + frame = pd.concat([bids_frame['b_sum'], bids_frame['b_size'], bids_frame['bids'], + asks_frame['asks'], asks_frame['a_size'], asks_frame['a_sum']], axis=1, + keys=['b_sum', 'b_size', 'bids', 'asks', 'a_size', 'a_sum']) + # logger.info('order book %s', frame ) + return frame diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 140ab2868..d34807090 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -11,6 +11,7 @@ from typing import Any, Callable, Dict, List, Optional import arrow import requests + from cachetools import TTLCache, cached from freqtrade import (DependencyException, OperationalException, @@ -21,6 +22,7 @@ from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver +from freqtrade.exchange.exchange_helpers import order_book_to_dataframe logger = logging.getLogger(__name__) @@ -180,6 +182,9 @@ class FreqtradeBot(object): final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list self.config['exchange']['pair_whitelist'] = final_list + # Refreshing candles + self.exchange.refresh_tickers(final_list, self.strategy.ticker_interval) + # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() @@ -267,16 +272,40 @@ class FreqtradeBot(object): return final_list - def get_target_bid(self, ticker: Dict[str, float]) -> float: + def get_target_bid(self, pair: str, ticker: Dict[str, float]) -> float: """ Calculates bid target between current ask price and last price :param ticker: Ticker to use for getting Ask and Last Price :return: float: Price """ if ticker['ask'] < ticker['last']: - return ticker['ask'] - balance = self.config['bid_strategy']['ask_last_balance'] - return ticker['ask'] + balance * (ticker['last'] - ticker['ask']) + ticker_rate = ticker['ask'] + else: + balance = self.config['bid_strategy']['ask_last_balance'] + ticker_rate = ticker['ask'] + balance * (ticker['last'] - ticker['ask']) + + used_rate = ticker_rate + config_bid_strategy = self.config.get('bid_strategy', {}) + if 'use_order_book' in config_bid_strategy and\ + config_bid_strategy.get('use_order_book', False): + logger.info('Getting price from order book') + order_book_top = config_bid_strategy.get('order_book_top', 1) + order_book = self.exchange.get_order_book(pair, order_book_top) + logger.debug('order_book %s', order_book) + # top 1 = index 0 + order_book_rate = order_book['bids'][order_book_top - 1][0] + # if ticker has lower rate, then use ticker ( usefull if down trending ) + logger.info('...top %s order book buy rate %0.8f', order_book_top, order_book_rate) + if ticker_rate < order_book_rate: + logger.info('...using ticker rate instead %0.8f', ticker_rate) + used_rate = ticker_rate + else: + used_rate = order_book_rate + else: + logger.info('Using Last Ask / Last Price') + used_rate = ticker_rate + + return used_rate def _get_trade_stake_amount(self) -> Optional[float]: """ @@ -333,7 +362,7 @@ class FreqtradeBot(object): amount_reserve_percent += self.strategy.stoploss # it should not be more than 50% amount_reserve_percent = max(amount_reserve_percent, 0.5) - return min(min_stake_amounts)/amount_reserve_percent + return min(min_stake_amounts) / amount_reserve_percent def create_trade(self) -> bool: """ @@ -365,23 +394,38 @@ class FreqtradeBot(object): if not whitelist: raise DependencyException('No currency pairs in whitelist') - - # EDGE - # WinRate and Expected Risk Reward should be calculated for all whitelisted pairs - # ASYNC: For each pair call backslap - # Save WR and ERR in Edge Dict - # Save last time updated for each pair in edge_last_update_time - # Calulate expectancy and position size and stop loss - - # whitelist = Edge.filter(whitelist) - - # Pick pair based on buy signals + # running get_signal on historical data fetched + # to find buy signals for _pair in whitelist: - thistory = self.exchange.get_candle_history(_pair, interval) - (buy, sell) = self.strategy.get_signal(_pair, interval, thistory) - + (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair)) if buy and not sell: + bidstrat_check_depth_of_market = self.config.get('bid_strategy', {}).\ + get('check_depth_of_market', {}) + if (bidstrat_check_depth_of_market.get('enabled', False)) and\ + (bidstrat_check_depth_of_market.get('bids_to_ask_delta', 0) > 0): + if self._check_depth_of_market_buy(_pair, bidstrat_check_depth_of_market): + return self.execute_buy(_pair, stake_amount) + else: + return False return self.execute_buy(_pair, stake_amount) + + return False + + def _check_depth_of_market_buy(self, pair: str, conf: Dict) -> bool: + """ + Checks depth of market before executing a buy + """ + conf_bids_to_ask_delta = conf.get('bids_to_ask_delta', 0) + logger.info('checking depth of market for %s', pair) + order_book = self.exchange.get_order_book(pair, 1000) + order_book_data_frame = order_book_to_dataframe(order_book['bids'], order_book['asks']) + order_book_bids = order_book_data_frame['b_size'].sum() + order_book_asks = order_book_data_frame['a_size'].sum() + bids_ask_delta = order_book_bids / order_book_asks + logger.info('bids: %s, asks: %s, delta: %s', order_book_bids, + order_book_asks, bids_ask_delta) + if bids_ask_delta >= conf_bids_to_ask_delta: + return True return False def execute_buy(self, pair: str, stake_amount: float) -> bool: @@ -396,7 +440,7 @@ class FreqtradeBot(object): fiat_currency = self.config.get('fiat_display_currency', None) # Calculate amount - buy_limit = self.get_target_bid(self.exchange.get_ticker(pair)) + buy_limit = self.get_target_bid(pair, self.exchange.get_ticker(pair)) min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit) if min_stake_amount is not None and min_stake_amount > stake_amount: @@ -539,22 +583,52 @@ class FreqtradeBot(object): raise ValueError(f'attempt to handle closed trade: {trade}') logger.debug('Handling %s ...', trade) - current_rate = self.exchange.get_ticker(trade.pair)['bid'] + sell_rate = self.exchange.get_ticker(trade.pair)['bid'] (buy, sell) = (False, False) experimental = self.config.get('experimental', {}) if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): - ticker = self.exchange.get_candle_history(trade.pair, self.strategy.ticker_interval) + ticker = self.exchange.klines.get(trade.pair) (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval, ticker) - should_sell = self.strategy.should_sell(trade, current_rate, datetime.utcnow(), buy, sell) - if should_sell.sell_flag: - self.execute_sell(trade, current_rate, should_sell.sell_type) - return True + config_ask_strategy = self.config.get('ask_strategy', {}) + if config_ask_strategy.get('use_order_book', False): + logger.info('Using order book for selling...') + # logger.debug('Order book %s',orderBook) + order_book_min = config_ask_strategy.get('order_book_min', 1) + order_book_max = config_ask_strategy.get('order_book_max', 1) + + order_book = self.exchange.get_order_book(trade.pair, order_book_max) + + for i in range(order_book_min, order_book_max + 1): + order_book_rate = order_book['asks'][i - 1][0] + + # if orderbook has higher rate (high profit), + # use orderbook, otherwise just use bids rate + logger.info(' order book asks top %s: %0.8f', i, order_book_rate) + if sell_rate < order_book_rate: + sell_rate = order_book_rate + + if self.check_sell(trade, sell_rate, buy, sell): + return True + break + else: + logger.info('checking sell') + if self.check_sell(trade, sell_rate, buy, sell): + return True + logger.info('Found no sell signals for whitelisted currencies. Trying again..') return False + def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: + should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) + if should_sell.sell_flag: + self.execute_sell(trade, sell_rate, should_sell.sell_type) + logger.info('excuted sell') + return True + return False + def check_handle_timedout(self) -> None: """ Check if any orders are timed out and cancel if neccessary diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 77caf6761..646f893e6 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -221,19 +221,18 @@ def download_backtesting_testdata(datadir: str, timerange: Optional[TimeRange] = None) -> None: """ - Download the latest ticker intervals from the exchange for the pairs passed in parameters + Download the latest ticker intervals from the exchange for the pair passed in parameters The data is downloaded starting from the last correct ticker interval data that - esists in a cache. If timerange starts earlier than the data in the cache, + exists in a cache. If timerange starts earlier than the data in the cache, the full data will be redownloaded Based on @Rybolov work: https://github.com/rybolov/freqtrade-data - :param pairs: list of pairs to download + :param pair: pair to download :param tick_interval: ticker interval :param timerange: range of time to download :return: None """ - path = make_testdata_path(datadir) filepair = pair.replace("/", "_") filename = os.path.join(path, f'{filepair}-{tick_interval}.json') @@ -249,8 +248,11 @@ def download_backtesting_testdata(datadir: str, logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') - new_data = exchange.get_candle_history(pair=pair, tick_interval=tick_interval, - since_ms=since_ms) + # Default since_ms to 30 days if nothing is given + new_data = exchange.get_history(pair=pair, tick_interval=tick_interval, + since_ms=since_ms if since_ms + else + int(arrow.utcnow().shift(days=-30).float_timestamp) * 1000) data.extend(new_data) logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 56b1b2282..c539d0154 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -434,15 +434,15 @@ class Backtesting(object): Run a backtesting end-to-end :return: None """ - data = {} + data: Dict[str, Any] = {} pairs = self.config['exchange']['pair_whitelist'] logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') - for pair in pairs: - data[pair] = self.exchange.get_ticker_history(pair, self.ticker_interval) + self.exchange.refresh_tickers(pairs, self.ticker_interval) + data = self.exchange.klines else: logger.info('Using local backtesting data (using whitelist in given config) ...') diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 1a2b88c9a..6afa4161b 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -6,7 +6,7 @@ import logging from abc import ABC, abstractmethod from datetime import datetime from enum import Enum -from typing import Dict, List, NamedTuple, Tuple +from typing import Dict, List, NamedTuple, Optional, Tuple import warnings import arrow @@ -70,8 +70,15 @@ class IStrategy(ABC): # associated ticker interval ticker_interval: str + # run "populate_indicators" only for new candle + process_only_new_candles: bool = False + + # Dict to determine if analysis is necessary + _last_candle_seen_per_pair: Dict[str, datetime] = {} + def __init__(self, config: dict) -> None: self.config = config + self._last_candle_seen_per_pair = {} @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -112,13 +119,34 @@ class IStrategy(ABC): add several TA indicators and buy signal to it :return DataFrame with ticker data and indicator data """ + dataframe = parse_ticker_dataframe(ticker_history) - dataframe = self.advise_indicators(dataframe, metadata) - dataframe = self.advise_buy(dataframe, metadata) - dataframe = self.advise_sell(dataframe, metadata) + + pair = str(metadata.get('pair')) + + # Test if seen this pair and last candle before. + # always run if process_only_new_candles is set to true + if (not self.process_only_new_candles or + self._last_candle_seen_per_pair.get(pair, None) != dataframe.iloc[-1]['date']): + # Defs that only make change on new candle data. + logging.debug("TA Analysis Launched") + dataframe = self.advise_indicators(dataframe, metadata) + dataframe = self.advise_buy(dataframe, metadata) + dataframe = self.advise_sell(dataframe, metadata) + self._last_candle_seen_per_pair[pair] = dataframe.iloc[-1]['date'] + else: + logging.debug("Skippinig TA Analysis for already analyzed candle") + dataframe['buy'] = 0 + dataframe['sell'] = 0 + + # Other Defs in strategy that want to be called every loop here + # twitter_sell = self.watch_twitter_feed(dataframe, metadata) + logging.debug("Loop Analysis Launched") + return dataframe - def get_signal(self, pair: str, interval: str, ticker_hist: List[Dict]) -> Tuple[bool, bool]: + def get_signal(self, pair: str, interval: str, + ticker_hist: Optional[List[Dict]]) -> Tuple[bool, bool]: """ Calculates current signal based several technical analysis indicators :param pair: pair in format ANT/BTC diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 5a44a2c57..aee47580c 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -66,6 +66,15 @@ class StrategyResolver(object): else: config['ticker_interval'] = self.strategy.ticker_interval + if 'process_only_new_candles' in config: + self.strategy.process_only_new_candles = config['process_only_new_candles'] + logger.info( + "Override process_only_new_candles 'process_only_new_candles' " + "with value in config file: %s.", config['process_only_new_candles'] + ) + else: + config['process_only_new_candles'] = self.strategy.process_only_new_candles + # Sort and apply type conversions self.strategy.minimal_roi = OrderedDict(sorted( {int(key): value for (key, value) in self.strategy.minimal_roi.items()}.items(), diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index d18016e16..af9062cab 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -102,7 +102,18 @@ def default_conf(): "sell": 30 }, "bid_strategy": { - "ask_last_balance": 0.0 + "ask_last_balance": 0.0, + "use_order_book": False, + "order_book_top": 1, + "check_depth_of_market": { + "enabled": False, + "bids_to_ask_delta": 1 + } + }, + "ask_strategy": { + "use_order_book": False, + "order_book_min": 1, + "order_book_max": 1 }, "exchange": { "name": "bittrex", @@ -403,6 +414,39 @@ def limit_sell_order(): } +@pytest.fixture +def order_book_l2(): + return MagicMock(return_value={ + 'bids': [ + [0.043936, 10.442], + [0.043935, 31.865], + [0.043933, 11.212], + [0.043928, 0.088], + [0.043925, 10.0], + [0.043921, 10.0], + [0.04392, 37.64], + [0.043899, 0.066], + [0.043885, 0.676], + [0.04387, 22.758] + ], + 'asks': [ + [0.043949, 0.346], + [0.04395, 0.608], + [0.043951, 3.948], + [0.043954, 0.288], + [0.043958, 9.277], + [0.043995, 1.566], + [0.044, 0.588], + [0.044002, 0.992], + [0.044003, 0.095], + [0.04402, 37.64] + ], + 'timestamp': None, + 'datetime': None, + 'nonce': 288004540 + }) + + @pytest.fixture def ticker_history(): return [ diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 5ee21c9e4..de720b3d9 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -3,8 +3,9 @@ import logging from datetime import datetime from random import randint -from unittest.mock import MagicMock, PropertyMock +from unittest.mock import Mock, MagicMock, PropertyMock +import arrow import ccxt import pytest @@ -13,6 +14,14 @@ from freqtrade.exchange import API_RETRY_COUNT, Exchange from freqtrade.tests.conftest import get_patched_exchange, log_has +# Source: https://stackoverflow.com/questions/29881236/how-to-mock-asyncio-coroutines +def get_mock_coro(return_value): + async def mock_coro(*args, **kwargs): + return return_value + + return Mock(wraps=mock_coro) + + def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs): with pytest.raises(TemporaryError): api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError) @@ -27,12 +36,32 @@ def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, * assert api_mock.__dict__[mock_ccxt_fun].call_count == 1 +async def async_ccxt_exception(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs): + with pytest.raises(TemporaryError): + api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + await getattr(exchange, fun)(**kwargs) + assert api_mock.__dict__[mock_ccxt_fun].call_count == API_RETRY_COUNT + 1 + + with pytest.raises(OperationalException): + api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + await getattr(exchange, fun)(**kwargs) + assert api_mock.__dict__[mock_ccxt_fun].call_count == 1 + + def test_init(default_conf, mocker, caplog): caplog.set_level(logging.INFO) get_patched_exchange(mocker, default_conf) assert log_has('Instance is running with dry_run enabled', caplog.record_tuples) +def test_destroy(default_conf, mocker, caplog): + caplog.set_level(logging.DEBUG) + get_patched_exchange(mocker, default_conf) + assert log_has('Exchange object destroyed, closing async loop', caplog.record_tuples) + + def test_init_exception(default_conf, mocker): default_conf['exchange']['name'] = 'wrong_exchange_name' @@ -64,6 +93,7 @@ def test_symbol_amount_prec(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) exchange = Exchange(default_conf) amount = 2.34559 @@ -87,6 +117,7 @@ def test_symbol_price_prec(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) exchange = Exchange(default_conf) price = 2.34559 @@ -108,6 +139,7 @@ def test_set_sandbox(default_conf, mocker): type(api_mock).urls = url_mock mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) exchange = Exchange(default_conf) liveurl = exchange._api.urls['api'] @@ -129,6 +161,7 @@ def test_set_sandbox_exception(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises(OperationalException, match=r'does not provide a sandbox api'): exchange = Exchange(default_conf) @@ -136,6 +169,20 @@ def test_set_sandbox_exception(default_conf, mocker): exchange.set_sandbox(exchange._api, default_conf['exchange'], 'Logname') +def test__load_async_markets(default_conf, mocker, caplog): + exchange = get_patched_exchange(mocker, default_conf) + exchange._api_async.load_markets = get_mock_coro(None) + exchange._load_async_markets() + assert exchange._api_async.load_markets.call_count == 1 + caplog.set_level(logging.DEBUG) + + exchange._api_async.load_markets = Mock(side_effect=ccxt.BaseError("deadbeef")) + exchange._load_async_markets() + + assert log_has('Could not load async markets. Reason: deadbeef', + caplog.record_tuples) + + def test_validate_pairs(default_conf, mocker): api_mock = MagicMock() api_mock.load_markets = MagicMock(return_value={ @@ -146,6 +193,7 @@ def test_validate_pairs(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) Exchange(default_conf) @@ -154,6 +202,7 @@ def test_validate_pairs_not_available(default_conf, mocker): api_mock.load_markets = MagicMock(return_value={}) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises(OperationalException, match=r'not available'): Exchange(default_conf) @@ -167,6 +216,7 @@ def test_validate_pairs_not_compatible(default_conf, mocker): default_conf['stake_currency'] = 'ETH' mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises(OperationalException, match=r'not compatible'): Exchange(default_conf) @@ -179,6 +229,7 @@ def test_validate_pairs_exception(default_conf, mocker, caplog): api_mock.load_markets = MagicMock(return_value={}) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available at Binance'): Exchange(default_conf) @@ -198,6 +249,7 @@ def test_validate_pairs_stake_exception(default_conf, mocker, caplog): api_mock.name = MagicMock(return_value='binance') mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises( OperationalException, @@ -515,6 +567,189 @@ def test_get_ticker(default_conf, mocker): exchange.get_ticker(pair='ETH/BTC', refresh=True) +def test_get_history(default_conf, mocker, caplog): + exchange = get_patched_exchange(mocker, default_conf) + tick = [ + [ + arrow.utcnow().timestamp * 1000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + pair = 'ETH/BTC' + + async def mock_candle_hist(pair, tick_interval, since_ms): + return pair, tick + + exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) + # one_call calculation * 1.8 should do 2 calls + since = 5 * 60 * 500 * 1.8 + print(f"since = {since}") + ret = exchange.get_history(pair, "5m", int((arrow.utcnow().timestamp - since) * 1000)) + + assert exchange._async_get_candle_history.call_count == 2 + # Returns twice the above tick + assert len(ret) == 2 + + +def test_refresh_tickers(mocker, default_conf, caplog) -> None: + tick = [ + [ + 1511686200000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + + caplog.set_level(logging.DEBUG) + exchange = get_patched_exchange(mocker, default_conf) + exchange._api_async.fetch_ohlcv = get_mock_coro(tick) + + pairs = ['IOTA/ETH', 'XRP/ETH'] + # empty dicts + assert not exchange.klines + exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m') + + assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples) + assert exchange.klines + for pair in pairs: + assert exchange.klines[pair] + + +@pytest.mark.asyncio +async def test__async_get_candle_history(default_conf, mocker, caplog): + tick = [ + [ + arrow.utcnow().timestamp * 1000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + + caplog.set_level(logging.DEBUG) + exchange = get_patched_exchange(mocker, default_conf) + # Monkey-patch async function + exchange._api_async.fetch_ohlcv = get_mock_coro(tick) + + exchange = Exchange(default_conf) + pair = 'ETH/BTC' + res = await exchange._async_get_candle_history(pair, "5m") + assert type(res) is tuple + assert len(res) == 2 + assert res[0] == pair + assert res[1] == tick + assert exchange._api_async.fetch_ohlcv.call_count == 1 + assert not log_has(f"Using cached klines data for {pair} ...", caplog.record_tuples) + # test caching + res = await exchange._async_get_candle_history(pair, "5m") + assert exchange._api_async.fetch_ohlcv.call_count == 1 + assert log_has(f"Using cached klines data for {pair} ...", caplog.record_tuples) + + # exchange = Exchange(default_conf) + await async_ccxt_exception(mocker, default_conf, MagicMock(), + "_async_get_candle_history", "fetch_ohlcv", + pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) + + api_mock = MagicMock() + with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'): + api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + await exchange._async_get_candle_history(pair, "5m", + (arrow.utcnow().timestamp - 2000) * 1000) + + +@pytest.mark.asyncio +async def test__async_get_candle_history_empty(default_conf, mocker, caplog): + """ Test empty exchange result """ + tick = [] + + caplog.set_level(logging.DEBUG) + exchange = get_patched_exchange(mocker, default_conf) + # Monkey-patch async function + exchange._api_async.fetch_ohlcv = get_mock_coro([]) + + exchange = Exchange(default_conf) + pair = 'ETH/BTC' + res = await exchange._async_get_candle_history(pair, "5m") + assert type(res) is tuple + assert len(res) == 2 + assert res[0] == pair + assert res[1] == tick + assert exchange._api_async.fetch_ohlcv.call_count == 1 + + +@pytest.mark.asyncio +async def test_async_get_candles_history(default_conf, mocker): + tick = [ + [ + 1511686200000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + + async def mock_get_candle_hist(pair, tick_interval, since_ms=None): + return (pair, tick) + + exchange = get_patched_exchange(mocker, default_conf) + # Monkey-patch async function + exchange._api_async.fetch_ohlcv = get_mock_coro(tick) + + exchange._async_get_candle_history = Mock(wraps=mock_get_candle_hist) + + pairs = ['ETH/BTC', 'XRP/BTC'] + res = await exchange.async_get_candles_history(pairs, "5m") + assert type(res) is list + assert len(res) == 2 + assert type(res[0]) is tuple + assert res[0][0] == pairs[0] + assert res[0][1] == tick + assert res[1][0] == pairs[1] + assert res[1][1] == tick + assert exchange._async_get_candle_history.call_count == 2 + + +def test_get_order_book(default_conf, mocker, order_book_l2): + default_conf['exchange']['name'] = 'binance' + api_mock = MagicMock() + + api_mock.fetch_l2_order_book = order_book_l2 + exchange = get_patched_exchange(mocker, default_conf, api_mock) + order_book = exchange.get_order_book(pair='ETH/BTC', limit=10) + assert 'bids' in order_book + assert 'asks' in order_book + assert len(order_book['bids']) == 10 + assert len(order_book['asks']) == 10 + + +def test_get_order_book_exception(default_conf, mocker): + api_mock = MagicMock() + with pytest.raises(OperationalException): + api_mock.fetch_l2_order_book = MagicMock(side_effect=ccxt.NotSupported) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_order_book(pair='ETH/BTC', limit=50) + with pytest.raises(TemporaryError): + api_mock.fetch_l2_order_book = MagicMock(side_effect=ccxt.NetworkError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_order_book(pair='ETH/BTC', limit=50) + with pytest.raises(OperationalException): + api_mock.fetch_l2_order_book = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_order_book(pair='ETH/BTC', limit=50) + + def make_fetch_ohlcv_mock(data): def fetch_ohlcv_mock(pair, timeframe, since): if since: diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 32a5229c0..a17867b3a 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -110,7 +110,7 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals return pairdata -# use for mock freqtrade.exchange.get_candle_history' +# use for mock ccxt.fetch_ohlvc' def _load_pair_as_ticks(pair, tickfreq): ticks = optimize.load_data(None, ticker_interval=tickfreq, pairs=[pair]) ticks = trim_dictlist(ticks, -201) @@ -455,7 +455,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.optimize.load_data', mocked_load_data) - mocker.patch('freqtrade.exchange.Exchange.get_candle_history') + mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', @@ -490,7 +490,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.optimize.load_data', MagicMock(return_value={})) - mocker.patch('freqtrade.exchange.Exchange.get_candle_history') + mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', @@ -733,9 +733,14 @@ def test_backtest_record(default_conf, fee, mocker): def test_backtest_start_live(default_conf, mocker, caplog): default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', - new=lambda s, n, i: _load_pair_as_ticks(n, i)) - patch_exchange(mocker) + + async def load_pairs(pair, timeframe, since): + return _load_pair_as_ticks(pair, timeframe) + + api_mock = MagicMock() + api_mock.fetch_ohlcv = load_pairs + + patch_exchange(mocker, api_mock) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock()) mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', MagicMock()) mocker.patch('freqtrade.configuration.open', mocker.mock_open( @@ -776,9 +781,13 @@ def test_backtest_start_live(default_conf, mocker, caplog): def test_backtest_start_multi_strat(default_conf, mocker, caplog): default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', - new=lambda s, n, i: _load_pair_as_ticks(n, i)) - patch_exchange(mocker) + + async def load_pairs(pair, timeframe, since): + return _load_pair_as_ticks(pair, timeframe) + api_mock = MagicMock() + api_mock.fetch_ohlcv = load_pairs + + patch_exchange(mocker, api_mock) backtestmock = MagicMock() mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) gen_table_mock = MagicMock() diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 13f65fbf5..77fa3e3b1 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -53,7 +53,7 @@ def _clean_test_file(file: str) -> None: def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-30m.json') _backup_file(file, copy_file=True) optimize.load_data(None, pairs=['UNITTEST/BTC'], ticker_interval='30m') @@ -63,7 +63,7 @@ def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-5m.json') _backup_file(file, copy_file=True) @@ -74,7 +74,7 @@ def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json') _backup_file(file, copy_file=True) optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC']) @@ -87,7 +87,7 @@ def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog, default_co """ Test load_data() with 1 min ticker """ - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) exchange = get_patched_exchange(mocker, default_conf) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') @@ -118,7 +118,7 @@ def test_testdata_path() -> None: def test_download_pairs(ticker_history, mocker, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) exchange = get_patched_exchange(mocker, default_conf) file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') @@ -261,7 +261,7 @@ def test_load_cached_data_for_updating(mocker) -> None: def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) mocker.patch('freqtrade.optimize.__init__.download_backtesting_testdata', side_effect=BaseException('File Error')) exchange = get_patched_exchange(mocker, default_conf) @@ -279,7 +279,7 @@ def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) def test_download_backtesting_testdata(ticker_history, mocker, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) exchange = get_patched_exchange(mocker, default_conf) # Download a 1 min ticker file @@ -304,7 +304,7 @@ def test_download_backtesting_testdata2(mocker, default_conf) -> None: [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199] ] json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=tick) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=tick) exchange = get_patched_exchange(mocker, default_conf) download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='1m') download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='3m') diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index 2c54e492a..fedd355af 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -89,7 +89,6 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog): def test_get_signal_handles_exceptions(mocker, default_conf): - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=MagicMock()) exchange = get_patched_exchange(mocker, default_conf) mocker.patch.object( _STRATEGY, 'analyze_ticker', @@ -129,3 +128,75 @@ def test_min_roi_reached(default_conf, fee) -> None: assert not strategy.min_roi_reached(trade, -0.01, arrow.utcnow().shift(minutes=-1).datetime) assert strategy.min_roi_reached(trade, 0.02, arrow.utcnow().shift(minutes=-1).datetime) + + +def test_analyze_ticker_default(ticker_history, mocker, caplog) -> None: + caplog.set_level(logging.DEBUG) + ind_mock = MagicMock(side_effect=lambda x, meta: x) + buy_mock = MagicMock(side_effect=lambda x, meta: x) + sell_mock = MagicMock(side_effect=lambda x, meta: x) + mocker.patch.multiple( + 'freqtrade.strategy.interface.IStrategy', + advise_indicators=ind_mock, + advise_buy=buy_mock, + advise_sell=sell_mock, + + ) + strategy = DefaultStrategy({}) + strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + assert ind_mock.call_count == 1 + assert buy_mock.call_count == 1 + assert buy_mock.call_count == 1 + + assert log_has('TA Analysis Launched', caplog.record_tuples) + assert not log_has('Skippinig TA Analysis for already analyzed candle', + caplog.record_tuples) + caplog.clear() + + strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + # No analysis happens as process_only_new_candles is true + assert ind_mock.call_count == 2 + assert buy_mock.call_count == 2 + assert buy_mock.call_count == 2 + assert log_has('TA Analysis Launched', caplog.record_tuples) + assert not log_has('Skippinig TA Analysis for already analyzed candle', + caplog.record_tuples) + + +def test_analyze_ticker_skip_analyze(ticker_history, mocker, caplog) -> None: + caplog.set_level(logging.DEBUG) + ind_mock = MagicMock(side_effect=lambda x, meta: x) + buy_mock = MagicMock(side_effect=lambda x, meta: x) + sell_mock = MagicMock(side_effect=lambda x, meta: x) + mocker.patch.multiple( + 'freqtrade.strategy.interface.IStrategy', + advise_indicators=ind_mock, + advise_buy=buy_mock, + advise_sell=sell_mock, + + ) + strategy = DefaultStrategy({}) + strategy.process_only_new_candles = True + + ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + assert ind_mock.call_count == 1 + assert buy_mock.call_count == 1 + assert buy_mock.call_count == 1 + assert log_has('TA Analysis Launched', caplog.record_tuples) + assert not log_has('Skippinig TA Analysis for already analyzed candle', + caplog.record_tuples) + caplog.clear() + + ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + # No analysis happens as process_only_new_candles is true + assert ind_mock.call_count == 1 + assert buy_mock.call_count == 1 + assert buy_mock.call_count == 1 + # only skipped analyze adds buy and sell columns, otherwise it's all mocked + assert 'buy' in ret + assert 'sell' in ret + assert ret['buy'].sum() == 0 + assert ret['sell'].sum() == 0 + assert not log_has('TA Analysis Launched', caplog.record_tuples) + assert log_has('Skippinig TA Analysis for already analyzed candle', + caplog.record_tuples) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index ca41d1d39..abc531689 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -165,6 +165,23 @@ def test_strategy_override_ticker_interval(caplog): ) in caplog.record_tuples +def test_strategy_override_process_only_new_candles(caplog): + caplog.set_level(logging.INFO) + + config = { + 'strategy': 'DefaultStrategy', + 'process_only_new_candles': True + } + resolver = StrategyResolver(config) + + assert resolver.strategy.process_only_new_candles + assert ('freqtrade.strategy.resolver', + logging.INFO, + "Override process_only_new_candles 'process_only_new_candles' " + "with value in config file: True." + ) in caplog.record_tuples + + def test_deprecate_populate_indicators(result): default_location = path.join(path.dirname(path.realpath(__file__))) resolver = StrategyResolver({'strategy': 'TestStrategyLegacy', diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index fa6bc7c2a..5e982f11a 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -43,7 +43,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: :return: None """ freqtrade.strategy.get_signal = lambda e, s, t: value - freqtrade.exchange.get_candle_history = lambda p, i: None + freqtrade.exchange.refresh_tickers = lambda p, i: None def patch_RPCManager(mocker) -> MagicMock: @@ -159,6 +159,15 @@ def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: assert whitelist == [] +def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) + + with pytest.raises(OperationalException): + freqtrade._gen_pair_whitelist(base_currency='BTC') + + @pytest.mark.skip(reason="Test not implemented") def test_refresh_whitelist() -> None: pass @@ -544,7 +553,6 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), - get_candle_history=MagicMock(return_value=20), get_balance=MagicMock(return_value=20), get_fee=fee, ) @@ -664,21 +672,21 @@ def test_balance_fully_ask_side(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 0.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) - assert freqtrade.get_target_bid({'ask': 20, 'last': 10}) == 20 + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 20, 'last': 10}) == 20 def test_balance_fully_last_side(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 1.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) - assert freqtrade.get_target_bid({'ask': 20, 'last': 10}) == 10 + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 20, 'last': 10}) == 10 def test_balance_bigger_last_ask(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 1.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) - assert freqtrade.get_target_bid({'ask': 5, 'last': 10}) == 5 + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 5, 'last': 10}) == 5 def test_process_maybe_execute_buy(mocker, default_conf) -> None: @@ -1878,6 +1886,191 @@ def test_get_real_amount_open_trade(default_conf, mocker): assert freqtrade.get_real_amount(trade, order) == amount +def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, markets, mocker, + order_book_l2): + default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True + default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 0.1 + patch_RPCManager(mocker) + mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + get_markets=markets + ) + + # Save state of current whitelist + whitelist = deepcopy(default_conf['exchange']['pair_whitelist']) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade is not None + assert trade.stake_amount == 0.001 + assert trade.is_open + assert trade.open_date is not None + assert trade.exchange == 'bittrex' + + # Simulate fulfilled LIMIT_BUY order for trade + trade.update(limit_buy_order) + + assert trade.open_rate == 0.00001099 + assert whitelist == default_conf['exchange']['pair_whitelist'] + + +def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_order, + fee, markets, mocker, order_book_l2): + default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True + # delta is 100 which is impossible to reach. hence check_depth_of_market will return false + default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 100 + patch_RPCManager(mocker) + mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + get_markets=markets + ) + # Save state of current whitelist + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade is None + + +def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2, markets) -> None: + """ + test if function get_target_bid will return the order book price + instead of the ask rate + """ + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_markets=markets, + get_order_book=order_book_l2 + ) + default_conf['exchange']['name'] = 'binance' + default_conf['bid_strategy']['use_order_book'] = True + default_conf['bid_strategy']['order_book_top'] = 2 + default_conf['bid_strategy']['ask_last_balance'] = 0 + default_conf['telegram']['enabled'] = False + + freqtrade = FreqtradeBot(default_conf) + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.045, 'last': 0.046}) == 0.043935 + + +def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets) -> None: + """ + test if function get_target_bid will return the ask rate (since its value is lower) + instead of the order book rate (even if enabled) + """ + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_markets=markets, + get_order_book=order_book_l2 + ) + default_conf['exchange']['name'] = 'binance' + default_conf['bid_strategy']['use_order_book'] = True + default_conf['bid_strategy']['order_book_top'] = 2 + default_conf['bid_strategy']['ask_last_balance'] = 0 + default_conf['telegram']['enabled'] = False + + freqtrade = FreqtradeBot(default_conf) + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.042, 'last': 0.046}) == 0.042 + + +def test_order_book_bid_strategy3(default_conf, mocker, order_book_l2, markets) -> None: + """ + test if function get_target_bid will return ask rate instead + of the order book rate + """ + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_markets=markets, + get_order_book=order_book_l2 + ) + default_conf['exchange']['name'] = 'binance' + default_conf['bid_strategy']['use_order_book'] = True + default_conf['bid_strategy']['order_book_top'] = 1 + default_conf['bid_strategy']['ask_last_balance'] = 0 + default_conf['telegram']['enabled'] = False + + freqtrade = FreqtradeBot(default_conf) + + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.03, 'last': 0.029}) == 0.03 + + +def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2, markets) -> None: + """ + test check depth of market + """ + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_markets=markets, + get_order_book=order_book_l2 + ) + default_conf['telegram']['enabled'] = False + default_conf['exchange']['name'] = 'binance' + default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True + # delta is 100 which is impossible to reach. hence function will return false + default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 100 + freqtrade = FreqtradeBot(default_conf) + + conf = default_conf['bid_strategy']['check_depth_of_market'] + assert freqtrade._check_depth_of_market_buy('ETH/BTC', conf) is False + + +def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order, + fee, markets, mocker, order_book_l2) -> None: + """ + test order book ask strategy + """ + mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) + default_conf['exchange']['name'] = 'binance' + default_conf['ask_strategy']['use_order_book'] = True + default_conf['ask_strategy']['order_book_min'] = 1 + default_conf['ask_strategy']['order_book_max'] = 2 + default_conf['telegram']['enabled'] = False + patch_RPCManager(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00001172, + 'ask': 0.00001173, + 'last': 0.00001172 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + sell=MagicMock(return_value={'id': limit_sell_order['id']}), + get_fee=fee, + get_markets=markets + ) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade + + time.sleep(0.01) # Race condition fix + trade.update(limit_buy_order) + assert trade.is_open is True + + patch_get_signal(freqtrade, value=(False, True)) + assert freqtrade.handle_trade(trade) is True + + def test_startup_messages(default_conf, mocker): default_conf['dynamic_whitelist'] = 20 freqtrade = get_patched_freqtradebot(mocker, default_conf) diff --git a/requirements.txt b/requirements.txt index 76b8f508f..99d0afcdb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -ccxt==1.17.170 +ccxt==1.17.283 SQLAlchemy==1.2.11 -python-telegram-bot==10.1.0 +python-telegram-bot==11.1.0 arrow==0.12.1 cachetools==2.1.0 requests==2.19.1 @@ -12,9 +12,10 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.1 TA-Lib==0.4.17 -pytest==3.7.2 +pytest==3.8.0 pytest-mock==1.10.0 -pytest-cov==2.5.1 +pytest-asyncio==0.9.0 +pytest-cov==2.6.0 tabulate==0.8.2 coinmarketcap==5.0.3 diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 686098f94..27c4c1e1c 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -"""This script generate json data from bittrex""" +"""This script generate json data""" import json import sys from pathlib import Path @@ -52,9 +52,10 @@ exchange = Exchange({'key': '', 'stake_currency': '', 'dry_run': True, 'exchange': { - 'name': args.exchange, - 'pair_whitelist': [] - } + 'name': args.exchange, + 'pair_whitelist': [], + 'ccxt_rate_limit': False + } }) pairs_not_available = [] diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index f2f2e0c7f..0f0a3d4cb 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -138,7 +138,8 @@ def plot_analyzed_dataframe(args: Namespace) -> None: tickers = {} if args.live: logger.info('Downloading pair.') - tickers[pair] = exchange.get_candle_history(pair, tick_interval) + exchange.refresh_tickers([pair], tick_interval) + tickers[pair] = exchange.klines[pair] else: tickers = optimize.load_data( datadir=_CONF.get("datadir"), diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 80c238d92..7c3892b77 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -45,6 +45,9 @@ class TestStrategy(IStrategy): # Optimal ticker interval for the strategy ticker_interval = '5m' + # run "populate_indicators" only for new candle + ta_on_candle = False + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame