diff --git a/.gitignore b/.gitignore index b52a31d8e..9ed046c40 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,7 @@ target/ # Jupyter Notebook .ipynb_checkpoints +*.ipynb # pyenv .python-version diff --git a/.travis.yml b/.travis.yml index 3dfcf6111..84f3c78d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,6 @@ jobs: - stage: tests script: - pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ - - coveralls name: pytest - script: - cp config.json.example config.json @@ -48,6 +47,8 @@ jobs: - build_helpers/publish_docker.sh name: "Build and test and push docker image" +after_success: + - coveralls notifications: slack: diff --git a/config_kraken.json.example b/config_kraken.json.example new file mode 100644 index 000000000..7a47b701f --- /dev/null +++ b/config_kraken.json.example @@ -0,0 +1,71 @@ +{ + "max_open_trades": 5, + "stake_currency": "EUR", + "stake_amount": 10, + "fiat_display_currency": "EUR", + "ticker_interval" : "5m", + "dry_run": true, + "db_url": "sqlite:///tradesv3.dryrun.sqlite", + "trailing_stop": false, + "unfilledtimeout": { + "buy": 10, + "sell": 30 + }, + "bid_strategy": { + "ask_last_balance": 0.0, + "use_order_book": false, + "order_book_top": 1, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + } + }, + "ask_strategy":{ + "use_order_book": false, + "order_book_min": 1, + "order_book_max": 9 + }, + "exchange": { + "name": "kraken", + "key": "", + "secret": "", + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": true, + "rateLimit": 1000 + }, + "pair_whitelist": [ + "ETH/EUR", + "BTC/EUR", + "BCH/EUR" + ], + "pair_blacklist": [ + + ] + }, + "edge": { + "enabled": false, + "process_throttle_secs": 3600, + "calculate_since_number_of_days": 7, + "capital_available_percentage": 0.5, + "allowed_risk": 0.01, + "stoploss_range_min": -0.01, + "stoploss_range_max": -0.1, + "stoploss_range_step": -0.01, + "minimum_winrate": 0.60, + "minimum_expectancy": 0.20, + "min_trade_number": 10, + "max_trade_duration_minute": 1440, + "remove_pumps": false + }, + "telegram": { + "enabled": false, + "token": "", + "chat_id": "" + }, + "initial_state": "running", + "forcebuy_enable": false, + "internals": { + "process_throttle_secs": 5 + } +} diff --git a/docs/faq.md b/docs/faq.md index 4bbf28fe6..3eaa771ee 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,4 +1,6 @@ -# freqtrade FAQ +# Freqtrade FAQ + +### Freqtrade commons #### I have waited 5 minutes, why hasn't the bot made any trades yet?! @@ -34,7 +36,9 @@ perform anymore BUYS? You can use the `/forcesell all` command from Telegram. -### How many epoch do I need to get a good Hyperopt result? +### Hyperopt module + +#### How many epoch do I need to get a good Hyperopt result? Per default Hyperopts without `-e` or `--epochs` parameter will only run 100 epochs, means 100 evals of your triggers, guards, .... Too few to find a great result (unless if you are very lucky), so you probably @@ -68,3 +72,18 @@ but it will give the idea. With only these triggers and guards there is already 8*10^9*10 evaluations. A roughly total of 80 billion evals. Did you run 100 000 evals? Congrats, you've done roughly 1 / 100 000 th of the search space. + +### Edge module + +#### Edge implements interesting approach for controlling position size, is there any theory behind it? + +The Edge module is mostly a result of brainstorming of [@mishaker](https://github.com/mishaker) and [@creslinux](https://github.com/creslinux) freqtrade team members. + +You can find further info on expectancy, winrate, risk management and position size in the following sources: +* https://www.tradeciety.com/ultimate-math-guide-for-traders/ +* http://www.vantharp.com/tharp-concepts/expectancy.asp +* https://samuraitradingacademy.com/trading-expectancy/ +* https://www.learningmarkets.com/determining-expectancy-in-your-trading/ +* http://www.lonestocktrader.com/make-money-trading-positive-expectancy/ +* https://www.babypips.com/trading/trade-expectancy-matter + diff --git a/docs/installation.md b/docs/installation.md index e5f514eb1..80223f954 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -115,10 +115,16 @@ Once you have Docker installed, simply create the config file (e.g. `config.json **1.1. Clone the git repository** +Linux/Mac/Windows with WSL ```bash git clone https://github.com/freqtrade/freqtrade.git ``` +Windows with docker +```bash +git clone --config core.autocrlf=input https://github.com/freqtrade/freqtrade.git +``` + **1.2. (Optional) Checkout the develop branch** ```bash diff --git a/freqtrade/constants.py b/freqtrade/constants.py index aa38449eb..1b3e25249 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -67,6 +67,7 @@ CONF_SCHEMA = { }, 'minProperties': 1 }, + 'amount_reserve_percent': {'type': 'number', 'minimum': 0.0, 'maximum': 0.5}, 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, 'trailing_stop': {'type': 'boolean'}, 'trailing_stop_positive': {'type': 'number', 'minimum': 0, 'maximum': 1}, diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 47886989e..204ed971e 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -1,730 +1,2 @@ -# pragma pylint: disable=W0603 -""" Cryptocurrency Exchanges support """ -import logging -import inspect -from random import randint -from typing import List, Dict, Tuple, Any, Optional -from datetime import datetime -from math import floor, ceil - -import arrow -import asyncio -import ccxt -import ccxt.async_support as ccxt_async -from pandas import DataFrame - -from freqtrade import constants, OperationalException, DependencyException, TemporaryError -from freqtrade.data.converter import parse_ticker_dataframe - -logger = logging.getLogger(__name__) - -API_RETRY_COUNT = 4 - - -# Urls to exchange markets, insert quote and base with .format() -_EXCHANGE_URLS = { - ccxt.bittrex.__name__: '/Market/Index?MarketName={quote}-{base}', - ccxt.binance.__name__: '/tradeDetail.html?symbol={base}_{quote}' -} - - -def retrier_async(f): - async def wrapper(*args, **kwargs): - count = kwargs.pop('count', API_RETRY_COUNT) - try: - return await f(*args, **kwargs) - except (TemporaryError, DependencyException) as ex: - logger.warning('%s() returned exception: "%s"', f.__name__, ex) - if count > 0: - count -= 1 - kwargs.update({'count': count}) - logger.warning('retrying %s() still for %s times', f.__name__, count) - return await wrapper(*args, **kwargs) - else: - logger.warning('Giving up retrying: %s()', f.__name__) - raise ex - return wrapper - - -def retrier(f): - def wrapper(*args, **kwargs): - count = kwargs.pop('count', API_RETRY_COUNT) - try: - return f(*args, **kwargs) - except (TemporaryError, DependencyException) as ex: - logger.warning('%s() returned exception: "%s"', f.__name__, ex) - if count > 0: - count -= 1 - kwargs.update({'count': count}) - logger.warning('retrying %s() still for %s times', f.__name__, count) - return wrapper(*args, **kwargs) - else: - logger.warning('Giving up retrying: %s()', f.__name__) - raise ex - return wrapper - - -class Exchange(object): - - _conf: Dict = {} - - def __init__(self, config: dict) -> None: - """ - Initializes this module with the given config, - it does basic validation whether the specified - exchange and pairs are valid. - :return: None - """ - self._conf.update(config) - - self._cached_ticker: Dict[str, Any] = {} - - # Holds last candle refreshed time of each pair - self._pairs_last_refresh_time: Dict[Tuple[str, str], int] = {} - - # Holds candles - self._klines: Dict[Tuple[str, str], DataFrame] = {} - - # Holds all open sell orders for dry_run - self._dry_run_open_orders: Dict[str, Any] = {} - - if config['dry_run']: - logger.info('Instance is running with dry_run enabled') - - exchange_config = config['exchange'] - self._api: ccxt.Exchange = self._init_ccxt( - exchange_config, ccxt_kwargs=exchange_config.get('ccxt_config')) - self._api_async: ccxt_async.Exchange = self._init_ccxt( - exchange_config, ccxt_async, ccxt_kwargs=exchange_config.get('ccxt_async_config')) - - logger.info('Using Exchange "%s"', self.name) - - self.markets = self._load_markets() - # Check if all pairs are available - self.validate_pairs(config['exchange']['pair_whitelist']) - self.validate_ordertypes(config.get('order_types', {})) - self.validate_order_time_in_force(config.get('order_time_in_force', {})) - if config.get('ticker_interval'): - # Check if timeframe is available - self.validate_timeframes(config['ticker_interval']) - - def __del__(self): - """ - Destructor - clean up async stuff - """ - logger.debug("Exchange object destroyed, closing async loop") - if self._api_async and inspect.iscoroutinefunction(self._api_async.close): - asyncio.get_event_loop().run_until_complete(self._api_async.close()) - - def _init_ccxt(self, exchange_config: dict, ccxt_module=ccxt, - ccxt_kwargs: dict = None) -> ccxt.Exchange: - """ - Initialize ccxt with given config and return valid - ccxt instance. - """ - # Find matching class for the given exchange name - name = exchange_config['name'] - - if name not in ccxt_module.exchanges: - raise OperationalException(f'Exchange {name} is not supported') - - ex_config = { - 'apiKey': exchange_config.get('key'), - 'secret': exchange_config.get('secret'), - 'password': exchange_config.get('password'), - 'uid': exchange_config.get('uid', ''), - 'enableRateLimit': exchange_config.get('ccxt_rate_limit', True) - } - if ccxt_kwargs: - logger.info('Applying additional ccxt config: %s', ccxt_kwargs) - ex_config.update(ccxt_kwargs) - try: - - api = getattr(ccxt_module, name.lower())(ex_config) - except (KeyError, AttributeError): - raise OperationalException(f'Exchange {name} is not supported') - - self.set_sandbox(api, exchange_config, name) - - return api - - @property - def name(self) -> str: - """exchange Name (from ccxt)""" - return self._api.name - - @property - def id(self) -> str: - """exchange ccxt id""" - return self._api.id - - def klines(self, pair_interval: Tuple[str, str], copy=True) -> DataFrame: - # create key tuple - if pair_interval in self._klines: - return self._klines[pair_interval].copy() if copy else self._klines[pair_interval] - else: - return DataFrame() - - def set_sandbox(self, api, exchange_config: dict, name: str): - if exchange_config.get('sandbox'): - if api.urls.get('test'): - api.urls['api'] = api.urls['test'] - logger.info("Enabled Sandbox API on %s", name) - else: - logger.warning(name, "No Sandbox URL in CCXT, exiting. " - "Please check your config.json") - raise OperationalException(f'Exchange {name} does not provide a sandbox api') - - def _load_async_markets(self) -> None: - try: - if self._api_async: - asyncio.get_event_loop().run_until_complete(self._api_async.load_markets()) - - except ccxt.BaseError as e: - logger.warning('Could not load async markets. Reason: %s', e) - return - - def _load_markets(self) -> Dict[str, Any]: - """ Initialize markets both sync and async """ - try: - markets = self._api.load_markets() - self._load_async_markets() - return markets - except ccxt.BaseError as e: - logger.warning('Unable to initialize markets. Reason: %s', e) - return {} - - def validate_pairs(self, pairs: List[str]) -> None: - """ - Checks if all given pairs are tradable on the current exchange. - Raises OperationalException if one pair is not available. - :param pairs: list of pairs - :return: None - """ - - if not self.markets: - logger.warning('Unable to validate pairs (assuming they are correct).') - # return - - stake_cur = self._conf['stake_currency'] - for pair in pairs: - # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs - # TODO: add a support for having coins in BTC/USDT format - if not pair.endswith(stake_cur): - raise OperationalException( - f'Pair {pair} not compatible with stake_currency: {stake_cur}') - if self.markets and pair not in self.markets: - raise OperationalException( - f'Pair {pair} is not available at {self.name}' - f'Please remove {pair} from your whitelist.') - - def validate_timeframes(self, timeframe: List[str]) -> None: - """ - Checks if ticker interval from config is a supported timeframe on the exchange - """ - timeframes = self._api.timeframes - if timeframe not in timeframes: - raise OperationalException( - f'Invalid ticker {timeframe}, this Exchange supports {timeframes}') - - def validate_ordertypes(self, order_types: Dict) -> None: - """ - Checks if order-types configured in strategy/config are supported - """ - if any(v == 'market' for k, v in order_types.items()): - if not self.exchange_has('createMarketOrder'): - raise OperationalException( - f'Exchange {self.name} does not support market orders.') - - if order_types.get('stoploss_on_exchange'): - if self.name != 'Binance': - raise OperationalException( - 'On exchange stoploss is not supported for %s.' % self.name - ) - - def validate_order_time_in_force(self, order_time_in_force: Dict) -> None: - """ - Checks if order time in force configured in strategy/config are supported - """ - if any(v != 'gtc' for k, v in order_time_in_force.items()): - if self.name != 'Binance': - raise OperationalException( - f'Time in force policies are not supporetd for {self.name} yet.') - - def exchange_has(self, endpoint: str) -> bool: - """ - Checks if exchange implements a specific API endpoint. - Wrapper around ccxt 'has' attribute - :param endpoint: Name of endpoint (e.g. 'fetchOHLCV', 'fetchTickers') - :return: bool - """ - return endpoint in self._api.has and self._api.has[endpoint] - - def symbol_amount_prec(self, pair, amount: float): - ''' - Returns the amount to buy or sell to a precision the Exchange accepts - Rounded down - ''' - if self._api.markets[pair]['precision']['amount']: - symbol_prec = self._api.markets[pair]['precision']['amount'] - big_amount = amount * pow(10, symbol_prec) - amount = floor(big_amount) / pow(10, symbol_prec) - return amount - - def symbol_price_prec(self, pair, price: float): - ''' - Returns the price buying or selling with to the precision the Exchange accepts - Rounds up - ''' - if self._api.markets[pair]['precision']['price']: - symbol_prec = self._api.markets[pair]['precision']['price'] - big_price = price * pow(10, symbol_prec) - price = ceil(big_price) / pow(10, symbol_prec) - return price - - def buy(self, pair: str, ordertype: str, amount: float, - rate: float, time_in_force) -> Dict: - if self._conf['dry_run']: - order_id = f'dry_run_buy_{randint(0, 10**6)}' - self._dry_run_open_orders[order_id] = { - 'pair': pair, - 'price': rate, - 'amount': amount, - 'type': ordertype, - 'side': 'buy', - 'remaining': 0.0, - 'datetime': arrow.utcnow().isoformat(), - 'status': 'closed', - 'fee': None - } - return {'id': order_id} - - try: - # Set the precision for amount and price(rate) as accepted by the exchange - amount = self.symbol_amount_prec(pair, amount) - rate = self.symbol_price_prec(pair, rate) if ordertype != 'market' else None - - if time_in_force == 'gtc': - return self._api.create_order(pair, ordertype, 'buy', amount, rate) - else: - return self._api.create_order(pair, ordertype, 'buy', - amount, rate, {'timeInForce': time_in_force}) - - except ccxt.InsufficientFunds as e: - raise DependencyException( - f'Insufficient funds to create limit buy order on market {pair}.' - f'Tried to buy amount {amount} at rate {rate} (total {rate*amount}).' - f'Message: {e}') - except ccxt.InvalidOrder as e: - raise DependencyException( - f'Could not create limit buy order on market {pair}.' - f'Tried to buy amount {amount} at rate {rate} (total {rate*amount}).' - f'Message: {e}') - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not place buy order due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - - def sell(self, pair: str, ordertype: str, amount: float, - rate: float, time_in_force='gtc') -> Dict: - if self._conf['dry_run']: - order_id = f'dry_run_sell_{randint(0, 10**6)}' - self._dry_run_open_orders[order_id] = { - 'pair': pair, - 'price': rate, - 'amount': amount, - 'type': ordertype, - 'side': 'sell', - 'remaining': 0.0, - 'datetime': arrow.utcnow().isoformat(), - 'status': 'closed' - } - return {'id': order_id} - - try: - # Set the precision for amount and price(rate) as accepted by the exchange - amount = self.symbol_amount_prec(pair, amount) - rate = self.symbol_price_prec(pair, rate) if ordertype != 'market' else None - - if time_in_force == 'gtc': - return self._api.create_order(pair, ordertype, 'sell', amount, rate) - else: - return self._api.create_order(pair, ordertype, 'sell', - amount, rate, {'timeInForce': time_in_force}) - - except ccxt.InsufficientFunds as e: - raise DependencyException( - f'Insufficient funds to create limit sell order on market {pair}.' - f'Tried to sell amount {amount} at rate {rate} (total {rate*amount}).' - f'Message: {e}') - except ccxt.InvalidOrder as e: - raise DependencyException( - f'Could not create limit sell order on market {pair}.' - f'Tried to sell amount {amount} at rate {rate} (total {rate*amount}).' - f'Message: {e}') - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - - def stoploss_limit(self, pair: str, amount: float, stop_price: float, rate: float) -> Dict: - """ - creates a stoploss limit order. - NOTICE: it is not supported by all exchanges. only binance is tested for now. - """ - - # Set the precision for amount and price(rate) as accepted by the exchange - amount = self.symbol_amount_prec(pair, amount) - rate = self.symbol_price_prec(pair, rate) - stop_price = self.symbol_price_prec(pair, stop_price) - - # Ensure rate is less than stop price - if stop_price <= rate: - raise OperationalException( - 'In stoploss limit order, stop price should be more than limit price') - - if self._conf['dry_run']: - order_id = f'dry_run_buy_{randint(0, 10**6)}' - self._dry_run_open_orders[order_id] = { - 'info': {}, - 'id': order_id, - 'pair': pair, - 'price': stop_price, - 'amount': amount, - 'type': 'stop_loss_limit', - 'side': 'sell', - 'remaining': amount, - 'datetime': arrow.utcnow().isoformat(), - 'status': 'open', - 'fee': None - } - return self._dry_run_open_orders[order_id] - - try: - order = self._api.create_order(pair, 'stop_loss_limit', 'sell', - amount, rate, {'stopPrice': stop_price}) - logger.info('stoploss limit order added for %s. ' - 'stop price: %s. limit: %s' % (pair, stop_price, rate)) - return order - - except ccxt.InsufficientFunds as e: - raise DependencyException( - f'Insufficient funds to place stoploss limit order on market {pair}. ' - f'Tried to put a stoploss amount {amount} with ' - f'stop {stop_price} and limit {rate} (total {rate*amount}).' - f'Message: {e}') - except ccxt.InvalidOrder as e: - raise DependencyException( - f'Could not place stoploss limit order on market {pair}.' - f'Tried to place stoploss amount {amount} with ' - f'stop {stop_price} and limit {rate} (total {rate*amount}).' - f'Message: {e}') - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not place stoploss limit order due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - - @retrier - def get_balance(self, currency: str) -> float: - if self._conf['dry_run']: - return 999.9 - - # ccxt exception is already handled by get_balances - balances = self.get_balances() - balance = balances.get(currency) - if balance is None: - raise TemporaryError( - f'Could not get {currency} balance due to malformed exchange response: {balances}') - return balance['free'] - - @retrier - def get_balances(self) -> dict: - if self._conf['dry_run']: - return {} - - try: - balances = self._api.fetch_balance() - # Remove additional info from ccxt results - balances.pop("info", None) - balances.pop("free", None) - balances.pop("total", None) - balances.pop("used", None) - - return balances - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not get balance due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - - @retrier - def get_tickers(self) -> Dict: - try: - return self._api.fetch_tickers() - except ccxt.NotSupported as e: - raise OperationalException( - f'Exchange {self._api.name} does not support fetching tickers in batch.' - f'Message: {e}') - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not load tickers due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - - @retrier - def get_ticker(self, pair: str, refresh: Optional[bool] = True) -> dict: - if refresh or pair not in self._cached_ticker.keys(): - try: - if pair not in self._api.markets: - raise DependencyException(f"Pair {pair} not available") - data = self._api.fetch_ticker(pair) - try: - self._cached_ticker[pair] = { - 'bid': float(data['bid']), - 'ask': float(data['ask']), - } - except KeyError: - logger.debug("Could not cache ticker data for %s", pair) - return data - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - else: - logger.info("returning cached ticker-data for %s", pair) - return self._cached_ticker[pair] - - def get_history(self, pair: str, tick_interval: str, - since_ms: int) -> List: - """ - Gets candle history using asyncio and returns the list of candles. - Handles all async doing. - """ - return asyncio.get_event_loop().run_until_complete( - self._async_get_history(pair=pair, tick_interval=tick_interval, - since_ms=since_ms)) - - async def _async_get_history(self, pair: str, - tick_interval: str, - since_ms: int) -> List: - # Assume exchange returns 500 candles - _LIMIT = 500 - - one_call = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 * _LIMIT * 1000 - logger.debug("one_call: %s", one_call) - input_coroutines = [self._async_get_candle_history( - pair, tick_interval, since) for since in - range(since_ms, arrow.utcnow().timestamp * 1000, one_call)] - - tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) - - # Combine tickers - data: List = [] - for p, ticker_interval, ticker in tickers: - if p == pair: - data.extend(ticker) - # Sort data again after extending the result - above calls return in "async order" - data = sorted(data, key=lambda x: x[0]) - logger.info("downloaded %s with length %s.", pair, len(data)) - return data - - def refresh_latest_ohlcv(self, pair_list: List[Tuple[str, str]]) -> List[Tuple[str, List]]: - """ - Refresh in-memory ohlcv asyncronously and set `_klines` with the result - """ - logger.debug("Refreshing ohlcv data for %d pairs", len(pair_list)) - - input_coroutines = [] - - # Gather corotines to run - for pair, ticker_interval in set(pair_list): - # Calculating ticker interval in second - interval_in_sec = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60 - - if not ((self._pairs_last_refresh_time.get((pair, ticker_interval), 0) - + interval_in_sec) >= arrow.utcnow().timestamp - and (pair, ticker_interval) in self._klines): - input_coroutines.append(self._async_get_candle_history(pair, ticker_interval)) - else: - logger.debug("Using cached ohlcv data for %s, %s ...", pair, ticker_interval) - - tickers = asyncio.get_event_loop().run_until_complete( - asyncio.gather(*input_coroutines, return_exceptions=True)) - - # handle caching - for res in tickers: - if isinstance(res, Exception): - logger.warning("Async code raised an exception: %s", res.__class__.__name__) - continue - pair = res[0] - tick_interval = res[1] - ticks = res[2] - # keeping last candle time as last refreshed time of the pair - if ticks: - self._pairs_last_refresh_time[(pair, tick_interval)] = ticks[-1][0] // 1000 - # keeping parsed dataframe in cache - self._klines[(pair, tick_interval)] = parse_ticker_dataframe( - ticks, tick_interval, fill_missing=True) - return tickers - - @retrier_async - async def _async_get_candle_history(self, pair: str, tick_interval: str, - since_ms: Optional[int] = None) -> Tuple[str, str, List]: - """ - Asyncronously gets candle histories using fetch_ohlcv - returns tuple: (pair, tick_interval, ohlcv_list) - """ - try: - # fetch ohlcv asynchronously - logger.debug("fetching %s, %s since %s ...", pair, tick_interval, since_ms) - - data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, - since=since_ms) - - # Because some exchange sort Tickers ASC and other DESC. - # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) - # when GDAX returns a list of tickers DESC (newest first, oldest last) - # Only sort if necessary to save computing time - try: - if data and data[0][0] > data[-1][0]: - data = sorted(data, key=lambda x: x[0]) - except IndexError: - logger.exception("Error loading %s. Result was %s.", pair, data) - return pair, tick_interval, [] - logger.debug("done fetching %s, %s ...", pair, tick_interval) - return pair, tick_interval, data - - except ccxt.NotSupported as e: - raise OperationalException( - f'Exchange {self._api.name} does not support fetching historical candlestick data.' - f'Message: {e}') - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not load ticker history due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(f'Could not fetch ticker data. Msg: {e}') - - @retrier - def cancel_order(self, order_id: str, pair: str) -> None: - if self._conf['dry_run']: - return - - try: - return self._api.cancel_order(order_id, pair) - except ccxt.InvalidOrder as e: - raise DependencyException( - f'Could not cancel order. Message: {e}') - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - - @retrier - def get_order(self, order_id: str, pair: str) -> Dict: - if self._conf['dry_run']: - order = self._dry_run_open_orders[order_id] - order.update({ - 'id': order_id - }) - return order - try: - return self._api.fetch_order(order_id, pair) - except ccxt.InvalidOrder as e: - raise DependencyException( - f'Could not get order. Message: {e}') - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not get order due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - - @retrier - def get_order_book(self, pair: str, limit: int = 100) -> dict: - """ - get order book level 2 from exchange - - Notes: - 20180619: bittrex doesnt support limits -.- - 20180619: binance support limits but only on specific range - """ - try: - if self._api.name == 'Binance': - limit_range = [5, 10, 20, 50, 100, 500, 1000] - # get next-higher step in the limit_range list - limit = min(list(filter(lambda x: limit <= x, limit_range))) - # above script works like loop below (but with slightly better performance): - # for limitx in limit_range: - # if limit <= limitx: - # limit = limitx - # break - - return self._api.fetch_l2_order_book(pair, limit) - except ccxt.NotSupported as e: - raise OperationalException( - f'Exchange {self._api.name} does not support fetching order book.' - f'Message: {e}') - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not get order book due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - - @retrier - def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: - if self._conf['dry_run']: - return [] - if not self.exchange_has('fetchMyTrades'): - return [] - try: - # Allow 5s offset to catch slight time offsets (discovered in #1185) - my_trades = self._api.fetch_my_trades(pair, since.timestamp() - 5) - matched_trades = [trade for trade in my_trades if trade['order'] == order_id] - - return matched_trades - - except ccxt.NetworkError as e: - raise TemporaryError( - f'Could not get trades due to networking error. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - - def get_pair_detail_url(self, pair: str) -> str: - try: - url_base = self._api.urls.get('www') - base, quote = pair.split('/') - - return url_base + _EXCHANGE_URLS[self._api.id].format(base=base, quote=quote) - except KeyError: - logger.warning('Could not get exchange url for %s', self.name) - return "" - - @retrier - def get_markets(self) -> List[dict]: - try: - return self._api.fetch_markets() - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not load markets due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) - - @retrier - def get_fee(self, symbol='ETH/BTC', type='', side='', amount=1, - price=1, taker_or_maker='maker') -> float: - try: - # validate that markets are loaded before trying to get fee - if self._api.markets is None or len(self._api.markets) == 0: - self._api.load_markets() - - return self._api.calculate_fee(symbol=symbol, type=type, side=side, amount=amount, - price=price, takerOrMaker=taker_or_maker)['rate'] - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not get fee info due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(e) +from freqtrade.exchange.exchange import Exchange # noqa: F401 +from freqtrade.exchange.kraken import Kraken # noqa: F401 diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py new file mode 100644 index 000000000..25d3e62c0 --- /dev/null +++ b/freqtrade/exchange/exchange.py @@ -0,0 +1,739 @@ +# pragma pylint: disable=W0603 +""" Cryptocurrency Exchanges support """ +import logging +import inspect +from random import randint +from typing import List, Dict, Tuple, Any, Optional +from datetime import datetime +from math import floor, ceil + +import arrow +import asyncio +import ccxt +import ccxt.async_support as ccxt_async +from pandas import DataFrame + +from freqtrade import constants, OperationalException, DependencyException, TemporaryError +from freqtrade.data.converter import parse_ticker_dataframe + +logger = logging.getLogger(__name__) + +API_RETRY_COUNT = 4 + + +# Urls to exchange markets, insert quote and base with .format() +_EXCHANGE_URLS = { + ccxt.bittrex.__name__: '/Market/Index?MarketName={quote}-{base}', + ccxt.binance.__name__: '/tradeDetail.html?symbol={base}_{quote}' +} + + +def retrier_async(f): + async def wrapper(*args, **kwargs): + count = kwargs.pop('count', API_RETRY_COUNT) + try: + return await f(*args, **kwargs) + except (TemporaryError, DependencyException) as ex: + logger.warning('%s() returned exception: "%s"', f.__name__, ex) + if count > 0: + count -= 1 + kwargs.update({'count': count}) + logger.warning('retrying %s() still for %s times', f.__name__, count) + return await wrapper(*args, **kwargs) + else: + logger.warning('Giving up retrying: %s()', f.__name__) + raise ex + return wrapper + + +def retrier(f): + def wrapper(*args, **kwargs): + count = kwargs.pop('count', API_RETRY_COUNT) + try: + return f(*args, **kwargs) + except (TemporaryError, DependencyException) as ex: + logger.warning('%s() returned exception: "%s"', f.__name__, ex) + if count > 0: + count -= 1 + kwargs.update({'count': count}) + logger.warning('retrying %s() still for %s times', f.__name__, count) + return wrapper(*args, **kwargs) + else: + logger.warning('Giving up retrying: %s()', f.__name__) + raise ex + return wrapper + + +class Exchange(object): + + _conf: Dict = {} + _params: Dict = {} + + def __init__(self, config: dict) -> None: + """ + Initializes this module with the given config, + it does basic validation whether the specified + exchange and pairs are valid. + :return: None + """ + self._conf.update(config) + + self._cached_ticker: Dict[str, Any] = {} + + # Holds last candle refreshed time of each pair + self._pairs_last_refresh_time: Dict[Tuple[str, str], int] = {} + + # Holds candles + self._klines: Dict[Tuple[str, str], DataFrame] = {} + + # Holds all open sell orders for dry_run + self._dry_run_open_orders: Dict[str, Any] = {} + + if config['dry_run']: + logger.info('Instance is running with dry_run enabled') + + exchange_config = config['exchange'] + self._api: ccxt.Exchange = self._init_ccxt( + exchange_config, ccxt_kwargs=exchange_config.get('ccxt_config')) + self._api_async: ccxt_async.Exchange = self._init_ccxt( + exchange_config, ccxt_async, ccxt_kwargs=exchange_config.get('ccxt_async_config')) + + logger.info('Using Exchange "%s"', self.name) + + self.markets = self._load_markets() + # Check if all pairs are available + self.validate_pairs(config['exchange']['pair_whitelist']) + self.validate_ordertypes(config.get('order_types', {})) + self.validate_order_time_in_force(config.get('order_time_in_force', {})) + if config.get('ticker_interval'): + # Check if timeframe is available + self.validate_timeframes(config['ticker_interval']) + + def __del__(self): + """ + Destructor - clean up async stuff + """ + logger.debug("Exchange object destroyed, closing async loop") + if self._api_async and inspect.iscoroutinefunction(self._api_async.close): + asyncio.get_event_loop().run_until_complete(self._api_async.close()) + + def _init_ccxt(self, exchange_config: dict, ccxt_module=ccxt, + ccxt_kwargs: dict = None) -> ccxt.Exchange: + """ + Initialize ccxt with given config and return valid + ccxt instance. + """ + # Find matching class for the given exchange name + name = exchange_config['name'] + + if name not in ccxt_module.exchanges: + raise OperationalException(f'Exchange {name} is not supported') + + ex_config = { + 'apiKey': exchange_config.get('key'), + 'secret': exchange_config.get('secret'), + 'password': exchange_config.get('password'), + 'uid': exchange_config.get('uid', ''), + 'enableRateLimit': exchange_config.get('ccxt_rate_limit', True) + } + if ccxt_kwargs: + logger.info('Applying additional ccxt config: %s', ccxt_kwargs) + ex_config.update(ccxt_kwargs) + try: + + api = getattr(ccxt_module, name.lower())(ex_config) + except (KeyError, AttributeError): + raise OperationalException(f'Exchange {name} is not supported') + + self.set_sandbox(api, exchange_config, name) + + return api + + @property + def name(self) -> str: + """exchange Name (from ccxt)""" + return self._api.name + + @property + def id(self) -> str: + """exchange ccxt id""" + return self._api.id + + def klines(self, pair_interval: Tuple[str, str], copy=True) -> DataFrame: + if pair_interval in self._klines: + return self._klines[pair_interval].copy() if copy else self._klines[pair_interval] + else: + return DataFrame() + + def set_sandbox(self, api, exchange_config: dict, name: str): + if exchange_config.get('sandbox'): + if api.urls.get('test'): + api.urls['api'] = api.urls['test'] + logger.info("Enabled Sandbox API on %s", name) + else: + logger.warning(name, "No Sandbox URL in CCXT, exiting. " + "Please check your config.json") + raise OperationalException(f'Exchange {name} does not provide a sandbox api') + + def _load_async_markets(self) -> None: + try: + if self._api_async: + asyncio.get_event_loop().run_until_complete(self._api_async.load_markets()) + + except ccxt.BaseError as e: + logger.warning('Could not load async markets. Reason: %s', e) + return + + def _load_markets(self) -> Dict[str, Any]: + """ Initialize markets both sync and async """ + try: + markets = self._api.load_markets() + self._load_async_markets() + return markets + except ccxt.BaseError as e: + logger.warning('Unable to initialize markets. Reason: %s', e) + return {} + + def validate_pairs(self, pairs: List[str]) -> None: + """ + Checks if all given pairs are tradable on the current exchange. + Raises OperationalException if one pair is not available. + :param pairs: list of pairs + :return: None + """ + + if not self.markets: + logger.warning('Unable to validate pairs (assuming they are correct).') + # return + + stake_cur = self._conf['stake_currency'] + for pair in pairs: + # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs + # TODO: add a support for having coins in BTC/USDT format + if not pair.endswith(stake_cur): + raise OperationalException( + f'Pair {pair} not compatible with stake_currency: {stake_cur}') + if self.markets and pair not in self.markets: + raise OperationalException( + f'Pair {pair} is not available at {self.name}' + f'Please remove {pair} from your whitelist.') + + def validate_timeframes(self, timeframe: List[str]) -> None: + """ + Checks if ticker interval from config is a supported timeframe on the exchange + """ + timeframes = self._api.timeframes + if timeframe not in timeframes: + raise OperationalException( + f'Invalid ticker {timeframe}, this Exchange supports {timeframes}') + + def validate_ordertypes(self, order_types: Dict) -> None: + """ + Checks if order-types configured in strategy/config are supported + """ + if any(v == 'market' for k, v in order_types.items()): + if not self.exchange_has('createMarketOrder'): + raise OperationalException( + f'Exchange {self.name} does not support market orders.') + + if order_types.get('stoploss_on_exchange'): + if self.name != 'Binance': + raise OperationalException( + 'On exchange stoploss is not supported for %s.' % self.name + ) + + def validate_order_time_in_force(self, order_time_in_force: Dict) -> None: + """ + Checks if order time in force configured in strategy/config are supported + """ + if any(v != 'gtc' for k, v in order_time_in_force.items()): + if self.name != 'Binance': + raise OperationalException( + f'Time in force policies are not supporetd for {self.name} yet.') + + def exchange_has(self, endpoint: str) -> bool: + """ + Checks if exchange implements a specific API endpoint. + Wrapper around ccxt 'has' attribute + :param endpoint: Name of endpoint (e.g. 'fetchOHLCV', 'fetchTickers') + :return: bool + """ + return endpoint in self._api.has and self._api.has[endpoint] + + def symbol_amount_prec(self, pair, amount: float): + ''' + Returns the amount to buy or sell to a precision the Exchange accepts + Rounded down + ''' + if self._api.markets[pair]['precision']['amount']: + symbol_prec = self._api.markets[pair]['precision']['amount'] + big_amount = amount * pow(10, symbol_prec) + amount = floor(big_amount) / pow(10, symbol_prec) + return amount + + def symbol_price_prec(self, pair, price: float): + ''' + Returns the price buying or selling with to the precision the Exchange accepts + Rounds up + ''' + if self._api.markets[pair]['precision']['price']: + symbol_prec = self._api.markets[pair]['precision']['price'] + big_price = price * pow(10, symbol_prec) + price = ceil(big_price) / pow(10, symbol_prec) + return price + + def buy(self, pair: str, ordertype: str, amount: float, + rate: float, time_in_force) -> Dict: + if self._conf['dry_run']: + order_id = f'dry_run_buy_{randint(0, 10**6)}' + self._dry_run_open_orders[order_id] = { + 'pair': pair, + 'price': rate, + 'amount': amount, + 'type': ordertype, + 'side': 'buy', + 'remaining': 0.0, + 'datetime': arrow.utcnow().isoformat(), + 'status': 'closed', + 'fee': None + } + return {'id': order_id} + + try: + # Set the precision for amount and price(rate) as accepted by the exchange + amount = self.symbol_amount_prec(pair, amount) + rate = self.symbol_price_prec(pair, rate) if ordertype != 'market' else None + + params = self._params.copy() + if time_in_force != 'gtc': + params.update({'timeInForce': time_in_force}) + + return self._api.create_order(pair, ordertype, 'buy', + amount, rate, params) + + except ccxt.InsufficientFunds as e: + raise DependencyException( + f'Insufficient funds to create limit buy order on market {pair}.' + f'Tried to buy amount {amount} at rate {rate} (total {rate*amount}).' + f'Message: {e}') + except ccxt.InvalidOrder as e: + raise DependencyException( + f'Could not create limit buy order on market {pair}.' + f'Tried to buy amount {amount} at rate {rate} (total {rate*amount}).' + f'Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not place buy order due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + + def sell(self, pair: str, ordertype: str, amount: float, + rate: float, time_in_force='gtc') -> Dict: + if self._conf['dry_run']: + order_id = f'dry_run_sell_{randint(0, 10**6)}' + self._dry_run_open_orders[order_id] = { + 'pair': pair, + 'price': rate, + 'amount': amount, + 'type': ordertype, + 'side': 'sell', + 'remaining': 0.0, + 'datetime': arrow.utcnow().isoformat(), + 'status': 'closed' + } + return {'id': order_id} + + try: + # Set the precision for amount and price(rate) as accepted by the exchange + amount = self.symbol_amount_prec(pair, amount) + rate = self.symbol_price_prec(pair, rate) if ordertype != 'market' else None + + params = self._params.copy() + if time_in_force != 'gtc': + params.update({'timeInForce': time_in_force}) + + return self._api.create_order(pair, ordertype, 'sell', + amount, rate, params) + + except ccxt.InsufficientFunds as e: + raise DependencyException( + f'Insufficient funds to create limit sell order on market {pair}.' + f'Tried to sell amount {amount} at rate {rate} (total {rate*amount}).' + f'Message: {e}') + except ccxt.InvalidOrder as e: + raise DependencyException( + f'Could not create limit sell order on market {pair}.' + f'Tried to sell amount {amount} at rate {rate} (total {rate*amount}).' + f'Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + + def stoploss_limit(self, pair: str, amount: float, stop_price: float, rate: float) -> Dict: + """ + creates a stoploss limit order. + NOTICE: it is not supported by all exchanges. only binance is tested for now. + """ + + # Set the precision for amount and price(rate) as accepted by the exchange + amount = self.symbol_amount_prec(pair, amount) + rate = self.symbol_price_prec(pair, rate) + stop_price = self.symbol_price_prec(pair, stop_price) + + # Ensure rate is less than stop price + if stop_price <= rate: + raise OperationalException( + 'In stoploss limit order, stop price should be more than limit price') + + if self._conf['dry_run']: + order_id = f'dry_run_buy_{randint(0, 10**6)}' + self._dry_run_open_orders[order_id] = { + 'info': {}, + 'id': order_id, + 'pair': pair, + 'price': stop_price, + 'amount': amount, + 'type': 'stop_loss_limit', + 'side': 'sell', + 'remaining': amount, + 'datetime': arrow.utcnow().isoformat(), + 'status': 'open', + 'fee': None + } + return self._dry_run_open_orders[order_id] + + try: + + params = self._params.copy() + params.update({'stopPrice': stop_price}) + + order = self._api.create_order(pair, 'stop_loss_limit', 'sell', + amount, rate, params) + logger.info('stoploss limit order added for %s. ' + 'stop price: %s. limit: %s' % (pair, stop_price, rate)) + return order + + except ccxt.InsufficientFunds as e: + raise DependencyException( + f'Insufficient funds to place stoploss limit order on market {pair}. ' + f'Tried to put a stoploss amount {amount} with ' + f'stop {stop_price} and limit {rate} (total {rate*amount}).' + f'Message: {e}') + except ccxt.InvalidOrder as e: + raise DependencyException( + f'Could not place stoploss limit order on market {pair}.' + f'Tried to place stoploss amount {amount} with ' + f'stop {stop_price} and limit {rate} (total {rate*amount}).' + f'Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not place stoploss limit order due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + + @retrier + def get_balance(self, currency: str) -> float: + if self._conf['dry_run']: + return 999.9 + + # ccxt exception is already handled by get_balances + balances = self.get_balances() + balance = balances.get(currency) + if balance is None: + raise TemporaryError( + f'Could not get {currency} balance due to malformed exchange response: {balances}') + return balance['free'] + + @retrier + def get_balances(self) -> dict: + if self._conf['dry_run']: + return {} + + try: + balances = self._api.fetch_balance() + # Remove additional info from ccxt results + balances.pop("info", None) + balances.pop("free", None) + balances.pop("total", None) + balances.pop("used", None) + + return balances + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get balance due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + + @retrier + def get_tickers(self) -> Dict: + try: + return self._api.fetch_tickers() + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching tickers in batch.' + f'Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not load tickers due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + + @retrier + def get_ticker(self, pair: str, refresh: Optional[bool] = True) -> dict: + if refresh or pair not in self._cached_ticker.keys(): + try: + if pair not in self._api.markets: + raise DependencyException(f"Pair {pair} not available") + data = self._api.fetch_ticker(pair) + try: + self._cached_ticker[pair] = { + 'bid': float(data['bid']), + 'ask': float(data['ask']), + } + except KeyError: + logger.debug("Could not cache ticker data for %s", pair) + return data + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + else: + logger.info("returning cached ticker-data for %s", pair) + return self._cached_ticker[pair] + + def get_history(self, pair: str, tick_interval: str, + since_ms: int) -> List: + """ + Gets candle history using asyncio and returns the list of candles. + Handles all async doing. + """ + return asyncio.get_event_loop().run_until_complete( + self._async_get_history(pair=pair, tick_interval=tick_interval, + since_ms=since_ms)) + + async def _async_get_history(self, pair: str, + tick_interval: str, + since_ms: int) -> List: + # Assume exchange returns 500 candles + _LIMIT = 500 + + one_call = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 * _LIMIT * 1000 + logger.debug("one_call: %s", one_call) + input_coroutines = [self._async_get_candle_history( + pair, tick_interval, since) for since in + range(since_ms, arrow.utcnow().timestamp * 1000, one_call)] + + tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) + + # Combine tickers + data: List = [] + for p, ticker_interval, ticker in tickers: + if p == pair: + data.extend(ticker) + # Sort data again after extending the result - above calls return in "async order" + data = sorted(data, key=lambda x: x[0]) + logger.info("downloaded %s with length %s.", pair, len(data)) + return data + + def refresh_latest_ohlcv(self, pair_list: List[Tuple[str, str]]) -> List[Tuple[str, List]]: + """ + Refresh in-memory ohlcv asyncronously and set `_klines` with the result + """ + logger.debug("Refreshing ohlcv data for %d pairs", len(pair_list)) + + input_coroutines = [] + + # Gather coroutines to run + for pair, ticker_interval in set(pair_list): + if (not ((pair, ticker_interval) in self._klines) + or self._now_is_time_to_refresh(pair, ticker_interval)): + input_coroutines.append(self._async_get_candle_history(pair, ticker_interval)) + else: + logger.debug("Using cached ohlcv data for %s, %s ...", pair, ticker_interval) + + tickers = asyncio.get_event_loop().run_until_complete( + asyncio.gather(*input_coroutines, return_exceptions=True)) + + # handle caching + for res in tickers: + if isinstance(res, Exception): + logger.warning("Async code raised an exception: %s", res.__class__.__name__) + continue + pair = res[0] + tick_interval = res[1] + ticks = res[2] + # keeping last candle time as last refreshed time of the pair + if ticks: + self._pairs_last_refresh_time[(pair, tick_interval)] = ticks[-1][0] // 1000 + # keeping parsed dataframe in cache + self._klines[(pair, tick_interval)] = parse_ticker_dataframe( + ticks, tick_interval, fill_missing=True) + return tickers + + def _now_is_time_to_refresh(self, pair: str, ticker_interval: str) -> bool: + # Calculating ticker interval in seconds + interval_in_sec = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60 + + return not ((self._pairs_last_refresh_time.get((pair, ticker_interval), 0) + + interval_in_sec) >= arrow.utcnow().timestamp) + + @retrier_async + async def _async_get_candle_history(self, pair: str, tick_interval: str, + since_ms: Optional[int] = None) -> Tuple[str, str, List]: + """ + Asyncronously gets candle histories using fetch_ohlcv + returns tuple: (pair, tick_interval, ohlcv_list) + """ + try: + # fetch ohlcv asynchronously + logger.debug("fetching %s, %s since %s ...", pair, tick_interval, since_ms) + + data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, + since=since_ms) + + # Because some exchange sort Tickers ASC and other DESC. + # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) + # when GDAX returns a list of tickers DESC (newest first, oldest last) + # Only sort if necessary to save computing time + try: + if data and data[0][0] > data[-1][0]: + data = sorted(data, key=lambda x: x[0]) + except IndexError: + logger.exception("Error loading %s. Result was %s.", pair, data) + return pair, tick_interval, [] + logger.debug("done fetching %s, %s ...", pair, tick_interval) + return pair, tick_interval, data + + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching historical candlestick data.' + f'Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not load ticker history due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(f'Could not fetch ticker data. Msg: {e}') + + @retrier + def cancel_order(self, order_id: str, pair: str) -> None: + if self._conf['dry_run']: + return + + try: + return self._api.cancel_order(order_id, pair) + except ccxt.InvalidOrder as e: + raise DependencyException( + f'Could not cancel order. Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + + @retrier + def get_order(self, order_id: str, pair: str) -> Dict: + if self._conf['dry_run']: + order = self._dry_run_open_orders[order_id] + order.update({ + 'id': order_id + }) + return order + try: + return self._api.fetch_order(order_id, pair) + except ccxt.InvalidOrder as e: + raise DependencyException( + f'Could not get order. Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get order due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + + @retrier + def get_order_book(self, pair: str, limit: int = 100) -> dict: + """ + get order book level 2 from exchange + + Notes: + 20180619: bittrex doesnt support limits -.- + 20180619: binance support limits but only on specific range + """ + try: + if self._api.name == 'Binance': + limit_range = [5, 10, 20, 50, 100, 500, 1000] + # get next-higher step in the limit_range list + limit = min(list(filter(lambda x: limit <= x, limit_range))) + # above script works like loop below (but with slightly better performance): + # for limitx in limit_range: + # if limit <= limitx: + # limit = limitx + # break + + return self._api.fetch_l2_order_book(pair, limit) + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching order book.' + f'Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get order book due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + + @retrier + def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: + if self._conf['dry_run']: + return [] + if not self.exchange_has('fetchMyTrades'): + return [] + try: + # Allow 5s offset to catch slight time offsets (discovered in #1185) + my_trades = self._api.fetch_my_trades(pair, since.timestamp() - 5) + matched_trades = [trade for trade in my_trades if trade['order'] == order_id] + + return matched_trades + + except ccxt.NetworkError as e: + raise TemporaryError( + f'Could not get trades due to networking error. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + + def get_pair_detail_url(self, pair: str) -> str: + try: + url_base = self._api.urls.get('www') + base, quote = pair.split('/') + + return url_base + _EXCHANGE_URLS[self._api.id].format(base=base, quote=quote) + except KeyError: + logger.warning('Could not get exchange url for %s', self.name) + return "" + + @retrier + def get_markets(self) -> List[dict]: + try: + return self._api.fetch_markets() + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not load markets due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + + @retrier + def get_fee(self, symbol='ETH/BTC', type='', side='', amount=1, + price=1, taker_or_maker='maker') -> float: + try: + # validate that markets are loaded before trying to get fee + if self._api.markets is None or len(self._api.markets) == 0: + self._api.load_markets() + + return self._api.calculate_fee(symbol=symbol, type=type, side=side, amount=amount, + price=price, takerOrMaker=taker_or_maker)['rate'] + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get fee info due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py new file mode 100644 index 000000000..91b41a159 --- /dev/null +++ b/freqtrade/exchange/kraken.py @@ -0,0 +1,12 @@ +""" Kraken exchange subclass """ +import logging +from typing import Dict + +from freqtrade.exchange import Exchange + +logger = logging.getLogger(__name__) + + +class Kraken(Exchange): + + _params: Dict = {"trading_agreement": "agree"} diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f67be724c..92bdbc042 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -17,10 +17,9 @@ from freqtrade import (DependencyException, OperationalException, from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge -from freqtrade.exchange import Exchange from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType -from freqtrade.resolvers import StrategyResolver, PairListResolver +from freqtrade.resolvers import ExchangeResolver, StrategyResolver, PairListResolver from freqtrade.state import State from freqtrade.strategy.interface import SellType, IStrategy from freqtrade.wallets import Wallets @@ -55,7 +54,10 @@ class FreqtradeBot(object): self.strategy: IStrategy = StrategyResolver(self.config).strategy self.rpc: RPCManager = RPCManager(self) - self.exchange = Exchange(self.config) + + exchange_name = self.config.get('exchange', {}).get('name', 'bittrex').title() + self.exchange = ExchangeResolver(exchange_name, self.config).exchange + self.wallets = Wallets(self.exchange) self.dataprovider = DataProvider(self.config, self.exchange) diff --git a/freqtrade/resolvers/__init__.py b/freqtrade/resolvers/__init__.py index da2987b27..5cf6c616a 100644 --- a/freqtrade/resolvers/__init__.py +++ b/freqtrade/resolvers/__init__.py @@ -1,4 +1,5 @@ from freqtrade.resolvers.iresolver import IResolver # noqa: F401 +from freqtrade.resolvers.exchange_resolver import ExchangeResolver # noqa: F401 from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver # noqa: F401 from freqtrade.resolvers.pairlist_resolver import PairListResolver # noqa: F401 from freqtrade.resolvers.strategy_resolver import StrategyResolver # noqa: F401 diff --git a/freqtrade/resolvers/exchange_resolver.py b/freqtrade/resolvers/exchange_resolver.py new file mode 100644 index 000000000..8d1845c71 --- /dev/null +++ b/freqtrade/resolvers/exchange_resolver.py @@ -0,0 +1,55 @@ +""" +This module loads custom exchanges +""" +import logging + +from freqtrade.exchange import Exchange +import freqtrade.exchange as exchanges +from freqtrade.resolvers import IResolver + +logger = logging.getLogger(__name__) + + +class ExchangeResolver(IResolver): + """ + This class contains all the logic to load a custom exchange class + """ + + __slots__ = ['exchange'] + + def __init__(self, exchange_name: str, config: dict) -> None: + """ + Load the custom class from config parameter + :param config: configuration dictionary + """ + try: + self.exchange = self._load_exchange(exchange_name, kwargs={'config': config}) + except ImportError: + logger.info( + f"No {exchange_name} specific subclass found. Using the generic class instead.") + self.exchange = Exchange(config) + + def _load_exchange( + self, exchange_name: str, kwargs: dict) -> Exchange: + """ + Loads the specified exchange. + Only checks for exchanges exported in freqtrade.exchanges + :param exchange_name: name of the module to import + :return: Exchange instance or None + """ + + try: + ex_class = getattr(exchanges, exchange_name) + + exchange = ex_class(kwargs['config']) + if exchange: + logger.info("Using resolved exchange %s", exchange_name) + return exchange + except AttributeError: + # Pass and raise ImportError instead + pass + + raise ImportError( + "Impossible to load Exchange '{}'. This class does not exist" + " or contains Python code errors".format(exchange_name) + ) diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index 6bf7fa17d..e7683bc78 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -63,7 +63,7 @@ class HyperOptResolver(IResolver): hyperopt = self._search_object(directory=_path, object_type=IHyperOpt, object_name=hyperopt_name) if hyperopt: - logger.info('Using resolved hyperopt %s from \'%s\'', hyperopt_name, _path) + logger.info("Using resolved hyperopt %s from '%s'", hyperopt_name, _path) return hyperopt except FileNotFoundError: logger.warning('Path "%s" does not exist', _path.relative_to(Path.cwd())) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index aee292926..852d1dc0c 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -47,7 +47,7 @@ class IResolver(object): :param directory: relative or absolute directory path :return: object instance """ - logger.debug('Searching for %s %s in \'%s\'', object_type.__name__, object_name, directory) + logger.debug("Searching for %s %s in '%s'", object_type.__name__, object_name, directory) for entry in directory.iterdir(): # Only consider python files if not str(entry).endswith('.py'): diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index 286cea5bf..4306a9669 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -48,7 +48,7 @@ class PairListResolver(IResolver): object_name=pairlist_name, kwargs=kwargs) if pairlist: - logger.info('Using resolved pairlist %s from \'%s\'', pairlist_name, _path) + logger.info("Using resolved pairlist %s from '%s'", pairlist_name, _path) return pairlist except FileNotFoundError: logger.warning('Path "%s" does not exist', _path.relative_to(Path.cwd())) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 01467b0a1..c49da9205 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -149,7 +149,7 @@ class StrategyResolver(IResolver): strategy = self._search_object(directory=_path, object_type=IStrategy, object_name=strategy_name, kwargs={'config': config}) if strategy: - logger.info('Using resolved strategy %s from \'%s\'', strategy_name, _path) + logger.info("Using resolved strategy %s from '%s'", strategy_name, _path) strategy._populate_fun_len = len( getfullargspec(strategy.populate_indicators).args) strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 809dc12e0..d6628d925 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -16,6 +16,7 @@ from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.exchange import Exchange from freqtrade.edge import Edge, PairInfo from freqtrade.freqtradebot import FreqtradeBot +from freqtrade.resolvers import ExchangeResolver logging.getLogger('').setLevel(logging.INFO) @@ -49,7 +50,11 @@ def patch_exchange(mocker, api_mock=None, id='bittrex') -> None: def get_patched_exchange(mocker, config, api_mock=None, id='bittrex') -> Exchange: patch_exchange(mocker, api_mock, id) - exchange = Exchange(config) + config["exchange"]["name"] = id + try: + exchange = ExchangeResolver(id.title(), config).exchange + except ImportError: + exchange = Exchange(config) return exchange diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index b384035b0..15da5e924 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -12,8 +12,10 @@ import pytest from pandas import DataFrame from freqtrade import DependencyException, OperationalException, TemporaryError -from freqtrade.exchange import API_RETRY_COUNT, Exchange -from freqtrade.tests.conftest import get_patched_exchange, log_has +from freqtrade.exchange import Exchange, Kraken +from freqtrade.exchange.exchange import API_RETRY_COUNT +from freqtrade.tests.conftest import get_patched_exchange, log_has, log_has_re +from freqtrade.resolvers.exchange_resolver import ExchangeResolver # Source: https://stackoverflow.com/questions/29881236/how-to-mock-asyncio-coroutines @@ -106,6 +108,24 @@ def test_init_exception(default_conf, mocker): Exchange(default_conf) +def test_exchange_resolver(default_conf, mocker, caplog): + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=MagicMock())) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + exchange = ExchangeResolver('Binance', default_conf).exchange + assert isinstance(exchange, Exchange) + assert log_has_re(r"No .* specific subclass found. Using the generic class instead.", + caplog.record_tuples) + caplog.clear() + + exchange = ExchangeResolver('Kraken', default_conf).exchange + assert isinstance(exchange, Exchange) + assert isinstance(exchange, Kraken) + assert not log_has_re(r"No .* specific subclass found. Using the generic class instead.", + caplog.record_tuples) + + def test_symbol_amount_prec(default_conf, mocker): ''' Test rounds down to 4 Decimal places @@ -531,6 +551,67 @@ def test_buy_considers_time_in_force(default_conf, mocker): assert api_mock.create_order.call_args[0][5] == {'timeInForce': 'ioc'} +def test_buy_kraken_trading_agreement(default_conf, mocker): + api_mock = MagicMock() + order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_type = 'market' + time_in_force = 'ioc' + api_mock.create_order = MagicMock(return_value={ + 'id': order_id, + 'info': { + 'foo': 'bar' + } + }) + default_conf['dry_run'] = False + + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kraken") + + order = exchange.buy(pair='ETH/BTC', ordertype=order_type, + amount=1, rate=200, time_in_force=time_in_force) + + assert 'id' in order + assert 'info' in order + assert order['id'] == order_id + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' + assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'buy' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] is None + assert api_mock.create_order.call_args[0][5] == {'timeInForce': 'ioc', + 'trading_agreement': 'agree'} + + +def test_sell_kraken_trading_agreement(default_conf, mocker): + api_mock = MagicMock() + order_id = 'test_prod_sell_{}'.format(randint(0, 10 ** 6)) + order_type = 'market' + api_mock.create_order = MagicMock(return_value={ + 'id': order_id, + 'info': { + 'foo': 'bar' + } + }) + default_conf['dry_run'] = False + + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kraken") + + order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + + assert 'id' in order + assert 'info' in order + assert order['id'] == order_id + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' + assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'sell' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] is None + assert api_mock.create_order.call_args[0][5] == {'trading_agreement': 'agree'} + + def test_sell_dry_run(default_conf, mocker): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf) @@ -552,11 +633,12 @@ def test_sell_prod(default_conf, mocker): }) default_conf['dry_run'] = False - exchange = get_patched_exchange(mocker, default_conf, api_mock) mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + exchange = get_patched_exchange(mocker, default_conf, api_mock) order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + assert 'id' in order assert 'info' in order assert order['id'] == order_id @@ -978,7 +1060,7 @@ async def test___async_get_candle_history_sort(default_conf, mocker): ] exchange = get_patched_exchange(mocker, default_conf) exchange._api_async.fetch_ohlcv = get_mock_coro(tick) - sort_mock = mocker.patch('freqtrade.exchange.sorted', MagicMock(side_effect=sort_data)) + sort_mock = mocker.patch('freqtrade.exchange.exchange.sorted', MagicMock(side_effect=sort_data)) # Test the ticker history sort res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) assert res[0] == 'ETH/BTC' diff --git a/requirements-dev.txt b/requirements-dev.txt index 1bbf5cbe1..34d59d802 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,12 +1,12 @@ # Include all requirements to run the bot. -r requirements.txt -flake8==3.7.5 +flake8==3.7.6 flake8-type-annotations==0.1.0 flake8-tidy-imports==2.0.0 -pytest==4.2.1 +pytest==4.3.0 pytest-mock==1.10.1 pytest-asyncio==0.10.0 pytest-cov==2.6.1 -coveralls==1.5.1 +coveralls==1.6.0 mypy==0.670 diff --git a/requirements.txt b/requirements.txt index c9cedcb0c..cd1d89c6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.247 +ccxt==1.18.281 SQLAlchemy==1.2.18 python-telegram-bot==11.1.0 arrow==0.13.1