diff --git a/README.md b/README.md index 1633bbf13..a46c1530f 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,10 @@ to understand the requirements before sending your pull-requests. ## Requirements +### Uptodate clock + +The clock must be accurate, syncronized to a NTP server very frequently to avoid problems with communication to the exchanges. + ### Min hardware required To run this bot we recommend you a cloud instance with a minimum of: diff --git a/docs/installation.md b/docs/installation.md index 0fecfcf78..1ceda6b1c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -267,7 +267,7 @@ Official webpage: https://mrjbq7.github.io/ta-lib/install.html wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz tar xvzf ta-lib-0.4.0-src.tar.gz cd ta-lib -sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h +sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h ./configure --prefix=/usr make make install diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 501c1784f..bb571b4ea 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -119,7 +119,6 @@ class Arguments(object): help='Override trades database URL, this is useful if dry_run is enabled' ' or in custom deployments (default: %(default)s)', dest='db_url', - default=constants.DEFAULT_DB_PROD_URL, type=str, metavar='PATH', ) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 3da432b1d..4e1137f33 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -110,9 +110,12 @@ class Configuration(object): '(not applicable with Backtesting and Hyperopt)' ) - if self.args.db_url != constants.DEFAULT_DB_PROD_URL: + if self.args.db_url and self.args.db_url != constants.DEFAULT_DB_PROD_URL: config.update({'db_url': self.args.db_url}) logger.info('Parameter --db-url detected ...') + else: + # Set default here + config.update({'db_url': constants.DEFAULT_DB_PROD_URL}) if config.get('dry_run', False): logger.info('Dry run is enabled') diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 5eaa2b377..d3c60c256 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -599,7 +599,8 @@ class Exchange(object): if not self.exchange_has('fetchMyTrades'): return [] try: - my_trades = self._api.fetch_my_trades(pair, since.timestamp()) + # Allow 5s offset to catch slight time offsets (discovered in #1185) + my_trades = self._api.fetch_my_trades(pair, since.timestamp() - 5) matched_trades = [trade for trade in my_trades if trade['order'] == order_id] return matched_trades diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 77e2b4915..cc9343659 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -10,7 +10,7 @@ from datetime import datetime from typing import Any, Callable, Dict, List, Optional import arrow -import requests +from requests.exceptions import RequestException from cachetools import TTLCache, cached @@ -667,7 +667,7 @@ class FreqtradeBot(object): if not trade.open_order_id: continue order = self.exchange.get_order(trade.open_order_id, trade.pair) - except requests.exceptions.RequestException: + except (RequestException, DependencyException): logger.info( 'Cannot query order for %s due to %s', trade, diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 535684b22..38df3cb38 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -2,38 +2,39 @@ from unittest.mock import MagicMock -import freqtrade.tests.conftest as tt # test tools +from freqtrade.tests.conftest import get_patched_freqtradebot + +import pytest # whitelist, blacklist, filtering, all of that will # eventually become some rules to run on a generic ACL engine # perhaps try to anticipate that by using some python package -def whitelist_conf(): - config = tt.default_conf() - config['stake_currency'] = 'BTC' - config['exchange']['pair_whitelist'] = [ +@pytest.fixture(scope="function") +def whitelist_conf(default_conf): + default_conf['stake_currency'] = 'BTC' + default_conf['exchange']['pair_whitelist'] = [ 'ETH/BTC', 'TKN/BTC', 'TRST/BTC', 'SWT/BTC', 'BCC/BTC' ] - config['exchange']['pair_blacklist'] = [ + default_conf['exchange']['pair_blacklist'] = [ 'BLK/BTC' ] - return config + return default_conf -def test_refresh_market_pair_not_in_whitelist(mocker, markets): - conf = whitelist_conf() +def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf): - freqtradebot = tt.get_patched_freqtradebot(mocker, conf) + freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) refreshedwhitelist = freqtradebot._refresh_whitelist( - conf['exchange']['pair_whitelist'] + ['XXX/BTC'] + whitelist_conf['exchange']['pair_whitelist'] + ['XXX/BTC'] ) # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] @@ -41,12 +42,12 @@ def test_refresh_market_pair_not_in_whitelist(mocker, markets): assert whitelist == refreshedwhitelist -def test_refresh_whitelist(mocker, markets): - conf = whitelist_conf() - freqtradebot = tt.get_patched_freqtradebot(mocker, conf) +def test_refresh_whitelist(mocker, markets, whitelist_conf): + freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) - refreshedwhitelist = freqtradebot._refresh_whitelist(conf['exchange']['pair_whitelist']) + refreshedwhitelist = freqtradebot._refresh_whitelist( + whitelist_conf['exchange']['pair_whitelist']) # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] @@ -54,9 +55,8 @@ def test_refresh_whitelist(mocker, markets): assert whitelist == refreshedwhitelist -def test_refresh_whitelist_dynamic(mocker, markets, tickers): - conf = whitelist_conf() - freqtradebot = tt.get_patched_freqtradebot(mocker, conf) +def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): + freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_markets=markets, @@ -68,21 +68,20 @@ def test_refresh_whitelist_dynamic(mocker, markets, tickers): whitelist = ['ETH/BTC', 'TKN/BTC'] refreshedwhitelist = freqtradebot._refresh_whitelist( - freqtradebot._gen_pair_whitelist(conf['stake_currency']) + freqtradebot._gen_pair_whitelist(whitelist_conf['stake_currency']) ) assert whitelist == refreshedwhitelist -def test_refresh_whitelist_dynamic_empty(mocker, markets_empty): - conf = whitelist_conf() - freqtradebot = tt.get_patched_freqtradebot(mocker, conf) +def test_refresh_whitelist_dynamic_empty(mocker, markets_empty, whitelist_conf): + freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets_empty) # argument: use the whitelist dynamically by exchange-volume whitelist = [] - conf['exchange']['pair_whitelist'] = [] + whitelist_conf['exchange']['pair_whitelist'] = [] freqtradebot._refresh_whitelist(whitelist) - pairslist = conf['exchange']['pair_whitelist'] + pairslist = whitelist_conf['exchange']['pair_whitelist'] assert set(whitelist) == set(pairslist) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index e558d726d..c302eeb43 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -993,6 +993,44 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fe assert nb_trades == 0 +def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_order_old, + fee, mocker) -> None: + rpc_mock = patch_RPCManager(mocker) + cancel_order_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_order=MagicMock(side_effect=DependencyException), + cancel_order=cancel_order_mock, + get_fee=fee + ) + freqtrade = FreqtradeBot(default_conf) + + trade_buy = Trade( + pair='ETH/BTC', + open_rate=0.00001099, + exchange='bittrex', + open_order_id='123456789', + amount=90.99181073, + fee_open=0.0, + fee_close=0.0, + stake_amount=1, + open_date=arrow.utcnow().shift(minutes=-601).datetime, + is_open=True + ) + + Trade.session.add(trade_buy) + + # check it does cancel buy orders over the time limit + freqtrade.check_handle_timedout() + assert cancel_order_mock.call_count == 0 + assert rpc_mock.call_count == 0 + trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() + nb_trades = len(trades) + assert nb_trades == 1 + + def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, mocker) -> None: rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() diff --git a/install_ta-lib.sh b/install_ta-lib.sh index 18e7b8bbb..d8ae2eeaa 100755 --- a/install_ta-lib.sh +++ b/install_ta-lib.sh @@ -1,6 +1,6 @@ if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then tar zxvf ta-lib-0.4.0-src.tar.gz - cd ta-lib && sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && ./configure && make && sudo make install && cd .. + cd ta-lib && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && ./configure && make && sudo make install && cd .. else echo "TA-lib already installed, skipping download and build." cd ta-lib && sudo make install && cd .. diff --git a/requirements.txt b/requirements.txt index 5b30ea0df..3b36fa5d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.350 +ccxt==1.17.351 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 0f0a3d4cb..68713f296 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -73,7 +73,7 @@ def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFram file = Path(args.exportfilename) # must align with columns in backtest.py columns = ["pair", "profit", "opents", "closets", "index", "duration", - "open_rate", "close_rate", "open_at_end"] + "open_rate", "close_rate", "open_at_end", "sell_reason"] with file.open() as f: data = json.load(f) trades = pd.DataFrame(data, columns=columns)