From e7d5cf9d9dde6bce0bbf643538c4873e53fb568b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 4 Oct 2018 20:11:02 +0200 Subject: [PATCH 1/7] Allow loading of any additional configuration to ccxt seperated by async and non-async --- freqtrade/exchange/__init__.py | 20 ++++++++++++----- freqtrade/tests/exchange/test_exchange.py | 27 +++++++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index d3c60c256..7fd0e5f43 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -93,8 +93,9 @@ class Exchange(object): logger.info('Instance is running with dry_run enabled') exchange_config = config['exchange'] - self._api = self._init_ccxt(exchange_config) - self._api_async = self._init_ccxt(exchange_config, ccxt_async) + self._api = self._init_ccxt(exchange_config, ccxt_kwargs=exchange_config.get('ccxt_config')) + self._api_async = self._init_ccxt(exchange_config, ccxt_async, + ccxt_kwargs=exchange_config.get('ccxt_async_config')) logger.info('Using Exchange "%s"', self.name) @@ -114,7 +115,8 @@ class Exchange(object): if self._api_async and inspect.iscoroutinefunction(self._api_async.close): asyncio.get_event_loop().run_until_complete(self._api_async.close()) - def _init_ccxt(self, exchange_config: dict, ccxt_module=ccxt) -> ccxt.Exchange: + def _init_ccxt(self, exchange_config: dict, ccxt_module=ccxt, + ccxt_kwargs: dict = None) -> ccxt.Exchange: """ Initialize ccxt with given config and return valid ccxt instance. @@ -124,14 +126,20 @@ class Exchange(object): if name not in ccxt_module.exchanges: raise OperationalException(f'Exchange {name} is not supported') - try: - api = getattr(ccxt_module, name.lower())({ + + ex_config = { 'apiKey': exchange_config.get('key'), 'secret': exchange_config.get('secret'), 'password': exchange_config.get('password'), 'uid': exchange_config.get('uid', ''), 'enableRateLimit': exchange_config.get('ccxt_rate_limit', True) - }) + } + if ccxt_kwargs: + logger.info('Applying additional ccxt config: %s', ccxt_kwargs) + ex_config.update(ccxt_kwargs) + try: + + api = getattr(ccxt_module, name.lower())(ex_config) except (KeyError, AttributeError): raise OperationalException(f'Exchange {name} is not supported') diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index d9d68c3b8..93cd1e546 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1,5 +1,6 @@ # pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement # pragma pylint: disable=protected-access +import copy import logging from datetime import datetime from random import randint @@ -56,6 +57,32 @@ def test_init(default_conf, mocker, caplog): assert log_has('Instance is running with dry_run enabled', caplog.record_tuples) +def test_init_ccxt_kwargs(default_conf, mocker, caplog): + mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) + caplog.set_level(logging.INFO) + conf = copy.deepcopy(default_conf) + conf['exchange']['ccxt_async_config'] = {'aiohttp_trust_env': True} + ex = Exchange(conf) + assert log_has("Applying additional ccxt config: {'aiohttp_trust_env': True}", + caplog.record_tuples) + assert ex._api_async.aiohttp_trust_env + assert not ex._api.aiohttp_trust_env + + # Reset logging and config + caplog.clear() + conf = copy.deepcopy(default_conf) + conf['exchange']['ccxt_config'] = {'TestKWARG': 11} + ex = Exchange(conf) + assert not log_has("Applying additional ccxt config: {'aiohttp_trust_env': True}", + caplog.record_tuples) + assert not ex._api_async.aiohttp_trust_env + assert hasattr(ex._api, 'TestKWARG') + assert ex._api.TestKWARG == 11 + assert not hasattr(ex._api_async, 'TestKWARG') + assert log_has("Applying additional ccxt config: {'TestKWARG': 11}", + caplog.record_tuples) + + def test_destroy(default_conf, mocker, caplog): caplog.set_level(logging.DEBUG) get_patched_exchange(mocker, default_conf) From d1edcf9dcd990c4b9ab0be0f65777b7eae419bfd Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 4 Oct 2018 20:17:19 +0200 Subject: [PATCH 2/7] Add documentation for ccxt_config --- config_full.json.example | 6 +++++- docs/configuration.md | 25 ++++++++++++++++++++++++- docs/telegram-usage.md | 6 ------ 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 7083bada6..71b119fb5 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -53,7 +53,11 @@ "pair_blacklist": [ "DOGE/BTC" ], - "outdated_offset": 5 + "outdated_offset": 5, + "ccxt_config": {}, + "ccxt_async_config": { + "aiohttp_trust_env": false + } }, "experimental": { "use_sell_signal": false, diff --git a/docs/configuration.md b/docs/configuration.md index 010e693d4..7c380e992 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -45,6 +45,8 @@ The table below will list all configuration parameters. | `exchange.pair_whitelist` | [] | No | List of currency to use by the bot. Can be overrided with `--dynamic-whitelist` param. | `exchange.pair_blacklist` | [] | No | List of currency the bot must avoid. Useful when using `--dynamic-whitelist` param. | `exchange.ccxt_rate_limit` | True | No | Have CCXT handle Exchange rate limits. Depending on the exchange, having this to false can lead to temporary bans from the exchange. +| `exchange.ccxt_config` | None | No | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) +| `exchange.ccxt_async_config` | None | No | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) | `experimental.use_sell_signal` | false | No | Use your sell strategy in addition of the `minimal_roi`. | `experimental.sell_profit_only` | false | No | waits until you have made a positive profit before taking a sell decision. | `experimental.ignore_roi_if_buy_signal` | false | No | Does not sell if the buy-signal is still active. Takes preference over `minimal_roi` and `use_sell_signal` @@ -204,8 +206,29 @@ you run it in production mode. } ``` + If you have not your Bittrex API key yet, [see our tutorial](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md). +### Using proxy with FreqTrade + +To use a proxy with freqtrade, add the kwarg `"aiohttp_trust_env"=true` to the `"ccxt_async_kwargs"` dict in the exchange section of the configuration. + +An example for this can be found in `config_full.json.example` + +``` json +"ccxt_async_config": { + "aiohttp_trust_env": true +} +``` + +Then, export your proxy settings using the variables `"HTTP_PROXY"` and `"HTTPS_PROXY"` set to the appropriate values + +``` bash +export HTTP_PROXY="http://addr:port" +export HTTPS_PROXY="http://addr:port" +freqtrade +``` + ### Embedding Strategies @@ -213,7 +236,7 @@ FreqTrade provides you with with an easy way to embed the strategy into your con This is done by utilizing BASE64 encoding and providing this string at the strategy configuration field, in your chosen config file. -##### Encoding a string as BASE64 +#### Encoding a string as BASE64 This is a quick example, how to generate the BASE64 string in python diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index ca764c9b4..945e31f9c 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -129,9 +129,3 @@ Day Profit BTC Profit USD ## /version > **Version:** `0.14.3` -### using proxy with telegram -``` -$ export HTTP_PROXY="http://addr:port" -$ export HTTPS_PROXY="http://addr:port" -$ freqtrade -``` From ddc15132866681063873d2c6dc0fe307cc0ca66d Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 4 Oct 2018 20:34:33 +0200 Subject: [PATCH 3/7] Add ccxt_config to both config_samples --- config.json.example | 5 ++++- config_full.json.example | 12 ++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/config.json.example b/config.json.example index 7a0bb6b9b..5198b9d81 100644 --- a/config.json.example +++ b/config.json.example @@ -28,7 +28,10 @@ "name": "bittrex", "key": "your_exchange_key", "secret": "your_exchange_secret", - "ccxt_rate_limit": true, + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": false + }, "pair_whitelist": [ "ETH/BTC", "LTC/BTC", diff --git a/config_full.json.example b/config_full.json.example index 71b119fb5..bdc6361d2 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -37,7 +37,11 @@ "name": "bittrex", "key": "your_exchange_key", "secret": "your_exchange_secret", - "ccxt_rate_limit": true, + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": false, + "aiohttp_trust_env": false + }, "pair_whitelist": [ "ETH/BTC", "LTC/BTC", @@ -53,11 +57,7 @@ "pair_blacklist": [ "DOGE/BTC" ], - "outdated_offset": 5, - "ccxt_config": {}, - "ccxt_async_config": { - "aiohttp_trust_env": false - } + "outdated_offset": 5 }, "experimental": { "use_sell_signal": false, From 37088cfb393d225c856bc9102aa44d9e7530f5bb Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 4 Oct 2018 20:34:48 +0200 Subject: [PATCH 4/7] add to constants --- freqtrade/constants.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index eadfa6eba..12f10d3b9 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -164,7 +164,9 @@ CONF_SCHEMA = { }, 'uniqueItems': True }, - 'outdated_offset': {'type': 'integer', 'minimum': 1} + 'outdated_offset': {'type': 'integer', 'minimum': 1}, + 'ccxt_config': {'type': 'object'}, + 'ccxt_async_config': {'type': 'object'} }, 'required': ['name', 'key', 'secret', 'pair_whitelist'] } From 3973d3697c7f964435af545bda25f2106988de2d Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 4 Oct 2018 20:35:28 +0200 Subject: [PATCH 5/7] deprecate ccxt_rate_limt --- docs/configuration.md | 2 +- freqtrade/configuration.py | 5 +++++ freqtrade/tests/test_configuration.py | 11 ++++++++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 7c380e992..3512f891b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -44,7 +44,7 @@ The table below will list all configuration parameters. | `exchange.secret` | secret | No | API secret to use for the exchange. Only required when you are in production mode. | `exchange.pair_whitelist` | [] | No | List of currency to use by the bot. Can be overrided with `--dynamic-whitelist` param. | `exchange.pair_blacklist` | [] | No | List of currency the bot must avoid. Useful when using `--dynamic-whitelist` param. -| `exchange.ccxt_rate_limit` | True | No | Have CCXT handle Exchange rate limits. Depending on the exchange, having this to false can lead to temporary bans from the exchange. +| `exchange.ccxt_rate_limit` | True | No | DEPRECATED!! Have CCXT handle Exchange rate limits. Depending on the exchange, having this to false can lead to temporary bans from the exchange. | `exchange.ccxt_config` | None | No | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) | `exchange.ccxt_async_config` | None | No | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) | `experimental.use_sell_signal` | false | No | Use your sell strategy in addition of the `minimal_roi`. diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 4e1137f33..93ae3deab 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -271,6 +271,11 @@ class Configuration(object): raise OperationalException( exception_msg ) + # Depreciation warning + if 'ccxt_rate_limit' in config.get('exchange', {}): + logger.warning("`ccxt_rate_limit` has been deprecated in favor of " + "`ccxt_config` and `ccxt_async_config` and will be removed " + "in future a future version.") logger.debug('Exchange "%s" supported', exchange) return True diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index bf41aab83..2c487c165 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -371,7 +371,7 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: assert log_has('Parameter -s/--spaces detected: [\'all\']', caplog.record_tuples) -def test_check_exchange(default_conf) -> None: +def test_check_exchange(default_conf, caplog) -> None: configuration = Configuration(Namespace()) # Test a valid exchange @@ -392,6 +392,15 @@ def test_check_exchange(default_conf) -> None: ): configuration.check_exchange(default_conf) + # Test ccxt_rate_limit depreciation + default_conf.get('exchange').update({'name': 'binance'}) + default_conf['exchange']['ccxt_rate_limit'] = True + configuration.check_exchange(default_conf) + assert log_has("`ccxt_rate_limit` has been deprecated in favor of " + "`ccxt_config` and `ccxt_async_config` and will be removed " + "in future a future version.", + caplog.record_tuples) + def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: mocker.patch('freqtrade.configuration.open', mocker.mock_open( From ce4f0696e1b46a9e1337095d5242da0d590e4344 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 4 Oct 2018 20:38:30 +0200 Subject: [PATCH 6/7] Add logging to download script and enable ccxt_async_config --- scripts/download_backtest_data.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 27c4c1e1c..b83eb2c4b 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -10,7 +10,14 @@ from freqtrade import arguments from freqtrade.arguments import TimeRange from freqtrade.exchange import Exchange from freqtrade.optimize import download_backtesting_testdata +from freqtrade.configuration import set_loggers +import logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', +) +set_loggers(0) DEFAULT_DL_PATH = 'user_data/data' @@ -54,7 +61,9 @@ exchange = Exchange({'key': '', 'exchange': { 'name': args.exchange, 'pair_whitelist': [], - 'ccxt_rate_limit': False + 'ccxt_async_config': { + "enableRateLimit": False + } } }) pairs_not_available = [] From 1d38c35e6a6f7db5d2e69f94f02f76fed6c897d3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 6 Oct 2018 09:23:40 +0200 Subject: [PATCH 7/7] Fix typo / word repetition --- freqtrade/configuration.py | 2 +- freqtrade/tests/test_configuration.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 93ae3deab..d61ff4dd7 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -275,7 +275,7 @@ class Configuration(object): if 'ccxt_rate_limit' in config.get('exchange', {}): logger.warning("`ccxt_rate_limit` has been deprecated in favor of " "`ccxt_config` and `ccxt_async_config` and will be removed " - "in future a future version.") + "in a future version.") logger.debug('Exchange "%s" supported', exchange) return True diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 2c487c165..547c38de9 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -398,7 +398,7 @@ def test_check_exchange(default_conf, caplog) -> None: configuration.check_exchange(default_conf) assert log_has("`ccxt_rate_limit` has been deprecated in favor of " "`ccxt_config` and `ccxt_async_config` and will be removed " - "in future a future version.", + "in a future version.", caplog.record_tuples)