From f5a70750f0f19eb1867d52950c27971cf5caba91 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 26 Nov 2018 21:06:32 +0100 Subject: [PATCH 01/62] edge real position sizing drafted --- freqtrade/edge/__init__.py | 16 +++++++++++++--- freqtrade/freqtradebot.py | 4 +++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 009b80664..6be09b19d 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -9,6 +9,7 @@ import utils_find_1st as utf1st from pandas import DataFrame import freqtrade.optimize as optimize +from freqtrade import constants, OperationalException from freqtrade.arguments import Arguments from freqtrade.arguments import TimeRange from freqtrade.strategy.interface import SellType @@ -53,7 +54,15 @@ class Edge(): self.edge_config = self.config.get('edge', {}) self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs - self._total_capital: float = self.config['stake_amount'] + # checking max_open_trades. it should be -1 as with Edge + # the number of trades is determined by position size + if self.config['max_open_trades'] != -1: + logger.critical('max_open_trades should be -1 in config !') + + if self.config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT: + raise OperationalException('Edge works only with unlimited stake amount') + + self._capital_percentage: float = self.edge_config.get('capital_available_percentage') self._allowed_risk: float = self.edge_config.get('allowed_risk') self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14) self._last_updated: int = 0 # Timestamp of pairs last updated time @@ -150,9 +159,10 @@ class Edge(): return True - def stake_amount(self, pair: str) -> float: + def stake_amount(self, pair: str, capital: float) -> float: stoploss = self._cached_pairs[pair].stoploss - allowed_capital_at_risk = round(self._total_capital * self._allowed_risk, 5) + available_capital = capital * self._capital_percentage + allowed_capital_at_risk = round(available_capital * self._allowed_risk, 5) position_size = abs(round((allowed_capital_at_risk / stoploss), 5)) return position_size diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 602c4ae2f..a6d2df954 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -333,7 +333,9 @@ class FreqtradeBot(object): :return: float: Stake Amount """ if self.edge: - stake_amount = self.edge.stake_amount(pair) + stake_amount = self.edge.stake_amount( + pair, self.wallets.get_free(self.config['stake_currency']) + ) else: stake_amount = self.config['stake_amount'] From 159ac6e6573a9d44a244482739cf199eff1155d4 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 27 Nov 2018 14:02:34 +0100 Subject: [PATCH 02/62] edge tests fixed for position sizing --- freqtrade/edge/__init__.py | 6 +++--- freqtrade/freqtradebot.py | 4 +--- freqtrade/tests/conftest.py | 4 ++++ freqtrade/tests/edge/test_edge.py | 20 ++++++++++---------- freqtrade/tests/test_freqtradebot.py | 4 ++-- 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 6be09b19d..f5961d9b4 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -162,8 +162,8 @@ class Edge(): def stake_amount(self, pair: str, capital: float) -> float: stoploss = self._cached_pairs[pair].stoploss available_capital = capital * self._capital_percentage - allowed_capital_at_risk = round(available_capital * self._allowed_risk, 5) - position_size = abs(round((allowed_capital_at_risk / stoploss), 5)) + allowed_capital_at_risk = round(available_capital * self._allowed_risk, 15) + position_size = abs(round((allowed_capital_at_risk / stoploss), 15)) return position_size def stoploss(self, pair: str) -> float: @@ -207,7 +207,7 @@ class Edge(): # 0.05% is 0.0005 # fee = 0.001 - stake = self.config.get('stake_amount') + stake = 0.015 fee = self.fee open_fee = fee / 2 diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a6d2df954..937fa0d3f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -336,6 +336,7 @@ class FreqtradeBot(object): stake_amount = self.edge.stake_amount( pair, self.wallets.get_free(self.config['stake_currency']) ) + return stake_amount else: stake_amount = self.config['stake_amount'] @@ -782,9 +783,6 @@ class FreqtradeBot(object): if sell_reason in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): sell_type = 'stoploss' - if self.config.get('dry_run', False) and sell_type == 'stoploss': - limit = trade.stop_loss - # Execute sell and update trade record order_id = self.exchange.sell(pair=str(trade.pair), ordertype=self.strategy.order_types[sell_type], diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index f7fe697b8..75c8032f1 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -10,6 +10,7 @@ import arrow import pytest from telegram import Chat, Message, Update +from freqtrade import constants from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe from freqtrade.exchange import Exchange from freqtrade.edge import Edge, PairInfo @@ -787,10 +788,13 @@ def buy_order_fee(): @pytest.fixture(scope="function") def edge_conf(default_conf): + default_conf['max_open_trades'] = -1 + default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT default_conf['edge'] = { "enabled": True, "process_throttle_secs": 1800, "calculate_since_number_of_days": 14, + "capital_available_percentage": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index fac055c17..40bb35209 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -123,9 +123,9 @@ def test_edge_results(edge_conf, mocker, caplog, data) -> None: assert res.close_time == _get_frame_time_from_offset(trade.close_tick) -def test_adjust(mocker, default_conf): - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) +def test_adjust(mocker, edge_conf): + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ 'E/F': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), @@ -138,9 +138,9 @@ def test_adjust(mocker, default_conf): assert(edge.adjust(pairs) == ['E/F', 'C/D']) -def test_stoploss(mocker, default_conf): - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) +def test_stoploss(mocker, edge_conf): + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ 'E/F': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), @@ -234,12 +234,12 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals return pairdata -def test_edge_process_downloaded_data(mocker, default_conf): - default_conf['datadir'] = None - freqtrade = get_patched_freqtradebot(mocker, default_conf) +def test_edge_process_downloaded_data(mocker, edge_conf): + edge_conf['datadir'] = None + freqtrade = get_patched_freqtradebot(mocker, edge_conf) mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) mocker.patch('freqtrade.optimize.load_data', mocked_load_data) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) assert edge.calculate() assert len(edge._cached_pairs) == 2 diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index eb5336c61..04d00d00a 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -260,8 +260,8 @@ def test_edge_overrides_stake_amount(mocker, edge_conf) -> None: patch_edge(mocker) freqtrade = FreqtradeBot(edge_conf) - assert freqtrade._get_trade_stake_amount('NEO/BTC') == (0.001 * 0.01) / 0.20 - assert freqtrade._get_trade_stake_amount('LTC/BTC') == (0.001 * 0.01) / 0.20 + assert freqtrade._get_trade_stake_amount('NEO/BTC') == (999.9 * 0.5 * 0.01) / 0.20 + assert freqtrade._get_trade_stake_amount('LTC/BTC') == (999.9 * 0.5 * 0.01) / 0.21 def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, edge_conf) -> None: From e9305b6592ddcba35fef7d647f8d58ef79ccf40a Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 28 Nov 2018 15:36:32 +0100 Subject: [PATCH 03/62] position size fixed --- freqtrade/edge/__init__.py | 11 ++++++++--- freqtrade/freqtradebot.py | 4 +++- freqtrade/wallets.py | 22 ++++++++++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index f5961d9b4..f7e8e5ea9 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -159,11 +159,16 @@ class Edge(): return True - def stake_amount(self, pair: str, capital: float) -> float: + def stake_amount(self, pair: str, free_capital: float, total_capital: float) -> float: stoploss = self._cached_pairs[pair].stoploss - available_capital = capital * self._capital_percentage + available_capital = total_capital * self._capital_percentage allowed_capital_at_risk = round(available_capital * self._allowed_risk, 15) - position_size = abs(round((allowed_capital_at_risk / stoploss), 15)) + max_position_size = abs(round((allowed_capital_at_risk / stoploss), 15)) + position_size = min(max_position_size, free_capital) + logger.info( + 'position size is %s for pair %s, stoploss %s and available capital of %s.', + position_size, pair, stoploss, available_capital + ) return position_size def stoploss(self, pair: str) -> float: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 937fa0d3f..90f100eac 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -334,7 +334,9 @@ class FreqtradeBot(object): """ if self.edge: stake_amount = self.edge.stake_amount( - pair, self.wallets.get_free(self.config['stake_currency']) + pair, + self.wallets.get_free(self.config['stake_currency']), + self.wallets.get_total(self.config['stake_currency']) ) return stake_amount else: diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index bf6f8b027..59d8fa3da 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -40,6 +40,28 @@ class Wallets(object): else: return 0 + def get_used(self, currency) -> float: + + if self.exchange._conf['dry_run']: + return 999.9 + + balance = self.wallets.get(currency) + if balance and balance.used: + return balance.used + else: + return 0 + + def get_total(self, currency) -> float: + + if self.exchange._conf['dry_run']: + return 999.9 + + balance = self.wallets.get(currency) + if balance and balance.total: + return balance.total + else: + return 0 + def update(self) -> None: balances = self.exchange.get_balances() From e698590bb25a3327b68a30897ae96bdec052cc1d Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 28 Nov 2018 20:04:56 +0100 Subject: [PATCH 04/62] avoid generating logs on each iteration --- freqtrade/edge/__init__.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index f7e8e5ea9..0b4227ed0 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -53,6 +53,7 @@ class Edge(): self.edge_config = self.config.get('edge', {}) self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs + self._final: list = [] # checking max_open_trades. it should be -1 as with Edge # the number of trades is determined by position size @@ -178,7 +179,6 @@ class Edge(): """ Filters out and sorts "pairs" according to Edge calculated pairs """ - final = [] for pair, info in self._cached_pairs.items(): if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ @@ -186,12 +186,14 @@ class Edge(): pair in pairs: final.append(pair) - if final: - logger.info('Edge validated only %s', final) - else: - logger.info('Edge removed all pairs as no pair with minimum expectancy was found !') + if self._final != final: + self._final = final + if self._final: + logger.info('Edge validated only %s', self._final) + else: + logger.info('Edge removed all pairs as no pair with minimum expectancy was found !') - return final + return self._final def _fill_calculable_fields(self, result: DataFrame) -> DataFrame: """ From a61daed8e95915f56ce36d5027335fac9b64f4bd Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 29 Nov 2018 12:24:04 +0100 Subject: [PATCH 05/62] logs enriched --- freqtrade/edge/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 0b4227ed0..f27309479 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -167,7 +167,10 @@ class Edge(): max_position_size = abs(round((allowed_capital_at_risk / stoploss), 15)) position_size = min(max_position_size, free_capital) logger.info( - 'position size is %s for pair %s, stoploss %s and available capital of %s.', + 'winrate: %s, expectancy: %s, position size: %s, pair: %s,' + ' stoploss: %s, available capital: %s.', + self._cached_pairs[pair].winrate, + self._cached_pairs[pair].expectancy, position_size, pair, stoploss, available_capital ) return position_size From 7767470af845279df6bea54436bf534ed7d41583 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 30 Nov 2018 17:50:03 +0100 Subject: [PATCH 06/62] =?UTF-8?q?return=20stake=20amount=20of=20strategy?= =?UTF-8?q?=20if=20edge=20doesn=E2=80=99t=20have=20any?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/edge/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index f27309479..74fabc409 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -161,6 +161,11 @@ class Edge(): return True def stake_amount(self, pair: str, free_capital: float, total_capital: float) -> float: + if pair not in self._cached_pairs: + logger.warning("cannot find %s in calculated pairs, " + "stake_amount of strategy is used instead.", pair) + return self.strategy.stake_amount + stoploss = self._cached_pairs[pair].stoploss available_capital = total_capital * self._capital_percentage allowed_capital_at_risk = round(available_capital * self._allowed_risk, 15) From 12471e012e1dd43155c84c063f33e6b7c4013dd6 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 30 Nov 2018 17:59:51 +0100 Subject: [PATCH 07/62] added tests for position sizing --- freqtrade/edge/__init__.py | 2 +- freqtrade/tests/edge/test_edge.py | 40 ++++++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 4800691e8..b76821da0 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -164,7 +164,7 @@ class Edge(): if pair not in self._cached_pairs: logger.warning("cannot find %s in calculated pairs, " "stake_amount of strategy is used instead.", pair) - return self.strategy.stake_amount + return self.config['stake_amount'] stoploss = self._cached_pairs[pair].stoploss available_capital = total_capital * self._capital_percentage diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index d5457f342..d453f6f27 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -5,6 +5,7 @@ import pytest import logging from freqtrade.tests.conftest import get_patched_freqtradebot from freqtrade.edge import Edge, PairInfo +from freqtrade import constants from pandas import DataFrame, to_datetime from freqtrade.strategy.interface import SellType from freqtrade.tests.optimize import (BTrade, BTContainer, _build_backtest_dataframe, @@ -152,9 +153,9 @@ def test_stoploss(mocker, edge_conf): assert edge.stoploss('E/F') == -0.01 -def test_nonexisting_stoploss(mocker, default_conf): - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) +def test_nonexisting_stoploss(mocker, edge_conf): + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ 'E/F': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), @@ -164,6 +165,39 @@ def test_nonexisting_stoploss(mocker, default_conf): assert edge.stoploss('N/O') == -0.1 +def test_stake_amount(mocker, edge_conf): + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) + mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( + return_value={ + 'E/F': PairInfo(-0.02, 0.66, 3.71, 0.50, 1.71, 10, 60), + } + )) + free = 100 + total = 100 + assert edge.stake_amount('E/F', free, total) == 25 + + free = 20 + total = 100 + assert edge.stake_amount('E/F', free, total) == 20 + + free = 0 + total = 100 + assert edge.stake_amount('E/F', free, total) == 0 + + +def test_nonexisting_stake_amount(mocker, edge_conf): + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) + mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( + return_value={ + 'E/F': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + } + )) + + assert edge.stake_amount('N/O', 1, 2) == constants.UNLIMITED_STAKE_AMOUNT + + def _validate_ohlc(buy_ohlc_sell_matrice): for index, ohlc in enumerate(buy_ohlc_sell_matrice): # if not high < open < low or not high < close < low From 11101e66686e858a74b9e405a6bf870e90f582da Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 30 Nov 2018 18:07:45 +0100 Subject: [PATCH 08/62] config full aded --- config_binance.json.example | 2 +- config_full.json.example | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/config_binance.json.example b/config_binance.json.example index 7773a8c39..3d11f317a 100644 --- a/config_binance.json.example +++ b/config_binance.json.example @@ -59,7 +59,7 @@ "enabled": false, "process_throttle_secs": 3600, "calculate_since_number_of_days": 7, - "total_capital_in_stake_currency": 0.5, + "capital_available_percentage": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, diff --git a/config_full.json.example b/config_full.json.example index b0719bcc6..7e54eb5ca 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -67,7 +67,8 @@ "edge": { "enabled": false, "process_throttle_secs": 3600, - "calculate_since_number_of_days": 2, + "calculate_since_number_of_days": 7, + "capital_available_percentage": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, From aadc9f052ab63d9a168f3518a4891cc4718ea985 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 30 Nov 2018 18:10:22 +0100 Subject: [PATCH 09/62] conf schema --- freqtrade/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 055fee3b2..82f392990 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -191,6 +191,7 @@ CONF_SCHEMA = { "process_throttle_secs": {'type': 'integer', 'minimum': 600}, "calculate_since_number_of_days": {'type': 'integer'}, "allowed_risk": {'type': 'number'}, + "capital_available_percentage": {'type': 'number'}, "stoploss_range_min": {'type': 'number'}, "stoploss_range_max": {'type': 'number'}, "stoploss_range_step": {'type': 'number'}, From c61ede4182048611d74436bc0c610cc350ae0a76 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 30 Nov 2018 18:20:29 +0100 Subject: [PATCH 10/62] documentation updated --- docs/edge.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index e5575554b..499b91e7b 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -82,9 +82,7 @@ Edge dictates the stake amount for each trade to the bot according to the follow Allowed capital at risk is calculated as follows: -**allowed capital at risk** = **total capital** X **allowed risk per trade** - -**total capital** is your stake amount. +**allowed capital at risk** = **capital_available_percentage** X **allowed risk per trade** **Stoploss** is calculated as described above against historical data. @@ -93,7 +91,7 @@ Your position size then will be: **position size** = **allowed capital at risk** / **stoploss** Example: -Let's say your stake amount is 3 ETH, you would allow 1% of risk for each trade. thus your allowed capital at risk would be **3 x 0.01 = 0.03 ETH**. Let's assume Edge has calculated that for **XLM/ETH** market your stoploss should be at 2%. So your position size will be **0.03 / 0.02= 1.5ETH**.
+Let's say the stake currency is ETH and you have 10 ETH on the exchange, your **capital_available_percentage** is 50% and you would allow 1% of risk for each trade. thus your available capital for trading is **10 x 0.5 = 5 ETH** and allowed capital at risk would be **5 x 0.01 = 0.05 ETH**. Let's assume Edge has calculated that for **XLM/ETH** market your stoploss should be at 2%. So your position size will be **0.05 / 0.02= 2.5ETH**.
## Configurations Edge has following configurations: @@ -111,6 +109,11 @@ Number of days of data against which Edge calculates Win Rate, Risk Reward and E Note that it downloads historical data so increasing this number would lead to slowing down the bot
(default to 7) +#### capital_available_percentage +This is the percentage of the total capital on exchange in stake currency.
+As an example if you have 100 USDT available in your wallet on the exchange and this value is 0.5 (which is 50%), then the bot will use a maximum amount of 50 USDT for trading and considers it as available capital. +(default to 0.5) + #### allowed_risk Percentage of allowed risk per trade
(default to 0.01 [1%]) From 7e86ec31be1e58ed93808ba660bc670ddc947859 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 30 Nov 2018 18:23:16 +0100 Subject: [PATCH 11/62] tests added for wallet additional functions --- freqtrade/tests/test_wallets.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/tests/test_wallets.py b/freqtrade/tests/test_wallets.py index 88366a869..8d9adc74c 100644 --- a/freqtrade/tests/test_wallets.py +++ b/freqtrade/tests/test_wallets.py @@ -58,6 +58,8 @@ def test_sync_wallet_at_boot(mocker, default_conf): assert freqtrade.wallets.wallets['GAS'].used == 0.1 assert freqtrade.wallets.wallets['GAS'].total == 0.260439 assert freqtrade.wallets.get_free('GAS') == 0.270739 + assert freqtrade.wallets.get_used('GAS') == 0.1 + assert freqtrade.wallets.get_total('GAS') == 0.260439 def test_sync_wallet_missing_data(mocker, default_conf): From 4a2d60370c652884407c0ce66ce90c2021a73d9a Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 30 Nov 2018 18:28:18 +0100 Subject: [PATCH 12/62] adding dots at the end of sentences --- docs/edge.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index 499b91e7b..27afcd507 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -97,7 +97,7 @@ Let's say the stake currency is ETH and you have 10 ETH on the exchange, your ** Edge has following configurations: #### enabled -If true, then Edge will run periodically
+If true, then Edge will run periodically.
(default to false) #### process_throttle_secs @@ -106,24 +106,24 @@ How often should Edge run in seconds?
#### calculate_since_number_of_days Number of days of data against which Edge calculates Win Rate, Risk Reward and Expectancy -Note that it downloads historical data so increasing this number would lead to slowing down the bot
+Note that it downloads historical data so increasing this number would lead to slowing down the bot.
(default to 7) #### capital_available_percentage This is the percentage of the total capital on exchange in stake currency.
-As an example if you have 100 USDT available in your wallet on the exchange and this value is 0.5 (which is 50%), then the bot will use a maximum amount of 50 USDT for trading and considers it as available capital. +As an example if you have 100 USDT available in your wallet on the exchange and this value is 0.5 (which is 50%), then the bot will use a maximum amount of 50 USDT for trading and considers it as available capital.
(default to 0.5) #### allowed_risk -Percentage of allowed risk per trade
+Percentage of allowed risk per trade.
(default to 0.01 [1%]) #### stoploss_range_min -Minimum stoploss
+Minimum stoploss.
(default to -0.01) #### stoploss_range_max -Maximum stoploss
+Maximum stoploss.
(default to -0.10) #### stoploss_range_step From 9c987fdedd3b22794027af01d2ad01df22c71f54 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 10:56:33 +0100 Subject: [PATCH 13/62] variable name changed (_final_pairs) --- freqtrade/edge/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index b76821da0..4c7ac7950 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -53,7 +53,7 @@ class Edge(): self.edge_config = self.config.get('edge', {}) self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs - self._final: list = [] + self._final_pairs: list = [] # checking max_open_trades. it should be -1 as with Edge # the number of trades is determined by position size @@ -199,14 +199,14 @@ class Edge(): pair in pairs: final.append(pair) - if self._final != final: - self._final = final - if self._final: - logger.info('Edge validated only %s', self._final) + if self._final_pairs != final: + self._final_pairs = final + if self._final_pairs: + logger.info('Edge validated only %s', self._final_pairs) else: logger.info('Edge removed all pairs as no pair with minimum expectancy was found !') - return self._final + return self._final_pairs def _fill_calculable_fields(self, result: DataFrame) -> DataFrame: """ From 86d9457ea13f1f802646400ac7dbc991637d9daf Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 10:58:05 +0100 Subject: [PATCH 14/62] removing unnecessary variable before returning the result --- freqtrade/freqtradebot.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 66e32ec4b..e687b4a8d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -333,12 +333,11 @@ class FreqtradeBot(object): :return: float: Stake Amount """ if self.edge: - stake_amount = self.edge.stake_amount( + return self.edge.stake_amount( pair, self.wallets.get_free(self.config['stake_currency']), self.wallets.get_total(self.config['stake_currency']) ) - return stake_amount else: stake_amount = self.config['stake_amount'] From c4f17f1c4588a61cbc4e15d4ea1a6d7d66a6fefa Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 10:58:47 +0100 Subject: [PATCH 15/62] config json updated --- config.json.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.json.example b/config.json.example index bbd9648da..323ff711e 100644 --- a/config.json.example +++ b/config.json.example @@ -57,7 +57,7 @@ "enabled": false, "process_throttle_secs": 3600, "calculate_since_number_of_days": 7, - "total_capital_in_stake_currency": 0.5, + "capital_available_percentage": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, From 9c0be99ff7b4e99bbe88d2b0cee3fb1ce1d3d0e8 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 11:00:33 +0100 Subject: [PATCH 16/62] rounding float at the end --- freqtrade/edge/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 4c7ac7950..a7bf2199a 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -168,8 +168,8 @@ class Edge(): stoploss = self._cached_pairs[pair].stoploss available_capital = total_capital * self._capital_percentage - allowed_capital_at_risk = round(available_capital * self._allowed_risk, 15) - max_position_size = abs(round((allowed_capital_at_risk / stoploss), 15)) + allowed_capital_at_risk = available_capital * self._allowed_risk + max_position_size = abs(allowed_capital_at_risk / stoploss) position_size = min(max_position_size, free_capital) logger.info( 'winrate: %s, expectancy: %s, position size: %s, pair: %s,' @@ -178,7 +178,7 @@ class Edge(): self._cached_pairs[pair].expectancy, position_size, pair, stoploss, available_capital ) - return position_size + return round(position_size, 15) def stoploss(self, pair: str) -> float: if pair in self._cached_pairs: From 88d277ea557ba70156ae30e2ebc71817a6bbd22d Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 11:08:18 +0100 Subject: [PATCH 17/62] adding required config for edge --- freqtrade/constants.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 82f392990..cd22815e9 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -200,7 +200,8 @@ CONF_SCHEMA = { "min_trade_number": {'type': 'number'}, "max_trade_duration_minute": {'type': 'integer'}, "remove_pumps": {'type': 'boolean'} - } + }, + 'required': ['process_throttle_secs', 'allowed_risk', 'capital_available_percentage'] } }, 'anyOf': [ From 4431e3bdb61bb7af1fa7db61d4d9e9f634d06b46 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 11:40:13 +0100 Subject: [PATCH 18/62] position size explanation enriched --- docs/edge.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index 27afcd507..7ecd4e75b 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -90,8 +90,14 @@ Your position size then will be: **position size** = **allowed capital at risk** / **stoploss** -Example: -Let's say the stake currency is ETH and you have 10 ETH on the exchange, your **capital_available_percentage** is 50% and you would allow 1% of risk for each trade. thus your available capital for trading is **10 x 0.5 = 5 ETH** and allowed capital at risk would be **5 x 0.01 = 0.05 ETH**. Let's assume Edge has calculated that for **XLM/ETH** market your stoploss should be at 2%. So your position size will be **0.05 / 0.02= 2.5ETH**.
+Example:
+Let's say the stake currency is ETH and you have 10 ETH on the exchange, your **capital_available_percentage** is 50% and you would allow 1% of risk for each trade. thus your available capital for trading is **10 x 0.5 = 5 ETH** and allowed capital at risk would be **5 x 0.01 = 0.05 ETH**.
+Let's assume Edge has calculated that for **XLM/ETH** market your stoploss should be at 2%. So your position size will be **0.05 / 0.02 = 2.5ETH**.
+Bot takes a position of 2.5ETH on XLM/ETH (call it trade 1). Up next, you receive another buy signal while trade 1 is still open. This time on BTC/ETH market. Edge calculated stoploss for this market at 4%. So your position size would be 0.05 / 0.04 = 1.25ETH (call it trade 2).
+Note that available capital for trading didn’t change for trade 2 even if you had already trade 1. The available capital doesn’t mean the free amount on your wallet.
+Now you have two trades open. The Bot receives yet another buy signal for another market: **ADA/ETH**. This time the stoploss is calculated at 1%. So your position size is **0.05 / 0.01 = 5ETH**. But there are already 4ETH blocked in two previous trades. So the position size for this third trade would be 1ETH.
+Available capital doesn’t change before a position is sold. Let’s assume that trade 1 receives a sell signal and it is sold with a profit of 1ETH. Your total capital on exchange would be 11 ETH and the available capital for trading becomes 5.5ETH.
+So the Bot receives another buy signal for trade 4 with a stoploss at 2% then your position size would be 0.055 / 0.02 = 2.75 ## Configurations Edge has following configurations: From ee62adf4f7c4d2d323dd4740c2e2fa765d7846b7 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 11:41:30 +0100 Subject: [PATCH 19/62] highlight --- docs/edge.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/edge.md b/docs/edge.md index 7ecd4e75b..5082b79d2 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -97,7 +97,7 @@ Bot takes a position of 2.5ETH on XLM/ETH (call it trade 1). Up next, you receiv Note that available capital for trading didn’t change for trade 2 even if you had already trade 1. The available capital doesn’t mean the free amount on your wallet.
Now you have two trades open. The Bot receives yet another buy signal for another market: **ADA/ETH**. This time the stoploss is calculated at 1%. So your position size is **0.05 / 0.01 = 5ETH**. But there are already 4ETH blocked in two previous trades. So the position size for this third trade would be 1ETH.
Available capital doesn’t change before a position is sold. Let’s assume that trade 1 receives a sell signal and it is sold with a profit of 1ETH. Your total capital on exchange would be 11 ETH and the available capital for trading becomes 5.5ETH.
-So the Bot receives another buy signal for trade 4 with a stoploss at 2% then your position size would be 0.055 / 0.02 = 2.75 +So the Bot receives another buy signal for trade 4 with a stoploss at 2% then your position size would be **0.055 / 0.02 = 2.75**. ## Configurations Edge has following configurations: From 1d41a917887b75188c2a31211346e1b778338318 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 11:48:41 +0100 Subject: [PATCH 20/62] =?UTF-8?q?stake=5Famount=20in=20case=20it=20doesn?= =?UTF-8?q?=E2=80=99t=20exist?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/edge/__init__.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index a7bf2199a..e5f913e2d 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -161,23 +161,19 @@ class Edge(): return True def stake_amount(self, pair: str, free_capital: float, total_capital: float) -> float: - if pair not in self._cached_pairs: - logger.warning("cannot find %s in calculated pairs, " - "stake_amount of strategy is used instead.", pair) - return self.config['stake_amount'] - - stoploss = self._cached_pairs[pair].stoploss + stoploss = self.stoploss(pair) available_capital = total_capital * self._capital_percentage allowed_capital_at_risk = available_capital * self._allowed_risk max_position_size = abs(allowed_capital_at_risk / stoploss) position_size = min(max_position_size, free_capital) - logger.info( - 'winrate: %s, expectancy: %s, position size: %s, pair: %s,' - ' stoploss: %s, available capital: %s.', - self._cached_pairs[pair].winrate, - self._cached_pairs[pair].expectancy, - position_size, pair, stoploss, available_capital - ) + if pair in self._cached_pairs: + logger.info( + 'winrate: %s, expectancy: %s, position size: %s, pair: %s,' + ' stoploss: %s, available capital: %s.', + self._cached_pairs[pair].winrate, + self._cached_pairs[pair].expectancy, + position_size, pair, stoploss, available_capital + ) return round(position_size, 15) def stoploss(self, pair: str) -> float: From 33f1cc13b3a3db99287c805d4293c657dc4ee763 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 11:56:16 +0100 Subject: [PATCH 21/62] fixing tests --- freqtrade/tests/conftest.py | 1 - freqtrade/tests/edge/test_edge.py | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 75c8032f1..29297ac61 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -63,7 +63,6 @@ def patch_edge(mocker) -> None: 'LTC/BTC': PairInfo(-0.21, 0.66, 3.71, 0.50, 1.71, 11, 20), } )) - mocker.patch('freqtrade.edge.Edge.stoploss', MagicMock(return_value=-0.20)) mocker.patch('freqtrade.edge.Edge.calculate', MagicMock(return_value=True)) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index d453f6f27..4b63ceb1a 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -191,11 +191,10 @@ def test_nonexisting_stake_amount(mocker, edge_conf): edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ - 'E/F': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + 'E/F': PairInfo(-0.11, 0.66, 3.71, 0.50, 1.71, 10, 60), } )) - - assert edge.stake_amount('N/O', 1, 2) == constants.UNLIMITED_STAKE_AMOUNT + assert edge.stake_amount('N/O', 1, 2) == 0.1 def _validate_ohlc(buy_ohlc_sell_matrice): From bd673178cec320cc6343e84f1586c2cc0f0360f2 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 11:56:53 +0100 Subject: [PATCH 22/62] constants removed --- freqtrade/tests/edge/test_edge.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 4b63ceb1a..74227cfe8 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -5,7 +5,6 @@ import pytest import logging from freqtrade.tests.conftest import get_patched_freqtradebot from freqtrade.edge import Edge, PairInfo -from freqtrade import constants from pandas import DataFrame, to_datetime from freqtrade.strategy.interface import SellType from freqtrade.tests.optimize import (BTrade, BTContainer, _build_backtest_dataframe, From 7ddbaa70ad8a9f5643dbec74eac7065353838348 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 12:06:48 +0100 Subject: [PATCH 23/62] USDT to ETH conversion. 1 USDT = 1 ETH --- docs/edge.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/edge.md b/docs/edge.md index 5082b79d2..829910484 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -117,7 +117,7 @@ Note that it downloads historical data so increasing this number would lead to s #### capital_available_percentage This is the percentage of the total capital on exchange in stake currency.
-As an example if you have 100 USDT available in your wallet on the exchange and this value is 0.5 (which is 50%), then the bot will use a maximum amount of 50 USDT for trading and considers it as available capital.
+As an example if you have 10 ETH available in your wallet on the exchange and this value is 0.5 (which is 50%), then the bot will use a maximum amount of 5 ETH for trading and considers it as available capital.
(default to 0.5) #### allowed_risk From 2d17346b0eec75df2597671c69e4e822eb8f16df Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 13:01:51 +0100 Subject: [PATCH 24/62] explaining arbitrary stake amount in comment --- freqtrade/edge/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index e5f913e2d..4dfcfc8ba 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -223,9 +223,13 @@ class Edge(): # 0.05% is 0.0005 # fee = 0.001 + # we set stake amount to an arbitraty amount. + # as it doesn't change the calculation. + # all returned values are relative. they are percentages. stake = 0.015 - fee = self.fee + + fee = self.fee open_fee = fee / 2 close_fee = fee / 2 From a5414b843703adf51eadbdba54f03397a91e4bf9 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 13:02:45 +0100 Subject: [PATCH 25/62] flake8 --- freqtrade/edge/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 4dfcfc8ba..a212c9849 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -223,12 +223,10 @@ class Edge(): # 0.05% is 0.0005 # fee = 0.001 - # we set stake amount to an arbitraty amount. + # we set stake amount to an arbitrary amount. # as it doesn't change the calculation. # all returned values are relative. they are percentages. stake = 0.015 - - fee = self.fee open_fee = fee / 2 close_fee = fee / 2 From e8fbe77ebc4cdcd2ab9e490fb69715b30993982f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Nov 2018 06:34:56 +0100 Subject: [PATCH 26/62] Refactor static whitelist to module --- freqtrade/freqtradebot.py | 47 ++++----------------- freqtrade/pairlist/StaticList.py | 70 ++++++++++++++++++++++++++++++++ freqtrade/pairlist/__init__.py | 0 3 files changed, 79 insertions(+), 38 deletions(-) create mode 100644 freqtrade/pairlist/StaticList.py create mode 100644 freqtrade/pairlist/__init__.py diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a73a2e98f..3e8874bb6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -25,6 +25,7 @@ from freqtrade.resolvers import StrategyResolver from freqtrade.state import State from freqtrade.strategy.interface import SellType, IStrategy from freqtrade.exchange.exchange_helpers import order_book_to_dataframe +from freqtrade.pairlist.StaticList import StaticList logger = logging.getLogger(__name__) @@ -59,6 +60,7 @@ class FreqtradeBot(object): self.persistence = None self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) + self.pairlists = StaticList(self, self.config) # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ @@ -148,11 +150,13 @@ class FreqtradeBot(object): try: nb_assets = self.config.get('dynamic_whitelist', None) # Refresh whitelist based on wallet maintenance - sanitized_list = self._refresh_whitelist( - self._gen_pair_whitelist( - self.config['stake_currency'] - ) if nb_assets else self.config['exchange']['pair_whitelist'] - ) + self.pairlists.refresh_whitelist() + sanitized_list = self.pairlists.whitelist + # sanitized_list = self._refresh_whitelist( + # self._gen_pair_whitelist( + # self.config['stake_currency'] + # ) if nb_assets else self.lists.get_whitelist() + # ) # Keep only the subsets of pairs wanted (up to nb_assets) self.active_pair_whitelist = sanitized_list[:nb_assets] if nb_assets else sanitized_list @@ -227,39 +231,6 @@ class FreqtradeBot(object): pairs = [s['symbol'] for s in sorted_tickers] return pairs - def _refresh_whitelist(self, whitelist: List[str]) -> List[str]: - """ - Check available markets and remove pair from whitelist if necessary - :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to - trade - :return: the list of pairs the user wants to trade without the one unavailable or - black_listed - """ - sanitized_whitelist = whitelist - markets = self.exchange.get_markets() - - markets = [m for m in markets if m['quote'] == self.config['stake_currency']] - known_pairs = set() - for market in markets: - pair = market['symbol'] - # pair is not int the generated dynamic market, or in the blacklist ... ignore it - if pair not in whitelist or pair in self.config['exchange'].get('pair_blacklist', []): - continue - # else the pair is valid - known_pairs.add(pair) - # Market is not active - if not market['active']: - sanitized_whitelist.remove(pair) - logger.info( - 'Ignoring %s from whitelist. Market is not active.', - pair - ) - - # We need to remove pairs that are unknown - final_list = [x for x in sanitized_whitelist if x in known_pairs] - - return final_list - def get_target_bid(self, pair: str, ticker: Dict[str, float]) -> float: """ Calculates bid target between current ask price and last price diff --git a/freqtrade/pairlist/StaticList.py b/freqtrade/pairlist/StaticList.py new file mode 100644 index 000000000..ea4a9ac98 --- /dev/null +++ b/freqtrade/pairlist/StaticList.py @@ -0,0 +1,70 @@ +""" +Static List provider + +Provides lists as configured in config.json + + """ +import logging +from typing import List, Optional + +logger = logging.getLogger(__name__) + + +class StaticList(object): + + def __init__(self, freqtrade, config: dict) -> None: + self._freqtrade = freqtrade + self._config = config + self._whitelist = self._config['exchange']['pair_whitelist'] + self._blacklist = self._config['exchange'].get('pair_blacklist', []) + self.refresh_whitelist() + + @property + def whitelist(self) -> List[str]: + """ Contains the current whitelist """ + return self._whitelist + + @property + def blacklist(self) -> List[str]: + return self._blacklist + + def refresh_whitelist(self) -> bool: + """ + Refreshes whitelist. + """ + return self.validate_whitelist(self._config['exchange']['pair_whitelist']) + + def validate_whitelist(self, whitelist: List[str]) -> bool: + """ + Check available markets and remove pair from whitelist if necessary + :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to + trade + :return: the list of pairs the user wants to trade without the one unavailable or + black_listed + """ + sanitized_whitelist = whitelist + markets = self._freqtrade.exchange.get_markets() + + # Filter to markets in stake currency + markets = [m for m in markets if m['quote'] == self._config['stake_currency']] + known_pairs = set() + + for market in markets: + pair = market['symbol'] + # pair is not int the generated dynamic market, or in the blacklist ... ignore it + if pair not in whitelist or pair in self.blacklist: + continue + # else the pair is valid + known_pairs.add(pair) + # Market is not active + if not market['active']: + sanitized_whitelist.remove(pair) + logger.info( + 'Ignoring %s from whitelist. Market is not active.', + pair + ) + + # We need to remove pairs that are unknown + self._whitelist = [x for x in sanitized_whitelist if x in known_pairs] + + return True diff --git a/freqtrade/pairlist/__init__.py b/freqtrade/pairlist/__init__.py new file mode 100644 index 000000000..e69de29bb From 1738633efc2696add32450cc30355dbf94721a1b Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Nov 2018 06:54:20 +0100 Subject: [PATCH 27/62] Fix refresh_whitelist tests --- freqtrade/pairlist/StaticList.py | 2 +- freqtrade/tests/test_acl_pair.py | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/freqtrade/pairlist/StaticList.py b/freqtrade/pairlist/StaticList.py index ea4a9ac98..8b0decdf6 100644 --- a/freqtrade/pairlist/StaticList.py +++ b/freqtrade/pairlist/StaticList.py @@ -17,7 +17,7 @@ class StaticList(object): self._config = config self._whitelist = self._config['exchange']['pair_whitelist'] self._blacklist = self._config['exchange'].get('pair_blacklist', []) - self.refresh_whitelist() + # self.refresh_whitelist() @property def whitelist(self) -> List[str]: diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 38df3cb38..58dd3dff2 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -6,9 +6,7 @@ from freqtrade.tests.conftest import get_patched_freqtradebot import pytest -# whitelist, blacklist, filtering, all of that will -# eventually become some rules to run on a generic ACL engine -# perhaps try to anticipate that by using some python package +# whitelist, blacklist, @pytest.fixture(scope="function") @@ -33,26 +31,26 @@ def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) - refreshedwhitelist = freqtradebot._refresh_whitelist( + freqtradebot.pairlists.validate_whitelist( whitelist_conf['exchange']['pair_whitelist'] + ['XXX/BTC'] ) # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed - assert whitelist == refreshedwhitelist + assert whitelist == freqtradebot.pairlists.whitelist def test_refresh_whitelist(mocker, markets, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) - refreshedwhitelist = freqtradebot._refresh_whitelist( + freqtradebot.pairlists.validate_whitelist( whitelist_conf['exchange']['pair_whitelist']) # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed - assert whitelist == refreshedwhitelist + assert whitelist == freqtradebot.pairlists.whitelist def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): @@ -67,11 +65,11 @@ def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): # argument: use the whitelist dynamically by exchange-volume whitelist = ['ETH/BTC', 'TKN/BTC'] - refreshedwhitelist = freqtradebot._refresh_whitelist( + freqtradebot._refresh_whitelist( freqtradebot._gen_pair_whitelist(whitelist_conf['stake_currency']) ) - assert whitelist == refreshedwhitelist + assert whitelist == freqtradebot.pairlists.whitelist def test_refresh_whitelist_dynamic_empty(mocker, markets_empty, whitelist_conf): From 8fd713f3ae09d465d205501eda28c98d5e7addfc Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Nov 2018 07:02:08 +0100 Subject: [PATCH 28/62] validate_whitelist should return the list again --- freqtrade/pairlist/StaticList.py | 10 ++++------ freqtrade/tests/test_acl_pair.py | 11 +++++------ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/freqtrade/pairlist/StaticList.py b/freqtrade/pairlist/StaticList.py index 8b0decdf6..225f89b2e 100644 --- a/freqtrade/pairlist/StaticList.py +++ b/freqtrade/pairlist/StaticList.py @@ -28,13 +28,13 @@ class StaticList(object): def blacklist(self) -> List[str]: return self._blacklist - def refresh_whitelist(self) -> bool: + def refresh_whitelist(self) -> None: """ Refreshes whitelist. """ - return self.validate_whitelist(self._config['exchange']['pair_whitelist']) + self._whitelist = self.validate_whitelist(self._config['exchange']['pair_whitelist']) - def validate_whitelist(self, whitelist: List[str]) -> bool: + def validate_whitelist(self, whitelist: List[str]) -> List[str]: """ Check available markets and remove pair from whitelist if necessary :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to @@ -65,6 +65,4 @@ class StaticList(object): ) # We need to remove pairs that are unknown - self._whitelist = [x for x in sanitized_whitelist if x in known_pairs] - - return True + return [x for x in sanitized_whitelist if x in known_pairs] diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 58dd3dff2..6ec11e664 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -31,22 +31,21 @@ def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) - freqtradebot.pairlists.validate_whitelist( - whitelist_conf['exchange']['pair_whitelist'] + ['XXX/BTC'] - ) + freqtradebot.pairlists.refresh_whitelist() # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed assert whitelist == freqtradebot.pairlists.whitelist + # Ensure config dict hasn't been changed + assert (whitelist_conf['exchange']['pair_whitelist'] == + freqtradebot.config['exchange']['pair_whitelist']) def test_refresh_whitelist(mocker, markets, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) - freqtradebot.pairlists.validate_whitelist( - whitelist_conf['exchange']['pair_whitelist']) - + freqtradebot.pairlists.refresh_whitelist() # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed From 58c7adae0aff2b6121243811f1aa02a3fd03d481 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Nov 2018 07:06:02 +0100 Subject: [PATCH 29/62] Test for blacklist --- freqtrade/pairlist/StaticList.py | 2 +- freqtrade/tests/test_acl_pair.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/pairlist/StaticList.py b/freqtrade/pairlist/StaticList.py index 225f89b2e..3c46fab11 100644 --- a/freqtrade/pairlist/StaticList.py +++ b/freqtrade/pairlist/StaticList.py @@ -30,7 +30,7 @@ class StaticList(object): def refresh_whitelist(self) -> None: """ - Refreshes whitelist. + Refreshes whitelist and assigns it to self._whitelist """ self._whitelist = self.validate_whitelist(self._config['exchange']['pair_whitelist']) diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 6ec11e664..d9fd93d09 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -6,7 +6,7 @@ from freqtrade.tests.conftest import get_patched_freqtradebot import pytest -# whitelist, blacklist, +# whitelist, blacklist @pytest.fixture(scope="function") @@ -41,7 +41,7 @@ def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf): freqtradebot.config['exchange']['pair_whitelist']) -def test_refresh_whitelist(mocker, markets, whitelist_conf): +def test_refresh_pairlists(mocker, markets, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) @@ -50,6 +50,7 @@ def test_refresh_whitelist(mocker, markets, whitelist_conf): whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed assert whitelist == freqtradebot.pairlists.whitelist + assert whitelist_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): From d09dbfe2e6e6a233174c9863aff4bb3b6fc1076a Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Nov 2018 20:08:50 +0100 Subject: [PATCH 30/62] Add volumePairList - refactor tests to correct file --- freqtrade/freqtradebot.py | 48 ++------- .../{StaticList.py => StaticPairList.py} | 6 +- freqtrade/pairlist/VolumePairList.py | 98 +++++++++++++++++++ freqtrade/tests/test_acl_pair.py | 46 +++++++-- freqtrade/tests/test_freqtradebot.py | 31 ------ 5 files changed, 149 insertions(+), 80 deletions(-) rename freqtrade/pairlist/{StaticList.py => StaticPairList.py} (91%) create mode 100644 freqtrade/pairlist/VolumePairList.py diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 3e8874bb6..d8f789e6a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -12,8 +12,6 @@ from typing import Any, Callable, Dict, List, Optional import arrow from requests.exceptions import RequestException -from cachetools import TTLCache, cached - from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.exchange import Exchange @@ -25,7 +23,8 @@ from freqtrade.resolvers import StrategyResolver from freqtrade.state import State from freqtrade.strategy.interface import SellType, IStrategy from freqtrade.exchange.exchange_helpers import order_book_to_dataframe -from freqtrade.pairlist.StaticList import StaticList +from freqtrade.pairlist.StaticPairList import StaticPairList +from freqtrade.pairlist.VolumePairList import VolumePairList logger = logging.getLogger(__name__) @@ -60,7 +59,11 @@ class FreqtradeBot(object): self.persistence = None self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) - self.pairlists = StaticList(self, self.config) + if self.config.get('dynamic_whitelist', None): + self.pairlists = VolumePairList(self, self.config) + + else: + self.pairlists = StaticPairList(self, self.config) # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ @@ -148,18 +151,9 @@ class FreqtradeBot(object): """ state_changed = False try: - nb_assets = self.config.get('dynamic_whitelist', None) - # Refresh whitelist based on wallet maintenance + # Refresh whitelist self.pairlists.refresh_whitelist() - sanitized_list = self.pairlists.whitelist - # sanitized_list = self._refresh_whitelist( - # self._gen_pair_whitelist( - # self.config['stake_currency'] - # ) if nb_assets else self.lists.get_whitelist() - # ) - - # Keep only the subsets of pairs wanted (up to nb_assets) - self.active_pair_whitelist = sanitized_list[:nb_assets] if nb_assets else sanitized_list + self.active_pair_whitelist = self.pairlists.whitelist # Calculating Edge positiong # Should be called before refresh_tickers @@ -207,30 +201,6 @@ class FreqtradeBot(object): self.state = State.STOPPED return state_changed - @cached(TTLCache(maxsize=1, ttl=1800)) - def _gen_pair_whitelist(self, base_currency: str, key: str = 'quoteVolume') -> List[str]: - """ - Updates the whitelist with with a dynamically generated list - :param base_currency: base currency as str - :param key: sort key (defaults to 'quoteVolume') - :return: List of pairs - """ - - if not self.exchange.exchange_has('fetchTickers'): - raise OperationalException( - 'Exchange does not support dynamic whitelist.' - 'Please edit your config and restart the bot' - ) - - tickers = self.exchange.get_tickers() - # check length so that we make sure that '/' is actually in the string - tickers = [v for k, v in tickers.items() - if len(k.split('/')) == 2 and k.split('/')[1] == base_currency] - - sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) - pairs = [s['symbol'] for s in sorted_tickers] - return pairs - def get_target_bid(self, pair: str, ticker: Dict[str, float]) -> float: """ Calculates bid target between current ask price and last price diff --git a/freqtrade/pairlist/StaticList.py b/freqtrade/pairlist/StaticPairList.py similarity index 91% rename from freqtrade/pairlist/StaticList.py rename to freqtrade/pairlist/StaticPairList.py index 3c46fab11..8466d7b82 100644 --- a/freqtrade/pairlist/StaticList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -10,7 +10,7 @@ from typing import List, Optional logger = logging.getLogger(__name__) -class StaticList(object): +class StaticPairList(object): def __init__(self, freqtrade, config: dict) -> None: self._freqtrade = freqtrade @@ -32,9 +32,9 @@ class StaticList(object): """ Refreshes whitelist and assigns it to self._whitelist """ - self._whitelist = self.validate_whitelist(self._config['exchange']['pair_whitelist']) + self._whitelist = self._validate_whitelist(self._config['exchange']['pair_whitelist']) - def validate_whitelist(self, whitelist: List[str]) -> List[str]: + def _validate_whitelist(self, whitelist: List[str]) -> List[str]: """ Check available markets and remove pair from whitelist if necessary :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py new file mode 100644 index 000000000..c79f3c5c0 --- /dev/null +++ b/freqtrade/pairlist/VolumePairList.py @@ -0,0 +1,98 @@ +""" +Static List provider + +Provides lists as configured in config.json + + """ +import logging +from typing import List +from cachetools import TTLCache, cached + +from freqtrade.pairlist.StaticPairList import StaticPairList +from freqtrade import OperationalException +logger = logging.getLogger(__name__) + + +class VolumePairList(StaticPairList): + + def __init__(self, freqtrade, config: dict) -> None: + self._freqtrade = freqtrade + self._config = config + self._whitelist = self._config['exchange']['pair_whitelist'] + self._blacklist = self._config['exchange'].get('pair_blacklist', []) + self._number_pairs = self._config.get('dynamic_whitelist', None) + if not self._freqtrade.exchange.exchange_has('fetchTickers'): + raise OperationalException( + 'Exchange does not support dynamic whitelist.' + 'Please edit your config and restart the bot' + ) + # self.refresh_whitelist() + + @property + def whitelist(self) -> List[str]: + """ Contains the current whitelist """ + return self._whitelist + + @property + def blacklist(self) -> List[str]: + return self._blacklist + + def refresh_whitelist(self) -> None: + """ + Refreshes whitelist and assigns it to self._whitelist + """ + # Generate dynamic whitelist + pairs = self._gen_pair_whitelist(self._config['stake_currency']) + # Validate whitelist to only have active market pairs + self._whitelist = self._validate_whitelist(pairs)[:self._number_pairs] + + @cached(TTLCache(maxsize=1, ttl=1800)) + def _gen_pair_whitelist(self, base_currency: str, key: str = 'quoteVolume') -> List[str]: + """ + Updates the whitelist with with a dynamically generated list + :param base_currency: base currency as str + :param key: sort key (defaults to 'quoteVolume') + :return: List of pairs + """ + + tickers = self._freqtrade.exchange.get_tickers() + # check length so that we make sure that '/' is actually in the string + tickers = [v for k, v in tickers.items() + if len(k.split('/')) == 2 and k.split('/')[1] == base_currency] + + sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) + pairs = [s['symbol'] for s in sorted_tickers] + return pairs + + def _validate_whitelist(self, whitelist: List[str]) -> List[str]: + """ + Check available markets and remove pair from whitelist if necessary + :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to + trade + :return: the list of pairs the user wants to trade without the one unavailable or + black_listed + """ + sanitized_whitelist = whitelist + markets = self._freqtrade.exchange.get_markets() + + # Filter to markets in stake currency + markets = [m for m in markets if m['quote'] == self._config['stake_currency']] + known_pairs = set() + + for market in markets: + pair = market['symbol'] + # pair is not int the generated dynamic market, or in the blacklist ... ignore it + if pair not in whitelist or pair in self.blacklist: + continue + # else the pair is valid + known_pairs.add(pair) + # Market is not active + if not market['active']: + sanitized_whitelist.remove(pair) + logger.info( + 'Ignoring %s from whitelist. Market is not active.', + pair + ) + + # We need to remove pairs that are unknown + return [x for x in sanitized_whitelist if x in known_pairs] diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index d9fd93d09..997c171ad 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -2,6 +2,7 @@ from unittest.mock import MagicMock +from freqtrade import OperationalException from freqtrade.tests.conftest import get_patched_freqtradebot import pytest @@ -35,7 +36,7 @@ def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf): # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed - assert whitelist == freqtradebot.pairlists.whitelist + assert set(whitelist) == set(freqtradebot.pairlists.whitelist) # Ensure config dict hasn't been changed assert (whitelist_conf['exchange']['pair_whitelist'] == freqtradebot.config['exchange']['pair_whitelist']) @@ -49,11 +50,12 @@ def test_refresh_pairlists(mocker, markets, whitelist_conf): # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed - assert whitelist == freqtradebot.pairlists.whitelist + assert set(whitelist)== set(freqtradebot.pairlists.whitelist) assert whitelist_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): + whitelist_conf['dynamic_whitelist'] = 5 freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -64,10 +66,7 @@ def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): # argument: use the whitelist dynamically by exchange-volume whitelist = ['ETH/BTC', 'TKN/BTC'] - - freqtradebot._refresh_whitelist( - freqtradebot._gen_pair_whitelist(whitelist_conf['stake_currency']) - ) + freqtradebot.pairlists.refresh_whitelist() assert whitelist == freqtradebot.pairlists.whitelist @@ -79,7 +78,40 @@ def test_refresh_whitelist_dynamic_empty(mocker, markets_empty, whitelist_conf): # argument: use the whitelist dynamically by exchange-volume whitelist = [] whitelist_conf['exchange']['pair_whitelist'] = [] - freqtradebot._refresh_whitelist(whitelist) + freqtradebot.pairlists.refresh_whitelist() pairslist = whitelist_conf['exchange']['pair_whitelist'] assert set(whitelist) == set(pairslist) + + +def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + + # Test to retrieved BTC sorted on quoteVolume (default) + freqtrade.pairlists.refresh_whitelist() + + whitelist = freqtrade.pairlists.whitelist + assert whitelist == ['ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC'] + + # Test to retrieve BTC sorted on bidVolume + whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC', key='bidVolume') + assert whitelist == ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'BLK/BTC'] + + # Test with USDT sorted on quoteVolume (default) + whitelist = freqtrade._gen_pair_whitelist(base_currency='USDT') + assert whitelist == ['TKN/USDT', 'ETH/USDT', 'LTC/USDT', 'BLK/USDT'] + + # Test with ETH (our fixture does not have ETH, so result should be empty) + whitelist = freqtrade._gen_pair_whitelist(base_currency='ETH') + assert whitelist == [] + + +def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) + + with pytest.raises(OperationalException): + freqtrade._gen_pair_whitelist(base_currency='BTC') diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index a9b3ffc5d..71d451b94 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -136,37 +136,6 @@ def test_throttle_with_assets(mocker, default_conf) -> None: assert result == -1 -def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) - mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) - - # Test to retrieved BTC sorted on quoteVolume (default) - whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC') - assert whitelist == ['ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC'] - - # Test to retrieve BTC sorted on bidVolume - whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC', key='bidVolume') - assert whitelist == ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'BLK/BTC'] - - # Test with USDT sorted on quoteVolume (default) - whitelist = freqtrade._gen_pair_whitelist(base_currency='USDT') - assert whitelist == ['TKN/USDT', 'ETH/USDT', 'LTC/USDT', 'BLK/USDT'] - - # Test with ETH (our fixture does not have ETH, so result should be empty) - whitelist = freqtrade._gen_pair_whitelist(base_currency='ETH') - assert whitelist == [] - - -def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) - mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) - - with pytest.raises(OperationalException): - freqtrade._gen_pair_whitelist(base_currency='BTC') - - def test_get_trade_stake_amount(default_conf, ticker, limit_buy_order, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) From 26187ef6c7863a3c25b9e02398690c8faf310d3d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 2 Dec 2018 22:12:12 +0100 Subject: [PATCH 31/62] patch exchange_has --- freqtrade/tests/rpc/test_rpc.py | 1 + freqtrade/tests/rpc/test_rpc_telegram.py | 1 + freqtrade/tests/test_freqtradebot.py | 1 + 3 files changed, 3 insertions(+) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index ff72ef634..b0b598654 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -663,6 +663,7 @@ def test_rpc_whitelist_dynamic(mocker, default_conf) -> None: patch_coinmarketcap(mocker) patch_exchange(mocker) default_conf['dynamic_whitelist'] = 4 + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) freqtradebot = FreqtradeBot(default_conf) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index a8e8bf003..4f29c5a03 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -1033,6 +1033,7 @@ def test_whitelist_dynamic(default_conf, update, mocker) -> None: _init=MagicMock(), _send_msg=msg_mock ) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) default_conf['dynamic_whitelist'] = 4 freqtradebot = get_patched_freqtradebot(mocker, default_conf) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 71d451b94..ca604340b 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -2533,5 +2533,6 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order def test_startup_messages(default_conf, mocker): default_conf['dynamic_whitelist'] = 20 + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, default_conf) assert freqtrade.state is State.RUNNING From 3a086aac58f4f2fd3530910d1adc3154e9c3ea73 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 2 Dec 2018 22:49:30 +0100 Subject: [PATCH 32/62] Add commit and message to container --- build_helpers/publish_docker.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh index c2b40ba81..9d82fc2d5 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker.sh @@ -4,6 +4,9 @@ TAG=$(echo "${TRAVIS_BRANCH}" | sed -e "s/\//_/") +# Add commit and commit_message to docker container +echo "${TRAVIS_COMMIT} ${TRAVIS_COMMIT_MESSAGE}" > freqtrade_commit + if [ "${TRAVIS_EVENT_TYPE}" = "cron" ]; then echo "event ${TRAVIS_EVENT_TYPE}: full rebuild - skipping cache" docker build -t freqtrade:${TAG} . From 3360e777a13d3ca910f097b381e0b83d516dde3b Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 3 Dec 2018 19:29:35 +0100 Subject: [PATCH 33/62] Fix flake adn mypy --- freqtrade/freqtradebot.py | 4 ++-- freqtrade/pairlist/StaticPairList.py | 2 +- freqtrade/tests/test_acl_pair.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d8f789e6a..ebe30cec6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -60,10 +60,10 @@ class FreqtradeBot(object): self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) if self.config.get('dynamic_whitelist', None): - self.pairlists = VolumePairList(self, self.config) + self.pairlists: StaticPairList = VolumePairList(self, self.config) else: - self.pairlists = StaticPairList(self, self.config) + self.pairlists: StaticPairList = StaticPairList(self, self.config) # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 8466d7b82..24ad30deb 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -5,7 +5,7 @@ Provides lists as configured in config.json """ import logging -from typing import List, Optional +from typing import List logger = logging.getLogger(__name__) diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 997c171ad..8e41ad860 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -50,7 +50,7 @@ def test_refresh_pairlists(mocker, markets, whitelist_conf): # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed - assert set(whitelist)== set(freqtradebot.pairlists.whitelist) + assert set(whitelist) == set(freqtradebot.pairlists.whitelist) assert whitelist_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist From b5192193fda4df5e3afa06399c22aa7a8f7889ba Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 3 Dec 2018 19:45:00 +0100 Subject: [PATCH 34/62] total amount passed to edge should consider open trades too --- freqtrade/freqtradebot.py | 3 ++- freqtrade/persistence.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ccb45be2c..1e4b2e678 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -305,7 +305,8 @@ class FreqtradeBot(object): return self.edge.stake_amount( pair, self.wallets.get_free(self.config['stake_currency']), - self.wallets.get_total(self.config['stake_currency']) + self.wallets.get_total(self.config['stake_currency']) + + Trade.calc_total_open_trades_in_stake_currency() ) else: stake_amount = self.config['stake_amount'] diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 592a88acb..20ae8c43a 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -14,6 +14,7 @@ from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker +from sqlalchemy import func from sqlalchemy.pool import StaticPool from freqtrade import OperationalException @@ -349,3 +350,13 @@ class Trade(_DECL_BASE): ) profit_percent = (close_trade_price / open_trade_price) - 1 return float(f"{profit_percent:.8f}") + + def calc_total_open_trades_in_stake_currency() -> float: + """ + Calculates total invested amount in open trades + in stake currency + """ + total_open_stake_amount = Trade.session.query(func.sum(Trade.stake_amount))\ + .filter(Trade.is_open.is_(True))\ + .scalar() + return total_open_stake_amount or 0 From 43bafc391f77e1b937b6e3b18ea74ab1d9b54e1e Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 3 Dec 2018 19:46:22 +0100 Subject: [PATCH 35/62] static method added --- freqtrade/persistence.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 20ae8c43a..aeab63f22 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -351,6 +351,7 @@ class Trade(_DECL_BASE): profit_percent = (close_trade_price / open_trade_price) - 1 return float(f"{profit_percent:.8f}") + @staticmethod def calc_total_open_trades_in_stake_currency() -> float: """ Calculates total invested amount in open trades From 108d9a111714c5d7a26aaa72980a03912e93819a Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 3 Dec 2018 19:55:37 +0100 Subject: [PATCH 36/62] function name refactored --- freqtrade/freqtradebot.py | 2 +- freqtrade/persistence.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1e4b2e678..6baef76bc 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -306,7 +306,7 @@ class FreqtradeBot(object): pair, self.wallets.get_free(self.config['stake_currency']), self.wallets.get_total(self.config['stake_currency']) + - Trade.calc_total_open_trades_in_stake_currency() + Trade.total_open_trades_stakes() ) else: stake_amount = self.config['stake_amount'] diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index aeab63f22..71752d58e 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -352,7 +352,7 @@ class Trade(_DECL_BASE): return float(f"{profit_percent:.8f}") @staticmethod - def calc_total_open_trades_in_stake_currency() -> float: + def total_open_trades_stakes() -> float: """ Calculates total invested amount in open trades in stake currency From 1b3ecb8343b7de2424a5d5bced73400d3a4cefa4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 3 Dec 2018 20:00:18 +0100 Subject: [PATCH 37/62] Deprecate --dynamic-whitelist --- freqtrade/arguments.py | 3 ++- freqtrade/configuration.py | 12 ++++++++---- freqtrade/constants.py | 8 ++++++++ freqtrade/freqtradebot.py | 3 +-- freqtrade/pairlist/VolumePairList.py | 3 ++- freqtrade/tests/test_acl_pair.py | 7 ++++--- freqtrade/tests/test_configuration.py | 8 +++++--- 7 files changed, 30 insertions(+), 14 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 84e1a0f77..a33848c5f 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -115,7 +115,8 @@ class Arguments(object): self.parser.add_argument( '--dynamic-whitelist', help='dynamically generate and update whitelist' - ' based on 24h BaseVolume (default: %(const)s)', + ' based on 24h BaseVolume (default: %(const)s)' + ' DEPRECATED.', dest='dynamic_whitelist', const=constants.DYNAMIC_WHITELIST, type=int, diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index feec0cb43..7ddcf1639 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -110,10 +110,14 @@ class Configuration(object): # Add dynamic_whitelist if found if 'dynamic_whitelist' in self.args and self.args.dynamic_whitelist: - config.update({'dynamic_whitelist': self.args.dynamic_whitelist}) - logger.info( - 'Parameter --dynamic-whitelist detected. ' - 'Using dynamically generated whitelist. ' + # Update to volumePairList (the previous default) + config['whitelist'] = {'method': 'VolumePairList', + 'config': {'number_assets': self.args.dynamic_whitelist} + } + logger.warning( + 'Parameter --dynamic-whitelist has been deprecated, ' + 'and will be completely replaced by the whitelist dict in the future. ' + 'For now: using dynamically generated whitelist based on VolumePairList. ' '(not applicable with Backtesting and Hyperopt)' ) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index f8fb91240..c200137cb 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -124,6 +124,14 @@ CONF_SCHEMA = { 'ignore_roi_if_buy_signal_true': {'type': 'boolean'} } }, + 'whitelist': { + 'type': 'object', + 'properties': { + 'method': {'type': 'string'}, + 'config': {'type': 'object'} + }, + 'required': ['method'] + }, 'telegram': { 'type': 'object', 'properties': { diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ebe30cec6..19a22b87d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -59,9 +59,8 @@ class FreqtradeBot(object): self.persistence = None self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) - if self.config.get('dynamic_whitelist', None): + if self.config.get('whitelist', {}).get('method') == 'VolumePairList': self.pairlists: StaticPairList = VolumePairList(self, self.config) - else: self.pairlists: StaticPairList = StaticPairList(self, self.config) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index c79f3c5c0..41f9943a1 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -18,9 +18,10 @@ class VolumePairList(StaticPairList): def __init__(self, freqtrade, config: dict) -> None: self._freqtrade = freqtrade self._config = config + self._whitelistconf = self._config.get('whitelist', {}).get('config') self._whitelist = self._config['exchange']['pair_whitelist'] self._blacklist = self._config['exchange'].get('pair_blacklist', []) - self._number_pairs = self._config.get('dynamic_whitelist', None) + self._number_pairs = self._whitelistconf.get('number_assets') if not self._freqtrade.exchange.exchange_has('fetchTickers'): raise OperationalException( 'Exchange does not support dynamic whitelist.' diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 8e41ad860..a12de9a81 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -4,7 +4,6 @@ from unittest.mock import MagicMock from freqtrade import OperationalException from freqtrade.tests.conftest import get_patched_freqtradebot - import pytest # whitelist, blacklist @@ -109,9 +108,11 @@ def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) + default_conf['whitelist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 10} + } mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) with pytest.raises(OperationalException): - freqtrade._gen_pair_whitelist(base_currency='BTC') + get_patched_freqtradebot(mocker, default_conf) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 23fefd3cd..951d831b2 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -119,7 +119,8 @@ def test_load_config_with_params(default_conf, mocker) -> None: configuration = Configuration(args) validated_conf = configuration.load_config() - assert validated_conf.get('dynamic_whitelist') == 10 + assert validated_conf.get('whitelist', {}).get('method') == 'VolumePairList' + assert validated_conf.get('whitelist', {}).get('config').get('number_assets') == 10 assert validated_conf.get('strategy') == 'TestStrategy' assert validated_conf.get('strategy_path') == '/some/path' assert validated_conf.get('db_url') == 'sqlite:///someurl' @@ -194,8 +195,9 @@ def test_show_info(default_conf, mocker, caplog) -> None: configuration.get_config() assert log_has( - 'Parameter --dynamic-whitelist detected. ' - 'Using dynamically generated whitelist. ' + 'Parameter --dynamic-whitelist has been deprecated, ' + 'and will be completely replaced by the whitelist dict in the future. ' + 'For now: using dynamically generated whitelist based on VolumePairList. ' '(not applicable with Backtesting and Hyperopt)', caplog.record_tuples ) From ef1208b3664d7ebe1dd6ff568b9592a46b8bfc7c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 3 Dec 2018 20:31:25 +0100 Subject: [PATCH 38/62] Fix rpc messages --- freqtrade/freqtradebot.py | 2 +- freqtrade/pairlist/StaticPairList.py | 24 +++++++++++++++++++++++- freqtrade/pairlist/VolumePairList.py | 16 +++++++--------- freqtrade/rpc/rpc.py | 3 ++- freqtrade/rpc/rpc_manager.py | 12 +++--------- freqtrade/rpc/telegram.py | 6 ++---- freqtrade/tests/rpc/test_rpc.py | 7 +++++-- freqtrade/tests/rpc/test_rpc_manager.py | 4 +++- freqtrade/tests/rpc/test_rpc_telegram.py | 4 +++- freqtrade/tests/test_acl_pair.py | 6 ++++-- freqtrade/tests/test_arguments.py | 3 ++- freqtrade/tests/test_configuration.py | 4 +--- freqtrade/tests/test_freqtradebot.py | 4 +++- 13 files changed, 59 insertions(+), 36 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 19a22b87d..17b818b27 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -112,7 +112,7 @@ class FreqtradeBot(object): }) logger.info('Changing state to: %s', state.name) if state == State.RUNNING: - self.rpc.startup_messages(self.config) + self.rpc.startup_messages(self.config, self.pairlists) if state == State.STOPPED: time.sleep(1) diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 24ad30deb..5b0e37357 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -19,15 +19,37 @@ class StaticPairList(object): self._blacklist = self._config['exchange'].get('pair_blacklist', []) # self.refresh_whitelist() + @property + def name(self) -> str: + """ + Gets name of the class + -> no need to overwrite in subclasses + """ + return self.__class__.__name__ + @property def whitelist(self) -> List[str]: - """ Contains the current whitelist """ + """ + Has the current whitelist + -> no need to overwrite in subclasses + """ return self._whitelist @property def blacklist(self) -> List[str]: + """ + Has the current blacklist + -> no need to overwrite in subclasses + """ return self._blacklist + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + -> Please overwrite in subclasses + """ + return f"{self.name}: {self.whitelist}" + def refresh_whitelist(self) -> None: """ Refreshes whitelist and assigns it to self._whitelist diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 41f9943a1..f45c57676 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -21,7 +21,7 @@ class VolumePairList(StaticPairList): self._whitelistconf = self._config.get('whitelist', {}).get('config') self._whitelist = self._config['exchange']['pair_whitelist'] self._blacklist = self._config['exchange'].get('pair_blacklist', []) - self._number_pairs = self._whitelistconf.get('number_assets') + self._number_pairs = self._whitelistconf['number_assets'] if not self._freqtrade.exchange.exchange_has('fetchTickers'): raise OperationalException( 'Exchange does not support dynamic whitelist.' @@ -29,14 +29,12 @@ class VolumePairList(StaticPairList): ) # self.refresh_whitelist() - @property - def whitelist(self) -> List[str]: - """ Contains the current whitelist """ - return self._whitelist - - @property - def blacklist(self) -> List[str]: - return self._blacklist + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + -> Please overwrite in subclasses + """ + return f"{self.name} - top {self._whitelistconf['number_assets']} volume pairs." def refresh_whitelist(self) -> None: """ diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 50cbd27ef..6ca32e4be 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -446,7 +446,8 @@ class RPC(object): def _rpc_whitelist(self) -> Dict: """ Returns the currently active whitelist""" - res = {'method': self._freqtrade.config.get('dynamic_whitelist', 0) or 'static', + res = {'method': self._freqtrade.pairlists.name, + 'length': len(self._freqtrade.pairlists.whitelist), 'whitelist': self._freqtrade.active_pair_whitelist } return res diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 74a4e3bdc..de861677d 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -52,7 +52,7 @@ class RPCManager(object): logger.debug('Forwarding message to rpc.%s', mod.name) mod.send_msg(msg) - def startup_messages(self, config) -> None: + def startup_messages(self, config, pairlist) -> None: if config.get('dry_run', False): self.send_msg({ 'type': RPCMessageType.WARNING_NOTIFICATION, @@ -72,14 +72,8 @@ class RPCManager(object): f'*Ticker Interval:* `{ticker_interval}`\n' f'*Strategy:* `{strategy_name}`' }) - if config.get('dynamic_whitelist', False): - top_pairs = 'top volume ' + str(config.get('dynamic_whitelist', 20)) - specific_pairs = '' - else: - top_pairs = 'whitelisted' - specific_pairs = '\n' + ', '.join(config['exchange'].get('pair_whitelist', '')) self.send_msg({ 'type': RPCMessageType.STATUS_NOTIFICATION, - 'status': f'Searching for {top_pairs} {stake_currency} pairs to buy and sell...' - f'{specific_pairs}' + 'status': f'Searching for {stake_currency} pairs to buy and sell ' + f'based on {pairlist.short_desc()}' }) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 55c5abef2..9a3abaea4 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -447,10 +447,8 @@ class Telegram(RPC): """ try: whitelist = self._rpc_whitelist() - if whitelist['method'] == 'static': - message = f"Using static whitelist with `{len(whitelist['whitelist'])}` pairs \n" - else: - message = f"Dynamic whitelist with `{whitelist['method']}` pairs\n" + + message = f"Using whitelist `{whitelist['method']}` with {whitelist['length']} pairs\n" message += f"`{', '.join(whitelist['whitelist'])}`" logger.debug(message) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index b0b598654..54bb45cd4 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -662,12 +662,15 @@ def test_rpc_whitelist(mocker, default_conf) -> None: def test_rpc_whitelist_dynamic(mocker, default_conf) -> None: patch_coinmarketcap(mocker) patch_exchange(mocker) - default_conf['dynamic_whitelist'] = 4 + default_conf['whitelist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 4} + } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) ret = rpc._rpc_whitelist() - assert ret['method'] == 4 + assert ret['method'] == 'VolumePairList' + assert ret['length'] == 4 assert ret['whitelist'] == default_conf['exchange']['pair_whitelist'] diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index cbb858522..9359fc927 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -128,7 +128,9 @@ def test_startupmessages_telegram_enabled(mocker, default_conf, caplog) -> None: telegram_mock.reset_mock() default_conf['dry_run'] = True - default_conf['dynamic_whitelist'] = 20 + default_conf['whitelist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 20} + } rpc_manager.startup_messages(default_conf) assert telegram_mock.call_count == 3 diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 4f29c5a03..8a97abf58 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -1034,7 +1034,9 @@ def test_whitelist_dynamic(default_conf, update, mocker) -> None: _send_msg=msg_mock ) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) - default_conf['dynamic_whitelist'] = 4 + default_conf['whitelist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 4} + } freqtradebot = get_patched_freqtradebot(mocker, default_conf) telegram = Telegram(freqtradebot) diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index a12de9a81..5acd9abda 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -54,14 +54,16 @@ def test_refresh_pairlists(mocker, markets, whitelist_conf): def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): - whitelist_conf['dynamic_whitelist'] = 5 - freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) + whitelist_conf['whitelist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 5} + } mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_markets=markets, get_tickers=tickers, exchange_has=MagicMock(return_value=True) ) + freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) # argument: use the whitelist dynamically by exchange-volume whitelist = ['ETH/BTC', 'TKN/BTC'] diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index e09aeb1df..d28ab30af 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -17,7 +17,8 @@ def test_parse_args_none() -> None: def test_parse_args_defaults() -> None: args = Arguments([], '').get_parsed_arg() assert args.config == 'config.json' - assert args.dynamic_whitelist is None + assert args.strategy_path is None + assert args.datadir is None assert args.loglevel == 0 diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 951d831b2..5eda9d871 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -102,7 +102,7 @@ def test_load_config(default_conf, mocker) -> None: assert validated_conf.get('strategy') == 'DefaultStrategy' assert validated_conf.get('strategy_path') is None - assert 'dynamic_whitelist' not in validated_conf + assert 'edge' not in validated_conf def test_load_config_with_params(default_conf, mocker) -> None: @@ -133,7 +133,6 @@ def test_load_config_with_params(default_conf, mocker) -> None: )) arglist = [ - '--dynamic-whitelist', '10', '--strategy', 'TestStrategy', '--strategy-path', '/some/path' ] @@ -152,7 +151,6 @@ def test_load_config_with_params(default_conf, mocker) -> None: )) arglist = [ - '--dynamic-whitelist', '10', '--strategy', 'TestStrategy', '--strategy-path', '/some/path' ] diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index ca604340b..0cdc9b0e6 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -2532,7 +2532,9 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order def test_startup_messages(default_conf, mocker): - default_conf['dynamic_whitelist'] = 20 + default_conf['whitelist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 20} + } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, default_conf) assert freqtrade.state is State.RUNNING From 18ad3388b4e23158bc540a7fb45dd102949c0071 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 3 Dec 2018 20:38:15 +0100 Subject: [PATCH 39/62] Some more tests adapted to pairlists --- freqtrade/tests/rpc/test_rpc.py | 2 +- freqtrade/tests/rpc/test_rpc_manager.py | 4 ++-- freqtrade/tests/rpc/test_rpc_telegram.py | 4 ++-- freqtrade/tests/test_freqtradebot.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 54bb45cd4..87124a9a7 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -655,7 +655,7 @@ def test_rpc_whitelist(mocker, default_conf) -> None: freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) ret = rpc._rpc_whitelist() - assert ret['method'] == 'static' + assert ret['method'] == 'StaticPairList' assert ret['whitelist'] == default_conf['exchange']['pair_whitelist'] diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index 9359fc927..15d9c20c6 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -121,7 +121,7 @@ def test_startupmessages_telegram_enabled(mocker, default_conf, caplog) -> None: freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc_manager = RPCManager(freqtradebot) - rpc_manager.startup_messages(default_conf) + rpc_manager.startup_messages(default_conf, freqtradebot.pairlists) assert telegram_mock.call_count == 3 assert "*Exchange:* `bittrex`" in telegram_mock.call_args_list[1][0][0]['status'] @@ -132,6 +132,6 @@ def test_startupmessages_telegram_enabled(mocker, default_conf, caplog) -> None: 'config': {'number_assets': 20} } - rpc_manager.startup_messages(default_conf) + rpc_manager.startup_messages(default_conf, freqtradebot.pairlists) assert telegram_mock.call_count == 3 assert "Dry run is enabled." in telegram_mock.call_args_list[0][0][0]['status'] diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 8a97abf58..f63393935 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -1021,7 +1021,7 @@ def test_whitelist_static(default_conf, update, mocker) -> None: telegram._whitelist(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 - assert ('Using static whitelist with `4` pairs \n`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`' + assert ('Using whitelist `StaticPairList` with 4 pairs\n`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`' in msg_mock.call_args_list[0][0][0]) @@ -1043,7 +1043,7 @@ def test_whitelist_dynamic(default_conf, update, mocker) -> None: telegram._whitelist(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 - assert ('Dynamic whitelist with `4` pairs\n`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`' + assert ('Using whitelist `VolumePairList` with 4 pairs\n`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`' in msg_mock.call_args_list[0][0][0]) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 0cdc9b0e6..a10389261 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -217,7 +217,7 @@ def test_edge_called_in_process(mocker, edge_conf) -> None: patch_exchange(mocker) freqtrade = FreqtradeBot(edge_conf) - freqtrade._refresh_whitelist = _refresh_whitelist + freqtrade.pairlists._validate_whitelist = _refresh_whitelist patch_get_signal(freqtrade) freqtrade._process() assert freqtrade.active_pair_whitelist == ['NEO/BTC', 'LTC/BTC'] From 0929f59680f302b82b61a6894b9424c784892796 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 3 Dec 2018 20:48:51 +0100 Subject: [PATCH 40/62] Refactor pairlist-tests --- freqtrade/constants.py | 4 ++-- freqtrade/tests/pairlist/__init__.py | 0 .../{test_acl_pair.py => pairlist/test_pairlist.py} | 12 ++++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 freqtrade/tests/pairlist/__init__.py rename freqtrade/tests/{test_acl_pair.py => pairlist/test_pairlist.py} (89%) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index c200137cb..e0799d46f 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -15,7 +15,7 @@ DEFAULT_DB_DRYRUN_URL = 'sqlite://' UNLIMITED_STAKE_AMOUNT = 'unlimited' REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange'] ORDERTYPE_POSSIBILITIES = ['limit', 'market'] - +AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList'] TICKER_INTERVAL_MINUTES = { '1m': 1, @@ -127,7 +127,7 @@ CONF_SCHEMA = { 'whitelist': { 'type': 'object', 'properties': { - 'method': {'type': 'string'}, + 'method': {'type': 'string', 'enum': AVAILABLE_PAIRLISTS}, 'config': {'type': 'object'} }, 'required': ['method'] diff --git a/freqtrade/tests/pairlist/__init__.py b/freqtrade/tests/pairlist/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/pairlist/test_pairlist.py similarity index 89% rename from freqtrade/tests/test_acl_pair.py rename to freqtrade/tests/pairlist/test_pairlist.py index 5acd9abda..d0a5bb1f6 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/pairlist/test_pairlist.py @@ -3,6 +3,7 @@ from unittest.mock import MagicMock from freqtrade import OperationalException +from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.tests.conftest import get_patched_freqtradebot import pytest @@ -22,6 +23,9 @@ def whitelist_conf(default_conf): default_conf['exchange']['pair_blacklist'] = [ 'BLK/BTC' ] + default_conf['whitelist'] = {'method': 'StaticPairList', + 'config': {'number_assets': 3} + } return default_conf @@ -118,3 +122,11 @@ def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None with pytest.raises(OperationalException): get_patched_freqtradebot(mocker, default_conf) + + +@pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS) +def test_pairlist_class(mocker, whitelist_conf, pairlist): + whitelist_conf['whitelist']['method'] = pairlist + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + assert freqtrade.pairlists.name == pairlist + # TODO: add more tests From ab60571ac7117ba5a140ae9d7c41316c7529f60a Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 4 Dec 2018 06:13:39 +0100 Subject: [PATCH 41/62] Add sample config --- config_full.json.example | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config_full.json.example b/config_full.json.example index 6134e9cad..da4de5cec 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -39,6 +39,12 @@ "stoploss": "market", "stoploss_on_exchange": "false" }, + "whitelist": { + "method": "VolumePairList", + "config": { + "number_assets": 20 + } + }, "exchange": { "name": "bittrex", "key": "your_exchange_key", From ba3218a87d7928ee697da0796182c78b323a7ba1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 4 Dec 2018 07:12:56 +0100 Subject: [PATCH 42/62] Support multiple sorting variants --- config_full.json.example | 3 ++- freqtrade/pairlist/VolumePairList.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index da4de5cec..551d39452 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -42,7 +42,8 @@ "whitelist": { "method": "VolumePairList", "config": { - "number_assets": 20 + "number_assets": 20, + "sort_key": "quoteVolume" } }, "exchange": { diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index f45c57676..aec35a727 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -12,6 +12,8 @@ from freqtrade.pairlist.StaticPairList import StaticPairList from freqtrade import OperationalException logger = logging.getLogger(__name__) +SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] + class VolumePairList(StaticPairList): @@ -22,13 +24,21 @@ class VolumePairList(StaticPairList): self._whitelist = self._config['exchange']['pair_whitelist'] self._blacklist = self._config['exchange'].get('pair_blacklist', []) self._number_pairs = self._whitelistconf['number_assets'] + self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume') + if not self._freqtrade.exchange.exchange_has('fetchTickers'): raise OperationalException( 'Exchange does not support dynamic whitelist.' 'Please edit your config and restart the bot' ) + if not self.validate_keys(self._sort_key): + raise OperationalException( + f'key {self._sort_key} not in {SORT_VALUES}') # self.refresh_whitelist() + def validate_keys(self, key): + return key in SORT_VALUES + def short_desc(self) -> str: """ Short whitelist method description - used for startup-messages @@ -41,7 +51,7 @@ class VolumePairList(StaticPairList): Refreshes whitelist and assigns it to self._whitelist """ # Generate dynamic whitelist - pairs = self._gen_pair_whitelist(self._config['stake_currency']) + pairs = self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key) # Validate whitelist to only have active market pairs self._whitelist = self._validate_whitelist(pairs)[:self._number_pairs] From bf678164c79cb6ee4445561cfd156ce3fb75f46f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 4 Dec 2018 07:16:34 +0100 Subject: [PATCH 43/62] remove default param - fix tests --- freqtrade/pairlist/VolumePairList.py | 2 +- freqtrade/tests/pairlist/test_pairlist.py | 44 ++++++++++++++++------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index aec35a727..30f60f252 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -56,7 +56,7 @@ class VolumePairList(StaticPairList): self._whitelist = self._validate_whitelist(pairs)[:self._number_pairs] @cached(TTLCache(maxsize=1, ttl=1800)) - def _gen_pair_whitelist(self, base_currency: str, key: str = 'quoteVolume') -> List[str]: + def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]: """ Updates the whitelist with with a dynamically generated list :param base_currency: base currency as str diff --git a/freqtrade/tests/pairlist/test_pairlist.py b/freqtrade/tests/pairlist/test_pairlist.py index d0a5bb1f6..12ca559f6 100644 --- a/freqtrade/tests/pairlist/test_pairlist.py +++ b/freqtrade/tests/pairlist/test_pairlist.py @@ -76,7 +76,7 @@ def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): assert whitelist == freqtradebot.pairlists.whitelist -def test_refresh_whitelist_dynamic_empty(mocker, markets_empty, whitelist_conf): +def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets_empty) @@ -89,27 +89,27 @@ def test_refresh_whitelist_dynamic_empty(mocker, markets_empty, whitelist_conf): assert set(whitelist) == set(pairslist) -def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) +def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers) -> None: + whitelist_conf['whitelist']['method'] = 'VolumePairList' mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) + mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) # Test to retrieved BTC sorted on quoteVolume (default) - freqtrade.pairlists.refresh_whitelist() - - whitelist = freqtrade.pairlists.whitelist + whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='quoteVolume') assert whitelist == ['ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC'] # Test to retrieve BTC sorted on bidVolume - whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC', key='bidVolume') + whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='bidVolume') assert whitelist == ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'BLK/BTC'] # Test with USDT sorted on quoteVolume (default) - whitelist = freqtrade._gen_pair_whitelist(base_currency='USDT') + whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='USDT', key='quoteVolume') assert whitelist == ['TKN/USDT', 'ETH/USDT', 'LTC/USDT', 'BLK/USDT'] # Test with ETH (our fixture does not have ETH, so result should be empty) - whitelist = freqtrade._gen_pair_whitelist(base_currency='ETH') + whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='ETH', key='quoteVolume') assert whitelist == [] @@ -125,8 +125,28 @@ def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None @pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS) -def test_pairlist_class(mocker, whitelist_conf, pairlist): +def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): whitelist_conf['whitelist']['method'] = pairlist + mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + assert freqtrade.pairlists.name == pairlist - # TODO: add more tests + assert pairlist in freqtrade.pairlists.short_desc() + assert isinstance(freqtrade.pairlists.whitelist, list) + assert isinstance(freqtrade.pairlists.blacklist, list) + + whitelist = ['ETH/BTC', 'TKN/BTC'] + new_whitelist = freqtrade.pairlists._validate_whitelist(whitelist) + + assert set(whitelist) == set(new_whitelist) + + whitelist = ['ETH/BTC', 'TKN/BTC', 'TRX/ETH'] + new_whitelist = freqtrade.pairlists._validate_whitelist(whitelist) + # TRX/ETH was removed + assert set(['ETH/BTC', 'TKN/BTC']) == set(new_whitelist) + + whitelist = ['ETH/BTC', 'TKN/BTC', 'BLK/BTC'] + new_whitelist = freqtrade.pairlists._validate_whitelist(whitelist) + # BLK/BTC is in blacklist ... + assert set(['ETH/BTC', 'TKN/BTC']) == set(new_whitelist) From e7684b446b3c737a01da2d88dd0be08a4f4b78d8 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 4 Dec 2018 17:05:35 +0100 Subject: [PATCH 44/62] capital in trade extracted to a separated argument --- freqtrade/edge/__init__.py | 10 +++++++--- freqtrade/freqtradebot.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index a212c9849..49acbd3e7 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -160,19 +160,23 @@ class Edge(): return True - def stake_amount(self, pair: str, free_capital: float, total_capital: float) -> float: + def stake_amount(self, pair: str, free_capital: float, + total_capital: float, capital_in_trade: float) -> float: stoploss = self.stoploss(pair) - available_capital = total_capital * self._capital_percentage + available_capital = (total_capital + capital_in_trade) * self._capital_percentage allowed_capital_at_risk = available_capital * self._allowed_risk max_position_size = abs(allowed_capital_at_risk / stoploss) position_size = min(max_position_size, free_capital) if pair in self._cached_pairs: logger.info( 'winrate: %s, expectancy: %s, position size: %s, pair: %s,' + ' capital in trade: %s, free capital: %s, total capital: %s,' ' stoploss: %s, available capital: %s.', self._cached_pairs[pair].winrate, self._cached_pairs[pair].expectancy, - position_size, pair, stoploss, available_capital + position_size, pair, + capital_in_trade, free_capital, total_capital, + stoploss, available_capital ) return round(position_size, 15) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 6baef76bc..d782f4342 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -305,7 +305,7 @@ class FreqtradeBot(object): return self.edge.stake_amount( pair, self.wallets.get_free(self.config['stake_currency']), - self.wallets.get_total(self.config['stake_currency']) + + self.wallets.get_total(self.config['stake_currency']), Trade.total_open_trades_stakes() ) else: From 24f9ea29c605351ee61a7d4c8dbdabe7b158ca53 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 4 Dec 2018 17:13:46 +0100 Subject: [PATCH 45/62] tests fixed --- freqtrade/tests/edge/test_edge.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 74227cfe8..008413ff1 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -174,15 +174,18 @@ def test_stake_amount(mocker, edge_conf): )) free = 100 total = 100 - assert edge.stake_amount('E/F', free, total) == 25 + in_trade = 25 + assert edge.stake_amount('E/F', free, total, in_trade) == 31.25 free = 20 total = 100 - assert edge.stake_amount('E/F', free, total) == 20 + in_trade = 25 + assert edge.stake_amount('E/F', free, total, in_trade) == 20 free = 0 total = 100 - assert edge.stake_amount('E/F', free, total) == 0 + in_trade = 25 + assert edge.stake_amount('E/F', free, total, in_trade) == 0 def test_nonexisting_stake_amount(mocker, edge_conf): @@ -193,7 +196,8 @@ def test_nonexisting_stake_amount(mocker, edge_conf): 'E/F': PairInfo(-0.11, 0.66, 3.71, 0.50, 1.71, 10, 60), } )) - assert edge.stake_amount('N/O', 1, 2) == 0.1 + # should use strategy stoploss + assert edge.stake_amount('N/O', 1, 2, 1) == 0.15 def _validate_ohlc(buy_ohlc_sell_matrice): From 6ab907bef1f11a0478b11a0423aa7e8041c61f39 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 4 Dec 2018 20:23:03 +0100 Subject: [PATCH 46/62] Rename config whitelist to pairlist --- config_full.json.example | 2 +- freqtrade/configuration.py | 6 +++--- freqtrade/constants.py | 2 +- freqtrade/freqtradebot.py | 2 +- freqtrade/pairlist/VolumePairList.py | 2 +- freqtrade/tests/pairlist/test_pairlist.py | 22 +++++++++++----------- freqtrade/tests/rpc/test_rpc.py | 6 +++--- freqtrade/tests/rpc/test_rpc_telegram.py | 6 +++--- freqtrade/tests/test_configuration.py | 4 ++-- freqtrade/tests/test_freqtradebot.py | 6 +++--- 10 files changed, 29 insertions(+), 29 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 551d39452..e21ccbe4f 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -39,7 +39,7 @@ "stoploss": "market", "stoploss_on_exchange": "false" }, - "whitelist": { + "pairlist": { "method": "VolumePairList", "config": { "number_assets": 20, diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 7ddcf1639..cb087a1ab 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -111,9 +111,9 @@ class Configuration(object): # Add dynamic_whitelist if found if 'dynamic_whitelist' in self.args and self.args.dynamic_whitelist: # Update to volumePairList (the previous default) - config['whitelist'] = {'method': 'VolumePairList', - 'config': {'number_assets': self.args.dynamic_whitelist} - } + config['pairlist'] = {'method': 'VolumePairList', + 'config': {'number_assets': self.args.dynamic_whitelist} + } logger.warning( 'Parameter --dynamic-whitelist has been deprecated, ' 'and will be completely replaced by the whitelist dict in the future. ' diff --git a/freqtrade/constants.py b/freqtrade/constants.py index e0799d46f..178aafe79 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -124,7 +124,7 @@ CONF_SCHEMA = { 'ignore_roi_if_buy_signal_true': {'type': 'boolean'} } }, - 'whitelist': { + 'pairlist': { 'type': 'object', 'properties': { 'method': {'type': 'string', 'enum': AVAILABLE_PAIRLISTS}, diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 17b818b27..54374b209 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -59,7 +59,7 @@ class FreqtradeBot(object): self.persistence = None self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) - if self.config.get('whitelist', {}).get('method') == 'VolumePairList': + if self.config.get('pairlist', {}).get('method') == 'VolumePairList': self.pairlists: StaticPairList = VolumePairList(self, self.config) else: self.pairlists: StaticPairList = StaticPairList(self, self.config) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 30f60f252..6565caae6 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -20,7 +20,7 @@ class VolumePairList(StaticPairList): def __init__(self, freqtrade, config: dict) -> None: self._freqtrade = freqtrade self._config = config - self._whitelistconf = self._config.get('whitelist', {}).get('config') + self._whitelistconf = self._config.get('pairlist', {}).get('config') self._whitelist = self._config['exchange']['pair_whitelist'] self._blacklist = self._config['exchange'].get('pair_blacklist', []) self._number_pairs = self._whitelistconf['number_assets'] diff --git a/freqtrade/tests/pairlist/test_pairlist.py b/freqtrade/tests/pairlist/test_pairlist.py index 12ca559f6..b94859efa 100644 --- a/freqtrade/tests/pairlist/test_pairlist.py +++ b/freqtrade/tests/pairlist/test_pairlist.py @@ -23,9 +23,9 @@ def whitelist_conf(default_conf): default_conf['exchange']['pair_blacklist'] = [ 'BLK/BTC' ] - default_conf['whitelist'] = {'method': 'StaticPairList', - 'config': {'number_assets': 3} - } + default_conf['pairlist'] = {'method': 'StaticPairList', + 'config': {'number_assets': 3} + } return default_conf @@ -58,9 +58,9 @@ def test_refresh_pairlists(mocker, markets, whitelist_conf): def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): - whitelist_conf['whitelist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 5} - } + whitelist_conf['pairlist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 5} + } mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_markets=markets, @@ -90,7 +90,7 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers) -> None: - whitelist_conf['whitelist']['method'] = 'VolumePairList' + whitelist_conf['pairlist']['method'] = 'VolumePairList' mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) @@ -114,9 +114,9 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers) def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: - default_conf['whitelist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 10} - } + default_conf['pairlist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 10} + } mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) @@ -126,7 +126,7 @@ def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None @pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS) def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): - whitelist_conf['whitelist']['method'] = pairlist + whitelist_conf['pairlist']['method'] = pairlist mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 87124a9a7..2b271af31 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -662,9 +662,9 @@ def test_rpc_whitelist(mocker, default_conf) -> None: def test_rpc_whitelist_dynamic(mocker, default_conf) -> None: patch_coinmarketcap(mocker) patch_exchange(mocker) - default_conf['whitelist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 4} - } + default_conf['pairlist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 4} + } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index f63393935..c1a19f2a4 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -1034,9 +1034,9 @@ def test_whitelist_dynamic(default_conf, update, mocker) -> None: _send_msg=msg_mock ) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) - default_conf['whitelist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 4} - } + default_conf['pairlist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 4} + } freqtradebot = get_patched_freqtradebot(mocker, default_conf) telegram = Telegram(freqtradebot) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 5eda9d871..d013f1059 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -119,8 +119,8 @@ def test_load_config_with_params(default_conf, mocker) -> None: configuration = Configuration(args) validated_conf = configuration.load_config() - assert validated_conf.get('whitelist', {}).get('method') == 'VolumePairList' - assert validated_conf.get('whitelist', {}).get('config').get('number_assets') == 10 + assert validated_conf.get('pairlist', {}).get('method') == 'VolumePairList' + assert validated_conf.get('pairlist', {}).get('config').get('number_assets') == 10 assert validated_conf.get('strategy') == 'TestStrategy' assert validated_conf.get('strategy_path') == '/some/path' assert validated_conf.get('db_url') == 'sqlite:///someurl' diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index a10389261..6c61b893c 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -2532,9 +2532,9 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order def test_startup_messages(default_conf, mocker): - default_conf['whitelist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 20} - } + default_conf['pairlist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 20} + } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, default_conf) assert freqtrade.state is State.RUNNING From 1c3ce265f15060539898a5f85ab7690f75c69b2b Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 4 Dec 2018 20:24:52 +0100 Subject: [PATCH 47/62] documentation for pairlists --- docs/bot-usage.md | 4 +++- docs/configuration.md | 35 +++++++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index ec8873b12..114e7613e 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -36,7 +36,7 @@ optional arguments: --strategy-path PATH specify additional strategy lookup path --dynamic-whitelist [INT] dynamically generate and update whitelist based on 24h - BaseVolume (default: 20) + BaseVolume (default: 20) DEPRECATED --db-url PATH Override trades database URL, this is useful if dry_run is enabled or in custom deployments (default: sqlite:///tradesv3.sqlite) @@ -89,6 +89,8 @@ This is very simple. Copy paste your strategy file into the folder ### How to use --dynamic-whitelist? +> Dynamic-whitelist is deprecated. Please move your configurations to the configuration as outlined [here](docs/configuration.md#Dynamic-Pairlists) + Per default `--dynamic-whitelist` will retrieve the 20 currencies based on BaseVolume. This value can be changed when you run the script. diff --git a/docs/configuration.md b/docs/configuration.md index 5b8baa43b..6d8db6600 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -34,8 +34,8 @@ The table below will list all configuration parameters. | `bid_strategy.ask_last_balance` | 0.0 | Yes | Set the bidding price. More information below. | `bid_strategy.use_order_book` | false | No | Allows buying of pair using the rates in Order Book Bids. | `bid_strategy.order_book_top` | 0 | No | Bot will use the top N rate in Order Book Bids. Ie. a value of 2 will allow the bot to pick the 2nd bid rate in Order Book Bids. -| `bid_strategy.check_depth_of_market.enabled` | false | No | Does not buy if the % difference of buy orders and sell orders is met in Order Book. -| `bid_strategy.check_depth_of_market.bids_to_ask_delta` | 0 | No | The % difference of buy orders and sell orders found in Order Book. A value lesser than 1 means sell orders is greater, while value greater than 1 means buy orders is higher. +| `bid_strategy. check_depth_of_market.enabled` | false | No | Does not buy if the % difference of buy orders and sell orders is met in Order Book. +| `bid_strategy. check_depth_of_market.bids_to_ask_delta` | 0 | No | The % difference of buy orders and sell orders found in Order Book. A value lesser than 1 means sell orders is greater, while value greater than 1 means buy orders is higher. | `ask_strategy.use_order_book` | false | No | Allows selling of open traded pair using the rates in Order Book Asks. | `ask_strategy.order_book_min` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `ask_strategy.order_book_max` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. @@ -52,6 +52,8 @@ The table below will list all configuration parameters. | `experimental.use_sell_signal` | false | No | Use your sell strategy in addition of the `minimal_roi`. | `experimental.sell_profit_only` | false | No | waits until you have made a positive profit before taking a sell decision. | `experimental.ignore_roi_if_buy_signal` | false | No | Does not sell if the buy-signal is still active. Takes preference over `minimal_roi` and `use_sell_signal` +| `pairlist.method` | StaticPairList | No | Use Static whitelist. [More information below](#dynamic-pairlists). +| `pairlist.config` | None | No | Additional configuration for dynamic pairlists. [More information below](#dynamic-pairlists). | `telegram.enabled` | true | Yes | Enable or not the usage of Telegram. | `telegram.token` | token | No | Your Telegram bot token. Only required if `telegram.enabled` is `true`. | `telegram.chat_id` | chat_id | No | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. @@ -147,7 +149,7 @@ This can be set in the configuration or in the strategy. Configuration overwrite If this is configured, all 4 values (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`) need to be present, otherwise the bot warn about it and will fail to start. The below is the default which is used if this is not configured in either Strategy or configuration. -``` json +``` python "order_types": { "buy": "limit", "sell": "limit", @@ -211,13 +213,38 @@ creating trades. Once you will be happy with your bot performance, you can switch it to production mode. +### Dynamic Pairlists + +Dynamic pairlists allow configuration of the pair-selection. +Basically, the bot runs against all pairs (with that stake) on the exchange, and a number of assets (`number_assets`) is selected based on different criteria. + +By *default*, a Static Pairlist is used (configured as `"pair_whitelist"` under the `"exchange"` section of this configuration). + +#### Available Pairlist methods + +* `"StaticPairList"` + * uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklist` +* `"VolumePairList"` + * Formerly available as `--dynamic-whitelist []` + * Selects `number_assets` top pairs based on `sort_key`, which can be one of `askVolume`, `bidVolume` and `quoteVolume`, defaults to `quoteVolume`. + +```json +"pairlist": { + "method": "VolumePairList", + "config": { + "number_assets": 20, + "sort_key": "quoteVolume" + } + }, +``` + ## Switch to production mode In production mode, the bot will engage your money. Be careful a wrong strategy can lose all your money. Be aware of what you are doing when you run it in production mode. -### To switch your bot in production mode: +### To switch your bot in production mode 1. Edit your `config.json` file From 37ebe05c6dee78a347493cd1f50b3aedec239539 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 5 Dec 2018 13:34:06 +0100 Subject: [PATCH 48/62] Update ccxt from 1.17.581 to 1.17.583 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2befc4d5c..b59104dfe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.581 +ccxt==1.17.583 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 21906e4892d2a04546fdfdf6875d670eac83dbda Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 5 Dec 2018 19:48:50 +0100 Subject: [PATCH 49/62] Remove duplicate code --- freqtrade/pairlist/VolumePairList.py | 36 ++-------------------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 6565caae6..bd562da83 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -31,12 +31,12 @@ class VolumePairList(StaticPairList): 'Exchange does not support dynamic whitelist.' 'Please edit your config and restart the bot' ) - if not self.validate_keys(self._sort_key): + if not self._validate_keys(self._sort_key): raise OperationalException( f'key {self._sort_key} not in {SORT_VALUES}') # self.refresh_whitelist() - def validate_keys(self, key): + def _validate_keys(self, key): return key in SORT_VALUES def short_desc(self) -> str: @@ -73,35 +73,3 @@ class VolumePairList(StaticPairList): pairs = [s['symbol'] for s in sorted_tickers] return pairs - def _validate_whitelist(self, whitelist: List[str]) -> List[str]: - """ - Check available markets and remove pair from whitelist if necessary - :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to - trade - :return: the list of pairs the user wants to trade without the one unavailable or - black_listed - """ - sanitized_whitelist = whitelist - markets = self._freqtrade.exchange.get_markets() - - # Filter to markets in stake currency - markets = [m for m in markets if m['quote'] == self._config['stake_currency']] - known_pairs = set() - - for market in markets: - pair = market['symbol'] - # pair is not int the generated dynamic market, or in the blacklist ... ignore it - if pair not in whitelist or pair in self.blacklist: - continue - # else the pair is valid - known_pairs.add(pair) - # Market is not active - if not market['active']: - sanitized_whitelist.remove(pair) - logger.info( - 'Ignoring %s from whitelist. Market is not active.', - pair - ) - - # We need to remove pairs that are unknown - return [x for x in sanitized_whitelist if x in known_pairs] From 1a10e1286193a8fa7d76b7dfc7dd479b17ca6ace Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 5 Dec 2018 19:48:59 +0100 Subject: [PATCH 50/62] Documentation and developer documentation --- CONTRIBUTING.md | 4 ++++ docs/developer.md | 60 +++++++++++++++++++++++++++++++++++++++++++++++ docs/index.md | 3 ++- 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 docs/developer.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 58185b27c..c9a967834 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -72,6 +72,10 @@ pip3.6 install mypy mypy freqtrade ``` +## Getting started + +Best start by reading the [documentation](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) to get a feel for what is possible with the bot, or head straight to the [Developer-documentation](https://github.com/freqtrade/freqtrade/blob/develop/docs/developer.md) (WIP) which should help you getting started. + ## (Core)-Committer Guide ### Process: Pull Requests diff --git a/docs/developer.md b/docs/developer.md new file mode 100644 index 000000000..35410de1f --- /dev/null +++ b/docs/developer.md @@ -0,0 +1,60 @@ +# Development Help + +This page is intended for developers of FreqTrade, people who want to contribute to the FreqTrade codebase or documentation, or people who want to understand the source code of the application they're running. + +All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel in [slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE) where you can ask questions. + + +## Module + +### Dynamic Pairlist + +You have a great idea for a new pair selection algorithm you would like to try out? Great. +Hopefully you also want to contribute this back upstream. + +Whatever your motivations are - This should get you off the ground in trying to develop a new Pairlist provider. + +First of all, have a look at the [VolumePairList](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/pairlist/VolumePairList.py) provider, and best copy this file with a name of your new Pairlist Provider. + +This is a simple provider, which however serves as a good example on how to start developing. + +Next, modify the classname of the provider (ideally align this with the Filename). + +Now, let's step through the methods which require actions: + +#### configuration + +Configuration for PairListProvider is done in the bot configuration file in the element `"pairlist"`. +This Pairlist-object may contain a `"config"` dict with additional configurations for the configured pairlist. +By convention, `"number_assets"` is used to specify the maximum number of pairs to keep in the whitelist. Please follow this to ensure a consistent user experience. + +Additional elements can be configured as needed. `VolumePairList` uses `"sort_key"` to specify the sorting value - however feel free to specify whatever is necessary for your great algorithm to be successfull and dynamic. + +#### short_desc + +Returns a description used for Telegram messages. +This should coutain the name of the Provider, as well as a short description containing the number of assets. Please follow the format `"PairlistName - top/bottom X pairs"`. + +#### refresh_whitelist + +Override this method and run all calculations needed in this method. +This is called with each iteration of the bot - so consider implementing caching for compute/network heavy calculations. + +Assign the resulting whiteslist to `self._whitelist` and `self._blacklist` respectively. These will then be used to run the bot in this iteration. Pairs with open trades will be added to the whitelist to have the sell-methods run correctly. + +Please also run `self._validate_whitelist(pairs)` and to check and remove pairs with inactive markets. This function is available in the Parent class (`StaticPairList`) and should ideally not be overwritten. + +##### sample + +``` python + def refresh_whitelist(self) -> None: + # Generate dynamic whitelist + pairs = self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key) + # Validate whitelist to only have active market pairs + self._whitelist = self._validate_whitelist(pairs)[:self._number_pairs] +``` + +#### _gen_pair_whitelist + +This is a simple method used by `VolumePairList` - however serves as a good example. +It implements caching (`@cached(TTLCache(maxsize=1, ttl=1800))`) as well as a configuration option to allow different (but similar) strategies to work with the same PairListProvider. diff --git a/docs/index.md b/docs/index.md index c64b9c188..2f579643e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -35,4 +35,5 @@ Pull-request. Do not hesitate to reach us on - [Run tests & Check PEP8 compliance](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) - [FAQ](https://github.com/freqtrade/freqtrade/blob/develop/docs/faq.md) - [SQL cheatsheet](https://github.com/freqtrade/freqtrade/blob/develop/docs/sql_cheatsheet.md) -- [Sandbox Testing](https://github.com/freqtrade/freqtrade/blob/develop/docs/sandbox-testing.md)) +- [Sandbox Testing](https://github.com/freqtrade/freqtrade/blob/develop/docs/sandbox-testing.md) +- [Developer Docs](https://github.com/freqtrade/freqtrade/blob/develop/docs/developer.md) From 43031aa3bb61828b764bde099075c2213c855023 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 5 Dec 2018 20:44:41 +0100 Subject: [PATCH 51/62] Add missing path-error handler for hyperopt --- freqtrade/resolvers/hyperopt_resolver.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index da7b65648..eb91c0e89 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -52,11 +52,14 @@ class HyperOptResolver(IResolver): abs_paths.insert(0, Path(extra_dir)) for _path in abs_paths: - hyperopt = self._search_object(directory=_path, object_type=IHyperOpt, - object_name=hyperopt_name) - if hyperopt: - logger.info('Using resolved hyperopt %s from \'%s\'', hyperopt_name, _path) - return hyperopt + try: + hyperopt = self._search_object(directory=_path, object_type=IHyperOpt, + object_name=hyperopt_name) + if hyperopt: + logger.info('Using resolved hyperopt %s from \'%s\'', hyperopt_name, _path) + return hyperopt + except FileNotFoundError: + logger.warning('Path "%s" does not exist', _path.relative_to(Path.cwd())) raise ImportError( "Impossible to load Hyperopt '{}'. This class does not exist" From 3e2fa580291066b3c4fd0948a4fdf0eea6935569 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 5 Dec 2018 20:44:56 +0100 Subject: [PATCH 52/62] load pairlists via resolver --- docs/developer.md | 14 +++- freqtrade/freqtradebot.py | 12 ++-- freqtrade/pairlist/IPairList.py | 91 ++++++++++++++++++++++++ freqtrade/pairlist/StaticPairList.py | 72 ++----------------- freqtrade/pairlist/VolumePairList.py | 16 ++--- freqtrade/resolvers/__init__.py | 1 + freqtrade/resolvers/pairlist_resolver.py | 59 +++++++++++++++ 7 files changed, 179 insertions(+), 86 deletions(-) create mode 100644 freqtrade/pairlist/IPairList.py create mode 100644 freqtrade/resolvers/pairlist_resolver.py diff --git a/docs/developer.md b/docs/developer.md index 35410de1f..c564acffb 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -20,6 +20,16 @@ This is a simple provider, which however serves as a good example on how to star Next, modify the classname of the provider (ideally align this with the Filename). +The base-class provides the an instance of the bot (`self._freqtrade`), as well as the configuration (`self._config`), and initiates both `_blacklist` and `_whitelist`. + +```python + self._freqtrade = freqtrade + self._config = config + self._whitelist = self._config['exchange']['pair_whitelist'] + self._blacklist = self._config['exchange'].get('pair_blacklist', []) +``` + + Now, let's step through the methods which require actions: #### configuration @@ -35,7 +45,7 @@ Additional elements can be configured as needed. `VolumePairList` uses `"sort_ke Returns a description used for Telegram messages. This should coutain the name of the Provider, as well as a short description containing the number of assets. Please follow the format `"PairlistName - top/bottom X pairs"`. -#### refresh_whitelist +#### refresh_pairlist Override this method and run all calculations needed in this method. This is called with each iteration of the bot - so consider implementing caching for compute/network heavy calculations. @@ -47,7 +57,7 @@ Please also run `self._validate_whitelist(pairs)` and to check and remove pairs ##### sample ``` python - def refresh_whitelist(self) -> None: + def refresh_pairlist(self) -> None: # Generate dynamic whitelist pairs = self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key) # Validate whitelist to only have active market pairs diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 54374b209..01de328f6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -19,12 +19,10 @@ from freqtrade.wallets import Wallets from freqtrade.edge import Edge from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType -from freqtrade.resolvers import StrategyResolver +from freqtrade.resolvers import StrategyResolver, PairListResolver from freqtrade.state import State from freqtrade.strategy.interface import SellType, IStrategy from freqtrade.exchange.exchange_helpers import order_book_to_dataframe -from freqtrade.pairlist.StaticPairList import StaticPairList -from freqtrade.pairlist.VolumePairList import VolumePairList logger = logging.getLogger(__name__) @@ -59,10 +57,8 @@ class FreqtradeBot(object): self.persistence = None self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) - if self.config.get('pairlist', {}).get('method') == 'VolumePairList': - self.pairlists: StaticPairList = VolumePairList(self, self.config) - else: - self.pairlists: StaticPairList = StaticPairList(self, self.config) + pairlistname = self.config.get('pairlist', {}).get('method', 'StaticPairList') + self.pairlists = PairListResolver(pairlistname, self, self.config).pairlist # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ @@ -151,7 +147,7 @@ class FreqtradeBot(object): state_changed = False try: # Refresh whitelist - self.pairlists.refresh_whitelist() + self.pairlists.refresh_pairlist() self.active_pair_whitelist = self.pairlists.whitelist # Calculating Edge positiong diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py new file mode 100644 index 000000000..6b5b0db4b --- /dev/null +++ b/freqtrade/pairlist/IPairList.py @@ -0,0 +1,91 @@ +""" +Static List provider + +Provides lists as configured in config.json + + """ +import logging +from abc import ABC, abstractmethod +from typing import List + +logger = logging.getLogger(__name__) + + +class IPairList(ABC): + + def __init__(self, freqtrade, config: dict) -> None: + self._freqtrade = freqtrade + self._config = config + self._whitelist = self._config['exchange']['pair_whitelist'] + self._blacklist = self._config['exchange'].get('pair_blacklist', []) + + @property + def name(self) -> str: + """ + Gets name of the class + -> no need to overwrite in subclasses + """ + return self.__class__.__name__ + + @property + def whitelist(self) -> List[str]: + """ + Has the current whitelist + -> no need to overwrite in subclasses + """ + return self._whitelist + + @property + def blacklist(self) -> List[str]: + """ + Has the current blacklist + -> no need to overwrite in subclasses + """ + return self._blacklist + + @abstractmethod + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + -> Please overwrite in subclasses + """ + + @abstractmethod + def refresh_pairlist(self) -> None: + """ + Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively + -> Please overwrite in subclasses + """ + + def _validate_whitelist(self, whitelist: List[str]) -> List[str]: + """ + Check available markets and remove pair from whitelist if necessary + :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to + trade + :return: the list of pairs the user wants to trade without the one unavailable or + black_listed + """ + sanitized_whitelist = whitelist + markets = self._freqtrade.exchange.get_markets() + + # Filter to markets in stake currency + markets = [m for m in markets if m['quote'] == self._config['stake_currency']] + known_pairs = set() + + for market in markets: + pair = market['symbol'] + # pair is not int the generated dynamic market, or in the blacklist ... ignore it + if pair not in whitelist or pair in self.blacklist: + continue + # else the pair is valid + known_pairs.add(pair) + # Market is not active + if not market['active']: + sanitized_whitelist.remove(pair) + logger.info( + 'Ignoring %s from whitelist. Market is not active.', + pair + ) + + # We need to remove pairs that are unknown + return [x for x in sanitized_whitelist if x in known_pairs] diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 5b0e37357..5896e814a 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -5,43 +5,16 @@ Provides lists as configured in config.json """ import logging -from typing import List + +from freqtrade.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) -class StaticPairList(object): +class StaticPairList(IPairList): def __init__(self, freqtrade, config: dict) -> None: - self._freqtrade = freqtrade - self._config = config - self._whitelist = self._config['exchange']['pair_whitelist'] - self._blacklist = self._config['exchange'].get('pair_blacklist', []) - # self.refresh_whitelist() - - @property - def name(self) -> str: - """ - Gets name of the class - -> no need to overwrite in subclasses - """ - return self.__class__.__name__ - - @property - def whitelist(self) -> List[str]: - """ - Has the current whitelist - -> no need to overwrite in subclasses - """ - return self._whitelist - - @property - def blacklist(self) -> List[str]: - """ - Has the current blacklist - -> no need to overwrite in subclasses - """ - return self._blacklist + super().__init__(freqtrade, config) def short_desc(self) -> str: """ @@ -50,41 +23,8 @@ class StaticPairList(object): """ return f"{self.name}: {self.whitelist}" - def refresh_whitelist(self) -> None: + def refresh_pairlist(self) -> None: """ - Refreshes whitelist and assigns it to self._whitelist + Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively """ self._whitelist = self._validate_whitelist(self._config['exchange']['pair_whitelist']) - - def _validate_whitelist(self, whitelist: List[str]) -> List[str]: - """ - Check available markets and remove pair from whitelist if necessary - :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to - trade - :return: the list of pairs the user wants to trade without the one unavailable or - black_listed - """ - sanitized_whitelist = whitelist - markets = self._freqtrade.exchange.get_markets() - - # Filter to markets in stake currency - markets = [m for m in markets if m['quote'] == self._config['stake_currency']] - known_pairs = set() - - for market in markets: - pair = market['symbol'] - # pair is not int the generated dynamic market, or in the blacklist ... ignore it - if pair not in whitelist or pair in self.blacklist: - continue - # else the pair is valid - known_pairs.add(pair) - # Market is not active - if not market['active']: - sanitized_whitelist.remove(pair) - logger.info( - 'Ignoring %s from whitelist. Market is not active.', - pair - ) - - # We need to remove pairs that are unknown - return [x for x in sanitized_whitelist if x in known_pairs] diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index bd562da83..9d3d46169 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -8,21 +8,18 @@ import logging from typing import List from cachetools import TTLCache, cached -from freqtrade.pairlist.StaticPairList import StaticPairList +from freqtrade.pairlist.IPairList import IPairList from freqtrade import OperationalException logger = logging.getLogger(__name__) SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] -class VolumePairList(StaticPairList): +class VolumePairList(IPairList): def __init__(self, freqtrade, config: dict) -> None: - self._freqtrade = freqtrade - self._config = config + super().__init__(freqtrade, config) self._whitelistconf = self._config.get('pairlist', {}).get('config') - self._whitelist = self._config['exchange']['pair_whitelist'] - self._blacklist = self._config['exchange'].get('pair_blacklist', []) self._number_pairs = self._whitelistconf['number_assets'] self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume') @@ -34,7 +31,6 @@ class VolumePairList(StaticPairList): if not self._validate_keys(self._sort_key): raise OperationalException( f'key {self._sort_key} not in {SORT_VALUES}') - # self.refresh_whitelist() def _validate_keys(self, key): return key in SORT_VALUES @@ -46,9 +42,10 @@ class VolumePairList(StaticPairList): """ return f"{self.name} - top {self._whitelistconf['number_assets']} volume pairs." - def refresh_whitelist(self) -> None: + def refresh_pairlist(self) -> None: """ - Refreshes whitelist and assigns it to self._whitelist + Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively + -> Please overwrite in subclasses """ # Generate dynamic whitelist pairs = self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key) @@ -72,4 +69,3 @@ class VolumePairList(StaticPairList): sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) pairs = [s['symbol'] for s in sorted_tickers] return pairs - diff --git a/freqtrade/resolvers/__init__.py b/freqtrade/resolvers/__init__.py index 84e3bcdcd..da2987b27 100644 --- a/freqtrade/resolvers/__init__.py +++ b/freqtrade/resolvers/__init__.py @@ -1,3 +1,4 @@ from freqtrade.resolvers.iresolver import IResolver # noqa: F401 from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver # noqa: F401 +from freqtrade.resolvers.pairlist_resolver import PairListResolver # noqa: F401 from freqtrade.resolvers.strategy_resolver import StrategyResolver # noqa: F401 diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py new file mode 100644 index 000000000..286cea5bf --- /dev/null +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -0,0 +1,59 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load custom hyperopts +""" +import logging +from pathlib import Path + +from freqtrade.pairlist.IPairList import IPairList +from freqtrade.resolvers import IResolver + +logger = logging.getLogger(__name__) + + +class PairListResolver(IResolver): + """ + This class contains all the logic to load custom hyperopt class + """ + + __slots__ = ['pairlist'] + + def __init__(self, pairlist_name: str, freqtrade, config: dict) -> None: + """ + Load the custom class from config parameter + :param config: configuration dictionary or None + """ + self.pairlist = self._load_pairlist(pairlist_name, kwargs={'freqtrade': freqtrade, + 'config': config}) + + def _load_pairlist( + self, pairlist_name: str, kwargs: dict) -> IPairList: + """ + Search and loads the specified pairlist. + :param pairlist_name: name of the module to import + :param extra_dir: additional directory to search for the given pairlist + :return: PairList instance or None + """ + current_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() + + abs_paths = [ + current_path.parent.parent.joinpath('user_data/pairlist'), + current_path, + ] + + for _path in abs_paths: + try: + pairlist = self._search_object(directory=_path, object_type=IPairList, + object_name=pairlist_name, + kwargs=kwargs) + if pairlist: + logger.info('Using resolved pairlist %s from \'%s\'', pairlist_name, _path) + return pairlist + except FileNotFoundError: + logger.warning('Path "%s" does not exist', _path.relative_to(Path.cwd())) + + raise ImportError( + "Impossible to load Pairlist '{}'. This class does not exist" + " or contains Python code errors".format(pairlist_name) + ) From 2f0d7a1aea8510266bbec118fd394991af622b15 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 5 Dec 2018 20:45:11 +0100 Subject: [PATCH 53/62] Add specific test --- freqtrade/tests/pairlist/test_pairlist.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/freqtrade/tests/pairlist/test_pairlist.py b/freqtrade/tests/pairlist/test_pairlist.py index b94859efa..83c3817a7 100644 --- a/freqtrade/tests/pairlist/test_pairlist.py +++ b/freqtrade/tests/pairlist/test_pairlist.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock from freqtrade import OperationalException from freqtrade.constants import AVAILABLE_PAIRLISTS +from freqtrade.resolvers import PairListResolver from freqtrade.tests.conftest import get_patched_freqtradebot import pytest @@ -30,12 +31,21 @@ def whitelist_conf(default_conf): return default_conf +def test_load_pairlist_noexist(mocker, markets, default_conf): + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) + with pytest.raises(ImportError, + match=r"Impossible to load Pairlist 'NonexistingPairList'." + r" This class does not exist or contains Python code errors"): + PairListResolver('NonexistingPairList', freqtradebot, default_conf).pairlist + + def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) - freqtradebot.pairlists.refresh_whitelist() + freqtradebot.pairlists.refresh_pairlist() # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed @@ -49,7 +59,7 @@ def test_refresh_pairlists(mocker, markets, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) - freqtradebot.pairlists.refresh_whitelist() + freqtradebot.pairlists.refresh_pairlist() # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed @@ -57,7 +67,7 @@ def test_refresh_pairlists(mocker, markets, whitelist_conf): assert whitelist_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist -def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): +def test_refresh_pairlist_dynamic(mocker, markets, tickers, whitelist_conf): whitelist_conf['pairlist'] = {'method': 'VolumePairList', 'config': {'number_assets': 5} } @@ -71,7 +81,7 @@ def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): # argument: use the whitelist dynamically by exchange-volume whitelist = ['ETH/BTC', 'TKN/BTC'] - freqtradebot.pairlists.refresh_whitelist() + freqtradebot.pairlists.refresh_pairlist() assert whitelist == freqtradebot.pairlists.whitelist @@ -83,7 +93,7 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): # argument: use the whitelist dynamically by exchange-volume whitelist = [] whitelist_conf['exchange']['pair_whitelist'] = [] - freqtradebot.pairlists.refresh_whitelist() + freqtradebot.pairlists.refresh_pairlist() pairslist = whitelist_conf['exchange']['pair_whitelist'] assert set(whitelist) == set(pairslist) From 0c10719037137778f3b81e7d90fe0ca5aa1058ba Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 6 Dec 2018 06:57:07 +0100 Subject: [PATCH 54/62] Specify JsonValidatorversion explicitly without doing that, it exclusiveMaximum raises an exception as jsonschema defaults to the latest version (Draft6) which changes behaviour of this property. fixes #1233 --- freqtrade/configuration.py | 2 +- freqtrade/tests/test_configuration.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index feec0cb43..e9daf4a99 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -275,7 +275,7 @@ class Configuration(object): :return: Returns the config if valid, otherwise throw an exception """ try: - validate(conf, constants.CONF_SCHEMA) + validate(conf, constants.CONF_SCHEMA, Draft4Validator) return conf except ValidationError as exception: logger.critical( diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 23fefd3cd..5a0ae2636 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -6,7 +6,7 @@ import logging from unittest.mock import MagicMock import pytest -from jsonschema import validate, ValidationError +from jsonschema import validate, ValidationError, Draft4Validator from freqtrade import constants from freqtrade import OperationalException @@ -486,4 +486,4 @@ def test_load_config_warn_forcebuy(default_conf, mocker, caplog) -> None: def test_validate_default_conf(default_conf) -> None: - validate(default_conf, constants.CONF_SCHEMA) + validate(default_conf, constants.CONF_SCHEMA, Draft4Validator) From bf1841d2a81b1231d91409d1053299963a87c057 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 6 Dec 2018 13:34:06 +0100 Subject: [PATCH 55/62] Update ccxt from 1.17.583 to 1.18.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b59104dfe..61a153d45 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.583 +ccxt==1.18.2 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 0ea7dc92722e82f059f43d71538395bf5c06a03d Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 6 Dec 2018 13:51:06 +0100 Subject: [PATCH 56/62] test added for total open trade stake amount from schalchemy --- freqtrade/tests/test_freqtradebot.py | 33 ++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index e8d80b91f..ed3271838 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -342,6 +342,39 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, assert freqtrade.handle_trade(trade) is False +def test_total_open_trades_stakes(mocker, default_conf, ticker, + limit_buy_order, fee, markets) -> None: + patch_RPCManager(mocker) + patch_exchange(mocker) + default_conf['stake_amount'] = 0.0000098751 + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=ticker, + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + get_markets=markets + ) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + freqtrade.create_trade() + trade = Trade.query.first() + + assert trade is not None + assert trade.stake_amount == 0.0000098751 + assert trade.is_open + assert trade.open_date is not None + + freqtrade.create_trade() + trade = Trade.query.order_by(Trade.id.desc()).first() + + assert trade is not None + assert trade.stake_amount == 0.0000098751 + assert trade.is_open + assert trade.open_date is not None + + assert Trade.total_open_trades_stakes() == 1.97502e-05 + + def test_get_min_pair_stake_amount(mocker, default_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) From a63f123b6ddc9ece400c5503b51f38e05fb6e948 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 6 Dec 2018 19:36:33 +0100 Subject: [PATCH 57/62] Check if number_assets is defined, as it's required by VolumePairList --- docs/developer.md | 2 +- freqtrade/pairlist/VolumePairList.py | 4 ++++ freqtrade/tests/pairlist/test_pairlist.py | 8 ++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/developer.md b/docs/developer.md index c564acffb..9137f16ca 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -43,7 +43,7 @@ Additional elements can be configured as needed. `VolumePairList` uses `"sort_ke #### short_desc Returns a description used for Telegram messages. -This should coutain the name of the Provider, as well as a short description containing the number of assets. Please follow the format `"PairlistName - top/bottom X pairs"`. +This should contain the name of the Provider, as well as a short description containing the number of assets. Please follow the format `"PairlistName - top/bottom X pairs"`. #### refresh_pairlist diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 9d3d46169..262e4bf59 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -20,6 +20,10 @@ class VolumePairList(IPairList): def __init__(self, freqtrade, config: dict) -> None: super().__init__(freqtrade, config) self._whitelistconf = self._config.get('pairlist', {}).get('config') + if 'number_assets' not in self._whitelistconf: + raise OperationalException( + f'`number_assets` not specified. Please check your configuration ' + 'for "pairlist.config.number_assets"') self._number_pairs = self._whitelistconf['number_assets'] self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume') diff --git a/freqtrade/tests/pairlist/test_pairlist.py b/freqtrade/tests/pairlist/test_pairlist.py index 83c3817a7..9f90aac6e 100644 --- a/freqtrade/tests/pairlist/test_pairlist.py +++ b/freqtrade/tests/pairlist/test_pairlist.py @@ -85,6 +85,14 @@ def test_refresh_pairlist_dynamic(mocker, markets, tickers, whitelist_conf): assert whitelist == freqtradebot.pairlists.whitelist + whitelist_conf['pairlist'] = {'method': 'VolumePairList', + 'config': {} + } + with pytest.raises(OperationalException, + match=r'`number_assets` not specified. Please check your configuration ' + r'for "pairlist.config.number_assets"'): + PairListResolver('VolumePairList', freqtradebot, whitelist_conf).pairlist + def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) From 8f19c83f6b8fb2f21d8f8ad8c925ae95d3ac907a Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 6 Dec 2018 19:39:25 +0100 Subject: [PATCH 58/62] Refrase documentation --- docs/configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 6d8db6600..c4c0bed28 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -215,8 +215,8 @@ production mode. ### Dynamic Pairlists -Dynamic pairlists allow configuration of the pair-selection. -Basically, the bot runs against all pairs (with that stake) on the exchange, and a number of assets (`number_assets`) is selected based on different criteria. +Dynamic pairlists select pairs for you based on the logic configured. +The bot runs against all pairs (with that stake) on the exchange, and a number of assets (`number_assets`) is selected based on the selected criteria. By *default*, a Static Pairlist is used (configured as `"pair_whitelist"` under the `"exchange"` section of this configuration). From ac9f19aee5f3932c2b63a0109dc6113ba6ee9fd9 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 7 Dec 2018 13:34:07 +0100 Subject: [PATCH 59/62] Update ccxt from 1.18.2 to 1.18.10 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 61a153d45..b3fe9f5a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.2 +ccxt==1.18.10 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From b3b6eda2ba6b0f367f4a10ddc4f7d07234fedc80 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 8 Dec 2018 13:33:06 +0100 Subject: [PATCH 60/62] Update ccxt from 1.18.10 to 1.18.11 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b3fe9f5a6..a434b2f2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.10 +ccxt==1.18.11 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 3b951c3817dd949a6d8f0e63802f8bd7293f00f3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Dec 2018 09:03:17 +0100 Subject: [PATCH 61/62] Drop indexes on renamed table avoid naming conflicts on recreate (indexes are not renamed, and keeping them on backup tables does not really make sense). fixes #1396 --- freqtrade/persistence.py | 3 +++ freqtrade/tests/test_persistence.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 71752d58e..a393eb318 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -100,6 +100,9 @@ def check_migrate(engine) -> None: # Schema migration necessary engine.execute(f"alter table trades rename to {table_back_name}") + # drop indexes on backup table + for index in inspector.get_indexes(table_back_name): + engine.execute(f"drop index {index['name']}") # let SQLAlchemy create the schema as required _DECL_BASE.metadata.create_all(engine) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index d0a209f40..a7b21bc1d 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -446,6 +446,8 @@ def test_migrate_new(mocker, default_conf, fee, caplog): # Create table using the old format engine.execute(create_table_old) + engine.execute("create index ix_trades_is_open on trades(is_open)") + engine.execute("create index ix_trades_pair on trades(pair)") engine.execute(insert_table_old) # fake previous backup From acb96eb501803aee9ad731c3e2534f40895ffdff Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 9 Dec 2018 13:33:06 +0100 Subject: [PATCH 62/62] Update ccxt from 1.18.11 to 1.18.13 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a434b2f2c..72c416b60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.11 +ccxt==1.18.13 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1