From 46ac46a5d3a9629b7f9d3ce8c704f693336b811f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 03:17:23 +0000 Subject: [PATCH 01/63] Bump ccxt from 1.79.81 to 1.80.61 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.79.81 to 1.80.61. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.79.81...1.80.61) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 571d1892c..de14b9f2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.22.3 pandas==1.4.2 pandas-ta==0.3.14b -ccxt==1.79.81 +ccxt==1.80.61 # Pin cryptography for now due to rust build errors with piwheels cryptography==36.0.2 aiohttp==3.8.1 From 5bfa2186a77f774cb3043edabafd56745dab3a53 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 03:17:29 +0000 Subject: [PATCH 02/63] Bump pytest from 7.1.1 to 7.1.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.1.1 to 7.1.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.1.1...7.1.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4fb4456f0..7f914875b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,7 +9,7 @@ flake8==4.0.1 flake8-tidy-imports==4.6.0 mypy==0.942 pre-commit==2.18.1 -pytest==7.1.1 +pytest==7.1.2 pytest-asyncio==0.18.3 pytest-cov==3.0.0 pytest-mock==3.7.0 From eee9fbb6690fa23749a4cf71940ec2e83bcb59c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 03:17:38 +0000 Subject: [PATCH 03/63] Bump types-python-dateutil from 2.8.11 to 2.8.12 Bumps [types-python-dateutil](https://github.com/python/typeshed) from 2.8.11 to 2.8.12. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-python-dateutil dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4fb4456f0..4f296b321 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -28,4 +28,4 @@ types-requests==2.27.19 types-tabulate==0.8.7 # Extensions to datetime library -types-python-dateutil==2.8.11 +types-python-dateutil==2.8.12 From 9b39c835867120a69041c36712700a45eab49694 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 03:17:44 +0000 Subject: [PATCH 04/63] Bump mkdocs-material from 8.2.9 to 8.2.10 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 8.2.9 to 8.2.10. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/8.2.9...8.2.10) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 8d03a38c3..c6b683e8f 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,5 +1,5 @@ mkdocs==1.3.0 -mkdocs-material==8.2.9 +mkdocs-material==8.2.10 mdx_truly_sane_lists==1.2 pymdown-extensions==9.3 jinja2==3.1.1 From 399be6f4e54566cf4e8e7d5d2f4719c9d0a9a765 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 03:17:53 +0000 Subject: [PATCH 05/63] Bump types-requests from 2.27.19 to 2.27.20 Bumps [types-requests](https://github.com/python/typeshed) from 2.27.19 to 2.27.20. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4fb4456f0..be23d14bf 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -24,7 +24,7 @@ nbconvert==6.5.0 # mypy types types-cachetools==5.0.1 types-filelock==3.2.5 -types-requests==2.27.19 +types-requests==2.27.20 types-tabulate==0.8.7 # Extensions to datetime library From b4afbb0b0ac2bdb96a6b653bc44240c3e5669112 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 04:54:53 +0000 Subject: [PATCH 06/63] Bump pymdown-extensions from 9.3 to 9.4 Bumps [pymdown-extensions](https://github.com/facelessuser/pymdown-extensions) from 9.3 to 9.4. - [Release notes](https://github.com/facelessuser/pymdown-extensions/releases) - [Commits](https://github.com/facelessuser/pymdown-extensions/compare/9.3...9.4) --- updated-dependencies: - dependency-name: pymdown-extensions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index c6b683e8f..97be17243 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,5 +1,5 @@ mkdocs==1.3.0 mkdocs-material==8.2.10 mdx_truly_sane_lists==1.2 -pymdown-extensions==9.3 +pymdown-extensions==9.4 jinja2==3.1.1 From 562e36c3ec20ce5d05c44fac78eb7e2ac71fb7a8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Apr 2022 07:01:27 +0200 Subject: [PATCH 07/63] Remove Interface V1 support --- freqtrade/resolvers/strategy_resolver.py | 18 +++++++++------ freqtrade/strategy/interface.py | 28 ++++-------------------- 2 files changed, 15 insertions(+), 31 deletions(-) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 0265ad6c3..44d590b67 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -217,15 +217,19 @@ class StrategyResolver(IResolver): raise OperationalException( "`populate_exit_trend` or `populate_sell_trend` must be implemented.") - strategy._populate_fun_len = len(getfullargspec(strategy.populate_indicators).args) - strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args) - strategy._sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args) + _populate_fun_len = len(getfullargspec(strategy.populate_indicators).args) + _buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args) + _sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args) if any(x == 2 for x in [ - strategy._populate_fun_len, - strategy._buy_fun_len, - strategy._sell_fun_len + _populate_fun_len, + _buy_fun_len, + _sell_fun_len ]): - strategy.INTERFACE_VERSION = 1 + raise OperationalException( + "Strategy Interface v1 is no longer supported. " + "Please update your strategy to implement " + "`populate_indicators`, `populate_entry_trend` and `populate_exit_trend` " + "with the metadata argument. ") return strategy @staticmethod diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index afcc1aa99..0ec3895bc 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -3,7 +3,6 @@ IStrategy interface This module defines the interface to apply for strategies """ import logging -import warnings from abc import ABC, abstractmethod from datetime import datetime, timedelta, timezone from typing import Dict, List, Optional, Tuple, Union @@ -44,14 +43,11 @@ class IStrategy(ABC, HyperStrategyMixin): """ # Strategy interface version # Default to version 2 - # Version 1 is the initial interface without metadata dict + # Version 1 is the initial interface without metadata dict - deprecated and no longer supported. # Version 2 populate_* include metadata dict # Version 3 - First version with short and leverage support INTERFACE_VERSION: int = 3 - _populate_fun_len: int = 0 - _buy_fun_len: int = 0 - _sell_fun_len: int = 0 _ft_params_from_file: Dict # associated minimal roi minimal_roi: Dict = {} @@ -1090,12 +1086,7 @@ class IStrategy(ABC, HyperStrategyMixin): dataframe = _create_and_merge_informative_pair( self, dataframe, metadata, inf_data, populate_fn) - if self._populate_fun_len == 2: - warnings.warn("deprecated - check out the Sample strategy to see " - "the current function headers!", DeprecationWarning) - return self.populate_indicators(dataframe) # type: ignore - else: - return self.populate_indicators(dataframe, metadata) + return self.populate_indicators(dataframe, metadata) def advise_entry(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ @@ -1109,12 +1100,7 @@ class IStrategy(ABC, HyperStrategyMixin): logger.debug(f"Populating enter signals for pair {metadata.get('pair')}.") - if self._buy_fun_len == 2: - warnings.warn("deprecated - check out the Sample strategy to see " - "the current function headers!", DeprecationWarning) - df = self.populate_buy_trend(dataframe) # type: ignore - else: - df = self.populate_entry_trend(dataframe, metadata) + df = self.populate_entry_trend(dataframe, metadata) if 'enter_long' not in df.columns: df = df.rename({'buy': 'enter_long', 'buy_tag': 'enter_tag'}, axis='columns') @@ -1129,14 +1115,8 @@ class IStrategy(ABC, HyperStrategyMixin): currently traded pair :return: DataFrame with exit column """ - logger.debug(f"Populating exit signals for pair {metadata.get('pair')}.") - if self._sell_fun_len == 2: - warnings.warn("deprecated - check out the Sample strategy to see " - "the current function headers!", DeprecationWarning) - df = self.populate_sell_trend(dataframe) # type: ignore - else: - df = self.populate_exit_trend(dataframe, metadata) + df = self.populate_exit_trend(dataframe, metadata) if 'exit_long' not in df.columns: df = df.rename({'sell': 'exit_long'}, axis='columns') return df From ec2582a4aed614dd1da552e9b76562e09f41c3d8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Apr 2022 07:02:09 +0200 Subject: [PATCH 08/63] Update tests to no longer use Strategy V1 --- tests/optimize/test_backtesting.py | 27 ++++----- tests/strategy/strats/legacy_strategy_v1.py | 60 +----------------- tests/strategy/test_interface.py | 2 +- tests/strategy/test_strategy_loading.py | 67 ++------------------- 4 files changed, 21 insertions(+), 135 deletions(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 4d32a7516..d7ee4a042 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -500,7 +500,7 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti Backtesting(default_conf) # Multiple strategies - default_conf['strategy_list'] = [CURRENT_TEST_STRATEGY, 'TestStrategyLegacyV1'] + default_conf['strategy_list'] = [CURRENT_TEST_STRATEGY, 'StrategyTestV2'] with pytest.raises(OperationalException, match='PrecisionFilter not allowed for backtesting multiple strategies.'): Backtesting(default_conf) @@ -1198,7 +1198,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): '--disable-max-market-positions', '--strategy-list', CURRENT_TEST_STRATEGY, - 'TestStrategyLegacyV1', + 'StrategyTestV2', ] args = get_args(args) start_backtesting(args) @@ -1221,14 +1221,13 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): 'up to 2017-11-14 22:58:00 (0 days).', 'Parameter --enable-position-stacking detected ...', f'Running backtesting for Strategy {CURRENT_TEST_STRATEGY}', - 'Running backtesting for Strategy TestStrategyLegacyV1', + 'Running backtesting for Strategy StrategyTestV2', ] for line in exists: assert log_has(line, caplog) -@pytest.mark.filterwarnings("ignore:deprecated") def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdatadir, capsys): default_conf.update({ "use_exit_signal": True, @@ -1310,7 +1309,7 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat '--breakdown', 'day', '--strategy-list', CURRENT_TEST_STRATEGY, - 'TestStrategyLegacyV1', + 'StrategyTestV2', ] args = get_args(args) start_backtesting(args) @@ -1327,7 +1326,7 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat 'up to 2017-11-14 22:58:00 (0 days).', 'Parameter --enable-position-stacking detected ...', f'Running backtesting for Strategy {CURRENT_TEST_STRATEGY}', - 'Running backtesting for Strategy TestStrategyLegacyV1', + 'Running backtesting for Strategy StrategyTestV2', ] for line in exists: @@ -1592,7 +1591,7 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda min_backtest_date = now - timedelta(weeks=4) load_backtest_metadata = MagicMock(return_value={ 'StrategyTestV2': {'run_id': '1', 'backtest_start_time': now.timestamp()}, - 'TestStrategyLegacyV1': {'run_id': run_id, 'backtest_start_time': start_time.timestamp()} + 'StrategyTestV3': {'run_id': run_id, 'backtest_start_time': start_time.timestamp()} }) load_backtest_stats = MagicMock(side_effect=[ { @@ -1601,9 +1600,9 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda 'strategy_comparison': [{'key': 'StrategyTestV2'}] }, { - 'metadata': {'TestStrategyLegacyV1': {'run_id': '2'}}, - 'strategy': {'TestStrategyLegacyV1': {}}, - 'strategy_comparison': [{'key': 'TestStrategyLegacyV1'}] + 'metadata': {'StrategyTestV3': {'run_id': '2'}}, + 'strategy': {'StrategyTestV3': {}}, + 'strategy_comparison': [{'key': 'StrategyTestV3'}] } ]) mocker.patch('pathlib.Path.glob', return_value=[ @@ -1627,7 +1626,7 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda '--cache', cache, '--strategy-list', 'StrategyTestV2', - 'TestStrategyLegacyV1', + 'StrategyTestV3', ] args = get_args(args) start_backtesting(args) @@ -1649,7 +1648,7 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda assert backtestmock.call_count == 2 exists = [ 'Running backtesting for Strategy StrategyTestV2', - 'Running backtesting for Strategy TestStrategyLegacyV1', + 'Running backtesting for Strategy StrategyTestV3', 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', 'Backtesting with data from 2017-11-14 21:17:00 up to 2017-11-14 22:58:00 (0 days).', ] @@ -1657,12 +1656,12 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda assert backtestmock.call_count == 0 exists = [ 'Reusing result of previous backtest for StrategyTestV2', - 'Reusing result of previous backtest for TestStrategyLegacyV1', + 'Reusing result of previous backtest for StrategyTestV3', ] else: exists = [ 'Reusing result of previous backtest for StrategyTestV2', - 'Running backtesting for Strategy TestStrategyLegacyV1', + 'Running backtesting for Strategy StrategyTestV3', 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', 'Backtesting with data from 2017-11-14 21:17:00 up to 2017-11-14 22:58:00 (0 days).', ] diff --git a/tests/strategy/strats/legacy_strategy_v1.py b/tests/strategy/strats/legacy_strategy_v1.py index bad2aa40d..a1fe4ee15 100644 --- a/tests/strategy/strats/legacy_strategy_v1.py +++ b/tests/strategy/strats/legacy_strategy_v1.py @@ -1,85 +1,29 @@ - -# --- Do not remove these libs --- -# Add your lib to import here -import talib.abstract as ta from pandas import DataFrame from freqtrade.strategy import IStrategy -# -------------------------------- - -# This class is a sample. Feel free to customize it. +# Dummy strategy - no longer loads but raises an exception. class TestStrategyLegacyV1(IStrategy): - """ - This is a test strategy using the legacy function headers, which will be - removed in a future update. - Please do not use this as a template, but refer to user_data/strategy/sample_strategy.py - for a uptodate version of this template. - """ - # Minimal ROI designed for the strategy. - # This attribute will be overridden if the config file contains "minimal_roi" minimal_roi = { "40": 0.0, "30": 0.01, "20": 0.02, "0": 0.04 } - - # Optimal stoploss designed for the strategy - # This attribute will be overridden if the config file contains "stoploss" stoploss = -0.10 timeframe = '5m' def populate_indicators(self, dataframe: DataFrame) -> DataFrame: - """ - Adds several different TA indicators to the given DataFrame - - Performance Note: For the best performance be frugal on the number of indicators - you are using. Let uncomment only the indicator you are using in your strategies - or your hyperopt configuration, otherwise you will waste your memory and CPU usage. - """ - - # Momentum Indicator - # ------------------------------------ - - # ADX - dataframe['adx'] = ta.ADX(dataframe) - - # TEMA - Triple Exponential Moving Average - dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) return dataframe def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: - """ - Based on TA indicators, populates the buy signal for the given dataframe - :param dataframe: DataFrame - :return: DataFrame with buy column - """ - dataframe.loc[ - ( - (dataframe['adx'] > 30) & - (dataframe['tema'] > dataframe['tema'].shift(1)) & - (dataframe['volume'] > 0) - ), - 'buy'] = 1 return dataframe def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: - """ - Based on TA indicators, populates the sell signal for the given dataframe - :param dataframe: DataFrame - :return: DataFrame with buy column - """ - dataframe.loc[ - ( - (dataframe['adx'] > 70) & - (dataframe['tema'] < dataframe['tema'].shift(1)) & - (dataframe['volume'] > 0) - ), - 'sell'] = 1 + return dataframe diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index a86d69135..ea81fe968 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -686,7 +686,7 @@ def test_is_pair_locked(default_conf): def test_is_informative_pairs_callback(default_conf): - default_conf.update({'strategy': 'TestStrategyLegacyV1'}) + default_conf.update({'strategy': 'StrategyTestV2'}) strategy = StrategyResolver.load_strategy(default_conf) # Should return empty # Uses fallback to base implementation diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index e74a2a022..85f7961ed 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -100,7 +100,7 @@ def test_load_strategy_noname(default_conf): @pytest.mark.filterwarnings("ignore:deprecated") -@pytest.mark.parametrize('strategy_name', ['StrategyTestV2', 'TestStrategyLegacyV1']) +@pytest.mark.parametrize('strategy_name', ['StrategyTestV2']) def test_strategy_pre_v3(result, default_conf, strategy_name): default_conf.update({'strategy': strategy_name}) @@ -346,40 +346,6 @@ def test_strategy_override_use_exit_profit_only(caplog, default_conf): assert log_has("Override strategy 'exit_profit_only' with value in config file: True.", caplog) -@pytest.mark.filterwarnings("ignore:deprecated") -def test_deprecate_populate_indicators(result, default_conf): - default_location = Path(__file__).parent / "strats" - default_conf.update({'strategy': 'TestStrategyLegacyV1', - 'strategy_path': default_location}) - strategy = StrategyResolver.load_strategy(default_conf) - with warnings.catch_warnings(record=True) as w: - # Cause all warnings to always be triggered. - warnings.simplefilter("always") - indicators = strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) - assert len(w) == 1 - assert issubclass(w[-1].category, DeprecationWarning) - assert "deprecated - check out the Sample strategy to see the current function headers!" \ - in str(w[-1].message) - - with warnings.catch_warnings(record=True) as w: - # Cause all warnings to always be triggered. - warnings.simplefilter("always") - strategy.advise_entry(indicators, {'pair': 'ETH/BTC'}) - assert len(w) == 1 - assert issubclass(w[-1].category, DeprecationWarning) - assert "deprecated - check out the Sample strategy to see the current function headers!" \ - in str(w[-1].message) - - with warnings.catch_warnings(record=True) as w: - # Cause all warnings to always be triggered. - warnings.simplefilter("always") - strategy.advise_exit(indicators, {'pair': 'ETH_BTC'}) - assert len(w) == 1 - assert issubclass(w[-1].category, DeprecationWarning) - assert "deprecated - check out the Sample strategy to see the current function headers!" \ - in str(w[-1].message) - - @pytest.mark.filterwarnings("ignore:deprecated") def test_missing_implements(default_conf, caplog): @@ -438,33 +404,14 @@ def test_missing_implements(default_conf, caplog): StrategyResolver.load_strategy(default_conf) -@pytest.mark.filterwarnings("ignore:deprecated") -def test_call_deprecated_function(result, default_conf, caplog): +def test_call_deprecated_function(default_conf): default_location = Path(__file__).parent / "strats" del default_conf['timeframe'] default_conf.update({'strategy': 'TestStrategyLegacyV1', 'strategy_path': default_location}) - strategy = StrategyResolver.load_strategy(default_conf) - metadata = {'pair': 'ETH/BTC'} - - # Make sure we are using a legacy function - assert strategy._populate_fun_len == 2 - assert strategy._buy_fun_len == 2 - assert strategy._sell_fun_len == 2 - assert strategy.INTERFACE_VERSION == 1 - assert strategy.timeframe == '5m' - - indicator_df = strategy.advise_indicators(result, metadata=metadata) - assert isinstance(indicator_df, DataFrame) - assert 'adx' in indicator_df.columns - - enterdf = strategy.advise_entry(result, metadata=metadata) - assert isinstance(enterdf, DataFrame) - assert 'enter_long' in enterdf.columns - - exitdf = strategy.advise_exit(result, metadata=metadata) - assert isinstance(exitdf, DataFrame) - assert 'exit_long' in exitdf + with pytest.raises(OperationalException, + match=r"Strategy Interface v1 is no longer supported.*"): + StrategyResolver.load_strategy(default_conf) def test_strategy_interface_versioning(result, default_conf): @@ -472,10 +419,6 @@ def test_strategy_interface_versioning(result, default_conf): strategy = StrategyResolver.load_strategy(default_conf) metadata = {'pair': 'ETH/BTC'} - # Make sure we are using a legacy function - assert strategy._populate_fun_len == 3 - assert strategy._buy_fun_len == 3 - assert strategy._sell_fun_len == 3 assert strategy.INTERFACE_VERSION == 2 indicator_df = strategy.advise_indicators(result, metadata=metadata) From 9bb0f1f675d211a3dd95359b6129c10628dcc958 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Apr 2022 07:09:08 +0200 Subject: [PATCH 09/63] Move legacy strategy to "broken strats" folder --- tests/commands/test_commands.py | 16 ++++++++-------- tests/rpc/test_rpc_apiserver.py | 1 - .../{ => broken_strats}/legacy_strategy_v1.py | 1 + tests/strategy/test_strategy_loading.py | 9 ++++----- 4 files changed, 13 insertions(+), 14 deletions(-) rename tests/strategy/strats/{ => broken_strats}/legacy_strategy_v1.py (97%) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index d1f54ad52..37eeda86a 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -859,8 +859,8 @@ def test_start_list_strategies(capsys): # pargs['config'] = None start_list_strategies(pargs) captured = capsys.readouterr() - assert "TestStrategyLegacyV1" in captured.out - assert "legacy_strategy_v1.py" not in captured.out + assert "StrategyTestV2" in captured.out + assert "strategy_test_v2.py" not in captured.out assert CURRENT_TEST_STRATEGY in captured.out # Test regular output @@ -874,8 +874,8 @@ def test_start_list_strategies(capsys): # pargs['config'] = None start_list_strategies(pargs) captured = capsys.readouterr() - assert "TestStrategyLegacyV1" in captured.out - assert "legacy_strategy_v1.py" in captured.out + assert "StrategyTestV2" in captured.out + assert "strategy_test_v2.py" in captured.out assert CURRENT_TEST_STRATEGY in captured.out # Test color output @@ -888,8 +888,8 @@ def test_start_list_strategies(capsys): # pargs['config'] = None start_list_strategies(pargs) captured = capsys.readouterr() - assert "TestStrategyLegacyV1" in captured.out - assert "legacy_strategy_v1.py" in captured.out + assert "StrategyTestV2" in captured.out + assert "strategy_test_v2.py" in captured.out assert CURRENT_TEST_STRATEGY in captured.out assert "LOAD FAILED" in captured.out # Recursive @@ -907,8 +907,8 @@ def test_start_list_strategies(capsys): # pargs['config'] = None start_list_strategies(pargs) captured = capsys.readouterr() - assert "TestStrategyLegacyV1" in captured.out - assert "legacy_strategy_v1.py" in captured.out + assert "StrategyTestV2" in captured.out + assert "strategy_test_v2.py" in captured.out assert "StrategyTestV2" in captured.out assert "TestStrategyNoImplements" in captured.out assert str(Path("broken_strats/broken_futures_strategies.py")) in captured.out diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index af8361571..4910213b4 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1389,7 +1389,6 @@ def test_api_strategies(botclient): 'StrategyTestV2', 'StrategyTestV3', 'StrategyTestV3Futures', - 'TestStrategyLegacyV1', ]} diff --git a/tests/strategy/strats/legacy_strategy_v1.py b/tests/strategy/strats/broken_strats/legacy_strategy_v1.py similarity index 97% rename from tests/strategy/strats/legacy_strategy_v1.py rename to tests/strategy/strats/broken_strats/legacy_strategy_v1.py index a1fe4ee15..f3b8c2696 100644 --- a/tests/strategy/strats/legacy_strategy_v1.py +++ b/tests/strategy/strats/broken_strats/legacy_strategy_v1.py @@ -1,3 +1,4 @@ +# type: ignore from pandas import DataFrame from freqtrade.strategy import IStrategy diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index 85f7961ed..3ed1eb0ce 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -1,6 +1,5 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 import logging -import warnings from base64 import urlsafe_b64encode from pathlib import Path @@ -35,7 +34,7 @@ def test_search_all_strategies_no_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver.search_all_objects(directory, enum_failed=False) assert isinstance(strategies, list) - assert len(strategies) == 6 + assert len(strategies) == 5 assert isinstance(strategies[0], dict) @@ -43,10 +42,10 @@ def test_search_all_strategies_with_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver.search_all_objects(directory, enum_failed=True) assert isinstance(strategies, list) - assert len(strategies) == 7 + assert len(strategies) == 6 # with enum_failed=True search_all_objects() shall find 2 good strategies # and 1 which fails to load - assert len([x for x in strategies if x['class'] is not None]) == 6 + assert len([x for x in strategies if x['class'] is not None]) == 5 assert len([x for x in strategies if x['class'] is None]) == 1 @@ -405,7 +404,7 @@ def test_missing_implements(default_conf, caplog): def test_call_deprecated_function(default_conf): - default_location = Path(__file__).parent / "strats" + default_location = Path(__file__).parent / "strats/broken_strats/" del default_conf['timeframe'] default_conf.update({'strategy': 'TestStrategyLegacyV1', 'strategy_path': default_location}) From ad6e5c53120053556b4cd0e19581a853e3e02432 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Apr 2022 07:41:51 +0200 Subject: [PATCH 10/63] Test informative fallback again --- tests/strategy/strats/strategy_test_v2.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/strategy/strats/strategy_test_v2.py b/tests/strategy/strats/strategy_test_v2.py index 8996b227a..85ff856e1 100644 --- a/tests/strategy/strats/strategy_test_v2.py +++ b/tests/strategy/strats/strategy_test_v2.py @@ -56,19 +56,6 @@ class StrategyTestV2(IStrategy): # By default this strategy does not use Position Adjustments position_adjustment_enable = False - def informative_pairs(self): - """ - Define additional, informative pair/interval combinations to be cached from the exchange. - These pair/interval combinations are non-tradeable, unless they are part - of the whitelist as well. - For more information, please consult the documentation - :return: List of tuples in the format (pair, interval) - Sample: return [("ETH/USDT", "5m"), - ("BTC/USDT", "15m"), - ] - """ - return [] - def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame From 86b3aac9ba8a0db45c986fa8ebda7bb19292fe33 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Apr 2022 08:38:02 +0200 Subject: [PATCH 11/63] Fix FTX not fetching the very latest data --- freqtrade/exchange/exchange.py | 9 ++++++--- freqtrade/exchange/ftx.py | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index ab063762b..2eb705b53 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -64,6 +64,7 @@ class Exchange: "ohlcv_params": {}, "ohlcv_candle_limit": 500, "ohlcv_partial_candle": True, + "ohlcv_require_since": False, # Check https://github.com/ccxt/ccxt/issues/10767 for removal of ohlcv_volume_currency "ohlcv_volume_currency": "base", # "base" or "quote" "tickers_have_quoteVolume": True, @@ -1710,7 +1711,8 @@ class Exchange: def _build_coroutine(self, pair: str, timeframe: str, candle_type: CandleType, since_ms: Optional[int]) -> Coroutine: - if not since_ms and self.required_candle_call_count > 1: + if (not since_ms + and (self._ft_has["ohlcv_require_since"] or self.required_candle_call_count > 1)): # Multiple calls for one pair - to get more history one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) move_to = one_call * self.required_candle_call_count @@ -1829,17 +1831,18 @@ class Exchange: pair, timeframe, since_ms, s ) params = deepcopy(self._ft_has.get('ohlcv_params', {})) + candle_limit = self.ohlcv_candle_limit(timeframe) if candle_type != CandleType.SPOT: params.update({'price': candle_type}) if candle_type != CandleType.FUNDING_RATE: data = await self._api_async.fetch_ohlcv( pair, timeframe=timeframe, since=since_ms, - limit=self.ohlcv_candle_limit(timeframe), params=params) + limit=candle_limit, params=params) else: # Funding rate data = await self._api_async.fetch_funding_rate_history( pair, since=since_ms, - limit=self.ohlcv_candle_limit(timeframe)) + limit=candle_limit) # Convert funding rate to candle pattern data = [[x['timestamp'], x['fundingRate'], 0, 0, 0, 0] for x in data] # Some exchanges sort OHLCV in ASC order and others in DESC. diff --git a/freqtrade/exchange/ftx.py b/freqtrade/exchange/ftx.py index f20aab138..d2dcf84a6 100644 --- a/freqtrade/exchange/ftx.py +++ b/freqtrade/exchange/ftx.py @@ -20,6 +20,7 @@ class Ftx(Exchange): _ft_has: Dict = { "stoploss_on_exchange": True, "ohlcv_candle_limit": 1500, + "ohlcv_require_since": True, "ohlcv_volume_currency": "quote", "mark_ohlcv_price": "index", "mark_ohlcv_timeframe": "1h", From 7b02114ad2826bf4260ec98d47d9cd9e9a191d78 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Apr 2022 09:49:51 +0200 Subject: [PATCH 12/63] Restrict trading pairs with too low precision closes #6606 --- freqtrade/exchange/exchange.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 2eb705b53..d4741bd64 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -369,6 +369,9 @@ class Exchange: return ( market.get('quote', None) is not None and market.get('base', None) is not None + and (self.precisionMode != TICK_SIZE + # Too low precision will falsify calculations + or market.get('precision', {}).get('price', None) > 1e-11) and ((self.trading_mode == TradingMode.SPOT and self.market_is_spot(market)) or (self.trading_mode == TradingMode.MARGIN and self.market_is_margin(market)) or (self.trading_mode == TradingMode.FUTURES and self.market_is_future(market))) From 6d576bc02d4474b29de678052b3a420d2aa2cc4d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Apr 2022 10:15:07 +0200 Subject: [PATCH 13/63] Check pre-commit verison updates --- .github/workflows/ci.yml | 21 ++++++++++++--- .pre-commit-config.yaml | 7 +++++ build_helpers/pre_commit_update.py | 42 ++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 build_helpers/pre_commit_update.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1902a6c45..35c237837 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -265,6 +265,21 @@ jobs: details: Test Failed webhookUrl: ${{ secrets.DISCORD_WEBHOOK }} + mypy_version_check: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: 3.9 + + - name: pre-commit dependencies + run: | + pip install pyaml + python build_helpers/pre_commit_update.py + docs_check: runs-on: ubuntu-20.04 steps: @@ -277,7 +292,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: 3.8 + python-version: 3.9 - name: Documentation build run: | @@ -304,7 +319,7 @@ jobs: # Notify only once - when CI completes (and after deploy) in case it's successfull notify-complete: - needs: [ build_linux, build_macos, build_windows, docs_check ] + needs: [ build_linux, build_macos, build_windows, docs_check, mypy_version_check ] runs-on: ubuntu-20.04 steps: @@ -325,7 +340,7 @@ jobs: webhookUrl: ${{ secrets.DISCORD_WEBHOOK }} deploy: - needs: [ build_linux, build_macos, build_windows, docs_check ] + needs: [ build_linux, build_macos, build_windows, docs_check, mypy_version_check ] runs-on: ubuntu-20.04 if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 316baf0e3..f223f0b9b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,6 +11,13 @@ repos: rev: "v0.942" hooks: - id: mypy + args: [ freqtrade ] + additional_dependencies: + - types-cachetools==5.0.1 + - types-filelock==3.2.5 + - types-requests==2.27.19 + - types-tabulate==0.8.7 + - types-python-dateutil==2.8.11 # stages: [push] - repo: https://github.com/pycqa/isort diff --git a/build_helpers/pre_commit_update.py b/build_helpers/pre_commit_update.py new file mode 100644 index 000000000..8724d8ade --- /dev/null +++ b/build_helpers/pre_commit_update.py @@ -0,0 +1,42 @@ +# File used in CI to ensure pre-commit dependencies are kept uptodate. + +import sys +from pathlib import Path + +import yaml + + +pre_commit_file = Path('.pre-commit-config.yaml') +require_dev = Path('requirements-dev.txt') + +with require_dev.open('r') as rfile: + requirements = rfile.readlines() + +# Extract types only +type_reqs = [r.strip('\n') for r in requirements if r.startswith('types-')] + +with pre_commit_file.open('r') as file: + f = yaml.load(file, Loader=yaml.FullLoader) + + +mypy_repo = [repo for repo in f['repos'] if repo['repo'] + == 'https://github.com/pre-commit/mirrors-mypy'] + +hooks = mypy_repo[0]['hooks'][0]['additional_dependencies'] + +errors = [] +for hook in hooks: + if hook not in type_reqs: + errors.append(f"{hook} is missing in requirements-dev.txt.") + +for req in type_reqs: + if req not in hooks: + errors.append(f"{req} is missing in pre-config file.") + + +if errors: + for e in errors: + print(e) + sys.exit(1) + +sys.exit(0) From fc118d0e95d38e7f5a655c79fc3de41ab9f2f537 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Apr 2022 10:19:31 +0200 Subject: [PATCH 14/63] Re-align dependencies --- .pre-commit-config.yaml | 4 ++-- requirements-dev.txt | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f223f0b9b..0dd343bb8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,9 +15,9 @@ repos: additional_dependencies: - types-cachetools==5.0.1 - types-filelock==3.2.5 - - types-requests==2.27.19 + - types-requests==2.27.20 - types-tabulate==0.8.7 - - types-python-dateutil==2.8.11 + - types-python-dateutil==2.8.12 # stages: [push] - repo: https://github.com/pycqa/isort diff --git a/requirements-dev.txt b/requirements-dev.txt index b0210d64a..c4fe366a5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -26,6 +26,4 @@ types-cachetools==5.0.1 types-filelock==3.2.5 types-requests==2.27.20 types-tabulate==0.8.7 - -# Extensions to datetime library types-python-dateutil==2.8.12 From 4143ebbeae53eb27706cf15573f4f36b905ac8e0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Apr 2022 10:51:11 +0200 Subject: [PATCH 15/63] Add CAGR calculation to backtesting --- docs/backtesting.md | 2 ++ freqtrade/data/btanalysis.py | 11 +++++++++++ freqtrade/optimize/optimize_reports.py | 4 +++- tests/data/test_btanalysis.py | 27 +++++++++++++++++++------- 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index f732068f1..a0a304400 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -299,6 +299,7 @@ A backtesting result will look like that: | Final balance | 0.01762792 BTC | | Absolute profit | 0.00762792 BTC | | Total profit % | 76.2% | +| CAGR % | 460.87% | | Trades per day | 3.575 | | Avg. stake amount | 0.001 BTC | | Total trade volume | 0.429 BTC | @@ -388,6 +389,7 @@ It contains some useful key metrics about performance of your strategy on backte | Final balance | 0.01762792 BTC | | Absolute profit | 0.00762792 BTC | | Total profit % | 76.2% | +| CAGR % | 460.87% | | Avg. stake amount | 0.001 BTC | | Total trade volume | 0.429 BTC | | | | diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 8abcc6747..206a6f5f3 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -553,3 +553,14 @@ def calculate_csum(trades: pd.DataFrame, starting_balance: float = 0) -> Tuple[f csum_max = csum_df['sum'].max() + starting_balance return csum_min, csum_max + + +def calculate_cagr(days_passed: int, starting_balance: float, final_balance: float) -> float: + """ + Calculate CAGR + :param days_passed: Days passed between start and ending balance + :param starting_balance: Starting balance + :param final_balance: Final balance to calculate CAGR against + :return: CAGR + """ + return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1 diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 0ceb3a411..e8bd035d1 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -9,7 +9,7 @@ from pandas import DataFrame, to_datetime from tabulate import tabulate from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT -from freqtrade.data.btanalysis import (calculate_csum, calculate_market_change, +from freqtrade.data.btanalysis import (calculate_cagr, calculate_csum, calculate_market_change, calculate_max_drawdown) from freqtrade.misc import (decimals_per_coin, file_dump_joblib, file_dump_json, get_backtest_metadata_filename, round_coin_value) @@ -446,6 +446,7 @@ def generate_strategy_stats(pairlist: List[str], 'profit_total_abs': results['profit_abs'].sum(), 'profit_total_long_abs': results.loc[~results['is_short'], 'profit_abs'].sum(), 'profit_total_short_abs': results.loc[results['is_short'], 'profit_abs'].sum(), + 'cagr': calculate_cagr(backtest_days, start_balance, content['final_balance']), 'backtest_start': min_date.strftime(DATETIME_PRINT_FORMAT), 'backtest_start_ts': int(min_date.timestamp() * 1000), 'backtest_end': max_date.strftime(DATETIME_PRINT_FORMAT), @@ -746,6 +747,7 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Absolute profit ', round_coin_value(strat_results['profit_total_abs'], strat_results['stake_currency'])), ('Total profit %', f"{strat_results['profit_total']:.2%}"), + ('CAGR %', f"{strat_results['cagr']:.2%}"), ('Trades per day', strat_results['trades_per_day']), ('Avg. daily profit %', f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"), diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 2b53e4900..eaf703b2d 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -8,13 +8,13 @@ from pandas import DataFrame, DateOffset, Timestamp, to_datetime from freqtrade.configuration import TimeRange from freqtrade.constants import LAST_BT_RESULT_FN -from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelism, calculate_csum, - calculate_market_change, calculate_max_drawdown, - calculate_underwater, combine_dataframes_with_mean, - create_cum_profit, extract_trades_of_period, - get_latest_backtest_filename, get_latest_hyperopt_file, - load_backtest_data, load_backtest_metadata, load_trades, - load_trades_from_db) +from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelism, calculate_cagr, + calculate_csum, calculate_market_change, + calculate_max_drawdown, calculate_underwater, + combine_dataframes_with_mean, create_cum_profit, + extract_trades_of_period, get_latest_backtest_filename, + get_latest_hyperopt_file, load_backtest_data, + load_backtest_metadata, load_trades, load_trades_from_db) from freqtrade.data.history import load_data, load_pair_history from freqtrade.exceptions import OperationalException from tests.conftest import CURRENT_TEST_STRATEGY, create_mock_trades @@ -336,6 +336,19 @@ def test_calculate_csum(testdatadir): csum_min, csum_max = calculate_csum(DataFrame()) +@pytest.mark.parametrize('start,end,days, expected', [ + (64900, 176000, 3 * 365, 0.3945), + (64900, 176000, 365, 1.7119), + (1000, 1000, 365, 0.0), + (1000, 1500, 365, 0.5), + (1000, 1500, 100, 3.3927), # sub year + (0.01000000, 0.01762792, 120, 4.6087), # sub year BTC values +]) +def test_calculate_cagr(start, end, days, expected): + + assert round(calculate_cagr(days, start, end), 4) == expected + + def test_calculate_max_drawdown2(): values = [0.011580, 0.010048, 0.011340, 0.012161, 0.010416, 0.010009, 0.020024, -0.024662, -0.022350, 0.020496, -0.029859, -0.030511, 0.010041, 0.010872, From 500fdc2759639d9275ab82bfbcaaa01e48017eae Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Apr 2022 11:12:35 +0200 Subject: [PATCH 16/63] run mypy also against tests --- .github/workflows/ci.yml | 4 ++-- .pre-commit-config.yaml | 2 +- freqtrade/strategy/informative_decorator.py | 2 +- freqtrade/strategy/interface.py | 2 +- setup.cfg | 4 ++++ 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35c237837..5bafe9cb8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,7 +100,7 @@ jobs: - name: Mypy run: | - mypy freqtrade scripts + mypy freqtrade scripts tests - name: Discord notification uses: rjstone/discord-webhook-notify@v1 @@ -255,7 +255,7 @@ jobs: - name: Mypy run: | - mypy freqtrade scripts + mypy freqtrade scripts tests - name: Discord notification uses: rjstone/discord-webhook-notify@v1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0dd343bb8..d980fc4e9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: rev: "v0.942" hooks: - id: mypy - args: [ freqtrade ] + args: [ freqtrade, scripts, tests ] additional_dependencies: - types-cachetools==5.0.1 - types-filelock==3.2.5 diff --git a/freqtrade/strategy/informative_decorator.py b/freqtrade/strategy/informative_decorator.py index 0dd5320cd..7dfdf5a8c 100644 --- a/freqtrade/strategy/informative_decorator.py +++ b/freqtrade/strategy/informative_decorator.py @@ -23,7 +23,7 @@ class InformativeData: def informative(timeframe: str, asset: str = '', fmt: Optional[Union[str, Callable[[Any], str]]] = None, *, - candle_type: Optional[CandleType] = None, + candle_type: Optional[Union[CandleType, str]] = None, ffill: bool = True) -> Callable[[PopulateIndicators], PopulateIndicators]: """ A decorator for populate_indicators_Nn(self, dataframe, metadata), allowing these functions to diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 0ec3895bc..300010b83 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -110,7 +110,7 @@ class IStrategy(ABC, HyperStrategyMixin): # Class level variables (intentional) containing # the dataprovider (dp) (access to other candles, historic data, ...) # and wallets - access to the current balance. - dp: Optional[DataProvider] + dp: DataProvider wallets: Optional[Wallets] = None # Filled from configuration stake_currency: str diff --git a/setup.cfg b/setup.cfg index a33ceda1f..edbd320c3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -53,6 +53,10 @@ exclude = [mypy] ignore_missing_imports = True warn_unused_ignores = True +exclude = (?x)( + ^build_helpers\.py$ + ) + [mypy-tests.*] ignore_errors = True From 2b3f68396081a41861797ef0a4abc3c08221765f Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Apr 2022 11:23:45 +0200 Subject: [PATCH 17/63] Update pre-commit to exclude build-helpers --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d980fc4e9..2170b704a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: rev: "v0.942" hooks: - id: mypy - args: [ freqtrade, scripts, tests ] + exclude: build_helpers additional_dependencies: - types-cachetools==5.0.1 - types-filelock==3.2.5 From 580da21ddaa3e9fa8a0016434bc0ffbaf9794276 Mon Sep 17 00:00:00 2001 From: froggleston Date: Mon, 25 Apr 2022 10:31:19 +0100 Subject: [PATCH 18/63] Move df append to pd concat --- freqtrade/optimize/backtesting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) mode change 100644 => 100755 freqtrade/optimize/backtesting.py diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py old mode 100644 new mode 100755 index 5442e425b..217e6ff54 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -10,6 +10,7 @@ from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Tuple from numpy import nan +import pandas as pd from pandas import DataFrame from freqtrade import constants @@ -1093,7 +1094,7 @@ class Backtesting: for t, v in pairresults.open_date.items(): allinds = pairdf.loc[(pairdf['date'] < v)] signal_inds = allinds.iloc[[-1]] - signal_candles_only_df = signal_candles_only_df.append(signal_inds) + signal_candles_only_df = pd.concat([signal_candles_only_df, signal_inds]) signal_candles_only[pair] = signal_candles_only_df From 431c539cbdf3e207d9a7afdc299a14f3b245f547 Mon Sep 17 00:00:00 2001 From: froggleston Date: Mon, 25 Apr 2022 10:42:24 +0100 Subject: [PATCH 19/63] Fix isort import order --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 217e6ff54..27e14ba93 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -9,8 +9,8 @@ from copy import deepcopy from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Tuple -from numpy import nan import pandas as pd +from numpy import nan from pandas import DataFrame from freqtrade import constants From 44000ae0b390ce4a4fa48e4fc588863fdafe8c87 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Apr 2022 17:37:25 +0200 Subject: [PATCH 20/63] Fix CAGR missing for old results --- freqtrade/optimize/optimize_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index e8bd035d1..dd058aff4 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -747,7 +747,7 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Absolute profit ', round_coin_value(strat_results['profit_total_abs'], strat_results['stake_currency'])), ('Total profit %', f"{strat_results['profit_total']:.2%}"), - ('CAGR %', f"{strat_results['cagr']:.2%}"), + ('CAGR %', f"{strat_results['cagr']:.2%}" if 'cagr' in strat_results else 'N/A'), ('Trades per day', strat_results['trades_per_day']), ('Avg. daily profit %', f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"), From 0b90e1d30921b8cec6e97e612e44e51ea225b0c8 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Fri, 1 Apr 2022 08:39:56 -0600 Subject: [PATCH 21/63] Added bot_start callback to strategy interface --- freqtrade/freqtradebot.py | 2 ++ freqtrade/strategy/interface.py | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 57d7cac3c..bee6c4746 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -122,6 +122,8 @@ class FreqtradeBot(LoggingMixin): self._schedule.every().day.at(t).do(update) self.last_process = datetime(1970, 1, 1, tzinfo=timezone.utc) + self.strategy.bot_start() + def notify_status(self, msg: str) -> None: """ Public method for users of this class (worker, etc.) to send notifications diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 300010b83..40744ed08 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -193,6 +193,13 @@ class IStrategy(ABC, HyperStrategyMixin): """ return self.populate_sell_trend(dataframe, metadata) + def bot_start(self, **kwargs) -> None: + """ + Called only once after bot instantiation. + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + """ + pass + def bot_loop_start(self, **kwargs) -> None: """ Called at the start of the bot iteration (one loop). From bf7da35e31a3d9a98fff35cd098ecaf83f6d91f2 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Fri, 1 Apr 2022 09:25:03 -0600 Subject: [PATCH 22/63] strategy callback on_whitelist_update --- freqtrade/freqtradebot.py | 4 ++++ freqtrade/strategy/interface.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index bee6c4746..949d821b3 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -179,6 +179,7 @@ class FreqtradeBot(LoggingMixin): # Query trades from persistence layer trades = Trade.get_open_trades() + current_pair_whitelist = self.active_pair_whitelist self.active_pair_whitelist = self._refresh_active_whitelist(trades) # Refreshing candles @@ -214,6 +215,9 @@ class FreqtradeBot(LoggingMixin): Trade.commit() self.last_process = datetime.now(timezone.utc) + if current_pair_whitelist != self.active_pair_whitelist: + self.strategy.on_whitelist_update() + def process_stopped(self) -> None: """ Close all orders that were left open diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 40744ed08..00d18fa7b 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -209,6 +209,13 @@ class IStrategy(ABC, HyperStrategyMixin): """ pass + def on_whitelist_update(self, **kwargs) -> None: + """ + Called every time the whitelist updates + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + """ + pass + def check_buy_timeout(self, pair: str, trade: Trade, order: Order, current_time: datetime, **kwargs) -> bool: """ From d92761b2b168a19eea764864c5fb9612a379c510 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sun, 3 Apr 2022 02:53:44 -0600 Subject: [PATCH 23/63] Revert "strategy callback on_whitelist_update" This reverts commit 39798dc1192161c3060830dd4684571aa86b7821. --- freqtrade/freqtradebot.py | 4 ---- freqtrade/strategy/interface.py | 7 ------- 2 files changed, 11 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 949d821b3..bee6c4746 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -179,7 +179,6 @@ class FreqtradeBot(LoggingMixin): # Query trades from persistence layer trades = Trade.get_open_trades() - current_pair_whitelist = self.active_pair_whitelist self.active_pair_whitelist = self._refresh_active_whitelist(trades) # Refreshing candles @@ -215,9 +214,6 @@ class FreqtradeBot(LoggingMixin): Trade.commit() self.last_process = datetime.now(timezone.utc) - if current_pair_whitelist != self.active_pair_whitelist: - self.strategy.on_whitelist_update() - def process_stopped(self) -> None: """ Close all orders that were left open diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 00d18fa7b..40744ed08 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -209,13 +209,6 @@ class IStrategy(ABC, HyperStrategyMixin): """ pass - def on_whitelist_update(self, **kwargs) -> None: - """ - Called every time the whitelist updates - :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. - """ - pass - def check_buy_timeout(self, pair: str, trade: Trade, order: Order, current_time: datetime, **kwargs) -> bool: """ From 4fd904e0a91ecf03020060c3d718382775bd143b Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sun, 3 Apr 2022 10:28:09 -0600 Subject: [PATCH 24/63] added bot_start to backtesting --- freqtrade/optimize/backtesting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 27e14ba93..1789290bc 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -182,6 +182,7 @@ class Backtesting: # since a "perfect" stoploss-exit is assumed anyway # And the regular "stoploss" function would not apply to that case self.strategy.order_types['stoploss_on_exchange'] = False + self.strategy.bot_start() def _load_protections(self, strategy: IStrategy): if self.config.get('enable_protections', False): From e09b4498fa419f0088b38cec2cef6c78307ee2e6 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Mon, 25 Apr 2022 17:30:49 -0600 Subject: [PATCH 25/63] added bot_start call to plot/plotting --- freqtrade/plot/plotting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 747248be7..5337016f3 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -610,6 +610,7 @@ def load_and_plot_trades(config: Dict[str, Any]): exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) IStrategy.dp = DataProvider(config, exchange) + strategy.bot_start() plot_elements = init_plotscript(config, list(exchange.markets), strategy.startup_candle_count) timerange = plot_elements['timerange'] trades = plot_elements['trades'] From e76c6e8ad31a525802ec35fa0dd8c6e264d32328 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Mon, 25 Apr 2022 17:32:48 -0600 Subject: [PATCH 26/63] added bot_start call to edge_positioning.__init__ --- freqtrade/edge/edge_positioning.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index 2fe41a17b..5dc84f172 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -90,6 +90,8 @@ class Edge: self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0]) except IndexError: self.fee = None + + strategy.bot_start() def calculate(self, pairs: List[str]) -> bool: if self.fee is None and pairs: From 810e190e164be61a530e4d550c246dddbcdb4f1c Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Mon, 25 Apr 2022 17:46:40 -0600 Subject: [PATCH 27/63] added tests for bot_start --- tests/optimize/test_backtesting.py | 1 + tests/optimize/test_edge_cli.py | 1 + tests/strategy/strats/strategy_test_v3.py | 5 +++++ tests/strategy/test_default_strategy.py | 1 + tests/test_plotting.py | 1 + 5 files changed, 9 insertions(+) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index d7ee4a042..16d1aa2a5 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -312,6 +312,7 @@ def test_backtesting_init(mocker, default_conf, order_types) -> None: get_fee.assert_called() assert backtesting.fee == 0.5 assert not backtesting.strategy.order_types["stoploss_on_exchange"] + assert backtesting.strategy.bot_started is True def test_backtesting_init_no_timeframe(mocker, default_conf, caplog) -> None: diff --git a/tests/optimize/test_edge_cli.py b/tests/optimize/test_edge_cli.py index f0f436a43..d9711b318 100644 --- a/tests/optimize/test_edge_cli.py +++ b/tests/optimize/test_edge_cli.py @@ -94,6 +94,7 @@ def test_edge_init(mocker, edge_conf) -> None: assert edge_cli.config == edge_conf assert edge_cli.config['stake_amount'] == 'unlimited' assert callable(edge_cli.edge.calculate) + assert edge_conf['strategy'].bot_started is True def test_edge_init_fee(mocker, edge_conf) -> None: diff --git a/tests/strategy/strats/strategy_test_v3.py b/tests/strategy/strats/strategy_test_v3.py index 372e29412..df83d3663 100644 --- a/tests/strategy/strats/strategy_test_v3.py +++ b/tests/strategy/strats/strategy_test_v3.py @@ -82,6 +82,11 @@ class StrategyTestV3(IStrategy): # }) # return prot + bot_started = False + + def bot_start(self): + self.bot_started = True + def informative_pairs(self): return [] diff --git a/tests/strategy/test_default_strategy.py b/tests/strategy/test_default_strategy.py index 5cb8fce16..a60274afd 100644 --- a/tests/strategy/test_default_strategy.py +++ b/tests/strategy/test_default_strategy.py @@ -32,6 +32,7 @@ def test_strategy_test_v3(result, fee, is_short, side): assert type(indicators) is DataFrame assert type(strategy.populate_buy_trend(indicators, metadata)) is DataFrame assert type(strategy.populate_sell_trend(indicators, metadata)) is DataFrame + assert strategy.bot_started is True trade = Trade( open_rate=19_000, diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 97f367608..f0a28c4eb 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -58,6 +58,7 @@ def test_init_plotscript(default_conf, mocker, testdatadir): assert "ohlcv" in ret assert "TRX/BTC" in ret["ohlcv"] assert "ADA/BTC" in ret["ohlcv"] + assert default_conf['strategy'].bot_started is True def test_add_indicators(default_conf, testdatadir, caplog): From 7f035a9d5384993dc195a0a5d8f2e22c5fc74d75 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Mon, 25 Apr 2022 17:59:50 -0600 Subject: [PATCH 28/63] added docs for bot_start --- docs/strategy-callbacks.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 7ec600a58..a53a6992c 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -7,6 +7,7 @@ Depending on the callback used, they may be called when entering / exiting a tra Currently available callbacks: +* [`bot_start()`](#bot-start) * [`bot_loop_start()`](#bot-loop-start) * [`custom_stake_amount()`](#stake-size-management) * [`custom_exit()`](#custom-exit-signal) @@ -21,6 +22,31 @@ Currently available callbacks: !!! Tip "Callback calling sequence" You can find the callback calling sequence in [bot-basics](bot-basics.md#bot-execution-logic) +## Bot start + +A simple callback which is called once when the bot starts. +This can be used to perform actions that must only be performed once and runs after dataprovider and wallet are set + +``` python +import asyncio + +class AwesomeStrategy(IStrategy): + + # ... populate_* methods + + async def some_asynchronous_task(self): + self.dp['remote_data'] = requests.get('https://some_remote_source.example.com') + + def bot_start(self, **kwargs) -> None: + """ + Called only once after bot instantiation. + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + """ + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + self.loop.run_until_complete(self.some_asynchronous_task()) + +``` ## Bot loop start A simple callback which is called once at the start of every bot throttling iteration (roughly every 5 seconds, unless configured differently). From 4cccf31a3eb005a2560a0479b42aa8ef2f4b5939 Mon Sep 17 00:00:00 2001 From: naveen <172697+naveensrinivasan@users.noreply.github.com> Date: Tue, 26 Apr 2022 01:07:59 +0000 Subject: [PATCH 29/63] chore: Set permissions for GitHub actions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restrict the GitHub token permissions only to the required ones; this way, even if the attackers will succeed in compromising your workflow, they won’t be able to do much. - Included permissions for the action. https://github.com/ossf/scorecard/blob/main/docs/checks.md#token-permissions https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs [Keeping your GitHub Actions and workflows secure Part 1: Preventing pwn requests](https://securitylab.github.com/research/github-actions-preventing-pwn-requests/) Signed-off-by: naveen <172697+naveensrinivasan@users.noreply.github.com> --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5bafe9cb8..932649f61 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -309,6 +309,9 @@ jobs: webhookUrl: ${{ secrets.DISCORD_WEBHOOK }} cleanup-prior-runs: + permissions: + actions: write # for rokroskar/workflow-run-cleanup-action to obtain workflow name & cancel it + contents: read # for rokroskar/workflow-run-cleanup-action to obtain branch runs-on: ubuntu-20.04 steps: - name: Cleanup previous runs on this branch From a35dc843ea75fe8a4c5452ab8534b0c077bfc106 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Mon, 25 Apr 2022 22:39:58 -0600 Subject: [PATCH 30/63] removed asyncio from bot_start example --- docs/strategy-callbacks.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index a53a6992c..e0cfbe8c1 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -34,17 +34,12 @@ class AwesomeStrategy(IStrategy): # ... populate_* methods - async def some_asynchronous_task(self): - self.dp['remote_data'] = requests.get('https://some_remote_source.example.com') - def bot_start(self, **kwargs) -> None: """ Called only once after bot instantiation. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. """ - self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.loop) - self.loop.run_until_complete(self.some_asynchronous_task()) + self.dp['remote_data'] = requests.get('https://some_remote_source.example.com') ``` ## Bot loop start From 6d99222320f05de1f7c2607f0cbe1beb8a38c29f Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Tue, 26 Apr 2022 09:39:15 +0300 Subject: [PATCH 31/63] Add 'exit_tag' parameter to 'custom_exit_price' callback. --- docs/strategy-callbacks.md | 2 +- freqtrade/freqtradebot.py | 3 ++- freqtrade/optimize/backtesting.py | 3 ++- freqtrade/strategy/interface.py | 3 ++- .../templates/subtemplates/strategy_methods_advanced.j2 | 3 ++- tests/test_freqtradebot.py | 6 +++--- 6 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 7ec600a58..005f94155 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -376,7 +376,7 @@ class AwesomeStrategy(IStrategy): def custom_exit_price(self, pair: str, trade: Trade, current_time: datetime, proposed_rate: float, - current_profit: float, **kwargs) -> float: + current_profit: float, exit_tag: Optional[str, **kwargs) -> float: dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 57d7cac3c..68623c748 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1373,7 +1373,8 @@ class FreqtradeBot(LoggingMixin): default_retval=proposed_limit_rate)( pair=trade.pair, trade=trade, current_time=datetime.now(timezone.utc), - proposed_rate=proposed_limit_rate, current_profit=current_profit) + proposed_rate=proposed_limit_rate, current_profit=current_profit, + exit_tag=exit_check.exit_reason) limit = self.get_valid_price(custom_exit_price, proposed_limit_rate) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 27e14ba93..21e124e72 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -540,7 +540,8 @@ class Backtesting: default_retval=closerate)( pair=trade.pair, trade=trade, current_time=exit_candle_time, - proposed_rate=closerate, current_profit=current_profit) + proposed_rate=closerate, current_profit=current_profit, + exit_tag=exit_.exit_reason) # We can't place orders lower than current low. # freqtrade does not support this in live, and the order would fill immediately if trade.is_short: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 300010b83..4f9e91b56 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -355,7 +355,7 @@ class IStrategy(ABC, HyperStrategyMixin): def custom_exit_price(self, pair: str, trade: Trade, current_time: datetime, proposed_rate: float, - current_profit: float, **kwargs) -> float: + current_profit: float, exit_tag: Optional[str], **kwargs) -> float: """ Custom exit price logic, returning the new exit price. @@ -368,6 +368,7 @@ class IStrategy(ABC, HyperStrategyMixin): :param current_time: datetime object, containing the current datetime :param proposed_rate: Rate, calculated based on pricing settings in exit_pricing. :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param exit_tag: Exit reason. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return float: New exit price value if provided """ diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index d5e2ea8ce..3fa36d506 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -32,7 +32,7 @@ def custom_entry_price(self, pair: str, current_time: 'datetime', proposed_rate: def custom_exit_price(self, pair: str, trade: 'Trade', current_time: 'datetime', proposed_rate: float, - current_profit: float, **kwargs) -> float: + current_profit: float, exit_tag: Optional[str, **kwargs) -> float: """ Custom exit price logic, returning the new exit price. @@ -45,6 +45,7 @@ def custom_exit_price(self, pair: str, trade: 'Trade', :param current_time: datetime object, containing the current datetime :param proposed_rate: Rate, calculated based on pricing settings in exit_pricing. :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param exit_tag: Exit reason. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return float: New exit price value if provided """ diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 3737c7c05..84d3c3324 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3221,7 +3221,7 @@ def test_execute_trade_exit_custom_exit_price( freqtrade.execute_trade_exit( trade=trade, limit=ticker_usdt_sell_up()['ask' if is_short else 'bid'], - exit_check=ExitCheckTuple(exit_type=ExitType.EXIT_SIGNAL) + exit_check=ExitCheckTuple(exit_type=ExitType.EXIT_SIGNAL, exit_reason='foo') ) # Sell price must be different to default bid price @@ -3249,8 +3249,8 @@ def test_execute_trade_exit_custom_exit_price( 'profit_ratio': profit_ratio, 'stake_currency': 'USDT', 'fiat_currency': 'USD', - 'sell_reason': ExitType.EXIT_SIGNAL.value, - 'exit_reason': ExitType.EXIT_SIGNAL.value, + 'sell_reason': 'foo', + 'exit_reason': 'foo', 'open_date': ANY, 'close_date': ANY, 'close_rate': ANY, From 108f11b1d796dc8a371250f2dbb028161bad5739 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 27 Apr 2022 06:42:56 +0200 Subject: [PATCH 32/63] Fix docs typos --- docs/strategy-callbacks.md | 2 +- freqtrade/templates/subtemplates/strategy_methods_advanced.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 005f94155..563b5a2cb 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -376,7 +376,7 @@ class AwesomeStrategy(IStrategy): def custom_exit_price(self, pair: str, trade: Trade, current_time: datetime, proposed_rate: float, - current_profit: float, exit_tag: Optional[str, **kwargs) -> float: + current_profit: float, exit_tag: Optional[str], **kwargs) -> float: dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index 3fa36d506..ed40ef509 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -32,7 +32,7 @@ def custom_entry_price(self, pair: str, current_time: 'datetime', proposed_rate: def custom_exit_price(self, pair: str, trade: 'Trade', current_time: 'datetime', proposed_rate: float, - current_profit: float, exit_tag: Optional[str, **kwargs) -> float: + current_profit: float, exit_tag: Optional[str], **kwargs) -> float: """ Custom exit price logic, returning the new exit price. From ad7fbfab1bacbfaa3778595d2f073411286f9619 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 27 Apr 2022 13:27:33 +0200 Subject: [PATCH 33/63] Slightly improved styling --- freqtrade/rpc/telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c2531fec3..1a9be4503 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -943,7 +943,7 @@ class Telegram(RPCHandler): else: fiat_currency = self._config.get('fiat_display_currency', '') try: - statlist, head, fiat_profit_sum = self._rpc._rpc_status_table( + statlist, _, _ = self._rpc._rpc_status_table( self._config['stake_currency'], fiat_currency) except RPCException: self._send_msg(msg='No open trade found.') From 30c9dc697530fda7add40cf7cae941df4cb06076 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 27 Apr 2022 13:53:11 +0200 Subject: [PATCH 34/63] Fix exit-signa being assigned when tag is set but no signal is present. --- freqtrade/optimize/backtesting.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 21e124e72..210eab39b 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -54,6 +54,11 @@ ESHORT_IDX = 8 # Exit short ENTER_TAG_IDX = 9 EXIT_TAG_IDX = 10 +# Every change to this headers list must evaluate further usages of the resulting tuple +# and eventually change the constants for indexes at the top +HEADERS = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long', + 'enter_short', 'exit_short', 'enter_tag', 'exit_tag'] + class Backtesting: """ @@ -305,10 +310,7 @@ class Backtesting: :param processed: a processed dictionary with format {pair, data}, which gets cleared to optimize memory usage! """ - # Every change to this headers list must evaluate further usages of the resulting tuple - # and eventually change the constants for indexes at the top - headers = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long', - 'enter_short', 'exit_short', 'enter_tag', 'exit_tag'] + data: Dict = {} self.progress.init_step(BacktestState.CONVERT, len(processed)) @@ -320,7 +322,7 @@ class Backtesting: if not pair_data.empty: # Cleanup from prior runs - pair_data.drop(headers[5:] + ['buy', 'sell'], axis=1, errors='ignore') + pair_data.drop(HEADERS[5:] + ['buy', 'sell'], axis=1, errors='ignore') df_analyzed = self.strategy.advise_exit( self.strategy.advise_entry(pair_data, {'pair': pair}), @@ -339,7 +341,7 @@ class Backtesting: # To avoid using data from future, we use entry/exit signals shifted # from the previous candle - for col in headers[5:]: + for col in HEADERS[5:]: tag_col = col in ('enter_tag', 'exit_tag') if col in df_analyzed.columns: df_analyzed.loc[:, col] = df_analyzed.loc[:, col].replace( @@ -351,7 +353,7 @@ class Backtesting: # Convert from Pandas to list for performance reasons # (Looping Pandas is slow.) - data[pair] = df_analyzed[headers].values.tolist() if not df_analyzed.empty else [] + data[pair] = df_analyzed[HEADERS].values.tolist() if not df_analyzed.empty else [] return data def _get_close_rate(self, row: Tuple, trade: LocalTrade, exit: ExitCheckTuple, @@ -515,10 +517,10 @@ class Backtesting: exit_candle_time: datetime = row[DATE_IDX].to_pydatetime() enter = row[SHORT_IDX] if trade.is_short else row[LONG_IDX] - exit_ = row[ESHORT_IDX] if trade.is_short else row[ELONG_IDX] + exit_sig = row[ESHORT_IDX] if trade.is_short else row[ELONG_IDX] exit_ = self.strategy.should_exit( trade, row[OPEN_IDX], exit_candle_time, # type: ignore - enter=enter, exit_=exit_, + enter=enter, exit_=exit_sig, low=row[LOW_IDX], high=row[HIGH_IDX] ) @@ -568,6 +570,7 @@ class Backtesting: len(row) > EXIT_TAG_IDX and row[EXIT_TAG_IDX] is not None and len(row[EXIT_TAG_IDX]) > 0 + and exit_.exit_type in (ExitType.EXIT_SIGNAL,) ): trade.exit_reason = row[EXIT_TAG_IDX] @@ -626,9 +629,7 @@ class Backtesting: detail_data.loc[:, 'exit_short'] = row[ESHORT_IDX] detail_data.loc[:, 'enter_tag'] = row[ENTER_TAG_IDX] detail_data.loc[:, 'exit_tag'] = row[EXIT_TAG_IDX] - headers = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long', - 'enter_short', 'exit_short', 'enter_tag', 'exit_tag'] - for det_row in detail_data[headers].values.tolist(): + for det_row in detail_data[HEADERS].values.tolist(): res = self._get_exit_trade_entry_for_candle(trade, det_row) if res: return res From 2c0a7c5d74df52a4ff166bc23515d318b5d08466 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 27 Apr 2022 17:01:23 +0200 Subject: [PATCH 35/63] Don't call interest_rate and isolated_liq twice --- freqtrade/freqtradebot.py | 12 ------------ tests/test_freqtradebot.py | 12 +++++------- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 68623c748..c82f2c8fe 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -664,16 +664,6 @@ class FreqtradeBot(LoggingMixin): amount = safe_value_fallback(order, 'filled', 'amount') enter_limit_filled_price = safe_value_fallback(order, 'average', 'price') - # TODO: this might be unnecessary, as we're calling it in update_trade_state. - isolated_liq = self.exchange.get_liquidation_price( - leverage=leverage, - pair=pair, - amount=amount, - open_rate=enter_limit_filled_price, - is_short=is_short - ) - interest_rate = self.exchange.get_interest_rate() - # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker') base_currency = self.exchange.get_pair_base_currency(pair) @@ -702,8 +692,6 @@ class FreqtradeBot(LoggingMixin): timeframe=timeframe_to_minutes(self.config['timeframe']), leverage=leverage, is_short=is_short, - interest_rate=interest_rate, - liquidation_price=isolated_liq, trading_mode=self.trading_mode, funding_fees=funding_fees ) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 84d3c3324..89fe88a2c 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -717,12 +717,12 @@ def test_process_informative_pairs_added(default_conf_usdt, ticker_usdt, mocker) (True, 'spot', 'gateio', None, 0.0, None), (False, 'spot', 'okx', None, 0.0, None), (True, 'spot', 'okx', None, 0.0, None), - (True, 'futures', 'binance', 'isolated', 0.0, 11.89108910891089), - (False, 'futures', 'binance', 'isolated', 0.0, 8.070707070707071), + (True, 'futures', 'binance', 'isolated', 0.0, 11.88151815181518), + (False, 'futures', 'binance', 'isolated', 0.0, 8.080471380471382), (True, 'futures', 'gateio', 'isolated', 0.0, 11.87413417771621), (False, 'futures', 'gateio', 'isolated', 0.0, 8.085708510208207), - (True, 'futures', 'binance', 'isolated', 0.05, 11.796534653465345), - (False, 'futures', 'binance', 'isolated', 0.05, 8.167171717171717), + (True, 'futures', 'binance', 'isolated', 0.05, 11.7874422442244), + (False, 'futures', 'binance', 'isolated', 0.05, 8.17644781144781), (True, 'futures', 'gateio', 'isolated', 0.05, 11.7804274688304), (False, 'futures', 'gateio', 'isolated', 0.05, 8.181423084697796), (True, 'futures', 'okx', 'isolated', 0.0, 11.87413417771621), @@ -845,6 +845,7 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, assert trade.open_order_id is None assert trade.open_rate == 10 assert trade.stake_amount == round(order['price'] * order['filled'] / leverage, 8) + assert pytest.approx(trade.liquidation_price) == liq_price # In case of rejected or expired order and partially filled order['status'] = 'expired' @@ -932,8 +933,6 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, assert trade.open_rate_requested == 10 # In case of custom entry price not float type - freqtrade.exchange.get_maintenance_ratio_and_amt = MagicMock(return_value=(0.01, 0.01)) - freqtrade.exchange.name = exchange_name order['status'] = 'open' order['id'] = '5568' freqtrade.strategy.custom_entry_price = lambda **kwargs: "string price" @@ -946,7 +945,6 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, trade.is_short = is_short assert trade assert trade.open_rate_requested == 10 - assert trade.liquidation_price == liq_price # In case of too high stake amount From 220927289d2419046efcaf7bae1499effa537a54 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 27 Apr 2022 19:10:04 +0200 Subject: [PATCH 36/63] Update documentation to highlight futures supported exchanges --- README.md | 8 ++++++++ docs/index.md | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/README.md b/README.md index 679dbcab0..cad39f9ac 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,14 @@ Please read the [exchange specific notes](docs/exchanges.md) to learn about even - [X] [OKX](https://okx.com/) (Former OKEX) - [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ +### Experimentally, freqtrade also supports futures on the following exchanges + +- [X] [Binance](https://www.binance.com/) +- [X] [Gate.io](https://www.gate.io/ref/6266643) +- [X] [OKX](https://okx.com/). + +Please make sure to read the [exchange specific notes](docs/exchanges.md), as well as the [trading with leverage](docs/leverage.md) documentation before diving in. + ### Community tested Exchanges confirmed working by the community: diff --git a/docs/index.md b/docs/index.md index 2aa80c240..e0a88a381 100644 --- a/docs/index.md +++ b/docs/index.md @@ -51,6 +51,14 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual, - [X] [OKX](https://okx.com/) (Former OKEX) - [ ] [potentially many others through ccxt](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ +### Experimentally, freqtrade also supports futures on the following exchanges: + +- [X] [Binance](https://www.binance.com/) +- [X] [Gate.io](https://www.gate.io/ref/6266643) +- [X] [OKX](https://okx.com/). + +Please make sure to read the [exchange specific notes](exchanges.md), as well as the [trading with leverage](leverage.md) documentation before diving in. + ### Community tested Exchanges confirmed working by the community: From 46855221aab34d0c0687474c9e8a1af08cfa9916 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 27 Apr 2022 19:58:19 +0200 Subject: [PATCH 37/63] Fix rounding issue with contract-sized pairs for dry-run orders --- freqtrade/exchange/exchange.py | 4 +++- freqtrade/freqtradebot.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index d4741bd64..b12751fff 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -785,7 +785,9 @@ class Exchange: rate: float, leverage: float, params: Dict = {}, stop_loss: bool = False) -> Dict[str, Any]: order_id = f'dry_run_{side}_{datetime.now().timestamp()}' - _amount = self.amount_to_precision(pair, amount) + # Rounding here must respect to contract sizes + _amount = self._contracts_to_amount( + pair, self.amount_to_precision(pair, self._amount_to_contracts(pair, amount))) dry_order: Dict[str, Any] = { 'id': order_id, 'symbol': pair, diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c82f2c8fe..7c20a7f60 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -585,7 +585,6 @@ class FreqtradeBot(LoggingMixin): Executes a limit buy for the given pair :param pair: pair for which we want to create a LIMIT_BUY :param stake_amount: amount of stake-currency for the pair - :param leverage: amount of leverage applied to this trade :return: True if a buy order is created, false if it fails. """ time_in_force = self.strategy.order_time_in_force['entry'] From ca49821df011c8ebb7ec8586c08daf675d210bd4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 28 Apr 2022 06:29:14 +0200 Subject: [PATCH 38/63] Fix race condition for loop --- freqtrade/exchange/exchange.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index d4741bd64..8ecccbce0 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -9,6 +9,7 @@ import logging from copy import deepcopy from datetime import datetime, timedelta, timezone from math import ceil +from threading import Lock from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union import arrow @@ -96,6 +97,9 @@ class Exchange: self._markets: Dict = {} self._trading_fees: Dict[str, Any] = {} self._leverage_tiers: Dict[str, List[Dict]] = {} + # Lock event loop. This is necessary to avoid race-conditions when using force* commands + # Due to funding fee fetching. + self._loop_lock = Lock() self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) self._config: Dict = {} @@ -1775,7 +1779,8 @@ class Exchange: async def gather_stuff(): return await asyncio.gather(*input_coro, return_exceptions=True) - results = self.loop.run_until_complete(gather_stuff()) + with self._loop_lock: + results = self.loop.run_until_complete(gather_stuff()) for res in results: if isinstance(res, Exception): @@ -2032,9 +2037,10 @@ class Exchange: if not self.exchange_has("fetchTrades"): raise OperationalException("This exchange does not support downloading Trades.") - return self.loop.run_until_complete( - self._async_get_trade_history(pair=pair, since=since, - until=until, from_id=from_id)) + with self._loop_lock: + return self.loop.run_until_complete( + self._async_get_trade_history(pair=pair, since=since, + until=until, from_id=from_id)) @retrier def _get_funding_fees_from_exchange(self, pair: str, since: Union[datetime, int]) -> float: From 1e835896415191469a369c30cf5848d96004d7e3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 28 Apr 2022 06:59:03 +0200 Subject: [PATCH 39/63] Fix hyperopt --- freqtrade/optimize/hyperopt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 3ae975ca7..1dafb483c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -468,6 +468,7 @@ class Hyperopt: self.backtesting.exchange._api = None self.backtesting.exchange._api_async = None self.backtesting.exchange.loop = None # type: ignore + self.backtesting.exchange._loop_lock = None # type: ignore # self.backtesting.exchange = None # type: ignore self.backtesting.pairlists = None # type: ignore From 2ef1181e16b8d2389f51687d3be67b78b26e01f8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 28 Apr 2022 07:33:30 +0200 Subject: [PATCH 40/63] Simplify trade __repr__ --- freqtrade/persistence/models.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index a9c07f12c..2cacc06e2 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -429,12 +429,10 @@ class LocalTrade(): def __repr__(self): open_since = self.open_date.strftime(DATETIME_PRINT_FORMAT) if self.is_open else 'closed' - leverage = self.leverage or 1.0 - is_short = self.is_short or False return ( f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, ' - f'is_short={is_short}, leverage={leverage}, ' + f'is_short={self.is_short or False}, leverage={self.leverage or 1.0}, ' f'open_rate={self.open_rate:.8f}, open_since={open_since})' ) From 64072f76b9760679bebb24b66cd4dc52ba85d40f Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 28 Apr 2022 14:40:47 +0200 Subject: [PATCH 41/63] Don't fail scheduled ci tasks due to notification --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5bafe9cb8..0a87e2c81 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -321,6 +321,8 @@ jobs: notify-complete: needs: [ build_linux, build_macos, build_windows, docs_check, mypy_version_check ] runs-on: ubuntu-20.04 + # Discord notification can't handle schedule events + if: (github.event_name != 'schedule') steps: - name: Check user permission From 4c95996069dc161fa3c6aaba8cdc6ad9629a6250 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 28 Apr 2022 14:50:50 +0200 Subject: [PATCH 42/63] Add Permissions for notify-complete job --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 932649f61..9e7ebfc6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -324,6 +324,8 @@ jobs: notify-complete: needs: [ build_linux, build_macos, build_windows, docs_check, mypy_version_check ] runs-on: ubuntu-20.04 + permissions: + repository-projects: read steps: - name: Check user permission From cb5c3316d1e3dd63658de539c94eb559d91e4e73 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 28 Apr 2022 19:43:52 +0200 Subject: [PATCH 43/63] Simplify log output --- freqtrade/exchange/exchange.py | 4 ++-- tests/exchange/test_exchange.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 6bb4eb446..82dcacb51 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -171,7 +171,7 @@ class Exchange: self._api_async = self._init_ccxt( exchange_config, ccxt_async, ccxt_kwargs=ccxt_async_config) - logger.info('Using Exchange "%s"', self.name) + logger.info(f'Using Exchange "{self.name}"') if validate: # Check if timeframe is available @@ -559,7 +559,7 @@ class Exchange: # Therefore we also show that. raise OperationalException( f"The ccxt library does not provide the list of timeframes " - f"for the exchange \"{self.name}\" and this exchange " + f"for the exchange {self.name} and this exchange " f"is therefore not supported. ccxt fetchOHLCV: {self.exchange_has('fetchOHLCV')}") if timeframe and (timeframe not in self.timeframes): diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index a6918b6d4..689ffa4ce 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -909,7 +909,7 @@ def test_validate_timeframes_emulated_ohlcv_1(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') with pytest.raises(OperationalException, match=r'The ccxt library does not provide the list of timeframes ' - r'for the exchange ".*" and this exchange ' + r'for the exchange .* and this exchange ' r'is therefore not supported. *'): Exchange(default_conf) @@ -930,7 +930,7 @@ def test_validate_timeframes_emulated_ohlcvi_2(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') with pytest.raises(OperationalException, match=r'The ccxt library does not provide the list of timeframes ' - r'for the exchange ".*" and this exchange ' + r'for the exchange .* and this exchange ' r'is therefore not supported. *'): Exchange(default_conf) From d1a61f9c615fb7e5d9717c126d9280bccf3e30ec Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 28 Apr 2022 20:05:19 +0200 Subject: [PATCH 44/63] Don't start futures backtest if leverage-tiers don't contain pair --- freqtrade/optimize/backtesting.py | 8 ++++++++ tests/optimize/test_backtesting.py | 33 ++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 210eab39b..0f816f295 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -269,10 +269,18 @@ class Backtesting: candle_type=CandleType.from_string(self.exchange._ft_has["mark_ohlcv_price"]) ) # Combine data to avoid combining the data per trade. + unavailable_pairs = [] for pair in self.pairlists.whitelist: + if pair not in self.exchange._leverage_tiers: + unavailable_pairs.append(pair) + continue self.futures_data[pair] = funding_rates_dict[pair].merge( mark_rates_dict[pair], on='date', how="inner", suffixes=["_fund", "_mark"]) + if unavailable_pairs: + raise OperationalException( + f"Pairs {', '.join(unavailable_pairs)} got no leverage tiers available. " + "It is therefore impossible to backtest with this pair at the moment.") else: self.futures_data = {} diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index d7ee4a042..a51e1b654 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1341,6 +1341,39 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat assert 'STRATEGY SUMMARY' in captured.out +@pytest.mark.filterwarnings("ignore:deprecated") +def test_backtest_start_futures_noliq(default_conf_usdt, mocker, + caplog, testdatadir, capsys): + # Tests detail-data loading + default_conf_usdt.update({ + "trading_mode": "futures", + "margin_mode": "isolated", + "use_exit_signal": True, + "exit_profit_only": False, + "exit_profit_offset": 0.0, + "ignore_roi_if_entry_signal": False, + "strategy": CURRENT_TEST_STRATEGY, + }) + patch_exchange(mocker) + + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT'])) + # mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) + + patched_configuration_load_config_file(mocker, default_conf_usdt) + + args = [ + 'backtesting', + '--config', 'config.json', + '--datadir', str(testdatadir), + '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'), + '--timeframe', '1h', + ] + args = get_args(args) + with pytest.raises(OperationalException, match=r"Pairs .* got no leverage tiers available\."): + start_backtesting(args) + + @pytest.mark.filterwarnings("ignore:deprecated") def test_backtest_start_nomock_futures(default_conf_usdt, mocker, caplog, testdatadir, capsys): From 21df1b0db32f04bbb362c33afa483f17c328bfb7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 27 Apr 2022 13:13:11 +0200 Subject: [PATCH 45/63] Use ORJSON for http responses --- freqtrade/rpc/api_server/webserver.py | 4 ++-- requirements.txt | 2 ++ setup.py | 1 + tests/rpc/test_rpc_apiserver.py | 3 +-- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index 63812f52f..0da129583 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -2,7 +2,7 @@ import logging from ipaddress import IPv4Address from typing import Any, Dict -import rapidjson +import orjson import uvicorn from fastapi import Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -24,7 +24,7 @@ class FTJSONResponse(JSONResponse): Use rapidjson for responses Handles NaN and Inf / -Inf in a javascript way by default. """ - return rapidjson.dumps(content).encode("utf-8") + return orjson.dumps(content, option=orjson.OPT_SERIALIZE_NUMPY) class ApiServer(RPCHandler): diff --git a/requirements.txt b/requirements.txt index de14b9f2c..ab8329979 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,6 +27,8 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.6 +# Properly format api responses +orjson==3.6.8 # Notify systemd sdnotify==0.3.2 diff --git a/setup.py b/setup.py index 250cafdc9..c5e418d0d 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ setup( 'pycoingecko', 'py_find_1st', 'python-rapidjson', + 'orjson', 'sdnotify', 'colorama', 'jinja2', diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 4910213b4..43f783a53 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -13,7 +13,6 @@ import uvicorn from fastapi import FastAPI from fastapi.exceptions import HTTPException from fastapi.testclient import TestClient -from numpy import isnan from requests.auth import _basic_auth_str from freqtrade.__init__ import __version__ @@ -985,7 +984,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets, is_short, assert_response(rc) resp_values = rc.json() assert len(resp_values) == 4 - assert isnan(resp_values[0]['profit_abs']) + assert resp_values[0]['profit_abs'] is None def test_api_version(botclient): From 48ff788e827ecb16a59d09260e03133ea13e6085 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 Apr 2022 09:53:05 +0000 Subject: [PATCH 46/63] Clarify that stoploss is required closes #6740 --- docs/strategy-callbacks.md | 36 ++++++++++++++++++------------------ freqtrade/constants.py | 2 ++ 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 563b5a2cb..c5639d581 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -122,11 +122,11 @@ See [Dataframe access](strategy-advanced.md#dataframe-access) for more informati ## Custom stoploss -Called for open trade every throttling iteration (roughly every 5 seconds) until a trade is closed. +Called for open trade every iteration (roughly every 5 seconds) until a trade is closed. The usage of the custom stoploss method must be enabled by setting `use_custom_stoploss=True` on the strategy object. -The stoploss price can only ever move upwards - if the stoploss value returned from `custom_stoploss` would result in a lower stoploss price than was previously set, it will be ignored. The traditional `stoploss` value serves as an absolute lower level and will be instated as the initial stoploss (before this method is called for the first time for a trade). +The stoploss price can only ever move upwards - if the stoploss value returned from `custom_stoploss` would result in a lower stoploss price than was previously set, it will be ignored. The traditional `stoploss` value serves as an absolute lower level and will be instated as the initial stoploss (before this method is called for the first time for a trade), and is still mandatory. The method must return a stoploss value (float / number) as a percentage of the current price. E.g. If the `current_rate` is 200 USD, then returning `0.02` will set the stoploss price 2% lower, at 196 USD. @@ -365,13 +365,13 @@ class AwesomeStrategy(IStrategy): # ... populate_* methods - def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) new_entryprice = dataframe['bollinger_10_lowerband'].iat[-1] - + return new_entryprice def custom_exit_price(self, pair: str, trade: Trade, @@ -381,14 +381,14 @@ class AwesomeStrategy(IStrategy): dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) new_exitprice = dataframe['bollinger_10_upperband'].iat[-1] - + return new_exitprice ``` !!! Warning - Modifying entry and exit prices will only work for limit orders. Depending on the price chosen, this can result in a lot of unfilled orders. By default the maximum allowed distance between the current price and the custom price is 2%, this value can be changed in config with the `custom_price_max_distance_ratio` parameter. - **Example**: + Modifying entry and exit prices will only work for limit orders. Depending on the price chosen, this can result in a lot of unfilled orders. By default the maximum allowed distance between the current price and the custom price is 2%, this value can be changed in config with the `custom_price_max_distance_ratio` parameter. + **Example**: If the new_entryprice is 97, the proposed_rate is 100 and the `custom_price_max_distance_ratio` is set to 2%, The retained valid custom entry price will be 98, which is 2% below the current (proposed) rate. !!! Warning "Backtesting" @@ -430,7 +430,7 @@ class AwesomeStrategy(IStrategy): 'exit': 60 * 25 } - def check_entry_timeout(self, pair: str, trade: 'Trade', order: 'Order', + def check_entry_timeout(self, pair: str, trade: 'Trade', order: 'Order', current_time: datetime, **kwargs) -> bool: if trade.open_rate > 100 and trade.open_date_utc < current_time - timedelta(minutes=5): return True @@ -508,7 +508,7 @@ class AwesomeStrategy(IStrategy): # ... populate_* methods def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float, - time_in_force: str, current_time: datetime, entry_tag: Optional[str], + time_in_force: str, current_time: datetime, entry_tag: Optional[str], side: str, **kwargs) -> bool: """ Called right before placing a entry order. @@ -616,35 +616,35 @@ from freqtrade.persistence import Trade class DigDeeperStrategy(IStrategy): - + position_adjustment_enable = True - + # Attempts to handle large drops with DCA. High stoploss is required. stoploss = -0.30 - + # ... populate_* methods - + # Example specific variables max_entry_position_adjustment = 3 # This number is explained a bit further down max_dca_multiplier = 5.5 - + # This is called when placing the initial order (opening trade) def custom_stake_amount(self, pair: str, current_time: datetime, current_rate: float, proposed_stake: float, min_stake: float, max_stake: float, entry_tag: Optional[str], side: str, **kwargs) -> float: - + # We need to leave most of the funds for possible further DCA orders # This also applies to fixed stakes return proposed_stake / self.max_dca_multiplier - + def adjust_trade_position(self, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, min_stake: float, max_stake: float, **kwargs): """ Custom trade adjustment logic, returning the stake amount that a trade should be increased. This means extra buy orders with additional fees. - + :param trade: trade object. :param current_time: datetime object, containing the current datetime :param current_rate: Current buy rate. @@ -654,7 +654,7 @@ class DigDeeperStrategy(IStrategy): :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return float: Stake amount to adjust your trade """ - + if current_profit > -0.05: return None diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 1a21ec77f..3b98ce56b 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -459,6 +459,8 @@ SCHEMA_BACKTEST_REQUIRED = [ 'stake_currency', 'stake_amount', 'dry_run_wallet', + 'stoploss', + 'minimal_roi', 'dataformat_ohlcv', 'dataformat_trades', ] From b6bee45e82604a06f3143aa10967c0641847e779 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 Apr 2022 09:54:54 +0000 Subject: [PATCH 47/63] Exclude user_data from isort --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 50f0242a8..e8d5ed47e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ exclude = ''' line_length = 100 multi_line_output=0 lines_after_imports=2 -skip_glob = ["**/.env*", "**/env/*", "**/.venv/*", "**/docs/*"] +skip_glob = ["**/.env*", "**/env/*", "**/.venv/*", "**/docs/*", "**/user_data/*"] [tool.pytest.ini_options] asyncio_mode = "auto" From da7a6f58f94db81d3f436dacadbdf2c7a903ee1d Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 Apr 2022 17:46:33 +0200 Subject: [PATCH 48/63] Revert requiring stoploss for backtest/hyperopt --- freqtrade/constants.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 3b98ce56b..1a21ec77f 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -459,8 +459,6 @@ SCHEMA_BACKTEST_REQUIRED = [ 'stake_currency', 'stake_amount', 'dry_run_wallet', - 'stoploss', - 'minimal_roi', 'dataformat_ohlcv', 'dataformat_trades', ] From f96c552c46a56ad5822d764591f7828a5a2dc4bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=AE=AE=E0=AE=A9=E0=AF=8B=E0=AE=9C=E0=AF=8D=E0=AE=95?= =?UTF-8?q?=E0=AF=81=E0=AE=AE=E0=AE=BE=E0=AE=B0=E0=AF=8D=20=E0=AE=AA?= =?UTF-8?q?=E0=AE=B4=E0=AE=A9=E0=AE=BF=E0=AE=9A=E0=AF=8D=E0=AE=9A=E0=AE=BE?= =?UTF-8?q?=E0=AE=AE=E0=AE=BF?= Date: Fri, 29 Apr 2022 22:14:02 +0530 Subject: [PATCH 49/63] Update PULL_REQUEST_TEMPLATE.md Added instructions as comments --- .github/PULL_REQUEST_TEMPLATE.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 7c0655b20..25a9761e2 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,17 +1,17 @@ -Thank you for sending your pull request. But first, have you included + ## Summary -Explain in one sentence the goal of this PR + Solve the issue: #___ ## Quick changelog -- -- +- +- ## What's new? -*Explain in details what this PR solve or improve. You can include visuals.* + From fbd142844fff0065bdac6a9e8770990faf289ab3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 Apr 2022 19:37:13 +0200 Subject: [PATCH 50/63] Refactor bt-caching stuff to it's own module --- freqtrade/data/btanalysis.py | 3 +- freqtrade/misc.py | 35 +----------------------- freqtrade/optimize/backtest_caching.py | 38 ++++++++++++++++++++++++++ freqtrade/optimize/backtesting.py | 2 +- freqtrade/optimize/optimize_reports.py | 4 +-- tests/optimize/test_backtesting.py | 2 +- 6 files changed, 45 insertions(+), 39 deletions(-) create mode 100644 freqtrade/optimize/backtest_caching.py diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 206a6f5f3..0c8e721c0 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -12,7 +12,8 @@ import pandas as pd from freqtrade.constants import LAST_BT_RESULT_FN from freqtrade.exceptions import OperationalException -from freqtrade.misc import get_backtest_metadata_filename, json_load +from freqtrade.misc import json_load +from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename from freqtrade.persistence import LocalTrade, Trade, init_db diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 55a533725..c3968e61c 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -2,13 +2,11 @@ Various tool function for Freqtrade and scripts """ import gzip -import hashlib import logging import re -from copy import deepcopy from datetime import datetime from pathlib import Path -from typing import Any, Iterator, List, Union +from typing import Any, Iterator, List from typing.io import IO from urllib.parse import urlparse @@ -251,34 +249,3 @@ def parse_db_uri_for_logging(uri: str): return uri pwd = parsed_db_uri.netloc.split(':')[1].split('@')[0] return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@') - - -def get_strategy_run_id(strategy) -> str: - """ - Generate unique identification hash for a backtest run. Identical config and strategy file will - always return an identical hash. - :param strategy: strategy object. - :return: hex string id. - """ - digest = hashlib.sha1() - config = deepcopy(strategy.config) - - # Options that have no impact on results of individual backtest. - not_important_keys = ('strategy_list', 'original_config', 'telegram', 'api_server') - for k in not_important_keys: - if k in config: - del config[k] - - # Explicitly allow NaN values (e.g. max_open_trades). - # as it does not matter for getting the hash. - digest.update(rapidjson.dumps(config, default=str, - number_mode=rapidjson.NM_NAN).encode('utf-8')) - with open(strategy.__file__, 'rb') as fp: - digest.update(fp.read()) - return digest.hexdigest().lower() - - -def get_backtest_metadata_filename(filename: Union[Path, str]) -> Path: - """Return metadata filename for specified backtest results file.""" - filename = Path(filename) - return filename.parent / Path(f'{filename.stem}.meta{filename.suffix}') diff --git a/freqtrade/optimize/backtest_caching.py b/freqtrade/optimize/backtest_caching.py new file mode 100644 index 000000000..c2a9903fa --- /dev/null +++ b/freqtrade/optimize/backtest_caching.py @@ -0,0 +1,38 @@ +import hashlib +from copy import deepcopy +from pathlib import Path +from typing import Union + +import rapidjson + + +def get_strategy_run_id(strategy) -> str: + """ + Generate unique identification hash for a backtest run. Identical config and strategy file will + always return an identical hash. + :param strategy: strategy object. + :return: hex string id. + """ + digest = hashlib.sha1() + config = deepcopy(strategy.config) + + # Options that have no impact on results of individual backtest. + not_important_keys = ('strategy_list', 'original_config', 'telegram', 'api_server') + for k in not_important_keys: + if k in config: + del config[k] + + # Explicitly allow NaN values (e.g. max_open_trades). + # as it does not matter for getting the hash. + digest.update(rapidjson.dumps(config, default=str, + number_mode=rapidjson.NM_NAN).encode('utf-8')) + + with open(strategy.__file__, 'rb') as fp: + digest.update(fp.read()) + return digest.hexdigest().lower() + + +def get_backtest_metadata_filename(filename: Union[Path, str]) -> Path: + """Return metadata filename for specified backtest results file.""" + filename = Path(filename) + return filename.parent / Path(f'{filename.stem}.meta{filename.suffix}') diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 0f816f295..db4496f0f 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -24,8 +24,8 @@ from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType TradingMode) from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds -from freqtrade.misc import get_strategy_run_id from freqtrade.mixins import LoggingMixin +from freqtrade.optimize.backtest_caching import get_strategy_run_id from freqtrade.optimize.bt_progress import BTProgress from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results, store_backtest_signal_candles, diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index dd058aff4..1d58dc339 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -11,8 +11,8 @@ from tabulate import tabulate from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT from freqtrade.data.btanalysis import (calculate_cagr, calculate_csum, calculate_market_change, calculate_max_drawdown) -from freqtrade.misc import (decimals_per_coin, file_dump_joblib, file_dump_json, - get_backtest_metadata_filename, round_coin_value) +from freqtrade.misc import decimals_per_coin, file_dump_joblib, file_dump_json, round_coin_value +from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename logger = logging.getLogger(__name__) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index a51e1b654..7494155b4 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -22,7 +22,7 @@ from freqtrade.data.history import get_timerange from freqtrade.enums import ExitType, RunMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange.exchange import timeframe_to_next_date -from freqtrade.misc import get_strategy_run_id +from freqtrade.optimize.backtest_caching import get_strategy_run_id from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import LocalTrade from freqtrade.resolvers import StrategyResolver From 43049e04652483ff039ee7765a0181d2ed1ca337 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 Apr 2022 19:44:17 +0200 Subject: [PATCH 51/63] Evict cache if parameter file changed closes #6735 --- freqtrade/optimize/backtest_caching.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtest_caching.py b/freqtrade/optimize/backtest_caching.py index c2a9903fa..d9d270072 100644 --- a/freqtrade/optimize/backtest_caching.py +++ b/freqtrade/optimize/backtest_caching.py @@ -26,7 +26,9 @@ def get_strategy_run_id(strategy) -> str: # as it does not matter for getting the hash. digest.update(rapidjson.dumps(config, default=str, number_mode=rapidjson.NM_NAN).encode('utf-8')) - + # Include _ft_params_from_file - so changing parameter files cause cache eviction + digest.update(rapidjson.dumps( + strategy._ft_params_from_file, default=str, number_mode=rapidjson.NM_NAN).encode('utf-8')) with open(strategy.__file__, 'rb') as fp: digest.update(fp.read()) return digest.hexdigest().lower() From f23faac36880027896de8d7bcc21f5af9edd6714 Mon Sep 17 00:00:00 2001 From: erdieee <58039191+erdieee@users.noreply.github.com> Date: Fri, 29 Apr 2022 20:10:50 +0200 Subject: [PATCH 52/63] Fix config_examples typo --- config_examples/config_binance.example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config_examples/config_binance.example.json b/config_examples/config_binance.example.json index ad8862afa..35b9fcd20 100644 --- a/config_examples/config_binance.example.json +++ b/config_examples/config_binance.example.json @@ -90,7 +90,7 @@ }, "bot_name": "freqtrade", "initial_state": "running", - "force_enter_enable": false, + "force_entry_enable": false, "internals": { "process_throttle_secs": 5 } From 788d9f5b55cef5657ad476d6feef8cbd227a3dec Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Fri, 29 Apr 2022 22:20:26 -0600 Subject: [PATCH 53/63] updated bot_start documentation with working example --- docs/strategy-callbacks.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index e0cfbe8c1..b01959e10 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -39,7 +39,10 @@ class AwesomeStrategy(IStrategy): Called only once after bot instantiation. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. """ - self.dp['remote_data'] = requests.get('https://some_remote_source.example.com') + if self.config['runmode'].value in ('live', 'dry_run'): + # Assign this to the class by using self.* + # can then be used by populate_* methods + self.remote_data = requests.get('https://some_remote_source.example.com') ``` ## Bot loop start From 23431a71064367496d8250ca7a7dd20b97da63d4 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Fri, 29 Apr 2022 22:21:22 -0600 Subject: [PATCH 54/63] removed invalid plotting and test_default_strategy tests for bot_start, edited edge test --- tests/optimize/test_edge_cli.py | 2 +- tests/strategy/test_default_strategy.py | 1 - tests/test_plotting.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/optimize/test_edge_cli.py b/tests/optimize/test_edge_cli.py index d9711b318..8241a5362 100644 --- a/tests/optimize/test_edge_cli.py +++ b/tests/optimize/test_edge_cli.py @@ -94,7 +94,7 @@ def test_edge_init(mocker, edge_conf) -> None: assert edge_cli.config == edge_conf assert edge_cli.config['stake_amount'] == 'unlimited' assert callable(edge_cli.edge.calculate) - assert edge_conf['strategy'].bot_started is True + assert edge_cli.strategy.bot_started is True def test_edge_init_fee(mocker, edge_conf) -> None: diff --git a/tests/strategy/test_default_strategy.py b/tests/strategy/test_default_strategy.py index a60274afd..5cb8fce16 100644 --- a/tests/strategy/test_default_strategy.py +++ b/tests/strategy/test_default_strategy.py @@ -32,7 +32,6 @@ def test_strategy_test_v3(result, fee, is_short, side): assert type(indicators) is DataFrame assert type(strategy.populate_buy_trend(indicators, metadata)) is DataFrame assert type(strategy.populate_sell_trend(indicators, metadata)) is DataFrame - assert strategy.bot_started is True trade = Trade( open_rate=19_000, diff --git a/tests/test_plotting.py b/tests/test_plotting.py index f0a28c4eb..97f367608 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -58,7 +58,6 @@ def test_init_plotscript(default_conf, mocker, testdatadir): assert "ohlcv" in ret assert "TRX/BTC" in ret["ohlcv"] assert "ADA/BTC" in ret["ohlcv"] - assert default_conf['strategy'].bot_started is True def test_add_indicators(default_conf, testdatadir, caplog): From 8756e7d9a1febecfae12649cc8ed21952d2f0086 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Fri, 29 Apr 2022 23:35:08 -0600 Subject: [PATCH 55/63] flake8 linting --- freqtrade/edge/edge_positioning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index 5dc84f172..fb9041574 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -90,7 +90,7 @@ class Edge: self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0]) except IndexError: self.fee = None - + strategy.bot_start() def calculate(self, pairs: List[str]) -> bool: From 09b74cebce95c80bec0f6982bf9b865523e49874 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 Apr 2022 08:55:07 +0200 Subject: [PATCH 56/63] Move edge bot_loop_start to edge_cli (otherwise it's called twice when running trade mode with edge on). --- docs/strategy-callbacks.md | 6 +++--- freqtrade/edge/edge_positioning.py | 2 -- freqtrade/optimize/edge_cli.py | 1 + 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 27777c2ce..5ff499b01 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -24,11 +24,11 @@ Currently available callbacks: ## Bot start -A simple callback which is called once when the bot starts. +A simple callback which is called once when the strategy is loaded. This can be used to perform actions that must only be performed once and runs after dataprovider and wallet are set ``` python -import asyncio +import requests class AwesomeStrategy(IStrategy): @@ -42,7 +42,7 @@ class AwesomeStrategy(IStrategy): if self.config['runmode'].value in ('live', 'dry_run'): # Assign this to the class by using self.* # can then be used by populate_* methods - self.remote_data = requests.get('https://some_remote_source.example.com') + self.cust_remote_data = requests.get('https://some_remote_source.example.com') ``` ## Bot loop start diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index fb9041574..2fe41a17b 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -91,8 +91,6 @@ class Edge: except IndexError: self.fee = None - strategy.bot_start() - def calculate(self, pairs: List[str]) -> bool: if self.fee is None and pairs: self.fee = self.exchange.get_fee(pairs[0]) diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index cc9bafb0b..30eabecd0 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -44,6 +44,7 @@ class EdgeCli: self.edge._timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) + self.strategy.bot_start() def start(self) -> None: result = self.edge.calculate(self.config['exchange']['pair_whitelist']) From 2acb68e6e24fdb6920b89fa2992d55d7f7c91bcf Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 Apr 2022 13:59:23 +0200 Subject: [PATCH 57/63] Move hyperopt-loss functions to their own package --- freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_calmar.py | 0 .../optimize/{ => hyperopt_loss}/hyperopt_loss_max_drawdown.py | 0 .../optimize/{ => hyperopt_loss}/hyperopt_loss_onlyprofit.py | 0 .../{ => hyperopt_loss}/hyperopt_loss_profit_drawdown.py | 0 freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_sharpe.py | 0 .../optimize/{ => hyperopt_loss}/hyperopt_loss_sharpe_daily.py | 0 .../{ => hyperopt_loss}/hyperopt_loss_short_trade_dur.py | 0 freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_sortino.py | 0 .../optimize/{ => hyperopt_loss}/hyperopt_loss_sortino_daily.py | 0 freqtrade/resolvers/hyperopt_resolver.py | 2 +- tests/optimize/test_hyperoptloss.py | 2 +- 11 files changed, 2 insertions(+), 2 deletions(-) rename freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_calmar.py (100%) rename freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_max_drawdown.py (100%) rename freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_onlyprofit.py (100%) rename freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_profit_drawdown.py (100%) rename freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_sharpe.py (100%) rename freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_sharpe_daily.py (100%) rename freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_short_trade_dur.py (100%) rename freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_sortino.py (100%) rename freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_sortino_daily.py (100%) diff --git a/freqtrade/optimize/hyperopt_loss_calmar.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_calmar.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py diff --git a/freqtrade/optimize/hyperopt_loss_max_drawdown.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_max_drawdown.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown.py diff --git a/freqtrade/optimize/hyperopt_loss_onlyprofit.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_onlyprofit.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_onlyprofit.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_onlyprofit.py diff --git a/freqtrade/optimize/hyperopt_loss_profit_drawdown.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_profit_drawdown.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py diff --git a/freqtrade/optimize/hyperopt_loss_sharpe.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_sharpe.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py diff --git a/freqtrade/optimize/hyperopt_loss_sharpe_daily.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe_daily.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_sharpe_daily.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe_daily.py diff --git a/freqtrade/optimize/hyperopt_loss_short_trade_dur.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_short_trade_dur.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_short_trade_dur.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_short_trade_dur.py diff --git a/freqtrade/optimize/hyperopt_loss_sortino.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_sortino.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py diff --git a/freqtrade/optimize/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino_daily.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_sortino_daily.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino_daily.py diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index e3c234f60..bcfe5e1d8 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -23,7 +23,7 @@ class HyperOptLossResolver(IResolver): object_type = IHyperOptLoss object_type_str = "HyperoptLoss" user_subdir = USERPATH_HYPEROPTS - initial_search_path = Path(__file__).parent.parent.joinpath('optimize').resolve() + initial_search_path = Path(__file__).parent.parent.joinpath('optimize/hyperopt_loss').resolve() @staticmethod def load_hyperoptloss(config: Dict) -> IHyperOptLoss: diff --git a/tests/optimize/test_hyperoptloss.py b/tests/optimize/test_hyperoptloss.py index e3f6daf6c..4ec80ef49 100644 --- a/tests/optimize/test_hyperoptloss.py +++ b/tests/optimize/test_hyperoptloss.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock import pytest from freqtrade.exceptions import OperationalException -from freqtrade.optimize.hyperopt_loss_short_trade_dur import ShortTradeDurHyperOptLoss +from freqtrade.optimize.hyperopt_loss.hyperopt_loss_short_trade_dur import ShortTradeDurHyperOptLoss from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver From c6c569b77288519c5a7d2016d6a291ee8d68b9ea Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 Apr 2022 14:47:27 +0200 Subject: [PATCH 58/63] chore: split BTAnalyais to metrics --- freqtrade/data/btanalysis.py | 167 +---------------- freqtrade/data/metrics.py | 173 ++++++++++++++++++ .../hyperopt_loss/hyperopt_loss_calmar.py | 2 +- .../hyperopt_loss_max_drawdown.py | 2 +- .../hyperopt_loss_profit_drawdown.py | 2 +- freqtrade/optimize/optimize_reports.py | 4 +- freqtrade/plot/plotting.py | 7 +- .../protections/max_drawdown_protection.py | 2 +- tests/data/test_btanalysis.py | 8 +- tests/test_plotting.py | 3 +- 10 files changed, 190 insertions(+), 180 deletions(-) create mode 100644 freqtrade/data/metrics.py diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 0c8e721c0..e29d9ebe4 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -5,7 +5,7 @@ import logging from copy import copy from datetime import datetime, timezone from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Union import numpy as np import pandas as pd @@ -400,168 +400,3 @@ def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame, trades = trades.loc[(trades['open_date'] >= trades_start) & (trades['close_date'] <= trades_stop)] return trades - - -def calculate_market_change(data: Dict[str, pd.DataFrame], column: str = "close") -> float: - """ - Calculate market change based on "column". - Calculation is done by taking the first non-null and the last non-null element of each column - and calculating the pctchange as "(last - first) / first". - Then the results per pair are combined as mean. - - :param data: Dict of Dataframes, dict key should be pair. - :param column: Column in the original dataframes to use - :return: - """ - tmp_means = [] - for pair, df in data.items(): - start = df[column].dropna().iloc[0] - end = df[column].dropna().iloc[-1] - tmp_means.append((end - start) / start) - - return float(np.mean(tmp_means)) - - -def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame], - column: str = "close") -> pd.DataFrame: - """ - Combine multiple dataframes "column" - :param data: Dict of Dataframes, dict key should be pair. - :param column: Column in the original dataframes to use - :return: DataFrame with the column renamed to the dict key, and a column - named mean, containing the mean of all pairs. - :raise: ValueError if no data is provided. - """ - df_comb = pd.concat([data[pair].set_index('date').rename( - {column: pair}, axis=1)[pair] for pair in data], axis=1) - - df_comb['mean'] = df_comb.mean(axis=1) - - return df_comb - - -def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, - timeframe: str) -> pd.DataFrame: - """ - Adds a column `col_name` with the cumulative profit for the given trades array. - :param df: DataFrame with date index - :param trades: DataFrame containing trades (requires columns close_date and profit_abs) - :param col_name: Column name that will be assigned the results - :param timeframe: Timeframe used during the operations - :return: Returns df with one additional column, col_name, containing the cumulative profit. - :raise: ValueError if trade-dataframe was found empty. - """ - if len(trades) == 0: - raise ValueError("Trade dataframe empty.") - from freqtrade.exchange import timeframe_to_minutes - timeframe_minutes = timeframe_to_minutes(timeframe) - # Resample to timeframe to make sure trades match candles - _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date' - )[['profit_abs']].sum() - df.loc[:, col_name] = _trades_sum['profit_abs'].cumsum() - # Set first value to 0 - df.loc[df.iloc[0].name, col_name] = 0 - # FFill to get continuous - df[col_name] = df[col_name].ffill() - return df - - -def _calc_drawdown_series(profit_results: pd.DataFrame, *, date_col: str, value_col: str - ) -> pd.DataFrame: - max_drawdown_df = pd.DataFrame() - max_drawdown_df['cumulative'] = profit_results[value_col].cumsum() - max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax() - max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value'] - max_drawdown_df['date'] = profit_results.loc[:, date_col] - return max_drawdown_df - - -def calculate_underwater(trades: pd.DataFrame, *, date_col: str = 'close_date', - value_col: str = 'profit_ratio' - ): - """ - Calculate max drawdown and the corresponding close dates - :param trades: DataFrame containing trades (requires columns close_date and profit_ratio) - :param date_col: Column in DataFrame to use for dates (defaults to 'close_date') - :param value_col: Column in DataFrame to use for values (defaults to 'profit_ratio') - :return: Tuple (float, highdate, lowdate, highvalue, lowvalue) with absolute max drawdown, - high and low time and high and low value. - :raise: ValueError if trade-dataframe was found empty. - """ - if len(trades) == 0: - raise ValueError("Trade dataframe empty.") - profit_results = trades.sort_values(date_col).reset_index(drop=True) - max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col) - - return max_drawdown_df - - -def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date', - value_col: str = 'profit_abs', starting_balance: float = 0 - ) -> Tuple[float, pd.Timestamp, pd.Timestamp, float, float, float]: - """ - Calculate max drawdown and the corresponding close dates - :param trades: DataFrame containing trades (requires columns close_date and profit_ratio) - :param date_col: Column in DataFrame to use for dates (defaults to 'close_date') - :param value_col: Column in DataFrame to use for values (defaults to 'profit_abs') - :param starting_balance: Portfolio starting balance - properly calculate relative drawdown. - :return: Tuple (float, highdate, lowdate, highvalue, lowvalue, relative_drawdown) - with absolute max drawdown, high and low time and high and low value, - and the relative account drawdown - :raise: ValueError if trade-dataframe was found empty. - """ - if len(trades) == 0: - raise ValueError("Trade dataframe empty.") - profit_results = trades.sort_values(date_col).reset_index(drop=True) - max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col) - - idxmin = max_drawdown_df['drawdown'].idxmin() - if idxmin == 0: - raise ValueError("No losing trade, therefore no drawdown.") - high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col] - low_date = profit_results.loc[idxmin, date_col] - high_val = max_drawdown_df.loc[max_drawdown_df.iloc[:idxmin] - ['high_value'].idxmax(), 'cumulative'] - low_val = max_drawdown_df.loc[idxmin, 'cumulative'] - max_drawdown_rel = 0.0 - if high_val + starting_balance != 0: - max_drawdown_rel = (high_val - low_val) / (high_val + starting_balance) - - return ( - abs(min(max_drawdown_df['drawdown'])), - high_date, - low_date, - high_val, - low_val, - max_drawdown_rel - ) - - -def calculate_csum(trades: pd.DataFrame, starting_balance: float = 0) -> Tuple[float, float]: - """ - Calculate min/max cumsum of trades, to show if the wallet/stake amount ratio is sane - :param trades: DataFrame containing trades (requires columns close_date and profit_percent) - :param starting_balance: Add starting balance to results, to show the wallets high / low points - :return: Tuple (float, float) with cumsum of profit_abs - :raise: ValueError if trade-dataframe was found empty. - """ - if len(trades) == 0: - raise ValueError("Trade dataframe empty.") - - csum_df = pd.DataFrame() - csum_df['sum'] = trades['profit_abs'].cumsum() - csum_min = csum_df['sum'].min() + starting_balance - csum_max = csum_df['sum'].max() + starting_balance - - return csum_min, csum_max - - -def calculate_cagr(days_passed: int, starting_balance: float, final_balance: float) -> float: - """ - Calculate CAGR - :param days_passed: Days passed between start and ending balance - :param starting_balance: Starting balance - :param final_balance: Final balance to calculate CAGR against - :return: CAGR - """ - return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1 diff --git a/freqtrade/data/metrics.py b/freqtrade/data/metrics.py new file mode 100644 index 000000000..44d5ce6ec --- /dev/null +++ b/freqtrade/data/metrics.py @@ -0,0 +1,173 @@ +import logging +from typing import Dict, Tuple + +import numpy as np +import pandas as pd + + +logger = logging.getLogger(__name__) + + +def calculate_market_change(data: Dict[str, pd.DataFrame], column: str = "close") -> float: + """ + Calculate market change based on "column". + Calculation is done by taking the first non-null and the last non-null element of each column + and calculating the pctchange as "(last - first) / first". + Then the results per pair are combined as mean. + + :param data: Dict of Dataframes, dict key should be pair. + :param column: Column in the original dataframes to use + :return: + """ + tmp_means = [] + for pair, df in data.items(): + start = df[column].dropna().iloc[0] + end = df[column].dropna().iloc[-1] + tmp_means.append((end - start) / start) + + return float(np.mean(tmp_means)) + + +def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame], + column: str = "close") -> pd.DataFrame: + """ + Combine multiple dataframes "column" + :param data: Dict of Dataframes, dict key should be pair. + :param column: Column in the original dataframes to use + :return: DataFrame with the column renamed to the dict key, and a column + named mean, containing the mean of all pairs. + :raise: ValueError if no data is provided. + """ + df_comb = pd.concat([data[pair].set_index('date').rename( + {column: pair}, axis=1)[pair] for pair in data], axis=1) + + df_comb['mean'] = df_comb.mean(axis=1) + + return df_comb + + +def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, + timeframe: str) -> pd.DataFrame: + """ + Adds a column `col_name` with the cumulative profit for the given trades array. + :param df: DataFrame with date index + :param trades: DataFrame containing trades (requires columns close_date and profit_abs) + :param col_name: Column name that will be assigned the results + :param timeframe: Timeframe used during the operations + :return: Returns df with one additional column, col_name, containing the cumulative profit. + :raise: ValueError if trade-dataframe was found empty. + """ + if len(trades) == 0: + raise ValueError("Trade dataframe empty.") + from freqtrade.exchange import timeframe_to_minutes + timeframe_minutes = timeframe_to_minutes(timeframe) + # Resample to timeframe to make sure trades match candles + _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date' + )[['profit_abs']].sum() + df.loc[:, col_name] = _trades_sum['profit_abs'].cumsum() + # Set first value to 0 + df.loc[df.iloc[0].name, col_name] = 0 + # FFill to get continuous + df[col_name] = df[col_name].ffill() + return df + + +def _calc_drawdown_series(profit_results: pd.DataFrame, *, date_col: str, value_col: str + ) -> pd.DataFrame: + max_drawdown_df = pd.DataFrame() + max_drawdown_df['cumulative'] = profit_results[value_col].cumsum() + max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax() + max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value'] + max_drawdown_df['date'] = profit_results.loc[:, date_col] + return max_drawdown_df + + +def calculate_underwater(trades: pd.DataFrame, *, date_col: str = 'close_date', + value_col: str = 'profit_ratio' + ): + """ + Calculate max drawdown and the corresponding close dates + :param trades: DataFrame containing trades (requires columns close_date and profit_ratio) + :param date_col: Column in DataFrame to use for dates (defaults to 'close_date') + :param value_col: Column in DataFrame to use for values (defaults to 'profit_ratio') + :return: Tuple (float, highdate, lowdate, highvalue, lowvalue) with absolute max drawdown, + high and low time and high and low value. + :raise: ValueError if trade-dataframe was found empty. + """ + if len(trades) == 0: + raise ValueError("Trade dataframe empty.") + profit_results = trades.sort_values(date_col).reset_index(drop=True) + max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col) + + return max_drawdown_df + + +def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date', + value_col: str = 'profit_abs', starting_balance: float = 0 + ) -> Tuple[float, pd.Timestamp, pd.Timestamp, float, float, float]: + """ + Calculate max drawdown and the corresponding close dates + :param trades: DataFrame containing trades (requires columns close_date and profit_ratio) + :param date_col: Column in DataFrame to use for dates (defaults to 'close_date') + :param value_col: Column in DataFrame to use for values (defaults to 'profit_abs') + :param starting_balance: Portfolio starting balance - properly calculate relative drawdown. + :return: Tuple (float, highdate, lowdate, highvalue, lowvalue, relative_drawdown) + with absolute max drawdown, high and low time and high and low value, + and the relative account drawdown + :raise: ValueError if trade-dataframe was found empty. + """ + if len(trades) == 0: + raise ValueError("Trade dataframe empty.") + profit_results = trades.sort_values(date_col).reset_index(drop=True) + max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col) + + idxmin = max_drawdown_df['drawdown'].idxmin() + if idxmin == 0: + raise ValueError("No losing trade, therefore no drawdown.") + high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col] + low_date = profit_results.loc[idxmin, date_col] + high_val = max_drawdown_df.loc[max_drawdown_df.iloc[:idxmin] + ['high_value'].idxmax(), 'cumulative'] + low_val = max_drawdown_df.loc[idxmin, 'cumulative'] + max_drawdown_rel = 0.0 + if high_val + starting_balance != 0: + max_drawdown_rel = (high_val - low_val) / (high_val + starting_balance) + + return ( + abs(min(max_drawdown_df['drawdown'])), + high_date, + low_date, + high_val, + low_val, + max_drawdown_rel + ) + + +def calculate_csum(trades: pd.DataFrame, starting_balance: float = 0) -> Tuple[float, float]: + """ + Calculate min/max cumsum of trades, to show if the wallet/stake amount ratio is sane + :param trades: DataFrame containing trades (requires columns close_date and profit_percent) + :param starting_balance: Add starting balance to results, to show the wallets high / low points + :return: Tuple (float, float) with cumsum of profit_abs + :raise: ValueError if trade-dataframe was found empty. + """ + if len(trades) == 0: + raise ValueError("Trade dataframe empty.") + + csum_df = pd.DataFrame() + csum_df['sum'] = trades['profit_abs'].cumsum() + csum_min = csum_df['sum'].min() + starting_balance + csum_max = csum_df['sum'].max() + starting_balance + + return csum_min, csum_max + + +def calculate_cagr(days_passed: int, starting_balance: float, final_balance: float) -> float: + """ + Calculate CAGR + :param days_passed: Days passed between start and ending balance + :param starting_balance: Starting balance + :param final_balance: Final balance to calculate CAGR against + :return: CAGR + """ + return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1 diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py index 846dae9ea..ea6c151e5 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py @@ -10,7 +10,7 @@ from typing import Any, Dict from pandas import DataFrame -from freqtrade.data.btanalysis import calculate_max_drawdown +from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.optimize.hyperopt import IHyperOptLoss diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown.py index ce955d928..a8af704cd 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown.py @@ -8,7 +8,7 @@ from datetime import datetime from pandas import DataFrame -from freqtrade.data.btanalysis import calculate_max_drawdown +from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.optimize.hyperopt import IHyperOptLoss diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py index 5bd12ff52..ed689edba 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py @@ -9,7 +9,7 @@ individual needs. """ from pandas import DataFrame -from freqtrade.data.btanalysis import calculate_max_drawdown +from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.optimize.hyperopt import IHyperOptLoss diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 1d58dc339..9c1a276a9 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -9,8 +9,8 @@ from pandas import DataFrame, to_datetime from tabulate import tabulate from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT -from freqtrade.data.btanalysis import (calculate_cagr, calculate_csum, calculate_market_change, - calculate_max_drawdown) +from freqtrade.data.metrics import (calculate_cagr, calculate_csum, calculate_market_change, + calculate_max_drawdown) from freqtrade.misc import decimals_per_coin, file_dump_joblib, file_dump_json, round_coin_value from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 5337016f3..773577d7b 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -5,12 +5,13 @@ from typing import Any, Dict, List, Optional import pandas as pd from freqtrade.configuration import TimeRange -from freqtrade.data.btanalysis import (analyze_trade_parallelism, calculate_max_drawdown, - calculate_underwater, combine_dataframes_with_mean, - create_cum_profit, extract_trades_of_period, load_trades) +from freqtrade.data.btanalysis import (analyze_trade_parallelism, extract_trades_of_period, + load_trades) from freqtrade.data.converter import trim_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import get_timerange, load_data +from freqtrade.data.metrics import (calculate_max_drawdown, calculate_underwater, + combine_dataframes_with_mean, create_cum_profit) from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_prev_date, timeframe_to_seconds diff --git a/freqtrade/plugins/protections/max_drawdown_protection.py b/freqtrade/plugins/protections/max_drawdown_protection.py index b6ef92bd5..4111b7ff4 100644 --- a/freqtrade/plugins/protections/max_drawdown_protection.py +++ b/freqtrade/plugins/protections/max_drawdown_protection.py @@ -5,7 +5,7 @@ from typing import Any, Dict import pandas as pd -from freqtrade.data.btanalysis import calculate_max_drawdown +from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index eaf703b2d..f9f49e280 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -8,14 +8,14 @@ from pandas import DataFrame, DateOffset, Timestamp, to_datetime from freqtrade.configuration import TimeRange from freqtrade.constants import LAST_BT_RESULT_FN -from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelism, calculate_cagr, - calculate_csum, calculate_market_change, - calculate_max_drawdown, calculate_underwater, - combine_dataframes_with_mean, create_cum_profit, +from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelism, extract_trades_of_period, get_latest_backtest_filename, get_latest_hyperopt_file, load_backtest_data, load_backtest_metadata, load_trades, load_trades_from_db) from freqtrade.data.history import load_data, load_pair_history +from freqtrade.data.metrics import (calculate_cagr, calculate_csum, calculate_market_change, + calculate_max_drawdown, calculate_underwater, + combine_dataframes_with_mean, create_cum_profit) from freqtrade.exceptions import OperationalException from tests.conftest import CURRENT_TEST_STRATEGY, create_mock_trades from tests.conftest_trades import MOCK_TRADE_COUNT diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 97f367608..65df2d84c 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -10,7 +10,8 @@ from plotly.subplots import make_subplots from freqtrade.commands import start_plot_dataframe, start_plot_profit from freqtrade.configuration import TimeRange from freqtrade.data import history -from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data +from freqtrade.data.btanalysis import load_backtest_data +from freqtrade.data.metrics import create_cum_profit from freqtrade.exceptions import OperationalException from freqtrade.plot.plotting import (add_areas, add_indicators, add_profit, create_plotconfig, generate_candlestick_graph, generate_plot_filename, From 4580127fa89f27d432177975b10e1b616842d26e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 Apr 2022 14:51:57 +0200 Subject: [PATCH 59/63] Small refactor --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 7456c40cf..dd8d82b13 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1038,7 +1038,7 @@ class Backtesting: timerange: TimeRange): self.progress.init_step(BacktestState.ANALYZE, 0) - logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) + logger.info(f"Running backtesting for Strategy {strat.get_strategy_name()}") backtest_start_time = datetime.now(timezone.utc) self._set_strategy(strat) From e4df2b0b966961e188664ed92030c9cb6e454f1f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 Apr 2022 14:55:52 +0200 Subject: [PATCH 60/63] Revert unwanted changes --- .github/PULL_REQUEST_TEMPLATE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 25a9761e2..90a10d4da 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -9,8 +9,8 @@ Solve the issue: #___ ## Quick changelog -- -- +- +- ## What's new? From 5c1ac3cf9503ca86d7e52f3f5f199fb356a647fc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 Apr 2022 19:55:13 +0200 Subject: [PATCH 61/63] Fix caching bug with freqUI backtesting --- freqtrade/rpc/api_server/api_backtest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index a902ea984..41712632b 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -84,6 +84,7 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac lastconfig['enable_protections'] = btconfig.get('enable_protections') lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet') + ApiServer._bt.strategylist = [strat] ApiServer._bt.results = {} ApiServer._bt.load_prior_backtest() From 0c921e01161a72c318f1e8a4b0049a2f77f08be0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 Apr 2022 20:08:40 +0200 Subject: [PATCH 62/63] Reorder api_backtesting test sequence --- tests/rpc/test_rpc_apiserver.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 43f783a53..ac2f1c3ec 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1483,7 +1483,7 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir): assert not result['running'] assert result['status_msg'] == 'Backtest reset' ftbot.config['export'] = 'trades' - ftbot.config['backtest_cache'] = 'none' + ftbot.config['backtest_cache'] = 'day' ftbot.config['user_data_dir'] = Path(tmpdir) ftbot.config['exportfilename'] = Path(tmpdir) / "backtest_results" ftbot.config['exportfilename'].mkdir() @@ -1556,19 +1556,19 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir): ApiServer._bgtask_running = False - mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest_one_strategy', - side_effect=DependencyException()) - rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data)) - assert log_has("Backtesting caused an error: ", caplog) - - ftbot.config['backtest_cache'] = 'day' - # Rerun backtest (should get previous result) rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data)) assert_response(rc) result = rc.json() assert log_has_re('Reusing result of previous backtest.*', caplog) + data['stake_amount'] = 101 + + mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest_one_strategy', + side_effect=DependencyException()) + rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data)) + assert log_has("Backtesting caused an error: ", caplog) + # Delete backtesting to avoid leakage since the backtest-object may stick around. rc = client_delete(client, f"{BASE_URI}/backtest") assert_response(rc) From d5fc923dcbe02daae51006494be03be6d68a874c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 1 May 2022 09:53:34 +0200 Subject: [PATCH 63/63] Properly validate stoploss existence for optimize commands closes #6740 --- freqtrade/configuration/config_setup.py | 2 +- freqtrade/configuration/config_validation.py | 11 +++++++---- freqtrade/constants.py | 4 ++++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/freqtrade/configuration/config_setup.py b/freqtrade/configuration/config_setup.py index 02f2d4089..d49bf61f6 100644 --- a/freqtrade/configuration/config_setup.py +++ b/freqtrade/configuration/config_setup.py @@ -22,6 +22,6 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str # Ensure these modes are using Dry-run config['dry_run'] = True - validate_config_consistency(config) + validate_config_consistency(config, preliminary=True) return config diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 6e4a4b0ef..ee846e7e6 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -39,7 +39,7 @@ def _extend_validator(validator_class): FreqtradeValidator = _extend_validator(Draft4Validator) -def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]: +def validate_config_schema(conf: Dict[str, Any], preliminary: bool = False) -> Dict[str, Any]: """ Validate the configuration follow the Config Schema :param conf: Config in JSON format @@ -49,7 +49,10 @@ def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]: if conf.get('runmode', RunMode.OTHER) in (RunMode.DRY_RUN, RunMode.LIVE): conf_schema['required'] = constants.SCHEMA_TRADE_REQUIRED elif conf.get('runmode', RunMode.OTHER) in (RunMode.BACKTEST, RunMode.HYPEROPT): - conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED + if preliminary: + conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED + else: + conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED_FINAL else: conf_schema['required'] = constants.SCHEMA_MINIMAL_REQUIRED try: @@ -64,7 +67,7 @@ def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]: ) -def validate_config_consistency(conf: Dict[str, Any]) -> None: +def validate_config_consistency(conf: Dict[str, Any], preliminary: bool = False) -> None: """ Validate the configuration consistency. Should be ran after loading both configuration and strategy, @@ -85,7 +88,7 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None: # validate configuration before returning logger.info('Validating configuration ...') - validate_config_schema(conf) + validate_config_schema(conf, preliminary=preliminary) def _validate_unlimited_amount(conf: Dict[str, Any]) -> None: diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 1a21ec77f..0ceabe917 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -462,6 +462,10 @@ SCHEMA_BACKTEST_REQUIRED = [ 'dataformat_ohlcv', 'dataformat_trades', ] +SCHEMA_BACKTEST_REQUIRED_FINAL = SCHEMA_BACKTEST_REQUIRED + [ + 'stoploss', + 'minimal_roi', +] SCHEMA_MINIMAL_REQUIRED = [ 'exchange',