From cd6620a044d9ebd53584febc9ef7500e6d2dd820 Mon Sep 17 00:00:00 2001 From: Bernd Zeimetz Date: Fri, 11 Jun 2021 15:01:13 +0200 Subject: [PATCH 01/14] Ignore broken symlinks while resolving strategies. Without this fix the resolver tries to read from the broken symlink, resulting in an exception that leads to the the rather confusing error message freqtrade.resolvers.iresolver - WARNING - Path "...../user_data/strategies" does not exist. as a result of a symlink matching .py not being readable. --- freqtrade/resolvers/iresolver.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index b51795e9e..5172e6fda 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -91,6 +91,9 @@ class IResolver: if not str(entry).endswith('.py'): logger.debug('Ignoring %s', entry) continue + if entry.is_symlink() and not entry.is_file(): + logger.debug('Ignoring broken symlink %s', entry) + continue module_path = entry.resolve() obj = next(cls._get_valid_object(module_path, object_name), None) From 6dc4259c6e6e664cf3bdb9f14b37447a1522152f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 05:18:54 +0000 Subject: [PATCH 02/14] Bump mkdocs-material from 7.1.7 to 7.1.8 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 7.1.7 to 7.1.8. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/docs/changelog.md) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/7.1.7...7.1.8) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 776d5592a..9b27d974b 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ mkdocs==1.2 -mkdocs-material==7.1.7 +mkdocs-material==7.1.8 mdx_truly_sane_lists==1.2 pymdown-extensions==8.2 From 4530ae28cd0786611926ceb036d002f3956cdfa6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 05:19:16 +0000 Subject: [PATCH 03/14] Bump sqlalchemy from 1.4.17 to 1.4.18 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 1.4.17 to 1.4.18. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/master/CHANGES) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index abfae55b9..fddf2cffe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ ccxt==1.51.3 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.7 aiohttp==3.7.4.post0 -SQLAlchemy==1.4.17 +SQLAlchemy==1.4.18 python-telegram-bot==13.6 arrow==1.1.0 cachetools==4.2.2 From 3f1d6d453cbde0cceab5b1b2239bce9b5f6b1635 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 05:19:23 +0000 Subject: [PATCH 04/14] Bump mypy from 0.812 to 0.902 Bumps [mypy](https://github.com/python/mypy) from 0.812 to 0.902. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.812...v0.902) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6fbe581a5..b0f970224 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ coveralls==3.1.0 flake8==3.9.2 flake8-type-annotations==0.1.0 flake8-tidy-imports==4.3.0 -mypy==0.812 +mypy==0.902 pytest==6.2.4 pytest-asyncio==0.15.1 pytest-cov==2.12.1 From fe933e78bde88a6e3424aeafd397fbc311770b36 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 05:19:33 +0000 Subject: [PATCH 05/14] Bump ccxt from 1.51.3 to 1.51.40 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.51.3 to 1.51.40. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/doc/exchanges-by-country.rst) - [Commits](https://github.com/ccxt/ccxt/compare/1.51.3...1.51.40) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index abfae55b9..781425ec5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.20.3 pandas==1.2.4 -ccxt==1.51.3 +ccxt==1.51.40 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.7 aiohttp==3.7.4.post0 From 63802aa7f66487d8a83342e757eaf69da89bc207 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 08:52:42 +0000 Subject: [PATCH 06/14] Bump mkdocs from 1.2 to 1.2.1 Bumps [mkdocs](https://github.com/mkdocs/mkdocs) from 1.2 to 1.2.1. - [Release notes](https://github.com/mkdocs/mkdocs/releases) - [Commits](https://github.com/mkdocs/mkdocs/compare/1.2...1.2.1) --- updated-dependencies: - dependency-name: mkdocs dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 9b27d974b..352103f8b 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ -mkdocs==1.2 +mkdocs==1.2.1 mkdocs-material==7.1.8 mdx_truly_sane_lists==1.2 pymdown-extensions==8.2 From 4ba7a2bbd290c2c6e5746995ceacd89660a04ab5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 14 Jun 2021 19:18:42 +0200 Subject: [PATCH 07/14] Fix mypy update problems --- freqtrade/resolvers/iresolver.py | 3 +++ freqtrade/rpc/webhook.py | 15 +++++++-------- requirements-dev.txt | 6 ++++++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 5172e6fda..5b6977b4b 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -58,6 +58,9 @@ class IResolver: # Generate spec based on absolute path # Pass object_name as first argument to have logging print a reasonable name. spec = importlib.util.spec_from_file_location(object_name or "", str(module_path)) + if not spec: + return iter([None]) + module = importlib.util.module_from_spec(spec) try: spec.loader.exec_module(module) # type: ignore # importlib does not use typehints diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 0e4a4bf6f..b4c55649e 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -77,14 +77,13 @@ class Webhook(RPCHandler): def _send_msg(self, payload: dict) -> None: """do the actual call to the webhook""" - if self._format == 'form': - kwargs = {'data': payload} - elif self._format == 'json': - kwargs = {'json': payload} - else: - raise NotImplementedError('Unknown format: {}'.format(self._format)) - try: - post(self._url, **kwargs) + if self._format == 'form': + post(self._url, data=payload) + elif self._format == 'json': + post(self._url, json=payload) + else: + raise NotImplementedError('Unknown format: {}'.format(self._format)) + except RequestException as exc: logger.warning("Could not call webhook url. Exception: %s", exc) diff --git a/requirements-dev.txt b/requirements-dev.txt index b0f970224..924b35e1a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -17,3 +17,9 @@ isort==5.8.0 # Convert jupyter notebooks to markdown documents nbconvert==6.0.7 + +# mypy types +types-cachetools==0.1.7 +types-filelock==0.1.3 +types-requests==0.1.11 +types-tabulate==0.1.0 From cf7394d01cb6798213bb4b08572885328756adda Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 14 Jun 2021 19:57:24 +0200 Subject: [PATCH 08/14] Export backtesting results by default closes #4977 --- docs/backtesting.md | 21 +++++++++++++-------- freqtrade/commands/cli_options.py | 5 +++-- freqtrade/constants.py | 2 ++ freqtrade/optimize/backtesting.py | 2 +- tests/conftest.py | 1 + tests/optimize/test_backtesting.py | 12 ++++++------ tests/test_configuration.py | 5 ++--- 7 files changed, 28 insertions(+), 20 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 2027c2079..26642ef8c 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -19,7 +19,7 @@ usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--enable-protections] [--dry-run-wallet DRY_RUN_WALLET] [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] - [--export EXPORT] [--export-filename PATH] + [--export {none,trades}] [--export-filename PATH] optional arguments: -h, --help show this help message and exit @@ -63,8 +63,8 @@ optional arguments: name is injected into the filename (so `backtest- data.json` becomes `backtest-data- DefaultStrategy.json` - --export EXPORT Export backtest results, argument are: trades. - Example: `--export=trades` + --export {none,trades} + Export backtest results (default: trades). --export-filename PATH Save backtest results to the file with this filename. Requires `--export` to be set as well. Example: @@ -100,7 +100,7 @@ Strategy arguments: Now you have good Buy and Sell strategies and some historic data, you want to test it against real data. This is what we call [backtesting](https://en.wikipedia.org/wiki/Backtesting). -Backtesting will use the crypto-currencies (pairs) from your config file and load historical candle (OHCLV) data from `user_data/data/` by default. +Backtesting will use the crypto-currencies (pairs) from your config file and load historical candle (OHLCV) data from `user_data/data/` by default. If no data is available for the exchange / pair / timeframe combination, backtesting will ask you to download them first using `freqtrade download-data`. For details on downloading, please refer to the [Data Downloading](data-download.md) section in the documentation. @@ -110,11 +110,16 @@ All profit calculations include fees, and freqtrade will use the exchange's defa !!! Warning "Using dynamic pairlists for backtesting" Using dynamic pairlists is possible, however it relies on the current market conditions - which will not reflect the historic status of the pairlist. - Also, when using pairlists other than StaticPairlist, reproducability of backtesting-results cannot be guaranteed. + Also, when using pairlists other than StaticPairlist, reproducibility of backtesting-results cannot be guaranteed. Please read the [pairlists documentation](plugins.md#pairlists) for more information. To achieve reproducible results, best generate a pairlist via the [`test-pairlist`](utils.md#test-pairlist) command and use that as static pairlist. +!!! Note + By default, Freqtrade will export backtesting results to `user_data/backtest_results`. + The exported trades can be used for [further analysis](#further-backtest-result-analysis) or can be used by the [plotting sub-command](plotting.md#plot-price-and-indicators) (`freqtrade plot-dataframe`) in the scripts directory. + + ### Starting balance Backtesting will require a starting balance, which can be provided as `--dry-run-wallet ` or `--starting-balance ` command line argument, or via `dry_run_wallet` configuration setting. @@ -174,13 +179,13 @@ Where `SampleStrategy1` and `AwesomeStrategy` refer to class names of strategies --- -Exporting trades to file +Prevent exporting trades to file ```bash -freqtrade backtesting --strategy backtesting --export trades --config config.json +freqtrade backtesting --strategy backtesting --export none --config config.json ``` -The exported trades can be used for [further analysis](#further-backtest-result-analysis), or can be used by the plotting script `plot_dataframe.py` in the scripts directory. +Only use this if you're sure you'll not want to plot or analyze your results further. --- diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index d832693ee..b226415e7 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -167,8 +167,9 @@ AVAILABLE_CLI_OPTIONS = { ), "export": Arg( '--export', - help='Export backtest results, argument are: trades. ' - 'Example: `--export=trades`', + help='Export backtest results (default: trades).', + choices=constants.EXPORT_OPTIONS, + ), "exportfilename": Arg( '--export-filename', diff --git a/freqtrade/constants.py b/freqtrade/constants.py index e42b9d4b8..259aa0e03 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -12,6 +12,7 @@ PROCESS_THROTTLE_SECS = 5 # sec HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec TIMEOUT_UNITS = ['minutes', 'seconds'] +EXPORT_OPTIONS = ['none', 'trades'] DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite' UNLIMITED_STAKE_AMOUNT = 'unlimited' @@ -308,6 +309,7 @@ CONF_SCHEMA = { 'required': ['enabled', 'listen_ip_address', 'listen_port', 'username', 'password'] }, 'db_url': {'type': 'string'}, + 'export': {'type': 'string', 'enum': EXPORT_OPTIONS, 'default': 'trades'}, 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, 'forcebuy_enable': {'type': 'boolean'}, 'disable_dataframe_checks': {'type': 'boolean'}, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 922f89c22..c72a8b5c5 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -520,7 +520,7 @@ class Backtesting: stats = generate_backtest_stats(data, self.all_results, min_date=min_date, max_date=max_date) - if self.config.get('export', False): + if self.config.get('export', 'none') == 'trades': store_backtest_stats(self.config['exportfilename'], stats) # Show backtest results diff --git a/tests/conftest.py b/tests/conftest.py index dd38ca610..c6a0dfcfd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -326,6 +326,7 @@ def get_default_conf(testdatadir): "strategy_path": str(Path(__file__).parent / "strategy" / "strats"), "strategy": "DefaultStrategy", "internals": {}, + "export": "none", } return configuration diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 7387c8865..60bd82d71 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -155,6 +155,7 @@ def test_setup_optimize_configuration_without_arguments(mocker, default_conf, ca 'backtesting', '--config', 'config.json', '--strategy', 'DefaultStrategy', + '--export', 'none' ] config = setup_optimize_configuration(get_args(args), RunMode.BACKTEST) @@ -172,7 +173,8 @@ def test_setup_optimize_configuration_without_arguments(mocker, default_conf, ca assert not log_has('Parameter --enable-position-stacking detected ...', caplog) assert 'timerange' not in config - assert 'export' not in config + assert 'export' in config + assert config['export'] == 'none' assert 'runmode' in config assert config['runmode'] == RunMode.BACKTEST @@ -193,7 +195,6 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> '--enable-position-stacking', '--disable-max-market-positions', '--timerange', ':100', - '--export', '/bar/foo', '--export-filename', 'foo_bar.json', '--fee', '0', ] @@ -223,7 +224,6 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog) assert 'export' in config - assert log_has('Parameter --export detected: {} ...'.format(config['export']), caplog) assert 'exportfilename' in config assert isinstance(config['exportfilename'], Path) assert log_has('Storing backtest results to {} ...'.format(config['exportfilename']), caplog) @@ -395,7 +395,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> default_conf['timeframe'] = "1m" default_conf['datadir'] = testdatadir - default_conf['export'] = None + default_conf['export'] = 'none' default_conf['timerange'] = '20180101-20180102' backtesting = Backtesting(default_conf) @@ -416,7 +416,7 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> default_conf['timeframe'] = "1m" default_conf['datadir'] = testdatadir - default_conf['export'] = None + default_conf['export'] = 'none' default_conf['timerange'] = '20180101-20180102' with pytest.raises(OperationalException, match='No pair in whitelist.'): @@ -440,7 +440,7 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti default_conf['ticker_interval'] = "1m" default_conf['datadir'] = testdatadir - default_conf['export'] = None + default_conf['export'] = 'none' # Use stoploss from strategy del default_conf['stoploss'] default_conf['timerange'] = '20180101-20180102' diff --git a/tests/test_configuration.py b/tests/test_configuration.py index aa121edfa..c5d0cd908 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -425,7 +425,6 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert not log_has('Parameter --enable-position-stacking detected ...', caplog) assert 'timerange' not in config - assert 'export' not in config def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> None: @@ -448,7 +447,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non '--enable-position-stacking', '--disable-max-market-positions', '--timerange', ':100', - '--export', '/bar/foo', + '--export', 'trades', '--stake-amount', 'unlimited' ] @@ -496,7 +495,7 @@ def test_setup_configuration_with_stratlist(mocker, default_conf, caplog) -> Non 'backtesting', '--config', 'config.json', '--ticker-interval', '1m', - '--export', '/bar/foo', + '--export', 'trades', '--strategy-list', 'DefaultStrategy', 'TestStrategy' From 6d5fc967147e0aae37423d1e0ee57de47a17a791 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sat, 12 Jun 2021 10:16:30 +0300 Subject: [PATCH 09/14] Implement most pessimistic handling of trailing stoploss. --- freqtrade/optimize/backtesting.py | 16 +++++++++ freqtrade/strategy/interface.py | 4 +-- tests/optimize/test_backtest_detail.py | 47 ++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index c72a8b5c5..19ae74ae6 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -225,6 +225,22 @@ class Backtesting: # sell at open price. return sell_row[OPEN_IDX] + # Special case: trailing triggers within same candle as trade opened. Assume most + # pessimistic price movement, which is moving just enough to arm stoploss and + # immediately going down to stop price. + if sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0 and \ + self.strategy.trailing_stop_positive: + if self.strategy.trailing_only_offset_is_reached: + # Worst case: price reaches stop_positive_offset and dives down. + stop_rate = sell_row[OPEN_IDX] * \ + (1 + abs(self.strategy.trailing_stop_positive_offset) - + abs(self.strategy.trailing_stop_positive)) + else: + # Worst case: price ticks tiny bit above open and dives down. + stop_rate = sell_row[OPEN_IDX] * (1 - abs(self.strategy.trailing_stop_positive)) + assert stop_rate < sell_row[HIGH_IDX] + return stop_rate + # Set close_rate to stoploss return trade.stop_loss elif sell.sell_type == (SellType.ROI): diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 8ea38f503..47d4259fc 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -610,7 +610,7 @@ class IStrategy(ABC, HyperStrategyMixin): # Initiate stoploss with open_rate. Does nothing if stoploss is already set. trade.adjust_stop_loss(trade.open_rate, stop_loss_value, initial=True) - if self.use_custom_stoploss: + if self.use_custom_stoploss and trade.stop_loss < current_rate: stop_loss_value = strategy_safe_wrapper(self.custom_stoploss, default_retval=None )(pair=trade.pair, trade=trade, current_time=current_time, @@ -623,7 +623,7 @@ class IStrategy(ABC, HyperStrategyMixin): else: logger.warning("CustomStoploss function did not return valid stoploss") - if self.trailing_stop: + if self.trailing_stop and trade.stop_loss < current_rate: # trailing stoploss handling sl_offset = self.trailing_stop_positive_offset diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index e5b969383..488425323 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -457,6 +457,50 @@ tc28 = BTContainer(data=[ trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=3)] ) +# Test 29: trailing_stop should be triggered by low of next candle, without adjusting stoploss using +# high of stoploss candle. +# stop-loss: 10%, ROI: 10% (should not apply) +tc29 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5050, 5000, 4900, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4900, 5250, 4500, 5100, 6172, 0, 0], # Triggers trailing-stoploss + [3, 5100, 5100, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.10}, profit_perc=-0.02, trailing_stop=True, + trailing_stop_positive=0.03, + trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=2)] +) + +# Test 30: trailing_stop should be triggered immediately on trade open candle. +# stop-loss: 10%, ROI: 10% (should not apply) +tc30 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5500, 5000, 4900, 6172, 0, 0], # enter trade (signal on last candle) and stop + [2, 4900, 5250, 4500, 5100, 6172, 0, 0], + [3, 5100, 5100, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.10}, profit_perc=-0.01, trailing_stop=True, + trailing_stop_positive=0.01, + trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=1)] +) + +# Test 31: trailing_stop should be triggered immediately on trade open candle. +# stop-loss: 10%, ROI: 10% (should not apply) +tc31 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5500, 5000, 4900, 6172, 0, 0], # enter trade (signal on last candle) and stop + [2, 4900, 5250, 4500, 5100, 6172, 0, 0], + [3, 5100, 5100, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.10}, profit_perc=0.01, trailing_stop=True, + trailing_only_offset_is_reached=True, trailing_stop_positive_offset=0.02, + trailing_stop_positive=0.01, + trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=1)] +) + TESTS = [ tc0, tc1, @@ -487,6 +531,9 @@ TESTS = [ tc26, tc27, tc28, + tc29, + tc30, + tc31, ] From 38ed49cef54d0bfe4606be46e2ec88c344e768b3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 13 Jun 2021 16:37:11 +0200 Subject: [PATCH 10/14] move low to stoploss_reached to clarify where which rate is used --- freqtrade/strategy/interface.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 47d4259fc..6358c6a4e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -524,15 +524,14 @@ class IStrategy(ABC, HyperStrategyMixin): :param force_stoploss: Externally provided stoploss :return: True if trade should be sold, False otherwise """ - # Set current rate to low for backtesting sell - current_rate = low or rate + current_rate = rate current_profit = trade.calc_profit_ratio(current_rate) trade.adjust_min_max_rates(high or current_rate) stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade, current_time=date, current_profit=current_profit, - force_stoploss=force_stoploss, high=high) + force_stoploss=force_stoploss, low=low, high=high) # Set current rate to high for backtesting sell current_rate = high or rate @@ -599,18 +598,21 @@ class IStrategy(ABC, HyperStrategyMixin): def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime, current_profit: float, - force_stoploss: float, high: float = None) -> SellCheckTuple: + force_stoploss: float, low: float = None, + high: float = None) -> SellCheckTuple: """ Based on current profit of the trade and configured (trailing) stoploss, decides to sell or not :param current_profit: current profit as ratio + :param low: Low value of this candle, only set in backtesting + :param high: High value of this candle, only set in backtesting """ stop_loss_value = force_stoploss if force_stoploss else self.stoploss # Initiate stoploss with open_rate. Does nothing if stoploss is already set. trade.adjust_stop_loss(trade.open_rate, stop_loss_value, initial=True) - if self.use_custom_stoploss and trade.stop_loss < current_rate: + if self.use_custom_stoploss and trade.stop_loss < (low or current_rate): stop_loss_value = strategy_safe_wrapper(self.custom_stoploss, default_retval=None )(pair=trade.pair, trade=trade, current_time=current_time, @@ -623,7 +625,7 @@ class IStrategy(ABC, HyperStrategyMixin): else: logger.warning("CustomStoploss function did not return valid stoploss") - if self.trailing_stop and trade.stop_loss < current_rate: + if self.trailing_stop and trade.stop_loss < (low or current_rate): # trailing stoploss handling sl_offset = self.trailing_stop_positive_offset @@ -643,7 +645,7 @@ class IStrategy(ABC, HyperStrategyMixin): # evaluate if the stoploss was hit if stoploss is not on exchange # in Dry-Run, this handles stoploss logic as well, as the logic will not be different to # regular stoploss handling. - if ((trade.stop_loss >= current_rate) and + if ((trade.stop_loss >= (low or current_rate)) and (not self.order_types.get('stoploss_on_exchange') or self.config['dry_run'])): sell_type = SellType.STOP_LOSS @@ -652,7 +654,7 @@ class IStrategy(ABC, HyperStrategyMixin): if trade.initial_stop_loss != trade.stop_loss: sell_type = SellType.TRAILING_STOP_LOSS logger.debug( - f"{trade.pair} - HIT STOP: current price at {current_rate:.6f}, " + f"{trade.pair} - HIT STOP: current price at {(low or current_rate):.6f}, " f"stoploss is {trade.stop_loss:.6f}, " f"initial stoploss was at {trade.initial_stop_loss:.6f}, " f"trade opened at {trade.open_rate:.6f}") From 1bb04bb0c24c8a959df1fbc77617f5e1c1752d29 Mon Sep 17 00:00:00 2001 From: barbarius Date: Wed, 16 Jun 2021 11:40:55 +0200 Subject: [PATCH 11/14] Moved daily avg trade row next to total trades on backtest results --- docs/backtesting.md | 8 +++----- freqtrade/optimize/optimize_reports.py | 4 ++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 26642ef8c..8e50aa356 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -284,7 +284,7 @@ A backtesting result will look like that: | Backtesting to | 2019-05-01 00:00:00 | | Max open trades | 3 | | | | -| Total trades | 429 | +| Total/Daily Avg Trades| 429 / 3.575 | | Starting balance | 0.01000000 BTC | | Final balance | 0.01762792 BTC | | Absolute profit | 0.00762792 BTC | @@ -373,12 +373,11 @@ It contains some useful key metrics about performance of your strategy on backte | Backtesting to | 2019-05-01 00:00:00 | | Max open trades | 3 | | | | -| Total trades | 429 | +| Total/Daily Avg Trades| 429 / 3.575 | | Starting balance | 0.01000000 BTC | | Final balance | 0.01762792 BTC | | Absolute profit | 0.00762792 BTC | | Total profit % | 76.2% | -| Trades per day | 3.575 | | Avg. stake amount | 0.001 BTC | | Total trade volume | 0.429 BTC | | | | @@ -409,12 +408,11 @@ It contains some useful key metrics about performance of your strategy on backte - `Backtesting from` / `Backtesting to`: Backtesting range (usually defined with the `--timerange` option). - `Max open trades`: Setting of `max_open_trades` (or `--max-open-trades`) - or number of pairs in the pairlist (whatever is lower). -- `Total trades`: Identical to the total trades of the backtest output table. +- `Total/Daily Avg Trades`: Identical to the total trades of the backtest output table / Total trades divided by the backtesting duration in days (this will give you information about how many trades to expect from the strategy). - `Starting balance`: Start balance - as given by dry-run-wallet (config or command line). - `Final balance`: Final balance - starting balance + absolute profit. - `Absolute profit`: Profit made in stake currency. - `Total profit %`: Total profit. Aligned to the `TOTAL` row's `Tot Profit %` from the first table. Calculated as `(End capital − Starting capital) / Starting capital`. -- `Trades per day`: Total trades divided by the backtesting duration in days (this will give you information about how many trades to expect from the strategy). - `Avg. stake amount`: Average stake amount, either `stake_amount` or the average when using dynamic stake amount. - `Total trade volume`: Volume generated on the exchange to reach the above profit. - `Best Pair` / `Worst Pair`: Best and worst performing pair, and it's corresponding `Cum Profit %`. diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 84e052ac4..64b043304 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -556,7 +556,8 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Backtesting to', strat_results['backtest_end']), ('Max open trades', strat_results['max_open_trades']), ('', ''), # Empty line to improve readability - ('Total trades', strat_results['total_trades']), + ('Total/Daily Avg Trades', + f"{strat_results['total_trades']} / {strat_results['trades_per_day']}"), ('Starting balance', round_coin_value(strat_results['starting_balance'], strat_results['stake_currency'])), ('Final balance', round_coin_value(strat_results['final_balance'], @@ -564,7 +565,6 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Absolute profit ', round_coin_value(strat_results['profit_total_abs'], strat_results['stake_currency'])), ('Total profit %', f"{round(strat_results['profit_total'] * 100, 2):}%"), - ('Trades per day', strat_results['trades_per_day']), ('Avg. stake amount', round_coin_value(strat_results['avg_stake_amount'], strat_results['stake_currency'])), ('Total trade volume', round_coin_value(strat_results['total_volume'], From 1c9def2fdbfad2e11a283b900cde6e8968145930 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 16 Jun 2021 20:17:44 +0100 Subject: [PATCH 12/14] Update freqtrade/optimize/optimize_reports.py --- freqtrade/optimize/optimize_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 64b043304..df7f721ec 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -556,7 +556,7 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Backtesting to', strat_results['backtest_end']), ('Max open trades', strat_results['max_open_trades']), ('', ''), # Empty line to improve readability - ('Total/Daily Avg Trades', + ('Total/Daily Avg Trades', f"{strat_results['total_trades']} / {strat_results['trades_per_day']}"), ('Starting balance', round_coin_value(strat_results['starting_balance'], strat_results['stake_currency'])), From b38ab84a13d23906b05b236f6eb9421088b9c09a Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 17 Jun 2021 06:48:41 +0200 Subject: [PATCH 13/14] Add documentation mention about new behaviour --- docs/backtesting.md | 1 + freqtrade/optimize/backtesting.py | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 26642ef8c..d34381f55 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -446,6 +446,7 @@ Since backtesting lacks some detailed information about what happens within a ca - Stoploss is evaluated before ROI within one candle. So you can often see more trades with the `stoploss` sell reason comparing to the results obtained with the same strategy in the Dry Run/Live Trade modes - Low happens before high for stoploss, protecting capital first - Trailing stoploss + - Trailing Stoploss is only adjusted if it's below the candle's low (otherwise it would be triggered) - High happens first - adjusting stoploss - Low uses the adjusted stoploss (so sells with large high-low difference are backtested correctly) - ROI applies before trailing-stop, ensuring profits are "top-capped" at ROI if both ROI and trailing stop applies diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 19ae74ae6..028a9eacd 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -228,13 +228,13 @@ class Backtesting: # Special case: trailing triggers within same candle as trade opened. Assume most # pessimistic price movement, which is moving just enough to arm stoploss and # immediately going down to stop price. - if sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0 and \ - self.strategy.trailing_stop_positive: + if (sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0 + and self.strategy.trailing_stop_positive): if self.strategy.trailing_only_offset_is_reached: # Worst case: price reaches stop_positive_offset and dives down. - stop_rate = sell_row[OPEN_IDX] * \ - (1 + abs(self.strategy.trailing_stop_positive_offset) - - abs(self.strategy.trailing_stop_positive)) + stop_rate = (sell_row[OPEN_IDX] * + (1 + abs(self.strategy.trailing_stop_positive_offset) - + abs(self.strategy.trailing_stop_positive))) else: # Worst case: price ticks tiny bit above open and dives down. stop_rate = sell_row[OPEN_IDX] * (1 - abs(self.strategy.trailing_stop_positive)) From a49ca9cbf78b1a0e458c3e4767344d2c71a66219 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 17 Jun 2021 06:57:35 +0200 Subject: [PATCH 14/14] Change log-level "Executing handler" msg to debug closes #5143 --- freqtrade/rpc/telegram.py | 2 +- tests/rpc/test_rpc_telegram.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c3ddfd644..aee513017 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -56,7 +56,7 @@ def authorized_only(command_handler: Callable[..., None]) -> Callable[..., Any]: ) return wrapper - logger.info( + logger.debug( 'Executing handler: %s for chat_id: %s', command_handler.__name__, chat_id diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index bbda55c3e..d091f3837 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -2,6 +2,7 @@ # pragma pylint: disable=protected-access, unused-argument, invalid-name # pragma pylint: disable=too-many-lines, too-many-arguments +import logging import re from datetime import datetime from functools import reduce @@ -120,7 +121,7 @@ def test_cleanup(default_conf, mocker, ) -> None: def test_authorized_only(default_conf, mocker, caplog, update) -> None: patch_exchange(mocker) - + caplog.set_level(logging.DEBUG) default_conf['telegram']['enabled'] = False bot = FreqtradeBot(default_conf) rpc = RPC(bot) @@ -136,6 +137,7 @@ def test_authorized_only(default_conf, mocker, caplog, update) -> None: def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: patch_exchange(mocker) + caplog.set_level(logging.DEBUG) chat = Chat(0xdeadbeef, 0) update = Update(randint(1, 100)) update.message = Message(randint(1, 100), datetime.utcnow(), chat)