From eac98dbbd69954573bea3671aebc91938639f2f6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 27 Jan 2021 07:29:40 +0100 Subject: [PATCH 001/834] Version bump to 2021.1 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index e96e7f530..74c8c412c 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = 'develop' +__version__ = '2021.1' if __version__ == 'develop': From 69d62ef38316df7aa33683badbcef6590253ac76 Mon Sep 17 00:00:00 2001 From: Eko Aprili Trisno Date: Thu, 4 Feb 2021 01:06:52 +0700 Subject: [PATCH 002/834] Add Refresh / Reload Button on rpc/Telegram --- freqtrade/rpc/telegram.py | 99 +++++++++++++++++++++++++++++++++------ 1 file changed, 84 insertions(+), 15 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 99f9a8a91..ad72e10e4 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -11,9 +11,9 @@ from typing import Any, Callable, Dict, List, Union import arrow from tabulate import tabulate -from telegram import KeyboardButton, ParseMode, ReplyKeyboardMarkup, Update +from telegram import KeyboardButton, ParseMode, ReplyKeyboardMarkup, Update, InlineKeyboardButton, InlineKeyboardMarkup from telegram.error import NetworkError, TelegramError -from telegram.ext import CallbackContext, CommandHandler, Updater +from telegram.ext import CallbackContext, CommandHandler, Updater, CallbackQueryHandler from telegram.utils.helpers import escape_markdown from freqtrade.__init__ import __version__ @@ -40,9 +40,13 @@ def authorized_only(command_handler: Callable[..., None]) -> Callable[..., Any]: update = kwargs.get('update') or args[0] # Reject unauthorized messages + if update.callback_query: + cchat_id = int(update.callback_query.message.chat.id) + else: + cchat_id = int(update.message.chat_id) + chat_id = int(self._config['telegram']['chat_id']) - - if int(update.message.chat_id) != chat_id: + if cchat_id != chat_id: logger.info( 'Rejected unauthorized message from: %s', update.message.chat_id @@ -150,10 +154,22 @@ class Telegram(RPCHandler): CommandHandler('logs', self._logs), CommandHandler('edge', self._edge), CommandHandler('help', self._help), - CommandHandler('version', self._version), + CommandHandler('version', self._version) + ] + callbacks = [ + CallbackQueryHandler(self._status_table, pattern='update_status_table'), + CallbackQueryHandler(self._daily, pattern='update_daily'), + CallbackQueryHandler(self._profit, pattern='update_profit'), + CallbackQueryHandler(self._profit, pattern='update_balance'), + CallbackQueryHandler(self._profit, pattern='update_performance'), + CallbackQueryHandler(self._profit, pattern='update_count') ] for handle in handles: self._updater.dispatcher.add_handler(handle) + + for handle in callbacks: + self._updater.dispatcher.add_handler(handle) + self._updater.start_polling( clean=True, bootstrap_retries=-1, @@ -336,9 +352,12 @@ class Telegram(RPCHandler): try: statlist, head = self._rpc._rpc_status_table( self._config['stake_currency'], self._config.get('fiat_display_currency', '')) - message = tabulate(statlist, headers=head, tablefmt='simple') - self._send_msg(f"
{message}
", parse_mode=ParseMode.HTML) + if(update.callback_query): + query = update.callback_query + self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, msg=f"
{message}
", parse_mode=ParseMode.HTML, callback_path="update_status_table", reload_able=True) + else: + self._send_msg(f"
{message}
", reload_able=True, callback_path="update_status_table", parse_mode=ParseMode.HTML) except RPCException as e: self._send_msg(str(e)) @@ -376,7 +395,11 @@ class Telegram(RPCHandler): ], tablefmt='simple') message = f'Daily Profit over the last {timescale} days:\n
{stats_tab}
' - self._send_msg(message, parse_mode=ParseMode.HTML) + if(update.callback_query): + query = update.callback_query + self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, msg=message, parse_mode=ParseMode.HTML, callback_path="update_daily", reload_able=True) + else: + self._send_msg(msg=message, parse_mode=ParseMode.HTML, callback_path="update_daily", reload_able=True) except RPCException as e: self._send_msg(str(e)) @@ -435,7 +458,11 @@ class Telegram(RPCHandler): if stats['closed_trade_count'] > 0: markdown_msg += (f"\n*Avg. Duration:* `{avg_duration}`\n" f"*Best Performing:* `{best_pair}: {best_rate:.2f}%`") - self._send_msg(markdown_msg) + if(update.callback_query): + query = update.callback_query + self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, msg=markdown_msg, callback_path="update_profit", reload_able=True) + else: + self._send_msg(msg=markdown_msg, callback_path="update_profit", reload_able=True) @authorized_only def _stats(self, update: Update, context: CallbackContext) -> None: @@ -514,7 +541,11 @@ class Telegram(RPCHandler): output += ("\n*Estimated Value*:\n" "\t`{stake}: {total: .8f}`\n" "\t`{symbol}: {value: .2f}`\n").format(**result) - self._send_msg(output) + if(update.callback_query): + query = update.callback_query + self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, msg=output, callback_path="update_balance", reload_able=True) + else: + self._send_msg(msg=output, callback_path="update_balance", reload_able=True) except RPCException as e: self._send_msg(str(e)) @@ -679,7 +710,11 @@ class Telegram(RPCHandler): count=trade['count'] ) for i, trade in enumerate(trades)) message = 'Performance:\n{}'.format(stats) - self._send_msg(message, parse_mode=ParseMode.HTML) + if(update.callback_query): + query = update.callback_query + self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, msg=message, parse_mode=ParseMode.HTML, callback_path="update_performance", reload_able=True) + else: + self._send_msg(msg=message, parse_mode=ParseMode.HTML, callback_path="update_performance", reload_able=True) except RPCException as e: self._send_msg(str(e)) @@ -699,7 +734,11 @@ class Telegram(RPCHandler): tablefmt='simple') message = "
{}
".format(message) logger.debug(message) - self._send_msg(message, parse_mode=ParseMode.HTML) + if(update.callback_query): + query = update.callback_query + self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, msg=message, parse_mode=ParseMode.HTML, callback_path="update_count", reload_able=True) + else: + self._send_msg(msg=message, parse_mode=ParseMode.HTML, callback_path="update_count", reload_able=True) except RPCException as e: self._send_msg(str(e)) @@ -901,8 +940,35 @@ class Telegram(RPCHandler): f"*Current state:* `{val['state']}`" ) - def _send_msg(self, msg: str, parse_mode: str = ParseMode.MARKDOWN, - disable_notification: bool = False) -> None: + def _update_msg(self, chat_id: str, message_id: str, msg: str, callback_path: str = "", reload_able: bool = False, parse_mode: str = ParseMode.MARKDOWN) -> None: + if reload_able: + reply_markup = InlineKeyboardMarkup([[InlineKeyboardButton("Refresh", callback_data=callback_path)]]) + else: + reply_markup = InlineKeyboardMarkup([[]]) + try: + try: + self._updater.bot.edit_message_text( + chat_id=chat_id, + message_id=message_id, + text=msg, + parse_mode=parse_mode, + reply_markup=reply_markup + ) + except BadRequest as e: + if 'not modified' in e.message.lower(): + pass + else: + logger.warning( + 'TelegramError: %s', + e.message + ) + except TelegramError as telegram_err: + logger.warning( + 'TelegramError: %s! Giving up on that message.', + telegram_err.message + ) + + def _send_msg(self, msg: str, parse_mode: str = ParseMode.MARKDOWN, disable_notification: bool = False, callback_path: str = "", reload_able: bool = False) -> None: """ Send given markdown message :param msg: message @@ -910,7 +976,10 @@ class Telegram(RPCHandler): :param parse_mode: telegram parse mode :return: None """ - reply_markup = ReplyKeyboardMarkup(self._keyboard) + if reload_able: + reply_markup = InlineKeyboardMarkup([[InlineKeyboardButton("Refresh", callback_data=callback_path)]]) + else: + reply_markup = ReplyKeyboardMarkup(self._keyboard) try: try: self._updater.bot.send_message( From 21d3635e8dcf9f41d241cb628534a8376f37fb1c Mon Sep 17 00:00:00 2001 From: Eko Aprili Trisno Date: Thu, 4 Feb 2021 01:16:27 +0700 Subject: [PATCH 003/834] Update telegram.py --- freqtrade/rpc/telegram.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index ad72e10e4..32af71a76 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -160,9 +160,9 @@ class Telegram(RPCHandler): CallbackQueryHandler(self._status_table, pattern='update_status_table'), CallbackQueryHandler(self._daily, pattern='update_daily'), CallbackQueryHandler(self._profit, pattern='update_profit'), - CallbackQueryHandler(self._profit, pattern='update_balance'), - CallbackQueryHandler(self._profit, pattern='update_performance'), - CallbackQueryHandler(self._profit, pattern='update_count') + CallbackQueryHandler(self._balance, pattern='update_balance'), + CallbackQueryHandler(self._performance, pattern='update_performance'), + CallbackQueryHandler(self._count, pattern='update_count') ] for handle in handles: self._updater.dispatcher.add_handler(handle) From 54d0ac9d20d077214c8df3b568b57f5114bb97da Mon Sep 17 00:00:00 2001 From: Eko Aprili Trisno Date: Thu, 4 Feb 2021 01:19:23 +0700 Subject: [PATCH 004/834] Update telegram.py --- freqtrade/rpc/telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 32af71a76..7cb05b04a 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -12,7 +12,7 @@ from typing import Any, Callable, Dict, List, Union import arrow from tabulate import tabulate from telegram import KeyboardButton, ParseMode, ReplyKeyboardMarkup, Update, InlineKeyboardButton, InlineKeyboardMarkup -from telegram.error import NetworkError, TelegramError +from telegram.error import NetworkError, TelegramError, BadRequest from telegram.ext import CallbackContext, CommandHandler, Updater, CallbackQueryHandler from telegram.utils.helpers import escape_markdown From ba32708ed44091b9eb2bc15ac8eeedfce98f3fd1 Mon Sep 17 00:00:00 2001 From: Eko Aprili Trisno Date: Sun, 14 Feb 2021 01:40:04 +0700 Subject: [PATCH 005/834] Update telegram.py --- freqtrade/rpc/telegram.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 7cb05b04a..80d7be60f 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -5,14 +5,14 @@ This module manage Telegram communication """ import json import logging -from datetime import timedelta +from datetime import timedelta, datetime from itertools import chain from typing import Any, Callable, Dict, List, Union import arrow from tabulate import tabulate from telegram import KeyboardButton, ParseMode, ReplyKeyboardMarkup, Update, InlineKeyboardButton, InlineKeyboardMarkup -from telegram.error import NetworkError, TelegramError, BadRequest +from telegram.error import NetworkError, TelegramError from telegram.ext import CallbackContext, CommandHandler, Updater, CallbackQueryHandler from telegram.utils.helpers import escape_markdown @@ -154,7 +154,7 @@ class Telegram(RPCHandler): CommandHandler('logs', self._logs), CommandHandler('edge', self._edge), CommandHandler('help', self._help), - CommandHandler('version', self._version) + CommandHandler('version', self._version), ] callbacks = [ CallbackQueryHandler(self._status_table, pattern='update_status_table'), @@ -945,6 +945,7 @@ class Telegram(RPCHandler): reply_markup = InlineKeyboardMarkup([[InlineKeyboardButton("Refresh", callback_data=callback_path)]]) else: reply_markup = InlineKeyboardMarkup([[]]) + msg+="\nUpdated: {}".format(datetime.now().ctime()) try: try: self._updater.bot.edit_message_text( @@ -976,10 +977,10 @@ class Telegram(RPCHandler): :param parse_mode: telegram parse mode :return: None """ - if reload_able: + if reload_able and self._config['telegram'].get('reload',True): reply_markup = InlineKeyboardMarkup([[InlineKeyboardButton("Refresh", callback_data=callback_path)]]) else: - reply_markup = ReplyKeyboardMarkup(self._keyboard) + reply_markup = ReplyKeyboardMarkup(self._keyboard, resize_keyboard=True) try: try: self._updater.bot.send_message( From aea8f05d10946ca488a2f1ec2564e92d0922feff Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 24 Feb 2021 06:39:59 +0100 Subject: [PATCH 006/834] Version bump 2021.2 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 74c8c412c..2205d284d 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2021.1' +__version__ = '2021.2' if __version__ == 'develop': From 834f00f5803431270f736d3a098eb9c5623cfbe8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 24 Feb 2021 06:46:07 +0100 Subject: [PATCH 007/834] Refresh slack link --- CONTRIBUTING.md | 2 +- README.md | 4 ++-- docs/developer.md | 2 +- docs/faq.md | 2 +- docs/index.md | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index afa41ed33..c29d6e632 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ Few pointers for contributions: - New features need to contain unit tests, must conform to PEP8 (max-line-length = 100) and should be documented with the introduction PR. - PR's can be declared as `[WIP]` - which signify Work in Progress Pull Requests (which are not finished). -If you are unsure, discuss the feature on our [discord server](https://discord.gg/MA9v74M), on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-l9d9iqgl-9cVBIeBkCBa8j6upSmd_NA) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. +If you are unsure, discuss the feature on our [discord server](https://discord.gg/MA9v74M), on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. ## Getting started diff --git a/README.md b/README.md index 7ef0d4ce7..c3a665c47 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,7 @@ For any questions not covered by the documentation or for further information ab Please check out our [discord server](https://discord.gg/MA9v74M). -You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-l9d9iqgl-9cVBIeBkCBa8j6upSmd_NA). +You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). ### [Bugs / Issues](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) @@ -178,7 +178,7 @@ to understand the requirements before sending your pull-requests. Coding is not a necessity to contribute - maybe start with improving our documentation? Issues labeled [good first issue](https://github.com/freqtrade/freqtrade/labels/good%20first%20issue) can be good first contributions, and will help get you familiar with the codebase. -**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [discord](https://discord.gg/MA9v74M) or [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-k9o2v5ut-jX8Mc4CwNM8CDc2Dyg96YA). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. +**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [discord](https://discord.gg/MA9v74M) or [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. **Important:** Always create your PR against the `develop` branch, not `stable`. diff --git a/docs/developer.md b/docs/developer.md index c09e528bf..4b8c64530 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -2,7 +2,7 @@ This page is intended for developers of Freqtrade, people who want to contribute to the Freqtrade codebase or documentation, or people who want to understand the source code of the application they're running. -All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel on [discord](https://discord.gg/MA9v74M) or [slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-l9d9iqgl-9cVBIeBkCBa8j6upSmd_NA) where you can ask questions. +All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel on [discord](https://discord.gg/MA9v74M) or [slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) where you can ask questions. ## Documentation diff --git a/docs/faq.md b/docs/faq.md index 87b0893bd..93b806dca 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -142,7 +142,7 @@ freqtrade hyperopt --hyperopt SampleHyperopt --hyperopt-loss SharpeHyperOptLossD ### Why does it take a long time to run hyperopt? -* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-l9d9iqgl-9cVBIeBkCBa8j6upSmd_NA) - or the Freqtrade [discord community](https://discord.gg/X89cVG). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you. +* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) - or the Freqtrade [discord community](https://discord.gg/X89cVG). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you. * If you wonder why it can take from 20 minutes to days to do 1000 epochs here are some answers: diff --git a/docs/index.md b/docs/index.md index db5088707..9d1a1532e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -81,7 +81,7 @@ For any questions not covered by the documentation or for further information ab Please check out our [discord server](https://discord.gg/MA9v74M). -You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-l9d9iqgl-9cVBIeBkCBa8j6upSmd_NA). +You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). ## Ready to try? From 2bed41da5dcc899b45c0d984b81b44cfc5094340 Mon Sep 17 00:00:00 2001 From: rextea Date: Fri, 26 Mar 2021 18:40:50 +0300 Subject: [PATCH 008/834] Add days breakdown table to backtesting --- docs/backtesting.md | 1 + freqtrade/commands/arguments.py | 2 +- freqtrade/commands/cli_options.py | 6 ++ freqtrade/configuration/configuration.py | 3 + freqtrade/optimize/optimize_reports.py | 71 +++++++++++++++++++++--- 5 files changed, 74 insertions(+), 9 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index d02c59f05..91faa07bb 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -67,6 +67,7 @@ optional arguments: Requires `--export` to be set as well. Example: `--export-filename=user_data/backtest_results/backtest _today.json` + --show-days Print a days breakdown table of the backtest results Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 9468a7f7d..b71819ef2 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -21,7 +21,7 @@ ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv", ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", "enable_protections", "dry_run_wallet", - "strategy_list", "export", "exportfilename"] + "strategy_list", "export", "exportfilename", "show_days"] ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", "position_stacking", "use_max_market_positions", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 15c13cec9..dc193ee4f 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -183,6 +183,12 @@ AVAILABLE_CLI_OPTIONS = { type=float, metavar='FLOAT', ), + "show_days": Arg( + '--show-days', + help='Print days breakdown for backtest results', + action='store_true', + default=False, + ), # Edge "stoploss_range": Arg( '--stoplosses', diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index a40a4fd83..1eb6351d0 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -260,6 +260,9 @@ class Configuration: self._args_to_config(config, argname='export', logstring='Parameter --export detected: {} ...') + self._args_to_config(config, argname='show_days', + logstring='Parameter --show-days detected ...') + # Edge section: if 'stoploss_range' in self.args and self.args["stoploss_range"]: txt_range = eval(self.args["stoploss_range"]) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 099976aa9..d15988669 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -13,7 +13,6 @@ from freqtrade.data.btanalysis import (calculate_csum, calculate_market_change, calculate_max_drawdown) from freqtrade.misc import decimals_per_coin, file_dump_json, round_coin_value - logger = logging.getLogger(__name__) @@ -32,7 +31,7 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N filename = Path.joinpath( recordfilename.parent, f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}' - ).with_suffix(recordfilename.suffix) + ).with_suffix(recordfilename.suffix) file_dump_json(filename, stats) latest_filename = Path.joinpath(filename.parent, LAST_BT_RESULT_FN) @@ -75,8 +74,8 @@ def _generate_result_line(result: DataFrame, starting_balance: int, first_column 'profit_total': profit_total, 'profit_total_pct': round(profit_total * 100.0, 2), 'duration_avg': str(timedelta( - minutes=round(result['trade_duration'].mean())) - ) if not result.empty else '0:00', + minutes=round(result['trade_duration'].mean())) + ) if not result.empty else '0:00', # 'duration_max': str(timedelta( # minutes=round(result['trade_duration'].max())) # ) if not result.empty else '0:00', @@ -161,12 +160,11 @@ def generate_strategy_metrics(all_results: Dict) -> List[Dict]: for strategy, results in all_results.items(): tabular_data.append(_generate_result_line( results['results'], results['config']['dry_run_wallet'], strategy) - ) + ) return tabular_data def generate_edge_table(results: dict) -> str: - floatfmt = ('s', '.10g', '.2f', '.2f', '.2f', '.2f', 'd', 'd', 'd') tabular_data = [] headers = ['Pair', 'Stoploss', 'Win Rate', 'Risk Reward Ratio', @@ -191,6 +189,29 @@ def generate_edge_table(results: dict) -> str: floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore +def generate_days_breakdown_stats(results: DataFrame, starting_balance: int) -> Dict[str, Any]: + days = results.resample('1d', on='close_date') + days_stats = [] + for name, day in days: + profit_abs = day['profit_abs'].sum().round(10) + profit_total = day['profit_abs'].sum() / starting_balance + wins = sum(day['profit_abs'] > 0) + draws = sum(day['profit_abs'] == 0) + loses = sum(day['profit_abs'] < 0) + profit_percentage = round(profit_total * 100.0, 2) + days_stats.append( + { + 'date': name.strftime('%d/%m/%Y'), + 'profit_percentage': profit_percentage, + 'profit_abs': profit_abs, + 'wins': wins, + 'draws': draws, + 'loses': loses + } + ) + return days_stats + + def generate_daily_stats(results: DataFrame) -> Dict[str, Any]: if len(results) == 0: return { @@ -266,6 +287,8 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame], starting_balance=starting_balance, results=results.loc[results['is_open']], skip_nan=True) + days_breakdown_stats = generate_days_breakdown_stats(results=results, + starting_balance=starting_balance) daily_stats = generate_daily_stats(results) best_pair = max([pair for pair in pair_results if pair['key'] != 'TOTAL'], key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None @@ -283,6 +306,7 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame], 'results_per_pair': pair_results, 'sell_reason_summary': sell_reason_stats, 'left_open_trades': left_open_results, + 'days_breakdown_stats': days_breakdown_stats, 'total_trades': len(results), 'total_volume': float(results['stake_amount'].sum()), 'avg_stake_amount': results['stake_amount'].mean() if len(results) > 0 else 0, @@ -425,6 +449,28 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right") +def text_table_days_breakdown(days_breakdown_stats: List[Dict[str, Any]], stake_currency: str) -> str: + """ + Generate small table with Backtest results by days + :param days_breakdown_stats: Days breakdown metrics + :param stake_currency: Stakecurrency used + :return: pretty printed table with tabulate as string + """ + headers = [ + 'Day', + 'Profit %', + f'Tot Profit {stake_currency}', + 'Wins', + 'Draws', + 'Losses', + ] + output = [[ + d['date'], d['profit_percentage'], round_coin_value(d['profit_abs'], stake_currency, False), + d['wins'], d['draws'], d['loses'], + ] for d in days_breakdown_stats] + return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right") + + def text_table_strategy(strategy_results, stake_currency: str) -> str: """ Generate summary table per strategy @@ -463,6 +509,8 @@ def text_table_add_metrics(strat_results: Dict) -> str: strat_results['stake_currency'])), ('Total profit %', f"{round(strat_results['profit_total'] * 100, 2)}%"), ('Trades per day', strat_results['trades_per_day']), + ('Avg. daily profit %', + f"{round(strat_results['profit_total'] / strat_results['backtest_days'] * 100, 2)}%"), ('Avg. stake amount', round_coin_value(strat_results['avg_stake_amount'], strat_results['stake_currency'])), ('Total trade volume', round_coin_value(strat_results['total_volume'], @@ -482,7 +530,7 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Worst day', round_coin_value(strat_results['backtest_worst_day_abs'], strat_results['stake_currency'])), ('Days win/draw/lose', f"{strat_results['winning_days']} / " - f"{strat_results['draw_days']} / {strat_results['losing_days']}"), + f"{strat_results['draw_days']} / {strat_results['losing_days']}"), ('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"), ('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"), ('', ''), # Empty line to improve readability @@ -510,7 +558,7 @@ def text_table_add_metrics(strat_results: Dict) -> str: strat_results['stake_currency']) stake_amount = round_coin_value( strat_results['stake_amount'], strat_results['stake_currency'] - ) if strat_results['stake_amount'] != UNLIMITED_STAKE_AMOUNT else 'unlimited' + ) if strat_results['stake_amount'] != UNLIMITED_STAKE_AMOUNT else 'unlimited' message = ("No trades made. " f"Your starting balance was {start_balance}, " @@ -542,6 +590,13 @@ def show_backtest_results(config: Dict, backtest_stats: Dict): print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '=')) print(table) + if config.get('show_days', False): + table = text_table_days_breakdown(days_breakdown_stats=results['days_breakdown_stats'], + stake_currency=stake_currency) + if isinstance(table, str) and len(table) > 0: + print(' DAYS BREAKDOWN '.center(len(table.splitlines()[0]), '=')) + print(table) + table = text_table_add_metrics(results) if isinstance(table, str) and len(table) > 0: print(' SUMMARY METRICS '.center(len(table.splitlines()[0]), '=')) From 76a02ff70aa4016ed6755fa1f00cbb6246edab97 Mon Sep 17 00:00:00 2001 From: rextea Date: Fri, 26 Mar 2021 18:49:17 +0300 Subject: [PATCH 009/834] fix indentations --- freqtrade/optimize/optimize_reports.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index d15988669..286fa5c46 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -13,6 +13,7 @@ from freqtrade.data.btanalysis import (calculate_csum, calculate_market_change, calculate_max_drawdown) from freqtrade.misc import decimals_per_coin, file_dump_json, round_coin_value + logger = logging.getLogger(__name__) @@ -31,7 +32,7 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N filename = Path.joinpath( recordfilename.parent, f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}' - ).with_suffix(recordfilename.suffix) + ).with_suffix(recordfilename.suffix) file_dump_json(filename, stats) latest_filename = Path.joinpath(filename.parent, LAST_BT_RESULT_FN) @@ -74,8 +75,8 @@ def _generate_result_line(result: DataFrame, starting_balance: int, first_column 'profit_total': profit_total, 'profit_total_pct': round(profit_total * 100.0, 2), 'duration_avg': str(timedelta( - minutes=round(result['trade_duration'].mean())) - ) if not result.empty else '0:00', + minutes=round(result['trade_duration'].mean())) + ) if not result.empty else '0:00', # 'duration_max': str(timedelta( # minutes=round(result['trade_duration'].max())) # ) if not result.empty else '0:00', @@ -530,7 +531,7 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Worst day', round_coin_value(strat_results['backtest_worst_day_abs'], strat_results['stake_currency'])), ('Days win/draw/lose', f"{strat_results['winning_days']} / " - f"{strat_results['draw_days']} / {strat_results['losing_days']}"), + f"{strat_results['draw_days']} / {strat_results['losing_days']}"), ('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"), ('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"), ('', ''), # Empty line to improve readability @@ -558,7 +559,7 @@ def text_table_add_metrics(strat_results: Dict) -> str: strat_results['stake_currency']) stake_amount = round_coin_value( strat_results['stake_amount'], strat_results['stake_currency'] - ) if strat_results['stake_amount'] != UNLIMITED_STAKE_AMOUNT else 'unlimited' + ) if strat_results['stake_amount'] != UNLIMITED_STAKE_AMOUNT else 'unlimited' message = ("No trades made. " f"Your starting balance was {start_balance}, " From 7fb34f7e25e1a9d08b3200fb8a0c7b57df773a52 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Mar 2021 11:34:11 +0100 Subject: [PATCH 010/834] Version bump 2021.3 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 2205d284d..5e2a1f88e 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2021.2' +__version__ = '2021.3' if __version__ == 'develop': From 2b78ee254cab129aed9448e201e496961b3cb788 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 28 Apr 2021 21:06:32 +0200 Subject: [PATCH 011/834] Version bump to 2021.4 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 5e2a1f88e..68bcad396 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2021.3' +__version__ = '2021.4' if __version__ == 'develop': From 1ffc53b3b5c770198bd664c534ae7a1f251a08d6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 2 May 2021 19:21:26 +0200 Subject: [PATCH 012/834] Fix docs typo for CategoryParameter closes #4852 --- docs/hyperopt.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index b3fdc699b..5f1f9bffa 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -251,9 +251,9 @@ We continue to define hyperoptable parameters: class MyAwesomeStrategy(IStrategy): buy_adx = IntParameter(20, 40, default=30) buy_rsi = IntParameter(20, 40, default=30) - buy_adx_enabled = CategoricalParameter([True, False]), - buy_rsi_enabled = CategoricalParameter([True, False]), - buy_trigger = CategoricalParameter(['bb_lower', 'macd_cross_signal']), + buy_adx_enabled = CategoricalParameter([True, False]) + buy_rsi_enabled = CategoricalParameter([True, False]) + buy_trigger = CategoricalParameter(['bb_lower', 'macd_cross_signal']) ``` Above definition says: I have five parameters I want to randomly combine to find the best combination. From 3d11df68e32e9415146f6920746dc4722a527116 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 3 May 2021 08:33:06 +0200 Subject: [PATCH 013/834] Be explicit with space assignment in documentation --- docs/hyperopt.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 5f1f9bffa..d8f4a8071 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -249,11 +249,11 @@ We continue to define hyperoptable parameters: ```python class MyAwesomeStrategy(IStrategy): - buy_adx = IntParameter(20, 40, default=30) - buy_rsi = IntParameter(20, 40, default=30) - buy_adx_enabled = CategoricalParameter([True, False]) - buy_rsi_enabled = CategoricalParameter([True, False]) - buy_trigger = CategoricalParameter(['bb_lower', 'macd_cross_signal']) + buy_adx = IntParameter(20, 40, default=30, space="buy") + buy_rsi = IntParameter(20, 40, default=30, space="buy") + buy_adx_enabled = CategoricalParameter([True, False], space="buy") + buy_rsi_enabled = CategoricalParameter([True, False], space="buy") + buy_trigger = CategoricalParameter(['bb_lower', 'macd_cross_signal'], space="buy") ``` Above definition says: I have five parameters I want to randomly combine to find the best combination. @@ -262,6 +262,10 @@ Then we have three category variables. First two are either `True` or `False`. We use these to either enable or disable the ADX and RSI guards. The last one we call `trigger` and use it to decide which buy trigger we want to use. +!!! Note "Parameter space assignment" + Parameters must either be assigned to a variable named `buy_*` or `sell_*` - or contain `space='buy'` | `space='sell'` to be assigned to a space correctly. + If no parameter is available for a space, you'll receive the error that no space was found when running hyperopt. + So let's write the buy strategy using these values: ```python From bd44deea0dbfbcf3a651d1533f04b019ec5291f5 Mon Sep 17 00:00:00 2001 From: Rikj000 Date: Mon, 24 May 2021 18:51:33 +0200 Subject: [PATCH 014/834] BugFix - hyperopt-show --print-json include non-optimized params --- freqtrade/optimize/hyperopt_tools.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) mode change 100644 => 100755 freqtrade/optimize/hyperopt_tools.py diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py old mode 100644 new mode 100755 index 49e70913f..8fa03a0d2 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -93,7 +93,7 @@ class HyperoptTools(): if print_json: result_dict: Dict = {} for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']: - HyperoptTools._params_update_for_json(result_dict, params, s) + HyperoptTools._params_update_for_json(result_dict, params, non_optimized, s) print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE)) else: @@ -106,11 +106,20 @@ class HyperoptTools(): HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:") @staticmethod - def _params_update_for_json(result_dict, params, space: str) -> None: + def _params_update_for_json(result_dict, params, non_optimized, space: str) -> None: if space in params: space_params = HyperoptTools._space_params(params, space) + space_non_optimized = HyperoptTools._space_params(non_optimized, space) + all_space_params = space_params + + # Include non optimized params if there are any + if len(space_non_optimized) > 0: + for non_optimized_param in space_non_optimized: + if non_optimized_param not in all_space_params: + all_space_params[non_optimized_param] = space_non_optimized[non_optimized_param] + if space in ['buy', 'sell']: - result_dict.setdefault('params', {}).update(space_params) + result_dict.setdefault('params', {}).update(all_space_params) elif space == 'roi': # TODO: get rid of OrderedDict when support for python 3.6 will be # dropped (dicts keep the order as the language feature) @@ -120,10 +129,10 @@ class HyperoptTools(): # OrderedDict is used to keep the numeric order of the items # in the dict. result_dict['minimal_roi'] = OrderedDict( - (str(k), v) for k, v in space_params.items() + (str(k), v) for k, v in all_space_params.items() ) else: # 'stoploss', 'trailing' - result_dict.update(space_params) + result_dict.update(all_space_params) @staticmethod def _params_pretty_print(params, space: str, header: str, non_optimized={}) -> None: From 0c9b913cad25e0f75cde562415014116e28c3153 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 May 2021 11:10:10 +0200 Subject: [PATCH 015/834] Version bump 2021.5 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 68bcad396..ed0c70417 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2021.4' +__version__ = '2021.5' if __version__ == 'develop': From 639c83575bd0094141f15d4f58814a1a59c565b1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 May 2021 13:08:28 +0200 Subject: [PATCH 016/834] Fix csv-export error with new hyperopt format --- freqtrade/optimize/hyperopt_tools.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 49e70913f..38cb0854e 100644 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -376,10 +376,11 @@ class HyperoptTools(): trials['Avg profit'] = trials['Avg profit'].apply( lambda x: f'{x * perc_multi:,.2f}%' if not isna(x) else "" ) - trials['Avg duration'] = trials['Avg duration'].apply( - lambda x: f'{x:,.1f} m' if isinstance( - x, float) else f"{x.total_seconds() // 60:,.1f} m" if not isna(x) else "" - ) + if perc_multi == 1: + trials['Avg duration'] = trials['Avg duration'].apply( + lambda x: f'{x:,.1f} m' if isinstance( + x, float) else f"{x.total_seconds() // 60:,.1f} m" if not isna(x) else "" + ) trials['Objective'] = trials['Objective'].apply( lambda x: f'{x:,.5f}' if x != 100000 else "" ) From 1e988c97ad1fc3a25ee4a40834a4795447ae370a Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 3 Jun 2021 20:55:18 +0200 Subject: [PATCH 017/834] Update dry-run order handling to use realistic fill prices closes #3389 --- freqtrade/exchange/exchange.py | 93 ++++++++++++++++++++++++++------- tests/exchange/test_exchange.py | 1 + tests/rpc/test_rpc_apiserver.py | 3 +- tests/rpc/test_rpc_telegram.py | 4 ++ tests/test_freqtradebot.py | 2 + 5 files changed, 84 insertions(+), 19 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 67676d4e0..87798e612 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -561,7 +561,7 @@ class Exchange: rate: float, params: Dict = {}) -> Dict[str, Any]: order_id = f'dry_run_{side}_{datetime.now().timestamp()}' _amount = self.amount_to_precision(pair, amount) - dry_order = { + dry_order: Dict[str, Any] = { 'id': order_id, 'symbol': pair, 'price': rate, @@ -577,26 +577,73 @@ class Exchange: 'fee': None, 'info': {} } - self._store_dry_order(dry_order, pair) + if dry_order["type"] in ["stop_loss_limit", "stop-loss-limit"]: + dry_order["info"] = {"stopPrice": dry_order["price"]} + + if dry_order["type"] == "market": + # Update market order pricing + average = self.get_dry_market_fill_price(pair, side, amount, rate) + dry_order.update({ + 'average': average, + 'cost': dry_order['amount'] * average, + }) + self.add_dry_order_fee(pair, dry_order) + + self._dry_run_open_orders[dry_order["id"]] = dry_order # Copy order and close it - so the returned order is open unless it's a market order return dry_order - def _store_dry_order(self, dry_order: Dict, pair: str) -> None: - closed_order = dry_order.copy() - if closed_order['type'] in ["market", "limit"]: - closed_order.update({ - 'status': 'closed', - 'filled': closed_order['amount'], - 'remaining': 0, - 'fee': { - 'currency': self.get_pair_quote_currency(pair), - 'cost': dry_order['cost'] * self.get_fee(pair), - 'rate': self.get_fee(pair) - } - }) - if closed_order["type"] in ["stop_loss_limit", "stop-loss-limit"]: - closed_order["info"].update({"stopPrice": closed_order["price"]}) - self._dry_run_open_orders[closed_order["id"]] = closed_order + def add_dry_order_fee(self, pair: str, dry_order: Dict[str, Any]): + dry_order.update({ + 'fee': { + 'currency': self.get_pair_quote_currency(pair), + 'cost': dry_order['cost'] * self.get_fee(pair), + 'rate': self.get_fee(pair) + } + }) + + def get_dry_market_fill_price(self, pair: str, side: str, amount: float, rate: float) -> float: + """ + Get the market order fill price based on orderbook interpolation + """ + if self.exchange_has('fetchL2OrderBook'): + ob = self.fetch_l2_order_book(pair, 20) + book_entry_type = 'asks' if side == 'buy' else 'bids' + + remaining_amount = amount + filled_amount = 0 + for book_entry in ob[book_entry_type]: + book_entry_price = book_entry[0] + book_entry_coin_volume = book_entry[1] + book_entry_ref_currency_volume = book_entry_price * book_entry_coin_volume + if remaining_amount > 0: + if remaining_amount < book_entry_ref_currency_volume: + filled_amount += remaining_amount * book_entry_price + else: + filled_amount += book_entry_ref_currency_volume * book_entry_price + remaining_amount -= book_entry_ref_currency_volume + else: + break + forecast_avg_filled_price = filled_amount / amount + return self.price_to_precision(pair, forecast_avg_filled_price) + + return rate + + def dry_limit_order_filled(self, pair: str, side: str, limit: float) -> bool: + if not self.exchange_has('fetchL2OrderBook'): + return True + ob = self.fetch_l2_order_book(pair, 1) + if side == 'buy': + price = ob['asks'][0][0] + logger.debug(f"{pair} checking dry buy-order: price={price}, limit={limit}") + if limit >= price: + return True + else: + price = ob['bids'][0][0] + logger.debug(f"{pair} checking dry sell-order: price={price}, limit={limit}") + if limit <= price: + return True + return False def fetch_dry_run_order(self, order_id) -> Dict[str, Any]: """ @@ -605,6 +652,16 @@ class Exchange: """ try: order = self._dry_run_open_orders[order_id] + pair = order['symbol'] + if order['status'] != "closed" and order['type'] in ["limit"]: + if self.dry_limit_order_filled(pair, order['side'], order['price']): + order.update({ + 'status': 'closed', + 'filled': order['amount'], + 'remaining': 0, + }) + self.add_dry_order_fee(pair, order) + return order except KeyError as e: # Gracefully handle errors with dry-run orders. diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 5fa94e6c1..73b8022d1 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -2117,6 +2117,7 @@ def test_get_historic_trades_notsupported(default_conf, mocker, caplog, exchange def test_cancel_order_dry_run(default_conf, mocker, exchange_name): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + mocker.patch('freqtrade.exchange.Exchange.dry_limit_order_filled', return_value=True) assert exchange.cancel_order(order_id='123', pair='TKN/BTC') == {} assert exchange.cancel_stoploss_order(order_id='123', pair='TKN/BTC') == {} diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index def2e43c6..f47819568 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -996,7 +996,8 @@ def test_api_forcesell(botclient, mocker, ticker, fee, markets): get_balances=MagicMock(return_value=ticker), fetch_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) + markets=PropertyMock(return_value=markets), + dry_limit_order_filled=MagicMock(return_value=True), ) patch_get_signal(ftbot, (True, False)) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 4c60bdad3..50c8a36ce 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -225,6 +225,7 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + dry_limit_order_filled=MagicMock(return_value=True), ) status_table = MagicMock() mocker.patch.multiple( @@ -671,6 +672,7 @@ def test_telegram_forcesell_handle(default_conf, update, ticker, fee, 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + dry_limit_order_filled=MagicMock(return_value=True), ) freqtradebot = FreqtradeBot(default_conf) @@ -729,6 +731,7 @@ def test_telegram_forcesell_down_handle(default_conf, update, ticker, fee, 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + dry_limit_order_filled=MagicMock(return_value=True), ) freqtradebot = FreqtradeBot(default_conf) @@ -789,6 +792,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + dry_limit_order_filled=MagicMock(return_value=True), ) default_conf['max_open_trades'] = 4 freqtradebot = FreqtradeBot(default_conf) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 66866a8fc..0866deead 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2750,6 +2750,7 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, ticker, fee, ticke price_to_precision=lambda s, x, y: y, stoploss=stoploss, cancel_stoploss_order=cancel_order, + dry_limit_order_filled=MagicMock(return_value=True), ) freqtrade = FreqtradeBot(default_conf) @@ -2792,6 +2793,7 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, ticker, f get_fee=fee, amount_to_precision=lambda s, x, y: y, price_to_precision=lambda s, x, y: y, + dry_limit_order_filled=MagicMock(return_value=True), ) stoploss = MagicMock(return_value={ From db03a2410958fff66dd943f35bc46527976876f3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 4 Jun 2021 06:44:51 +0200 Subject: [PATCH 018/834] Add tests for fill methods --- freqtrade/exchange/exchange.py | 14 ++++---- tests/conftest.py | 34 ++++++++++++++++++ tests/exchange/test_exchange.py | 64 +++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 6 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 87798e612..ea3a7d7cd 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -608,22 +608,24 @@ class Exchange: """ if self.exchange_has('fetchL2OrderBook'): ob = self.fetch_l2_order_book(pair, 20) - book_entry_type = 'asks' if side == 'buy' else 'bids' + ob_type = 'asks' if side == 'buy' else 'bids' remaining_amount = amount filled_amount = 0 - for book_entry in ob[book_entry_type]: + for book_entry in ob[ob_type]: book_entry_price = book_entry[0] book_entry_coin_volume = book_entry[1] - book_entry_ref_currency_volume = book_entry_price * book_entry_coin_volume if remaining_amount > 0: - if remaining_amount < book_entry_ref_currency_volume: + if remaining_amount < book_entry_coin_volume: filled_amount += remaining_amount * book_entry_price else: - filled_amount += book_entry_ref_currency_volume * book_entry_price - remaining_amount -= book_entry_ref_currency_volume + filled_amount += book_entry_coin_volume * book_entry_price + remaining_amount -= book_entry_coin_volume else: break + else: + # If remaining_amount wasn't consumed completely (break was not called) + filled_amount += remaining_amount * book_entry_price forecast_avg_filled_price = filled_amount / amount return self.price_to_precision(pair, forecast_avg_filled_price) diff --git a/tests/conftest.py b/tests/conftest.py index 43a98647f..8ce41cf9f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1087,6 +1087,40 @@ def order_book_l2(): }) +@pytest.fixture +def order_book_l2_usd(): + return MagicMock(return_value={ + 'symbol': 'LTC/USDT', + 'bids': [ + [25.563, 49.269], + [25.562, 83.0], + [25.56, 106.0], + [25.559, 15.381], + [25.558, 29.299], + [25.557, 34.624], + [25.556, 10.0], + [25.555, 14.684], + [25.554, 45.91], + [25.553, 50.0] + ], + 'asks': [ + [25.566, 14.27], + [25.567, 48.484], + [25.568, 92.349], + [25.572, 31.48], + [25.573, 23.0], + [25.574, 20.0], + [25.575, 89.606], + [25.576, 262.016], + [25.577, 178.557], + [25.578, 78.614] + ], + 'timestamp': None, + 'datetime': None, + 'nonce': 2372149736 + }) + + @pytest.fixture def ohlcv_history_list(): return [ diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 73b8022d1..2c4ddacb4 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -947,6 +947,70 @@ def test_create_dry_run_order(default_conf, mocker, side, exchange_name): assert order["symbol"] == "ETH/BTC" +@pytest.mark.parametrize("side,startprice,endprice", [ + ("buy", 25.563, 25.566), + ("sell", 25.566, 25.563) +]) +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, endprice, + exchange_name, order_book_l2_usd): + default_conf['dry_run'] = True + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + mocker.patch.multiple('freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + fetch_l2_order_book=order_book_l2_usd, + ) + + order = exchange.create_dry_run_order( + pair='LTC/USDT', ordertype='limit', side=side, amount=1, rate=startprice) + assert 'id' in order + assert f'dry_run_{side}_' in order["id"] + assert order["side"] == side + assert order["type"] == "limit" + assert order["symbol"] == "LTC/USDT" + + order_closed = exchange.fetch_dry_run_order(order['id']) + assert order_book_l2_usd.call_count == 1 + assert order_closed['status'] == 'open' + assert not order['fee'] + + order_book_l2_usd.reset_mock() + order_closed['price'] = endprice + + order_closed = exchange.fetch_dry_run_order(order['id']) + assert order_closed['status'] == 'closed' + assert order['fee'] + + +@pytest.mark.parametrize("side,amount,endprice", [ + ("buy", 1, 25.566), + ("buy", 100, 25.5672), # Requires interpolation + ("buy", 1000, 25.575), # More than orderbook return + ("sell", 1, 25.563), + ("sell", 100, 25.5625), # Requires interpolation + ("sell", 1000, 25.5555), # More than orderbook return +]) +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test_create_dry_run_order_market_fill(default_conf, mocker, side, amount, endprice, + exchange_name, order_book_l2_usd): + default_conf['dry_run'] = True + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + mocker.patch.multiple('freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + fetch_l2_order_book=order_book_l2_usd, + ) + + order = exchange.create_dry_run_order( + pair='LTC/USDT', ordertype='market', side=side, amount=amount, rate=25.5) + assert 'id' in order + assert f'dry_run_{side}_' in order["id"] + assert order["side"] == side + assert order["type"] == "market" + assert order["symbol"] == "LTC/USDT" + assert order['status'] == 'closed' + assert round(order["average"], 4) == round(endprice, 4) + + @pytest.mark.parametrize("side", [ ("buy"), ("sell") From c389d44e9ac35cdf1f3c4a903f8d89ca8bb1b6f3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Jun 2021 15:22:52 +0200 Subject: [PATCH 019/834] Improve filling logic --- freqtrade/exchange/exchange.py | 36 +++++++++++++++++++++------------ tests/exchange/test_exchange.py | 4 +++- tests/rpc/test_rpc.py | 15 ++++++++------ tests/rpc/test_rpc_apiserver.py | 2 +- tests/rpc/test_rpc_telegram.py | 14 ++++++------- tests/test_freqtradebot.py | 11 ++++++++-- 6 files changed, 52 insertions(+), 30 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index ea3a7d7cd..19b646a93 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -587,13 +587,15 @@ class Exchange: 'average': average, 'cost': dry_order['amount'] * average, }) - self.add_dry_order_fee(pair, dry_order) + dry_order = self.add_dry_order_fee(pair, dry_order) + + dry_order = self.check_dry_limit_order_filled(dry_order) self._dry_run_open_orders[dry_order["id"]] = dry_order # Copy order and close it - so the returned order is open unless it's a market order return dry_order - def add_dry_order_fee(self, pair: str, dry_order: Dict[str, Any]): + def add_dry_order_fee(self, pair: str, dry_order: Dict[str, Any]) -> Dict[str, Any]: dry_order.update({ 'fee': { 'currency': self.get_pair_quote_currency(pair), @@ -601,6 +603,7 @@ class Exchange: 'rate': self.get_fee(pair) } }) + return dry_order def get_dry_market_fill_price(self, pair: str, side: str, amount: float, rate: float) -> float: """ @@ -631,7 +634,7 @@ class Exchange: return rate - def dry_limit_order_filled(self, pair: str, side: str, limit: float) -> bool: + def _is_dry_limit_order_filled(self, pair: str, side: str, limit: float) -> bool: if not self.exchange_has('fetchL2OrderBook'): return True ob = self.fetch_l2_order_book(pair, 1) @@ -647,6 +650,22 @@ class Exchange: return True return False + def check_dry_limit_order_filled(self, order: Dict[str, Any]) -> Dict[str, Any]: + """ + Check dry-run limit order fill and update fee (if it filled). + """ + if order['status'] != "closed" and order['type'] in ["limit"]: + pair = order['symbol'] + if self._is_dry_limit_order_filled(pair, order['side'], order['price']): + order.update({ + 'status': 'closed', + 'filled': order['amount'], + 'remaining': 0, + }) + self.add_dry_order_fee(pair, order) + + return order + def fetch_dry_run_order(self, order_id) -> Dict[str, Any]: """ Return dry-run order @@ -654,16 +673,7 @@ class Exchange: """ try: order = self._dry_run_open_orders[order_id] - pair = order['symbol'] - if order['status'] != "closed" and order['type'] in ["limit"]: - if self.dry_limit_order_filled(pair, order['side'], order['price']): - order.update({ - 'status': 'closed', - 'filled': order['amount'], - 'remaining': 0, - }) - self.add_dry_order_fee(pair, order) - + order = self.check_dry_limit_order_filled(order) return order except KeyError as e: # Gracefully handle errors with dry-run orders. diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 2c4ddacb4..42bb07175 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -963,11 +963,13 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, order = exchange.create_dry_run_order( pair='LTC/USDT', ordertype='limit', side=side, amount=1, rate=startprice) + assert order_book_l2_usd.call_count == 1 assert 'id' in order assert f'dry_run_{side}_' in order["id"] assert order["side"] == side assert order["type"] == "limit" assert order["symbol"] == "LTC/USDT" + order_book_l2_usd.reset_mock() order_closed = exchange.fetch_dry_run_order(order['id']) assert order_book_l2_usd.call_count == 1 @@ -2181,7 +2183,7 @@ def test_get_historic_trades_notsupported(default_conf, mocker, caplog, exchange def test_cancel_order_dry_run(default_conf, mocker, exchange_name): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) - mocker.patch('freqtrade.exchange.Exchange.dry_limit_order_filled', return_value=True) + mocker.patch('freqtrade.exchange.Exchange._is_dry_limit_order_filled', return_value=True) assert exchange.cancel_order(order_id='123', pair='TKN/BTC') == {} assert exchange.cancel_stoploss_order(order_id='123', pair='TKN/BTC') == {} diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 7556dde6d..fd231b614 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -679,6 +679,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: 'filled': 0.0, } ), + _is_dry_limit_order_filled=MagicMock(return_value=True), get_fee=fee, ) mocker.patch('freqtrade.wallets.Wallets.get_free', return_value=1000) @@ -703,8 +704,8 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: assert msg == {'result': 'Created sell orders for all open trades.'} freqtradebot.enter_positions() - msg = rpc._rpc_forcesell('1') - assert msg == {'result': 'Created sell order for trade 1.'} + msg = rpc._rpc_forcesell('2') + assert msg == {'result': 'Created sell order for trade 2.'} freqtradebot.state = State.STOPPED with pytest.raises(RPCException, match=r'.*trader is not running*'): @@ -715,9 +716,11 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: freqtradebot.state = State.RUNNING assert cancel_order_mock.call_count == 0 + mocker.patch( + 'freqtrade.exchange.Exchange._is_dry_limit_order_filled', MagicMock(return_value=False)) freqtradebot.enter_positions() # make an limit-buy open trade - trade = Trade.query.filter(Trade.id == '1').first() + trade = Trade.query.filter(Trade.id == '3').first() filled_amount = trade.amount / 2 # Fetch order - it's open first, and closed after cancel_order is called. mocker.patch( @@ -738,7 +741,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: ) # check that the trade is called, which is done by ensuring exchange.cancel_order is called # and trade amount is updated - rpc._rpc_forcesell('1') + rpc._rpc_forcesell('3') assert cancel_order_mock.call_count == 1 assert trade.amount == filled_amount @@ -766,8 +769,8 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: } ) # check that the trade is called, which is done by ensuring exchange.cancel_order is called - msg = rpc._rpc_forcesell('2') - assert msg == {'result': 'Created sell order for trade 2.'} + msg = rpc._rpc_forcesell('4') + assert msg == {'result': 'Created sell order for trade 4.'} assert cancel_order_mock.call_count == 2 assert trade.amount == amount diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index f47819568..f30825b7b 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -997,7 +997,7 @@ def test_api_forcesell(botclient, mocker, ticker, fee, markets): fetch_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets), - dry_limit_order_filled=MagicMock(return_value=True), + _is_dry_limit_order_filled=MagicMock(return_value=False), ) patch_get_signal(ftbot, (True, False)) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 50c8a36ce..c933ac648 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -225,7 +225,7 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, - dry_limit_order_filled=MagicMock(return_value=True), + _is_dry_limit_order_filled=MagicMock(return_value=True), ) status_table = MagicMock() mocker.patch.multiple( @@ -672,7 +672,7 @@ def test_telegram_forcesell_handle(default_conf, update, ticker, fee, 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, - dry_limit_order_filled=MagicMock(return_value=True), + _is_dry_limit_order_filled=MagicMock(return_value=True), ) freqtradebot = FreqtradeBot(default_conf) @@ -731,7 +731,7 @@ def test_telegram_forcesell_down_handle(default_conf, update, ticker, fee, 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, - dry_limit_order_filled=MagicMock(return_value=True), + _is_dry_limit_order_filled=MagicMock(return_value=True), ) freqtradebot = FreqtradeBot(default_conf) @@ -792,7 +792,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, - dry_limit_order_filled=MagicMock(return_value=True), + _is_dry_limit_order_filled=MagicMock(return_value=True), ) default_conf['max_open_trades'] = 4 freqtradebot = FreqtradeBot(default_conf) @@ -809,9 +809,9 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None context.args = ["all"] telegram._forcesell(update=update, context=context) - # Called for each trade 4 times - assert msg_mock.call_count == 12 - msg = msg_mock.call_args_list[2][0][0] + # Called for each trade 2 times + assert msg_mock.call_count == 8 + msg = msg_mock.call_args_list[1][0][0] assert { 'type': RPCMessageType.SELL, 'trade_id': 1, diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 0866deead..4bcc578c4 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -305,6 +305,7 @@ def test_total_open_trades_stakes(mocker, default_conf, ticker, fee) -> None: 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -334,6 +335,7 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, mocker) -> Non 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) # Save state of current whitelist @@ -2533,6 +2535,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) @@ -2596,6 +2599,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker) 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) @@ -2648,6 +2652,7 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) @@ -2750,7 +2755,7 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, ticker, fee, ticke price_to_precision=lambda s, x, y: y, stoploss=stoploss, cancel_stoploss_order=cancel_order, - dry_limit_order_filled=MagicMock(return_value=True), + _is_dry_limit_order_filled=MagicMock(side_effect=[True, False]), ) freqtrade = FreqtradeBot(default_conf) @@ -2793,7 +2798,7 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, ticker, f get_fee=fee, amount_to_precision=lambda s, x, y: y, price_to_precision=lambda s, x, y: y, - dry_limit_order_filled=MagicMock(return_value=True), + _is_dry_limit_order_filled=MagicMock(side_effect=[False, True]), ) stoploss = MagicMock(return_value={ @@ -2862,6 +2867,7 @@ def test_execute_sell_market_order(default_conf, ticker, fee, 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) @@ -3467,6 +3473,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_b }), buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) default_conf['ask_strategy'] = { 'ignore_roi_if_buy_signal': False From c76848e089ad722aa5f851d6da0a9823621f1ad5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jun 2021 13:51:42 +0200 Subject: [PATCH 020/834] Update dry-run description with new filling logic --- docs/configuration.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index ef6f34094..63f55505a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -503,7 +503,8 @@ Once you will be happy with your bot performance running in the Dry-run mode, yo * API-keys may or may not be provided. Only Read-Only operations (i.e. operations that do not alter account state) on the exchange are performed in dry-run mode. * Wallets (`/balance`) are simulated based on `dry_run_wallet`. * Orders are simulated, and will not be posted to the exchange. -* Orders are assumed to fill immediately, and will never time out. +* Market orders fill based on orderbook volume the moment the order is placed. +* Limit orders fill once price reaches the defined level - or time out based on `unfilledtimeout` settings. * In combination with `stoploss_on_exchange`, the stop_loss price is assumed to be filled. * Open orders (not trades, which are stored in the database) are reset on bot restart. From 03eff698291b16077163bfe153860276a2a48d28 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 13 Jun 2021 20:21:43 +0200 Subject: [PATCH 021/834] Simplify update message sending --- freqtrade/rpc/telegram.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index ba9c6c0f6..921fdfe59 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -10,12 +10,12 @@ from datetime import date, datetime, timedelta from html import escape from itertools import chain from math import isnan -from typing import Any, Callable, Dict, List, Union, cast +from typing import Any, Callable, Dict, List, Union import arrow from tabulate import tabulate -from telegram import (InlineKeyboardButton, InlineKeyboardMarkup, KeyboardButton, ParseMode, - ReplyKeyboardMarkup, Update) +from telegram import (CallbackQuery, InlineKeyboardButton, InlineKeyboardMarkup, KeyboardButton, + ParseMode, ReplyKeyboardMarkup, Update) from telegram.error import BadRequest, NetworkError, TelegramError from telegram.ext import CallbackContext, CallbackQueryHandler, CommandHandler, Updater from telegram.utils.helpers import escape_markdown @@ -180,8 +180,8 @@ class Telegram(RPCHandler): for handle in handles: self._updater.dispatcher.add_handler(handle) - for handle in callbacks: - self._updater.dispatcher.add_handler(handle) + for callback in callbacks: + self._updater.dispatcher.add_handler(callback) self._updater.start_polling( bootstrap_retries=-1, @@ -422,9 +422,7 @@ class Telegram(RPCHandler): lines = message.split("\n") message = "\n".join(lines[:-1] + [lines[1]] + [lines[-1]]) if(messages_count == 1 and update.callback_query): - query = update.callback_query - self._update_msg(chat_id=query.message.chat_id, - message_id=query.message.message_id, + self._update_msg(query=update.callback_query, msg=f"
{message}
", parse_mode=ParseMode.HTML, callback_path="update_status_table", reload_able=True) @@ -469,8 +467,7 @@ class Telegram(RPCHandler): tablefmt='simple') message = f'Daily Profit over the last {timescale} days:\n
{stats_tab}
' if(update.callback_query): - query = update.callback_query - self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, + self._update_msg(query=update.callback_query, msg=message, parse_mode=ParseMode.HTML, callback_path="update_daily", reload_able=True) else: @@ -548,8 +545,7 @@ class Telegram(RPCHandler): markdown_msg += (f"\n*Avg. Duration:* `{avg_duration}`\n" f"*Best Performing:* `{best_pair}: {best_rate:.2f}%`") if(update.callback_query): - query = update.callback_query - self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, + self._update_msg(query=update.callback_query, msg=markdown_msg, callback_path="update_profit", reload_able=True) else: self._send_msg(msg=markdown_msg, callback_path="update_profit", reload_able=True) @@ -640,8 +636,7 @@ class Telegram(RPCHandler): f"\t`{result['symbol']}: " f"{round_coin_value(result['value'], result['symbol'], False)}`\n") if(update.callback_query): - query = update.callback_query - self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, + self._update_msg(query=update.callback_query, msg=output, callback_path="update_balance", reload_able=True) else: self._send_msg(msg=output, callback_path="update_balance", reload_able=True) @@ -841,8 +836,7 @@ class Telegram(RPCHandler): output += stat_line if(sent_messages == 0 and update.callback_query): - query = update.callback_query - self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, + self._update_msg(query=update.callback_query, msg=output, parse_mode=ParseMode.HTML, callback_path="update_performance", reload_able=True) else: @@ -868,8 +862,7 @@ class Telegram(RPCHandler): message = "
{}
".format(message) logger.debug(message) if(update.callback_query): - query = update.callback_query - self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, + self._update_msg(query=update.callback_query, msg=message, parse_mode=ParseMode.HTML, callback_path="update_count", reload_able=True) else: @@ -1106,7 +1099,7 @@ class Telegram(RPCHandler): f"*Current state:* `{val['state']}`" ) - def _update_msg(self, chat_id: str, message_id: str, msg: str, callback_path: str = "", + def _update_msg(self, query: CallbackQuery, msg: str, callback_path: str = "", reload_able: bool = False, parse_mode: str = ParseMode.MARKDOWN) -> None: if reload_able: reply_markup = InlineKeyboardMarkup([ @@ -1115,6 +1108,11 @@ class Telegram(RPCHandler): else: reply_markup = InlineKeyboardMarkup([[]]) msg += "\nUpdated: {}".format(datetime.now().ctime()) + if not query.message: + return + chat_id = query.message.chat_id + message_id = query.message.message_id + try: try: self._updater.bot.edit_message_text( From a95f760ff7e46462ca7116aa5f6616fcc54724e3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 13 Jun 2021 20:34:08 +0200 Subject: [PATCH 022/834] Simplify update logic by moving it to send_msg --- freqtrade/rpc/telegram.py | 65 +++++++++++++-------------------------- 1 file changed, 22 insertions(+), 43 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 921fdfe59..0fb322eb8 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -10,7 +10,7 @@ from datetime import date, datetime, timedelta from html import escape from itertools import chain from math import isnan -from typing import Any, Callable, Dict, List, Union +from typing import Any, Callable, Dict, List, Optional, Union import arrow from tabulate import tabulate @@ -421,14 +421,9 @@ class Telegram(RPCHandler): # insert separators line between Total lines = message.split("\n") message = "\n".join(lines[:-1] + [lines[1]] + [lines[-1]]) - if(messages_count == 1 and update.callback_query): - self._update_msg(query=update.callback_query, - msg=f"
{message}
", - parse_mode=ParseMode.HTML, - callback_path="update_status_table", reload_able=True) - else: - self._send_msg(f"
{message}
", reload_able=True, - callback_path="update_status_table", parse_mode=ParseMode.HTML) + self._send_msg(f"
{message}
", reload_able=True, + callback_path="update_status_table", parse_mode=ParseMode.HTML, + query=update.callback_query) except RPCException as e: self._send_msg(str(e)) @@ -466,13 +461,8 @@ class Telegram(RPCHandler): ], tablefmt='simple') message = f'Daily Profit over the last {timescale} days:\n
{stats_tab}
' - if(update.callback_query): - self._update_msg(query=update.callback_query, - msg=message, parse_mode=ParseMode.HTML, - callback_path="update_daily", reload_able=True) - else: - self._send_msg(msg=message, parse_mode=ParseMode.HTML, callback_path="update_daily", - reload_able=True) + self._send_msg(message, parse_mode=ParseMode.HTML, callback_path="update_daily", + reload_able=True, query=update.callback_query) except RPCException as e: self._send_msg(str(e)) @@ -544,11 +534,8 @@ class Telegram(RPCHandler): if stats['closed_trade_count'] > 0: markdown_msg += (f"\n*Avg. Duration:* `{avg_duration}`\n" f"*Best Performing:* `{best_pair}: {best_rate:.2f}%`") - if(update.callback_query): - self._update_msg(query=update.callback_query, - msg=markdown_msg, callback_path="update_profit", reload_able=True) - else: - self._send_msg(msg=markdown_msg, callback_path="update_profit", reload_able=True) + self._send_msg(markdown_msg, callback_path="update_profit", reload_able=True, + query=update.callback_query) @authorized_only def _stats(self, update: Update, context: CallbackContext) -> None: @@ -635,11 +622,8 @@ class Telegram(RPCHandler): f"\t`{result['stake']}: {result['total']: .8f}`\n" f"\t`{result['symbol']}: " f"{round_coin_value(result['value'], result['symbol'], False)}`\n") - if(update.callback_query): - self._update_msg(query=update.callback_query, - msg=output, callback_path="update_balance", reload_able=True) - else: - self._send_msg(msg=output, callback_path="update_balance", reload_able=True) + self._send_msg(output, callback_path="update_balance", reload_able=True, + query=update.callback_query) except RPCException as e: self._send_msg(str(e)) @@ -820,7 +804,6 @@ class Telegram(RPCHandler): try: trades = self._rpc._rpc_performance() output = "Performance:\n" - sent_messages = 0 for i, trade in enumerate(trades): stat_line = ( f"{i+1}.\t {trade['pair']}\t" @@ -831,17 +814,12 @@ class Telegram(RPCHandler): if len(output + stat_line) >= MAX_TELEGRAM_MESSAGE_LENGTH: self._send_msg(output, parse_mode=ParseMode.HTML) output = stat_line - sent_messages += 1 else: output += stat_line - if(sent_messages == 0 and update.callback_query): - self._update_msg(query=update.callback_query, - msg=output, parse_mode=ParseMode.HTML, - callback_path="update_performance", reload_able=True) - else: - self._send_msg(msg=output, parse_mode=ParseMode.HTML, - callback_path="update_performance", reload_able=True) + self._send_msg(output, parse_mode=ParseMode.HTML, + callback_path="update_performance", reload_able=True, + query=update.callback_query) except RPCException as e: self._send_msg(str(e)) @@ -861,13 +839,9 @@ class Telegram(RPCHandler): tablefmt='simple') message = "
{}
".format(message) logger.debug(message) - if(update.callback_query): - self._update_msg(query=update.callback_query, - msg=message, parse_mode=ParseMode.HTML, - callback_path="update_count", reload_able=True) - else: - self._send_msg(msg=message, parse_mode=ParseMode.HTML, - callback_path="update_count", reload_able=True) + self._send_msg(message, parse_mode=ParseMode.HTML, + callback_path="update_count", reload_able=True, + query=update.callback_query) except RPCException as e: self._send_msg(str(e)) @@ -1140,7 +1114,8 @@ class Telegram(RPCHandler): disable_notification: bool = False, keyboard: List[List[Union[str, KeyboardButton, InlineKeyboardButton]]] = None, callback_path: str = "", - reload_able: bool = False) -> None: + reload_able: bool = False, + query: Optional[CallbackQuery] = None) -> None: """ Send given markdown message :param msg: message @@ -1148,6 +1123,10 @@ class Telegram(RPCHandler): :param parse_mode: telegram parse mode :return: None """ + if query: + self._update_msg(query=query, msg=msg, parse_mode=parse_mode, + callback_path=callback_path, reload_able=reload_able) + return if reload_able and self._config['telegram'].get('reload', True): reply_markup = InlineKeyboardMarkup([ [InlineKeyboardButton("Refresh", callback_data=callback_path)]]) From e226252921c87370f173631221ca5f522ce1ccba Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 13 Jun 2021 20:39:25 +0200 Subject: [PATCH 023/834] Always use the same parameter sequence --- freqtrade/rpc/telegram.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 0fb322eb8..8f8627ece 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -421,8 +421,8 @@ class Telegram(RPCHandler): # insert separators line between Total lines = message.split("\n") message = "\n".join(lines[:-1] + [lines[1]] + [lines[-1]]) - self._send_msg(f"
{message}
", reload_able=True, - callback_path="update_status_table", parse_mode=ParseMode.HTML, + self._send_msg(f"
{message}
", parse_mode=ParseMode.HTML, + reload_able=True, callback_path="update_status_table", query=update.callback_query) except RPCException as e: self._send_msg(str(e)) @@ -461,8 +461,8 @@ class Telegram(RPCHandler): ], tablefmt='simple') message = f'Daily Profit over the last {timescale} days:\n
{stats_tab}
' - self._send_msg(message, parse_mode=ParseMode.HTML, callback_path="update_daily", - reload_able=True, query=update.callback_query) + self._send_msg(message, parse_mode=ParseMode.HTML, reload_able=True, + callback_path="update_daily", query=update.callback_query) except RPCException as e: self._send_msg(str(e)) @@ -534,7 +534,7 @@ class Telegram(RPCHandler): if stats['closed_trade_count'] > 0: markdown_msg += (f"\n*Avg. Duration:* `{avg_duration}`\n" f"*Best Performing:* `{best_pair}: {best_rate:.2f}%`") - self._send_msg(markdown_msg, callback_path="update_profit", reload_able=True, + self._send_msg(markdown_msg, reload_able=True, callback_path="update_profit", query=update.callback_query) @authorized_only @@ -622,7 +622,7 @@ class Telegram(RPCHandler): f"\t`{result['stake']}: {result['total']: .8f}`\n" f"\t`{result['symbol']}: " f"{round_coin_value(result['value'], result['symbol'], False)}`\n") - self._send_msg(output, callback_path="update_balance", reload_able=True, + self._send_msg(output, reload_able=True, callback_path="update_balance", query=update.callback_query) except RPCException as e: self._send_msg(str(e)) @@ -818,7 +818,7 @@ class Telegram(RPCHandler): output += stat_line self._send_msg(output, parse_mode=ParseMode.HTML, - callback_path="update_performance", reload_able=True, + reload_able=True, callback_path="update_performance", query=update.callback_query) except RPCException as e: self._send_msg(str(e)) @@ -840,7 +840,7 @@ class Telegram(RPCHandler): message = "
{}
".format(message) logger.debug(message) self._send_msg(message, parse_mode=ParseMode.HTML, - callback_path="update_count", reload_able=True, + reload_able=True, callback_path="update_count", query=update.callback_query) except RPCException as e: self._send_msg(str(e)) From 3f1d6d453cbde0cceab5b1b2239bce9b5f6b1635 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 05:19:23 +0000 Subject: [PATCH 024/834] Bump mypy from 0.812 to 0.902 Bumps [mypy](https://github.com/python/mypy) from 0.812 to 0.902. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.812...v0.902) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6fbe581a5..b0f970224 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ coveralls==3.1.0 flake8==3.9.2 flake8-type-annotations==0.1.0 flake8-tidy-imports==4.3.0 -mypy==0.812 +mypy==0.902 pytest==6.2.4 pytest-asyncio==0.15.1 pytest-cov==2.12.1 From 4ba7a2bbd290c2c6e5746995ceacd89660a04ab5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 14 Jun 2021 19:18:42 +0200 Subject: [PATCH 025/834] Fix mypy update problems --- freqtrade/resolvers/iresolver.py | 3 +++ freqtrade/rpc/webhook.py | 15 +++++++-------- requirements-dev.txt | 6 ++++++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 5172e6fda..5b6977b4b 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -58,6 +58,9 @@ class IResolver: # Generate spec based on absolute path # Pass object_name as first argument to have logging print a reasonable name. spec = importlib.util.spec_from_file_location(object_name or "", str(module_path)) + if not spec: + return iter([None]) + module = importlib.util.module_from_spec(spec) try: spec.loader.exec_module(module) # type: ignore # importlib does not use typehints diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 0e4a4bf6f..b4c55649e 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -77,14 +77,13 @@ class Webhook(RPCHandler): def _send_msg(self, payload: dict) -> None: """do the actual call to the webhook""" - if self._format == 'form': - kwargs = {'data': payload} - elif self._format == 'json': - kwargs = {'json': payload} - else: - raise NotImplementedError('Unknown format: {}'.format(self._format)) - try: - post(self._url, **kwargs) + if self._format == 'form': + post(self._url, data=payload) + elif self._format == 'json': + post(self._url, json=payload) + else: + raise NotImplementedError('Unknown format: {}'.format(self._format)) + except RequestException as exc: logger.warning("Could not call webhook url. Exception: %s", exc) diff --git a/requirements-dev.txt b/requirements-dev.txt index b0f970224..924b35e1a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -17,3 +17,9 @@ isort==5.8.0 # Convert jupyter notebooks to markdown documents nbconvert==6.0.7 + +# mypy types +types-cachetools==0.1.7 +types-filelock==0.1.3 +types-requests==0.1.11 +types-tabulate==0.1.0 From cf7394d01cb6798213bb4b08572885328756adda Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 14 Jun 2021 19:57:24 +0200 Subject: [PATCH 026/834] Export backtesting results by default closes #4977 --- docs/backtesting.md | 21 +++++++++++++-------- freqtrade/commands/cli_options.py | 5 +++-- freqtrade/constants.py | 2 ++ freqtrade/optimize/backtesting.py | 2 +- tests/conftest.py | 1 + tests/optimize/test_backtesting.py | 12 ++++++------ tests/test_configuration.py | 5 ++--- 7 files changed, 28 insertions(+), 20 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 2027c2079..26642ef8c 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -19,7 +19,7 @@ usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--enable-protections] [--dry-run-wallet DRY_RUN_WALLET] [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] - [--export EXPORT] [--export-filename PATH] + [--export {none,trades}] [--export-filename PATH] optional arguments: -h, --help show this help message and exit @@ -63,8 +63,8 @@ optional arguments: name is injected into the filename (so `backtest- data.json` becomes `backtest-data- DefaultStrategy.json` - --export EXPORT Export backtest results, argument are: trades. - Example: `--export=trades` + --export {none,trades} + Export backtest results (default: trades). --export-filename PATH Save backtest results to the file with this filename. Requires `--export` to be set as well. Example: @@ -100,7 +100,7 @@ Strategy arguments: Now you have good Buy and Sell strategies and some historic data, you want to test it against real data. This is what we call [backtesting](https://en.wikipedia.org/wiki/Backtesting). -Backtesting will use the crypto-currencies (pairs) from your config file and load historical candle (OHCLV) data from `user_data/data/` by default. +Backtesting will use the crypto-currencies (pairs) from your config file and load historical candle (OHLCV) data from `user_data/data/` by default. If no data is available for the exchange / pair / timeframe combination, backtesting will ask you to download them first using `freqtrade download-data`. For details on downloading, please refer to the [Data Downloading](data-download.md) section in the documentation. @@ -110,11 +110,16 @@ All profit calculations include fees, and freqtrade will use the exchange's defa !!! Warning "Using dynamic pairlists for backtesting" Using dynamic pairlists is possible, however it relies on the current market conditions - which will not reflect the historic status of the pairlist. - Also, when using pairlists other than StaticPairlist, reproducability of backtesting-results cannot be guaranteed. + Also, when using pairlists other than StaticPairlist, reproducibility of backtesting-results cannot be guaranteed. Please read the [pairlists documentation](plugins.md#pairlists) for more information. To achieve reproducible results, best generate a pairlist via the [`test-pairlist`](utils.md#test-pairlist) command and use that as static pairlist. +!!! Note + By default, Freqtrade will export backtesting results to `user_data/backtest_results`. + The exported trades can be used for [further analysis](#further-backtest-result-analysis) or can be used by the [plotting sub-command](plotting.md#plot-price-and-indicators) (`freqtrade plot-dataframe`) in the scripts directory. + + ### Starting balance Backtesting will require a starting balance, which can be provided as `--dry-run-wallet ` or `--starting-balance ` command line argument, or via `dry_run_wallet` configuration setting. @@ -174,13 +179,13 @@ Where `SampleStrategy1` and `AwesomeStrategy` refer to class names of strategies --- -Exporting trades to file +Prevent exporting trades to file ```bash -freqtrade backtesting --strategy backtesting --export trades --config config.json +freqtrade backtesting --strategy backtesting --export none --config config.json ``` -The exported trades can be used for [further analysis](#further-backtest-result-analysis), or can be used by the plotting script `plot_dataframe.py` in the scripts directory. +Only use this if you're sure you'll not want to plot or analyze your results further. --- diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index d832693ee..b226415e7 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -167,8 +167,9 @@ AVAILABLE_CLI_OPTIONS = { ), "export": Arg( '--export', - help='Export backtest results, argument are: trades. ' - 'Example: `--export=trades`', + help='Export backtest results (default: trades).', + choices=constants.EXPORT_OPTIONS, + ), "exportfilename": Arg( '--export-filename', diff --git a/freqtrade/constants.py b/freqtrade/constants.py index e42b9d4b8..259aa0e03 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -12,6 +12,7 @@ PROCESS_THROTTLE_SECS = 5 # sec HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec TIMEOUT_UNITS = ['minutes', 'seconds'] +EXPORT_OPTIONS = ['none', 'trades'] DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite' UNLIMITED_STAKE_AMOUNT = 'unlimited' @@ -308,6 +309,7 @@ CONF_SCHEMA = { 'required': ['enabled', 'listen_ip_address', 'listen_port', 'username', 'password'] }, 'db_url': {'type': 'string'}, + 'export': {'type': 'string', 'enum': EXPORT_OPTIONS, 'default': 'trades'}, 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, 'forcebuy_enable': {'type': 'boolean'}, 'disable_dataframe_checks': {'type': 'boolean'}, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 922f89c22..c72a8b5c5 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -520,7 +520,7 @@ class Backtesting: stats = generate_backtest_stats(data, self.all_results, min_date=min_date, max_date=max_date) - if self.config.get('export', False): + if self.config.get('export', 'none') == 'trades': store_backtest_stats(self.config['exportfilename'], stats) # Show backtest results diff --git a/tests/conftest.py b/tests/conftest.py index dd38ca610..c6a0dfcfd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -326,6 +326,7 @@ def get_default_conf(testdatadir): "strategy_path": str(Path(__file__).parent / "strategy" / "strats"), "strategy": "DefaultStrategy", "internals": {}, + "export": "none", } return configuration diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 7387c8865..60bd82d71 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -155,6 +155,7 @@ def test_setup_optimize_configuration_without_arguments(mocker, default_conf, ca 'backtesting', '--config', 'config.json', '--strategy', 'DefaultStrategy', + '--export', 'none' ] config = setup_optimize_configuration(get_args(args), RunMode.BACKTEST) @@ -172,7 +173,8 @@ def test_setup_optimize_configuration_without_arguments(mocker, default_conf, ca assert not log_has('Parameter --enable-position-stacking detected ...', caplog) assert 'timerange' not in config - assert 'export' not in config + assert 'export' in config + assert config['export'] == 'none' assert 'runmode' in config assert config['runmode'] == RunMode.BACKTEST @@ -193,7 +195,6 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> '--enable-position-stacking', '--disable-max-market-positions', '--timerange', ':100', - '--export', '/bar/foo', '--export-filename', 'foo_bar.json', '--fee', '0', ] @@ -223,7 +224,6 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog) assert 'export' in config - assert log_has('Parameter --export detected: {} ...'.format(config['export']), caplog) assert 'exportfilename' in config assert isinstance(config['exportfilename'], Path) assert log_has('Storing backtest results to {} ...'.format(config['exportfilename']), caplog) @@ -395,7 +395,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> default_conf['timeframe'] = "1m" default_conf['datadir'] = testdatadir - default_conf['export'] = None + default_conf['export'] = 'none' default_conf['timerange'] = '20180101-20180102' backtesting = Backtesting(default_conf) @@ -416,7 +416,7 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> default_conf['timeframe'] = "1m" default_conf['datadir'] = testdatadir - default_conf['export'] = None + default_conf['export'] = 'none' default_conf['timerange'] = '20180101-20180102' with pytest.raises(OperationalException, match='No pair in whitelist.'): @@ -440,7 +440,7 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti default_conf['ticker_interval'] = "1m" default_conf['datadir'] = testdatadir - default_conf['export'] = None + default_conf['export'] = 'none' # Use stoploss from strategy del default_conf['stoploss'] default_conf['timerange'] = '20180101-20180102' diff --git a/tests/test_configuration.py b/tests/test_configuration.py index aa121edfa..c5d0cd908 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -425,7 +425,6 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert not log_has('Parameter --enable-position-stacking detected ...', caplog) assert 'timerange' not in config - assert 'export' not in config def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> None: @@ -448,7 +447,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non '--enable-position-stacking', '--disable-max-market-positions', '--timerange', ':100', - '--export', '/bar/foo', + '--export', 'trades', '--stake-amount', 'unlimited' ] @@ -496,7 +495,7 @@ def test_setup_configuration_with_stratlist(mocker, default_conf, caplog) -> Non 'backtesting', '--config', 'config.json', '--ticker-interval', '1m', - '--export', '/bar/foo', + '--export', 'trades', '--strategy-list', 'DefaultStrategy', 'TestStrategy' From 6d5fc967147e0aae37423d1e0ee57de47a17a791 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sat, 12 Jun 2021 10:16:30 +0300 Subject: [PATCH 027/834] Implement most pessimistic handling of trailing stoploss. --- freqtrade/optimize/backtesting.py | 16 +++++++++ freqtrade/strategy/interface.py | 4 +-- tests/optimize/test_backtest_detail.py | 47 ++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index c72a8b5c5..19ae74ae6 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -225,6 +225,22 @@ class Backtesting: # sell at open price. return sell_row[OPEN_IDX] + # Special case: trailing triggers within same candle as trade opened. Assume most + # pessimistic price movement, which is moving just enough to arm stoploss and + # immediately going down to stop price. + if sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0 and \ + self.strategy.trailing_stop_positive: + if self.strategy.trailing_only_offset_is_reached: + # Worst case: price reaches stop_positive_offset and dives down. + stop_rate = sell_row[OPEN_IDX] * \ + (1 + abs(self.strategy.trailing_stop_positive_offset) - + abs(self.strategy.trailing_stop_positive)) + else: + # Worst case: price ticks tiny bit above open and dives down. + stop_rate = sell_row[OPEN_IDX] * (1 - abs(self.strategy.trailing_stop_positive)) + assert stop_rate < sell_row[HIGH_IDX] + return stop_rate + # Set close_rate to stoploss return trade.stop_loss elif sell.sell_type == (SellType.ROI): diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 8ea38f503..47d4259fc 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -610,7 +610,7 @@ class IStrategy(ABC, HyperStrategyMixin): # Initiate stoploss with open_rate. Does nothing if stoploss is already set. trade.adjust_stop_loss(trade.open_rate, stop_loss_value, initial=True) - if self.use_custom_stoploss: + if self.use_custom_stoploss and trade.stop_loss < current_rate: stop_loss_value = strategy_safe_wrapper(self.custom_stoploss, default_retval=None )(pair=trade.pair, trade=trade, current_time=current_time, @@ -623,7 +623,7 @@ class IStrategy(ABC, HyperStrategyMixin): else: logger.warning("CustomStoploss function did not return valid stoploss") - if self.trailing_stop: + if self.trailing_stop and trade.stop_loss < current_rate: # trailing stoploss handling sl_offset = self.trailing_stop_positive_offset diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index e5b969383..488425323 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -457,6 +457,50 @@ tc28 = BTContainer(data=[ trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=3)] ) +# Test 29: trailing_stop should be triggered by low of next candle, without adjusting stoploss using +# high of stoploss candle. +# stop-loss: 10%, ROI: 10% (should not apply) +tc29 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5050, 5000, 4900, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4900, 5250, 4500, 5100, 6172, 0, 0], # Triggers trailing-stoploss + [3, 5100, 5100, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.10}, profit_perc=-0.02, trailing_stop=True, + trailing_stop_positive=0.03, + trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=2)] +) + +# Test 30: trailing_stop should be triggered immediately on trade open candle. +# stop-loss: 10%, ROI: 10% (should not apply) +tc30 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5500, 5000, 4900, 6172, 0, 0], # enter trade (signal on last candle) and stop + [2, 4900, 5250, 4500, 5100, 6172, 0, 0], + [3, 5100, 5100, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.10}, profit_perc=-0.01, trailing_stop=True, + trailing_stop_positive=0.01, + trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=1)] +) + +# Test 31: trailing_stop should be triggered immediately on trade open candle. +# stop-loss: 10%, ROI: 10% (should not apply) +tc31 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5500, 5000, 4900, 6172, 0, 0], # enter trade (signal on last candle) and stop + [2, 4900, 5250, 4500, 5100, 6172, 0, 0], + [3, 5100, 5100, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.10}, profit_perc=0.01, trailing_stop=True, + trailing_only_offset_is_reached=True, trailing_stop_positive_offset=0.02, + trailing_stop_positive=0.01, + trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=1)] +) + TESTS = [ tc0, tc1, @@ -487,6 +531,9 @@ TESTS = [ tc26, tc27, tc28, + tc29, + tc30, + tc31, ] From 38ed49cef54d0bfe4606be46e2ec88c344e768b3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 13 Jun 2021 16:37:11 +0200 Subject: [PATCH 028/834] move low to stoploss_reached to clarify where which rate is used --- freqtrade/strategy/interface.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 47d4259fc..6358c6a4e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -524,15 +524,14 @@ class IStrategy(ABC, HyperStrategyMixin): :param force_stoploss: Externally provided stoploss :return: True if trade should be sold, False otherwise """ - # Set current rate to low for backtesting sell - current_rate = low or rate + current_rate = rate current_profit = trade.calc_profit_ratio(current_rate) trade.adjust_min_max_rates(high or current_rate) stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade, current_time=date, current_profit=current_profit, - force_stoploss=force_stoploss, high=high) + force_stoploss=force_stoploss, low=low, high=high) # Set current rate to high for backtesting sell current_rate = high or rate @@ -599,18 +598,21 @@ class IStrategy(ABC, HyperStrategyMixin): def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime, current_profit: float, - force_stoploss: float, high: float = None) -> SellCheckTuple: + force_stoploss: float, low: float = None, + high: float = None) -> SellCheckTuple: """ Based on current profit of the trade and configured (trailing) stoploss, decides to sell or not :param current_profit: current profit as ratio + :param low: Low value of this candle, only set in backtesting + :param high: High value of this candle, only set in backtesting """ stop_loss_value = force_stoploss if force_stoploss else self.stoploss # Initiate stoploss with open_rate. Does nothing if stoploss is already set. trade.adjust_stop_loss(trade.open_rate, stop_loss_value, initial=True) - if self.use_custom_stoploss and trade.stop_loss < current_rate: + if self.use_custom_stoploss and trade.stop_loss < (low or current_rate): stop_loss_value = strategy_safe_wrapper(self.custom_stoploss, default_retval=None )(pair=trade.pair, trade=trade, current_time=current_time, @@ -623,7 +625,7 @@ class IStrategy(ABC, HyperStrategyMixin): else: logger.warning("CustomStoploss function did not return valid stoploss") - if self.trailing_stop and trade.stop_loss < current_rate: + if self.trailing_stop and trade.stop_loss < (low or current_rate): # trailing stoploss handling sl_offset = self.trailing_stop_positive_offset @@ -643,7 +645,7 @@ class IStrategy(ABC, HyperStrategyMixin): # evaluate if the stoploss was hit if stoploss is not on exchange # in Dry-Run, this handles stoploss logic as well, as the logic will not be different to # regular stoploss handling. - if ((trade.stop_loss >= current_rate) and + if ((trade.stop_loss >= (low or current_rate)) and (not self.order_types.get('stoploss_on_exchange') or self.config['dry_run'])): sell_type = SellType.STOP_LOSS @@ -652,7 +654,7 @@ class IStrategy(ABC, HyperStrategyMixin): if trade.initial_stop_loss != trade.stop_loss: sell_type = SellType.TRAILING_STOP_LOSS logger.debug( - f"{trade.pair} - HIT STOP: current price at {current_rate:.6f}, " + f"{trade.pair} - HIT STOP: current price at {(low or current_rate):.6f}, " f"stoploss is {trade.stop_loss:.6f}, " f"initial stoploss was at {trade.initial_stop_loss:.6f}, " f"trade opened at {trade.open_rate:.6f}") From 1bb04bb0c24c8a959df1fbc77617f5e1c1752d29 Mon Sep 17 00:00:00 2001 From: barbarius Date: Wed, 16 Jun 2021 11:40:55 +0200 Subject: [PATCH 029/834] Moved daily avg trade row next to total trades on backtest results --- docs/backtesting.md | 8 +++----- freqtrade/optimize/optimize_reports.py | 4 ++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 26642ef8c..8e50aa356 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -284,7 +284,7 @@ A backtesting result will look like that: | Backtesting to | 2019-05-01 00:00:00 | | Max open trades | 3 | | | | -| Total trades | 429 | +| Total/Daily Avg Trades| 429 / 3.575 | | Starting balance | 0.01000000 BTC | | Final balance | 0.01762792 BTC | | Absolute profit | 0.00762792 BTC | @@ -373,12 +373,11 @@ It contains some useful key metrics about performance of your strategy on backte | Backtesting to | 2019-05-01 00:00:00 | | Max open trades | 3 | | | | -| Total trades | 429 | +| Total/Daily Avg Trades| 429 / 3.575 | | Starting balance | 0.01000000 BTC | | Final balance | 0.01762792 BTC | | Absolute profit | 0.00762792 BTC | | Total profit % | 76.2% | -| Trades per day | 3.575 | | Avg. stake amount | 0.001 BTC | | Total trade volume | 0.429 BTC | | | | @@ -409,12 +408,11 @@ It contains some useful key metrics about performance of your strategy on backte - `Backtesting from` / `Backtesting to`: Backtesting range (usually defined with the `--timerange` option). - `Max open trades`: Setting of `max_open_trades` (or `--max-open-trades`) - or number of pairs in the pairlist (whatever is lower). -- `Total trades`: Identical to the total trades of the backtest output table. +- `Total/Daily Avg Trades`: Identical to the total trades of the backtest output table / Total trades divided by the backtesting duration in days (this will give you information about how many trades to expect from the strategy). - `Starting balance`: Start balance - as given by dry-run-wallet (config or command line). - `Final balance`: Final balance - starting balance + absolute profit. - `Absolute profit`: Profit made in stake currency. - `Total profit %`: Total profit. Aligned to the `TOTAL` row's `Tot Profit %` from the first table. Calculated as `(End capital − Starting capital) / Starting capital`. -- `Trades per day`: Total trades divided by the backtesting duration in days (this will give you information about how many trades to expect from the strategy). - `Avg. stake amount`: Average stake amount, either `stake_amount` or the average when using dynamic stake amount. - `Total trade volume`: Volume generated on the exchange to reach the above profit. - `Best Pair` / `Worst Pair`: Best and worst performing pair, and it's corresponding `Cum Profit %`. diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 84e052ac4..64b043304 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -556,7 +556,8 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Backtesting to', strat_results['backtest_end']), ('Max open trades', strat_results['max_open_trades']), ('', ''), # Empty line to improve readability - ('Total trades', strat_results['total_trades']), + ('Total/Daily Avg Trades', + f"{strat_results['total_trades']} / {strat_results['trades_per_day']}"), ('Starting balance', round_coin_value(strat_results['starting_balance'], strat_results['stake_currency'])), ('Final balance', round_coin_value(strat_results['final_balance'], @@ -564,7 +565,6 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Absolute profit ', round_coin_value(strat_results['profit_total_abs'], strat_results['stake_currency'])), ('Total profit %', f"{round(strat_results['profit_total'] * 100, 2):}%"), - ('Trades per day', strat_results['trades_per_day']), ('Avg. stake amount', round_coin_value(strat_results['avg_stake_amount'], strat_results['stake_currency'])), ('Total trade volume', round_coin_value(strat_results['total_volume'], From 1c9def2fdbfad2e11a283b900cde6e8968145930 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 16 Jun 2021 20:17:44 +0100 Subject: [PATCH 030/834] Update freqtrade/optimize/optimize_reports.py --- freqtrade/optimize/optimize_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 64b043304..df7f721ec 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -556,7 +556,7 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Backtesting to', strat_results['backtest_end']), ('Max open trades', strat_results['max_open_trades']), ('', ''), # Empty line to improve readability - ('Total/Daily Avg Trades', + ('Total/Daily Avg Trades', f"{strat_results['total_trades']} / {strat_results['trades_per_day']}"), ('Starting balance', round_coin_value(strat_results['starting_balance'], strat_results['stake_currency'])), From b38ab84a13d23906b05b236f6eb9421088b9c09a Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 17 Jun 2021 06:48:41 +0200 Subject: [PATCH 031/834] Add documentation mention about new behaviour --- docs/backtesting.md | 1 + freqtrade/optimize/backtesting.py | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 26642ef8c..d34381f55 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -446,6 +446,7 @@ Since backtesting lacks some detailed information about what happens within a ca - Stoploss is evaluated before ROI within one candle. So you can often see more trades with the `stoploss` sell reason comparing to the results obtained with the same strategy in the Dry Run/Live Trade modes - Low happens before high for stoploss, protecting capital first - Trailing stoploss + - Trailing Stoploss is only adjusted if it's below the candle's low (otherwise it would be triggered) - High happens first - adjusting stoploss - Low uses the adjusted stoploss (so sells with large high-low difference are backtested correctly) - ROI applies before trailing-stop, ensuring profits are "top-capped" at ROI if both ROI and trailing stop applies diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 19ae74ae6..028a9eacd 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -228,13 +228,13 @@ class Backtesting: # Special case: trailing triggers within same candle as trade opened. Assume most # pessimistic price movement, which is moving just enough to arm stoploss and # immediately going down to stop price. - if sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0 and \ - self.strategy.trailing_stop_positive: + if (sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0 + and self.strategy.trailing_stop_positive): if self.strategy.trailing_only_offset_is_reached: # Worst case: price reaches stop_positive_offset and dives down. - stop_rate = sell_row[OPEN_IDX] * \ - (1 + abs(self.strategy.trailing_stop_positive_offset) - - abs(self.strategy.trailing_stop_positive)) + stop_rate = (sell_row[OPEN_IDX] * + (1 + abs(self.strategy.trailing_stop_positive_offset) - + abs(self.strategy.trailing_stop_positive))) else: # Worst case: price ticks tiny bit above open and dives down. stop_rate = sell_row[OPEN_IDX] * (1 - abs(self.strategy.trailing_stop_positive)) From a49ca9cbf78b1a0e458c3e4767344d2c71a66219 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 17 Jun 2021 06:57:35 +0200 Subject: [PATCH 032/834] Change log-level "Executing handler" msg to debug closes #5143 --- freqtrade/rpc/telegram.py | 2 +- tests/rpc/test_rpc_telegram.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c3ddfd644..aee513017 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -56,7 +56,7 @@ def authorized_only(command_handler: Callable[..., None]) -> Callable[..., Any]: ) return wrapper - logger.info( + logger.debug( 'Executing handler: %s for chat_id: %s', command_handler.__name__, chat_id diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index bbda55c3e..d091f3837 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -2,6 +2,7 @@ # pragma pylint: disable=protected-access, unused-argument, invalid-name # pragma pylint: disable=too-many-lines, too-many-arguments +import logging import re from datetime import datetime from functools import reduce @@ -120,7 +121,7 @@ def test_cleanup(default_conf, mocker, ) -> None: def test_authorized_only(default_conf, mocker, caplog, update) -> None: patch_exchange(mocker) - + caplog.set_level(logging.DEBUG) default_conf['telegram']['enabled'] = False bot = FreqtradeBot(default_conf) rpc = RPC(bot) @@ -136,6 +137,7 @@ def test_authorized_only(default_conf, mocker, caplog, update) -> None: def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: patch_exchange(mocker) + caplog.set_level(logging.DEBUG) chat = Chat(0xdeadbeef, 0) update = Update(randint(1, 100)) update.message = Message(randint(1, 100), datetime.utcnow(), chat) From a9f111dca0063790dadaebfad03c265c8e0842ea Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 17 Jun 2021 19:50:49 +0200 Subject: [PATCH 033/834] Fix some types --- freqtrade/rpc/telegram.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index f83d5a238..6a0e98a75 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -95,7 +95,7 @@ class Telegram(RPCHandler): Validates the keyboard configuration from telegram config section. """ - self._keyboard: List[List[Union[str, KeyboardButton, InlineKeyboardButton]]] = [ + self._keyboard: List[List[Union[str, KeyboardButton]]] = [ ['/daily', '/profit', '/balance'], ['/status', '/status table', '/performance'], ['/count', '/start', '/stop', '/help'] @@ -1112,7 +1112,7 @@ class Telegram(RPCHandler): def _send_msg(self, msg: str, parse_mode: str = ParseMode.MARKDOWN, disable_notification: bool = False, - keyboard: List[List[Union[str, KeyboardButton, InlineKeyboardButton]]] = None, + keyboard: List[List[InlineKeyboardButton]] = None, callback_path: str = "", reload_able: bool = False, query: Optional[CallbackQuery] = None) -> None: @@ -1123,6 +1123,7 @@ class Telegram(RPCHandler): :param parse_mode: telegram parse mode :return: None """ + reply_markup: Union[InlineKeyboardMarkup, ReplyKeyboardMarkup] if query: self._update_msg(query=query, msg=msg, parse_mode=parse_mode, callback_path=callback_path, reload_able=reload_able) From 8562e19776433d182d7406ad1594ed2220d37ae9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 17 Jun 2021 20:15:53 +0200 Subject: [PATCH 034/834] Document protections to come from the strategy --- docs/configuration.md | 3 +- docs/includes/protections.md | 67 +++++++----------------------------- 2 files changed, 14 insertions(+), 56 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index ef6f34094..3788ef57c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -105,7 +105,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `edge.*` | Please refer to [edge configuration document](edge.md) for detailed explanation. | `experimental.block_bad_exchanges` | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
*Defaults to `true`.*
**Datatype:** Boolean | `pairlists` | Define one or more pairlists to be used. [More information](plugins.md#pairlists-and-pairlist-handlers).
*Defaults to `StaticPairList`.*
**Datatype:** List of Dicts -| `protections` | Define one or more protections to be used. [More information](plugins.md#protections). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** List of Dicts +| `protections` | Define one or more protections to be used. [More information](plugins.md#protections).
**Datatype:** List of Dicts | `telegram.enabled` | Enable the usage of Telegram.
**Datatype:** Boolean | `telegram.token` | Your Telegram bot token. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `telegram.chat_id` | Your personal Telegram account id. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String @@ -156,7 +156,6 @@ Values set in the configuration file always overwrite values set in the strategy * `order_time_in_force` * `unfilledtimeout` * `disable_dataframe_checks` -* `protections` * `use_sell_signal` (ask_strategy) * `sell_profit_only` (ask_strategy) * `sell_profit_offset` (ask_strategy) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 6bc57153e..3ea2dde61 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -8,7 +8,6 @@ All protection end times are rounded up to the next candle to avoid sudden, unex !!! Note Not all Protections will work for all strategies, and parameters will need to be tuned for your strategy to improve performance. - To align your protection with your strategy, you can define protections in the strategy. !!! Tip Each Protection can be configured multiple times with different parameters, to allow different levels of protection (short-term / long-term). @@ -47,16 +46,16 @@ This applies across all pairs, unless `only_per_pair` is set to true, which will The below example stops trading for all pairs for 4 candles after the last trade if the bot hit stoploss 4 times within the last 24 candles. -```json -"protections": [ +``` python +protections = [ { "method": "StoplossGuard", "lookback_period_candles": 24, "trade_limit": 4, "stop_duration_candles": 4, - "only_per_pair": false + "only_per_pair": False } -], +] ``` !!! Note @@ -69,8 +68,8 @@ The below example stops trading for all pairs for 4 candles after the last trade The below sample stops trading for 12 candles if max-drawdown is > 20% considering all pairs - with a minimum of `trade_limit` trades - within the last 48 candles. If desired, `lookback_period` and/or `stop_duration` can be used. -```json -"protections": [ +``` python +protections = [ { "method": "MaxDrawdown", "lookback_period_candles": 48, @@ -78,7 +77,7 @@ The below sample stops trading for 12 candles if max-drawdown is > 20% consideri "stop_duration_candles": 12, "max_allowed_drawdown": 0.2 }, -], +] ``` #### Low Profit Pairs @@ -88,8 +87,8 @@ If that ratio is below `required_profit`, that pair will be locked for `stop_dur The below example will stop trading a pair for 60 minutes if the pair does not have a required profit of 2% (and a minimum of 2 trades) within the last 6 candles. -```json -"protections": [ +``` python +protections = [ { "method": "LowProfitPairs", "lookback_period_candles": 6, @@ -97,7 +96,7 @@ The below example will stop trading a pair for 60 minutes if the pair does not h "stop_duration": 60, "required_profit": 0.02 } -], +] ``` #### Cooldown Period @@ -106,13 +105,13 @@ The below example will stop trading a pair for 60 minutes if the pair does not h The below example will stop trading a pair for 2 candles after closing a trade, allowing this pair to "cool down". -```json -"protections": [ +``` python +protections = [ { "method": "CooldownPeriod", "stop_duration_candles": 2 } -], +] ``` !!! Note @@ -132,46 +131,6 @@ The below example assumes a timeframe of 1 hour: * Locks all pairs that had 4 Trades within the last 6 hours (`6 * 1h candles`) with a combined profit ratio of below 0.02 (<2%) (`LowProfitPairs`). * Locks all pairs for 2 candles that had a profit of below 0.01 (<1%) within the last 24h (`24 * 1h candles`), a minimum of 4 trades. -```json -"timeframe": "1h", -"protections": [ - { - "method": "CooldownPeriod", - "stop_duration_candles": 5 - }, - { - "method": "MaxDrawdown", - "lookback_period_candles": 48, - "trade_limit": 20, - "stop_duration_candles": 4, - "max_allowed_drawdown": 0.2 - }, - { - "method": "StoplossGuard", - "lookback_period_candles": 24, - "trade_limit": 4, - "stop_duration_candles": 2, - "only_per_pair": false - }, - { - "method": "LowProfitPairs", - "lookback_period_candles": 6, - "trade_limit": 2, - "stop_duration_candles": 60, - "required_profit": 0.02 - }, - { - "method": "LowProfitPairs", - "lookback_period_candles": 24, - "trade_limit": 4, - "stop_duration_candles": 2, - "required_profit": 0.01 - } - ], -``` - -You can use the same in your strategy, the syntax is only slightly different: - ``` python from freqtrade.strategy import IStrategy From 546ca0107178f0a95b41c433aeb3e6497c9f6a48 Mon Sep 17 00:00:00 2001 From: Rik Helsen Date: Thu, 17 Jun 2021 20:33:21 +0200 Subject: [PATCH 035/834] :recycle: Fixed flake8 warning --- freqtrade/optimize/hyperopt_tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 92ec6f194..742db07cc 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -114,7 +114,8 @@ class HyperoptTools(): if len(space_non_optimized) > 0: for non_optimized_param in space_non_optimized: if non_optimized_param not in all_space_params: - all_space_params[non_optimized_param] = space_non_optimized[non_optimized_param] + all_space_params[non_optimized_param] = \ + space_non_optimized[non_optimized_param] if space in ['buy', 'sell']: result_dict.setdefault('params', {}).update(all_space_params) From 15678045096f75b26449dcb964c9d579654e41ea Mon Sep 17 00:00:00 2001 From: Rik Helsen Date: Thu, 17 Jun 2021 22:41:49 +0200 Subject: [PATCH 036/834] :zap: kwargs merge dictionaries instead of using loops --- freqtrade/optimize/hyperopt_tools.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 742db07cc..dac299dc6 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -110,12 +110,9 @@ class HyperoptTools(): space_non_optimized = HyperoptTools._space_params(non_optimized, space) all_space_params = space_params - # Include non optimized params if there are any + # Merge non optimized params if there are any if len(space_non_optimized) > 0: - for non_optimized_param in space_non_optimized: - if non_optimized_param not in all_space_params: - all_space_params[non_optimized_param] = \ - space_non_optimized[non_optimized_param] + all_space_params = {**space_non_optimized, **space_params} if space in ['buy', 'sell']: result_dict.setdefault('params', {}).update(all_space_params) From 0a1e15988f9149670a8c7df2f77ad2a04a20e046 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Fri, 18 Jun 2021 09:48:59 +0200 Subject: [PATCH 037/834] Fix errors during ubuntu install Encountering the python header error on a fresh ubuntu install: ``` utils_find_1st/find_1st.cpp:3:10: fatal error: Python.h: No such file or directory #include "Python.h" ^~~~~~~~~~ compilation terminated. ``` solved by installing python3.7-dev. Also need to ensure python3.7-venv for fresh install. --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index c19965a18..25994fdc0 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -60,7 +60,7 @@ OS Specific steps are listed first, the [Common](#common) section below is neces sudo apt-get update # install packages - sudo apt install -y python3-pip python3-venv python3-pandas git + sudo apt install -y python3-pip python3.7-venv python3.7-dev python3-pandas git ``` === "RaspberryPi/Raspbian" From e1010ff5923e4c68900c6e786bb3e6138ea1265c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 17 Jun 2021 21:01:22 +0200 Subject: [PATCH 038/834] Don't load protections from config if strategy defines a property --- freqtrade/freqtradebot.py | 2 +- freqtrade/optimize/backtesting.py | 2 +- freqtrade/plugins/protectionmanager.py | 4 ++-- freqtrade/resolvers/strategy_resolver.py | 4 +++- freqtrade/strategy/interface.py | 2 +- tests/plugins/test_protections.py | 3 +-- 6 files changed, 9 insertions(+), 8 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a2e7fcb5d..e8a321e94 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -70,7 +70,7 @@ class FreqtradeBot(LoggingMixin): PairLocks.timeframe = self.config['timeframe'] - self.protections = ProtectionManager(self.config) + self.protections = ProtectionManager(self.config, self.strategy.protections) # RPC runs in separate threads, can start handling external commands just after # initialization, even before Freqtradebot has a chance to start its throttling, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 028a9eacd..8b75fe438 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -137,7 +137,7 @@ class Backtesting: if hasattr(strategy, 'protections'): conf = deepcopy(conf) conf['protections'] = strategy.protections - self.protections = ProtectionManager(conf) + self.protections = ProtectionManager(self.config, strategy.protections) def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]: """ diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index a8edd4e4b..f33e5b4bc 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -15,11 +15,11 @@ logger = logging.getLogger(__name__) class ProtectionManager(): - def __init__(self, config: dict) -> None: + def __init__(self, config: Dict, protections: List) -> None: self._config = config self._protection_handlers: List[IProtection] = [] - for protection_handler_config in self._config.get('protections', []): + for protection_handler_config in protections: protection_handler = ProtectionResolver.load_protection( protection_handler_config['method'], config=config, diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 6484f900b..e76d1e3e5 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -113,7 +113,9 @@ class StrategyResolver(IResolver): - Strategy - default (if not None) """ - if attribute in config: + if (attribute in config + and not isinstance(getattr(type(strategy), 'my_property', None), property)): + # Ensure Properties are not overwritten setattr(strategy, attribute, config[attribute]) logger.info("Override strategy '%s' with value in config file: %s.", attribute, config[attribute]) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 6358c6a4e..b259a7977 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -107,7 +107,7 @@ class IStrategy(ABC, HyperStrategyMixin): startup_candle_count: int = 0 # Protections - protections: List + protections: List = [] # Class level variables (intentional) containing # the dataprovider (dp) (access to other candles, historic data, ...) diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 10ab64690..9ec47dade 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -70,8 +70,7 @@ def test_protectionmanager(mocker, default_conf): ]) def test_protections_init(mocker, default_conf, timeframe, expected, protconf): default_conf['timeframe'] = timeframe - default_conf['protections'] = protconf - man = ProtectionManager(default_conf) + man = ProtectionManager(default_conf, protconf) assert len(man._protection_handlers) == len(protconf) assert man._protection_handlers[0]._lookback_period == expected[0] assert man._protection_handlers[0]._stop_duration == expected[1] From 6e89fbd14665f4aa5473af33a7cd1b64ec0f0e0d Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 18 Jun 2021 21:06:58 +0200 Subject: [PATCH 039/834] Remove Dockerfile.aarch64 it's identical to the real image except for the "--platform" tag, which is unnecessary if building from a arm64 architecture --- docker/Dockerfile.aarch64 | 58 --------------------------------------- docs/docker_quickstart.md | 2 +- 2 files changed, 1 insertion(+), 59 deletions(-) delete mode 100644 docker/Dockerfile.aarch64 diff --git a/docker/Dockerfile.aarch64 b/docker/Dockerfile.aarch64 deleted file mode 100644 index e5d3f0ee9..000000000 --- a/docker/Dockerfile.aarch64 +++ /dev/null @@ -1,58 +0,0 @@ -FROM --platform=linux/arm64/v8 python:3.9.4-slim-buster as base - -# Setup env -ENV LANG C.UTF-8 -ENV LC_ALL C.UTF-8 -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONFAULTHANDLER 1 -ENV PATH=/home/ftuser/.local/bin:$PATH -ENV FT_APP_ENV="docker" - -# Prepare environment -RUN mkdir /freqtrade \ - && apt-get update \ - && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-serial-dev \ - && apt-get clean \ - && useradd -u 1000 -G sudo -U -m ftuser \ - && chown ftuser:ftuser /freqtrade \ - # Allow sudoers - && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers - -WORKDIR /freqtrade - -# Install dependencies -FROM base as python-deps -RUN apt-get update \ - && apt-get -y install build-essential libssl-dev git libffi-dev libgfortran5 pkg-config cmake gcc \ - && apt-get clean \ - && pip install --upgrade pip - -# Install TA-lib -COPY build_helpers/* /tmp/ -RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib* -ENV LD_LIBRARY_PATH /usr/local/lib - -# Install dependencies -COPY --chown=ftuser:ftuser requirements.txt requirements-hyperopt.txt /freqtrade/ -USER ftuser -RUN pip install --user --no-cache-dir numpy \ - && pip install --user --no-cache-dir -r requirements-hyperopt.txt - -# Copy dependencies to runtime-image -FROM base as runtime-image -COPY --from=python-deps /usr/local/lib /usr/local/lib -ENV LD_LIBRARY_PATH /usr/local/lib - -COPY --from=python-deps --chown=ftuser:ftuser /home/ftuser/.local /home/ftuser/.local - -USER ftuser -# Install and execute -COPY --chown=ftuser:ftuser . /freqtrade/ - -RUN pip install -e . --user --no-cache-dir --no-build-isolation\ - && mkdir /freqtrade/user_data/ \ - && freqtrade install-ui - -ENTRYPOINT ["freqtrade"] -# Default to trade mode -CMD [ "trade" ] diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index 3a85aa885..cb66fc7e2 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -98,7 +98,7 @@ Create a new directory and place the [docker-compose file](https://raw.githubuse image: freqtradeorg/freqtrade:custom_arm64 build: context: . - dockerfile: "./docker/Dockerfile.aarch64" + dockerfile: "Dockerfile" ``` The above snippet creates a new directory called `ft_userdata`, downloads the latest compose file and pulls the freqtrade image. From 656bebd4da833dc008b3487ebbe4d7cb134c1d64 Mon Sep 17 00:00:00 2001 From: Rik Helsen Date: Fri, 18 Jun 2021 22:03:04 +0200 Subject: [PATCH 040/834] :beetle: Included completely non_optimized spaces in json + swapped merge dictionary order --- freqtrade/optimize/hyperopt_tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index dac299dc6..9eee42a8d 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -105,14 +105,14 @@ class HyperoptTools(): @staticmethod def _params_update_for_json(result_dict, params, non_optimized, space: str) -> None: - if space in params: + if (space in params) or (space in non_optimized): space_params = HyperoptTools._space_params(params, space) space_non_optimized = HyperoptTools._space_params(non_optimized, space) all_space_params = space_params # Merge non optimized params if there are any if len(space_non_optimized) > 0: - all_space_params = {**space_non_optimized, **space_params} + all_space_params = {**space_params, **space_non_optimized} if space in ['buy', 'sell']: result_dict.setdefault('params', {}).update(all_space_params) From 39b876e37a674384fb9e6d4fbc6a777ddec38acd Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 10 Jun 2021 20:09:25 +0200 Subject: [PATCH 041/834] Log exchange responses if configured --- docs/configuration.md | 1 + freqtrade/exchange/binance.py | 1 + freqtrade/exchange/exchange.py | 21 +++++++++++++++++---- freqtrade/exchange/ftx.py | 7 ++++++- freqtrade/exchange/kraken.py | 1 + tests/exchange/test_exchange.py | 4 +++- 6 files changed, 29 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 3788ef57c..8b85e9e96 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -102,6 +102,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exchange.markets_refresh_interval` | The interval in minutes in which markets are reloaded.
*Defaults to `60` minutes.*
**Datatype:** Positive Integer | `exchange.skip_pair_validation` | Skip pairlist validation on startup.
*Defaults to `false`
**Datatype:** Boolean | `exchange.skip_open_order_update` | Skips open order updates on startup should the exchange cause problems. Only relevant in live conditions.
*Defaults to `false`
**Datatype:** Boolean +| `exchange.log_responses` | Log relevant exchange responses. For debug mode only - use with care.
*Defaults to `false`
**Datatype:** Boolean | `edge.*` | Please refer to [edge configuration document](edge.md) for detailed explanation. | `experimental.block_bad_exchanges` | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
*Defaults to `true`.*
**Datatype:** Boolean | `pairlists` | Define one or more pairlists to be used. [More information](plugins.md#pairlists-and-pairlist-handlers).
*Defaults to `StaticPairList`.*
**Datatype:** List of Dicts diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 0bcfa5e17..0c470cb24 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -68,6 +68,7 @@ class Binance(Exchange): amount=amount, price=rate, params=params) logger.info('stoploss limit order added for %s. ' 'stop price: %s. limit: %s', pair, stop_price, rate) + self._log_exchange_response('create_stoploss_order', order) return order except ccxt.InsufficientFunds as e: raise InsufficientFundsError( diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 67676d4e0..07ac337fc 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -104,6 +104,7 @@ class Exchange: logger.info('Instance is running with dry_run enabled') logger.info(f"Using CCXT {ccxt.__version__}") exchange_config = config['exchange'] + self.log_responses = exchange_config.get('log_responses', False) # Deep merge ft_has with default ft_has options self._ft_has = deep_merge_dicts(self._ft_has, deepcopy(self._ft_has_default)) @@ -226,6 +227,11 @@ class Exchange: """exchange ccxt precisionMode""" return self._api.precisionMode + def _log_exchange_response(self, endpoint, response) -> None: + """ Log exchange responses """ + if self.log_responses: + logger.info(f"API {endpoint}: {response}") + def ohlcv_candle_limit(self, timeframe: str) -> int: """ Exchange ohlcv candle limit @@ -622,8 +628,10 @@ class Exchange: or self._api.options.get("createMarketBuyOrderRequiresPrice", False)) rate_for_order = self.price_to_precision(pair, rate) if needs_price else None - return self._api.create_order(pair, ordertype, side, - amount, rate_for_order, params) + order = self._api.create_order(pair, ordertype, side, + amount, rate_for_order, params) + self._log_exchange_response('create_order', order) + return order except ccxt.InsufficientFunds as e: raise InsufficientFundsError( @@ -694,7 +702,9 @@ class Exchange: if self._config['dry_run']: return self.fetch_dry_run_order(order_id) try: - return self._api.fetch_order(order_id, pair) + order = self._api.fetch_order(order_id, pair) + self._log_exchange_response('fetch_order', order) + return order except ccxt.OrderNotFound as e: raise RetryableOrderError( f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e @@ -744,7 +754,9 @@ class Exchange: return {} try: - return self._api.cancel_order(order_id, pair) + order = self._api.cancel_order(order_id, pair) + self._log_exchange_response('cancel_order', order) + return order except ccxt.InvalidOrder as e: raise InvalidOrderException( f'Could not cancel order. Message: {e}') from e @@ -1042,6 +1054,7 @@ class Exchange: pair, int((since.replace(tzinfo=timezone.utc).timestamp() - 5) * 1000)) matched_trades = [trade for trade in my_trades if trade['order'] == order_id] + self._log_exchange_response('get_trades_for_order', matched_trades) return matched_trades except ccxt.DDoSProtection as e: raise DDosProtection(e) from e diff --git a/freqtrade/exchange/ftx.py b/freqtrade/exchange/ftx.py index 3184c2524..6cd549d60 100644 --- a/freqtrade/exchange/ftx.py +++ b/freqtrade/exchange/ftx.py @@ -69,6 +69,7 @@ class Ftx(Exchange): order = self._api.create_order(symbol=pair, type=ordertype, side='sell', amount=amount, params=params) + self._log_exchange_response('create_stoploss_order', order) logger.info('stoploss order added for %s. ' 'stop price: %s.', pair, stop_price) return order @@ -99,12 +100,14 @@ class Ftx(Exchange): orders = self._api.fetch_orders(pair, None, params={'type': 'stop'}) order = [order for order in orders if order['id'] == order_id] + self._log_exchange_response('fetch_stoploss_order', order) if len(order) == 1: if order[0].get('status') == 'closed': # Trigger order was triggered ... real_order_id = order[0].get('info', {}).get('orderId') order1 = self._api.fetch_order(real_order_id, pair) + self._log_exchange_response('fetch_stoploss_order1', order1) # Fake type to stop - as this was really a stop order. order1['id_stop'] = order1['id'] order1['id'] = order_id @@ -131,7 +134,9 @@ class Ftx(Exchange): if self._config['dry_run']: return {} try: - return self._api.cancel_order(order_id, pair, params={'type': 'stop'}) + order = self._api.cancel_order(order_id, pair, params={'type': 'stop'}) + self._log_exchange_response('cancel_stoploss_order', order) + return order except ccxt.InvalidOrder as e: raise InvalidOrderException( f'Could not cancel order. Message: {e}') from e diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index 6f1fa409a..8f7cbe590 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -103,6 +103,7 @@ class Kraken(Exchange): order = self._api.create_order(symbol=pair, type=ordertype, side='sell', amount=amount, price=stop_price, params=params) + self._log_exchange_response('create_stoploss_order', order) logger.info('stoploss order added for %s. ' 'stop price: %s.', pair, stop_price) return order diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 5fa94e6c1..f5becc274 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -2271,8 +2271,9 @@ def test_cancel_stoploss_order_with_result(default_conf, mocker, exchange_name): @pytest.mark.parametrize("exchange_name", EXCHANGES) -def test_fetch_order(default_conf, mocker, exchange_name): +def test_fetch_order(default_conf, mocker, exchange_name, caplog): default_conf['dry_run'] = True + default_conf['exchange']['log_responses'] = True order = MagicMock() order.myid = 123 exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) @@ -2287,6 +2288,7 @@ def test_fetch_order(default_conf, mocker, exchange_name): api_mock.fetch_order = MagicMock(return_value=456) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) assert exchange.fetch_order('X', 'TKN/BTC') == 456 + assert log_has("API fetch_order: 456", caplog) with pytest.raises(InvalidOrderException): api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) From 6e99e3fbbbb903b8e91cb5373b55e9309f151c56 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 19 Jun 2021 09:31:34 +0200 Subject: [PATCH 042/834] Implement tests for message updating --- freqtrade/rpc/telegram.py | 33 +++++++++++++-------------------- tests/rpc/test_rpc_telegram.py | 30 ++++++++++++++++++++++++++---- 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 6a0e98a75..6cb48aef1 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1088,27 +1088,20 @@ class Telegram(RPCHandler): message_id = query.message.message_id try: - try: - self._updater.bot.edit_message_text( - chat_id=chat_id, - message_id=message_id, - text=msg, - parse_mode=parse_mode, - reply_markup=reply_markup - ) - except BadRequest as e: - if 'not modified' in e.message.lower(): - pass - else: - logger.warning( - 'TelegramError: %s', - e.message - ) - except TelegramError as telegram_err: - logger.warning( - 'TelegramError: %s! Giving up on that message.', - telegram_err.message + self._updater.bot.edit_message_text( + chat_id=chat_id, + message_id=message_id, + text=msg, + parse_mode=parse_mode, + reply_markup=reply_markup ) + except BadRequest as e: + if 'not modified' in e.message.lower(): + pass + else: + logger.warning('TelegramError: %s', e.message) + except TelegramError as telegram_err: + logger.warning('TelegramError: %s! Giving up on that message.', telegram_err.message) def _send_msg(self, msg: str, parse_mode: str = ParseMode.MARKDOWN, disable_notification: bool = False, diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 830ef200e..39ef6a1ab 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -13,7 +13,7 @@ from unittest.mock import ANY, MagicMock import arrow import pytest from telegram import Chat, Message, ReplyKeyboardMarkup, Update -from telegram.error import NetworkError +from telegram.error import BadRequest, NetworkError, TelegramError from freqtrade import __version__ from freqtrade.constants import CANCEL_REASON @@ -25,8 +25,8 @@ from freqtrade.loggers import setup_logging from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc import RPC from freqtrade.rpc.telegram import Telegram, authorized_only -from tests.conftest import (create_mock_trades, get_patched_freqtradebot, log_has, patch_exchange, - patch_get_signal, patch_whitelist) +from tests.conftest import (create_mock_trades, get_patched_freqtradebot, log_has, log_has_re, + patch_exchange, patch_get_signal, patch_whitelist) class DummyCls(Telegram): @@ -1561,7 +1561,7 @@ def test__sell_emoji(default_conf, mocker, msg, expected): assert telegram._get_sell_emoji(msg) == expected -def test__send_msg(default_conf, mocker) -> None: +def test_telegram__send_msg(default_conf, mocker, caplog) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) bot = MagicMock() telegram, _, _ = get_telegram_testobject(mocker, default_conf, mock=False) @@ -1572,6 +1572,28 @@ def test__send_msg(default_conf, mocker) -> None: telegram._send_msg('test') assert len(bot.method_calls) == 1 + # Test update + query = MagicMock() + telegram._send_msg('test', callback_path="DeadBeef", query=query, reload_able=True) + edit_message_text = telegram._updater.bot.edit_message_text + assert edit_message_text.call_count == 1 + assert "Updated: " in edit_message_text.call_args_list[0][1]['text'] + + telegram._updater.bot.edit_message_text = MagicMock(side_effect=BadRequest("not modified")) + telegram._send_msg('test', callback_path="DeadBeef", query=query) + assert telegram._updater.bot.edit_message_text.call_count == 1 + assert not log_has_re(r"TelegramError: .*", caplog) + + telegram._updater.bot.edit_message_text = MagicMock(side_effect=BadRequest("")) + telegram._send_msg('test2', callback_path="DeadBeef", query=query) + assert telegram._updater.bot.edit_message_text.call_count == 1 + assert log_has_re(r"TelegramError: .*", caplog) + + telegram._updater.bot.edit_message_text = MagicMock(side_effect=TelegramError("DeadBEEF")) + telegram._send_msg('test3', callback_path="DeadBeef", query=query) + + assert log_has_re(r"TelegramError: DeadBEEF! Giving up.*", caplog) + def test__send_msg_network_error(default_conf, mocker, caplog) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) From a7f8342171354ab361dc99554f2dbfae3f2f171d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 19 Jun 2021 16:49:54 +0200 Subject: [PATCH 043/834] Add small documentation about reload disabling --- config_full.json.example | 4 +++- docs/telegram-usage.md | 2 ++ freqtrade/constants.py | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 6aeb756f3..bc9f33f96 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -178,7 +178,9 @@ "sell_fill": "on", "buy_cancel": "on", "sell_cancel": "on" - } + }, + "reload": true, + "balance_dust_level": 0.01 }, "api_server": { "enabled": false, diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 87ff38881..f5d9744b4 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -95,6 +95,7 @@ Example configuration showing the different settings: "buy_fill": "off", "sell_fill": "off" }, + "reload": true, "balance_dust_level": 0.01 }, ``` @@ -105,6 +106,7 @@ Example configuration showing the different settings: `balance_dust_level` will define what the `/balance` command takes as "dust" - Currencies with a balance below this will be shown. +`reload` allows you to disable reload-buttons on selected messages. ## Create a custom keyboard (command shortcut buttons) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 259aa0e03..013e9df41 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -275,7 +275,8 @@ CONF_SCHEMA = { 'default': 'off' }, } - } + }, + 'reload': {'type': 'boolean'}, }, 'required': ['enabled', 'token', 'chat_id'], }, From 96fbb226c5783fb9e50c37bc3a0f0593101ebb4c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 19 Jun 2021 19:32:29 +0200 Subject: [PATCH 044/834] Implement better strategy checks part of #2696 --- freqtrade/strategy/interface.py | 19 +++++++++++++------ tests/strategy/test_interface.py | 14 ++++++++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index b259a7977..65e27a2c2 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -453,18 +453,25 @@ class IStrategy(ABC, HyperStrategyMixin): """ Ensure dataframe (length, last candle) was not modified, and has all elements we need. """ + message_template = "Dataframe returned from strategy has mismatching {}." message = "" - if df_len != len(dataframe): - message = "length" + if dataframe is None: + message = "No dataframe returned (return statement missing?)." + elif 'buy' not in dataframe: + message = "Buy column not set." + elif 'sell' not in dataframe: + message = "Sell column not set." + elif df_len != len(dataframe): + message = message_template.format("length") elif df_close != dataframe["close"].iloc[-1]: - message = "last close price" + message = message_template.format("last close price") elif df_date != dataframe["date"].iloc[-1]: - message = "last date" + message = message_template.format("last date") if message: if self.disable_dataframe_checks: - logger.warning(f"Dataframe returned from strategy has mismatching {message}.") + logger.warning(message) else: - raise StrategyError(f"Dataframe returned from strategy has mismatching {message}.") + raise StrategyError(message) def get_signal(self, pair: str, timeframe: str, dataframe: DataFrame) -> Tuple[bool, bool]: """ diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 64081fa37..04d12a51f 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -153,6 +153,8 @@ def test_assert_df_raise(mocker, caplog, ohlcv_history): def test_assert_df(ohlcv_history, caplog): df_len = len(ohlcv_history) - 1 + ohlcv_history.loc[:, 'buy'] = 0 + ohlcv_history.loc[:, 'sell'] = 0 # Ensure it's running when passed correctly _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history), ohlcv_history.loc[df_len, 'close'], ohlcv_history.loc[df_len, 'date']) @@ -170,6 +172,18 @@ def test_assert_df(ohlcv_history, caplog): match=r"Dataframe returned from strategy.*last date\."): _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history), ohlcv_history.loc[df_len, 'close'], ohlcv_history.loc[0, 'date']) + with pytest.raises(StrategyError, + match=r"No dataframe returned \(return statement missing\?\)."): + _STRATEGY.assert_df(None, len(ohlcv_history), + ohlcv_history.loc[df_len, 'close'], ohlcv_history.loc[0, 'date']) + with pytest.raises(StrategyError, + match="Buy column not set"): + _STRATEGY.assert_df(ohlcv_history.drop('buy', axis=1), len(ohlcv_history), + ohlcv_history.loc[df_len, 'close'], ohlcv_history.loc[0, 'date']) + with pytest.raises(StrategyError, + match="Sell column not set"): + _STRATEGY.assert_df(ohlcv_history.drop('sell', axis=1), len(ohlcv_history), + ohlcv_history.loc[df_len, 'close'], ohlcv_history.loc[0, 'date']) _STRATEGY.disable_dataframe_checks = True caplog.clear() From 122943d835e56a5aad6be3e1e172b351174e428d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 19 Jun 2021 19:37:27 +0200 Subject: [PATCH 045/834] Don't run filter again for pairlist generator The generator implicitly runs filter - so it should not be ran again as that would void generator caching. closes #5103 --- freqtrade/plugins/pairlistmanager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index d1cdd2c5b..03f4760b8 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -83,7 +83,8 @@ class PairListManager(): pairlist = self._pairlist_handlers[0].gen_pairlist(tickers) # Process all Pairlist Handlers in the chain - for pairlist_handler in self._pairlist_handlers: + # except for the first one, which is the generator. + for pairlist_handler in self._pairlist_handlers[1:]: pairlist = pairlist_handler.filter_pairlist(pairlist, tickers) # Validation against blacklist happens after the chain of Pairlist Handlers From 347eceeda5b675474f0294a6db99bc660909995f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 19 Jun 2021 20:30:40 +0200 Subject: [PATCH 046/834] Try fix fluky test --- tests/plugins/test_pairlist.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 5e2274ce3..ae8f6e958 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -75,7 +75,7 @@ def whitelist_conf_agefilter(default_conf): "method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", - "refresh_period": 0, + "refresh_period": -1, }, { "method": "AgeFilter", @@ -687,7 +687,6 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o freqtrade.pairlists.refresh_pairlist() assert len(freqtrade.pairlists.whitelist) == 3 assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0 - # freqtrade.config['exchange']['pair_whitelist'].append('HOT/BTC') previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count freqtrade.pairlists.refresh_pairlist() From 7f434c041389ec7edf2137db2f4172be8ad1d36b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 20 Jun 2021 09:37:32 +0200 Subject: [PATCH 047/834] Simplify mkdocs jquery inclusion by using overrides instead of partials --- docs/overrides/main.html | 10 ++++++ docs/partials/header.html | 72 --------------------------------------- mkdocs.yml | 2 +- 3 files changed, 11 insertions(+), 73 deletions(-) create mode 100644 docs/overrides/main.html delete mode 100644 docs/partials/header.html diff --git a/docs/overrides/main.html b/docs/overrides/main.html new file mode 100644 index 000000000..5b116de4b --- /dev/null +++ b/docs/overrides/main.html @@ -0,0 +1,10 @@ +{% extends "base.html" %} + +{% block footer %} + {{ super() }} + + + + +{% endblock %} diff --git a/docs/partials/header.html b/docs/partials/header.html deleted file mode 100644 index 22132bc96..000000000 --- a/docs/partials/header.html +++ /dev/null @@ -1,72 +0,0 @@ -{#- -This file was automatically generated - do not edit --#} -{% set site_url = config.site_url | d(nav.homepage.url, true) | url %} -{% if not config.use_directory_urls and site_url[0] == site_url[-1] == "." %} -{% set site_url = site_url ~ "/index.html" %} -{% endif %} -
- - - - -
diff --git a/mkdocs.yml b/mkdocs.yml index cc5747225..e3e1ade86 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -42,7 +42,7 @@ theme: name: material logo: 'images/logo.png' favicon: 'images/logo.png' - custom_dir: 'docs' + custom_dir: 'docs/overrides' palette: primary: 'blue grey' accent: 'tear' From 97351c95c0e21ca906d54e9264b93d5b01be06f3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 20 Jun 2021 10:36:18 +0200 Subject: [PATCH 048/834] Add section about GPU support #5158 #5085 #3704 #2754 --- docs/faq.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/faq.md b/docs/faq.md index e5da550fd..d015ae50e 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -136,6 +136,22 @@ On Windows, the `--logfile` option is also supported by Freqtrade and you can us > type \path\to\mylogfile.log | findstr "something" ``` +### Why does freqtrade not have GPU support? + +First of all, most indicator libraries don't have GPU support - as such, there would be little benefit for indicator calculations. +The GPU improvements would only apply to pandas-native calculations - or ones written by yourself. + +For hyperopt, freqtrade is using scikit-optimize, which is built on top of scikit-learn. +Their statement about GPU support is [pretty clear](https://scikit-learn.org/stable/faq.html#will-you-add-gpu-support). + +GPU's also are only good at crunching numbers (floating point operations). +For hyperopt, we need both number-crunching (find next parameters) and running python code (running backtesting). +As such, GPU's are not too well suited for most parts of hyperopt. + +The benefit of using GPU would therefore be pretty slim - and will not justify the complexity introduced by trying to add GPU support. + +There is however nothing preventing you from using GPU-enabled indicators within your strategy if you think you must have this - you will however probably be disappointed by the slim gain that will give you (compared to the complexity). + ## Hyperopt module ### How many epochs do I need to get a good Hyperopt result? From 17f8936f420b83901d8b78c0f7a5a7099fa07637 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 03:00:59 +0000 Subject: [PATCH 049/834] Bump scipy from 1.6.3 to 1.7.0 Bumps [scipy](https://github.com/scipy/scipy) from 1.6.3 to 1.7.0. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.6.3...v1.7.0) --- updated-dependencies: - dependency-name: scipy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 5e7e9d9d2..83e23e3ec 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -2,7 +2,7 @@ -r requirements.txt # Required for hyperopt -scipy==1.6.3 +scipy==1.7.0 scikit-learn==0.24.2 scikit-optimize==0.8.1 filelock==3.0.12 From fc7b372ce43e58aa46a5ab0cff6a71a278a69196 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 03:01:06 +0000 Subject: [PATCH 050/834] Bump ccxt from 1.51.40 to 1.51.77 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.51.40 to 1.51.77. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.51.40...1.51.77) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3d0b0256e..dab9861e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.20.3 pandas==1.2.4 -ccxt==1.51.40 +ccxt==1.51.77 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.7 aiohttp==3.7.4.post0 From eab6399490445bacc8a17fc97defa2a764fd0f8b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 03:01:17 +0000 Subject: [PATCH 051/834] Bump prompt-toolkit from 3.0.18 to 3.0.19 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.18 to 3.0.19. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.18...3.0.19) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3d0b0256e..7dee5cb01 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,4 +40,4 @@ aiofiles==0.7.0 colorama==0.4.4 # Building config files interactively questionary==1.9.0 -prompt-toolkit==3.0.18 +prompt-toolkit==3.0.19 From a6628fc65f9e0b7d0be0dbec154a47d7eadc4264 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 03:01:21 +0000 Subject: [PATCH 052/834] Bump types-requests from 0.1.11 to 0.1.13 Bumps [types-requests](https://github.com/python/typeshed) from 0.1.11 to 0.1.13. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 924b35e1a..1867c543d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -21,5 +21,5 @@ nbconvert==6.0.7 # mypy types types-cachetools==0.1.7 types-filelock==0.1.3 -types-requests==0.1.11 +types-requests==0.1.13 types-tabulate==0.1.0 From bb0ee837bc5f03a59dcc09956cab8f91e9530633 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 03:01:22 +0000 Subject: [PATCH 053/834] Bump pycoingecko from 2.1.0 to 2.2.0 Bumps [pycoingecko](https://github.com/man-c/pycoingecko) from 2.1.0 to 2.2.0. - [Release notes](https://github.com/man-c/pycoingecko/releases) - [Changelog](https://github.com/man-c/pycoingecko/blob/master/CHANGELOG.md) - [Commits](https://github.com/man-c/pycoingecko/commits) --- updated-dependencies: - dependency-name: pycoingecko dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3d0b0256e..b3419e437 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ jsonschema==3.2.0 TA-Lib==0.4.20 technical==1.3.0 tabulate==0.8.9 -pycoingecko==2.1.0 +pycoingecko==2.2.0 jinja2==3.0.1 tables==3.6.1 blosc==1.10.4 From fdc04e27a4e6a71314dad58b0001bc947aa3b305 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 04:25:59 +0000 Subject: [PATCH 054/834] Bump types-tabulate from 0.1.0 to 0.1.1 Bumps [types-tabulate](https://github.com/python/typeshed) from 0.1.0 to 0.1.1. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-tabulate dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1867c543d..927e1c813 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -22,4 +22,4 @@ nbconvert==6.0.7 types-cachetools==0.1.7 types-filelock==0.1.3 types-requests==0.1.13 -types-tabulate==0.1.0 +types-tabulate==0.1.1 From 2d05a8bea1d656f2d2f7c2261b6bd99537c8686b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 04:35:56 +0000 Subject: [PATCH 055/834] Bump types-cachetools from 0.1.7 to 0.1.8 Bumps [types-cachetools](https://github.com/python/typeshed) from 0.1.7 to 0.1.8. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-cachetools dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1867c543d..bdac6d0f4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -19,7 +19,7 @@ isort==5.8.0 nbconvert==6.0.7 # mypy types -types-cachetools==0.1.7 +types-cachetools==0.1.8 types-filelock==0.1.3 types-requests==0.1.13 types-tabulate==0.1.0 From 8c1484ed5e95ca0cef7af498e5870e886b85f6ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 06:06:08 +0000 Subject: [PATCH 056/834] Bump types-filelock from 0.1.3 to 0.1.4 Bumps [types-filelock](https://github.com/python/typeshed) from 0.1.3 to 0.1.4. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-filelock dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 328830454..30044058b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -20,6 +20,6 @@ nbconvert==6.0.7 # mypy types types-cachetools==0.1.8 -types-filelock==0.1.3 +types-filelock==0.1.4 types-requests==0.1.13 types-tabulate==0.1.1 From 0605cbb06eb26b4a879727447f2f76ea5fcd5232 Mon Sep 17 00:00:00 2001 From: Eugene Schava Date: Tue, 22 Jun 2021 12:20:12 +0300 Subject: [PATCH 057/834] make "/profit N" command output be consistent with "/daily" and "/status table" commands --- freqtrade/rpc/rpc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 2a7721af0..296793930 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -339,7 +339,10 @@ class RPC: self, stake_currency: str, fiat_display_currency: str, start_date: datetime = datetime.fromtimestamp(0)) -> Dict[str, Any]: """ Returns cumulative profit statistics """ - trades = Trade.get_trades([Trade.open_date >= start_date]).order_by(Trade.id).all() + trade_filter = \ + (Trade.is_open.is_(False) & (Trade.close_date >= start_date)) | \ + Trade.is_open.is_(True) + trades = Trade.get_trades(trade_filter).order_by(Trade.id).all() profit_all_coin = [] profit_all_ratio = [] From e97c82c51490b6c915da417c74abc1a6a484913b Mon Sep 17 00:00:00 2001 From: Eugene Schava Date: Tue, 22 Jun 2021 12:22:19 +0300 Subject: [PATCH 058/834] make "/profit N" command output be consistent with "/daily" and "/status table" commands --- freqtrade/rpc/rpc.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 296793930..b155de673 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -339,9 +339,8 @@ class RPC: self, stake_currency: str, fiat_display_currency: str, start_date: datetime = datetime.fromtimestamp(0)) -> Dict[str, Any]: """ Returns cumulative profit statistics """ - trade_filter = \ - (Trade.is_open.is_(False) & (Trade.close_date >= start_date)) | \ - Trade.is_open.is_(True) + trade_filter = ((Trade.is_open.is_(False) & (Trade.close_date >= start_date)) | + Trade.is_open.is_(True)) trades = Trade.get_trades(trade_filter).order_by(Trade.id).all() profit_all_coin = [] From 10e94350e9c9052af7ba869eb9812a1ca72a03e4 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Tue, 22 Jun 2021 14:59:43 +0200 Subject: [PATCH 059/834] Update installation.md --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 25994fdc0..5c6ac001f 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -60,7 +60,7 @@ OS Specific steps are listed first, the [Common](#common) section below is neces sudo apt-get update # install packages - sudo apt install -y python3-pip python3.7-venv python3.7-dev python3-pandas git + sudo apt install -y python3-pip python3-venv python3-dev python3-pandas git ``` === "RaspberryPi/Raspbian" From 3c70768e18dd43d6ac96790c36e7f9382d53071a Mon Sep 17 00:00:00 2001 From: Eugene Schava Date: Wed, 23 Jun 2021 07:30:08 +0300 Subject: [PATCH 060/834] make "/profit N" command output be consistent with "/daily" and "/status table" commands --- freqtrade/rpc/telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 6cb48aef1..16c9fddcc 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -482,7 +482,7 @@ class Telegram(RPCHandler): timescale = None try: if context.args: - timescale = int(context.args[0]) + timescale = int(context.args[0]) - 1 today_start = datetime.combine(date.today(), datetime.min.time()) start_date = today_start - timedelta(days=timescale) except (TypeError, ValueError, IndexError): From f7c09ba63a146bd5a24e138d1de9e19da69ffd39 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 24 Jun 2021 18:17:40 +0200 Subject: [PATCH 061/834] Log endpoint should use static rpc class --- freqtrade/rpc/api_server/api_v1.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index e907b92f0..965664028 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -162,8 +162,8 @@ def delete_lock_pair(payload: DeleteLockRequest, rpc: RPC = Depends(get_rpc)): @router.get('/logs', response_model=Logs, tags=['info']) -def logs(limit: Optional[int] = None, rpc: RPC = Depends(get_rpc)): - return rpc._rpc_get_logs(limit) +def logs(limit: Optional[int] = None): + return RPC._rpc_get_logs(limit) @router.post('/start', response_model=StatusMsg, tags=['botcontrol']) From c938edc01bada0e5fbbb3d1641615eb9ea848b06 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 24 Jun 2021 18:18:01 +0200 Subject: [PATCH 062/834] Apply dataprovider to /pair_history endpoint --- freqtrade/rpc/rpc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index b155de673..506df4cca 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -825,7 +825,10 @@ class RPC: if pair not in _data: raise RPCException(f"No data for {pair}, {timeframe} in {timerange} found.") from freqtrade.resolvers.strategy_resolver import StrategyResolver + from freqtrade.data.dataprovider import DataProvider strategy = StrategyResolver.load_strategy(config) + strategy.dp = DataProvider(config, exchange=None, pairlists=None) + df_analyzed = strategy.analyze_ticker(_data[pair], {'pair': pair}) return RPC._convert_dataframe_to_dict(strategy.get_strategy_name(), pair, timeframe, From e0d3ca6c6d7a24a0f7ba3e4beaacf0203a22001b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 24 Jun 2021 18:44:59 +0200 Subject: [PATCH 063/834] Fix import sorting --- freqtrade/rpc/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 506df4cca..8f806f555 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -824,8 +824,8 @@ class RPC: ) if pair not in _data: raise RPCException(f"No data for {pair}, {timeframe} in {timerange} found.") - from freqtrade.resolvers.strategy_resolver import StrategyResolver from freqtrade.data.dataprovider import DataProvider + from freqtrade.resolvers.strategy_resolver import StrategyResolver strategy = StrategyResolver.load_strategy(config) strategy.dp = DataProvider(config, exchange=None, pairlists=None) From 538a1acdb5b2ac5c20a57edbc10fb71f0255fd8c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 24 Jun 2021 22:53:27 +0200 Subject: [PATCH 064/834] Add Binance Broker ad to documentation page --- docs/overrides/main.html | 58 +++++++++++++++++++++++++++++++++++ docs/stylesheets/ft.extra.css | 11 +++++++ 2 files changed, 69 insertions(+) diff --git a/docs/overrides/main.html b/docs/overrides/main.html index 5b116de4b..e3d1a5d4a 100644 --- a/docs/overrides/main.html +++ b/docs/overrides/main.html @@ -1,5 +1,41 @@ {% extends "base.html" %} + + +{% block site_nav %} + + + {% if nav %} + {% if page and page.meta and page.meta.hide %} + {% set hidden = "hidden" if "navigation" in page.meta.hide %} + {% endif %} + + {% endif %} + + + {% if page.toc and not "toc.integrate" in features %} + {% if page and page.meta and page.meta.hide %} + {% set hidden = "hidden" if "toc" in page.meta.hide %} + {% endif %} + + {% endif %} +{% endblock %} + {% block footer %} {{ super() }} @@ -7,4 +43,26 @@ + + // Load binance SDK + + + {% endblock %} diff --git a/docs/stylesheets/ft.extra.css b/docs/stylesheets/ft.extra.css index 3369fa177..f7e1f48d8 100644 --- a/docs/stylesheets/ft.extra.css +++ b/docs/stylesheets/ft.extra.css @@ -11,3 +11,14 @@ .rst-versions .rst-other-versions { color: white; } + + +#widget-wrapper { + height: calc(220px * 0.5625 + 18px); + width: 220px; + margin: 0 auto 16px auto; + border-style: solid; + border-color: var(--md-code-bg-color); + border-width: 1px; + border-radius: 5px; +} From f585ffa264e1a44e193b6e5e43bb30f4e55c70c9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 24 Jun 2021 22:53:46 +0200 Subject: [PATCH 065/834] Add Dark theme to Documentation --- mkdocs.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index e3e1ade86..854939ca0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -44,8 +44,18 @@ theme: favicon: 'images/logo.png' custom_dir: 'docs/overrides' palette: - primary: 'blue grey' - accent: 'tear' + - scheme: default + primary: 'blue grey' + accent: 'tear' + toggle: + icon: material/toggle-switch-off-outline + name: Switch to dark mode + - scheme: slate + primary: 'blue grey' + accent: 'tear' + toggle: + icon: material/toggle-switch-off-outline + name: Switch to dark mode extra_css: - 'stylesheets/ft.extra.css' extra_javascript: From 2ade3ec7b9b788bdbfd069a8127961388a07c476 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 24 Jun 2021 23:09:06 +0200 Subject: [PATCH 066/834] Add max-width query to hide on small screens --- docs/stylesheets/ft.extra.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/stylesheets/ft.extra.css b/docs/stylesheets/ft.extra.css index f7e1f48d8..8e6f9929a 100644 --- a/docs/stylesheets/ft.extra.css +++ b/docs/stylesheets/ft.extra.css @@ -22,3 +22,7 @@ border-width: 1px; border-radius: 5px; } + +@media screen and (max-width: 700px) { + #widget-wrapper { display: none; } +} From 9e91240283cc2499683574e60853a21ab0e1de69 Mon Sep 17 00:00:00 2001 From: Carlo Revelli Date: Fri, 25 Jun 2021 10:43:40 +0100 Subject: [PATCH 067/834] binance-portal --- docs/overrides/main.html | 2 +- docs/stylesheets/ft.extra.css | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/overrides/main.html b/docs/overrides/main.html index e3d1a5d4a..b2138dd7b 100644 --- a/docs/overrides/main.html +++ b/docs/overrides/main.html @@ -58,7 +58,7 @@ window.binanceBrokerPortalSdk.initBrokerSDK('#widget', { apiHost: 'https://www.binance.com', brokerId: 'R4BD3S82', - site: "site" + slideTime: 4e4, }); } catch(err) { console.log(err) diff --git a/docs/stylesheets/ft.extra.css b/docs/stylesheets/ft.extra.css index 8e6f9929a..930f2038a 100644 --- a/docs/stylesheets/ft.extra.css +++ b/docs/stylesheets/ft.extra.css @@ -23,6 +23,6 @@ border-radius: 5px; } -@media screen and (max-width: 700px) { +@media screen and (max-width: calc(76.25em + 1px)) { #widget-wrapper { display: none; } } From 69a3aee01e3d4174693977bdb7103b8234969bca Mon Sep 17 00:00:00 2001 From: Carlo Revelli Date: Fri, 25 Jun 2021 10:53:52 +0100 Subject: [PATCH 068/834] minor edits --- docs/overrides/main.html | 2 +- docs/stylesheets/ft.extra.css | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/overrides/main.html b/docs/overrides/main.html index b2138dd7b..dfc5264be 100644 --- a/docs/overrides/main.html +++ b/docs/overrides/main.html @@ -44,7 +44,7 @@ - // Load binance SDK +