diff --git a/docs/assets/show-config-output.png b/docs/assets/show-config-output.png new file mode 100644 index 000000000..010c5b8a4 Binary files /dev/null and b/docs/assets/show-config-output.png differ diff --git a/docs/configuration.md b/docs/configuration.md index 51c97ae80..f9348e497 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -49,6 +49,9 @@ FREQTRADE__EXCHANGE__SECRET= !!! Note Environment variables detected are logged at startup - so if you can't find why a value is not what you think it should be based on the configuration, make sure it's not loaded from an environment variable. +!!! Tip "Validate combined result" + You can use the [show-config subcommand](utils.md#show-config) to see the final, combined configuration. + ### Multiple configuration files Multiple configuration files can be specified and used by the bot or the bot can read its configuration parameters from the process standard input stream. @@ -56,6 +59,9 @@ Multiple configuration files can be specified and used by the bot or the bot can You can specify additional configuration files in `add_config_files`. Files specified in this parameter will be loaded and merged with the initial config file. The files are resolved relative to the initial configuration file. This is similar to using multiple `--config` parameters, but simpler in usage as you don't have to specify all files for all commands. +!!! Tip "Validate combined result" + You can use the [show-config subcommand](utils.md#show-config) to see the final, combined configuration. + !!! Tip "Use multiple configuration files to keep secrets secret" You can use a 2nd configuration file containing your secrets. That way you can share your "primary" configuration file, while still keeping your API keys for yourself. The 2nd file should only specify what you intend to override. diff --git a/docs/utils.md b/docs/utils.md index ea52737d6..a11e7b698 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -66,6 +66,53 @@ $ freqtrade new-config --config user_data/config_binance.json ? Do you want to enable Telegram? No ``` +## Show config + +Show configuration file (with sensitive values redacted by default). +Especially useful with [split configuration files](configuration.md#multiple-configuration-files) or [environment variables](configuration.md#environment-variables), where this command will show the merged configuration. + +![Show config output](assets/show-config-output.png) + +``` +usage: freqtrade show-config [-h] [--userdir PATH] [-c PATH] + [--show-sensitive] + +options: + -h, --help show this help message and exit + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + -c PATH, --config PATH + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. + --show-sensitive Show secrets in the output. +``` + +``` output +Your combined configuration is: +{ + "exit_pricing": { + "price_side": "other", + "use_order_book": true, + "order_book_top": 1 + }, + "stake_currency": "USDT", + "exchange": { + "name": "binance", + "key": "REDACTED", + "secret": "REDACTED", + "ccxt_config": {}, + "ccxt_async_config": {}, + } + // ... +} +``` + +!!! Warning "Sharing information provided by this command" + We try to remove all known sensitive information from the default output (without `--show-sensitive`). + Yet, please do double-check for sensitive values in your output to make sure you're not accidentally exposing some private info. + ## Create new strategy Creates a new strategy from a template similar to SampleStrategy. diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 98e7cb084..e0fa0fd51 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -8,7 +8,7 @@ Note: Be careful with file-scoped imports in these subfiles. """ from freqtrade.commands.analyze_commands import start_analysis_entries_exits from freqtrade.commands.arguments import Arguments -from freqtrade.commands.build_config_commands import start_new_config +from freqtrade.commands.build_config_commands import start_new_config, start_show_config from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades, start_download_data, start_list_data) from freqtrade.commands.db_commands import start_convert_db diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 191f07910..55c3aa586 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -62,6 +62,7 @@ ARGS_TEST_PAIRLIST = ["user_data_dir", "verbosity", "config", "quote_currencies" ARGS_CREATE_USERDIR = ["user_data_dir", "reset"] ARGS_BUILD_CONFIG = ["config"] +ARGS_SHOW_CONFIG = ["user_data_dir", "config", "show_sensitive"] ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] @@ -209,9 +210,9 @@ class Arguments: start_list_strategies, start_list_timeframes, start_lookahead_analysis, start_new_config, start_new_strategy, start_plot_dataframe, start_plot_profit, - start_recursive_analysis, start_show_trades, - start_strategy_update, start_test_pairlist, start_trading, - start_webserver) + start_recursive_analysis, start_show_config, + start_show_trades, start_strategy_update, + start_test_pairlist, start_trading, start_webserver) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added @@ -244,6 +245,14 @@ class Arguments: build_config_cmd.set_defaults(func=start_new_config) self._build_args(optionlist=ARGS_BUILD_CONFIG, parser=build_config_cmd) + # add show-config subcommand + show_config_cmd = subparsers.add_parser( + 'show-config', + help="Show resolved config", + ) + show_config_cmd.set_defaults(func=start_show_config) + self._build_args(optionlist=ARGS_SHOW_CONFIG, parser=show_config_cmd) + # add new-strategy subcommand build_strategy_cmd = subparsers.add_parser( 'new-strategy', diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 5b3447d70..63862c3bf 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -5,9 +5,12 @@ from typing import Any, Dict, List from questionary import Separator, prompt +from freqtrade.configuration import sanitize_config +from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.configuration.detect_environment import running_in_docker from freqtrade.configuration.directory_operations import chown_user_directory from freqtrade.constants import UNLIMITED_STAKE_AMOUNT +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS, available_exchanges from freqtrade.util import render_template @@ -264,3 +267,19 @@ def start_new_config(args: Dict[str, Any]) -> None: "Please delete it or use a different configuration file name.") selections = ask_user_config() deploy_new_config(config_path, selections) + + +def start_show_config(args: Dict[str, Any]) -> None: + + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE, set_dry=False) + + # TODO: Sanitize from sensitive info before printing + + print("Your combined configuration is:") + config_sanitized = sanitize_config( + config['original_config'], + show_sensitive=args.get('show_sensitive', False) + ) + + from rich import print_json + print_json(data=config_sanitized) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 4d2dcb1fe..287ab2594 100755 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -716,4 +716,10 @@ AVAILABLE_CLI_OPTIONS = { help='Specify startup candles to be checked (`199`, `499`, `999`, `1999`).', nargs='+', ), + "show_sensitive": Arg( + '--show-sensitive', + help='Show secrets in the output.', + action='store_true', + default=False, + ), } diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index 05921606c..aa06a70c9 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,5 +1,6 @@ # flake8: noqa: F401 +from freqtrade.configuration.config_secrets import sanitize_config from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.configuration.config_validation import validate_config_consistency from freqtrade.configuration.configuration import Configuration diff --git a/freqtrade/configuration/config_secrets.py b/freqtrade/configuration/config_secrets.py new file mode 100644 index 000000000..47ee741bf --- /dev/null +++ b/freqtrade/configuration/config_secrets.py @@ -0,0 +1,36 @@ +from copy import deepcopy + +from freqtrade.constants import Config + + +def sanitize_config(config: Config, *, show_sensitive: bool = False) -> Config: + """ + Remove sensitive information from the config. + :param config: Configuration + :param show_sensitive: Show sensitive information + :return: Configuration + """ + if show_sensitive: + return config + keys_to_remove = [ + "exchange.key", + "exchange.secret", + "exchange.password", + "exchange.uid", + "telegram.token", + "telegram.chat_id", + "discord.webhook_url", + "api_server.password", + ] + config = deepcopy(config) + for key in keys_to_remove: + if '.' in key: + nested_keys = key.split('.') + nested_config = config + for nested_key in nested_keys[:-1]: + nested_config = nested_config.get(nested_key, {}) + nested_config[nested_keys[-1]] = 'REDACTED' + else: + config[key] = 'REDACTED' + + return config diff --git a/freqtrade/configuration/config_setup.py b/freqtrade/configuration/config_setup.py index d49bf61f6..154ba5175 100644 --- a/freqtrade/configuration/config_setup.py +++ b/freqtrade/configuration/config_setup.py @@ -10,7 +10,8 @@ from .configuration import Configuration logger = logging.getLogger(__name__) -def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]: +def setup_utils_configuration( + args: Dict[str, Any], method: RunMode, *, set_dry: bool = True) -> Dict[str, Any]: """ Prepare the configuration for utils subcommands :param args: Cli args from Arguments() @@ -21,7 +22,8 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str config = configuration.get_config() # Ensure these modes are using Dry-run - config['dry_run'] = True + if set_dry: + config['dry_run'] = True validate_config_consistency(config, preliminary=True) return config diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 58b1fdf84..0da307175 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -200,6 +200,12 @@ class Configuration: config['exportfilename'] = (config['user_data_dir'] / 'backtest_results') + if self.args.get('show_sensitive'): + logger.warning( + "Sensitive information will be shown in the upcomming output. " + "Please make sure to never share this output without redacting " + "the information yourself.") + def _process_optimize_options(self, config: Config) -> None: # This will override the strategy configuration diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 90bd1c270..789535a46 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -12,9 +12,9 @@ from freqtrade.commands import (start_backtesting_show, start_convert_data, star start_create_userdir, start_download_data, start_hyperopt_list, start_hyperopt_show, start_install_ui, start_list_data, start_list_exchanges, start_list_markets, start_list_strategies, - start_list_timeframes, start_new_strategy, start_show_trades, - start_strategy_update, start_test_pairlist, start_trading, - start_webserver) + start_list_timeframes, start_new_strategy, start_show_config, + start_show_trades, start_strategy_update, start_test_pairlist, + start_trading, start_webserver) from freqtrade.commands.db_commands import start_convert_db from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui, get_ui_download_url, read_ui_version) @@ -39,6 +39,14 @@ def test_setup_utils_configuration(): assert "exchange" in config assert config['dry_run'] is True + args = [ + 'list-exchanges', '--config', 'tests/testdata/testconfigs/testconfig.json', + ] + + config = setup_utils_configuration(get_args(args), RunMode.OTHER, set_dry=False) + assert "exchange" in config + assert config['dry_run'] is False + def test_start_trading_fail(mocker, caplog): @@ -1572,3 +1580,33 @@ def test_start_strategy_updater(mocker, tmp_path): start_strategy_update(pargs) # Number of strategies in the test directory assert sc_mock.call_count == 2 + + +def test_start_show_config(capsys, caplog): + args = [ + "show-config", + "--config", + "tests/testdata/testconfigs/main_test_config.json", + ] + pargs = get_args(args) + start_show_config(pargs) + + captured = capsys.readouterr() + assert "Your combined configuration is:" in captured.out + assert '"max_open_trades":' in captured.out + assert '"secret": "REDACTED"' in captured.out + + args = [ + "show-config", + "--config", + "tests/testdata/testconfigs/main_test_config.json", + "--show-sensitive" + ] + pargs = get_args(args) + start_show_config(pargs) + + captured = capsys.readouterr() + assert "Your combined configuration is:" in captured.out + assert '"max_open_trades":' in captured.out + assert '"secret": "REDACTED"' not in captured.out + assert log_has_re(r'Sensitive information will be shown in the upcomming output.*', caplog) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 71585cac3..124258c04 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -10,6 +10,7 @@ from jsonschema import ValidationError from freqtrade.commands import Arguments from freqtrade.configuration import Configuration, validate_config_consistency +from freqtrade.configuration.config_secrets import sanitize_config from freqtrade.configuration.config_validation import validate_config_schema from freqtrade.configuration.deprecated_settings import (check_conflicting_settings, process_deprecated_setting, @@ -1426,7 +1427,7 @@ def test_flat_vars_to_nested_dict(caplog): assert not log_has("Loading variable 'NOT_RELEVANT'", caplog) -def test_setup_hyperopt_freqai(mocker, default_conf, caplog) -> None: +def test_setup_hyperopt_freqai(mocker, default_conf) -> None: patched_configuration_load_config_file(mocker, default_conf) mocker.patch( 'freqtrade.configuration.configuration.create_datadir', @@ -1459,7 +1460,7 @@ def test_setup_hyperopt_freqai(mocker, default_conf, caplog) -> None: validate_config_consistency(config) -def test_setup_freqai_backtesting(mocker, default_conf, caplog) -> None: +def test_setup_freqai_backtesting(mocker, default_conf) -> None: patched_configuration_load_config_file(mocker, default_conf) mocker.patch( 'freqtrade.configuration.configuration.create_datadir', @@ -1506,3 +1507,17 @@ def test_setup_freqai_backtesting(mocker, default_conf, caplog) -> None: OperationalException, match=r".* pass --timerange if you intend to use FreqAI .*" ): validate_config_consistency(conf) + + +def test_sanitize_config(default_conf_usdt): + assert default_conf_usdt['exchange']['key'] != 'REDACTED' + res = sanitize_config(default_conf_usdt) + # Didn't modify original dict + assert default_conf_usdt['exchange']['key'] != 'REDACTED' + + assert res['exchange']['key'] == 'REDACTED' + assert res['exchange']['secret'] == 'REDACTED' + + res = sanitize_config(default_conf_usdt, show_sensitive=True) + assert res['exchange']['key'] == default_conf_usdt['exchange']['key'] + assert res['exchange']['secret'] == default_conf_usdt['exchange']['secret'] diff --git a/tests/testdata/testconfigs/test_base_config.json b/tests/testdata/testconfigs/test_base_config.json index d15c5890b..40a114150 100644 --- a/tests/testdata/testconfigs/test_base_config.json +++ b/tests/testdata/testconfigs/test_base_config.json @@ -1,6 +1,6 @@ { "stake_currency": "", - "dry_run": true, + "dry_run": false, "exchange": { "name": "", "key": "",