diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 3a8100972..f209b9528 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -490,6 +490,8 @@ freqtrade hyperopt --config config.json --hyperopt-loss --str ``` The `-e` option will set how many evaluations hyperopt will do. Since hyperopt uses Bayesian search, running too many epochs at once may not produce greater results. Experience has shown that best results are usually not improving much after 500-1000 epochs. +The `-es` option will set ofter how many batches of evaluations with no improvements hyperopt will stop. A good value is 20-30% of the total epochs. Early stop is by default disabled (`-es=0`) + Doing multiple runs (executions) with a few 1000 epochs and different random state will most likely produce different results. The `--spaces all` option determines that all possible parameters should be optimized. Possibilities are listed below. diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index c42e46711..52beb3dfa 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -78,6 +78,7 @@ ARGS_HYPEROPT = [ "disableparamexport", "hyperopt_ignore_missing_space", "analyze_per_epoch", + "early_stop", ] ARGS_EDGE = [*ARGS_COMMON_OPTIMIZE, "stoploss_range"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index f05907808..12b5ccb5b 100755 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -262,6 +262,14 @@ AVAILABLE_CLI_OPTIONS = { metavar="INT", default=constants.HYPEROPT_EPOCH, ), + "early_stop": Arg( + "-es", + "--early-stop", + help="Early stop hyperopt if no improvement after (default: %(default)d) epochs.", + type=check_int_positive, + metavar="INT", + default=0, # 0 to disable by default + ), "spaces": Arg( "--spaces", help="Specify which parameters to hyperopt. Space-separated list.", diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 11aff879f..6cde18816 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -334,6 +334,11 @@ class Configuration: ("print_all", "Parameter --print-all detected ..."), ] self._args_to_config_loop(config, configurations) + if self.args.get("early_stop", 0) > 0: + logger.info( + f"Parameter --early-stop detected ... Will early stop hyperopt if no improvement " + f"after {self.args.get('early_stop')} epochs ..." + ) configurations = [ ("print_json", "Parameter --print-json detected ..."), diff --git a/freqtrade/optimize/hyperopt/hyperopt.py b/freqtrade/optimize/hyperopt/hyperopt.py index 3db5ec9fe..3ddee162c 100644 --- a/freqtrade/optimize/hyperopt/hyperopt.py +++ b/freqtrade/optimize/hyperopt/hyperopt.py @@ -317,6 +317,13 @@ class Hyperopt: logging_mp_handle(log_queue) gc.collect() + if ( + self.hyperopter.es_batches > 0 + and self.hyperopter.es_terminator.should_terminate(self.opt) + ): + logger.info(f"Early stopping after {(i + 1) * jobs} epochs") + break + except KeyboardInterrupt: print("User interrupted..") diff --git a/freqtrade/optimize/hyperopt/hyperopt_optimizer.py b/freqtrade/optimize/hyperopt/hyperopt_optimizer.py index 99e81e4b3..467475b6d 100644 --- a/freqtrade/optimize/hyperopt/hyperopt_optimizer.py +++ b/freqtrade/optimize/hyperopt/hyperopt_optimizer.py @@ -14,6 +14,7 @@ import optuna from joblib import delayed, dump, load, wrap_non_picklable_objects from joblib.externals import cloudpickle from optuna.exceptions import ExperimentalWarning +from optuna.terminator import BestValueStagnationEvaluator, Terminator from pandas import DataFrame from freqtrade.constants import DATETIME_PRINT_FORMAT, Config @@ -104,6 +105,11 @@ class HyperOptimizer: self.market_change = 0.0 + self.es_epochs = config.get("early_stop", 0) + self.es_batches = self.es_epochs // config.get("hyperopt_jobs", 1) + if self.es_epochs > 0 and self.es_epochs < 0.2 * config.get("epochs", 0): + logger.warning(f"Easly stop epochs {self.es_epochs} lower than 20% of total epochs") + if HyperoptTools.has_space(self.config, "sell"): # Make sure use_exit_signal is enabled self.config["use_exit_signal"] = True @@ -424,6 +430,11 @@ class HyperOptimizer: else: sampler = o_sampler + if self.es_batches > 0: + with warnings.catch_warnings(): + warnings.filterwarnings(action="ignore", category=ExperimentalWarning) + self.es_terminator = Terminator(BestValueStagnationEvaluator(self.es_batches)) + logger.info(f"Using optuna sampler {o_sampler}.") return optuna.create_study(sampler=sampler, direction="minimize")