From 9a6a89c238629cdb8e8695dc5efff2f305076e49 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 3 Sep 2019 19:54:28 +0300 Subject: [PATCH 1/4] allow simplified hyperopt interface --- freqtrade/optimize/hyperopt_interface.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index f1f123653..2642e9de8 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -9,6 +9,15 @@ from typing import Dict, Any, Callable, List from pandas import DataFrame from skopt.space import Dimension, Integer, Real +from freqtrade import OperationalException + + +def _format_exception_message(method: str, space: str) -> str: + return (f"The '{space}' space is included into the hyperoptimization " + f"but {method}() method is not found in your " + f"custom Hyperopt class. You should either implement this " + f"method or remove the '{space}' space from hyperoptimization.") + class IHyperOpt(ABC): """ @@ -32,32 +41,32 @@ class IHyperOpt(ABC): """ @staticmethod - @abstractmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ Create a buy strategy generator. """ + raise OperationalException(_format_exception_message('buy_strategy_generator', 'buy')) @staticmethod - @abstractmethod def sell_strategy_generator(params: Dict[str, Any]) -> Callable: """ Create a sell strategy generator. """ + raise OperationalException(_format_exception_message('sell_strategy_generator', 'sell')) @staticmethod - @abstractmethod def indicator_space() -> List[Dimension]: """ Create an indicator space. """ + raise OperationalException(_format_exception_message('indicator_space', 'buy')) @staticmethod - @abstractmethod def sell_indicator_space() -> List[Dimension]: """ Create a sell indicator space. """ + raise OperationalException(_format_exception_message('sell_indicator_space', 'sell')) @staticmethod def generate_roi_table(params: Dict) -> Dict[int, float]: From 3343b34725b6c0cbf3c79fbacf5f60f50606b30a Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Thu, 5 Sep 2019 00:38:15 +0300 Subject: [PATCH 2/4] Add tests for simplified hyperopt interface --- freqtrade/tests/optimize/test_hyperopt.py | 268 ++++++++++++++++++++++ 1 file changed, 268 insertions(+) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 9583de510..fd79a38f2 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -693,3 +693,271 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> assert dumper.called # Should be called twice, once for tickerdata, once to save evaluations assert dumper.call_count == 2 + + +def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) -> None: + dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch( + 'freqtrade.optimize.hyperopt.get_timeframe', + MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) + ) + + parallel = mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', + MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}}]) + ) + patch_exchange(mocker) + + default_conf.update({'config': 'config.json.example', + 'epochs': 1, + 'timerange': None, + 'spaces': 'roi stoploss', + 'hyperopt_jobs': 1, }) + + hyperopt = Hyperopt(default_conf) + hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) + + del hyperopt.custom_hyperopt.__class__.buy_strategy_generator + del hyperopt.custom_hyperopt.__class__.sell_strategy_generator + del hyperopt.custom_hyperopt.__class__.indicator_space + del hyperopt.custom_hyperopt.__class__.sell_indicator_space + + hyperopt.start() + + parallel.assert_called_once() + + out, err = capsys.readouterr() + assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out + assert dumper.called + # Should be called twice, once for tickerdata, once to save evaluations + assert dumper.call_count == 2 + assert hasattr(hyperopt.backtesting, "advise_sell") + assert hasattr(hyperopt.backtesting, "advise_buy") + assert hasattr(hyperopt, "max_open_trades") + assert hyperopt.max_open_trades == default_conf['max_open_trades'] + assert hasattr(hyperopt, "position_stacking") + + +def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) -> None: + mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch( + 'freqtrade.optimize.hyperopt.get_timeframe', + MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) + ) + + patch_exchange(mocker) + + default_conf.update({'config': 'config.json.example', + 'epochs': 1, + 'timerange': None, + 'spaces': 'all', + 'hyperopt_jobs': 1, }) + + hyperopt = Hyperopt(default_conf) + hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) + + del hyperopt.custom_hyperopt.__class__.buy_strategy_generator + del hyperopt.custom_hyperopt.__class__.sell_strategy_generator + del hyperopt.custom_hyperopt.__class__.indicator_space + del hyperopt.custom_hyperopt.__class__.sell_indicator_space + + with pytest.raises(OperationalException, match=r"The 'buy' space is included into *"): + hyperopt.start() + + +def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None: + dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch( + 'freqtrade.optimize.hyperopt.get_timeframe', + MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) + ) + + parallel = mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', + MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}}]) + ) + patch_exchange(mocker) + + default_conf.update({'config': 'config.json.example', + 'epochs': 1, + 'timerange': None, + 'spaces': 'buy', + 'hyperopt_jobs': 1, }) + + hyperopt = Hyperopt(default_conf) + hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) + + # TODO: sell_strategy_generator() is actually not called because + # run_optimizer_parallel() is mocked + del hyperopt.custom_hyperopt.__class__.sell_strategy_generator + del hyperopt.custom_hyperopt.__class__.sell_indicator_space + + hyperopt.start() + + parallel.assert_called_once() + + out, err = capsys.readouterr() + assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out + assert dumper.called + # Should be called twice, once for tickerdata, once to save evaluations + assert dumper.call_count == 2 + assert hasattr(hyperopt.backtesting, "advise_sell") + assert hasattr(hyperopt.backtesting, "advise_buy") + assert hasattr(hyperopt, "max_open_trades") + assert hyperopt.max_open_trades == default_conf['max_open_trades'] + assert hasattr(hyperopt, "position_stacking") + + +def test_simplified_interface_buy_failed_1(mocker, default_conf, caplog, capsys) -> None: + mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch( + 'freqtrade.optimize.hyperopt.get_timeframe', + MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) + ) + + patch_exchange(mocker) + + default_conf.update({'config': 'config.json.example', + 'epochs': 1, + 'timerange': None, + 'spaces': 'buy', + 'hyperopt_jobs': 1, }) + + hyperopt = Hyperopt(default_conf) + hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) + + del hyperopt.custom_hyperopt.__class__.buy_strategy_generator + + with pytest.raises(OperationalException, match=r"The 'buy' space is included into *"): + hyperopt.start() + + +def test_simplified_interface_buy_failed_2(mocker, default_conf, caplog, capsys) -> None: + mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch( + 'freqtrade.optimize.hyperopt.get_timeframe', + MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) + ) + + patch_exchange(mocker) + + default_conf.update({'config': 'config.json.example', + 'epochs': 1, + 'timerange': None, + 'spaces': 'buy', + 'hyperopt_jobs': 1, }) + + hyperopt = Hyperopt(default_conf) + hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) + + del hyperopt.custom_hyperopt.__class__.indicator_space + + with pytest.raises(OperationalException, match=r"The 'buy' space is included into *"): + hyperopt.start() + + +def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None: + dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch( + 'freqtrade.optimize.hyperopt.get_timeframe', + MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) + ) + + parallel = mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', + MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}}]) + ) + patch_exchange(mocker) + + default_conf.update({'config': 'config.json.example', + 'epochs': 1, + 'timerange': None, + 'spaces': 'sell', + 'hyperopt_jobs': 1, }) + + hyperopt = Hyperopt(default_conf) + hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) + + # TODO: buy_strategy_generator() is actually not called because + # run_optimizer_parallel() is mocked + del hyperopt.custom_hyperopt.__class__.buy_strategy_generator + del hyperopt.custom_hyperopt.__class__.indicator_space + + hyperopt.start() + + parallel.assert_called_once() + + out, err = capsys.readouterr() + assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out + assert dumper.called + # Should be called twice, once for tickerdata, once to save evaluations + assert dumper.call_count == 2 + assert hasattr(hyperopt.backtesting, "advise_sell") + assert hasattr(hyperopt.backtesting, "advise_buy") + assert hasattr(hyperopt, "max_open_trades") + assert hyperopt.max_open_trades == default_conf['max_open_trades'] + assert hasattr(hyperopt, "position_stacking") + + +def test_simplified_interface_sell_failed_1(mocker, default_conf, caplog, capsys) -> None: + mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch( + 'freqtrade.optimize.hyperopt.get_timeframe', + MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) + ) + + patch_exchange(mocker) + + default_conf.update({'config': 'config.json.example', + 'epochs': 1, + 'timerange': None, + 'spaces': 'sell', + 'hyperopt_jobs': 1, }) + + hyperopt = Hyperopt(default_conf) + hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) + + del hyperopt.custom_hyperopt.__class__.sell_strategy_generator + + with pytest.raises(OperationalException, match=r"The 'sell' space is included into *"): + hyperopt.start() + + +def test_simplified_interface_sell_failed_2(mocker, default_conf, caplog, capsys) -> None: + mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch( + 'freqtrade.optimize.hyperopt.get_timeframe', + MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) + ) + + patch_exchange(mocker) + + default_conf.update({'config': 'config.json.example', + 'epochs': 1, + 'timerange': None, + 'spaces': 'sell', + 'hyperopt_jobs': 1, }) + + hyperopt = Hyperopt(default_conf) + hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) + + del hyperopt.custom_hyperopt.__class__.sell_indicator_space + + with pytest.raises(OperationalException, match=r"The 'sell' space is included into *"): + hyperopt.start() From 7e56704767932fa1ac1ccb4a505ee3cc73568ff1 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 6 Sep 2019 15:08:44 +0300 Subject: [PATCH 3/4] Parametrize tests for hyperopt simplified failed --- freqtrade/tests/optimize/test_hyperopt.py | 92 +++-------------------- 1 file changed, 10 insertions(+), 82 deletions(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index fd79a38f2..6fba59be9 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -814,58 +814,6 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None: assert hasattr(hyperopt, "position_stacking") -def test_simplified_interface_buy_failed_1(mocker, default_conf, caplog, capsys) -> None: - mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', - MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) - ) - - patch_exchange(mocker) - - default_conf.update({'config': 'config.json.example', - 'epochs': 1, - 'timerange': None, - 'spaces': 'buy', - 'hyperopt_jobs': 1, }) - - hyperopt = Hyperopt(default_conf) - hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() - hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) - - del hyperopt.custom_hyperopt.__class__.buy_strategy_generator - - with pytest.raises(OperationalException, match=r"The 'buy' space is included into *"): - hyperopt.start() - - -def test_simplified_interface_buy_failed_2(mocker, default_conf, caplog, capsys) -> None: - mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', - MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) - ) - - patch_exchange(mocker) - - default_conf.update({'config': 'config.json.example', - 'epochs': 1, - 'timerange': None, - 'spaces': 'buy', - 'hyperopt_jobs': 1, }) - - hyperopt = Hyperopt(default_conf) - hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() - hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) - - del hyperopt.custom_hyperopt.__class__.indicator_space - - with pytest.raises(OperationalException, match=r"The 'buy' space is included into *"): - hyperopt.start() - - def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) @@ -911,7 +859,13 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None assert hasattr(hyperopt, "position_stacking") -def test_simplified_interface_sell_failed_1(mocker, default_conf, caplog, capsys) -> None: +@pytest.mark.parametrize("method,space", [ + ('buy_strategy_generator', 'buy'), + ('indicator_space', 'buy'), + ('sell_strategy_generator', 'sell'), + ('sell_indicator_space', 'sell'), +]) +def test_simplified_interface_failed(mocker, default_conf, caplog, capsys, method, space) -> None: mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch( @@ -924,40 +878,14 @@ def test_simplified_interface_sell_failed_1(mocker, default_conf, caplog, capsys default_conf.update({'config': 'config.json.example', 'epochs': 1, 'timerange': None, - 'spaces': 'sell', + 'spaces': space, 'hyperopt_jobs': 1, }) hyperopt = Hyperopt(default_conf) hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) - del hyperopt.custom_hyperopt.__class__.sell_strategy_generator + delattr(hyperopt.custom_hyperopt.__class__, method) - with pytest.raises(OperationalException, match=r"The 'sell' space is included into *"): - hyperopt.start() - - -def test_simplified_interface_sell_failed_2(mocker, default_conf, caplog, capsys) -> None: - mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mocker.patch( - 'freqtrade.optimize.hyperopt.get_timeframe', - MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) - ) - - patch_exchange(mocker) - - default_conf.update({'config': 'config.json.example', - 'epochs': 1, - 'timerange': None, - 'spaces': 'sell', - 'hyperopt_jobs': 1, }) - - hyperopt = Hyperopt(default_conf) - hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() - hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) - - del hyperopt.custom_hyperopt.__class__.sell_indicator_space - - with pytest.raises(OperationalException, match=r"The 'sell' space is included into *"): + with pytest.raises(OperationalException, match=f"The '{space}' space is included into *"): hyperopt.start() From 4fdf8a75cd6e5f4fd1a6eff0c671a907527b65a4 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 6 Sep 2019 16:09:03 +0300 Subject: [PATCH 4/4] Adjust hyperopt tests after the merge with develop --- freqtrade/tests/optimize/test_hyperopt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index a8b2f2e67..976cb3d53 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -706,7 +706,8 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', - MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}}]) + MagicMock(return_value=[{ + 'loss': 1, 'results_explanation': 'foo result', 'params': {'stoploss': 0.0}}]) ) patch_exchange(mocker)