diff --git a/tests/freqai/test_freqai_backtesting.py b/tests/freqai/test_freqai_backtesting.py index 6dad158fd..e689d3927 100644 --- a/tests/freqai/test_freqai_backtesting.py +++ b/tests/freqai/test_freqai_backtesting.py @@ -29,26 +29,34 @@ def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir, c patch_exchange(mocker) now = datetime.now(timezone.utc) - mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', - PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT'])) - mocker.patch('freqtrade.optimize.backtesting.history.load_data') - mocker.patch('freqtrade.optimize.backtesting.history.get_timerange', return_value=(now, now)) + mocker.patch( + "freqtrade.plugins.pairlistmanager.PairListManager.whitelist", + PropertyMock(return_value=["HULUMULU/USDT", "XRP/USDT"]), + ) + mocker.patch("freqtrade.optimize.backtesting.history.load_data") + mocker.patch("freqtrade.optimize.backtesting.history.get_timerange", return_value=(now, now)) patched_configuration_load_config_file(mocker, freqai_conf) args = [ - 'backtesting', - '--config', 'config.json', - '--datadir', str(testdatadir), - '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'), - '--timeframe', '1m', - '--strategy-list', CURRENT_TEST_STRATEGY + "backtesting", + "--config", + "config.json", + "--datadir", + str(testdatadir), + "--strategy-path", + str(Path(__file__).parents[1] / "strategy/strats"), + "--timeframe", + "1m", + "--strategy-list", + CURRENT_TEST_STRATEGY, ] args = get_args(args) bt_config = setup_optimize_configuration(args, RunMode.BACKTEST) Backtesting(bt_config) - assert log_has_re('Using --strategy-list with FreqAI REQUIRES all strategies to have identical', - caplog) + assert log_has_re( + "Using --strategy-list with FreqAI REQUIRES all strategies to have identical", caplog + ) Backtesting.cleanup() @@ -60,23 +68,29 @@ def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir, c ("1d", 302), ], ) -def test_freqai_backtest_load_data(freqai_conf, mocker, caplog, - timeframe, expected_startup_candle_count): +def test_freqai_backtest_load_data( + freqai_conf, mocker, caplog, timeframe, expected_startup_candle_count +): patch_exchange(mocker) now = datetime.now(timezone.utc) - mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', - PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT'])) - mocker.patch('freqtrade.optimize.backtesting.history.load_data') - mocker.patch('freqtrade.optimize.backtesting.history.get_timerange', return_value=(now, now)) - freqai_conf['timeframe'] = timeframe - freqai_conf.get('freqai', {}).get('feature_parameters', {}).update({'include_timeframes': []}) + mocker.patch( + "freqtrade.plugins.pairlistmanager.PairListManager.whitelist", + PropertyMock(return_value=["HULUMULU/USDT", "XRP/USDT"]), + ) + mocker.patch("freqtrade.optimize.backtesting.history.load_data") + mocker.patch("freqtrade.optimize.backtesting.history.get_timerange", return_value=(now, now)) + freqai_conf["timeframe"] = timeframe + freqai_conf.get("freqai", {}).get("feature_parameters", {}).update({"include_timeframes": []}) backtesting = Backtesting(deepcopy(freqai_conf)) backtesting.load_bt_data() - assert log_has_re(f'Increasing startup_candle_count for freqai on {timeframe} ' - f'to {expected_startup_candle_count}', caplog) - assert history.load_data.call_args[1]['startup_candles'] == expected_startup_candle_count + assert log_has_re( + f"Increasing startup_candle_count for freqai on {timeframe} " + f"to {expected_startup_candle_count}", + caplog, + ) + assert history.load_data.call_args[1]["startup_candles"] == expected_startup_candle_count Backtesting.cleanup() @@ -85,45 +99,55 @@ def test_freqai_backtest_live_models_model_not_found(freqai_conf, mocker, testda patch_exchange(mocker) now = datetime.now(timezone.utc) - mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', - PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT'])) - mocker.patch('freqtrade.optimize.backtesting.history.load_data') - mocker.patch('freqtrade.optimize.backtesting.history.get_timerange', return_value=(now, now)) + mocker.patch( + "freqtrade.plugins.pairlistmanager.PairListManager.whitelist", + PropertyMock(return_value=["HULUMULU/USDT", "XRP/USDT"]), + ) + mocker.patch("freqtrade.optimize.backtesting.history.load_data") + mocker.patch("freqtrade.optimize.backtesting.history.get_timerange", return_value=(now, now)) freqai_conf["timerange"] = "" freqai_conf.get("freqai", {}).update({"backtest_using_historic_predictions": False}) patched_configuration_load_config_file(mocker, freqai_conf) args = [ - 'backtesting', - '--config', 'config.json', - '--datadir', str(testdatadir), - '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'), - '--timeframe', '5m', - '--freqai-backtest-live-models' + "backtesting", + "--config", + "config.json", + "--datadir", + str(testdatadir), + "--strategy-path", + str(Path(__file__).parents[1] / "strategy/strats"), + "--timeframe", + "5m", + "--freqai-backtest-live-models", ] args = get_args(args) bt_config = setup_optimize_configuration(args, RunMode.BACKTEST) - with pytest.raises(OperationalException, - match=r".* Historic predictions data is required to run backtest .*"): + with pytest.raises( + OperationalException, match=r".* Historic predictions data is required to run backtest .*" + ): Backtesting(bt_config) Backtesting.cleanup() def test_freqai_backtest_consistent_timerange(mocker, freqai_conf): - freqai_conf['runmode'] = 'backtest' - mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', - PropertyMock(return_value=['XRP/USDT:USDT'])) + freqai_conf["runmode"] = "backtest" + mocker.patch( + "freqtrade.plugins.pairlistmanager.PairListManager.whitelist", + PropertyMock(return_value=["XRP/USDT:USDT"]), + ) - gbs = mocker.patch('freqtrade.optimize.backtesting.generate_backtest_stats') + gbs = mocker.patch("freqtrade.optimize.backtesting.generate_backtest_stats") - freqai_conf['candle_type_def'] = CandleType.FUTURES - freqai_conf.get('exchange', {}).update({'pair_whitelist': ['XRP/USDT:USDT']}) - freqai_conf.get('freqai', {}).get('feature_parameters', {}).update( - {'include_timeframes': ['5m', '1h'], 'include_corr_pairlist': []}) - freqai_conf['timerange'] = '20211120-20211121' + freqai_conf["candle_type_def"] = CandleType.FUTURES + freqai_conf.get("exchange", {}).update({"pair_whitelist": ["XRP/USDT:USDT"]}) + freqai_conf.get("freqai", {}).get("feature_parameters", {}).update( + {"include_timeframes": ["5m", "1h"], "include_corr_pairlist": []} + ) + freqai_conf["timerange"] = "20211120-20211121" strategy = get_patched_freqai_strategy(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf) @@ -139,6 +163,6 @@ def test_freqai_backtest_consistent_timerange(mocker, freqai_conf): backtesting = Backtesting(deepcopy(freqai_conf)) backtesting.start() - assert gbs.call_args[1]['min_date'] == datetime(2021, 11, 20, 0, 0, tzinfo=timezone.utc) - assert gbs.call_args[1]['max_date'] == datetime(2021, 11, 21, 0, 0, tzinfo=timezone.utc) + assert gbs.call_args[1]["min_date"] == datetime(2021, 11, 20, 0, 0, tzinfo=timezone.utc) + assert gbs.call_args[1]["max_date"] == datetime(2021, 11, 21, 0, 0, tzinfo=timezone.utc) Backtesting.cleanup() diff --git a/tests/freqai/test_freqai_datadrawer.py b/tests/freqai/test_freqai_datadrawer.py index 548fad650..037691d50 100644 --- a/tests/freqai/test_freqai_datadrawer.py +++ b/tests/freqai/test_freqai_datadrawer.py @@ -1,4 +1,3 @@ - import shutil from pathlib import Path from unittest.mock import patch @@ -15,7 +14,7 @@ from tests.freqai.conftest import get_patched_freqai_strategy def test_update_historic_data(mocker, freqai_conf): - freqai_conf['runmode'] = 'backtest' + freqai_conf["runmode"] = "backtest" strategy = get_patched_freqai_strategy(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf) strategy.dp = DataProvider(freqai_conf, exchange) @@ -99,7 +98,7 @@ def test_use_strategy_to_populate_indicators(mocker, freqai_conf): sub_timerange = TimeRange.parse_timerange("20180111-20180114") corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk) - df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, 'LTC/BTC') + df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC") assert len(df.columns) == 33 shutil.rmtree(Path(freqai.dk.full_path)) @@ -133,10 +132,7 @@ def test_get_timerange_from_backtesting_live_df_pred_not_found(mocker, freqai_co exchange = get_patched_exchange(mocker, freqai_conf) strategy.dp = DataProvider(freqai_conf, exchange) freqai = strategy.freqai - with pytest.raises( - OperationalException, - match=r'Historic predictions not found.*' - ): + with pytest.raises(OperationalException, match=r"Historic predictions not found.*"): freqai.dd.get_timerange_from_live_historic_predictions() @@ -158,13 +154,10 @@ def test_set_initial_return_values(mocker, freqai_conf): start_x_plus_1 = "2023-08-30" end_x_plus_5 = "2023-09-03" - historic_data = { - 'date_pred': pd.date_range(end=end_x, periods=5), - 'value': range(1, 6) - } + historic_data = {"date_pred": pd.date_range(end=end_x, periods=5), "value": range(1, 6)} new_data = { - 'date': pd.date_range(start=start_x_plus_1, end=end_x_plus_5), - 'value': range(6, 11) + "date": pd.date_range(start=start_x_plus_1, end=end_x_plus_5), + "value": range(6, 11), } freqai.dd.historic_predictions[pair] = pd.DataFrame(historic_data) @@ -173,20 +166,21 @@ def test_set_initial_return_values(mocker, freqai_conf): dataframe = pd.DataFrame(new_data) # Action - with patch('logging.Logger.warning') as mock_logger_warning: + with patch("logging.Logger.warning") as mock_logger_warning: freqai.dd.set_initial_return_values(pair, new_pred_df, dataframe) # Assertions hist_pred_df = freqai.dd.historic_predictions[pair] model_return_df = freqai.dd.model_return_values[pair] - assert hist_pred_df['date_pred'].iloc[-1] == pd.Timestamp(end_x_plus_5) - assert 'date_pred' in hist_pred_df.columns + assert hist_pred_df["date_pred"].iloc[-1] == pd.Timestamp(end_x_plus_5) + assert "date_pred" in hist_pred_df.columns assert hist_pred_df.shape[0] == 8 # compare values in model_return_df with hist_pred_df - assert (model_return_df["value"].values == - hist_pred_df.tail(len(dataframe))["value"].values).all() + assert ( + model_return_df["value"].values == hist_pred_df.tail(len(dataframe))["value"].values + ).all() assert model_return_df.shape[0] == len(dataframe) # Ensure logger error is not called @@ -212,13 +206,10 @@ def test_set_initial_return_values_warning(mocker, freqai_conf): start_x_plus_1 = "2023-09-01" end_x_plus_5 = "2023-09-05" - historic_data = { - 'date_pred': pd.date_range(end=end_x, periods=5), - 'value': range(1, 6) - } + historic_data = {"date_pred": pd.date_range(end=end_x, periods=5), "value": range(1, 6)} new_data = { - 'date': pd.date_range(start=start_x_plus_1, end=end_x_plus_5), - 'value': range(6, 11) + "date": pd.date_range(start=start_x_plus_1, end=end_x_plus_5), + "value": range(6, 11), } freqai.dd.historic_predictions[pair] = pd.DataFrame(historic_data) @@ -227,20 +218,21 @@ def test_set_initial_return_values_warning(mocker, freqai_conf): dataframe = pd.DataFrame(new_data) # Action - with patch('logging.Logger.warning') as mock_logger_warning: + with patch("logging.Logger.warning") as mock_logger_warning: freqai.dd.set_initial_return_values(pair, new_pred_df, dataframe) # Assertions hist_pred_df = freqai.dd.historic_predictions[pair] model_return_df = freqai.dd.model_return_values[pair] - assert hist_pred_df['date_pred'].iloc[-1] == pd.Timestamp(end_x_plus_5) - assert 'date_pred' in hist_pred_df.columns + assert hist_pred_df["date_pred"].iloc[-1] == pd.Timestamp(end_x_plus_5) + assert "date_pred" in hist_pred_df.columns assert hist_pred_df.shape[0] == 10 # compare values in model_return_df with hist_pred_df - assert (model_return_df["value"].values == hist_pred_df.tail( - len(dataframe))["value"].values).all() + assert ( + model_return_df["value"].values == hist_pred_df.tail(len(dataframe))["value"].values + ).all() assert model_return_df.shape[0] == len(dataframe) # Ensure logger error is not called diff --git a/tests/freqai/test_freqai_datakitchen.py b/tests/freqai/test_freqai_datakitchen.py index 0f4bbfd08..27efc3a66 100644 --- a/tests/freqai/test_freqai_datakitchen.py +++ b/tests/freqai/test_freqai_datakitchen.py @@ -67,7 +67,6 @@ def test_split_timerange( def test_check_if_model_expired(mocker, freqai_conf): - dk = get_patched_data_kitchen(mocker, freqai_conf) now = datetime.now(tz=timezone.utc).timestamp() assert dk.check_if_model_expired(now) is False @@ -81,10 +80,10 @@ def test_filter_features(mocker, freqai_conf): freqai.dk.find_features(unfiltered_dataframe) filtered_df, _labels = freqai.dk.filter_features( - unfiltered_dataframe, - freqai.dk.training_features_list, - freqai.dk.label_list, - training_filter=True, + unfiltered_dataframe, + freqai.dk.training_features_list, + freqai.dk.label_list, + training_filter=True, ) assert len(filtered_df.columns) == 14 @@ -95,22 +94,20 @@ def test_make_train_test_datasets(mocker, freqai_conf): freqai.dk.find_features(unfiltered_dataframe) features_filtered, labels_filtered = freqai.dk.filter_features( - unfiltered_dataframe, - freqai.dk.training_features_list, - freqai.dk.label_list, - training_filter=True, - ) + unfiltered_dataframe, + freqai.dk.training_features_list, + freqai.dk.label_list, + training_filter=True, + ) data_dictionary = freqai.dk.make_train_test_datasets(features_filtered, labels_filtered) assert data_dictionary assert len(data_dictionary) == 7 - assert len(data_dictionary['train_features'].index) == 1916 + assert len(data_dictionary["train_features"].index) == 1916 -@pytest.mark.parametrize('model', [ - 'LightGBMRegressor' - ]) +@pytest.mark.parametrize("model", ["LightGBMRegressor"]) def test_get_full_model_path(mocker, freqai_conf, model): freqai_conf.update({"freqaimodel": model}) freqai_conf.update({"timerange": "20180110-20180130"}) @@ -134,9 +131,10 @@ def test_get_full_model_path(mocker, freqai_conf, model): data_load_timerange = TimeRange.parse_timerange("20180110-20180130") new_timerange = TimeRange.parse_timerange("20180120-20180130") - freqai.dk.set_paths('ADA/BTC', None) + freqai.dk.set_paths("ADA/BTC", None) freqai.extract_data_and_train_model( - new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) + new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange + ) model_path = freqai.dk.get_full_models_path(freqai_conf) assert model_path.is_dir() is True @@ -161,7 +159,7 @@ def test_get_pair_data_for_features_with_prealoaded_data(mocker, freqai_conf): def test_get_pair_data_for_features_without_preloaded_data(mocker, freqai_conf): freqai_conf.update({"timerange": "20180115-20180130"}) - freqai_conf['runmode'] = 'backtest' + freqai_conf["runmode"] = "backtest" strategy = get_patched_freqai_strategy(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf) @@ -172,13 +170,13 @@ def test_get_pair_data_for_features_without_preloaded_data(mocker, freqai_conf): timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(timerange, freqai.dk) - base_df = {'5m': pd.DataFrame()} + base_df = {"5m": pd.DataFrame()} df = freqai.dk.get_pair_data_for_features("LTC/BTC", "5m", strategy, base_dataframes=base_df) assert df is not base_df["5m"] assert not df.empty - assert df.iloc[0]['date'].strftime("%Y-%m-%d %H:%M:%S") == "2018-01-11 23:00:00" - assert df.iloc[-1]['date'].strftime("%Y-%m-%d %H:%M:%S") == "2018-01-30 00:00:00" + assert df.iloc[0]["date"].strftime("%Y-%m-%d %H:%M:%S") == "2018-01-11 23:00:00" + assert df.iloc[-1]["date"].strftime("%Y-%m-%d %H:%M:%S") == "2018-01-30 00:00:00" def test_populate_features(mocker, freqai_conf): @@ -192,12 +190,14 @@ def test_populate_features(mocker, freqai_conf): freqai.dd.load_all_pair_histories(timerange, freqai.dk) corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(timerange, "LTC/BTC", freqai.dk) - mocker.patch.object(strategy, 'feature_engineering_expand_all', return_value=base_df["5m"]) - df = freqai.dk.populate_features(base_df["5m"], "LTC/BTC", strategy, - base_dataframes=base_df, corr_dataframes=corr_df) + mocker.patch.object(strategy, "feature_engineering_expand_all", return_value=base_df["5m"]) + df = freqai.dk.populate_features( + base_df["5m"], "LTC/BTC", strategy, base_dataframes=base_df, corr_dataframes=corr_df + ) strategy.feature_engineering_expand_all.assert_called_once() - pd.testing.assert_frame_equal(base_df["5m"], - strategy.feature_engineering_expand_all.call_args[0][0]) + pd.testing.assert_frame_equal( + base_df["5m"], strategy.feature_engineering_expand_all.call_args[0][0] + ) - assert df.iloc[0]['date'].strftime("%Y-%m-%d %H:%M:%S") == "2018-01-15 00:00:00" + assert df.iloc[0]["date"].strftime("%Y-%m-%d %H:%M:%S") == "2018-01-15 00:00:00" diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 6c72329c1..21a558548 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -24,7 +24,7 @@ from tests.freqai.conftest import ( def can_run_model(model: str) -> None: - is_pytorch_model = 'Reinforcement' in model or 'PyTorch' in model + is_pytorch_model = "Reinforcement" in model or "PyTorch" in model if is_arm() and "Catboost" in model: pytest.skip("CatBoost is not supported on ARM.") @@ -33,57 +33,59 @@ def can_run_model(model: str) -> None: pytest.skip("Reinforcement learning / PyTorch module not available on intel based Mac OS.") -@pytest.mark.parametrize('model, pca, dbscan, float32, can_short, shuffle, buffer, noise', [ - ('LightGBMRegressor', True, False, True, True, False, 0, 0), - ('XGBoostRegressor', False, True, False, True, False, 10, 0.05), - ('XGBoostRFRegressor', False, False, False, True, False, 0, 0), - ('CatboostRegressor', False, False, False, True, True, 0, 0), - ('PyTorchMLPRegressor', False, False, False, False, False, 0, 0), - ('PyTorchTransformerRegressor', False, False, False, False, False, 0, 0), - ('ReinforcementLearner', False, True, False, True, False, 0, 0), - ('ReinforcementLearner_multiproc', False, False, False, True, False, 0, 0), - ('ReinforcementLearner_test_3ac', False, False, False, False, False, 0, 0), - ('ReinforcementLearner_test_3ac', False, False, False, True, False, 0, 0), - ('ReinforcementLearner_test_4ac', False, False, False, True, False, 0, 0), - ]) -def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, - dbscan, float32, can_short, shuffle, - buffer, noise): - +@pytest.mark.parametrize( + "model, pca, dbscan, float32, can_short, shuffle, buffer, noise", + [ + ("LightGBMRegressor", True, False, True, True, False, 0, 0), + ("XGBoostRegressor", False, True, False, True, False, 10, 0.05), + ("XGBoostRFRegressor", False, False, False, True, False, 0, 0), + ("CatboostRegressor", False, False, False, True, True, 0, 0), + ("PyTorchMLPRegressor", False, False, False, False, False, 0, 0), + ("PyTorchTransformerRegressor", False, False, False, False, False, 0, 0), + ("ReinforcementLearner", False, True, False, True, False, 0, 0), + ("ReinforcementLearner_multiproc", False, False, False, True, False, 0, 0), + ("ReinforcementLearner_test_3ac", False, False, False, False, False, 0, 0), + ("ReinforcementLearner_test_3ac", False, False, False, True, False, 0, 0), + ("ReinforcementLearner_test_4ac", False, False, False, True, False, 0, 0), + ], +) +def test_extract_data_and_train_model_Standard( + mocker, freqai_conf, model, pca, dbscan, float32, can_short, shuffle, buffer, noise +): can_run_model(model) test_tb = True if is_mac(): test_tb = False - model_save_ext = 'joblib' + model_save_ext = "joblib" freqai_conf.update({"freqaimodel": model}) freqai_conf.update({"timerange": "20180110-20180130"}) freqai_conf.update({"strategy": "freqai_test_strat"}) - freqai_conf['freqai']['feature_parameters'].update({"principal_component_analysis": pca}) - freqai_conf['freqai']['feature_parameters'].update({"use_DBSCAN_to_remove_outliers": dbscan}) + freqai_conf["freqai"]["feature_parameters"].update({"principal_component_analysis": pca}) + freqai_conf["freqai"]["feature_parameters"].update({"use_DBSCAN_to_remove_outliers": dbscan}) freqai_conf.update({"reduce_df_footprint": float32}) - freqai_conf['freqai']['feature_parameters'].update({"shuffle_after_split": shuffle}) - freqai_conf['freqai']['feature_parameters'].update({"buffer_train_data_candles": buffer}) - freqai_conf['freqai']['feature_parameters'].update({"noise_standard_deviation": noise}) + freqai_conf["freqai"]["feature_parameters"].update({"shuffle_after_split": shuffle}) + freqai_conf["freqai"]["feature_parameters"].update({"buffer_train_data_candles": buffer}) + freqai_conf["freqai"]["feature_parameters"].update({"noise_standard_deviation": noise}) - if 'ReinforcementLearner' in model: - model_save_ext = 'zip' + if "ReinforcementLearner" in model: + model_save_ext = "zip" freqai_conf = make_rl_config(freqai_conf) # test the RL guardrails - freqai_conf['freqai']['feature_parameters'].update({"use_SVM_to_remove_outliers": True}) - freqai_conf['freqai']['feature_parameters'].update({"DI_threshold": 2}) - freqai_conf['freqai']['data_split_parameters'].update({'shuffle': True}) + freqai_conf["freqai"]["feature_parameters"].update({"use_SVM_to_remove_outliers": True}) + freqai_conf["freqai"]["feature_parameters"].update({"DI_threshold": 2}) + freqai_conf["freqai"]["data_split_parameters"].update({"shuffle": True}) - if 'test_3ac' in model or 'test_4ac' in model: + if "test_3ac" in model or "test_4ac" in model: freqai_conf["freqaimodel_path"] = str(Path(__file__).parents[1] / "freqai" / "test_models") freqai_conf["freqai"]["rl_config"]["drop_ohlc_from_features"] = True - if 'PyTorch' in model: - model_save_ext = 'zip' + if "PyTorch" in model: + model_save_ext = "zip" pytorch_mlp_mtp = mock_pytorch_mlp_model_training_parameters() - freqai_conf['freqai']['model_training_parameters'].update(pytorch_mlp_mtp) - if 'Transformer' in model: + freqai_conf["freqai"]["model_training_parameters"].update(pytorch_mlp_mtp) + if "Transformer" in model: # transformer model takes a window, unlike the MLP regressor freqai_conf.update({"conv_width": 10}) @@ -97,7 +99,7 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, freqai.can_short = can_short freqai.dk = FreqaiDataKitchen(freqai_conf) freqai.dk.live = True - freqai.dk.set_paths('ADA/BTC', 10000) + freqai.dk.set_paths("ADA/BTC", 10000) timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(timerange, freqai.dk) @@ -105,32 +107,37 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, data_load_timerange = TimeRange.parse_timerange("20180125-20180130") new_timerange = TimeRange.parse_timerange("20180127-20180130") - freqai.dk.set_paths('ADA/BTC', None) + freqai.dk.set_paths("ADA/BTC", None) freqai.train_timer("start", "ADA/BTC") freqai.extract_data_and_train_model( - new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) + new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange + ) freqai.train_timer("stop", "ADA/BTC") freqai.dd.save_metric_tracker_to_disk() freqai.dd.save_drawer_to_disk() assert Path(freqai.dk.full_path / "metric_tracker.json").is_file() assert Path(freqai.dk.full_path / "pair_dictionary.json").is_file() - assert Path(freqai.dk.data_path / - f"{freqai.dk.model_filename}_model.{model_save_ext}").is_file() + assert Path( + freqai.dk.data_path / f"{freqai.dk.model_filename}_model.{model_save_ext}" + ).is_file() assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").is_file() assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").is_file() shutil.rmtree(Path(freqai.dk.full_path)) -@pytest.mark.parametrize('model, strat', [ - ('LightGBMRegressorMultiTarget', "freqai_test_multimodel_strat"), - ('XGBoostRegressorMultiTarget', "freqai_test_multimodel_strat"), - ('CatboostRegressorMultiTarget', "freqai_test_multimodel_strat"), - ('LightGBMClassifierMultiTarget', "freqai_test_multimodel_classifier_strat"), - ('CatboostClassifierMultiTarget', "freqai_test_multimodel_classifier_strat") - ]) +@pytest.mark.parametrize( + "model, strat", + [ + ("LightGBMRegressorMultiTarget", "freqai_test_multimodel_strat"), + ("XGBoostRegressorMultiTarget", "freqai_test_multimodel_strat"), + ("CatboostRegressorMultiTarget", "freqai_test_multimodel_strat"), + ("LightGBMClassifierMultiTarget", "freqai_test_multimodel_classifier_strat"), + ("CatboostClassifierMultiTarget", "freqai_test_multimodel_classifier_strat"), + ], +) def test_extract_data_and_train_model_MultiTargets(mocker, freqai_conf, model, strat): can_run_model(model) @@ -152,28 +159,32 @@ def test_extract_data_and_train_model_MultiTargets(mocker, freqai_conf, model, s data_load_timerange = TimeRange.parse_timerange("20180110-20180130") new_timerange = TimeRange.parse_timerange("20180120-20180130") - freqai.dk.set_paths('ADA/BTC', None) + freqai.dk.set_paths("ADA/BTC", None) freqai.extract_data_and_train_model( - new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) + new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange + ) assert len(freqai.dk.label_list) == 2 assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").is_file() assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").is_file() assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").is_file() - assert len(freqai.dk.data['training_features_list']) == 14 + assert len(freqai.dk.data["training_features_list"]) == 14 shutil.rmtree(Path(freqai.dk.full_path)) -@pytest.mark.parametrize('model', [ - 'LightGBMClassifier', - 'CatboostClassifier', - 'XGBoostClassifier', - 'XGBoostRFClassifier', - 'SKLearnRandomForestClassifier', - 'PyTorchMLPClassifier', - ]) +@pytest.mark.parametrize( + "model", + [ + "LightGBMClassifier", + "CatboostClassifier", + "XGBoostClassifier", + "XGBoostRFClassifier", + "SKLearnRandomForestClassifier", + "PyTorchMLPClassifier", + ], +) def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model): can_run_model(model) @@ -196,25 +207,28 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model): data_load_timerange = TimeRange.parse_timerange("20180110-20180130") new_timerange = TimeRange.parse_timerange("20180120-20180130") - freqai.dk.set_paths('ADA/BTC', None) + freqai.dk.set_paths("ADA/BTC", None) - freqai.extract_data_and_train_model(new_timerange, "ADA/BTC", - strategy, freqai.dk, data_load_timerange) + freqai.extract_data_and_train_model( + new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange + ) - if 'PyTorchMLPClassifier': + if "PyTorchMLPClassifier": pytorch_mlp_mtp = mock_pytorch_mlp_model_training_parameters() - freqai_conf['freqai']['model_training_parameters'].update(pytorch_mlp_mtp) + freqai_conf["freqai"]["model_training_parameters"].update(pytorch_mlp_mtp) - if freqai.dd.model_type == 'joblib': + if freqai.dd.model_type == "joblib": model_file_extension = ".joblib" elif freqai.dd.model_type == "pytorch": model_file_extension = ".zip" else: - raise Exception(f"Unsupported model type: {freqai.dd.model_type}," - f" can't assign model_file_extension") + raise Exception( + f"Unsupported model type: {freqai.dd.model_type}," f" can't assign model_file_extension" + ) - assert Path(freqai.dk.data_path / - f"{freqai.dk.model_filename}_model{model_file_extension}").exists() + assert Path( + freqai.dk.data_path / f"{freqai.dk.model_filename}_model{model_file_extension}" + ).exists() assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").exists() assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").exists() @@ -233,9 +247,9 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model): ("XGBoostClassifier", 2, "freqai_test_classifier"), ("LightGBMClassifier", 2, "freqai_test_classifier"), ("CatboostClassifier", 2, "freqai_test_classifier"), - ("PyTorchMLPClassifier", 2, "freqai_test_classifier") + ("PyTorchMLPClassifier", 2, "freqai_test_classifier"), ], - ) +) def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog): can_run_model(model) test_tb = True @@ -243,7 +257,7 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog) test_tb = False freqai_conf.get("freqai", {}).update({"save_backtest_models": True}) - freqai_conf['runmode'] = RunMode.BACKTEST + freqai_conf["runmode"] = RunMode.BACKTEST Trade.use_db = False @@ -251,21 +265,22 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog) freqai_conf.update({"timerange": "20180120-20180130"}) freqai_conf.update({"strategy": strat}) - if 'ReinforcementLearner' in model: + if "ReinforcementLearner" in model: freqai_conf = make_rl_config(freqai_conf) - if 'test_4ac' in model: + if "test_4ac" in model: freqai_conf["freqaimodel_path"] = str(Path(__file__).parents[1] / "freqai" / "test_models") - if 'PyTorch' in model: + if "PyTorch" in model: pytorch_mlp_mtp = mock_pytorch_mlp_model_training_parameters() - freqai_conf['freqai']['model_training_parameters'].update(pytorch_mlp_mtp) - if 'Transformer' in model: + freqai_conf["freqai"]["model_training_parameters"].update(pytorch_mlp_mtp) + if "Transformer" in model: # transformer model takes a window, unlike the MLP regressor freqai_conf.update({"conv_width": 10}) freqai_conf.get("freqai", {}).get("feature_parameters", {}).update( - {"indicator_periods_candles": [2]}) + {"indicator_periods_candles": [2]} + ) strategy = get_patched_freqai_strategy(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf) @@ -282,7 +297,7 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog) df = base_df[freqai_conf["timeframe"]] metadata = {"pair": "LTC/BTC"} - freqai.dk.set_paths('LTC/BTC', None) + freqai.dk.set_paths("LTC/BTC", None) freqai.start_backtesting(df, metadata, freqai.dk, strategy) model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] @@ -294,13 +309,16 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog) def test_start_backtesting_subdaily_backtest_period(mocker, freqai_conf): freqai_conf.update({"timerange": "20180120-20180124"}) - freqai_conf['runmode'] = 'backtest' - freqai_conf.get("freqai", {}).update({ - "backtest_period_days": 0.5, - "save_backtest_models": True, - }) + freqai_conf["runmode"] = "backtest" + freqai_conf.get("freqai", {}).update( + { + "backtest_period_days": 0.5, + "save_backtest_models": True, + } + ) freqai_conf.get("freqai", {}).get("feature_parameters", {}).update( - {"indicator_periods_candles": [2]}) + {"indicator_periods_candles": [2]} + ) strategy = get_patched_freqai_strategy(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf) strategy.dp = DataProvider(freqai_conf, exchange) @@ -325,10 +343,11 @@ def test_start_backtesting_subdaily_backtest_period(mocker, freqai_conf): def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog): freqai_conf.update({"timerange": "20180120-20180130"}) - freqai_conf['runmode'] = 'backtest' + freqai_conf["runmode"] = "backtest" freqai_conf.get("freqai", {}).update({"save_backtest_models": True}) freqai_conf.get("freqai", {}).get("feature_parameters", {}).update( - {"indicator_periods_candles": [2]}) + {"indicator_periods_candles": [2]} + ) strategy = get_patched_freqai_strategy(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf) strategy.dp = DataProvider(freqai_conf, exchange) @@ -381,7 +400,7 @@ def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog): freqai.dk.pair = pair freqai.start_backtesting(df, metadata, freqai.dk, strategy) - path = (freqai.dd.full_path / freqai.dk.backtest_predictions_folder) + path = freqai.dd.full_path / freqai.dk.backtest_predictions_folder prediction_files = [x for x in path.iterdir() if x.is_file()] assert len(prediction_files) == 2 @@ -389,7 +408,7 @@ def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog): def test_backtesting_fit_live_predictions(mocker, freqai_conf, caplog): - freqai_conf['runmode'] = 'backtest' + freqai_conf["runmode"] = "backtest" freqai_conf.get("freqai", {}).update({"fit_live_predictions_candles": 10}) strategy = get_patched_freqai_strategy(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf) @@ -418,12 +437,12 @@ def test_backtesting_fit_live_predictions(mocker, freqai_conf, caplog): def test_plot_feature_importance(mocker, freqai_conf): - from freqtrade.freqai.utils import plot_feature_importance freqai_conf.update({"timerange": "20180110-20180130"}) freqai_conf.get("freqai", {}).get("feature_parameters", {}).update( - {"princpial_component_analysis": "true"}) + {"princpial_component_analysis": "true"} + ) strategy = get_patched_freqai_strategy(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf) @@ -436,15 +455,22 @@ def test_plot_feature_importance(mocker, freqai_conf): timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(timerange, freqai.dk) - freqai.dd.pair_dict = {"ADA/BTC": {"model_filename": "fake_name", - "trained_timestamp": 1, "data_path": "", "extras": {}}} + freqai.dd.pair_dict = { + "ADA/BTC": { + "model_filename": "fake_name", + "trained_timestamp": 1, + "data_path": "", + "extras": {}, + } + } data_load_timerange = TimeRange.parse_timerange("20180110-20180130") new_timerange = TimeRange.parse_timerange("20180120-20180130") - freqai.dk.set_paths('ADA/BTC', None) + freqai.dk.set_paths("ADA/BTC", None) freqai.extract_data_and_train_model( - new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) + new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange + ) model = freqai.dd.load_data("ADA/BTC", freqai.dk) @@ -455,17 +481,21 @@ def test_plot_feature_importance(mocker, freqai_conf): shutil.rmtree(Path(freqai.dk.full_path)) -@pytest.mark.parametrize('timeframes,corr_pairs', [ - (['5m'], ['ADA/BTC', 'DASH/BTC']), - (['5m'], ['ADA/BTC', 'DASH/BTC', 'ETH/USDT']), - (['5m', '15m'], ['ADA/BTC', 'DASH/BTC', 'ETH/USDT']), -]) +@pytest.mark.parametrize( + "timeframes,corr_pairs", + [ + (["5m"], ["ADA/BTC", "DASH/BTC"]), + (["5m"], ["ADA/BTC", "DASH/BTC", "ETH/USDT"]), + (["5m", "15m"], ["ADA/BTC", "DASH/BTC", "ETH/USDT"]), + ], +) def test_freqai_informative_pairs(mocker, freqai_conf, timeframes, corr_pairs): - freqai_conf['freqai']['feature_parameters'].update({ - 'include_timeframes': timeframes, - 'include_corr_pairlist': corr_pairs, - - }) + freqai_conf["freqai"]["feature_parameters"].update( + { + "include_timeframes": timeframes, + "include_corr_pairlist": corr_pairs, + } + ) strategy = get_patched_freqai_strategy(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf) pairlists = PairListManager(exchange, freqai_conf) @@ -507,8 +537,8 @@ def test_download_all_data_for_training(mocker, freqai_conf, caplog, tmp_path): exchange = get_patched_exchange(mocker, freqai_conf) pairlist = PairListManager(exchange, freqai_conf) strategy.dp = DataProvider(freqai_conf, exchange, pairlist) - freqai_conf['pairs'] = freqai_conf['exchange']['pair_whitelist'] - freqai_conf['datadir'] = tmp_path + freqai_conf["pairs"] = freqai_conf["exchange"]["pair_whitelist"] + freqai_conf["datadir"] = tmp_path download_all_data_for_training(strategy.dp, freqai_conf) assert log_has_re( @@ -518,9 +548,8 @@ def test_download_all_data_for_training(mocker, freqai_conf, caplog, tmp_path): @pytest.mark.usefixtures("init_persistence") -@pytest.mark.parametrize('dp_exists', [(False), (True)]) +@pytest.mark.parametrize("dp_exists", [(False), (True)]) def test_get_state_info(mocker, freqai_conf, dp_exists, caplog, tickers): - if is_mac(): pytest.skip("Reinforcement learning module not available on intel based Mac OS") @@ -528,12 +557,12 @@ def test_get_state_info(mocker, freqai_conf, dp_exists, caplog, tickers): freqai_conf.update({"timerange": "20180110-20180130"}) freqai_conf.update({"strategy": "freqai_rl_test_strat"}) freqai_conf = make_rl_config(freqai_conf) - freqai_conf['entry_pricing']['price_side'] = 'same' - freqai_conf['exit_pricing']['price_side'] = 'same' + freqai_conf["entry_pricing"]["price_side"] = "same" + freqai_conf["exit_pricing"]["price_side"] = "same" strategy = get_patched_freqai_strategy(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf) - ticker_mock = MagicMock(return_value=tickers()['ETH/BTC']) + ticker_mock = MagicMock(return_value=tickers()["ETH/BTC"]) mocker.patch(f"{EXMS}.fetch_ticker", ticker_mock) strategy.dp = DataProvider(freqai_conf, exchange)