diff --git a/docs/backtesting.md b/docs/backtesting.md index 981f889a2..4d91b3ca6 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -435,6 +435,20 @@ To save time, by default backtest will reuse a cached result from within the las To further analyze your backtest results, freqtrade will export the trades to file by default. You can then load the trades to perform further analysis as shown in the [data analysis](strategy_analysis_example.md#load-backtest-results-to-pandas-dataframe) backtesting section. +### Backtest output file + +The output file freqtrade produces is a zip file containing the following files: + +- The backtest report in json format +- the market change data in feather format +- a copy of the strategy file +- a copy of the strategy parameters (if a parameter file was used) +- a sanitized copy of the config file + +This will ensure results are reproducible - under the assumption that the same data is available. + +Only the strategy file and the config file are included in the zip file, eventual dependencies are not included. + ## Assumptions made by backtesting Since backtesting lacks some detailed information about what happens within a candle, it needs to take a few assumptions: diff --git a/freqtrade/ft_types/backtest_result_type.py b/freqtrade/ft_types/backtest_result_type.py index 27cdb6126..b8ddbb4b6 100644 --- a/freqtrade/ft_types/backtest_result_type.py +++ b/freqtrade/ft_types/backtest_result_type.py @@ -1,4 +1,5 @@ -from typing import Any +from copy import deepcopy +from typing import Any, cast from typing_extensions import TypedDict @@ -15,11 +16,16 @@ class BacktestResultType(TypedDict): def get_BacktestResultType_default() -> BacktestResultType: - return { - "metadata": {}, - "strategy": {}, - "strategy_comparison": [], - } + return cast( + BacktestResultType, + deepcopy( + { + "metadata": {}, + "strategy": {}, + "strategy_comparison": [], + } + ), + ) class BacktestHistoryEntryType(BacktestMetadataType): diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d2efc78b5..9a5e58c92 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1792,6 +1792,7 @@ class Backtesting: dt_appendix, market_change_data=combined_res, analysis_results=self.analysis_results, + strategy_files={s.get_strategy_name(): s.__file__ for s in self.strategylist}, ) # Results may be mixed up now. Sort them so they follow --strategy-list order. diff --git a/freqtrade/optimize/optimize_reports/bt_storage.py b/freqtrade/optimize/optimize_reports/bt_storage.py index d0c5d7fb7..1e6a67e81 100644 --- a/freqtrade/optimize/optimize_reports/bt_storage.py +++ b/freqtrade/optimize/optimize_reports/bt_storage.py @@ -6,6 +6,7 @@ from zipfile import ZIP_DEFLATED, ZipFile from pandas import DataFrame +from freqtrade.configuration import sanitize_config from freqtrade.constants import LAST_BT_RESULT_FN from freqtrade.enums.runmode import RunMode from freqtrade.ft_types import BacktestResultType @@ -52,6 +53,7 @@ def store_backtest_results( *, market_change_data: DataFrame | None = None, analysis_results: dict[str, dict[str, DataFrame]] | None = None, + strategy_files: dict[str, str] | None = None, ) -> Path: """ Stores backtest results and analysis data in a zip file, with metadata stored separately @@ -85,6 +87,32 @@ def store_backtest_results( dump_json_to_file(stats_buf, stats_copy) zipf.writestr(json_filename.name, stats_buf.getvalue()) + config_buf = StringIO() + dump_json_to_file(config_buf, sanitize_config(config["original_config"])) + zipf.writestr(f"{base_filename.stem}_config.json", config_buf.getvalue()) + + for strategy_name, strategy_file in (strategy_files or {}).items(): + # Store the strategy file and its parameters + strategy_buf = BytesIO() + strategy_path = Path(strategy_file) + if not strategy_path.is_file(): + logger.warning(f"Strategy file '{strategy_path}' does not exist. Skipping.") + continue + with strategy_path.open("rb") as strategy_file_obj: + strategy_buf.write(strategy_file_obj.read()) + strategy_buf.seek(0) + zipf.writestr(f"{base_filename.stem}_{strategy_name}.py", strategy_buf.getvalue()) + strategy_params = strategy_path.with_suffix(".json") + if strategy_params.is_file(): + strategy_params_buf = BytesIO() + with strategy_params.open("rb") as strategy_params_obj: + strategy_params_buf.write(strategy_params_obj.read()) + strategy_params_buf.seek(0) + zipf.writestr( + f"{base_filename.stem}_{strategy_name}.json", + strategy_params_buf.getvalue(), + ) + # Add market change data if present if market_change_data is not None: market_change_name = f"{base_filename.stem}_market_change.feather" diff --git a/freqtrade/optimize/optimize_reports/optimize_reports.py b/freqtrade/optimize/optimize_reports/optimize_reports.py index c0188673a..e5b526779 100644 --- a/freqtrade/optimize/optimize_reports/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports/optimize_reports.py @@ -18,7 +18,7 @@ from freqtrade.data.metrics import ( calculate_sortino, calculate_sqn, ) -from freqtrade.ft_types import BacktestResultType +from freqtrade.ft_types import BacktestResultType, get_BacktestResultType_default from freqtrade.util import decimals_per_coin, fmt_coin, get_dry_run_wallet @@ -587,11 +587,7 @@ def generate_backtest_stats( :param max_date: Backtest end date :return: Dictionary containing results per strategy and a strategy summary. """ - result: BacktestResultType = { - "metadata": {}, - "strategy": {}, - "strategy_comparison": [], - } + result: BacktestResultType = get_BacktestResultType_default() market_change = calculate_market_change(btdata, "close") metadata = {} pairlist = list(btdata.keys()) diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index 278922b7d..5d06e8a48 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -108,6 +108,9 @@ def __run_backtest_bg(btconfig: Config): ApiBG.bt["bt"].results, datetime.now().strftime("%Y-%m-%d_%H-%M-%S"), market_change_data=combined_res, + strategy_files={ + s.get_strategy_name(): s.__file__ for s in ApiBG.bt["bt"].strategylist + }, ) ApiBG.bt["bt"].results["metadata"][strategy_name]["filename"] = str(fn.stem) ApiBG.bt["bt"].results["metadata"][strategy_name]["strategy"] = strategy_name diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 98e02cdd6..5cd873f23 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -132,6 +132,7 @@ class IStrategy(ABC, HyperStrategyMixin): stake_currency: str # container variable for strategy source code __source__: str = "" + __file__: str = "" # Definition of plot_config. See plotting documentation for more details. plot_config: dict = {} diff --git a/tests/conftest.py b/tests/conftest.py index b7e766b3e..06dd23f79 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -652,6 +652,7 @@ def get_default_conf(testdatadir): "trading_mode": "spot", "margin_mode": "", "candle_type_def": CandleType.SPOT, + "original_config": {}, } return configuration diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index d0c970b33..9fb5ccae9 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -1,5 +1,6 @@ import json import re +import shutil from datetime import timedelta from pathlib import Path from shutil import copyfile @@ -41,7 +42,7 @@ from freqtrade.optimize.optimize_reports.optimize_reports import ( from freqtrade.resolvers.strategy_resolver import StrategyResolver from freqtrade.util import dt_ts from freqtrade.util.datetime_helpers import dt_from_ts, dt_utc -from tests.conftest import CURRENT_TEST_STRATEGY +from tests.conftest import CURRENT_TEST_STRATEGY, log_has_re from tests.data.test_history import _clean_test_file @@ -253,8 +254,9 @@ def test_store_backtest_results(testdatadir, mocker): dump_mock = mocker.patch("freqtrade.optimize.optimize_reports.bt_storage.file_dump_json") zip_mock = mocker.patch("freqtrade.optimize.optimize_reports.bt_storage.ZipFile") data = {"metadata": {}, "strategy": {}, "strategy_comparison": []} - - store_backtest_results({"exportfilename": testdatadir}, data, "2022_01_01_15_05_13") + store_backtest_results( + {"exportfilename": testdatadir, "original_config": {}}, data, "2022_01_01_15_05_13" + ) assert dump_mock.call_count == 2 assert zip_mock.call_count == 1 @@ -264,7 +266,9 @@ def test_store_backtest_results(testdatadir, mocker): dump_mock.reset_mock() zip_mock.reset_mock() filename = testdatadir / "testresult.json" - store_backtest_results({"exportfilename": filename}, data, "2022_01_01_15_05_13") + store_backtest_results( + {"exportfilename": filename, "original_config": {}}, data, "2022_01_01_15_05_13" + ) assert dump_mock.call_count == 2 assert zip_mock.call_count == 1 assert isinstance(dump_mock.call_args_list[0][0][0], Path) @@ -272,9 +276,16 @@ def test_store_backtest_results(testdatadir, mocker): assert str(dump_mock.call_args_list[0][0][0]).startswith(str(testdatadir / "testresult")) -def test_store_backtest_results_real(tmp_path): +def test_store_backtest_results_real(tmp_path, caplog): data = {"metadata": {}, "strategy": {}, "strategy_comparison": []} - store_backtest_results({"exportfilename": tmp_path}, data, "2022_01_01_15_05_13") + config = { + "exportfilename": tmp_path, + "original_config": {}, + } + store_backtest_results( + config, data, "2022_01_01_15_05_13", strategy_files={"DefStrat": "NoFile"} + ) + assert log_has_re(r"Strategy file .* does not exist\. Skipping\.", caplog) zip_file = tmp_path / "backtest-result-2022_01_01_15_05_13.zip" assert zip_file.is_file() @@ -287,8 +298,19 @@ def test_store_backtest_results_real(tmp_path): fn = get_latest_backtest_filename(tmp_path) assert fn == "backtest-result-2022_01_01_15_05_13.zip" + strategy_test_dir = Path(__file__).parent.parent / "strategy" / "strats" + + shutil.copy(strategy_test_dir / "strategy_test_v3.py", tmp_path) + params_file = tmp_path / "strategy_test_v3.json" + with params_file.open("w") as f: + f.write("""{"strategy_name": "TurtleStrategyX5","params":{}}""") + store_backtest_results( - {"exportfilename": tmp_path}, data, "2024_01_01_15_05_25", market_change_data=pd.DataFrame() + config, + data, + "2024_01_01_15_05_25", + market_change_data=pd.DataFrame(), + strategy_files={"DefStrat": str(tmp_path / "strategy_test_v3.py")}, ) zip_file = tmp_path / "backtest-result-2024_01_01_15_05_25.zip" assert zip_file.is_file() @@ -298,6 +320,22 @@ def test_store_backtest_results_real(tmp_path): with ZipFile(zip_file, "r") as zipf: assert "backtest-result-2024_01_01_15_05_25.json" in zipf.namelist() assert "backtest-result-2024_01_01_15_05_25_market_change.feather" in zipf.namelist() + assert "backtest-result-2024_01_01_15_05_25_config.json" in zipf.namelist() + # strategy file is copied to the zip file + assert "backtest-result-2024_01_01_15_05_25_DefStrat.py" in zipf.namelist() + # compare the content of the strategy file + with zipf.open("backtest-result-2024_01_01_15_05_25_DefStrat.py") as strategy_file: + strategy_content = strategy_file.read() + with (strategy_test_dir / "strategy_test_v3.py").open("rb") as original_file: + original_content = original_file.read() + assert strategy_content == original_content + assert "backtest-result-2024_01_01_15_05_25_DefStrat.py" in zipf.namelist() + with zipf.open("backtest-result-2024_01_01_15_05_25_DefStrat.json") as pf: + params_content = pf.read() + with params_file.open("rb") as original_file: + original_content = original_file.read() + assert params_content == original_content + assert (tmp_path / LAST_BT_RESULT_FN).is_file() # Last file reference should be updated @@ -313,6 +351,7 @@ def test_write_read_backtest_candles(tmp_path): "exportfilename": tmp_path, "export": "signals", "runmode": "backtest", + "original_config": {}, } # test directory exporting sample_date = "2022_01_01_15_05_13"