From 23b10161d519ec60114030ada279b20e8d29bf2a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 23 Mar 2025 15:42:38 +0100 Subject: [PATCH 01/10] test: improve test resiliance --- tests/conftest.py | 1 + tests/optimize/test_optimize_reports.py | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b7e766b3e..06dd23f79 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -652,6 +652,7 @@ def get_default_conf(testdatadir): "trading_mode": "spot", "margin_mode": "", "candle_type_def": CandleType.SPOT, + "original_config": {}, } return configuration diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index d0c970b33..c69761fe0 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -253,8 +253,9 @@ def test_store_backtest_results(testdatadir, mocker): dump_mock = mocker.patch("freqtrade.optimize.optimize_reports.bt_storage.file_dump_json") zip_mock = mocker.patch("freqtrade.optimize.optimize_reports.bt_storage.ZipFile") data = {"metadata": {}, "strategy": {}, "strategy_comparison": []} - - store_backtest_results({"exportfilename": testdatadir}, data, "2022_01_01_15_05_13") + store_backtest_results( + {"exportfilename": testdatadir, "original_config": {}}, data, "2022_01_01_15_05_13" + ) assert dump_mock.call_count == 2 assert zip_mock.call_count == 1 @@ -264,7 +265,9 @@ def test_store_backtest_results(testdatadir, mocker): dump_mock.reset_mock() zip_mock.reset_mock() filename = testdatadir / "testresult.json" - store_backtest_results({"exportfilename": filename}, data, "2022_01_01_15_05_13") + store_backtest_results( + {"exportfilename": filename, "original_config": {}}, data, "2022_01_01_15_05_13" + ) assert dump_mock.call_count == 2 assert zip_mock.call_count == 1 assert isinstance(dump_mock.call_args_list[0][0][0], Path) @@ -274,7 +277,11 @@ def test_store_backtest_results(testdatadir, mocker): def test_store_backtest_results_real(tmp_path): data = {"metadata": {}, "strategy": {}, "strategy_comparison": []} - store_backtest_results({"exportfilename": tmp_path}, data, "2022_01_01_15_05_13") + config = { + "exportfilename": tmp_path, + "original_config": {}, + } + store_backtest_results(config, data, "2022_01_01_15_05_13") zip_file = tmp_path / "backtest-result-2022_01_01_15_05_13.zip" assert zip_file.is_file() @@ -287,9 +294,7 @@ def test_store_backtest_results_real(tmp_path): fn = get_latest_backtest_filename(tmp_path) assert fn == "backtest-result-2022_01_01_15_05_13.zip" - store_backtest_results( - {"exportfilename": tmp_path}, data, "2024_01_01_15_05_25", market_change_data=pd.DataFrame() - ) + store_backtest_results(config, data, "2024_01_01_15_05_25", market_change_data=pd.DataFrame()) zip_file = tmp_path / "backtest-result-2024_01_01_15_05_25.zip" assert zip_file.is_file() assert (tmp_path / "backtest-result-2024_01_01_15_05_25.meta.json").is_file() @@ -313,6 +318,7 @@ def test_write_read_backtest_candles(tmp_path): "exportfilename": tmp_path, "export": "signals", "runmode": "backtest", + "original_config": {}, } # test directory exporting sample_date = "2022_01_01_15_05_13" From 9fa3a15e245a2214bd0a226495a8432ad3a5b882 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 23 Mar 2025 15:44:55 +0100 Subject: [PATCH 02/10] feat: store (sanitized) configuration --- freqtrade/optimize/optimize_reports/bt_storage.py | 5 +++++ tests/optimize/test_optimize_reports.py | 2 ++ 2 files changed, 7 insertions(+) diff --git a/freqtrade/optimize/optimize_reports/bt_storage.py b/freqtrade/optimize/optimize_reports/bt_storage.py index d0c5d7fb7..00c8c1913 100644 --- a/freqtrade/optimize/optimize_reports/bt_storage.py +++ b/freqtrade/optimize/optimize_reports/bt_storage.py @@ -6,6 +6,7 @@ from zipfile import ZIP_DEFLATED, ZipFile from pandas import DataFrame +from freqtrade.configuration import sanitize_config from freqtrade.constants import LAST_BT_RESULT_FN from freqtrade.enums.runmode import RunMode from freqtrade.ft_types import BacktestResultType @@ -85,6 +86,10 @@ def store_backtest_results( dump_json_to_file(stats_buf, stats_copy) zipf.writestr(json_filename.name, stats_buf.getvalue()) + config_buf = StringIO() + dump_json_to_file(config_buf, sanitize_config(config["original_config"])) + zipf.writestr(f"{base_filename.stem}_config.json", config_buf.getvalue()) + # Add market change data if present if market_change_data is not None: market_change_name = f"{base_filename.stem}_market_change.feather" diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index c69761fe0..838dcb14f 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -303,6 +303,8 @@ def test_store_backtest_results_real(tmp_path): with ZipFile(zip_file, "r") as zipf: assert "backtest-result-2024_01_01_15_05_25.json" in zipf.namelist() assert "backtest-result-2024_01_01_15_05_25_market_change.feather" in zipf.namelist() + assert "backtest-result-2024_01_01_15_05_25_config.json" in zipf.namelist() + assert (tmp_path / LAST_BT_RESULT_FN).is_file() # Last file reference should be updated From 04a28b25508d76d71465b93d226dac88dcce9756 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 23 Mar 2025 15:51:48 +0100 Subject: [PATCH 03/10] fix: backtestResulttype - simplify --- freqtrade/ft_types/backtest_result_type.py | 13 ++++++++----- .../optimize/optimize_reports/optimize_reports.py | 8 ++------ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/freqtrade/ft_types/backtest_result_type.py b/freqtrade/ft_types/backtest_result_type.py index 27cdb6126..4cd8f2a58 100644 --- a/freqtrade/ft_types/backtest_result_type.py +++ b/freqtrade/ft_types/backtest_result_type.py @@ -1,3 +1,4 @@ +from copy import deepcopy from typing import Any from typing_extensions import TypedDict @@ -15,11 +16,13 @@ class BacktestResultType(TypedDict): def get_BacktestResultType_default() -> BacktestResultType: - return { - "metadata": {}, - "strategy": {}, - "strategy_comparison": [], - } + return deepcopy( + { + "metadata": {}, + "strategy": {}, + "strategy_comparison": [], + } + ) class BacktestHistoryEntryType(BacktestMetadataType): diff --git a/freqtrade/optimize/optimize_reports/optimize_reports.py b/freqtrade/optimize/optimize_reports/optimize_reports.py index c0188673a..e5b526779 100644 --- a/freqtrade/optimize/optimize_reports/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports/optimize_reports.py @@ -18,7 +18,7 @@ from freqtrade.data.metrics import ( calculate_sortino, calculate_sqn, ) -from freqtrade.ft_types import BacktestResultType +from freqtrade.ft_types import BacktestResultType, get_BacktestResultType_default from freqtrade.util import decimals_per_coin, fmt_coin, get_dry_run_wallet @@ -587,11 +587,7 @@ def generate_backtest_stats( :param max_date: Backtest end date :return: Dictionary containing results per strategy and a strategy summary. """ - result: BacktestResultType = { - "metadata": {}, - "strategy": {}, - "strategy_comparison": [], - } + result: BacktestResultType = get_BacktestResultType_default() market_change = calculate_market_change(btdata, "close") metadata = {} pairlist = list(btdata.keys()) From 85fc9364319b03d2fc75c714f3855349dc0e107d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 23 Mar 2025 17:22:50 +0100 Subject: [PATCH 04/10] feat: add Strategy and parameter file to backtest zip file --- freqtrade/optimize/backtesting.py | 1 + .../optimize/optimize_reports/bt_storage.py | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d2efc78b5..9a5e58c92 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1792,6 +1792,7 @@ class Backtesting: dt_appendix, market_change_data=combined_res, analysis_results=self.analysis_results, + strategy_files={s.get_strategy_name(): s.__file__ for s in self.strategylist}, ) # Results may be mixed up now. Sort them so they follow --strategy-list order. diff --git a/freqtrade/optimize/optimize_reports/bt_storage.py b/freqtrade/optimize/optimize_reports/bt_storage.py index 00c8c1913..202db7c21 100644 --- a/freqtrade/optimize/optimize_reports/bt_storage.py +++ b/freqtrade/optimize/optimize_reports/bt_storage.py @@ -53,6 +53,7 @@ def store_backtest_results( *, market_change_data: DataFrame | None = None, analysis_results: dict[str, dict[str, DataFrame]] | None = None, + strategy_files: dict[str, str] | None = None, ) -> Path: """ Stores backtest results and analysis data in a zip file, with metadata stored separately @@ -90,6 +91,25 @@ def store_backtest_results( dump_json_to_file(config_buf, sanitize_config(config["original_config"])) zipf.writestr(f"{base_filename.stem}_config.json", config_buf.getvalue()) + for strategy_name, strategy_file in (strategy_files or {}).items(): + # Store the strategy file and its parameters + strategy_buf = BytesIO() + strategy_path = Path(strategy_file) + with strategy_path.open("rb") as strategy_file_obj: + strategy_buf.write(strategy_file_obj.read()) + strategy_buf.seek(0) + zipf.writestr(f"{base_filename.stem}_{strategy_name}.py", strategy_buf.getvalue()) + strategy_params = strategy_path.with_suffix(".json") + if strategy_params.is_file(): + strategy_params_buf = BytesIO() + with strategy_params.open("rb") as strategy_params_obj: + strategy_params_buf.write(strategy_params_obj.read()) + strategy_params_buf.seek(0) + zipf.writestr( + f"{base_filename.stem}_{strategy_name}.json", + strategy_params_buf.getvalue(), + ) + # Add market change data if present if market_change_data is not None: market_change_name = f"{base_filename.stem}_market_change.feather" From 5b0b44069aad729bbb6f3f61d35032672cd6b779 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 23 Mar 2025 17:23:05 +0100 Subject: [PATCH 05/10] test: Ensure strategy file is added to the test file. --- tests/optimize/test_optimize_reports.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 838dcb14f..3e00e4d85 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -294,7 +294,15 @@ def test_store_backtest_results_real(tmp_path): fn = get_latest_backtest_filename(tmp_path) assert fn == "backtest-result-2022_01_01_15_05_13.zip" - store_backtest_results(config, data, "2024_01_01_15_05_25", market_change_data=pd.DataFrame()) + strategy_test_dir = Path(__file__).parent.parent / "strategy" / "strats" + + store_backtest_results( + config, + data, + "2024_01_01_15_05_25", + market_change_data=pd.DataFrame(), + strategy_files={"DefStrat": str(strategy_test_dir / "strategy_test_v3.py")}, + ) zip_file = tmp_path / "backtest-result-2024_01_01_15_05_25.zip" assert zip_file.is_file() assert (tmp_path / "backtest-result-2024_01_01_15_05_25.meta.json").is_file() @@ -304,6 +312,14 @@ def test_store_backtest_results_real(tmp_path): assert "backtest-result-2024_01_01_15_05_25.json" in zipf.namelist() assert "backtest-result-2024_01_01_15_05_25_market_change.feather" in zipf.namelist() assert "backtest-result-2024_01_01_15_05_25_config.json" in zipf.namelist() + # strategy file is copied to the zip file + assert "backtest-result-2024_01_01_15_05_25_DefStrat.py" in zipf.namelist() + # compare the content of the strategy file + with zipf.open("backtest-result-2024_01_01_15_05_25_DefStrat.py") as strategy_file: + strategy_content = strategy_file.read() + with (strategy_test_dir / "strategy_test_v3.py").open("rb") as original_file: + original_content = original_file.read() + assert strategy_content == original_content assert (tmp_path / LAST_BT_RESULT_FN).is_file() From 7b44dd17aef21f2b9d7c55ebb6b945946389e76b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 23 Mar 2025 17:24:00 +0100 Subject: [PATCH 06/10] feat: strategy-file saving for api backtest --- freqtrade/rpc/api_server/api_backtest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index 278922b7d..5d06e8a48 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -108,6 +108,9 @@ def __run_backtest_bg(btconfig: Config): ApiBG.bt["bt"].results, datetime.now().strftime("%Y-%m-%d_%H-%M-%S"), market_change_data=combined_res, + strategy_files={ + s.get_strategy_name(): s.__file__ for s in ApiBG.bt["bt"].strategylist + }, ) ApiBG.bt["bt"].results["metadata"][strategy_name]["filename"] = str(fn.stem) ApiBG.bt["bt"].results["metadata"][strategy_name]["strategy"] = strategy_name From 3ffc69ad024df680fb88d9e73817d02207c7d6ae Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 23 Mar 2025 19:09:11 +0100 Subject: [PATCH 07/10] chore: fix typing errors --- freqtrade/ft_types/backtest_result_type.py | 17 ++++++++++------- freqtrade/strategy/interface.py | 1 + 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/freqtrade/ft_types/backtest_result_type.py b/freqtrade/ft_types/backtest_result_type.py index 4cd8f2a58..b8ddbb4b6 100644 --- a/freqtrade/ft_types/backtest_result_type.py +++ b/freqtrade/ft_types/backtest_result_type.py @@ -1,5 +1,5 @@ from copy import deepcopy -from typing import Any +from typing import Any, cast from typing_extensions import TypedDict @@ -16,12 +16,15 @@ class BacktestResultType(TypedDict): def get_BacktestResultType_default() -> BacktestResultType: - return deepcopy( - { - "metadata": {}, - "strategy": {}, - "strategy_comparison": [], - } + return cast( + BacktestResultType, + deepcopy( + { + "metadata": {}, + "strategy": {}, + "strategy_comparison": [], + } + ), ) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 98e02cdd6..5cd873f23 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -132,6 +132,7 @@ class IStrategy(ABC, HyperStrategyMixin): stake_currency: str # container variable for strategy source code __source__: str = "" + __file__: str = "" # Definition of plot_config. See plotting documentation for more details. plot_config: dict = {} From 286371c1e482fa85eb61dc09c9ec769c67db0de6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 23 Mar 2025 19:35:02 +0100 Subject: [PATCH 08/10] docs: add documentation for output zip file format --- docs/backtesting.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/backtesting.md b/docs/backtesting.md index 981f889a2..4d91b3ca6 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -435,6 +435,20 @@ To save time, by default backtest will reuse a cached result from within the las To further analyze your backtest results, freqtrade will export the trades to file by default. You can then load the trades to perform further analysis as shown in the [data analysis](strategy_analysis_example.md#load-backtest-results-to-pandas-dataframe) backtesting section. +### Backtest output file + +The output file freqtrade produces is a zip file containing the following files: + +- The backtest report in json format +- the market change data in feather format +- a copy of the strategy file +- a copy of the strategy parameters (if a parameter file was used) +- a sanitized copy of the config file + +This will ensure results are reproducible - under the assumption that the same data is available. + +Only the strategy file and the config file are included in the zip file, eventual dependencies are not included. + ## Assumptions made by backtesting Since backtesting lacks some detailed information about what happens within a candle, it needs to take a few assumptions: From 1fdce89806b6f638cf88146fb4bf8ca0851df7ce Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 24 Mar 2025 06:41:58 +0100 Subject: [PATCH 09/10] chore: Implement safeguard against non-existing strategy files --- freqtrade/optimize/optimize_reports/bt_storage.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/optimize/optimize_reports/bt_storage.py b/freqtrade/optimize/optimize_reports/bt_storage.py index 202db7c21..1e6a67e81 100644 --- a/freqtrade/optimize/optimize_reports/bt_storage.py +++ b/freqtrade/optimize/optimize_reports/bt_storage.py @@ -95,6 +95,9 @@ def store_backtest_results( # Store the strategy file and its parameters strategy_buf = BytesIO() strategy_path = Path(strategy_file) + if not strategy_path.is_file(): + logger.warning(f"Strategy file '{strategy_path}' does not exist. Skipping.") + continue with strategy_path.open("rb") as strategy_file_obj: strategy_buf.write(strategy_file_obj.read()) strategy_buf.seek(0) From 799ce4e1f2ec3096744b6925bbbe2dd04e0d0ee1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 24 Mar 2025 07:10:43 +0100 Subject: [PATCH 10/10] test: Enhance test to also cover params file saving --- tests/optimize/test_optimize_reports.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 3e00e4d85..9fb5ccae9 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -1,5 +1,6 @@ import json import re +import shutil from datetime import timedelta from pathlib import Path from shutil import copyfile @@ -41,7 +42,7 @@ from freqtrade.optimize.optimize_reports.optimize_reports import ( from freqtrade.resolvers.strategy_resolver import StrategyResolver from freqtrade.util import dt_ts from freqtrade.util.datetime_helpers import dt_from_ts, dt_utc -from tests.conftest import CURRENT_TEST_STRATEGY +from tests.conftest import CURRENT_TEST_STRATEGY, log_has_re from tests.data.test_history import _clean_test_file @@ -275,13 +276,16 @@ def test_store_backtest_results(testdatadir, mocker): assert str(dump_mock.call_args_list[0][0][0]).startswith(str(testdatadir / "testresult")) -def test_store_backtest_results_real(tmp_path): +def test_store_backtest_results_real(tmp_path, caplog): data = {"metadata": {}, "strategy": {}, "strategy_comparison": []} config = { "exportfilename": tmp_path, "original_config": {}, } - store_backtest_results(config, data, "2022_01_01_15_05_13") + store_backtest_results( + config, data, "2022_01_01_15_05_13", strategy_files={"DefStrat": "NoFile"} + ) + assert log_has_re(r"Strategy file .* does not exist\. Skipping\.", caplog) zip_file = tmp_path / "backtest-result-2022_01_01_15_05_13.zip" assert zip_file.is_file() @@ -296,12 +300,17 @@ def test_store_backtest_results_real(tmp_path): strategy_test_dir = Path(__file__).parent.parent / "strategy" / "strats" + shutil.copy(strategy_test_dir / "strategy_test_v3.py", tmp_path) + params_file = tmp_path / "strategy_test_v3.json" + with params_file.open("w") as f: + f.write("""{"strategy_name": "TurtleStrategyX5","params":{}}""") + store_backtest_results( config, data, "2024_01_01_15_05_25", market_change_data=pd.DataFrame(), - strategy_files={"DefStrat": str(strategy_test_dir / "strategy_test_v3.py")}, + strategy_files={"DefStrat": str(tmp_path / "strategy_test_v3.py")}, ) zip_file = tmp_path / "backtest-result-2024_01_01_15_05_25.zip" assert zip_file.is_file() @@ -320,6 +329,12 @@ def test_store_backtest_results_real(tmp_path): with (strategy_test_dir / "strategy_test_v3.py").open("rb") as original_file: original_content = original_file.read() assert strategy_content == original_content + assert "backtest-result-2024_01_01_15_05_25_DefStrat.py" in zipf.namelist() + with zipf.open("backtest-result-2024_01_01_15_05_25_DefStrat.json") as pf: + params_content = pf.read() + with params_file.open("rb") as original_file: + original_content = original_file.read() + assert params_content == original_content assert (tmp_path / LAST_BT_RESULT_FN).is_file()