From 6571ef4e45b4249cb741478ede933398641e5e25 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Mon, 21 Oct 2024 11:14:28 +0200 Subject: [PATCH 1/4] fix: guarantee crash resiliency, as long as users reload bot gracefully --- freqtrade/freqai/freqai_interface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 950f40ca4..f935e97ef 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -185,6 +185,7 @@ class IFreqaiModel(ABC): Callback for Subclasses to override to include logic for shutting down resources when SIGINT is sent. """ + self.dd.save_historic_predictions_to_disk() return def shutdown(self): From a1513b7c7cd10087098e8da161ee46274bc290af Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Mon, 21 Oct 2024 11:40:46 +0200 Subject: [PATCH 2/4] fix: allow user to break training --- freqtrade/freqai/freqai_interface.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index f935e97ef..b0c376272 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -199,9 +199,16 @@ class IFreqaiModel(ABC): self.data_provider = None self._on_stop() - logger.info("Waiting on Training iteration") - for _thread in self._threads: - _thread.join() + if self.freqai_info.get("wait_for_training_iteration_on_reload", True): + logger.info("Waiting on Training iteration") + for _thread in self._threads: + _thread.join() + else: + logger.warning( + "Breaking current training iteration because " + "you set wait_for_training_iteration_on_reload to " + " False." + ) def start_scanning(self, *args, **kwargs) -> None: """ From d215e3ca680340d1631275327061501f3e890911 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Sun, 27 Oct 2024 19:20:19 +0100 Subject: [PATCH 3/4] chore: add documentation and add wait_for_training to config schema --- docs/freqai-parameter-table.md | 1 + freqtrade/configuration/config_schema.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/docs/freqai-parameter-table.md b/docs/freqai-parameter-table.md index 8a02faad2..3bb289313 100644 --- a/docs/freqai-parameter-table.md +++ b/docs/freqai-parameter-table.md @@ -22,6 +22,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the | `write_metrics_to_disk` | Collect train timings, inference timings and cpu usage in json file.
**Datatype:** Boolean.
Default: `False` | `data_kitchen_thread_count` |
Designate the number of threads you want to use for data processing (outlier methods, normalization, etc.). This has no impact on the number of threads used for training. If user does not set it (default), FreqAI will use max number of threads - 2 (leaving 1 physical core available for Freqtrade bot and FreqUI)
**Datatype:** Positive integer. | `activate_tensorboard` |
Indicate whether or not to activate tensorboard for the tensorboard enabled modules (currently Reinforcment Learning, XGBoost, Catboost, and PyTorch). Tensorboard needs Torch installed, which means you will need the torch/RL docker image or you need to answer "yes" to the install question about whether or not you wish to install Torch.
**Datatype:** Boolean.
Default: `True`. +| `wait_for_training_iteration_on_reload` |
When using /reload or ctrl-c, wait for the current training iteration to finish before completing graceful shutdown. If set to `False`, FreqAI will break the current training iteration, allowing you to shutdown gracefully more quickly, but you will lose your current training iteration.
**Datatype:** Boolean.
Default: `True`. ### Feature parameters diff --git a/freqtrade/configuration/config_schema.py b/freqtrade/configuration/config_schema.py index 9eb9c0477..2106d928d 100644 --- a/freqtrade/configuration/config_schema.py +++ b/freqtrade/configuration/config_schema.py @@ -995,6 +995,13 @@ CONF_SCHEMA = { "type": "string", "default": "example", }, + "wait_for_training_iteration_on_reload": { + "description": ( + "Wait for the next training iteration to complete after /reload or ctrl+c." + ), + "type": "boolean", + "default": True, + }, "feature_parameters": { "description": "The parameters used to engineer the feature set", "type": "object", From a3bcc9d91eefce840db1aa04e66e329e02011104 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Oct 2024 21:28:36 +0100 Subject: [PATCH 4/4] chore: schema.json updated --- build_helpers/schema.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build_helpers/schema.json b/build_helpers/schema.json index 6a73e75b0..6227829d9 100644 --- a/build_helpers/schema.json +++ b/build_helpers/schema.json @@ -1383,6 +1383,11 @@ "type": "string", "default": "example" }, + "wait_for_training_iteration_on_reload": { + "description": "Wait for the next training iteration to complete after /reload or ctrl+c.", + "type": "boolean", + "default": true + }, "feature_parameters": { "description": "The parameters used to engineer the feature set", "type": "object",