From 48a8d7de1c61e0825efbe0370df3c2d108b56872 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Thu, 3 Oct 2024 11:33:52 +0800 Subject: [PATCH 01/15] Postpone imports on demand --- freqtrade/commands/analyze_commands.py | 3 +- freqtrade/commands/build_config_commands.py | 23 ++++++---- freqtrade/commands/data_commands.py | 29 +++++++----- freqtrade/commands/db_commands.py | 5 +-- freqtrade/commands/deploy_commands.py | 15 ++++--- freqtrade/commands/hyperopt_commands.py | 8 ++-- freqtrade/commands/list_commands.py | 44 ++++++++++++++----- freqtrade/commands/optimize_commands.py | 8 +++- freqtrade/commands/pairlist_commands.py | 4 +- freqtrade/commands/plot_commands.py | 3 +- freqtrade/commands/strategy_utils_commands.py | 7 +-- freqtrade/data/btanalysis.py | 10 ++++- freqtrade/main.py | 3 +- freqtrade/{util/gc_setup.py => system.py} | 12 +++++ freqtrade/util/migrations/funding_rate_mig.py | 3 +- pyproject.toml | 2 + tests/commands/test_build_config.py | 15 +++---- tests/commands/test_commands.py | 34 +++++++------- tests/data/test_btanalysis.py | 2 +- tests/test_main.py | 6 +-- 20 files changed, 151 insertions(+), 85 deletions(-) rename freqtrade/{util/gc_setup.py => system.py} (61%) diff --git a/freqtrade/commands/analyze_commands.py b/freqtrade/commands/analyze_commands.py index 7d605a228..0b07d335b 100644 --- a/freqtrade/commands/analyze_commands.py +++ b/freqtrade/commands/analyze_commands.py @@ -2,7 +2,6 @@ import logging from pathlib import Path from typing import Any, Dict -from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode from freqtrade.exceptions import ConfigurationError, OperationalException @@ -17,6 +16,8 @@ def setup_analyze_configuration(args: Dict[str, Any], method: RunMode) -> Dict[s :param method: Bot running mode :return: Configuration """ + from freqtrade.configuration import setup_utils_configuration + config = setup_utils_configuration(args, method) no_unlimited_runmodes = { diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index cb64e4da9..3bcc5254f 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -3,17 +3,9 @@ import secrets from pathlib import Path from typing import Any, Dict, List -from questionary import Separator, prompt - -from freqtrade.configuration import sanitize_config -from freqtrade.configuration.config_setup import setup_utils_configuration -from freqtrade.configuration.detect_environment import running_in_docker -from freqtrade.configuration.directory_operations import chown_user_directory from freqtrade.constants import UNLIMITED_STAKE_AMOUNT from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS, available_exchanges -from freqtrade.util import render_template logger = logging.getLogger(__name__) @@ -36,6 +28,8 @@ def validate_is_float(val): def ask_user_overwrite(config_path: Path) -> bool: + from questionary import prompt + questions = [ { "type": "confirm", @@ -54,6 +48,11 @@ def ask_user_config() -> Dict[str, Any]: Interactive questions built using https://github.com/tmbo/questionary :returns: Dict with keys to put into template """ + from questionary import Separator, prompt + + from freqtrade.configuration.detect_environment import running_in_docker + from freqtrade.exchange import available_exchanges + questions: List[Dict[str, Any]] = [ { "type": "confirm", @@ -227,6 +226,9 @@ def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None: """ from jinja2.exceptions import TemplateNotFound + from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS + from freqtrade.util import render_template + try: exchange_template = MAP_EXCHANGE_CHILDCLASS.get( selections["exchange_name"], selections["exchange_name"] @@ -256,6 +258,8 @@ def start_new_config(args: Dict[str, Any]) -> None: Asking the user questions to fill out the template accordingly. """ + from freqtrade.configuration.directory_operations import chown_user_directory + config_path = Path(args["config"][0]) chown_user_directory(config_path.parent) if config_path.exists(): @@ -272,6 +276,9 @@ def start_new_config(args: Dict[str, Any]) -> None: def start_show_config(args: Dict[str, Any]) -> None: + from freqtrade.configuration import sanitize_config + from freqtrade.configuration.config_setup import setup_utils_configuration + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE, set_dry=False) print("Your combined configuration is:") diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index a114444b3..9ce039b22 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -3,22 +3,10 @@ import sys from collections import defaultdict from typing import Any, Dict -from freqtrade.configuration import TimeRange, setup_utils_configuration from freqtrade.constants import DATETIME_PRINT_FORMAT, DL_DATA_TIMEFRAMES, Config -from freqtrade.data.converter import ( - convert_ohlcv_format, - convert_trades_format, - convert_trades_to_ohlcv, -) -from freqtrade.data.history import download_data_main from freqtrade.enums import CandleType, RunMode, TradingMode from freqtrade.exceptions import ConfigurationError -from freqtrade.exchange import timeframe_to_minutes -from freqtrade.misc import plural from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist -from freqtrade.resolvers import ExchangeResolver -from freqtrade.util import print_rich_table -from freqtrade.util.migrations import migrate_data logger = logging.getLogger(__name__) @@ -42,6 +30,9 @@ def start_download_data(args: Dict[str, Any]) -> None: """ Download data (former download_backtest_data.py script) """ + from freqtrade.configuration import setup_utils_configuration + from freqtrade.data.history import download_data_main + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) _check_data_config_download_sanity(config) @@ -54,6 +45,10 @@ def start_download_data(args: Dict[str, Any]) -> None: def start_convert_trades(args: Dict[str, Any]) -> None: + from freqtrade.configuration import TimeRange, setup_utils_configuration + from freqtrade.data.converter import convert_trades_to_ohlcv + from freqtrade.resolvers import ExchangeResolver + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) timerange = TimeRange() @@ -96,6 +91,10 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: """ Convert data from one format to another """ + from freqtrade.configuration import setup_utils_configuration + from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format + from freqtrade.util.migrations import migrate_data + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) if ohlcv: migrate_data(config) @@ -118,6 +117,9 @@ def start_list_data(args: Dict[str, Any]) -> None: """ List available OHLCV data """ + from freqtrade.configuration import setup_utils_configuration + from freqtrade.exchange import timeframe_to_minutes + from freqtrade.util import print_rich_table if args["trades"]: start_list_trades_data(args) @@ -181,6 +183,9 @@ def start_list_trades_data(args: Dict[str, Any]) -> None: """ List available Trades data """ + from freqtrade.configuration import setup_utils_configuration + from freqtrade.misc import plural + from freqtrade.util import print_rich_table config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) diff --git a/freqtrade/commands/db_commands.py b/freqtrade/commands/db_commands.py index 98af38ca4..1c346ad7a 100644 --- a/freqtrade/commands/db_commands.py +++ b/freqtrade/commands/db_commands.py @@ -1,9 +1,6 @@ import logging from typing import Any, Dict -from sqlalchemy import func, select - -from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.enums import RunMode @@ -11,8 +8,10 @@ logger = logging.getLogger(__name__) def start_convert_db(args: Dict[str, Any]) -> None: + from sqlalchemy import func, select from sqlalchemy.orm import make_transient + from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.persistence import Order, Trade, init_db from freqtrade.persistence.migrations import set_sequence_ids from freqtrade.persistence.pairlock import PairLock diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index 3a784bda9..f96716d7b 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -3,14 +3,9 @@ import sys from pathlib import Path from typing import Any, Dict, Optional, Tuple -import requests - -from freqtrade.configuration import setup_utils_configuration -from freqtrade.configuration.directory_operations import copy_sample_files, create_userdata_dir from freqtrade.constants import USERPATH_STRATEGIES from freqtrade.enums import RunMode from freqtrade.exceptions import ConfigurationError, OperationalException -from freqtrade.util import render_template, render_template_with_fallback logger = logging.getLogger(__name__) @@ -26,6 +21,8 @@ def start_create_userdir(args: Dict[str, Any]) -> None: :param args: Cli args from Arguments() :return: None """ + from freqtrade.configuration.directory_operations import copy_sample_files, create_userdata_dir + if "user_data_dir" in args and args["user_data_dir"]: userdir = create_userdata_dir(args["user_data_dir"], create_dir=True) copy_sample_files(userdir, overwrite=args["reset"]) @@ -38,6 +35,8 @@ def deploy_new_strategy(strategy_name: str, strategy_path: Path, subtemplate: st """ Deploy new strategy from template to strategy_path """ + from freqtrade.util import render_template, render_template_with_fallback + fallback = "full" attributes = render_template_with_fallback( templatefile=f"strategy_subtemplates/strategy_attributes_{subtemplate}.j2", @@ -82,6 +81,8 @@ def deploy_new_strategy(strategy_name: str, strategy_path: Path, subtemplate: st def start_new_strategy(args: Dict[str, Any]) -> None: + from freqtrade.configuration import setup_utils_configuration + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) if "strategy" in args and args["strategy"]: @@ -124,6 +125,8 @@ def download_and_install_ui(dest_folder: Path, dl_url: str, version: str): from io import BytesIO from zipfile import ZipFile + import requests + logger.info(f"Downloading {dl_url}") resp = requests.get(dl_url, timeout=req_timeout).content dest_folder.mkdir(parents=True, exist_ok=True) @@ -140,6 +143,8 @@ def download_and_install_ui(dest_folder: Path, dl_url: str, version: str): def get_ui_download_url(version: Optional[str] = None) -> Tuple[str, str]: + import requests + base_url = "https://api.github.com/repos/freqtrade/frequi/" # Get base UI Repo path diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index d89d25796..ccf319e9e 100644 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -2,11 +2,8 @@ import logging from operator import itemgetter from typing import Any, Dict -from freqtrade.configuration import setup_utils_configuration -from freqtrade.data.btanalysis import get_latest_hyperopt_file from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.optimize.optimize_reports import show_backtest_result logger = logging.getLogger(__name__) @@ -16,6 +13,8 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: """ List hyperopt epochs previously evaluated """ + from freqtrade.configuration import setup_utils_configuration + from freqtrade.data.btanalysis import get_latest_hyperopt_file from freqtrade.optimize.hyperopt_output import HyperoptOutput from freqtrade.optimize.hyperopt_tools import HyperoptTools @@ -61,7 +60,10 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: """ Show details of a hyperopt epoch previously evaluated """ + from freqtrade.configuration import setup_utils_configuration + from freqtrade.data.btanalysis import get_latest_hyperopt_file from freqtrade.optimize.hyperopt_tools import HyperoptTools + from freqtrade.optimize.optimize_reports import show_backtest_result config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index af6e4571f..c99830aa8 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -1,21 +1,19 @@ +from __future__ import annotations + import csv import logging import sys +import typing from typing import Any, Dict, List, Union import rapidjson -from rich.console import Console -from rich.table import Table -from rich.text import Text -from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode from freqtrade.exceptions import ConfigurationError, OperationalException -from freqtrade.exchange import list_available_exchanges, market_is_active -from freqtrade.ft_types import ValidExchangesType -from freqtrade.misc import parse_db_uri_for_logging, plural -from freqtrade.resolvers import ExchangeResolver, StrategyResolver -from freqtrade.util import print_rich_table + + +if typing.TYPE_CHECKING: + from freqtrade.ft_types import ValidExchangesType logger = logging.getLogger(__name__) @@ -27,6 +25,12 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: :param args: Cli args from Arguments() :return: None """ + from rich.console import Console + from rich.table import Table + from rich.text import Text + + from freqtrade.exchange import list_available_exchanges + available_exchanges: List[ValidExchangesType] = list_available_exchanges( args["list_exchanges_all"] ) @@ -86,6 +90,10 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: def _print_objs_tabular(objs: List, print_colorized: bool) -> None: + from rich.console import Console + from rich.table import Table + from rich.text import Text + names = [s["name"] for s in objs] objs_to_print: List[Dict[str, Union[Text, str]]] = [ { @@ -129,6 +137,9 @@ def start_list_strategies(args: Dict[str, Any]) -> None: """ Print files with Strategy custom classes available in the directory """ + from freqtrade.configuration import setup_utils_configuration + from freqtrade.resolvers import StrategyResolver + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) strategy_objs = StrategyResolver.search_all_objects( @@ -152,9 +163,11 @@ def start_list_freqAI_models(args: Dict[str, Any]) -> None: """ Print files with FreqAI models custom classes available in the directory """ - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + from freqtrade.configuration import setup_utils_configuration from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + model_objs = FreqaiModelResolver.search_all_objects(config, not args["print_one_column"]) # Sort alphabetically model_objs = sorted(model_objs, key=lambda x: x["name"]) @@ -168,6 +181,9 @@ def start_list_timeframes(args: Dict[str, Any]) -> None: """ Print timeframes available on Exchange """ + from freqtrade.configuration import setup_utils_configuration + from freqtrade.resolvers import ExchangeResolver + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) # Do not use timeframe set in the config config["timeframe"] = None @@ -191,6 +207,12 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: :param pairs_only: if True print only pairs, otherwise print all instruments (markets) :return: None """ + from freqtrade.configuration import setup_utils_configuration + from freqtrade.exchange import market_is_active + from freqtrade.misc import plural + from freqtrade.resolvers import ExchangeResolver + from freqtrade.util import print_rich_table + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) # Init exchange @@ -302,6 +324,8 @@ def start_show_trades(args: Dict[str, Any]) -> None: """ import json + from freqtrade.configuration import setup_utils_configuration + from freqtrade.misc import parse_db_uri_for_logging from freqtrade.persistence import Trade, init_db config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index aa055469a..d8240f660 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -2,10 +2,8 @@ import logging from typing import Any, Dict from freqtrade import constants -from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode from freqtrade.exceptions import ConfigurationError, OperationalException -from freqtrade.util import fmt_coin logger = logging.getLogger(__name__) @@ -18,6 +16,9 @@ def setup_optimize_configuration(args: Dict[str, Any], method: RunMode) -> Dict[ :param method: Bot running mode :return: Configuration """ + from freqtrade.configuration import setup_utils_configuration + from freqtrade.util import fmt_coin + config = setup_utils_configuration(args, method) no_unlimited_runmodes = { @@ -64,6 +65,7 @@ def start_backtesting_show(args: Dict[str, Any]) -> None: """ Show previous backtest result """ + from freqtrade.configuration import setup_utils_configuration config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) @@ -144,6 +146,7 @@ def start_lookahead_analysis(args: Dict[str, Any]) -> None: :param args: Cli args from Arguments() :return: None """ + from freqtrade.configuration import setup_utils_configuration from freqtrade.optimize.analysis.lookahead_helpers import LookaheadAnalysisSubFunctions config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) @@ -156,6 +159,7 @@ def start_recursive_analysis(args: Dict[str, Any]) -> None: :param args: Cli args from Arguments() :return: None """ + from freqtrade.configuration import setup_utils_configuration from freqtrade.optimize.analysis.recursive_helpers import RecursiveAnalysisSubFunctions config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) diff --git a/freqtrade/commands/pairlist_commands.py b/freqtrade/commands/pairlist_commands.py index 8b2963563..b027a222d 100644 --- a/freqtrade/commands/pairlist_commands.py +++ b/freqtrade/commands/pairlist_commands.py @@ -3,9 +3,7 @@ from typing import Any, Dict import rapidjson -from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode -from freqtrade.resolvers import ExchangeResolver logger = logging.getLogger(__name__) @@ -15,8 +13,10 @@ def start_test_pairlist(args: Dict[str, Any]) -> None: """ Test Pairlist configuration """ + from freqtrade.configuration import setup_utils_configuration from freqtrade.persistence import FtNoDBContext from freqtrade.plugins.pairlistmanager import PairListManager + from freqtrade.resolvers import ExchangeResolver config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) diff --git a/freqtrade/commands/plot_commands.py b/freqtrade/commands/plot_commands.py index 4b939cc80..4ed698bf2 100644 --- a/freqtrade/commands/plot_commands.py +++ b/freqtrade/commands/plot_commands.py @@ -1,6 +1,5 @@ from typing import Any, Dict -from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode from freqtrade.exceptions import ConfigurationError @@ -18,6 +17,7 @@ def start_plot_dataframe(args: Dict[str, Any]) -> None: Entrypoint for dataframe plotting """ # Import here to avoid errors if plot-dependencies are not installed. + from freqtrade.configuration import setup_utils_configuration from freqtrade.plot.plotting import load_and_plot_trades validate_plot_args(args) @@ -31,6 +31,7 @@ def start_plot_profit(args: Dict[str, Any]) -> None: Entrypoint for plot_profit """ # Import here to avoid errors if plot-dependencies are not installed. + from freqtrade.configuration import setup_utils_configuration from freqtrade.plot.plotting import plot_profit validate_plot_args(args) diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index 761a7262c..785e89aee 100644 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -4,10 +4,7 @@ import time from pathlib import Path from typing import Any, Dict -from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode -from freqtrade.resolvers import StrategyResolver -from freqtrade.strategy.strategyupdater import StrategyUpdater logger = logging.getLogger(__name__) @@ -19,6 +16,8 @@ def start_strategy_update(args: Dict[str, Any]) -> None: :param args: Cli args from Arguments() :return: None """ + from freqtrade.configuration import setup_utils_configuration + from freqtrade.resolvers import StrategyResolver if sys.version_info == (3, 8): # pragma: no cover sys.exit("Freqtrade strategy updater requires Python version >= 3.9") @@ -49,6 +48,8 @@ def start_strategy_update(args: Dict[str, Any]) -> None: def start_conversion(strategy_obj, config): + from freqtrade.strategy.strategyupdater import StrategyUpdater + print(f"Conversion of {Path(strategy_obj['location']).name} started.") instance_strategy_updater = StrategyUpdater() start = time.perf_counter() diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 580807a76..ad90014c2 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -2,7 +2,10 @@ Helpers when analyzing backtest data """ +from __future__ import annotations + import logging +import typing from copy import copy from datetime import datetime, timezone from pathlib import Path @@ -16,7 +19,10 @@ from freqtrade.exceptions import ConfigurationError, OperationalException from freqtrade.ft_types import BacktestHistoryEntryType, BacktestResultType from freqtrade.misc import file_dump_json, json_load from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename -from freqtrade.persistence import LocalTrade, Trade, init_db + + +if typing.TYPE_CHECKING: + from freqtrade.persistence import LocalTrade, Trade logger = logging.getLogger(__name__) @@ -460,6 +466,8 @@ def load_trades_from_db(db_url: str, strategy: Optional[str] = None) -> pd.DataF Can also serve as protection to load the correct result. :return: Dataframe containing Trades """ + from freqtrade.persistence import Trade, init_db + init_db(db_url) filters = [] diff --git a/freqtrade/main.py b/freqtrade/main.py index 67584c5b7..dd034477a 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -15,11 +15,10 @@ if sys.version_info < (3, 9): # pragma: no cover from freqtrade import __version__ from freqtrade.commands import Arguments -from freqtrade.configuration import asyncio_setup from freqtrade.constants import DOCS_LINK from freqtrade.exceptions import ConfigurationError, FreqtradeException, OperationalException from freqtrade.loggers import setup_logging_pre -from freqtrade.util.gc_setup import gc_set_threshold +from freqtrade.system import asyncio_setup, gc_set_threshold logger = logging.getLogger("freqtrade") diff --git a/freqtrade/util/gc_setup.py b/freqtrade/system.py similarity index 61% rename from freqtrade/util/gc_setup.py rename to freqtrade/system.py index a3532cbab..c7be18ec0 100644 --- a/freqtrade/util/gc_setup.py +++ b/freqtrade/system.py @@ -1,11 +1,23 @@ +"""System specific or performance tuning""" + import gc import logging import platform +import sys logger = logging.getLogger(__name__) +def asyncio_setup() -> None: # pragma: no cover + # Set eventloop for win32 setups + + if sys.platform == "win32": + import asyncio + + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + + def gc_set_threshold(): """ Reduce number of GC runs to improve performance (explanation video) diff --git a/freqtrade/util/migrations/funding_rate_mig.py b/freqtrade/util/migrations/funding_rate_mig.py index 16ca60732..e53ec339e 100644 --- a/freqtrade/util/migrations/funding_rate_mig.py +++ b/freqtrade/util/migrations/funding_rate_mig.py @@ -2,7 +2,6 @@ import logging from typing import Optional from freqtrade.constants import Config -from freqtrade.data.history import get_datahandler from freqtrade.enums import TradingMode from freqtrade.exchange import Exchange @@ -11,6 +10,8 @@ logger = logging.getLogger(__name__) def migrate_funding_fee_timeframe(config: Config, exchange: Optional[Exchange]): + from freqtrade.data.history import get_datahandler + if config.get("trading_mode", TradingMode.SPOT) != TradingMode.FUTURES: # only act on futures return diff --git a/pyproject.toml b/pyproject.toml index bd36d15c6..0229b8284 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -160,6 +160,8 @@ extend-ignore = [ "S607", # Starting a process with a partial executable path "S608", # Possible SQL injection vector through string-based query construction "NPY002", # Numpy legacy random generator + "UP006", # non-pep585-annotation + "UP007", # non-pep604-annotation ] [tool.ruff.lint.mccabe] diff --git a/tests/commands/test_build_config.py b/tests/commands/test_build_config.py index 5d287a35f..6efda2ff8 100644 --- a/tests/commands/test_build_config.py +++ b/tests/commands/test_build_config.py @@ -90,16 +90,13 @@ def test_ask_user_overwrite(mocker): """ Once https://github.com/tmbo/questionary/issues/35 is implemented, improve this test. """ - prompt_mock = mocker.patch( - "freqtrade.commands.build_config_commands.prompt", return_value={"overwrite": False} - ) + prompt_mock = mocker.patch("questionary.prompt", return_value={"overwrite": False}) assert not ask_user_overwrite(Path("test.json")) assert prompt_mock.call_count == 1 prompt_mock.reset_mock() - prompt_mock = mocker.patch( - "freqtrade.commands.build_config_commands.prompt", return_value={"overwrite": True} - ) + + prompt_mock = mocker.patch("questionary.prompt", return_value={"overwrite": True}) assert ask_user_overwrite(Path("test.json")) assert prompt_mock.call_count == 1 @@ -108,14 +105,12 @@ def test_ask_user_config(mocker): """ Once https://github.com/tmbo/questionary/issues/35 is implemented, improve this test. """ - prompt_mock = mocker.patch( - "freqtrade.commands.build_config_commands.prompt", return_value={"overwrite": False} - ) + prompt_mock = mocker.patch("questionary.prompt", return_value={"overwrite": False}) answers = ask_user_config() assert isinstance(answers, dict) assert prompt_mock.call_count == 1 - prompt_mock = mocker.patch("freqtrade.commands.build_config_commands.prompt", return_value={}) + prompt_mock = mocker.patch("questionary.prompt", return_value={}) with pytest.raises(OperationalException, match=r"User interrupted interactive questions\."): ask_user_config() diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index c55126db1..9be7f9506 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -571,8 +571,12 @@ def test_create_datadir_failed(caplog): def test_create_datadir(caplog, mocker): - cud = mocker.patch("freqtrade.commands.deploy_commands.create_userdata_dir", MagicMock()) - csf = mocker.patch("freqtrade.commands.deploy_commands.copy_sample_files", MagicMock()) + cud = mocker.patch( + "freqtrade.configuration.directory_operations.create_userdata_dir", MagicMock() + ) + csf = mocker.patch( + "freqtrade.configuration.directory_operations.copy_sample_files", MagicMock() + ) args = ["create-userdir", "--userdir", "/temp/freqtrade/test"] start_create_userdir(get_args(args)) @@ -591,7 +595,7 @@ def test_start_new_strategy(mocker, caplog): assert "CoolNewStrategy" in wt_mock.call_args_list[0][0][0] assert log_has_re("Writing strategy to .*", caplog) - mocker.patch("freqtrade.commands.deploy_commands.setup_utils_configuration") + mocker.patch("freqtrade.configuration.setup_utils_configuration") mocker.patch.object(Path, "exists", MagicMock(return_value=True)) with pytest.raises( OperationalException, match=r".* already exists. Please choose another Strategy Name\." @@ -664,7 +668,7 @@ def test_download_and_install_ui(mocker, tmp_path): file_like_object.seek(0) requests_mock.content = file_like_object.read() - mocker.patch("freqtrade.commands.deploy_commands.requests.get", return_value=requests_mock) + mocker.patch("requests.get", return_value=requests_mock) mocker.patch("freqtrade.commands.deploy_commands.Path.is_dir", side_effect=[True, False]) wb_mock = mocker.patch("freqtrade.commands.deploy_commands.Path.write_bytes") @@ -689,9 +693,7 @@ def test_get_ui_download_url(mocker): [{"browser_download_url": "http://download.zip"}], ] ) - get_mock = mocker.patch( - "freqtrade.commands.deploy_commands.requests.get", return_value=response - ) + get_mock = mocker.patch("requests.get", return_value=response) x, last_version = get_ui_download_url() assert get_mock.call_count == 2 assert last_version == "0.0.1" @@ -714,9 +716,7 @@ def test_get_ui_download_url_direct(mocker): }, ] ) - get_mock = mocker.patch( - "freqtrade.commands.deploy_commands.requests.get", return_value=response - ) + get_mock = mocker.patch("requests.get", return_value=response) x, last_version = get_ui_download_url() assert get_mock.call_count == 1 assert last_version == "0.0.2" @@ -734,7 +734,7 @@ def test_get_ui_download_url_direct(mocker): def test_download_data_keyboardInterrupt(mocker, markets): dl_mock = mocker.patch( - "freqtrade.commands.data_commands.download_data_main", + "freqtrade.data.history.download_data_main", MagicMock(side_effect=KeyboardInterrupt), ) patch_exchange(mocker) @@ -972,7 +972,7 @@ def test_download_data_data_invalid(mocker): def test_start_convert_trades(mocker): convert_mock = mocker.patch( - "freqtrade.commands.data_commands.convert_trades_to_ohlcv", MagicMock(return_value=[]) + "freqtrade.data.converter.convert_trades_to_ohlcv", MagicMock(return_value=[]) ) patch_exchange(mocker) mocker.patch(f"{EXMS}.get_markets") @@ -1522,7 +1522,7 @@ def test_hyperopt_show(mocker, capsys): mocker.patch( "freqtrade.optimize.hyperopt_tools.HyperoptTools._read_results", side_effect=fake_iterator ) - mocker.patch("freqtrade.commands.hyperopt_commands.show_backtest_result") + mocker.patch("freqtrade.optimize.optimize_reports.show_backtest_result") args = [ "hyperopt-show", @@ -1579,8 +1579,8 @@ def test_hyperopt_show(mocker, capsys): def test_convert_data(mocker, testdatadir): - ohlcv_mock = mocker.patch("freqtrade.commands.data_commands.convert_ohlcv_format") - trades_mock = mocker.patch("freqtrade.commands.data_commands.convert_trades_format") + ohlcv_mock = mocker.patch("freqtrade.data.converter.convert_ohlcv_format") + trades_mock = mocker.patch("freqtrade.data.converter.convert_trades_format") args = [ "convert-data", "--format-from", @@ -1601,8 +1601,8 @@ def test_convert_data(mocker, testdatadir): def test_convert_data_trades(mocker, testdatadir): - ohlcv_mock = mocker.patch("freqtrade.commands.data_commands.convert_ohlcv_format") - trades_mock = mocker.patch("freqtrade.commands.data_commands.convert_trades_format") + ohlcv_mock = mocker.patch("freqtrade.data.converter.convert_ohlcv_format") + trades_mock = mocker.patch("freqtrade.data.converter.convert_trades_format") args = [ "convert-trade-data", "--format-from", diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index b9dee6d59..691b02fc0 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -146,7 +146,7 @@ def test_load_backtest_data_multi(testdatadir): def test_load_trades_from_db(default_conf, fee, is_short, mocker): create_mock_trades(fee, is_short) # remove init so it does not init again - init_mock = mocker.patch("freqtrade.data.btanalysis.init_db", MagicMock()) + init_mock = mocker.patch("freqtrade.persistence.init_db", MagicMock()) trades = load_trades_from_db(db_url=default_conf["db_url"]) assert init_mock.call_count == 1 diff --git a/tests/test_main.py b/tests/test_main.py index b230d4e99..3c8150c38 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -121,7 +121,7 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None: def test_main_operational_exception1(mocker, default_conf, caplog) -> None: patch_exchange(mocker) mocker.patch( - "freqtrade.commands.list_commands.list_available_exchanges", + "freqtrade.exchange.list_available_exchanges", MagicMock(side_effect=ValueError("Oh snap!")), ) patched_configuration_load_config_file(mocker, default_conf) @@ -135,7 +135,7 @@ def test_main_operational_exception1(mocker, default_conf, caplog) -> None: assert log_has("Fatal exception!", caplog) assert not log_has_re(r"SIGINT.*", caplog) mocker.patch( - "freqtrade.commands.list_commands.list_available_exchanges", + "freqtrade.exchange.list_available_exchanges", MagicMock(side_effect=KeyboardInterrupt), ) with pytest.raises(SystemExit): @@ -147,7 +147,7 @@ def test_main_operational_exception1(mocker, default_conf, caplog) -> None: def test_main_ConfigurationError(mocker, default_conf, caplog) -> None: patch_exchange(mocker) mocker.patch( - "freqtrade.commands.list_commands.list_available_exchanges", + "freqtrade.exchange.list_available_exchanges", MagicMock(side_effect=ConfigurationError("Oh snap!")), ) patched_configuration_load_config_file(mocker, default_conf) From 9c28a6ff4a75e2adbac9b185d5a826baaab9e13b Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Thu, 3 Oct 2024 12:07:03 +0800 Subject: [PATCH 02/15] Add test --- tests/commands/test_startup_time.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 tests/commands/test_startup_time.py diff --git a/tests/commands/test_startup_time.py b/tests/commands/test_startup_time.py new file mode 100644 index 000000000..6ffaa51c6 --- /dev/null +++ b/tests/commands/test_startup_time.py @@ -0,0 +1,14 @@ +import subprocess +import time + + +MAXIMUM_STARTUP_TIME = 0.5 + + +def test_startup_time(): + start = time.time() + subprocess.run(["freqtrade", "-h"]) + elapsed = time.time() - start + assert ( + elapsed < MAXIMUM_STARTUP_TIME + ), "The startup time is too long, try to use lazy import in the command entry function" From e6dedee56f3eae6d58078760c8f1c0c323906ec7 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Fri, 4 Oct 2024 11:05:49 +0800 Subject: [PATCH 03/15] Split freqtrade.system into a package --- freqtrade/configuration/__init__.py | 1 - freqtrade/main.py | 3 ++- freqtrade/system/__init__.py | 1 + .../{configuration => system}/asyncio_config.py | 0 freqtrade/{system.py => system/gc_setup.py} | 12 ------------ 5 files changed, 3 insertions(+), 14 deletions(-) create mode 100644 freqtrade/system/__init__.py rename freqtrade/{configuration => system}/asyncio_config.py (100%) rename freqtrade/{system.py => system/gc_setup.py} (61%) diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index 8fe65a9b0..aa06a70c9 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,6 +1,5 @@ # flake8: noqa: F401 -from freqtrade.configuration.asyncio_config import asyncio_setup from freqtrade.configuration.config_secrets import sanitize_config from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.configuration.config_validation import validate_config_consistency diff --git a/freqtrade/main.py b/freqtrade/main.py index 8047497e3..2dcc5a7a8 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -18,7 +18,8 @@ from freqtrade.commands import Arguments from freqtrade.constants import DOCS_LINK from freqtrade.exceptions import ConfigurationError, FreqtradeException, OperationalException from freqtrade.loggers import setup_logging_pre -from freqtrade.system import asyncio_setup, gc_set_threshold +from freqtrade.system.asyncio_config import asyncio_setup +from freqtrade.system.gc_setup import gc_set_threshold logger = logging.getLogger("freqtrade") diff --git a/freqtrade/system/__init__.py b/freqtrade/system/__init__.py new file mode 100644 index 000000000..323ef7196 --- /dev/null +++ b/freqtrade/system/__init__.py @@ -0,0 +1 @@ +"""system specific and performance tuning""" diff --git a/freqtrade/configuration/asyncio_config.py b/freqtrade/system/asyncio_config.py similarity index 100% rename from freqtrade/configuration/asyncio_config.py rename to freqtrade/system/asyncio_config.py diff --git a/freqtrade/system.py b/freqtrade/system/gc_setup.py similarity index 61% rename from freqtrade/system.py rename to freqtrade/system/gc_setup.py index c7be18ec0..a3532cbab 100644 --- a/freqtrade/system.py +++ b/freqtrade/system/gc_setup.py @@ -1,23 +1,11 @@ -"""System specific or performance tuning""" - import gc import logging import platform -import sys logger = logging.getLogger(__name__) -def asyncio_setup() -> None: # pragma: no cover - # Set eventloop for win32 setups - - if sys.platform == "win32": - import asyncio - - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - - def gc_set_threshold(): """ Reduce number of GC runs to improve performance (explanation video) From 0fe772efa82536a9c995dd72ff261490f39e9716 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Fri, 4 Oct 2024 11:10:39 +0800 Subject: [PATCH 04/15] Eliminate unnecessary TYPE_CHECKING --- freqtrade/commands/list_commands.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index c99830aa8..d4ae4cf43 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -1,19 +1,13 @@ -from __future__ import annotations - import csv import logging import sys -import typing from typing import Any, Dict, List, Union import rapidjson from freqtrade.enums import RunMode from freqtrade.exceptions import ConfigurationError, OperationalException - - -if typing.TYPE_CHECKING: - from freqtrade.ft_types import ValidExchangesType +from freqtrade.ft_types import ValidExchangesType logger = logging.getLogger(__name__) From 903614140ae8549c6659af7aef8332df39354c2e Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Fri, 4 Oct 2024 11:22:47 +0800 Subject: [PATCH 05/15] Eliminate unnecessary lazy import --- freqtrade/commands/build_config_commands.py | 5 ++--- tests/commands/test_build_config.py | 16 ++++++++++++---- tests/commands/test_commands.py | 8 ++++++-- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 3bcc5254f..27e3f395a 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -3,6 +3,8 @@ import secrets from pathlib import Path from typing import Any, Dict, List +from questionary import Separator, prompt + from freqtrade.constants import UNLIMITED_STAKE_AMOUNT from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException @@ -28,8 +30,6 @@ def validate_is_float(val): def ask_user_overwrite(config_path: Path) -> bool: - from questionary import prompt - questions = [ { "type": "confirm", @@ -48,7 +48,6 @@ def ask_user_config() -> Dict[str, Any]: Interactive questions built using https://github.com/tmbo/questionary :returns: Dict with keys to put into template """ - from questionary import Separator, prompt from freqtrade.configuration.detect_environment import running_in_docker from freqtrade.exchange import available_exchanges diff --git a/tests/commands/test_build_config.py b/tests/commands/test_build_config.py index 6efda2ff8..805d76e25 100644 --- a/tests/commands/test_build_config.py +++ b/tests/commands/test_build_config.py @@ -90,13 +90,17 @@ def test_ask_user_overwrite(mocker): """ Once https://github.com/tmbo/questionary/issues/35 is implemented, improve this test. """ - prompt_mock = mocker.patch("questionary.prompt", return_value={"overwrite": False}) + prompt_mock = mocker.patch( + "freqtrade.commands.build_config_commands.prompt", return_value={"overwrite": False} + ) assert not ask_user_overwrite(Path("test.json")) assert prompt_mock.call_count == 1 prompt_mock.reset_mock() - prompt_mock = mocker.patch("questionary.prompt", return_value={"overwrite": True}) + prompt_mock = mocker.patch( + "freqtrade.commands.build_config_commands.prompt", return_value={"overwrite": False} + ) assert ask_user_overwrite(Path("test.json")) assert prompt_mock.call_count == 1 @@ -105,12 +109,16 @@ def test_ask_user_config(mocker): """ Once https://github.com/tmbo/questionary/issues/35 is implemented, improve this test. """ - prompt_mock = mocker.patch("questionary.prompt", return_value={"overwrite": False}) + prompt_mock = mocker.patch( + "freqtrade.commands.build_config_commands.prompt", return_value={"overwrite": False} + ) answers = ask_user_config() assert isinstance(answers, dict) assert prompt_mock.call_count == 1 - prompt_mock = mocker.patch("questionary.prompt", return_value={}) + prompt_mock = mocker.patch( + "freqtrade.commands.build_config_commands.prompt", return_value={"overwrite": False} + ) with pytest.raises(OperationalException, match=r"User interrupted interactive questions\."): ask_user_config() diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 9be7f9506..ebc83e5e9 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -693,7 +693,9 @@ def test_get_ui_download_url(mocker): [{"browser_download_url": "http://download.zip"}], ] ) - get_mock = mocker.patch("requests.get", return_value=response) + get_mock = mocker.patch( + "freqtrade.commands.deploy_commands.requests.get", return_value=response + ) x, last_version = get_ui_download_url() assert get_mock.call_count == 2 assert last_version == "0.0.1" @@ -716,7 +718,9 @@ def test_get_ui_download_url_direct(mocker): }, ] ) - get_mock = mocker.patch("requests.get", return_value=response) + get_mock = mocker.patch( + "freqtrade.commands.deploy_commands.requests.get", return_value=response + ) x, last_version = get_ui_download_url() assert get_mock.call_count == 1 assert last_version == "0.0.2" From 30a273c946d3a939dd675e4600466ce087c07572 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Fri, 4 Oct 2024 11:36:51 +0800 Subject: [PATCH 06/15] Revert lazy import requests --- freqtrade/commands/deploy_commands.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index f96716d7b..d610156ba 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -3,6 +3,8 @@ import sys from pathlib import Path from typing import Any, Dict, Optional, Tuple +import requests + from freqtrade.constants import USERPATH_STRATEGIES from freqtrade.enums import RunMode from freqtrade.exceptions import ConfigurationError, OperationalException @@ -125,8 +127,6 @@ def download_and_install_ui(dest_folder: Path, dl_url: str, version: str): from io import BytesIO from zipfile import ZipFile - import requests - logger.info(f"Downloading {dl_url}") resp = requests.get(dl_url, timeout=req_timeout).content dest_folder.mkdir(parents=True, exist_ok=True) @@ -143,8 +143,6 @@ def download_and_install_ui(dest_folder: Path, dl_url: str, version: str): def get_ui_download_url(version: Optional[str] = None) -> Tuple[str, str]: - import requests - base_url = "https://api.github.com/repos/freqtrade/frequi/" # Get base UI Repo path From 852a1900b46fa94f8aa113133ff0348d53474233 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Fri, 4 Oct 2024 11:52:07 +0800 Subject: [PATCH 07/15] Fix test --- tests/commands/test_startup_time.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/commands/test_startup_time.py b/tests/commands/test_startup_time.py index 6ffaa51c6..37e2eeea2 100644 --- a/tests/commands/test_startup_time.py +++ b/tests/commands/test_startup_time.py @@ -6,6 +6,9 @@ MAXIMUM_STARTUP_TIME = 0.5 def test_startup_time(): + # warm up to generate pyc + subprocess.run(["freqtrade", "-h"]) + start = time.time() subprocess.run(["freqtrade", "-h"]) elapsed = time.time() - start From f889c2449713ef34cfcd1e6f77d3a79661e44942 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Fri, 4 Oct 2024 11:56:38 +0800 Subject: [PATCH 08/15] Fix test --- tests/commands/test_build_config.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/commands/test_build_config.py b/tests/commands/test_build_config.py index 805d76e25..5d287a35f 100644 --- a/tests/commands/test_build_config.py +++ b/tests/commands/test_build_config.py @@ -97,9 +97,8 @@ def test_ask_user_overwrite(mocker): assert prompt_mock.call_count == 1 prompt_mock.reset_mock() - prompt_mock = mocker.patch( - "freqtrade.commands.build_config_commands.prompt", return_value={"overwrite": False} + "freqtrade.commands.build_config_commands.prompt", return_value={"overwrite": True} ) assert ask_user_overwrite(Path("test.json")) assert prompt_mock.call_count == 1 @@ -116,9 +115,7 @@ def test_ask_user_config(mocker): assert isinstance(answers, dict) assert prompt_mock.call_count == 1 - prompt_mock = mocker.patch( - "freqtrade.commands.build_config_commands.prompt", return_value={"overwrite": False} - ) + prompt_mock = mocker.patch("freqtrade.commands.build_config_commands.prompt", return_value={}) with pytest.raises(OperationalException, match=r"User interrupted interactive questions\."): ask_user_config() From 1ad4f0c9b51060ca8aa8c297f59b0725f2032495 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Oct 2024 11:30:17 +0200 Subject: [PATCH 09/15] chore: re-export from system module --- freqtrade/main.py | 3 +-- freqtrade/system/__init__.py | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 2dcc5a7a8..8047497e3 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -18,8 +18,7 @@ from freqtrade.commands import Arguments from freqtrade.constants import DOCS_LINK from freqtrade.exceptions import ConfigurationError, FreqtradeException, OperationalException from freqtrade.loggers import setup_logging_pre -from freqtrade.system.asyncio_config import asyncio_setup -from freqtrade.system.gc_setup import gc_set_threshold +from freqtrade.system import asyncio_setup, gc_set_threshold logger = logging.getLogger("freqtrade") diff --git a/freqtrade/system/__init__.py b/freqtrade/system/__init__.py index 323ef7196..10a23aa4b 100644 --- a/freqtrade/system/__init__.py +++ b/freqtrade/system/__init__.py @@ -1 +1,7 @@ """system specific and performance tuning""" + +from freqtrade.system.asyncio_config import asyncio_setup +from freqtrade.system.gc_setup import gc_set_threshold + + +__all__ = ["asyncio_setup", "gc_set_threshold"] From 3fe97e1709ec3daa8cb07139b360c5eaac5062d3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Oct 2024 11:32:57 +0200 Subject: [PATCH 10/15] chore: Remove __futures__ import and corresponding ruff skips --- freqtrade/data/btanalysis.py | 2 -- pyproject.toml | 2 -- 2 files changed, 4 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index ad90014c2..9e9e6b4de 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -2,8 +2,6 @@ Helpers when analyzing backtest data """ -from __future__ import annotations - import logging import typing from copy import copy diff --git a/pyproject.toml b/pyproject.toml index 98bda823a..ad1c02928 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -157,8 +157,6 @@ extend-ignore = [ "S607", # Starting a process with a partial executable path "S608", # Possible SQL injection vector through string-based query construction "NPY002", # Numpy legacy random generator - "UP006", # non-pep585-annotation - "UP007", # non-pep604-annotation ] [tool.ruff.lint.mccabe] From 092275a981897094656bab4be795b6376d082528 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Oct 2024 13:00:11 +0200 Subject: [PATCH 11/15] refactor: move config deployment to it's own file --- freqtrade/commands/build_config_commands.py | 249 +------------------ freqtrade/configuration/deploy_config.py | 250 ++++++++++++++++++++ tests/commands/test_build_config.py | 18 +- 3 files changed, 265 insertions(+), 252 deletions(-) create mode 100644 freqtrade/configuration/deploy_config.py diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 27e3f395a..a35a48edc 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -1,11 +1,7 @@ import logging -import secrets from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict -from questionary import Separator, prompt - -from freqtrade.constants import UNLIMITED_STAKE_AMOUNT from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException @@ -13,250 +9,17 @@ from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) -def validate_is_int(val): - try: - _ = int(val) - return True - except Exception: - return False - - -def validate_is_float(val): - try: - _ = float(val) - return True - except Exception: - return False - - -def ask_user_overwrite(config_path: Path) -> bool: - questions = [ - { - "type": "confirm", - "name": "overwrite", - "message": f"File {config_path} already exists. Overwrite?", - "default": False, - }, - ] - answers = prompt(questions) - return answers["overwrite"] - - -def ask_user_config() -> Dict[str, Any]: - """ - Ask user a few questions to build the configuration. - Interactive questions built using https://github.com/tmbo/questionary - :returns: Dict with keys to put into template - """ - - from freqtrade.configuration.detect_environment import running_in_docker - from freqtrade.exchange import available_exchanges - - questions: List[Dict[str, Any]] = [ - { - "type": "confirm", - "name": "dry_run", - "message": "Do you want to enable Dry-run (simulated trades)?", - "default": True, - }, - { - "type": "text", - "name": "stake_currency", - "message": "Please insert your stake currency:", - "default": "USDT", - }, - { - "type": "text", - "name": "stake_amount", - "message": f"Please insert your stake amount (Number or '{UNLIMITED_STAKE_AMOUNT}'):", - "default": "unlimited", - "validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_float(val), - "filter": lambda val: ( - '"' + UNLIMITED_STAKE_AMOUNT + '"' if val == UNLIMITED_STAKE_AMOUNT else val - ), - }, - { - "type": "text", - "name": "max_open_trades", - "message": "Please insert max_open_trades (Integer or -1 for unlimited open trades):", - "default": "3", - "validate": lambda val: validate_is_int(val), - }, - { - "type": "select", - "name": "timeframe_in_config", - "message": "Time", - "choices": ["Have the strategy define timeframe.", "Override in configuration."], - }, - { - "type": "text", - "name": "timeframe", - "message": "Please insert your desired timeframe (e.g. 5m):", - "default": "5m", - "when": lambda x: x["timeframe_in_config"] == "Override in configuration.", - }, - { - "type": "text", - "name": "fiat_display_currency", - "message": ( - "Please insert your display Currency for reporting " - "(leave empty to disable FIAT conversion):" - ), - "default": "USD", - }, - { - "type": "select", - "name": "exchange_name", - "message": "Select exchange", - "choices": [ - "binance", - "binanceus", - "bingx", - "gate", - "htx", - "kraken", - "kucoin", - "okx", - Separator("------------------"), - "other", - ], - }, - { - "type": "confirm", - "name": "trading_mode", - "message": "Do you want to trade Perpetual Swaps (perpetual futures)?", - "default": False, - "filter": lambda val: "futures" if val else "spot", - "when": lambda x: x["exchange_name"] in ["binance", "gate", "okx", "bybit"], - }, - { - "type": "autocomplete", - "name": "exchange_name", - "message": "Type your exchange name (Must be supported by ccxt)", - "choices": available_exchanges(), - "when": lambda x: x["exchange_name"] == "other", - }, - { - "type": "password", - "name": "exchange_key", - "message": "Insert Exchange Key", - "when": lambda x: not x["dry_run"], - }, - { - "type": "password", - "name": "exchange_secret", - "message": "Insert Exchange Secret", - "when": lambda x: not x["dry_run"], - }, - { - "type": "password", - "name": "exchange_key_password", - "message": "Insert Exchange API Key password", - "when": lambda x: not x["dry_run"] and x["exchange_name"] in ("kucoin", "okx"), - }, - { - "type": "confirm", - "name": "telegram", - "message": "Do you want to enable Telegram?", - "default": False, - }, - { - "type": "password", - "name": "telegram_token", - "message": "Insert Telegram token", - "when": lambda x: x["telegram"], - }, - { - "type": "password", - "name": "telegram_chat_id", - "message": "Insert Telegram chat id", - "when": lambda x: x["telegram"], - }, - { - "type": "confirm", - "name": "api_server", - "message": "Do you want to enable the Rest API (includes FreqUI)?", - "default": False, - }, - { - "type": "text", - "name": "api_server_listen_addr", - "message": ( - "Insert Api server Listen Address (0.0.0.0 for docker, " - "otherwise best left untouched)" - ), - "default": "127.0.0.1" if not running_in_docker() else "0.0.0.0", # noqa: S104 - "when": lambda x: x["api_server"], - }, - { - "type": "text", - "name": "api_server_username", - "message": "Insert api-server username", - "default": "freqtrader", - "when": lambda x: x["api_server"], - }, - { - "type": "password", - "name": "api_server_password", - "message": "Insert api-server password", - "when": lambda x: x["api_server"], - }, - ] - answers = prompt(questions) - - if not answers: - # Interrupted questionary sessions return an empty dict. - raise OperationalException("User interrupted interactive questions.") - # Ensure default is set for non-futures exchanges - answers["trading_mode"] = answers.get("trading_mode", "spot") - answers["margin_mode"] = "isolated" if answers.get("trading_mode") == "futures" else "" - # Force JWT token to be a random string - answers["api_server_jwt_key"] = secrets.token_hex() - answers["api_server_ws_token"] = secrets.token_urlsafe(25) - - return answers - - -def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None: - """ - Applies selections to the template and writes the result to config_path - :param config_path: Path object for new config file. Should not exist yet - :param selections: Dict containing selections taken by the user. - """ - from jinja2.exceptions import TemplateNotFound - - from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS - from freqtrade.util import render_template - - try: - exchange_template = MAP_EXCHANGE_CHILDCLASS.get( - selections["exchange_name"], selections["exchange_name"] - ) - - selections["exchange"] = render_template( - templatefile=f"subtemplates/exchange_{exchange_template}.j2", arguments=selections - ) - except TemplateNotFound: - selections["exchange"] = render_template( - templatefile="subtemplates/exchange_generic.j2", arguments=selections - ) - - config_text = render_template(templatefile="base_config.json.j2", arguments=selections) - - logger.info(f"Writing config to `{config_path}`.") - logger.info( - "Please make sure to check the configuration contents and adjust settings to your needs." - ) - - config_path.write_text(config_text) - - def start_new_config(args: Dict[str, Any]) -> None: """ Create a new strategy from a template Asking the user questions to fill out the template accordingly. """ + from freqtrade.configuration.deploy_config import ( + ask_user_config, + ask_user_overwrite, + deploy_new_config, + ) from freqtrade.configuration.directory_operations import chown_user_directory config_path = Path(args["config"][0]) diff --git a/freqtrade/configuration/deploy_config.py b/freqtrade/configuration/deploy_config.py new file mode 100644 index 000000000..cd10cd0d8 --- /dev/null +++ b/freqtrade/configuration/deploy_config.py @@ -0,0 +1,250 @@ +import logging +import secrets +from pathlib import Path +from typing import Any, Dict, List + +from questionary import Separator, prompt + +from freqtrade.constants import UNLIMITED_STAKE_AMOUNT +from freqtrade.exceptions import OperationalException + + +logger = logging.getLogger(__name__) + + +def validate_is_int(val): + try: + _ = int(val) + return True + except Exception: + return False + + +def validate_is_float(val): + try: + _ = float(val) + return True + except Exception: + return False + + +def ask_user_overwrite(config_path: Path) -> bool: + questions = [ + { + "type": "confirm", + "name": "overwrite", + "message": f"File {config_path} already exists. Overwrite?", + "default": False, + }, + ] + answers = prompt(questions) + return answers["overwrite"] + + +def ask_user_config() -> Dict[str, Any]: + """ + Ask user a few questions to build the configuration. + Interactive questions built using https://github.com/tmbo/questionary + :returns: Dict with keys to put into template + """ + + from freqtrade.configuration.detect_environment import running_in_docker + from freqtrade.exchange import available_exchanges + + questions: List[Dict[str, Any]] = [ + { + "type": "confirm", + "name": "dry_run", + "message": "Do you want to enable Dry-run (simulated trades)?", + "default": True, + }, + { + "type": "text", + "name": "stake_currency", + "message": "Please insert your stake currency:", + "default": "USDT", + }, + { + "type": "text", + "name": "stake_amount", + "message": f"Please insert your stake amount (Number or '{UNLIMITED_STAKE_AMOUNT}'):", + "default": "unlimited", + "validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_float(val), + "filter": lambda val: ( + '"' + UNLIMITED_STAKE_AMOUNT + '"' if val == UNLIMITED_STAKE_AMOUNT else val + ), + }, + { + "type": "text", + "name": "max_open_trades", + "message": "Please insert max_open_trades (Integer or -1 for unlimited open trades):", + "default": "3", + "validate": lambda val: validate_is_int(val), + }, + { + "type": "select", + "name": "timeframe_in_config", + "message": "Time", + "choices": ["Have the strategy define timeframe.", "Override in configuration."], + }, + { + "type": "text", + "name": "timeframe", + "message": "Please insert your desired timeframe (e.g. 5m):", + "default": "5m", + "when": lambda x: x["timeframe_in_config"] == "Override in configuration.", + }, + { + "type": "text", + "name": "fiat_display_currency", + "message": ( + "Please insert your display Currency for reporting " + "(leave empty to disable FIAT conversion):" + ), + "default": "USD", + }, + { + "type": "select", + "name": "exchange_name", + "message": "Select exchange", + "choices": [ + "binance", + "binanceus", + "bingx", + "gate", + "htx", + "kraken", + "kucoin", + "okx", + Separator("------------------"), + "other", + ], + }, + { + "type": "confirm", + "name": "trading_mode", + "message": "Do you want to trade Perpetual Swaps (perpetual futures)?", + "default": False, + "filter": lambda val: "futures" if val else "spot", + "when": lambda x: x["exchange_name"] in ["binance", "gate", "okx", "bybit"], + }, + { + "type": "autocomplete", + "name": "exchange_name", + "message": "Type your exchange name (Must be supported by ccxt)", + "choices": available_exchanges(), + "when": lambda x: x["exchange_name"] == "other", + }, + { + "type": "password", + "name": "exchange_key", + "message": "Insert Exchange Key", + "when": lambda x: not x["dry_run"], + }, + { + "type": "password", + "name": "exchange_secret", + "message": "Insert Exchange Secret", + "when": lambda x: not x["dry_run"], + }, + { + "type": "password", + "name": "exchange_key_password", + "message": "Insert Exchange API Key password", + "when": lambda x: not x["dry_run"] and x["exchange_name"] in ("kucoin", "okx"), + }, + { + "type": "confirm", + "name": "telegram", + "message": "Do you want to enable Telegram?", + "default": False, + }, + { + "type": "password", + "name": "telegram_token", + "message": "Insert Telegram token", + "when": lambda x: x["telegram"], + }, + { + "type": "password", + "name": "telegram_chat_id", + "message": "Insert Telegram chat id", + "when": lambda x: x["telegram"], + }, + { + "type": "confirm", + "name": "api_server", + "message": "Do you want to enable the Rest API (includes FreqUI)?", + "default": False, + }, + { + "type": "text", + "name": "api_server_listen_addr", + "message": ( + "Insert Api server Listen Address (0.0.0.0 for docker, " + "otherwise best left untouched)" + ), + "default": "127.0.0.1" if not running_in_docker() else "0.0.0.0", # noqa: S104 + "when": lambda x: x["api_server"], + }, + { + "type": "text", + "name": "api_server_username", + "message": "Insert api-server username", + "default": "freqtrader", + "when": lambda x: x["api_server"], + }, + { + "type": "password", + "name": "api_server_password", + "message": "Insert api-server password", + "when": lambda x: x["api_server"], + }, + ] + answers = prompt(questions) + + if not answers: + # Interrupted questionary sessions return an empty dict. + raise OperationalException("User interrupted interactive questions.") + # Ensure default is set for non-futures exchanges + answers["trading_mode"] = answers.get("trading_mode", "spot") + answers["margin_mode"] = "isolated" if answers.get("trading_mode") == "futures" else "" + # Force JWT token to be a random string + answers["api_server_jwt_key"] = secrets.token_hex() + answers["api_server_ws_token"] = secrets.token_urlsafe(25) + + return answers + + +def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None: + """ + Applies selections to the template and writes the result to config_path + :param config_path: Path object for new config file. Should not exist yet + :param selections: Dict containing selections taken by the user. + """ + from jinja2.exceptions import TemplateNotFound + + from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS + from freqtrade.util import render_template + + try: + exchange_template = MAP_EXCHANGE_CHILDCLASS.get( + selections["exchange_name"], selections["exchange_name"] + ) + + selections["exchange"] = render_template( + templatefile=f"subtemplates/exchange_{exchange_template}.j2", arguments=selections + ) + except TemplateNotFound: + selections["exchange"] = render_template( + templatefile="subtemplates/exchange_generic.j2", arguments=selections + ) + + config_text = render_template(templatefile="base_config.json.j2", arguments=selections) + + logger.info(f"Writing config to `{config_path}`.") + logger.info( + "Please make sure to check the configuration contents and adjust settings to your needs." + ) + + config_path.write_text(config_text) diff --git a/tests/commands/test_build_config.py b/tests/commands/test_build_config.py index 5d287a35f..9d4a57db1 100644 --- a/tests/commands/test_build_config.py +++ b/tests/commands/test_build_config.py @@ -4,10 +4,10 @@ from unittest.mock import MagicMock import pytest import rapidjson -from freqtrade.commands.build_config_commands import ( +from freqtrade.commands.build_config_commands import start_new_config +from freqtrade.configuration.deploy_config import ( ask_user_config, ask_user_overwrite, - start_new_config, validate_is_float, validate_is_int, ) @@ -39,7 +39,7 @@ def test_start_new_config(mocker, caplog, exchange): wt_mock = mocker.patch.object(Path, "write_text", MagicMock()) mocker.patch.object(Path, "exists", MagicMock(return_value=True)) unlink_mock = mocker.patch.object(Path, "unlink", MagicMock()) - mocker.patch("freqtrade.commands.build_config_commands.ask_user_overwrite", return_value=True) + mocker.patch("freqtrade.configuration.deploy_config.ask_user_overwrite", return_value=True) sample_selections = { "max_open_trades": 3, @@ -62,7 +62,7 @@ def test_start_new_config(mocker, caplog, exchange): "api_server_password": "MoneyMachine", } mocker.patch( - "freqtrade.commands.build_config_commands.ask_user_config", return_value=sample_selections + "freqtrade.configuration.deploy_config.ask_user_config", return_value=sample_selections ) args = ["new-config", "--config", "coolconfig.json"] start_new_config(get_args(args)) @@ -80,7 +80,7 @@ def test_start_new_config(mocker, caplog, exchange): def test_start_new_config_exists(mocker, caplog): mocker.patch.object(Path, "exists", MagicMock(return_value=True)) - mocker.patch("freqtrade.commands.build_config_commands.ask_user_overwrite", return_value=False) + mocker.patch("freqtrade.configuration.deploy_config.ask_user_overwrite", return_value=False) args = ["new-config", "--config", "coolconfig.json"] with pytest.raises(OperationalException, match=r"Configuration .* already exists\."): start_new_config(get_args(args)) @@ -91,14 +91,14 @@ def test_ask_user_overwrite(mocker): Once https://github.com/tmbo/questionary/issues/35 is implemented, improve this test. """ prompt_mock = mocker.patch( - "freqtrade.commands.build_config_commands.prompt", return_value={"overwrite": False} + "freqtrade.configuration.deploy_config.prompt", return_value={"overwrite": False} ) assert not ask_user_overwrite(Path("test.json")) assert prompt_mock.call_count == 1 prompt_mock.reset_mock() prompt_mock = mocker.patch( - "freqtrade.commands.build_config_commands.prompt", return_value={"overwrite": True} + "freqtrade.configuration.deploy_config.prompt", return_value={"overwrite": True} ) assert ask_user_overwrite(Path("test.json")) assert prompt_mock.call_count == 1 @@ -109,13 +109,13 @@ def test_ask_user_config(mocker): Once https://github.com/tmbo/questionary/issues/35 is implemented, improve this test. """ prompt_mock = mocker.patch( - "freqtrade.commands.build_config_commands.prompt", return_value={"overwrite": False} + "freqtrade.configuration.deploy_config.prompt", return_value={"overwrite": False} ) answers = ask_user_config() assert isinstance(answers, dict) assert prompt_mock.call_count == 1 - prompt_mock = mocker.patch("freqtrade.commands.build_config_commands.prompt", return_value={}) + prompt_mock = mocker.patch("freqtrade.configuration.deploy_config.prompt", return_value={}) with pytest.raises(OperationalException, match=r"User interrupted interactive questions\."): ask_user_config() From 24c09d0319bd6046c9363a98880b8d2898ff2600 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Oct 2024 13:06:19 +0200 Subject: [PATCH 12/15] refactor: move ui deployment to it's own file --- freqtrade/commands/deploy_commands.py | 84 +++------------------------ freqtrade/commands/deploy_ui.py | 84 +++++++++++++++++++++++++++ tests/commands/test_commands.py | 34 +++++------ 3 files changed, 107 insertions(+), 95 deletions(-) create mode 100644 freqtrade/commands/deploy_ui.py diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index d610156ba..e389fc806 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -1,9 +1,7 @@ import logging import sys from pathlib import Path -from typing import Any, Dict, Optional, Tuple - -import requests +from typing import Any, Dict from freqtrade.constants import USERPATH_STRATEGIES from freqtrade.enums import RunMode @@ -101,80 +99,14 @@ def start_new_strategy(args: Dict[str, Any]) -> None: raise ConfigurationError("`new-strategy` requires --strategy to be set.") -def clean_ui_subdir(directory: Path): - if directory.is_dir(): - logger.info("Removing UI directory content.") - - for p in reversed(list(directory.glob("**/*"))): # iterate contents from leaves to root - if p.name in (".gitkeep", "fallback_file.html"): - continue - if p.is_file(): - p.unlink() - elif p.is_dir(): - p.rmdir() - - -def read_ui_version(dest_folder: Path) -> Optional[str]: - file = dest_folder / ".uiversion" - if not file.is_file(): - return None - - with file.open("r") as f: - return f.read() - - -def download_and_install_ui(dest_folder: Path, dl_url: str, version: str): - from io import BytesIO - from zipfile import ZipFile - - logger.info(f"Downloading {dl_url}") - resp = requests.get(dl_url, timeout=req_timeout).content - dest_folder.mkdir(parents=True, exist_ok=True) - with ZipFile(BytesIO(resp)) as zf: - for fn in zf.filelist: - with zf.open(fn) as x: - destfile = dest_folder / fn.filename - if fn.is_dir(): - destfile.mkdir(exist_ok=True) - else: - destfile.write_bytes(x.read()) - with (dest_folder / ".uiversion").open("w") as f: - f.write(version) - - -def get_ui_download_url(version: Optional[str] = None) -> Tuple[str, str]: - base_url = "https://api.github.com/repos/freqtrade/frequi/" - # Get base UI Repo path - - resp = requests.get(f"{base_url}releases", timeout=req_timeout) - resp.raise_for_status() - r = resp.json() - - if version: - tmp = [x for x in r if x["name"] == version] - if tmp: - latest_version = tmp[0]["name"] - assets = tmp[0].get("assets", []) - else: - raise ValueError("UI-Version not found.") - else: - latest_version = r[0]["name"] - assets = r[0].get("assets", []) - dl_url = "" - if assets and len(assets) > 0: - dl_url = assets[0]["browser_download_url"] - - # URL not found - try assets url - if not dl_url: - assets = r[0]["assets_url"] - resp = requests.get(assets, timeout=req_timeout) - r = resp.json() - dl_url = r[0]["browser_download_url"] - - return dl_url, latest_version - - def start_install_ui(args: Dict[str, Any]) -> None: + from freqtrade.commands.deploy_ui import ( + clean_ui_subdir, + download_and_install_ui, + get_ui_download_url, + read_ui_version, + ) + dest_folder = Path(__file__).parents[1] / "rpc/api_server/ui/installed/" # First make sure the assets are removed. dl_url, latest_version = get_ui_download_url(args.get("ui_version")) diff --git a/freqtrade/commands/deploy_ui.py b/freqtrade/commands/deploy_ui.py new file mode 100644 index 000000000..e9a9d2c43 --- /dev/null +++ b/freqtrade/commands/deploy_ui.py @@ -0,0 +1,84 @@ +import logging +from pathlib import Path +from typing import Optional, Tuple + +import requests + + +logger = logging.getLogger(__name__) + +# Timeout for requests +req_timeout = 30 + + +def clean_ui_subdir(directory: Path): + if directory.is_dir(): + logger.info("Removing UI directory content.") + + for p in reversed(list(directory.glob("**/*"))): # iterate contents from leaves to root + if p.name in (".gitkeep", "fallback_file.html"): + continue + if p.is_file(): + p.unlink() + elif p.is_dir(): + p.rmdir() + + +def read_ui_version(dest_folder: Path) -> Optional[str]: + file = dest_folder / ".uiversion" + if not file.is_file(): + return None + + with file.open("r") as f: + return f.read() + + +def download_and_install_ui(dest_folder: Path, dl_url: str, version: str): + from io import BytesIO + from zipfile import ZipFile + + logger.info(f"Downloading {dl_url}") + resp = requests.get(dl_url, timeout=req_timeout).content + dest_folder.mkdir(parents=True, exist_ok=True) + with ZipFile(BytesIO(resp)) as zf: + for fn in zf.filelist: + with zf.open(fn) as x: + destfile = dest_folder / fn.filename + if fn.is_dir(): + destfile.mkdir(exist_ok=True) + else: + destfile.write_bytes(x.read()) + with (dest_folder / ".uiversion").open("w") as f: + f.write(version) + + +def get_ui_download_url(version: Optional[str] = None) -> Tuple[str, str]: + base_url = "https://api.github.com/repos/freqtrade/frequi/" + # Get base UI Repo path + + resp = requests.get(f"{base_url}releases", timeout=req_timeout) + resp.raise_for_status() + r = resp.json() + + if version: + tmp = [x for x in r if x["name"] == version] + if tmp: + latest_version = tmp[0]["name"] + assets = tmp[0].get("assets", []) + else: + raise ValueError("UI-Version not found.") + else: + latest_version = r[0]["name"] + assets = r[0].get("assets", []) + dl_url = "" + if assets and len(assets) > 0: + dl_url = assets[0]["browser_download_url"] + + # URL not found - try assets url + if not dl_url: + assets = r[0]["assets_url"] + resp = requests.get(assets, timeout=req_timeout) + r = resp.json() + dl_url = r[0]["browser_download_url"] + + return dl_url, latest_version diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index ebc83e5e9..9ed3a2dd2 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -31,7 +31,7 @@ from freqtrade.commands import ( start_webserver, ) from freqtrade.commands.db_commands import start_convert_db -from freqtrade.commands.deploy_commands import ( +from freqtrade.commands.deploy_ui import ( clean_ui_subdir, download_and_install_ui, get_ui_download_url, @@ -612,13 +612,13 @@ def test_start_new_strategy_no_arg(mocker, caplog): def test_start_install_ui(mocker): - clean_mock = mocker.patch("freqtrade.commands.deploy_commands.clean_ui_subdir") + clean_mock = mocker.patch("freqtrade.commands.deploy_ui.clean_ui_subdir") get_url_mock = mocker.patch( - "freqtrade.commands.deploy_commands.get_ui_download_url", + "freqtrade.commands.deploy_ui.get_ui_download_url", return_value=("https://example.com/whatever", "0.0.1"), ) - download_mock = mocker.patch("freqtrade.commands.deploy_commands.download_and_install_ui") - mocker.patch("freqtrade.commands.deploy_commands.read_ui_version", return_value=None) + download_mock = mocker.patch("freqtrade.commands.deploy_ui.download_and_install_ui") + mocker.patch("freqtrade.commands.deploy_ui.read_ui_version", return_value=None) args = [ "install-ui", ] @@ -642,13 +642,13 @@ def test_start_install_ui(mocker): def test_clean_ui_subdir(mocker, tmp_path, caplog): - mocker.patch("freqtrade.commands.deploy_commands.Path.is_dir", side_effect=[True, True]) - mocker.patch("freqtrade.commands.deploy_commands.Path.is_file", side_effect=[False, True]) - rd_mock = mocker.patch("freqtrade.commands.deploy_commands.Path.rmdir") - ul_mock = mocker.patch("freqtrade.commands.deploy_commands.Path.unlink") + mocker.patch("freqtrade.commands.deploy_ui.Path.is_dir", side_effect=[True, True]) + mocker.patch("freqtrade.commands.deploy_ui.Path.is_file", side_effect=[False, True]) + rd_mock = mocker.patch("freqtrade.commands.deploy_ui.Path.rmdir") + ul_mock = mocker.patch("freqtrade.commands.deploy_ui.Path.unlink") mocker.patch( - "freqtrade.commands.deploy_commands.Path.glob", + "freqtrade.commands.deploy_ui.Path.glob", return_value=[Path("test1"), Path("test2"), Path(".gitkeep")], ) folder = tmp_path / "uitests" @@ -668,10 +668,10 @@ def test_download_and_install_ui(mocker, tmp_path): file_like_object.seek(0) requests_mock.content = file_like_object.read() - mocker.patch("requests.get", return_value=requests_mock) + mocker.patch("freqtrade.commands.deploy_ui.requests.get", return_value=requests_mock) - mocker.patch("freqtrade.commands.deploy_commands.Path.is_dir", side_effect=[True, False]) - wb_mock = mocker.patch("freqtrade.commands.deploy_commands.Path.write_bytes") + mocker.patch("freqtrade.commands.deploy_ui.Path.is_dir", side_effect=[True, False]) + wb_mock = mocker.patch("freqtrade.commands.deploy_ui.Path.write_bytes") folder = tmp_path / "uitests_dl" folder.mkdir(exist_ok=True) @@ -693,9 +693,7 @@ def test_get_ui_download_url(mocker): [{"browser_download_url": "http://download.zip"}], ] ) - get_mock = mocker.patch( - "freqtrade.commands.deploy_commands.requests.get", return_value=response - ) + get_mock = mocker.patch("freqtrade.commands.deploy_ui.requests.get", return_value=response) x, last_version = get_ui_download_url() assert get_mock.call_count == 2 assert last_version == "0.0.1" @@ -718,9 +716,7 @@ def test_get_ui_download_url_direct(mocker): }, ] ) - get_mock = mocker.patch( - "freqtrade.commands.deploy_commands.requests.get", return_value=response - ) + get_mock = mocker.patch("freqtrade.commands.deploy_ui.requests.get", return_value=response) x, last_version = get_ui_download_url() assert get_mock.call_count == 1 assert last_version == "0.0.2" From 672a23dbcda05acb8fe3bcf0245e01a169c27606 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Oct 2024 13:09:30 +0200 Subject: [PATCH 13/15] chore: revert changes to btanalysis - they have no impact --- freqtrade/data/btanalysis.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 9e9e6b4de..580807a76 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -3,7 +3,6 @@ Helpers when analyzing backtest data """ import logging -import typing from copy import copy from datetime import datetime, timezone from pathlib import Path @@ -17,10 +16,7 @@ from freqtrade.exceptions import ConfigurationError, OperationalException from freqtrade.ft_types import BacktestHistoryEntryType, BacktestResultType from freqtrade.misc import file_dump_json, json_load from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename - - -if typing.TYPE_CHECKING: - from freqtrade.persistence import LocalTrade, Trade +from freqtrade.persistence import LocalTrade, Trade, init_db logger = logging.getLogger(__name__) @@ -464,8 +460,6 @@ def load_trades_from_db(db_url: str, strategy: Optional[str] = None) -> pd.DataF Can also serve as protection to load the correct result. :return: Dataframe containing Trades """ - from freqtrade.persistence import Trade, init_db - init_db(db_url) filters = [] From ed5c929b238506702cd06356ee30c8f508b6a485 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Oct 2024 13:17:35 +0200 Subject: [PATCH 14/15] chore: remove rapidjson from "default" imports --- freqtrade/commands/list_commands.py | 4 ++-- tests/data/test_btanalysis.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index d4ae4cf43..4922eac3c 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -3,8 +3,6 @@ import logging import sys from typing import Any, Dict, List, Union -import rapidjson - from freqtrade.enums import RunMode from freqtrade.exceptions import ConfigurationError, OperationalException from freqtrade.ft_types import ValidExchangesType @@ -297,6 +295,8 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: elif args.get("print_one_column", False): print("\n".join(pairs.keys())) elif args.get("list_pairs_print_json", False): + import rapidjson + print(rapidjson.dumps(list(pairs.keys()), default=str)) elif args.get("print_csv", False): writer = csv.DictWriter(sys.stdout, fieldnames=headers) diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 691b02fc0..b9dee6d59 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -146,7 +146,7 @@ def test_load_backtest_data_multi(testdatadir): def test_load_trades_from_db(default_conf, fee, is_short, mocker): create_mock_trades(fee, is_short) # remove init so it does not init again - init_mock = mocker.patch("freqtrade.persistence.init_db", MagicMock()) + init_mock = mocker.patch("freqtrade.data.btanalysis.init_db", MagicMock()) trades = load_trades_from_db(db_url=default_conf["db_url"]) assert init_mock.call_count == 1 From aa2c1501dacf3d300df4a7a68386c87ac0c14f3e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Oct 2024 13:20:43 +0200 Subject: [PATCH 15/15] chore: fix missed dependencies for deploy_ui --- freqtrade/rpc/api_server/web_ui.py | 2 +- tests/rpc/test_rpc_apiserver.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/api_server/web_ui.py b/freqtrade/rpc/api_server/web_ui.py index 8f9dd7f12..bf37c6fa7 100644 --- a/freqtrade/rpc/api_server/web_ui.py +++ b/freqtrade/rpc/api_server/web_ui.py @@ -21,7 +21,7 @@ async def fallback(): @router_ui.get("/ui_version", include_in_schema=False) async def ui_version(): - from freqtrade.commands.deploy_commands import read_ui_version + from freqtrade.commands.deploy_ui import read_ui_version uibase = Path(__file__).parent / "ui/installed/" version = read_ui_version(uibase) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index c6b4bb994..02ee6ab89 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -185,7 +185,7 @@ def test_api_ui_fallback(botclient, mocker): def test_api_ui_version(botclient, mocker): _ftbot, client = botclient - mocker.patch("freqtrade.commands.deploy_commands.read_ui_version", return_value="0.1.2") + mocker.patch("freqtrade.commands.deploy_ui.read_ui_version", return_value="0.1.2") rc = client_get(client, "/ui_version") assert rc.status_code == 200 assert rc.json()["version"] == "0.1.2"