diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index ddc0dc489..c94c02a8b 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -47,7 +47,7 @@ class Arguments(object): return self.parsed_arg - def parse_args(self) -> argparse.Namespace: + def parse_args(self, no_default_config: bool = False) -> argparse.Namespace: """ Parses given arguments and returns an argparse Namespace instance. """ @@ -55,7 +55,7 @@ class Arguments(object): # Workaround issue in argparse with action='append' and default value # (see https://bugs.python.org/issue16399) - if parsed_arg.config is None: + if parsed_arg.config is None and not no_default_config: parsed_arg.config = [constants.DEFAULT_CONFIG] return parsed_arg @@ -427,26 +427,24 @@ class Arguments(object): default=None ) - def testdata_dl_options(self) -> None: + def download_data_options(self) -> None: """ Parses given arguments for testdata download """ self.parser.add_argument( - '--pairs-file', - help='File containing a list of pairs to download.', - dest='pairs_file', - default=None, - metavar='PATH', + '-v', '--verbose', + help='Verbose mode (-vv for more, -vvv to get all messages).', + action='count', + dest='loglevel', + default=0, ) - self.parser.add_argument( - '--export', - help='Export files to given dir.', - dest='export', - default=None, - metavar='PATH', + '--logfile', + help='Log to the file specified', + dest='logfile', + type=str, + metavar='FILE' ) - self.parser.add_argument( '-c', '--config', help='Specify configuration file (default: %(default)s). ' @@ -456,7 +454,21 @@ class Arguments(object): type=str, metavar='PATH', ) - + self.parser.add_argument( + '-d', '--datadir', + help='Path to backtest data.', + dest='datadir', + default=None, + type=str, + metavar='PATH', + ) + self.parser.add_argument( + '--pairs-file', + help='File containing a list of pairs to download.', + dest='pairs_file', + default=None, + metavar='PATH', + ) self.parser.add_argument( '--days', help='Download data for given number of days.', @@ -465,7 +477,6 @@ class Arguments(object): metavar='INT', default=None ) - self.parser.add_argument( '--exchange', help='Exchange name (default: %(default)s). Only valid if no config is provided.', @@ -473,7 +484,6 @@ class Arguments(object): type=str, default='bittrex' ) - self.parser.add_argument( '-t', '--timeframes', help='Specify which tickers to download. Space separated list. \ @@ -484,7 +494,6 @@ class Arguments(object): nargs='+', dest='timeframes', ) - self.parser.add_argument( '--erase', help='Clean all existing data for the selected exchange/pairs/timeframes.', diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index c19580c36..58fd1d6d9 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -122,12 +122,11 @@ class Configuration(object): return conf - def _load_common_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + def _load_logging_config(self, config: Dict[str, Any]) -> None: """ - Extract information for sys.argv and load common configuration - :return: configuration as dictionary + Extract information for sys.argv and load logging configuration: + the --loglevel, --logfile options """ - # Log level if 'loglevel' in self.args and self.args.loglevel: config.update({'verbosity': self.args.loglevel}) @@ -153,6 +152,13 @@ class Configuration(object): set_loggers(config['verbosity']) logger.info('Verbosity set to %s', config['verbosity']) + def _load_common_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + """ + Extract information for sys.argv and load common configuration + :return: configuration as dictionary + """ + self._load_logging_config(config) + # Support for sd_notify if self.args.sd_notify: config['internals'].update({'sd_notify': True}) @@ -228,6 +234,17 @@ class Configuration(object): else: logger.info(logstring.format(config[argname])) + def _load_datadir_config(self, config: Dict[str, Any]) -> None: + """ + Extract information for sys.argv and load datadir configuration: + the --datadir option + """ + if 'datadir' in self.args and self.args.datadir: + config.update({'datadir': self._create_datadir(config, self.args.datadir)}) + else: + config.update({'datadir': self._create_datadir(config, None)}) + logger.info('Using data folder: %s ...', config.get('datadir')) + def _load_optimize_config(self, config: Dict[str, Any]) -> Dict[str, Any]: """ Extract information for sys.argv and load Optimize configuration @@ -263,11 +280,7 @@ class Configuration(object): self._args_to_config(config, argname='timerange', logstring='Parameter --timerange detected: {} ...') - if 'datadir' in self.args and self.args.datadir: - config.update({'datadir': self._create_datadir(config, self.args.datadir)}) - else: - config.update({'datadir': self._create_datadir(config, None)}) - logger.info('Using data folder: %s ...', config.get('datadir')) + self._load_datadir_config(config) self._args_to_config(config, argname='refresh_pairs', logstring='Parameter -r/--refresh-pairs-cached detected ...') diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index ecd108b5e..afa42f287 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -170,18 +170,18 @@ def test_parse_args_hyperopt_custom() -> None: assert call_args.func is not None -def test_testdata_dl_options() -> None: +def test_download_data_options() -> None: args = [ '--pairs-file', 'file_with_pairs', - '--export', 'export/folder', + '--datadir', 'datadir/folder', '--days', '30', '--exchange', 'binance' ] arguments = Arguments(args, '') - arguments.testdata_dl_options() + arguments.download_data_options() args = arguments.parse_args() assert args.pairs_file == 'file_with_pairs' - assert args.export == 'export/folder' + assert args.datadir == 'datadir/folder' assert args.days == 30 assert args.exchange == 'binance' diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 42b305778..acf86d0d8 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -1,39 +1,39 @@ #!/usr/bin/env python3 """ -This script generates json data +This script generates json files with pairs history data """ +import arrow import json import sys from pathlib import Path -import arrow -from typing import Any, Dict +from typing import Any, Dict, List -from freqtrade.arguments import Arguments -from freqtrade.arguments import TimeRange -from freqtrade.exchange import Exchange +from freqtrade.arguments import Arguments, TimeRange +from freqtrade.configuration import Configuration from freqtrade.data.history import download_pair_history -from freqtrade.configuration import Configuration, set_loggers +from freqtrade.exchange import Exchange from freqtrade.misc import deep_merge_dicts import logging -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', -) -set_loggers(0) + +logger = logging.getLogger('download_backtest_data') DEFAULT_DL_PATH = 'user_data/data' arguments = Arguments(sys.argv[1:], 'download utility') -arguments.testdata_dl_options() -args = arguments.parse_args() +arguments.download_data_options() + +# Do not read the default config if config is not specified +# in the command line options explicitely +args = arguments.parse_args(no_default_config=True) timeframes = args.timeframes +pairs: List = [] + +configuration = Configuration(args) +config: Dict[str, Any] = {} if args.config: - configuration = Configuration(args) - - config: Dict[str, Any] = {} # Now expecting a list of config filenames here, not a string for path in args.config: print(f"Using config: {path}...") @@ -42,9 +42,19 @@ if args.config: config['stake_currency'] = '' # Ensure we do not use Exchange credentials + config['exchange']['dry_run'] = True config['exchange']['key'] = '' config['exchange']['secret'] = '' + + if args.exchange: + config['exchange']['name'] = args.exchange + + pairs = config['exchange']['pair_whitelist'] + timeframes = [config['ticker_interval']] + else: + if not args.exchange: + sys.exit("No exchange specified.") config = { 'stake_currency': '', 'dry_run': True, @@ -60,55 +70,59 @@ else: } } +configuration._load_logging_config(config) +configuration._load_datadir_config(config) -dl_path = Path(DEFAULT_DL_PATH).joinpath(config['exchange']['name']) -if args.export: - dl_path = Path(args.export) - -if not dl_path.is_dir(): - sys.exit(f'Directory {dl_path} does not exist.') +dl_path = Path(config['datadir']) pairs_file = Path(args.pairs_file) if args.pairs_file else dl_path.joinpath('pairs.json') -if not pairs_file.exists(): - sys.exit(f'No pairs file found with path {pairs_file}.') -with pairs_file.open() as file: - PAIRS = list(set(json.load(file))) +if not pairs or args.pairs_file: + print(f'Reading pairs file "{pairs_file}".') + # Download pairs from the pairs file if no config is specified + # or if pairs file is specified explicitely + if not pairs_file.exists(): + sys.exit(f'No pairs file found with path "{pairs_file}".') -PAIRS.sort() + with pairs_file.open() as file: + pairs = list(set(json.load(file))) + pairs.sort() timerange = TimeRange() if args.days: time_since = arrow.utcnow().shift(days=-args.days).strftime("%Y%m%d") timerange = arguments.parse_timerange(f'{time_since}-') +print(f'About to download pairs: {pairs}, intervals: {timeframes} to {dl_path}') -print(f'About to download pairs: {PAIRS} to {dl_path}') - -# Init exchange -exchange = Exchange(config) pairs_not_available = [] -for pair in PAIRS: - if pair not in exchange._api.markets: - pairs_not_available.append(pair) - print(f"skipping pair {pair}") - continue - for ticker_interval in timeframes: - pair_print = pair.replace('/', '_') - filename = f'{pair_print}-{ticker_interval}.json' - dl_file = dl_path.joinpath(filename) - if args.erase and dl_file.exists(): - print(f'Deleting existing data for pair {pair}, interval {ticker_interval}') - dl_file.unlink() +try: + # Init exchange + exchange = Exchange(config) - print(f'downloading pair {pair}, interval {ticker_interval}') - download_pair_history(datadir=dl_path, exchange=exchange, - pair=pair, - ticker_interval=ticker_interval, - timerange=timerange) + for pair in pairs: + if pair not in exchange._api.markets: + pairs_not_available.append(pair) + print(f"skipping pair {pair}") + continue + for ticker_interval in timeframes: + pair_print = pair.replace('/', '_') + filename = f'{pair_print}-{ticker_interval}.json' + dl_file = dl_path.joinpath(filename) + if args.erase and dl_file.exists(): + print(f'Deleting existing data for pair {pair}, interval {ticker_interval}') + dl_file.unlink() + print(f'downloading pair {pair}, interval {ticker_interval}') + download_pair_history(datadir=dl_path, exchange=exchange, + pair=pair, ticker_interval=ticker_interval, + timerange=timerange) -if pairs_not_available: - print(f"Pairs [{','.join(pairs_not_available)}] not availble.") +except KeyboardInterrupt: + sys.exit("SIGINT received, aborting ...") + +finally: + if pairs_not_available: + print(f"Pairs [{','.join(pairs_not_available)}] not availble.")