diff --git a/docs/backtesting.md b/docs/backtesting.md index 45d5f486f..004425323 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -7,7 +7,7 @@ Backtesting. To download data (candles / OHLCV) needed for backtesting and hyperoptimization use the `freqtrade download-data` command. -If no additional parameter is specified, freqtrade will download data for `"1m"` and `"5m"` timeframes. +If no additional parameter is specified, freqtrade will download data for `"1m"` and `"5m"` timeframes for 30 days. Exchange and pairs will come from `config.json` (if specified using `-c/--config`). Otherwise `--exchange` becomes mandatory. Alternatively, a `pairs.json` file can be used. @@ -37,6 +37,10 @@ This will download ticker data for all the currency pairs you defined in `pairs. - Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers. - To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options. +!!! Tip Updating existing data + If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data. + Be carefull though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded. + ## Test your strategy with Backtesting Now you have good Buy and Sell strategies and some historic data, you want to test it against diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 0b0561d3d..f44400e32 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -184,10 +184,6 @@ optional arguments: Specify max_open_trades to use. --stake_amount STAKE_AMOUNT Specify stake_amount. - -r, --refresh-pairs-cached - Refresh the pairs files in tests/testdata with the - latest data from the exchange. Use it if you want to - run your optimization commands with up-to-date data. --eps, --enable-position-stacking Allow buying the same pair multiple times (position stacking). @@ -245,10 +241,6 @@ optional arguments: Specify max_open_trades to use. --stake_amount STAKE_AMOUNT Specify stake_amount. - -r, --refresh-pairs-cached - Refresh the pairs files in tests/testdata with the - latest data from the exchange. Use it if you want to - run your optimization commands with up-to-date data. --customhyperopt NAME Specify hyperopt class name (default: `DefaultHyperOpts`). @@ -310,10 +302,6 @@ optional arguments: Specify max_open_trades to use. --stake_amount STAKE_AMOUNT Specify stake_amount. - -r, --refresh-pairs-cached - Refresh the pairs files in tests/testdata with the - latest data from the exchange. Use it if you want to - run your optimization commands with up-to-date data. --stoplosses STOPLOSS_RANGE Defines a range of stoploss against which edge will assess the strategy the format is "min,max,step" diff --git a/docs/deprecated.md b/docs/deprecated.md index ed70b1936..349d41a09 100644 --- a/docs/deprecated.md +++ b/docs/deprecated.md @@ -4,7 +4,7 @@ This page contains description of the command line arguments, configuration para and the bot features that were declared as DEPRECATED by the bot development team and are no longer supported. Please avoid their usage in your configuration. -## Deprecated +## Removed features ### the `--refresh-pairs-cached` command line option @@ -12,9 +12,7 @@ and are no longer supported. Please avoid their usage in your configuration. Since this leads to much confusion, and slows down backtesting (while not being part of backtesting) this has been singled out as a seperate freqtrade subcommand `freqtrade download-data`. -This command line option was deprecated in `2019.7-dev` and will be removed after the next release. - -## Removed features +This command line option was deprecated in 2019.7-dev (develop branch) and removed in 2019.9 (master branch). ### The **--dynamic-whitelist** command line option diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index bb961173b..6e2ecea2e 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -15,7 +15,7 @@ ARGS_STRATEGY = ["strategy", "strategy_path"] ARGS_MAIN = ARGS_COMMON + ARGS_STRATEGY + ["db_url", "sd_notify"] ARGS_COMMON_OPTIMIZE = ["ticker_interval", "timerange", - "max_open_trades", "stake_amount", "refresh_pairs"] + "max_open_trades", "stake_amount"] ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", "strategy_list", "export", "exportfilename"] diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/configuration/cli_options.py index bf1ec3620..cb07dbdba 100644 --- a/freqtrade/configuration/cli_options.py +++ b/freqtrade/configuration/cli_options.py @@ -107,13 +107,6 @@ AVAILABLE_CLI_OPTIONS = { help='Specify stake_amount.', type=float, ), - "refresh_pairs": Arg( - '-r', '--refresh-pairs-cached', - help='Refresh the pairs files in tests/testdata with the latest data from the ' - 'exchange. Use it if you want to run your optimization commands with ' - 'up-to-date data.', - action='store_true', - ), # Backtesting "position_stacking": Arg( '--eps', '--enable-position-stacking', diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index df893b3ad..764593d0f 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -215,10 +215,6 @@ class Configuration: self._process_datadir_options(config) - self._args_to_config(config, argname='refresh_pairs', - logstring='Parameter -r/--refresh-pairs-cached detected ...', - deprecated_msg='-r/--refresh-pairs-cached will be removed soon.') - self._args_to_config(config, argname='strategy_list', logstring='Using strategy list of {} strategies', logfun=len) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index eb6ec0f2a..7d5e4540b 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -65,7 +65,6 @@ class DataProvider: """ return load_pair_history(pair=pair, ticker_interval=ticker_interval or self._config['ticker_interval'], - refresh_pairs=False, datadir=Path(self._config['datadir']) ) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index b4776fab0..865289f38 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -129,8 +129,7 @@ def load_pair_history(pair: str, else: logger.warning( f'No history data for pair: "{pair}", interval: {ticker_interval}. ' - 'Use --refresh-pairs-cached option or `freqtrade download-data` ' - 'script to download the data' + 'Use `freqtrade download-data` to download the data' ) return None @@ -142,33 +141,25 @@ def load_data(datadir: Path, exchange: Optional[Exchange] = None, timerange: TimeRange = TimeRange(None, None, 0, 0), fill_up_missing: bool = True, - live: bool = False ) -> Dict[str, DataFrame]: """ - Loads ticker history data for a list of pairs the given parameters + Loads ticker history data for a list of pairs :return: dict(:) + TODO: refresh_pairs is still used by edge to keep the data uptodate. + This should be replaced in the future. Instead, writing the current candles to disk + from dataprovider should be implemented, as this would avoid loading ohlcv data twice. + exchange and refresh_pairs are then not needed here nor in load_pair_history. """ result: Dict[str, DataFrame] = {} - if live: - if exchange: - logger.info('Live: Downloading data for all defined pairs ...') - exchange.refresh_latest_ohlcv([(pair, ticker_interval) for pair in pairs]) - result = {key[0]: value for key, value in exchange._klines.items() if value is not None} - else: - raise OperationalException( - "Exchange needs to be initialized when using live data." - ) - else: - logger.info('Using local backtesting data ...') - for pair in pairs: - hist = load_pair_history(pair=pair, ticker_interval=ticker_interval, - datadir=datadir, timerange=timerange, - refresh_pairs=refresh_pairs, - exchange=exchange, - fill_up_missing=fill_up_missing) - if hist is not None: - result[pair] = hist + for pair in pairs: + hist = load_pair_history(pair=pair, ticker_interval=ticker_interval, + datadir=datadir, timerange=timerange, + refresh_pairs=refresh_pairs, + exchange=exchange, + fill_up_missing=fill_up_missing) + if hist is not None: + result[pair] = hist return result diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4956907fb..6074b281b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -414,8 +414,6 @@ class Backtesting: datadir=Path(self.config['datadir']), pairs=pairs, ticker_interval=self.ticker_interval, - refresh_pairs=self.config.get('refresh_pairs', False), - exchange=self.exchange, timerange=timerange, ) diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index f22bcb642..0cf5a009b 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -39,7 +39,8 @@ class EdgeCli: self.strategy = StrategyResolver(self.config).strategy self.edge = Edge(config, self.exchange, self.strategy) - self.edge._refresh_pairs = self.config.get('refresh_pairs', False) + # Set refresh_pairs to false for edge-cli (it must be true for edge) + self.edge._refresh_pairs = False self.timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index c511aa5ac..61623d3df 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -362,8 +362,6 @@ class Hyperopt: datadir=Path(self.config['datadir']), pairs=self.config['exchange']['pair_whitelist'], ticker_interval=self.backtesting.ticker_interval, - refresh_pairs=self.config.get('refresh_pairs', False), - exchange=self.backtesting.exchange, timerange=timerange ) diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 72da47e76..e773a970e 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -24,7 +24,6 @@ def test_parse_ticker_dataframe(ticker_history_list, caplog): def test_ohlcv_fill_up_missing_data(testdatadir, caplog): data = load_pair_history(datadir=testdatadir, ticker_interval='1m', - refresh_pairs=False, pair='UNITTEST/BTC', fill_up_missing=False) caplog.set_level(logging.DEBUG) diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index ec176d889..39e2f7d2e 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -45,7 +45,6 @@ def test_historic_ohlcv(mocker, default_conf, ticker_history): data = dp.historic_ohlcv("UNITTEST/BTC", "5m") assert isinstance(data, DataFrame) assert historymock.call_count == 1 - assert historymock.call_args_list[0][1]["refresh_pairs"] is False assert historymock.call_args_list[0][1]["ticker_interval"] == "5m" diff --git a/tests/data/test_history.py b/tests/data/test_history.py index e747794e5..e386c3506 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -74,8 +74,7 @@ def test_load_data_7min_ticker(mocker, caplog, default_conf, testdatadir) -> Non assert ld is None assert log_has( 'No history data for pair: "UNITTEST/BTC", interval: 7m. ' - 'Use --refresh-pairs-cached option or `freqtrade download-data` ' - 'script to download the data', caplog + 'Use `freqtrade download-data` to download the data', caplog ) @@ -105,13 +104,11 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, # do not download a new pair if refresh_pairs isn't set history.load_pair_history(datadir=testdatadir, ticker_interval='1m', - refresh_pairs=False, pair='MEME/BTC') assert os.path.isfile(file) is False assert log_has( 'No history data for pair: "MEME/BTC", interval: 1m. ' - 'Use --refresh-pairs-cached option or `freqtrade download-data` ' - 'script to download the data', caplog + 'Use `freqtrade download-data` to download the data', caplog ) # download a new pair if refresh_pairs is set @@ -134,31 +131,6 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, _clean_test_file(file) -def test_load_data_live(default_conf, mocker, caplog, testdatadir) -> None: - refresh_mock = MagicMock() - mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock) - exchange = get_patched_exchange(mocker, default_conf) - - history.load_data(datadir=testdatadir, ticker_interval='5m', - pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'], - live=True, - exchange=exchange) - assert refresh_mock.call_count == 1 - assert len(refresh_mock.call_args_list[0][0][0]) == 2 - assert log_has('Live: Downloading data for all defined pairs ...', caplog) - - -def test_load_data_live_noexchange(default_conf, mocker, caplog, testdatadir) -> None: - - with pytest.raises(OperationalException, - match=r'Exchange needs to be initialized when using live data.'): - history.load_data(datadir=testdatadir, ticker_interval='5m', - pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'], - exchange=None, - live=True, - ) - - def test_testdata_path(testdatadir) -> None: assert str(Path('tests') / 'testdata') in str(testdatadir) @@ -349,7 +321,6 @@ def test_load_partial_missing(testdatadir, caplog) -> None: start = arrow.get('2018-01-01T00:00:00') end = arrow.get('2018-01-11T00:00:00') tickerdata = history.load_data(testdatadir, '5m', ['UNITTEST/BTC'], - refresh_pairs=False, timerange=TimeRange('date', 'date', start.timestamp, end.timestamp)) # timedifference in 5 minutes @@ -364,7 +335,7 @@ def test_load_partial_missing(testdatadir, caplog) -> None: start = arrow.get('2018-01-10T00:00:00') end = arrow.get('2018-02-20T00:00:00') tickerdata = history.load_data(datadir=testdatadir, ticker_interval='5m', - pairs=['UNITTEST/BTC'], refresh_pairs=False, + pairs=['UNITTEST/BTC'], timerange=TimeRange('date', 'date', start.timestamp, end.timestamp)) # timedifference in 5 minutes diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 7b50f2b18..fa40809d8 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -188,16 +188,12 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'position_stacking' not in config assert not log_has('Parameter --enable-position-stacking detected ...', caplog) - assert 'refresh_pairs' not in config - assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) - assert 'timerange' not in config assert 'export' not in config assert 'runmode' in config assert config['runmode'] == RunMode.BACKTEST -@pytest.mark.filterwarnings("ignore:DEPRECATED") def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) mocker.patch( @@ -213,7 +209,6 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> '--ticker-interval', '1m', '--enable-position-stacking', '--disable-max-market-positions', - '--refresh-pairs-cached', '--timerange', ':100', '--export', '/bar/foo', '--export-filename', 'foo_bar.json' @@ -240,9 +235,6 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> assert log_has('Parameter --disable-max-market-positions detected ...', caplog) assert log_has('max_open_trades set to unlimited ...', caplog) - assert 'refresh_pairs' in config - assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) - assert 'timerange' in config assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog) diff --git a/tests/optimize/test_edge_cli.py b/tests/optimize/test_edge_cli.py index f312bba2c..97103da55 100644 --- a/tests/optimize/test_edge_cli.py +++ b/tests/optimize/test_edge_cli.py @@ -3,8 +3,6 @@ from unittest.mock import MagicMock -import pytest - from freqtrade.edge import PairInfo from freqtrade.optimize import setup_configuration, start_edge from freqtrade.optimize.edge_cli import EdgeCli @@ -35,14 +33,10 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'ticker_interval' in config assert not log_has_re('Parameter -i/--ticker-interval detected .*', caplog) - assert 'refresh_pairs' not in config - assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) - assert 'timerange' not in config assert 'stoploss_range' not in config -@pytest.mark.filterwarnings("ignore:DEPRECATED") def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> None: patched_configuration_load_config_file(mocker, edge_conf) mocker.patch( @@ -56,7 +50,6 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N '--datadir', '/foo/bar', 'edge', '--ticker-interval', '1m', - '--refresh-pairs-cached', '--timerange', ':100', '--stoplosses=-0.01,-0.10,-0.001' ] @@ -74,8 +67,6 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N assert log_has('Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...', caplog) - assert 'refresh_pairs' in config - assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) assert 'timerange' in config assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 2e383c839..0888c79d4 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -86,15 +86,11 @@ def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, ca assert 'position_stacking' not in config assert not log_has('Parameter --enable-position-stacking detected ...', caplog) - assert 'refresh_pairs' not in config - assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) - assert 'timerange' not in config assert 'runmode' in config assert config['runmode'] == RunMode.HYPEROPT -@pytest.mark.filterwarnings("ignore:DEPRECATED") def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) mocker.patch( @@ -108,7 +104,6 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo 'hyperopt', '--ticker-interval', '1m', '--timerange', ':100', - '--refresh-pairs-cached', '--enable-position-stacking', '--disable-max-market-positions', '--epochs', '1000', @@ -137,9 +132,6 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo assert log_has('Parameter --disable-max-market-positions detected ...', caplog) assert log_has('max_open_trades set to unlimited ...', caplog) - assert 'refresh_pairs' in config - assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) - assert 'timerange' in config assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog) diff --git a/tests/test_arguments.py b/tests/test_arguments.py index 7b18aa356..cf0104c01 100644 --- a/tests/test_arguments.py +++ b/tests/test_arguments.py @@ -99,7 +99,6 @@ def test_parse_args_backtesting_custom() -> None: '-c', 'test_conf.json', 'backtesting', '--ticker-interval', '1m', - '--refresh-pairs-cached', '--strategy-list', 'DefaultStrategy', 'SampleStrategy' @@ -110,7 +109,6 @@ def test_parse_args_backtesting_custom() -> None: assert call_args["subparser"] == 'backtesting' assert call_args["func"] is not None assert call_args["ticker_interval"] == '1m' - assert call_args["refresh_pairs"] is True assert type(call_args["strategy_list"]) is list assert len(call_args["strategy_list"]) == 2 diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 7cb30f1b3..330b82d39 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -358,14 +358,10 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'position_stacking' not in config assert not log_has('Parameter --enable-position-stacking detected ...', caplog) - assert 'refresh_pairs' not in config - assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) - assert 'timerange' not in config assert 'export' not in config -@pytest.mark.filterwarnings("ignore:DEPRECATED") def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) mocker.patch( @@ -385,7 +381,6 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non '--ticker-interval', '1m', '--enable-position-stacking', '--disable-max-market-positions', - '--refresh-pairs-cached', '--timerange', ':100', '--export', '/bar/foo' ] @@ -415,8 +410,6 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non assert log_has('Parameter --disable-max-market-positions detected ...', caplog) assert log_has('max_open_trades set to unlimited ...', caplog) - assert 'refresh_pairs'in config - assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) assert 'timerange' in config assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog)