From e3227a741c650cd60cf46273e40f55645e2b6164 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 3 Jun 2018 14:52:03 +0200 Subject: [PATCH 01/70] add --export-filename for backtesting --- freqtrade/arguments.py | 10 ++++++++++ freqtrade/configuration.py | 5 +++++ freqtrade/optimize/backtesting.py | 8 +++++--- freqtrade/tests/optimize/test_backtesting.py | 8 +++++++- 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 97c3d8cb2..7e895177a 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -137,6 +137,16 @@ class Arguments(object): default=None, dest='export', ) + parser.add_argument( + '--export-filename', + help='Save backtest results to this filename \ + requires --export to be set as well\ + Example --export-filename=backtest_today.json\ + (default: %(default)s', + type=str, + default='backtest-result.json', + dest='exportfilename', + ) @staticmethod def optimizer_shared_options(parser: argparse.ArgumentParser) -> None: diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index a800bde78..77b5b4447 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -157,6 +157,11 @@ class Configuration(object): config.update({'export': self.args.export}) logger.info('Parameter --export detected: %s ...', self.args.export) + # If --export-filename is used we add it to the configuration + if 'export' in config and 'exportfilename' in self.args and self.args.exportfilename: + config.update({'exportfilename': self.args.exportfilename}) + logger.info('Storing backtest results to %s ...', self.args.exportfilename) + return config def _load_hyperopt_config(self, config: Dict[str, Any]) -> Dict[str, Any]: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 1d560d309..d7ed45955 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -154,6 +154,7 @@ class Backtesting(object): max_open_trades = args.get('max_open_trades', 0) realistic = args.get('realistic', False) record = args.get('record', None) + recordfilename = args.get('recordfn', 'backtest-result.json') records = [] trades = [] trade_count_lock: Dict = {} @@ -196,8 +197,8 @@ class Backtesting(object): # For now export inside backtest(), maybe change so that backtest() # returns a tuple like: (dataframe, records, logs, etc) if record and record.find('trades') >= 0: - logger.info('Dumping backtest results') - file_dump_json('backtest-result.json', records) + logger.info('Dumping backtest results to %s', recordfilename) + file_dump_json(recordfilename, records) labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] return DataFrame.from_records(trades, columns=labels) @@ -257,7 +258,8 @@ class Backtesting(object): 'realistic': self.config.get('realistic_simulation', False), 'sell_profit_only': sell_profit_only, 'use_sell_signal': use_sell_signal, - 'record': self.config.get('export') + 'record': self.config.get('export'), + 'recordfn': self.config.get('exportfilename'), } ) logger.info( diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 1b1872404..65820ac09 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -218,7 +218,8 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non '--realistic-simulation', '--refresh-pairs-cached', '--timerange', ':100', - '--export', '/bar/foo' + '--export', '/bar/foo', + '--export-filename', 'foo_bar.json' ] config = setup_configuration(get_args(args)) @@ -259,6 +260,11 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non 'Parameter --export detected: {} ...'.format(config['export']), caplog.record_tuples ) + assert 'exportfilename' in config + assert log_has( + 'Storing backtest results to {} ...'.format(config['exportfilename']), + caplog.record_tuples + ) def test_start(mocker, fee, default_conf, caplog) -> None: From 482d0636389f8b40b2bb05d7a30cef0be2c895ef Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 3 Jun 2018 19:41:34 +0200 Subject: [PATCH 02/70] update documentation for --export-filename --- docs/backtesting.md | 6 ++++++ docs/bot-usage.md | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/docs/backtesting.md b/docs/backtesting.md index 8c4c4180d..0b53d45b7 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -62,6 +62,12 @@ Where `-s TestStrategy` refers to the class name within the strategy file `test_ python3 ./freqtrade/main.py backtesting --export trades ``` +**Exporting trades to file specifying a custom filename** +```bash +python3 ./freqtrade/main.py backtesting --export trades --export-filename=backtest_teststrategy.json +``` + + **Running backtest with smaller testset** Use the `--timerange` argument to change how much of the testset you want to use. The last N ticks/timeframes will be used. diff --git a/docs/bot-usage.md b/docs/bot-usage.md index e2c18473c..cfffd04e9 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -120,6 +120,8 @@ Backtesting also uses the config specified via `-c/--config`. ``` usage: main.py backtesting [-h] [-i TICKER_INTERVAL] [--realistic-simulation] [--timerange TIMERANGE] [-l] [-r] [--export EXPORT] + [--export-filename EXPORTFILENAME] + optional arguments: -h, --help show this help message and exit @@ -137,6 +139,11 @@ optional arguments: run your backtesting with up-to-date data. --export EXPORT export backtest results, argument are: trades Example --export=trades + --export-filename EXPORTFILENAME + Save backtest results to this filename requires + --export to be set as well Example --export- + filename=backtest_today.json (default: backtest- + result.json ``` ### How to use --refresh-pairs-cached parameter? From 26120ff6758745d57c1529cff90fce0a84d7ff11 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 3 Jun 2018 23:06:37 +0200 Subject: [PATCH 03/70] remove unnecessary .gitkeep --- docs/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 docs/.gitkeep diff --git a/docs/.gitkeep b/docs/.gitkeep deleted file mode 100644 index e69de29bb..000000000 From 5ef2654eb4d75d8c25cbba7cd1e0c5b3bec4ea7d Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 3 Jun 2018 23:07:00 +0200 Subject: [PATCH 04/70] replace references to old url replace garq with freqtrade --- .github/ISSUE_TEMPLATE.md | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 2 +- CONTRIBUTING.md | 2 +- README.md | 48 ++++++++++++++++---------------- docs/backtesting.md | 4 +-- docs/bot-optimization.md | 8 +++--- docs/bot-usage.md | 6 ++-- docs/configuration.md | 8 +++--- docs/faq.md | 2 +- docs/hyperopt.md | 12 ++++---- docs/index.md | 48 ++++++++++++++++---------------- docs/installation.md | 16 +++++------ docs/sql_cheatsheet.md | 2 +- docs/telegram-usage.md | 2 +- 14 files changed, 81 insertions(+), 81 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 2cbbc59cb..2a6d3f18f 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,6 +1,6 @@ ## Step 1: Have you search for this issue before posting it? -If you have discovered a bug in the bot, please [search our issue tracker](https://github.com/gcarq/freqtrade/issues?q=is%3Aissue). +If you have discovered a bug in the bot, please [search our issue tracker](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue). If it hasn't been reported, please create a new issue. ## Step 2: Describe your environment diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index ae79dd8f0..20ef27f0f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,5 +1,5 @@ Thank you for sending your pull request. But first, have you included -unit tests, and is your code PEP8 conformant? [More details](https://github.com/gcarq/freqtrade/blob/develop/CONTRIBUTING.md) +unit tests, and is your code PEP8 conformant? [More details](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) ## Summary Explain in one sentence the goal of this PR diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 93089495b..0cbe9167d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,7 @@ Feel like our bot is missing a feature? We welcome your pull requests! Few point conformant (max-line-length = 100). If you are unsure, discuss the feature on our [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE) -or in a [issue](https://github.com/gcarq/freqtrade/issues) before a PR. +or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. **Before sending the PR:** diff --git a/README.md b/README.md index 3ded0099a..24e01531c 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ # freqtrade -[![Build Status](https://travis-ci.org/gcarq/freqtrade.svg?branch=develop)](https://travis-ci.org/gcarq/freqtrade) -[![Coverage Status](https://coveralls.io/repos/github/gcarq/freqtrade/badge.svg?branch=develop&service=github)](https://coveralls.io/github/gcarq/freqtrade?branch=develop) -[![Maintainability](https://api.codeclimate.com/v1/badges/5737e6d668200b7518ff/maintainability)](https://codeclimate.com/github/gcarq/freqtrade/maintainability) +[![Build Status](https://travis-ci.org/freqtrade/freqtrade.svg?branch=develop)](https://travis-ci.org/freqtrade/freqtrade) +[![Coverage Status](https://coveralls.io/repos/github/freqtrade/freqtrade/badge.svg?branch=develop&service=github)](https://coveralls.io/github/freqtrade/freqtrade?branch=develop) +[![Maintainability](https://api.codeclimate.com/v1/badges/5737e6d668200b7518ff/maintainability)](https://codeclimate.com/github/freqtrade/freqtrade/maintainability) Simple High frequency trading bot for crypto currencies designed to support multi exchanges and be controlled via Telegram. -![freqtrade](https://raw.githubusercontent.com/gcarq/freqtrade/develop/docs/assets/freqtrade-screenshot.png) +![freqtrade](https://raw.githubusercontent.com/freqtrade/freqtrade/develop/docs/assets/freqtrade-screenshot.png) ## Disclaimer This software is for educational purposes only. Do not risk money which @@ -25,12 +25,12 @@ hesitate to read the source code and understand the mechanism of this bot. ## Table of Contents - [Features](#features) - [Quick start](#quick-start) -- [Documentations](https://github.com/gcarq/freqtrade/blob/develop/docs/index.md) - - [Installation](https://github.com/gcarq/freqtrade/blob/develop/docs/installation.md) - - [Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md) - - [Strategy Optimization](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md) - - [Backtesting](https://github.com/gcarq/freqtrade/blob/develop/docs/backtesting.md) - - [Hyperopt](https://github.com/gcarq/freqtrade/blob/develop/docs/hyperopt.md) +- [Documentations](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) + - [Installation](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md) + - [Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md) + - [Strategy Optimization](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md) + - [Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) + - [Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Support](#support) - [Help](#help--slack) - [Bugs](#bugs--issues) @@ -73,7 +73,7 @@ strategy parameters with real exchange data. ## Quick start This quick start section is a very short explanation on how to test the bot in dry-run. We invite you to read the -[bot documentation](https://github.com/gcarq/freqtrade/blob/develop/docs/index.md) +[bot documentation](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) to ensure you understand how the bot is working. ### Easy installation @@ -87,7 +87,7 @@ The following steps are made for Linux/MacOS environment **1. Clone the repo** ```bash -git clone git@github.com:gcarq/freqtrade.git +git clone git@github.com:freqtrade/freqtrade.git git checkout develop cd freqtrade ``` @@ -109,26 +109,26 @@ For any questions not covered by the documentation or for further information about the bot, we encourage you to join our slack channel. - [Click here to join Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE). -### [Bugs / Issues](https://github.com/gcarq/freqtrade/issues?q=is%3Aissue) +### [Bugs / Issues](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) If you discover a bug in the bot, please -[search our issue tracker](https://github.com/gcarq/freqtrade/issues?q=is%3Aissue) +[search our issue tracker](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) first. If it hasn't been reported, please -[create a new issue](https://github.com/gcarq/freqtrade/issues/new) and +[create a new issue](https://github.com/freqtrade/freqtrade/issues/new) and ensure you follow the template guide so that our team can assist you as quickly as possible. -### [Feature Requests](https://github.com/gcarq/freqtrade/labels/enhancement) +### [Feature Requests](https://github.com/freqtrade/freqtrade/labels/enhancement) Have you a great idea to improve the bot you want to share? Please, -first search if this feature was not [already discussed](https://github.com/gcarq/freqtrade/labels/enhancement). +first search if this feature was not [already discussed](https://github.com/freqtrade/freqtrade/labels/enhancement). If it hasn't been requested, please -[create a new request](https://github.com/gcarq/freqtrade/issues/new) +[create a new request](https://github.com/freqtrade/freqtrade/issues/new) and ensure you follow the template guide so that it does not get lost in the bug reports. -### [Pull Requests](https://github.com/gcarq/freqtrade/pulls) +### [Pull Requests](https://github.com/freqtrade/freqtrade/pulls) Feel like our bot is missing a feature? We welcome your pull requests! Please read our -[Contributing document](https://github.com/gcarq/freqtrade/blob/develop/CONTRIBUTING.md) +[Contributing document](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) to understand the requirements before sending your pull-requests. **Important:** Always create your PR against the `develop` branch, not @@ -171,14 +171,14 @@ optional arguments: only if dry_run is enabled. ``` More details on: -- [How to run the bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md#bot-commands) -- [How to use Backtesting](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md#backtesting-commands) -- [How to use Hyperopt](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md#hyperopt-commands) +- [How to run the bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#bot-commands) +- [How to use Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#backtesting-commands) +- [How to use Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#hyperopt-commands) ### Telegram RPC commands Telegram is not mandatory. However, this is a great way to control your bot. More details on our -[documentation](https://github.com/gcarq/freqtrade/blob/develop/docs/index.md) +[documentation](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) - `/start`: Starts the trader - `/stop`: Stops the trader diff --git a/docs/backtesting.md b/docs/backtesting.md index 0b53d45b7..ac743dcac 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -14,7 +14,7 @@ real data. This is what we call Backtesting will use the crypto-currencies (pair) from your config file and load static tickers located in -[/freqtrade/tests/testdata](https://github.com/gcarq/freqtrade/tree/develop/freqtrade/tests/testdata). +[/freqtrade/tests/testdata](https://github.com/freqtrade/freqtrade/tree/develop/freqtrade/tests/testdata). If the 5 min and 1 min ticker for the crypto-currencies to test is not already in the `testdata` folder, backtesting will download them automatically. Testdata files will not be updated until you specify it. @@ -166,4 +166,4 @@ strategies, your configuration, and the crypto-currency you have set up. ## Next step Great, your strategy is profitable. What if the bot can give your the optimal parameters to use for your strategy? -Your next step is to learn [how to find optimal parameters with Hyperopt](https://github.com/gcarq/freqtrade/blob/develop/docs/hyperopt.md) +Your next step is to learn [how to find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index b9ff3fe40..bdb21928e 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -49,7 +49,7 @@ If you want to use a strategy from a different folder you can pass `--strategy-p python3 ./freqtrade/main.py --strategy AwesomeStrategy --strategy-path /some/folder ``` -**For the following section we will use the [user_data/strategies/test_strategy.py](https://github.com/gcarq/freqtrade/blob/develop/user_data/strategies/test_strategy.py) +**For the following section we will use the [user_data/strategies/test_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py) file as reference.** ### Buy strategy @@ -138,15 +138,15 @@ def populate_indicators(dataframe: DataFrame) -> DataFrame: ``` **Want more indicators example?** -Look into the [user_data/strategies/test_strategy.py](https://github.com/gcarq/freqtrade/blob/develop/user_data/strategies/test_strategy.py). +Look into the [user_data/strategies/test_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py). Then uncomment indicators you need. ### Where is the default strategy? The default buy strategy is located in the file -[freqtrade/default_strategy.py](https://github.com/gcarq/freqtrade/blob/develop/freqtrade/strategy/default_strategy.py). +[freqtrade/default_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/strategy/default_strategy.py). ## Next step Now you have a perfect strategy you probably want to backtesting it. -Your next step is to learn [How to use the Backtesting](https://github.com/gcarq/freqtrade/blob/develop/docs/backtesting.md). +Your next step is to learn [How to use the Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md). diff --git a/docs/bot-usage.md b/docs/bot-usage.md index cfffd04e9..815fed672 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -68,7 +68,7 @@ python3 ./freqtrade/main.py --strategy AwesomeStrategy If the bot does not find your strategy file, it will display in an error message the reason (File not found, or errors in your code). -Learn more about strategy file in [optimize your bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md). +Learn more about strategy file in [optimize your bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md). ### How to use --strategy-path? This parameter allows you to add an additional strategy lookup path, which gets @@ -189,9 +189,9 @@ optional arguments: ## A parameter missing in the configuration? All parameters for `main.py`, `backtesting`, `hyperopt` are referenced -in [misc.py](https://github.com/gcarq/freqtrade/blob/develop/freqtrade/misc.py#L84) +in [misc.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/misc.py#L84) ## Next step The optimal strategy of the bot will change with time depending of the market trends. The next step is to -[optimize your bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md). +[optimize your bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md). diff --git a/docs/configuration.md b/docs/configuration.md index a2df3f2fe..bd867857c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -40,7 +40,7 @@ The table below will list all configuration parameters. | `internals.process_throttle_secs` | 5 | Yes | Set the process throttle. Value in second. The definition of each config parameters is in -[misc.py](https://github.com/gcarq/freqtrade/blob/develop/freqtrade/misc.py#L205). +[misc.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/misc.py#L205). ### Understand minimal_roi `minimal_roi` is a JSON object where the key is a duration @@ -141,12 +141,12 @@ you run it in production mode. "key": "af8ddd35195e9dc500b9a6f799f6f5c93d89193b", "secret": "08a9dc6db3d7b53e1acebd9275677f4b0a04f1a5", ... -} +} ``` If you have not your Bittrex API key yet, -[see our tutorial](https://github.com/gcarq/freqtrade/blob/develop/docs/pre-requisite.md). +[see our tutorial](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md). ## Next step Now you have configured your config.json, the next step is to -[start your bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md). +[start your bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md). diff --git a/docs/faq.md b/docs/faq.md index b3f15613a..31a302067 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -27,7 +27,7 @@ like pauses. You can stop your bot, adjust settings and start it again. #### I want to improve the bot with a new strategy That's great. We have a nice backtesting and hyperoptimizing setup. See -the tutorial [here|Testing-new-strategies-with-Hyperopt](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md#hyperopt-commands). +the tutorial [here|Testing-new-strategies-with-Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#hyperopt-commands). #### Is there a setting to only SELL the coins being held and not perform anymore BUYS? diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 0bfa9ae2d..a079e34df 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -14,13 +14,13 @@ parameters with Hyperopt. ## Prepare Hyperopt Before we start digging in Hyperopt, we recommend you to take a look at -your strategy file located into [user_data/strategies/](https://github.com/gcarq/freqtrade/blob/develop/user_data/strategies/test_strategy.py) +your strategy file located into [user_data/strategies/](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py) ### 1. Configure your Guards and Triggers There are two places you need to change in your strategy file to add a new buy strategy for testing: -- Inside [populate_buy_trend()](https://github.com/gcarq/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L278-L294). -- Inside [hyperopt_space()](https://github.com/gcarq/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L244-L297) known as `SPACE`. +- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L278-L294). +- Inside [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L244-L297) known as `SPACE`. There you have two different type of indicators: 1. `guards` and 2. `triggers`. @@ -110,13 +110,13 @@ cannot use your config file. It is also made on purpose to allow you testing your strategy with different configurations. The Hyperopt configuration is located in -[user_data/hyperopt_conf.py](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopt_conf.py). +[user_data/hyperopt_conf.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopt_conf.py). ## Advanced notions ### Understand the Guards and Triggers When you need to add the new guards and triggers to be hyperopt -parameters, you do this by adding them into the [hyperopt_space()](https://github.com/gcarq/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L244-L297). +parameters, you do this by adding them into the [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L244-L297). If it's a trigger, you add one line to the 'trigger' choice group and that's it. @@ -312,4 +312,4 @@ def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: ## Next step Now you have a perfect bot and want to control it from Telegram. Your -next step is to learn the [Telegram usage](https://github.com/gcarq/freqtrade/blob/develop/docs/telegram-usage.md). +next step is to learn the [Telegram usage](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md). diff --git a/docs/index.md b/docs/index.md index ed940d132..afde2d5eb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,27 +6,27 @@ Pull-request. Do not hesitate to reach us on if you do not find the answer to your questions. ## Table of Contents -- [Pre-requisite](https://github.com/gcarq/freqtrade/blob/develop/docs/pre-requisite.md) - - [Setup your Bittrex account](https://github.com/gcarq/freqtrade/blob/develop/docs/pre-requisite.md#setup-your-bittrex-account) - - [Setup your Telegram bot](https://github.com/gcarq/freqtrade/blob/develop/docs/pre-requisite.md#setup-your-telegram-bot) -- [Bot Installation](https://github.com/gcarq/freqtrade/blob/develop/docs/installation.md) - - [Install with Docker (all platforms)](https://github.com/gcarq/freqtrade/blob/develop/docs/installation.md#docker) - - [Install on Linux Ubuntu](https://github.com/gcarq/freqtrade/blob/develop/docs/installation.md#21-linux---ubuntu-1604) - - [Install on MacOS](https://github.com/gcarq/freqtrade/blob/develop/docs/installation.md#23-macos-installation) - - [Install on Windows](https://github.com/gcarq/freqtrade/blob/develop/docs/installation.md#windows) -- [Bot Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md) -- [Bot usage (Start your bot)](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md) - - [Bot commands](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md#bot-commands) - - [Backtesting commands](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md#backtesting-commands) - - [Hyperopt commands](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md#hyperopt-commands) -- [Bot Optimization](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md) - - [Change your strategy](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md#change-your-strategy) - - [Add more Indicator](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md#add-more-indicator) - - [Test your strategy with Backtesting](https://github.com/gcarq/freqtrade/blob/develop/docs/backtesting.md) - - [Find optimal parameters with Hyperopt](https://github.com/gcarq/freqtrade/blob/develop/docs/hyperopt.md) -- [Control the bot with telegram](https://github.com/gcarq/freqtrade/blob/develop/docs/telegram-usage.md) -- [Contribute to the project](https://github.com/gcarq/freqtrade/blob/develop/CONTRIBUTING.md) - - [How to contribute](https://github.com/gcarq/freqtrade/blob/develop/CONTRIBUTING.md) - - [Run tests & Check PEP8 compliance](https://github.com/gcarq/freqtrade/blob/develop/CONTRIBUTING.md) -- [FAQ](https://github.com/gcarq/freqtrade/blob/develop/docs/faq.md) - - [SQL cheatsheet](https://github.com/gcarq/freqtrade/blob/develop/docs/sql_cheatsheet.md) \ No newline at end of file +- [Pre-requisite](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md) + - [Setup your Bittrex account](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md#setup-your-bittrex-account) + - [Setup your Telegram bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md#setup-your-telegram-bot) +- [Bot Installation](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md) + - [Install with Docker (all platforms)](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md#docker) + - [Install on Linux Ubuntu](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md#21-linux---ubuntu-1604) + - [Install on MacOS](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md#23-macos-installation) + - [Install on Windows](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md#windows) +- [Bot Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md) +- [Bot usage (Start your bot)](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md) + - [Bot commands](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#bot-commands) + - [Backtesting commands](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#backtesting-commands) + - [Hyperopt commands](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#hyperopt-commands) +- [Bot Optimization](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md) + - [Change your strategy](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#change-your-strategy) + - [Add more Indicator](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#add-more-indicator) + - [Test your strategy with Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) + - [Find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) +- [Control the bot with telegram](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md) +- [Contribute to the project](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) + - [How to contribute](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) + - [Run tests & Check PEP8 compliance](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) +- [FAQ](https://github.com/freqtrade/freqtrade/blob/develop/docs/faq.md) + - [SQL cheatsheet](https://github.com/freqtrade/freqtrade/blob/develop/docs/sql_cheatsheet.md) diff --git a/docs/installation.md b/docs/installation.md index be8e2e501..850b2c255 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -2,7 +2,7 @@ This page explains how to prepare your environment for running the bot. -To understand how to set up the bot please read the [Bot Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md) page. +To understand how to set up the bot please read the [Bot Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md) page. ## Table of Contents @@ -69,7 +69,7 @@ Once you have Docker installed, simply create the config file (e.g. `config.json #### 1.1. Clone the git repository ```bash -git clone https://github.com/gcarq/freqtrade.git +git clone https://github.com/freqtrade/freqtrade.git ``` #### 1.2. (Optional) Checkout the develop branch @@ -90,7 +90,7 @@ cd freqtrade cp -n config.json.example config.json ``` -> To edit the config please refer to the [Bot Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md) page. +> To edit the config please refer to the [Bot Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md) page. #### 1.5. Create your database file *(optional - the bot will create it if it is missing)* @@ -237,7 +237,7 @@ sudo apt-get install mongodb-org Clone the git repository: ```bash -git clone https://github.com/gcarq/freqtrade.git +git clone https://github.com/freqtrade/freqtrade.git ``` Optionally checkout the develop branch: @@ -287,7 +287,7 @@ export PATH=/env/mongodb/bin:$PATH Clone the git repository: ```bash -git clone https://github.com/gcarq/freqtrade.git +git clone https://github.com/freqtrade/freqtrade.git ``` Optionally checkout the develop branch: @@ -306,7 +306,7 @@ cd freqtrade cp config.json.example config.json ``` -> *To edit the config please refer to [Bot Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md).* +> *To edit the config please refer to [Bot Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md).* #### 2. Setup your Python virtual environment (virtualenv) @@ -350,8 +350,8 @@ copy paste `config.json` to ``\path\freqtrade-develop\freqtrade` >python main.py ``` -> Thanks [Owdr](https://github.com/Owdr) for the commands. Source: [Issue #222](https://github.com/gcarq/freqtrade/issues/222) +> Thanks [Owdr](https://github.com/Owdr) for the commands. Source: [Issue #222](https://github.com/freqtrade/freqtrade/issues/222) Now you have an environment ready, the next step is -[Bot Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md)... +[Bot Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md)... diff --git a/docs/sql_cheatsheet.md b/docs/sql_cheatsheet.md index 141eaeafe..ba26f1707 100644 --- a/docs/sql_cheatsheet.md +++ b/docs/sql_cheatsheet.md @@ -85,7 +85,7 @@ INSERT INTO trades (exchange, pair, is_open, fee_open, fee_close, open_rate, sta ## Fix wrong fees in the table If your DB was created before -[PR#200](https://github.com/gcarq/freqtrade/pull/200) was merged +[PR#200](https://github.com/freqtrade/freqtrade/pull/200) was merged (before 12/23/17). ```sql diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 20e80269b..4f878dbd1 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -4,7 +4,7 @@ This page explains how to command your bot with Telegram. ## Pre-requisite To control your bot with Telegram, you need first to -[set up a Telegram bot](https://github.com/gcarq/freqtrade/blob/develop/docs/pre-requisite.md) +[set up a Telegram bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md) and add your Telegram API keys into your config file. ## Telegram commands From eeda93a3595435f9baf7ffa82c606614b5c45b08 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 10:01:26 +0300 Subject: [PATCH 05/70] Fix folder names in custom datadir documentation --- docs/backtesting.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index ac743dcac..f843d87e0 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -97,17 +97,17 @@ The full timerange specification: To update your testdata directory, or download into another testdata directory: ```bash mkdir -p user_data/data/testdata-20180113 -cp freqtrade/tests/testdata/pairs.json user_data/data-20180113 -cd user_data/data-20180113 +cp freqtrade/tests/testdata/pairs.json user_data/data/testdata-20180113 +cd user_data/data/testdata-20180113 ``` -Possibly edit pairs.json file to include/exclude pairs +Possibly edit `pairs.json` file to include/exclude pairs ```bash python3 freqtrade/tests/testdata/download_backtest_data.py -p pairs.json ``` -The script will read your pairs.json file, and download ticker data +The script will read your `pairs.json` file, and download ticker data into the current working directory. From a0c79bd7272754024ce2b796e94cf45530c119c3 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 11:47:27 +0300 Subject: [PATCH 06/70] make --pairs-file required --- freqtrade/arguments.py | 1 + scripts/download_backtest_data.py | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 7e895177a..04d94a572 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -291,6 +291,7 @@ class Arguments(object): '--pairs-file', help='File containing a list of pairs to download', dest='pairs_file', + required=True, default=None ) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 1c73eae03..85debe703 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -15,12 +15,9 @@ arguments.testdata_dl_options() args = arguments.parse_args() TICKER_INTERVALS = ['1m', '5m'] -PAIRS = [] -if args.pairs_file: - with open(args.pairs_file) as file: - PAIRS = json.load(file) -PAIRS = list(set(PAIRS)) +with open(args.pairs_file) as file: + PAIRS = list(set(json.load(file))) dl_path = DEFAULT_DL_PATH if args.export and os.path.exists(args.export): From e10279b7b4af2dd2b9a0dea28bf668ecbce986e8 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 11:50:33 +0300 Subject: [PATCH 07/70] show default exchange in download_backtest_data.py --- freqtrade/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 04d94a572..bf2ba4d6d 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -310,7 +310,7 @@ class Arguments(object): self.parser.add_argument( '--exchange', - help='Exchange name', + help='Exchange name (default: %(default)s)', dest='exchange', type=str, default='bittrex') From 6891054b8470738a00ac2d79a009812d5e1e580f Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 12:20:17 +0300 Subject: [PATCH 08/70] use folder user_data/data/exchangename by default and pick pairs.json from that folder by default --- freqtrade/arguments.py | 1 - scripts/download_backtest_data.py | 16 ++++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index bf2ba4d6d..fd4a88826 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -291,7 +291,6 @@ class Arguments(object): '--pairs-file', help='File containing a list of pairs to download', dest='pairs_file', - required=True, default=None ) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 85debe703..c66103769 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -8,7 +8,7 @@ import arrow from freqtrade import (exchange, arguments, misc) -DEFAULT_DL_PATH = 'freqtrade/tests/testdata' +DEFAULT_DL_PATH = 'user_data/data' arguments = arguments.Arguments(sys.argv[1:], 'download utility') arguments.testdata_dl_options() @@ -16,12 +16,16 @@ args = arguments.parse_args() TICKER_INTERVALS = ['1m', '5m'] -with open(args.pairs_file) as file: - PAIRS = list(set(json.load(file))) +dl_path = args.export if args.export and os.path.exists(args.export) else os.path.join(DEFAULT_DL_PATH, args.exchange) +if not os.path.isdir(dl_path): + sys.exit(f'Directory {dl_path} does not exist.') -dl_path = DEFAULT_DL_PATH -if args.export and os.path.exists(args.export): - dl_path = args.export +pairs_file = args.pairs_file if args.pairs_file else os.path.join(dl_path, 'pairs.json') +if not os.path.isfile(pairs_file): + sys.exit(f'No pairs file found with path {pairs_file}.') + +with open(pairs_file) as file: + PAIRS = list(set(json.load(file))) since_time = None if args.days: From d4b431a3350e0d1313b3ffc993ab2c88ade1de61 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 12:30:21 +0300 Subject: [PATCH 09/70] update documentation about download_backtesting_data.py script --- docs/backtesting.md | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index f843d87e0..ab47923c6 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -93,22 +93,30 @@ The full timerange specification: `--timerange=1527595200-1527618600` -**Update testdata directory** -To update your testdata directory, or download into another testdata directory: -```bash -mkdir -p user_data/data/testdata-20180113 -cp freqtrade/tests/testdata/pairs.json user_data/data/testdata-20180113 -cd user_data/data/testdata-20180113 -``` +**Downloading new set of ticker data** +To download new set of backtesting ticker data, you can use a download script. -Possibly edit `pairs.json` file to include/exclude pairs +If you are using Binance for example: +- create a folder `user_data/data/binance` and copy `pairs.json` in that folder. +- update the `pairs.json` to contain the currency pairs you are interested in. ```bash -python3 freqtrade/tests/testdata/download_backtest_data.py -p pairs.json +mkdir -p user_data/data/binance +cp freqtrade/tests/testdata/pairs.json user_data/data/binance ``` -The script will read your `pairs.json` file, and download ticker data -into the current working directory. +Then run: + +```bash +python scripts/download_backtest_data --exchange binance +``` + +This will download ticker data for all the currency pairs you defined in `pairs.json`. + +- To use a different folder than the exchange specific default, use `--export user_data/data/some_directory`. +- To change the exchange used to download the tickers, use `--exchange`. Default is `bittrex`. +- To use `pairs.json` from some other folder, use `--pairs-file some_other_dir/pairs.json`. +- To download ticker data for only 10 days, use `--days 10`. For help about backtesting usage, please refer to From 5c7899ae9864e07efb5817dbd4bc56038fa4dc0d Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 12:45:23 +0300 Subject: [PATCH 10/70] flake8 fix --- scripts/download_backtest_data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index c66103769..cd64f36bb 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -16,7 +16,8 @@ args = arguments.parse_args() TICKER_INTERVALS = ['1m', '5m'] -dl_path = args.export if args.export and os.path.exists(args.export) else os.path.join(DEFAULT_DL_PATH, args.exchange) +dl_path = args.export if args.export and os.path.exists(args.export) \ + else os.path.join(DEFAULT_DL_PATH, args.exchange) if not os.path.isdir(dl_path): sys.exit(f'Directory {dl_path} does not exist.') From af1ba1e191e42c00c7bec80a74cd06ebcdd756d3 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 12:58:35 +0300 Subject: [PATCH 11/70] split ugly ternary to regular if --- scripts/download_backtest_data.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index cd64f36bb..18a543c4e 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -16,8 +16,10 @@ args = arguments.parse_args() TICKER_INTERVALS = ['1m', '5m'] -dl_path = args.export if args.export and os.path.exists(args.export) \ - else os.path.join(DEFAULT_DL_PATH, args.exchange) +dl_path = os.path.join(DEFAULT_DL_PATH, args.exchange) +if args.export: + dl_path = args.export + if not os.path.isdir(dl_path): sys.exit(f'Directory {dl_path} does not exist.') From 639b6bc4f6156e04907bc70f182490887c56d88e Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 14:13:19 +0300 Subject: [PATCH 12/70] set and create default datadir based on used exchange --- freqtrade/arguments.py | 5 ++--- freqtrade/configuration.py | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index fd4a88826..9e88d08ca 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -4,7 +4,6 @@ This module contains the argument manager class import argparse import logging -import os import re import arrow from typing import List, Tuple, Optional @@ -72,9 +71,9 @@ class Arguments(object): ) self.parser.add_argument( '-d', '--datadir', - help='path to backtest data (default: %(default)s', + help='path to backtest data', dest='datadir', - default=os.path.join('freqtrade', 'tests', 'testdata'), + default=None, type=str, metavar='PATH', ) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 77b5b4447..54146199d 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -1,7 +1,7 @@ """ This module contains the configuration class """ - +import os import json import logging from argparse import Namespace @@ -113,6 +113,14 @@ class Configuration(object): return config + def _create_default_datadir(self, config: Dict[str, Any]) -> str: + exchange_name = config.get('exchange', {}).get('name').lower() + default_path = os.path.join('user_data', 'data', exchange_name) + if not os.path.isdir(default_path): + os.makedirs(default_path) + logger.info(f'Created data directory: {default_path}') + return default_path + def _load_backtesting_config(self, config: Dict[str, Any]) -> Dict[str, Any]: """ Extract information for sys.argv and load Backtesting configuration @@ -145,7 +153,9 @@ class Configuration(object): # If --datadir is used we add it to the configuration if 'datadir' in self.args and self.args.datadir: config.update({'datadir': self.args.datadir}) - logger.info('Using data folder: %s ...', self.args.datadir) + else: + config.update({'datadir': self._create_default_datadir(config)}) + logger.info('Using data folder: %s ...', config.get('datadir')) # If -r/--refresh-pairs-cached is used we add it to the configuration if 'refresh_pairs' in self.args and self.args.refresh_pairs: From 3321e4cafd0761acd82e8fc9fd96c2e8d2ff9442 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 14:14:04 +0300 Subject: [PATCH 13/70] travis should run hyperopt and backtesting using tests/testdata tickers --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1cff5c04b..c3c118654 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,10 +21,10 @@ jobs: - script: pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ - script: - cp config.json.example config.json - - python freqtrade/main.py backtesting + - python freqtrade/main.py --datadir freqtrade/tests/testdata backtesting - script: - cp config.json.example config.json - - python freqtrade/main.py hyperopt -e 5 + - python freqtrade/main.py --datadir freqtrade/tests/testdata hyperopt -e 5 - script: flake8 freqtrade - script: mypy freqtrade after_success: From 5ff405b0b0e870b176cf027df68499fbb50f793c Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 13:31:52 +0300 Subject: [PATCH 14/70] allow defining of timeframes to download --- docs/backtesting.md | 1 + freqtrade/arguments.py | 11 +++++++++++ scripts/download_backtest_data.py | 4 ++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index ab47923c6..8364d77e4 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -117,6 +117,7 @@ This will download ticker data for all the currency pairs you defined in `pairs. - To change the exchange used to download the tickers, use `--exchange`. Default is `bittrex`. - To use `pairs.json` from some other folder, use `--pairs-file some_other_dir/pairs.json`. - To download ticker data for only 10 days, use `--days 10`. +- Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers. For help about backtesting usage, please refer to diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 9e88d08ca..ff9c8da2c 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -312,3 +312,14 @@ class Arguments(object): dest='exchange', type=str, default='bittrex') + + self.parser.add_argument( + '-t', '--timeframes', + help='Specify which tickers to download. Space separated list. \ + Default: %(default)s', + choices=['5m', '15m', '30m', '1h', '2h', '4h', + '6h', '8h', '12h', '1d', '3d', '1w', '1M'], + default=['1m', '5m'], + nargs='+', + dest='timeframes', + ) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 18a543c4e..6185edaf7 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -14,7 +14,7 @@ arguments = arguments.Arguments(sys.argv[1:], 'download utility') arguments.testdata_dl_options() args = arguments.parse_args() -TICKER_INTERVALS = ['1m', '5m'] +timeframes = args.timeframes dl_path = os.path.join(DEFAULT_DL_PATH, args.exchange) if args.export: @@ -44,7 +44,7 @@ exchange._API = exchange.init_ccxt({'key': '', for pair in PAIRS: - for tick_interval in TICKER_INTERVALS: + for tick_interval in timeframes: print(f'downloading pair {pair}, interval {tick_interval}') data = exchange.get_ticker_history(pair, tick_interval, since_ms=since_time) From 0f3dc821f21b2d37b8ce2c7e41713106b2125f0a Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 15:03:42 +0300 Subject: [PATCH 15/70] add missing timeframes to allowed values --- freqtrade/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index ff9c8da2c..e63e74419 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -317,7 +317,7 @@ class Arguments(object): '-t', '--timeframes', help='Specify which tickers to download. Space separated list. \ Default: %(default)s', - choices=['5m', '15m', '30m', '1h', '2h', '4h', + choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', '6h', '8h', '12h', '1d', '3d', '1w', '1M'], default=['1m', '5m'], nargs='+', From 7df77b1b286f4d8c0e6a443e2ea40509c0128534 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 4 Jun 2018 16:35:00 +0300 Subject: [PATCH 16/70] match timeframes to arguments --- freqtrade/arguments.py | 2 +- freqtrade/constants.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index e63e74419..a45dd20b3 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -318,7 +318,7 @@ class Arguments(object): help='Specify which tickers to download. Space separated list. \ Default: %(default)s', choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', - '6h', '8h', '12h', '1d', '3d', '1w', '1M'], + '6h', '8h', '12h', '1d', '3d', '1w'], default=['1m', '5m'], nargs='+', dest='timeframes', diff --git a/freqtrade/constants.py b/freqtrade/constants.py index a22a06ebe..910ac9b65 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -12,6 +12,7 @@ DEFAULT_STRATEGY = 'DefaultStrategy' TICKER_INTERVAL_MINUTES = { '1m': 1, + '3m': 3, '5m': 5, '15m': 15, '30m': 30, @@ -19,8 +20,10 @@ TICKER_INTERVAL_MINUTES = { '2h': 120, '4h': 240, '6h': 360, + '8h': 480, '12h': 720, '1d': 1440, + '3d': 4320, '1w': 10080, } From 7c8bf95f8f7b2f04286e8f48208da43b5455bc8e Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 4 Jun 2018 16:45:47 +0300 Subject: [PATCH 17/70] To be able to start bot with USDT in fiat_display_currency in config.json There are use case that build the base pair to consider price of whitelist pairs. On Binance this is USDT not USD. --- freqtrade/constants.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index a22a06ebe..9560afae8 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -28,7 +28,8 @@ SUPPORTED_FIAT = [ "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", - "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD" + "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD", + "USDT" ] # Required json-schema for user specified config From a44978a0680b489a5123b1c7f46d28faba24b5c3 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 4 Jun 2018 21:48:15 +0300 Subject: [PATCH 18/70] Per steer from project core member, add other valid coinmarketcap listed crypto base currencies that are valid during conversion lookup Here is the test of USDT working: https://api.coinmarketcap.com/v2/ticker/1027/?convert=USDT&limit=10 CMK page lists: "BTC", "ETH" "XRP", "LTC", and "BCH" as valid. --- freqtrade/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 9560afae8..f40efec9a 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -29,7 +29,7 @@ SUPPORTED_FIAT = [ "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD", - "USDT" + "BTC", "ETH", "XRP", "LTC", "BCH", "USDT" ] # Required json-schema for user specified config From b13658b3198c5e6c855ae0801f231dae9bddc24e Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 4 Jun 2018 22:17:10 +0300 Subject: [PATCH 19/70] Updated configuration doc with new fiat values accepted. --- docs/configuration.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index bd867857c..4c101e8a3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -92,8 +92,10 @@ The bot was tested with the following exchanges: Feel free to test other exchanges and submit your PR to improve the bot. ### What values for fiat_display_currency? -`fiat_display_currency` set the fiat to use for the conversion form coin to fiat in Telegram. -The valid value are: "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD". +`fiat_display_currency` set the base currency to use for the conversion form coin to fiat in Telegram. +The valid values are: "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD". +In addition to central bank currencies, a range of cryto currencies are supported. +The valid values are: "BTC", "ETH", "XRP", "LTC", "BCH", "USDT". ## Switch to dry-run mode We recommend starting the bot in dry-run mode to see how your bot will From e52ec145886212dde7fdc50a7ba1f46e4f01b904 Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Mon, 4 Jun 2018 22:19:25 +0300 Subject: [PATCH 20/70] Update configuration.md typo, form to from. --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 4c101e8a3..2cc6294f4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -92,7 +92,7 @@ The bot was tested with the following exchanges: Feel free to test other exchanges and submit your PR to improve the bot. ### What values for fiat_display_currency? -`fiat_display_currency` set the base currency to use for the conversion form coin to fiat in Telegram. +`fiat_display_currency` set the base currency to use for the conversion from coin to fiat in Telegram. The valid values are: "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD". In addition to central bank currencies, a range of cryto currencies are supported. The valid values are: "BTC", "ETH", "XRP", "LTC", "BCH", "USDT". From 5683f9e10e76c12a8bf1b91f84d3b07b3825e315 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 3 Jun 2018 13:58:00 -0700 Subject: [PATCH 21/70] Remove hardcoded backtest-result.json in Plot scripts --- .gitignore | 1 + freqtrade/arguments.py | 13 +++++++------ scripts/plot_dataframe.py | 13 +++++-------- scripts/plot_profit.py | 33 +++++++++++++++++--------------- user_data/backtest_data/.gitkeep | 0 5 files changed, 31 insertions(+), 29 deletions(-) create mode 100644 user_data/backtest_data/.gitkeep diff --git a/.gitignore b/.gitignore index 219a9fb40..b52a31d8e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ logfile.txt hyperopt_trials.pickle user_data/ freqtrade-plot.html +freqtrade-profit-plot.html # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index a45dd20b3..c9cecb1f5 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -123,8 +123,8 @@ class Arguments(object): ) parser.add_argument( '-r', '--refresh-pairs-cached', - help='refresh the pairs files in tests/testdata with the latest data from the exchange. \ - Use it if you want to run your backtesting with up-to-date data.', + help='refresh the pairs files in tests/testdata with the latest data from the ' + 'exchange. Use it if you want to run your backtesting with up-to-date data.', action='store_true', dest='refresh_pairs', ) @@ -140,11 +140,12 @@ class Arguments(object): '--export-filename', help='Save backtest results to this filename \ requires --export to be set as well\ - Example --export-filename=backtest_today.json\ + Example --export-filename=user_data/backtest_data/backtest_today.json\ (default: %(default)s', type=str, - default='backtest-result.json', + default=os.path.join('user_data', 'backtest_data', 'backtest-result.json'), dest='exportfilename', + metavar='PATH', ) @staticmethod @@ -220,8 +221,8 @@ class Arguments(object): self.hyperopt_options(hyperopt_cmd) @staticmethod - def parse_timerange(text: Optional[str]) -> Optional[Tuple[Tuple, - Optional[int], Optional[int]]]: + def parse_timerange(text: Optional[str]) -> \ + Optional[Tuple[Tuple, Optional[int], Optional[int]]]: """ Parse the value of the argument --timerange to determine what is the range desired :param text: value from --timerange diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 108c0b609..47ce0e746 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -13,18 +13,14 @@ Optional Cli parameters -db / --db-url: Show trades stored in database """ import logging +import os import sys from argparse import Namespace - -from typing import List - +from typing import Dict, List, Any +from sqlalchemy import create_engine from plotly import tools from plotly.offline import plot import plotly.graph_objs as go - -from typing import Dict, List, Any -from sqlalchemy import create_engine - from freqtrade.arguments import Arguments from freqtrade.analyze import Analyze from freqtrade import exchange @@ -35,6 +31,7 @@ from freqtrade.persistence import Trade logger = logging.getLogger(__name__) _CONF: Dict[str, Any] = {} + def plot_analyzed_dataframe(args: Namespace) -> None: """ Calls analyze() and plots the returned dataframe @@ -187,7 +184,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: fig['layout']['yaxis2'].update(title='Volume') fig['layout']['yaxis3'].update(title='MACD') - plot(fig, filename='freqtrade-plot.html') + plot(fig, filename=os.path.join('user_data', 'freqtrade-plot.html')) def plot_parse_args(args: List[str]) -> Namespace: diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index daa16ddc9..a5ac00169 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -8,9 +8,12 @@ Mandatory Cli parameters: Optional Cli parameters -c / --config: specify configuration file -s / --strategy: strategy to use ---timerange: specify what timerange of data to use. +-d / --datadir: path to pair backtest data +--timerange: specify what timerange of data to use +--export-filename: Specify where the backtest export is located. """ import logging +import os import sys import json from argparse import Namespace @@ -90,7 +93,18 @@ def plot_profit(args: Namespace) -> None: 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', config.get('strategy') ) - exit() + exit(0) + + # Load the profits results + try: + filename = args.exportfilename + with open(filename) as file: + data = json.load(file) + except FileNotFoundError: + logger.critical( + 'File "backtest-result.json" not found. This script require backtesting ' + 'results to run.\nPlease run a backtesting with the parameter --export.') + exit(0) # Take pairs from the cli otherwise switch to the pair in the config file if args.pair: @@ -140,18 +154,7 @@ def plot_profit(args: Namespace) -> None: num += 1 avgclose /= num - # Load the profits results - # And make an profits-growth array - - try: - filename = 'backtest-result.json' - with open(filename) as file: - data = json.load(file) - except FileNotFoundError: - logger.critical('File "backtest-result.json" not found. This script require backtesting ' - 'results to run.\nPlease run a backtesting with the parameter --export.') - exit(0) - + # make an profits-growth array pg = make_profit_array(data, num_iterations, min_date, tick_interval, filter_pairs) # @@ -184,7 +187,7 @@ def plot_profit(args: Namespace) -> None: ) fig.append_trace(pair_profit, 3, 1) - plot(fig, filename='freqtrade-profit-plot.html') + plot(fig, filename=os.path.join('user_data', 'freqtrade-profit-plot.html')) def define_index(min_date: int, max_date: int, interval: str) -> int: diff --git a/user_data/backtest_data/.gitkeep b/user_data/backtest_data/.gitkeep new file mode 100644 index 000000000..e69de29bb From af76d5f0e003fa0e3b3475228ef08360eb639cc7 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 3 Jun 2018 14:36:22 -0700 Subject: [PATCH 22/70] Breakdown the script in functions the improve maintainability --- scripts/plot_dataframe.py | 46 +++++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 47ce0e746..b031118f0 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -23,6 +23,8 @@ from plotly.offline import plot import plotly.graph_objs as go from freqtrade.arguments import Arguments from freqtrade.analyze import Analyze +from freqtrade.optimize.backtesting import setup_configuration +from freqtrade.configuration import Configuration from freqtrade import exchange import freqtrade.optimize as optimize from freqtrade import persistence @@ -37,12 +39,19 @@ def plot_analyzed_dataframe(args: Namespace) -> None: Calls analyze() and plots the returned dataframe :return: None """ - pair = args.pair.replace('-', '_') + + # Load the configuration + config = setup_configuration(args) + + # Set the pair to audit + pair = args.pair + + # Set timerange to use timerange = Arguments.parse_timerange(args.timerange) - # Init strategy + # Load the strategy try: - analyze = Analyze({'strategy': args.strategy}) + analyze = Analyze(config) except AttributeError: logger.critical( 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', @@ -50,13 +59,15 @@ def plot_analyzed_dataframe(args: Namespace) -> None: ) exit() - tick_interval = analyze.strategy.ticker_interval + # Set the ticker to use + tick_interval = analyze.get_ticker_interval() + # Load pqir tickers tickers = {} if args.live: logger.info('Downloading pair.') # Init Bittrex to use public API - exchange.init({'key': '', 'secret': ''}) + exchange.init(config) tickers[pair] = exchange.get_ticker_history(pair, tick_interval) else: tickers = optimize.load_data( @@ -66,20 +77,31 @@ def plot_analyzed_dataframe(args: Namespace) -> None: refresh_pairs=False, timerange=timerange ) - dataframes = analyze.tickerdata_to_dataframe(tickers) - dataframe = dataframes[pair] - dataframe = analyze.populate_buy_trend(dataframe) - dataframe = analyze.populate_sell_trend(dataframe) + # Get trades already made from the DB trades = [] if args.db_url: engine = create_engine('sqlite:///' + args.db_url) persistence.init(_CONF, engine) trades = Trade.query.filter(Trade.pair.is_(pair)).all() + + dataframes = analyze.tickerdata_to_dataframe(tickers) + dataframe = dataframes[pair] + dataframe = analyze.populate_buy_trend(dataframe) + dataframe = analyze.populate_sell_trend(dataframe) + if len(dataframe.index) > 750: logger.warning('Ticker contained more than 750 candles, clipping.') - data = dataframe.tail(750) + + generate_graph( + pair=pair, + trades=trades, + data=dataframe.tail(750) + ) + + +def generate_graph(pair, trades, data): candles = go.Candlestick( x=data.date, @@ -168,6 +190,8 @@ def plot_analyzed_dataframe(args: Namespace) -> None: vertical_spacing=0.0001, ) + # Row 1 + fig.append_trace(candles, 1, 1) fig.append_trace(bb_lower, 1, 1) fig.append_trace(bb_upper, 1, 1) @@ -179,7 +203,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: fig.append_trace(trade_buys, 1, 1) fig.append_trace(trade_sells, 1, 1) - fig['layout'].update(title=args.pair) + fig['layout'].update(title=pair) fig['layout']['yaxis1'].update(title='Price') fig['layout']['yaxis2'].update(title='Volume') fig['layout']['yaxis3'].update(title='MACD') From 64504e67779afbba995f4d93d6fd86ffb6e8de72 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 3 Jun 2018 15:53:15 -0700 Subject: [PATCH 23/70] Add support of --refresh-pairs-cached param --- scripts/plot_dataframe.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index b031118f0..94e72d2dd 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -24,7 +24,6 @@ import plotly.graph_objs as go from freqtrade.arguments import Arguments from freqtrade.analyze import Analyze from freqtrade.optimize.backtesting import setup_configuration -from freqtrade.configuration import Configuration from freqtrade import exchange import freqtrade.optimize as optimize from freqtrade import persistence @@ -52,6 +51,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: # Load the strategy try: analyze = Analyze(config) + exchange.init(config) except AttributeError: logger.critical( 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', @@ -67,17 +67,20 @@ def plot_analyzed_dataframe(args: Namespace) -> None: if args.live: logger.info('Downloading pair.') # Init Bittrex to use public API - exchange.init(config) tickers[pair] = exchange.get_ticker_history(pair, tick_interval) else: tickers = optimize.load_data( datadir=args.datadir, pairs=[pair], ticker_interval=tick_interval, - refresh_pairs=False, + refresh_pairs=config.get('refresh_pairs', False), timerange=timerange ) + # No ticker found, or impossible to download + if tickers == {}: + exit() + # Get trades already made from the DB trades = [] if args.db_url: From 1c75bfdddde8a7c59ee3fa0cec5fdfc87acf9bff Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 3 Jun 2018 18:41:28 -0700 Subject: [PATCH 24/70] Add more indicators --- scripts/plot_dataframe.py | 167 +++++++++++++++++++++++++++++--------- 1 file changed, 127 insertions(+), 40 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 94e72d2dd..247f530d2 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -33,6 +33,20 @@ logger = logging.getLogger(__name__) _CONF: Dict[str, Any] = {} +# Update the global variable TO_DISPLAY to select which indicator you want to display +TO_DISPLAY = { + 'sma': False, # On Row 1 + 'ema': True, # On Row 1 + + 'macd': True, # On Row 3 + 'rsi': False, # On Row 3 + 'fisher_rsi': False, # On Row 3 + 'mfi': False, # On Row 3 + 'slow': False, # On Row 3 + 'fast': False # On Row 3 +} + + def plot_analyzed_dataframe(args: Namespace) -> None: """ Calls analyze() and plots the returned dataframe @@ -88,7 +102,6 @@ def plot_analyzed_dataframe(args: Namespace) -> None: persistence.init(_CONF, engine) trades = Trade.query.filter(Trade.pair.is_(pair)).all() - dataframes = analyze.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] dataframe = analyze.populate_buy_trend(dataframe) @@ -97,15 +110,35 @@ def plot_analyzed_dataframe(args: Namespace) -> None: if len(dataframe.index) > 750: logger.warning('Ticker contained more than 750 candles, clipping.') - generate_graph( + fig = generate_graph( pair=pair, trades=trades, data=dataframe.tail(750) ) + plot(fig, filename=os.path.join('user_data', 'freqtrade-plot.html')) -def generate_graph(pair, trades, data): +def generate_graph(pair, trades, data) -> None: + """ + Generate the graph from the data generated by Backtesting or from DB + :param pair: Pair to Display on the graph + :param trades: All trades created + :param data: Dataframe + :return: None + """ + + # Define the graph + fig = tools.make_subplots( + rows=3, + cols=1, + shared_xaxes=True, + row_width=[1, 1, 4], + vertical_spacing=0.0001, + ) + fig['layout'].update(title=pair) + + # Common information candles = go.Candlestick( x=data.date, open=data.open, @@ -167,51 +200,105 @@ def generate_graph(pair, trades, data): ) ) - bb_lower = go.Scatter( - x=data.date, - y=data.bb_lowerband, - name='BB lower', - line={'color': "transparent"}, - ) - bb_upper = go.Scatter( - x=data.date, - y=data.bb_upperband, - name='BB upper', - fill="tonexty", - fillcolor="rgba(0,176,246,0.2)", - line={'color': "transparent"}, - ) - macd = go.Scattergl(x=data['date'], y=data['macd'], name='MACD') - macdsignal = go.Scattergl(x=data['date'], y=data['macdsignal'], name='MACD signal') - volume = go.Bar(x=data['date'], y=data['volume'], name='Volume') - - fig = tools.make_subplots( - rows=3, - cols=1, - shared_xaxes=True, - row_width=[1, 1, 4], - vertical_spacing=0.0001, - ) - # Row 1 - fig.append_trace(candles, 1, 1) - fig.append_trace(bb_lower, 1, 1) - fig.append_trace(bb_upper, 1, 1) + + if 'bb_lowerband' in data and 'bb_upperband' in data: + bb_lower = go.Scatter( + x=data.date, + y=data.bb_lowerband, + name='BB lower', + line={'color': "transparent"}, + ) + bb_upper = go.Scatter( + x=data.date, + y=data.bb_upperband, + name='BB upper', + fill="tonexty", + fillcolor="rgba(0,176,246,0.2)", + line={'color': "transparent"}, + ) + fig.append_trace(bb_lower, 1, 1) + fig.append_trace(bb_upper, 1, 1) + + if TO_DISPLAY['sma'] and 'sma' in data: + sma = generate_scattergl(index='sma', name='SMA', data=data) + fig.append_trace(sma, 1, 1) + + if TO_DISPLAY['ema'] and 'ema10' in data and 'ema50' in data: + ema10 = generate_scattergl(index='ema10', name='EMA10', data=data) + ema50 = generate_scattergl(index='ema50', name='EMA50', data=data) + fig.append_trace(ema10, 1, 1) + fig.append_trace(ema50, 1, 1) + fig.append_trace(buys, 1, 1) fig.append_trace(sells, 1, 1) - fig.append_trace(volume, 2, 1) - fig.append_trace(macd, 3, 1) - fig.append_trace(macdsignal, 3, 1) fig.append_trace(trade_buys, 1, 1) fig.append_trace(trade_sells, 1, 1) - - fig['layout'].update(title=pair) fig['layout']['yaxis1'].update(title='Price') - fig['layout']['yaxis2'].update(title='Volume') - fig['layout']['yaxis3'].update(title='MACD') - plot(fig, filename=os.path.join('user_data', 'freqtrade-plot.html')) + # Row 2 + volume = go.Bar( + x=data['date'], + y=data['volume'], + name='Volume' + ) + fig.append_trace(volume, 2, 1) + fig['layout']['yaxis2'].update(title='Volume') + + # Row 3 (On Row 3, we can only display one indicator) + if TO_DISPLAY['macd'] and 'macd' in data and 'macdsignal' in data: + macd = generate_scattergl(index='macd', name='MACD', data=data) + macdsignal = generate_scattergl(index='macdsignal', name='MACD Signal', data=data) + fig.append_trace(macd, 3, 1) + fig.append_trace(macdsignal, 3, 1) + fig['layout']['yaxis3'].update(title='MACD') + + elif TO_DISPLAY['fast'] and 'fastd' in data and 'fastk' in data: + fastd = generate_scattergl(index='fastd', name='fastd', data=data) + fastk = generate_scattergl(index='fastk', name='fastk', data=data) + fig.append_trace(fastd, 3, 1) + fig.append_trace(fastk, 3, 1) + fig['layout']['yaxis3'].update(title='Stoch Fast') + + elif TO_DISPLAY['slow'] and 'slowd' in data and 'slowk' in data: + slowd = generate_scattergl(index='slowd', name='slowd', data=data) + slowk = generate_scattergl(index='slowk', name='slowk', data=data) + fig.append_trace(slowd, 3, 1) + fig.append_trace(slowk, 3, 1) + fig['layout']['yaxis3'].update(title='Stoch Slow') + + elif TO_DISPLAY['rsi'] and 'rsi' in data: + rsi = generate_scattergl(index='rsi', name='RSI', data=data) + fig.append_trace(rsi, 3, 1) + fig['layout']['yaxis3'].update(title='RSI') + + elif TO_DISPLAY['mfi'] and 'mfi' in data: + mfi = generate_scattergl(index='mfi', name='MFI', data=data) + fig.append_trace(mfi, 3, 1) + fig['layout']['yaxis3'].update(title='MFI') + + elif TO_DISPLAY['fisher_rsi'] and 'fisher_rsi' in data: + fisher_rsi = generate_scattergl(index='fisher_rsi', name='Fisher RSI', data=data) + fig.append_trace(fisher_rsi, 3, 1) + fig['layout']['yaxis3'].update(title='Fisher RSI') + + return fig + + +def generate_scattergl(index, name, data) -> go.Scattergl: + """ + Generate a Scattergl element + :param index: code of the Indicator to generate + :param name: Name that will be display in the graph legend + :param data: Dataframe + :return: Scattergl + """ + return go.Scattergl( + x=data['date'], + y=data[index], + name=name + ) def plot_parse_args(args: List[str]) -> Namespace: From e16fb45d84cc3d013669b00e8c2633f9c9bbfc36 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Mon, 4 Jun 2018 19:25:25 -0700 Subject: [PATCH 25/70] Fix typo, remove Bittrex mention --- scripts/plot_dataframe.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 247f530d2..99d17c0b6 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -76,11 +76,10 @@ def plot_analyzed_dataframe(args: Namespace) -> None: # Set the ticker to use tick_interval = analyze.get_ticker_interval() - # Load pqir tickers + # Load pair tickers tickers = {} if args.live: logger.info('Downloading pair.') - # Init Bittrex to use public API tickers[pair] = exchange.get_ticker_history(pair, tick_interval) else: tickers = optimize.load_data( From 662436acd291d4ca4bc385aa47c0f86844f5b74e Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Mon, 4 Jun 2018 20:40:36 -0700 Subject: [PATCH 26/70] Fix typo in Argument() --- freqtrade/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index c9cecb1f5..a95b6ae52 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -141,7 +141,7 @@ class Arguments(object): help='Save backtest results to this filename \ requires --export to be set as well\ Example --export-filename=user_data/backtest_data/backtest_today.json\ - (default: %(default)s', + (default: %(default)s)', type=str, default=os.path.join('user_data', 'backtest_data', 'backtest-result.json'), dest='exportfilename', From 8edcef6d3215a215fd7821060e02783882e822ba Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Mon, 4 Jun 2018 20:41:16 -0700 Subject: [PATCH 27/70] Add two params to select what indicators to display --- scripts/plot_dataframe.py | 139 +++++++++++++++++--------------------- 1 file changed, 63 insertions(+), 76 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 99d17c0b6..1a7b5b875 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -5,12 +5,20 @@ Script to display when the bot will buy a specific pair Mandatory Cli parameters: -p / --pair: pair to examine -Optional Cli parameters +Option but recommended -s / --strategy: strategy to use + + +Optional Cli parameters -d / --datadir: path to pair backtest data --timerange: specify what timerange of data to use. -l / --live: Live, to download the latest ticker for the pair -db / --db-url: Show trades stored in database + + +Indicators recommended +Row 1: sma, ema3, ema5, ema10, ema50 +Row 3: macd, rsi, fisher_rsi, mfi, slowd, slowk, fastd, fastk """ import logging import os @@ -33,20 +41,6 @@ logger = logging.getLogger(__name__) _CONF: Dict[str, Any] = {} -# Update the global variable TO_DISPLAY to select which indicator you want to display -TO_DISPLAY = { - 'sma': False, # On Row 1 - 'ema': True, # On Row 1 - - 'macd': True, # On Row 3 - 'rsi': False, # On Row 3 - 'fisher_rsi': False, # On Row 3 - 'mfi': False, # On Row 3 - 'slow': False, # On Row 3 - 'fast': False # On Row 3 -} - - def plot_analyzed_dataframe(args: Namespace) -> None: """ Calls analyze() and plots the returned dataframe @@ -59,6 +53,15 @@ def plot_analyzed_dataframe(args: Namespace) -> None: # Set the pair to audit pair = args.pair + if pair is None: + logger.critical('Parameter --pair mandatory;. E.g --pair ETH/BTC') + exit() + + if '/' not in pair: + logger.critical('--pair format must be XXX/YYY') + exit() + + # Set timerange to use timerange = Arguments.parse_timerange(args.timerange) @@ -112,13 +115,14 @@ def plot_analyzed_dataframe(args: Namespace) -> None: fig = generate_graph( pair=pair, trades=trades, - data=dataframe.tail(750) + data=dataframe.tail(750), + args=args ) plot(fig, filename=os.path.join('user_data', 'freqtrade-plot.html')) -def generate_graph(pair, trades, data) -> None: +def generate_graph(pair, trades, data, args) -> tools.make_subplots: """ Generate the graph from the data generated by Backtesting or from DB :param pair: Pair to Display on the graph @@ -136,6 +140,9 @@ def generate_graph(pair, trades, data) -> None: vertical_spacing=0.0001, ) fig['layout'].update(title=pair) + fig['layout']['yaxis1'].update(title='Price') + fig['layout']['yaxis2'].update(title='Volume') + fig['layout']['yaxis3'].update(title='Other') # Common information candles = go.Candlestick( @@ -220,21 +227,11 @@ def generate_graph(pair, trades, data) -> None: fig.append_trace(bb_lower, 1, 1) fig.append_trace(bb_upper, 1, 1) - if TO_DISPLAY['sma'] and 'sma' in data: - sma = generate_scattergl(index='sma', name='SMA', data=data) - fig.append_trace(sma, 1, 1) - - if TO_DISPLAY['ema'] and 'ema10' in data and 'ema50' in data: - ema10 = generate_scattergl(index='ema10', name='EMA10', data=data) - ema50 = generate_scattergl(index='ema50', name='EMA50', data=data) - fig.append_trace(ema10, 1, 1) - fig.append_trace(ema50, 1, 1) - + fig = generate_row(fig=fig, row=1, raw_indicators=args.indicators1, data=data) fig.append_trace(buys, 1, 1) fig.append_trace(sells, 1, 1) fig.append_trace(trade_buys, 1, 1) fig.append_trace(trade_sells, 1, 1) - fig['layout']['yaxis1'].update(title='Price') # Row 2 volume = go.Bar( @@ -243,61 +240,33 @@ def generate_graph(pair, trades, data) -> None: name='Volume' ) fig.append_trace(volume, 2, 1) - fig['layout']['yaxis2'].update(title='Volume') - # Row 3 (On Row 3, we can only display one indicator) - if TO_DISPLAY['macd'] and 'macd' in data and 'macdsignal' in data: - macd = generate_scattergl(index='macd', name='MACD', data=data) - macdsignal = generate_scattergl(index='macdsignal', name='MACD Signal', data=data) - fig.append_trace(macd, 3, 1) - fig.append_trace(macdsignal, 3, 1) - fig['layout']['yaxis3'].update(title='MACD') - - elif TO_DISPLAY['fast'] and 'fastd' in data and 'fastk' in data: - fastd = generate_scattergl(index='fastd', name='fastd', data=data) - fastk = generate_scattergl(index='fastk', name='fastk', data=data) - fig.append_trace(fastd, 3, 1) - fig.append_trace(fastk, 3, 1) - fig['layout']['yaxis3'].update(title='Stoch Fast') - - elif TO_DISPLAY['slow'] and 'slowd' in data and 'slowk' in data: - slowd = generate_scattergl(index='slowd', name='slowd', data=data) - slowk = generate_scattergl(index='slowk', name='slowk', data=data) - fig.append_trace(slowd, 3, 1) - fig.append_trace(slowk, 3, 1) - fig['layout']['yaxis3'].update(title='Stoch Slow') - - elif TO_DISPLAY['rsi'] and 'rsi' in data: - rsi = generate_scattergl(index='rsi', name='RSI', data=data) - fig.append_trace(rsi, 3, 1) - fig['layout']['yaxis3'].update(title='RSI') - - elif TO_DISPLAY['mfi'] and 'mfi' in data: - mfi = generate_scattergl(index='mfi', name='MFI', data=data) - fig.append_trace(mfi, 3, 1) - fig['layout']['yaxis3'].update(title='MFI') - - elif TO_DISPLAY['fisher_rsi'] and 'fisher_rsi' in data: - fisher_rsi = generate_scattergl(index='fisher_rsi', name='Fisher RSI', data=data) - fig.append_trace(fisher_rsi, 3, 1) - fig['layout']['yaxis3'].update(title='Fisher RSI') + # Row 3 + fig = generate_row(fig=fig, row=3, raw_indicators=args.indicators2, data=data) return fig -def generate_scattergl(index, name, data) -> go.Scattergl: +def generate_row(fig, row, raw_indicators, data) -> tools.make_subplots: """ - Generate a Scattergl element - :param index: code of the Indicator to generate - :param name: Name that will be display in the graph legend - :param data: Dataframe - :return: Scattergl + Generator all the indicator selected by the user for a specific row """ - return go.Scattergl( - x=data['date'], - y=data[index], - name=name - ) + for indicator in raw_indicators.split(','): + if indicator in data: + scattergl = go.Scattergl( + x=data['date'], + y=data[indicator], + name=indicator + ) + fig.append_trace(scattergl, row, 1) + else: + logger.info( + 'Indicator "%s" ignored. Reason: This indicator is not found ' + 'in your strategy.', + indicator + ) + + return fig def plot_parse_args(args: List[str]) -> Namespace: @@ -308,6 +277,24 @@ def plot_parse_args(args: List[str]) -> Namespace: """ arguments = Arguments(args, 'Graph dataframe') arguments.scripts_options() + arguments.parser.add_argument( + '--indicators1', + help='Set indicators from your strategy you want in the first row of the graph. Separate ' + 'them with a coma. E.g: ema3,ema5 (default: %(default)s)', + type=str, + default='sma,ema3,ema5', + dest='indicators1', + ) + + arguments.parser.add_argument( + '--indicators2', + help='Set indicators from your strategy you want in the third row of the graph. Separate ' + 'them with a coma. E.g: fastd,fastk (default: %(default)s)', + type=str, + default='macd', + dest='indicators2', + ) + arguments.common_args_parser() arguments.optimizer_shared_options(arguments.parser) arguments.backtesting_options(arguments.parser) From 1b071b1f4a812203507fd38260949c6361779048 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Mon, 4 Jun 2018 20:42:34 -0700 Subject: [PATCH 28/70] Add example on how to start the script --- scripts/plot_dataframe.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 1a7b5b875..c3fb1af18 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -19,6 +19,10 @@ Optional Cli parameters Indicators recommended Row 1: sma, ema3, ema5, ema10, ema50 Row 3: macd, rsi, fisher_rsi, mfi, slowd, slowk, fastd, fastk + +Example of usage: +> python3 scripts/plot_dataframe.py --pair BTC/EUR -d user_data/data/ --indicators1 sma,ema3 +--indicators2 fastk,fastd """ import logging import os From 3778bcda24343e1c6d7d23e60b060788478632d9 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Mon, 4 Jun 2018 20:50:44 -0700 Subject: [PATCH 29/70] Ok! you won Flake8 --- freqtrade/arguments.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index a95b6ae52..bb59627de 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -173,7 +173,7 @@ class Arguments(object): @staticmethod def hyperopt_options(parser: argparse.ArgumentParser) -> None: """ - Parses given arguments for Hyperopt scripts. + Parses given arguments foér Hyperopt scripts. """ parser.add_argument( '-e', '--epochs', @@ -222,7 +222,7 @@ class Arguments(object): @staticmethod def parse_timerange(text: Optional[str]) -> \ - Optional[Tuple[Tuple, Optional[int], Optional[int]]]: + Optional[Tuple[Tuple, Optional[int], Optional[int]]]: """ Parse the value of the argument --timerange to determine what is the range desired :param text: value from --timerange From 947462e1342b50992072157d2bd293d3b24557e8 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Mon, 4 Jun 2018 21:29:53 -0700 Subject: [PATCH 30/70] Add back 'import os' in Arguments() --- freqtrade/arguments.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index bb59627de..ee57dc1be 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -2,6 +2,7 @@ This module contains the argument manager class """ +import os import argparse import logging import re From c29c13dfd72e72832821f3b12aa1eb67432903b6 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Mon, 4 Jun 2018 22:42:24 -0700 Subject: [PATCH 31/70] Fix a typo in Arguments() comment --- freqtrade/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index ee57dc1be..d79a52af2 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -174,7 +174,7 @@ class Arguments(object): @staticmethod def hyperopt_options(parser: argparse.ArgumentParser) -> None: """ - Parses given arguments foér Hyperopt scripts. + Parses given arguments for Hyperopt scripts. """ parser.add_argument( '-e', '--epochs', From 5024cd52afd6d27d8192d5feffd367d99dedb3bf Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Mon, 4 Jun 2018 23:49:16 -0700 Subject: [PATCH 32/70] Update docstring for generate_graph() --- scripts/plot_dataframe.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index c3fb1af18..e7737a5c7 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -132,6 +132,7 @@ def generate_graph(pair, trades, data, args) -> tools.make_subplots: :param pair: Pair to Display on the graph :param trades: All trades created :param data: Dataframe + :param args: sys.argv that contrains the two params indicators1, and indicators2 :return: None """ From 7cc36eee0f88f9adbee06dee7d21553a8bb2fdad Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Tue, 5 Jun 2018 13:27:24 +0300 Subject: [PATCH 33/70] Docs: point links to freqtrade org --- setup.py | 2 +- user_data/strategies/test_strategy.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 856a47181..ee6b7ae38 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ from freqtrade import __version__ setup(name='freqtrade', version=__version__, description='Simple High Frequency Trading Bot for crypto currencies', - url='https://github.com/gcarq/freqtrade', + url='https://github.com/freqtrade/freqtrade', author='gcarq and contributors', author_email='michael.egger@tsn.at', license='GPLv3', diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index af28388be..34f496e38 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -14,7 +14,7 @@ import numpy # noqa class TestStrategy(IStrategy): """ This is a test strategy to inspire you. - More information in https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md + More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md You can: - Rename the class name (Do not forget to update class_name) From 456d0a050f422bfe1d94b1c2a7d170a839bf2978 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Tue, 5 Jun 2018 13:49:59 +0200 Subject: [PATCH 34/70] update doc for process_throttle_secs --- docs/configuration.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index a2df3f2fe..b9e633c57 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -73,6 +73,12 @@ value. This parameter is optional. If you use it, it will take over the Possible values are `running` or `stopped`. (default=`running`) If the value is `stopped` the bot has to be started with `/start` first. +### Understand process_throttle_secs +`process_throttle_secs` is an optional field in seconds that defines in seconds how long the bot should wait +before asking the strategy if we should buy or a sell an asset. After each wait period, the strategy is asked for +every opened trade wether or not we should buy, and for all the remaining pairs (either the dynamic list of pairs or +the static list of pairs) if we should buy. + ### Understand ask_last_balance `ask_last_balance` sets the bidding price. Value `0.0` will use `ask` price, `1.0` will use the `last` price and values between those interpolate between ask and last @@ -146,7 +152,6 @@ you run it in production mode. If you have not your Bittrex API key yet, [see our tutorial](https://github.com/gcarq/freqtrade/blob/develop/docs/pre-requisite.md). - ## Next step Now you have configured your config.json, the next step is to [start your bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md). From 608fc170d937541f59c40bbc8b0456d6e8daad25 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Tue, 5 Jun 2018 13:51:30 +0200 Subject: [PATCH 35/70] fix doc --- docs/configuration.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index b9e633c57..8acf2986c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -74,9 +74,9 @@ Possible values are `running` or `stopped`. (default=`running`) If the value is `stopped` the bot has to be started with `/start` first. ### Understand process_throttle_secs -`process_throttle_secs` is an optional field in seconds that defines in seconds how long the bot should wait -before asking the strategy if we should buy or a sell an asset. After each wait period, the strategy is asked for -every opened trade wether or not we should buy, and for all the remaining pairs (either the dynamic list of pairs or +`process_throttle_secs` is an optional field that defines in seconds how long the bot should wait +before asking the strategy if we should buy or a sell an asset. After each wait period, the strategy is asked again for +every opened trade wether or not we should sell, and for all the remaining pairs (either the dynamic list of pairs or the static list of pairs) if we should buy. ### Understand ask_last_balance From 7a34578b4da58c0f80523f0a9db10836e67a532f Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 5 Jun 2018 23:34:26 +0200 Subject: [PATCH 36/70] refactor timerange to named tuple --- freqtrade/arguments.py | 21 ++++++++----- freqtrade/optimize/__init__.py | 51 ++++++++++++++++--------------- freqtrade/optimize/backtesting.py | 2 +- freqtrade/optimize/hyperopt.py | 2 +- 4 files changed, 42 insertions(+), 34 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index d79a52af2..bf5abb8ee 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -7,11 +7,19 @@ import argparse import logging import re import arrow -from typing import List, Tuple, Optional +from typing import List, Optional, NamedTuple from freqtrade import __version__, constants +class TimeRange(NamedTuple): + + starttype: Optional[str] = None + stoptype: Optional[str] = None + startts: int = 0 + stopts: int = 0 + + class Arguments(object): """ Arguments Class. Manage the arguments received by the cli @@ -222,15 +230,14 @@ class Arguments(object): self.hyperopt_options(hyperopt_cmd) @staticmethod - def parse_timerange(text: Optional[str]) -> \ - Optional[Tuple[Tuple, Optional[int], Optional[int]]]: + def parse_timerange(text: Optional[str]) -> TimeRange: """ Parse the value of the argument --timerange to determine what is the range desired :param text: value from --timerange :return: Start and End range period """ if text is None: - return None + return TimeRange() syntax = [(r'^-(\d{8})$', (None, 'date')), (r'^(\d{8})-$', ('date', None)), (r'^(\d{8})-(\d{8})$', ('date', 'date')), @@ -246,8 +253,8 @@ class Arguments(object): if match: # Regex has matched rvals = match.groups() index = 0 - start: Optional[int] = None - stop: Optional[int] = None + start: int = 0 + stop: int = 0 if stype[0]: starts = rvals[index] if stype[0] == 'date': @@ -263,7 +270,7 @@ class Arguments(object): else arrow.get(stops, 'YYYYMMDD').timestamp else: stop = int(stops) - return stype, start, stop + return TimeRange(stype[0], stype[1], start, stop) raise Exception('Incorrect syntax for timerange "%s"' % text) def scripts_options(self) -> None: diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 711adfd28..dc48a64ec 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -9,39 +9,40 @@ import arrow from freqtrade import misc, constants from freqtrade.exchange import get_ticker_history +from freqtrade.arguments import TimeRange from user_data.hyperopt_conf import hyperopt_optimize_conf logger = logging.getLogger(__name__) -def trim_tickerlist(tickerlist: List[Dict], timerange: Tuple[Tuple, int, int]) -> List[Dict]: +def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: if not tickerlist: return tickerlist - stype, start, stop = timerange - start_index = 0 stop_index = len(tickerlist) - if stype[0] == 'line': - stop_index = start - if stype[0] == 'index': - start_index = start - elif stype[0] == 'date': - while start_index < len(tickerlist) and tickerlist[start_index][0] < start * 1000: + if timerange.starttype == 'line': + stop_index = timerange.startts + if timerange.starttype == 'index': + start_index = timerange.startts + elif timerange.starttype == 'date': + while (start_index < len(tickerlist) and + tickerlist[start_index][0] < timerange.startts * 1000): start_index += 1 - if stype[1] == 'line': - start_index = len(tickerlist) + stop - if stype[1] == 'index': - stop_index = stop - elif stype[1] == 'date': - while stop_index > 0 and tickerlist[stop_index-1][0] > stop * 1000: + if timerange.stoptype == 'line': + start_index = len(tickerlist) + timerange.stopts + if timerange.stoptype == 'index': + stop_index = timerange.stopts + elif timerange.stoptype == 'date': + while (stop_index > 0 and + tickerlist[stop_index-1][0] > timerange.stopts * 1000): stop_index -= 1 if start_index > stop_index: - raise ValueError(f'The timerange [{start},{stop}] is incorrect') + raise ValueError(f'The timerange [{timerange.startts},{timerange.stopts}] is incorrect') return tickerlist[start_index:stop_index] @@ -49,7 +50,7 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: Tuple[Tuple, int, int]) - def load_tickerdata_file( datadir: str, pair: str, ticker_interval: str, - timerange: Optional[Tuple[Tuple, int, int]] = None) -> Optional[List[Dict]]: + timerange: TimeRange) -> Optional[List[Dict]]: """ Load a pair from file, :return dict OR empty if unsuccesful @@ -84,7 +85,7 @@ def load_data(datadir: str, ticker_interval: str, pairs: Optional[List[str]] = None, refresh_pairs: Optional[bool] = False, - timerange: Optional[Tuple[Tuple, int, int]] = None) -> Dict[str, List]: + timerange: TimeRange = TimeRange()) -> Dict[str, List]: """ Loads ticker history data for the given parameters :return: dict @@ -124,7 +125,7 @@ def make_testdata_path(datadir: str) -> str: def download_pairs(datadir, pairs: List[str], ticker_interval: str, - timerange: Optional[Tuple[Tuple, int, int]] = None) -> bool: + timerange: TimeRange = TimeRange()) -> bool: """For each pairs passed in parameters, download the ticker intervals""" for pair in pairs: try: @@ -144,7 +145,7 @@ def download_pairs(datadir, pairs: List[str], def load_cached_data_for_updating(filename: str, tick_interval: str, - timerange: Optional[Tuple[Tuple, int, int]]) -> Tuple[ + timerange: Optional[TimeRange]) -> Tuple[ List[Any], Optional[int]]: """ @@ -155,10 +156,10 @@ def load_cached_data_for_updating(filename: str, # user sets timerange, so find the start time if timerange: - if timerange[0][0] == 'date': - since_ms = timerange[1] * 1000 - elif timerange[0][1] == 'line': - num_minutes = timerange[2] * constants.TICKER_INTERVAL_MINUTES[tick_interval] + if timerange.starttype == 'date': + since_ms = timerange.startts * 1000 + elif timerange.stoptype == 'line': + num_minutes = timerange.stopts * constants.TICKER_INTERVAL_MINUTES[tick_interval] since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000 # read the cached file @@ -188,7 +189,7 @@ def load_cached_data_for_updating(filename: str, def download_backtesting_testdata(datadir: str, pair: str, tick_interval: str = '5m', - timerange: Optional[Tuple[Tuple, int, int]] = None) -> None: + timerange: Optional[TimeRange] = None) -> None: """ Download the latest ticker intervals from the exchange for the pairs passed in parameters diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d7ed45955..3dd643561 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -221,7 +221,7 @@ class Backtesting(object): timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - data = optimize.load_data( # type: ignore # timerange will be refactored + data = optimize.load_data( self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 74b39b445..878acc2dc 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -497,7 +497,7 @@ class Hyperopt(Backtesting): def start(self) -> None: timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - data = load_data( # type: ignore # timerange will be refactored + data = load_data( datadir=str(self.config.get('datadir')), pairs=self.config['exchange']['pair_whitelist'], ticker_interval=self.ticker_interval, From 270ccbb0dac9d4b996831721d9d5f4249e2ae173 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 5 Jun 2018 23:41:50 +0200 Subject: [PATCH 37/70] fix args test --- freqtrade/tests/test_arguments.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 474aa2507..9c24a6789 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -9,7 +9,7 @@ import logging import pytest -from freqtrade.arguments import Arguments +from freqtrade.arguments import Arguments, TimeRange def test_arguments_object() -> None: @@ -107,20 +107,25 @@ def test_parse_args_dynamic_whitelist_invalid_values() -> None: def test_parse_timerange_incorrect() -> None: - assert ((None, 'line'), None, -200) == Arguments.parse_timerange('-200') - assert (('line', None), 200, None) == Arguments.parse_timerange('200-') - assert (('index', 'index'), 200, 500) == Arguments.parse_timerange('200-500') + assert TimeRange(None, 'line', 0, -200) == Arguments.parse_timerange('-200') + assert TimeRange('line', None, 200, 0) == Arguments.parse_timerange('200-') + assert TimeRange('index', 'index', 200, 500) == Arguments.parse_timerange('200-500') - assert (('date', None), 1274486400, None) == Arguments.parse_timerange('20100522-') - assert ((None, 'date'), None, 1274486400) == Arguments.parse_timerange('-20100522') + assert TimeRange('date', None, 1274486400, 0) == Arguments.parse_timerange('20100522-') + assert TimeRange(None, 'date', 0, 1274486400) == Arguments.parse_timerange('-20100522') timerange = Arguments.parse_timerange('20100522-20150730') - assert timerange == (('date', 'date'), 1274486400, 1438214400) + assert timerange == TimeRange('date', 'date', 1274486400, 1438214400) # Added test for unix timestamp - BTC genesis date - assert (('date', None), 1231006505, None) == Arguments.parse_timerange('1231006505-') - assert ((None, 'date'), None, 1233360000) == Arguments.parse_timerange('-1233360000') + assert TimeRange('date', None, 1231006505, 0) == Arguments.parse_timerange('1231006505-') + assert TimeRange(None, 'date', 0, 1233360000) == Arguments.parse_timerange('-1233360000') timerange = Arguments.parse_timerange('1231006505-1233360000') - assert timerange == (('date', 'date'), 1231006505, 1233360000) + assert TimeRange('date', 'date', 1231006505, 1233360000) == timerange + + # TODO: Find solution for the following case (passing timestamp in ms) + timerange = Arguments.parse_timerange('1231006505000-1233360000000') + assert TimeRange('date', 'date', 1231006505, 1233360000) != timerange + with pytest.raises(Exception, match=r'Incorrect syntax.*'): Arguments.parse_timerange('-') From f37c5b70ba3ea81f09ba6f24f61c7047f6b9542e Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 5 Jun 2018 23:53:49 +0200 Subject: [PATCH 38/70] Fix tests - read optional argument --- freqtrade/optimize/__init__.py | 2 +- freqtrade/tests/optimize/test_backtesting.py | 6 ++-- freqtrade/tests/optimize/test_optimize.py | 33 ++++++++++---------- freqtrade/tests/test_analyze.py | 3 +- freqtrade/tests/test_arguments.py | 1 - 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index dc48a64ec..00f05cc46 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -50,7 +50,7 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: def load_tickerdata_file( datadir: str, pair: str, ticker_interval: str, - timerange: TimeRange) -> Optional[List[Dict]]: + timerange: Optional[TimeRange] = None) -> Optional[List[Dict]]: """ Load a pair from file, :return dict OR empty if unsuccesful diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 65820ac09..efcee3839 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -13,7 +13,7 @@ from arrow import Arrow from freqtrade import optimize from freqtrade.analyze import Analyze -from freqtrade.arguments import Arguments +from freqtrade.arguments import Arguments, TimeRange from freqtrade.optimize.backtesting import Backtesting, start, setup_configuration from freqtrade.tests.conftest import log_has @@ -30,7 +30,7 @@ def trim_dictlist(dict_list, num): def load_data_test(what): - timerange = ((None, 'line'), None, -100) + timerange = TimeRange(None, 'line', 0, -100) data = optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC'], timerange=timerange) pair = data['UNITTEST/BTC'] @@ -311,7 +311,7 @@ def test_tickerdata_to_dataframe(default_conf, mocker) -> None: Test Backtesting.tickerdata_to_dataframe() method """ mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) - timerange = ((None, 'line'), None, -100) + timerange = TimeRange(None, 'line', 0, -100) tick = optimize.load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) tickerlist = {'UNITTEST/BTC': tick} diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 349fa3be3..3f358cfb8 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -11,6 +11,7 @@ from freqtrade.misc import file_dump_json from freqtrade.optimize.__init__ import make_testdata_path, download_pairs, \ download_backtesting_testdata, load_tickerdata_file, trim_tickerlist, \ load_cached_data_for_updating +from freqtrade.arguments import TimeRange from freqtrade.tests.conftest import log_has # Change this if modifying UNITTEST/BTC testdatafile @@ -176,7 +177,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # timeframe starts earlier than the cached data # should fully update data - timerange = (('date', None), test_data[0][0] / 1000 - 1, None) + timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0) data, start_ts = load_cached_data_for_updating(test_filename, '1m', timerange) @@ -187,13 +188,13 @@ def test_load_cached_data_for_updating(mocker) -> None: num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 120 data, start_ts = load_cached_data_for_updating(test_filename, '1m', - ((None, 'line'), None, -num_lines)) + TimeRange(None, 'line', 0, -num_lines)) assert data == [] assert start_ts < test_data[0][0] - 1 # timeframe starts in the center of the cached data # should return the chached data w/o the last item - timerange = (('date', None), test_data[0][0] / 1000 + 1, None) + timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0) data, start_ts = load_cached_data_for_updating(test_filename, '1m', timerange) @@ -202,7 +203,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # same with 'line' timeframe num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 30 - timerange = ((None, 'line'), None, -num_lines) + timerange = TimeRange(None, 'line', 0, -num_lines) data, start_ts = load_cached_data_for_updating(test_filename, '1m', timerange) @@ -211,7 +212,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # timeframe starts after the chached data # should return the chached data w/o the last item - timerange = (('date', None), test_data[-1][0] / 1000 + 1, None) + timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 1, 0) data, start_ts = load_cached_data_for_updating(test_filename, '1m', timerange) @@ -220,7 +221,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # same with 'line' timeframe num_lines = 30 - timerange = ((None, 'line'), None, -num_lines) + timerange = TimeRange(None, 'line', 0, -num_lines) data, start_ts = load_cached_data_for_updating(test_filename, '1m', timerange) @@ -230,7 +231,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # no timeframe is set # should return the chached data w/o the last item num_lines = 30 - timerange = ((None, 'line'), None, -num_lines) + timerange = TimeRange(None, 'line', 0, -num_lines) data, start_ts = load_cached_data_for_updating(test_filename, '1m', timerange) @@ -239,7 +240,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # no datafile exist # should return timestamp start time - timerange = (('date', None), now_ts - 10000, None) + timerange = TimeRange('date', None, now_ts - 10000, 0) data, start_ts = load_cached_data_for_updating(test_filename + 'unexist', '1m', timerange) @@ -248,7 +249,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # same with 'line' timeframe num_lines = 30 - timerange = ((None, 'line'), None, -num_lines) + timerange = TimeRange(None, 'line', 0, -num_lines) data, start_ts = load_cached_data_for_updating(test_filename + 'unexist', '1m', timerange) @@ -343,7 +344,7 @@ def test_trim_tickerlist() -> None: # Test the pattern ^(-\d+)$ # This pattern uses the latest N elements - timerange = ((None, 'line'), None, -5) + timerange = TimeRange(None, 'line', 0, -5) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) @@ -353,7 +354,7 @@ def test_trim_tickerlist() -> None: # Test the pattern ^(\d+)-$ # This pattern keep X element from the end - timerange = (('line', None), 5, None) + timerange = TimeRange('line', None, 5, 0) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) @@ -363,7 +364,7 @@ def test_trim_tickerlist() -> None: # Test the pattern ^(\d+)-(\d+)$ # This pattern extract a window - timerange = (('index', 'index'), 5, 10) + timerange = TimeRange('index', 'index', 5, 10) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) @@ -374,7 +375,7 @@ def test_trim_tickerlist() -> None: # Test the pattern ^(\d{8})-(\d{8})$ # This pattern extract a window between the dates - timerange = (('date', 'date'), ticker_list[5][0] / 1000, ticker_list[10][0] / 1000 - 1) + timerange = TimeRange('date', 'date', ticker_list[5][0] / 1000, ticker_list[10][0] / 1000 - 1) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) @@ -385,7 +386,7 @@ def test_trim_tickerlist() -> None: # Test the pattern ^-(\d{8})$ # This pattern extracts elements from the start to the date - timerange = ((None, 'date'), None, ticker_list[10][0] / 1000 - 1) + timerange = TimeRange(None, 'date', 0, ticker_list[10][0] / 1000 - 1) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) @@ -395,7 +396,7 @@ def test_trim_tickerlist() -> None: # Test the pattern ^(\d{8})-$ # This pattern extracts elements from the date to now - timerange = (('date', None), ticker_list[10][0] / 1000 - 1, None) + timerange = TimeRange('date', None, ticker_list[10][0] / 1000 - 1, None) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) @@ -405,7 +406,7 @@ def test_trim_tickerlist() -> None: # Test a wrong pattern # This pattern must return the list unchanged - timerange = ((None, None), None, 5) + timerange = TimeRange(None, None, None, 5) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 01033ce7d..418f31851 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -13,6 +13,7 @@ from pandas import DataFrame from freqtrade.analyze import Analyze, SignalType from freqtrade.optimize.__init__ import load_tickerdata_file +from freqtrade.arguments import TimeRange from freqtrade.tests.conftest import log_has # Avoid to reinit the same object again and again @@ -183,7 +184,7 @@ def test_tickerdata_to_dataframe(default_conf) -> None: """ analyze = Analyze(default_conf) - timerange = ((None, 'line'), None, -100) + timerange = TimeRange(None, 'line', 0, -100) tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) tickerlist = {'UNITTEST/BTC': tick} data = analyze.tickerdata_to_dataframe(tickerlist) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 9c24a6789..6c3ecb913 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -126,7 +126,6 @@ def test_parse_timerange_incorrect() -> None: timerange = Arguments.parse_timerange('1231006505000-1233360000000') assert TimeRange('date', 'date', 1231006505, 1233360000) != timerange - with pytest.raises(Exception, match=r'Incorrect syntax.*'): Arguments.parse_timerange('-') From cac6e0d7156de739f2d46180334d334d7f3f5403 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 6 Jun 2018 00:10:18 +0200 Subject: [PATCH 39/70] Add docstring to TimeRange class --- freqtrade/arguments.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index bf5abb8ee..8f36af150 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -13,7 +13,11 @@ from freqtrade import __version__, constants class TimeRange(NamedTuple): - + """ + NamedTuple Defining timerange inputs. + [start/stop]type defines if [start/stop]ts shall be used. + if *type is none, don't use corresponding startvalue. + """ starttype: Optional[str] = None stoptype: Optional[str] = None startts: int = 0 From e6900036215587ba4f00bc1dfffcf859553d131d Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 6 Jun 2018 20:18:16 +0200 Subject: [PATCH 40/70] reinstate caching for get_ticker --- freqtrade/exchange/__init__.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 3c67db71f..89347b0dc 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -18,6 +18,8 @@ _API: ccxt.Exchange = None _CONF: Dict = {} API_RETRY_COUNT = 4 +_CACHED_TICKER: Dict[str, Any] = {} + # Holds all open sell orders for dry_run _DRY_RUN_OPEN_ORDERS: Dict[str, Any] = {} @@ -264,17 +266,28 @@ def get_tickers() -> Dict: raise OperationalException(e) -# TODO: remove refresh argument, keeping it to keep track of where it was intended to be used @retrier def get_ticker(pair: str, refresh: Optional[bool] = True) -> dict: - try: - return _API.fetch_ticker(pair) - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - 'Could not load ticker history due to {}. Message: {}'.format( - e.__class__.__name__, e)) - except ccxt.BaseError as e: - raise OperationalException(e) + global _CACHED_TICKER + if refresh or pair not in _CACHED_TICKER.keys(): + try: + data = _API.fetch_ticker(pair) + try: + _CACHED_TICKER[pair] = { + 'bid': float(data['bid']), + } + except KeyError as e: + logger.debug("Could not cache ticker data for %s", pair) + return data + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + 'Could not load ticker history due to {}. Message: {}'.format( + e.__class__.__name__, e)) + except ccxt.BaseError as e: + raise OperationalException(e) + else: + logger.info("returning cached data for %s", pair) + return _CACHED_TICKER[pair] @retrier From a901f21bcd3f70dc014f0d33d2c9d43a74bfa45a Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 6 Jun 2018 20:24:47 +0200 Subject: [PATCH 41/70] test ticker caching --- freqtrade/exchange/__init__.py | 1 + freqtrade/tests/exchange/test_exchange.py | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 89347b0dc..98cb8db0b 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -275,6 +275,7 @@ def get_ticker(pair: str, refresh: Optional[bool] = True) -> dict: try: _CACHED_TICKER[pair] = { 'bid': float(data['bid']), + 'ask': float(data['ask']), } except KeyError as e: logger.debug("Could not cache ticker data for %s", pair) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 56812c75e..ff4233eaf 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -310,9 +310,19 @@ def test_get_ticker(default_conf, mocker): # if not fetching a new result we should get the cached ticker ticker = get_ticker(pair='ETH/BTC') + assert api_mock.fetch_ticker.call_count == 1 assert ticker['bid'] == 0.5 assert ticker['ask'] == 1 + assert 'ETH/BTC' in exchange._CACHED_TICKER + assert exchange._CACHED_TICKER['ETH/BTC']['bid'] == 0.5 + assert exchange._CACHED_TICKER['ETH/BTC']['ask'] == 1 + + # Test caching + api_mock.fetch_ticker = MagicMock() + get_ticker(pair='ETH/BTC', refresh=False) + assert api_mock.fetch_ticker.call_count == 0 + with pytest.raises(TemporaryError): # test retrier api_mock.fetch_ticker = MagicMock(side_effect=ccxt.NetworkError) mocker.patch('freqtrade.exchange._API', api_mock) From 4a17671f45fb00a1be52dca832b4f031c95369f2 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 6 Jun 2018 20:30:42 +0200 Subject: [PATCH 42/70] improve log message --- freqtrade/exchange/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 98cb8db0b..5e768518b 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -287,7 +287,7 @@ def get_ticker(pair: str, refresh: Optional[bool] = True) -> dict: except ccxt.BaseError as e: raise OperationalException(e) else: - logger.info("returning cached data for %s", pair) + logger.info("returning cached ticker-data for %s", pair) return _CACHED_TICKER[pair] From 771449053042ae43a6f950f3b2c53e4441193746 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 6 Jun 2018 21:24:57 +0200 Subject: [PATCH 43/70] Test keyerror exception --- freqtrade/tests/exchange/test_exchange.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index ff4233eaf..97a723929 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -333,6 +333,10 @@ def test_get_ticker(default_conf, mocker): mocker.patch('freqtrade.exchange._API', api_mock) get_ticker(pair='ETH/BTC', refresh=True) + api_mock.fetch_ticker = MagicMock(return_value={}) + mocker.patch('freqtrade.exchange._API', api_mock) + get_ticker(pair='ETH/BTC', refresh=True) + def make_fetch_ohlcv_mock(data): def fetch_ohlcv_mock(pair, timeframe, since): From 2ba5e2053acbff036b18143c9f8c2fcf5a3fb22d Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 7 Jun 2018 00:55:09 +0200 Subject: [PATCH 44/70] create pyup.io config file --- .pyup.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .pyup.yml diff --git a/.pyup.yml b/.pyup.yml new file mode 100644 index 000000000..a0833af39 --- /dev/null +++ b/.pyup.yml @@ -0,0 +1,4 @@ +# autogenerated pyup.io config file +# see https://pyup.io/docs/configuration/ for all available options + +schedule: every day From 02671a7e103a7b55c311e5431f0d448d0f444b75 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 7 Jun 2018 01:12:46 +0200 Subject: [PATCH 45/70] pin networkx with pyup ignore filter --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6e7550515..43043775e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ pytest-mock==1.10.0 pytest-cov==2.5.1 hyperopt==0.1 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 -networkx==1.11 +networkx==1.11 # pyup: ignore tabulate==0.8.2 coinmarketcap==5.0.3 From 8583e89550f886126163b57340182e6742a226b8 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 7 Jun 2018 05:25:53 +0200 Subject: [PATCH 46/70] persistence: simplify init and pass db_url via config dict --- freqtrade/persistence.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index f9a7d1e3c..63c29dc4f 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -10,13 +10,11 @@ from typing import Dict, Optional, Any import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, create_engine) -from sqlalchemy.engine import Engine +from sqlalchemy import inspect from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker from sqlalchemy.pool import StaticPool -from sqlalchemy import inspect - logger = logging.getLogger(__name__) @@ -24,30 +22,30 @@ _CONF = {} _DECL_BASE: Any = declarative_base() -def init(config: dict, engine: Optional[Engine] = None) -> None: +def init(config: Dict) -> None: """ Initializes this module with the given config, registers all known command handlers and starts polling for message updates :param config: config to use - :param engine: database engine for sqlalchemy (Optional) :return: None """ _CONF.update(config) - if not engine: - if _CONF.get('dry_run', False): - # the user wants dry run to use a DB - if _CONF.get('dry_run_db', False): - engine = create_engine('sqlite:///tradesv3.dry_run.sqlite') - # Otherwise dry run will store in memory - else: - engine = create_engine('sqlite://', - connect_args={'check_same_thread': False}, - poolclass=StaticPool, - echo=False) - else: - engine = create_engine('sqlite:///tradesv3.sqlite') + db_url = _CONF.get('db_url', None) + kwargs = {} + + if not db_url and _CONF.get('dry_run', False): + # Default to in-memory db if not specified + # and take care of thread ownership if in-memory db + db_url = 'sqlite://' + kwargs.update({ + 'connect_args': {'check_same_thread': False}, + 'poolclass': StaticPool, + 'echo': False, + }) + + engine = create_engine(db_url, **kwargs) session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) Trade.session = session() Trade.query = session.query_property() @@ -55,7 +53,7 @@ def init(config: dict, engine: Optional[Engine] = None) -> None: check_migrate(engine) # Clean dry_run DB - if _CONF.get('dry_run', False) and _CONF.get('dry_run_db', False): + if _CONF.get('dry_run', False) and db_url != 'sqlite://': clean_dry_run_db() From 58a6f217051e57b78905eb7f91dfae8f35dcef20 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 7 Jun 2018 05:26:39 +0200 Subject: [PATCH 47/70] remove dry_run_db and replace it with db_url in config --- freqtrade/arguments.py | 24 ++++++++++-------------- freqtrade/configuration.py | 23 ++++++++++++++--------- freqtrade/constants.py | 2 ++ 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index d79a52af2..659d39d09 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -95,20 +95,23 @@ class Arguments(object): ) self.parser.add_argument( '--dynamic-whitelist', - help='dynamically generate and update whitelist \ - based on 24h BaseVolume (Default 20 currencies)', # noqa + help='dynamically generate and update whitelist' + ' based on 24h BaseVolume (default: %(default)s)', dest='dynamic_whitelist', const=constants.DYNAMIC_WHITELIST, + default=constants.DYNAMIC_WHITELIST, type=int, metavar='INT', nargs='?', ) self.parser.add_argument( - '--dry-run-db', - help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" \ - instead of memory DB. Work only if dry_run is enabled.', - action='store_true', - dest='dry_run_db', + '--db-url', + help='Override trades database URL, this is useful if dry_run is enabled' + ' or in custom deployments (default: %(default)s)', + dest='db_url', + default=constants.DEFAULT_DB_URL, + type=str, + metavar='PATH', ) @staticmethod @@ -277,13 +280,6 @@ class Arguments(object): default=None ) - self.parser.add_argument( - '-db', '--db-url', - help='Show trades stored in database.', - dest='db_url', - default=None - ) - def testdata_dl_options(self) -> None: """ Parses given arguments for testdata download diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 54146199d..afabfe225 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -97,16 +97,21 @@ class Configuration(object): '(not applicable with Backtesting and Hyperopt)' ) - # Add dry_run_db if found and the bot in dry run - if self.args.dry_run_db and config.get('dry_run', False): - config.update({'dry_run_db': True}) - logger.info('Parameter --dry-run-db detected ...') + if self.args.db_url and config.get('db_url', None): + config.update({'db_url': self.args.db_url}) + logger.info('Parameter --db-url detected ...') - if config.get('dry_run_db', False): - if config.get('dry_run', False): - logger.info('Dry_run will use the DB file: "tradesv3.dry_run.sqlite"') - else: - logger.info('Dry run is disabled. (--dry_run_db ignored)') + if config.get('dry_run', False): + logger.info('Dry run is enabled') + if config.get('db_url') in [None, constants.DEFAULT_DB_URL]: + # Default to in-memory db for dry_run if not specified + config['db_url'] = 'sqlite://' + else: + if not config.get('db_url', None): + config['db_url'] = constants.DEFAULT_DB_URL + logger.info('Dry run is disabled') + + logger.info('Using DB: "{}"'.format(config['db_url'])) # Check if the exchange set by the user is supported self.check_exchange(config) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 5469d9b60..204c6fb36 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -9,6 +9,7 @@ TICKER_INTERVAL = 5 # min HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec DEFAULT_STRATEGY = 'DefaultStrategy' +DEFAULT_DB_URL = 'sqlite:///tradesv3.sqlite' TICKER_INTERVAL_MINUTES = { '1m': 1, @@ -83,6 +84,7 @@ CONF_SCHEMA = { }, 'required': ['enabled', 'token', 'chat_id'] }, + 'db_url': {'type': 'string'}, 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, 'internals': { 'type': 'object', From e2aa78c11b3a1a06bcb839de9558ed975464cd07 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 7 Jun 2018 05:27:27 +0200 Subject: [PATCH 48/70] remove obsolete param --- freqtrade/freqtradebot.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 41841e911..cd0c4b6d4 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -33,12 +33,11 @@ class FreqtradeBot(object): This is from here the bot start its logic. """ - def __init__(self, config: Dict[str, Any], db_url: Optional[str] = None)-> None: + def __init__(self, config: Dict[str, Any])-> None: """ Init all variables and object the bot need to work :param config: configuration dict, you can use the Configuration.get_config() method to get the config dict. - :param db_url: database connector string for sqlalchemy (Optional) """ logger.info( @@ -57,17 +56,16 @@ class FreqtradeBot(object): self.persistence = None self.exchange = None - self._init_modules(db_url=db_url) + self._init_modules() - def _init_modules(self, db_url: Optional[str] = None) -> None: + def _init_modules(self) -> None: """ Initializes all modules and updates the config - :param db_url: database connector string for sqlalchemy (Optional) :return: None """ # Initialize all modules - persistence.init(self.config, db_url) + persistence.init(self.config) exchange.init(self.config) # Set initial application state From a29ac44d640136558f09a765fd24374f2c31f840 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 7 Jun 2018 05:27:55 +0200 Subject: [PATCH 49/70] adapt tests --- freqtrade/tests/conftest.py | 6 +- freqtrade/tests/rpc/test_rpc.py | 24 ++++--- freqtrade/tests/rpc/test_rpc_telegram.py | 53 ++++++++------- freqtrade/tests/test_configuration.py | 26 ++------ freqtrade/tests/test_freqtradebot.py | 77 +++++++++++----------- freqtrade/tests/test_persistence.py | 83 +++++------------------- 6 files changed, 99 insertions(+), 170 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 5d6195a2f..1311687b7 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -9,7 +9,6 @@ from unittest.mock import MagicMock import arrow import pytest from jsonschema import validate -from sqlalchemy import create_engine from telegram import Chat, Message, Update from freqtrade.analyze import Analyze @@ -45,7 +44,7 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) mocker.patch('freqtrade.freqtradebot.Analyze.get_signal', MagicMock()) - return FreqtradeBot(config, create_engine('sqlite://')) + return FreqtradeBot(config) def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]] = None) -> None: @@ -108,7 +107,8 @@ def default_conf(): "chat_id": "0" }, "initial_state": "running", - "loglevel": logging.DEBUG + "db_url": "sqlite://", + "loglevel": logging.DEBUG, } validate(configuration, constants.CONF_SCHEMA) return configuration diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 1cf374b6b..6a7de9796 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -7,8 +7,6 @@ Unit test file for rpc/rpc.py from datetime import datetime from unittest.mock import MagicMock -from sqlalchemy import create_engine - from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc.rpc import RPC @@ -39,7 +37,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: get_fee=fee ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED @@ -88,7 +86,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: get_fee=fee ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED @@ -123,7 +121,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, get_fee=fee ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] @@ -180,7 +178,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, get_fee=fee ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] @@ -243,7 +241,7 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, get_fee=fee ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] @@ -314,7 +312,7 @@ def test_rpc_balance_handle(default_conf, mocker): get_balances=MagicMock(return_value=mock_balance) ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) (error, res) = rpc.rpc_balance(default_conf['fiat_display_currency']) @@ -344,7 +342,7 @@ def test_rpc_start(mocker, default_conf) -> None: get_ticker=MagicMock() ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED @@ -372,7 +370,7 @@ def test_rpc_stop(mocker, default_conf) -> None: get_ticker=MagicMock() ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) freqtradebot.state = State.RUNNING @@ -411,7 +409,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: get_fee=fee, ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED @@ -521,7 +519,7 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, get_fee=fee ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) # Create some test data @@ -560,7 +558,7 @@ def test_rpc_count(mocker, default_conf, ticker, fee) -> None: get_fee=fee, ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) (error, trades) = rpc.rpc_count() diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 28fdc7902..2e6e3b285 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -11,7 +11,6 @@ from datetime import datetime from random import randint from unittest.mock import MagicMock -from sqlalchemy import create_engine from telegram import Update, Message, Chat from telegram.error import NetworkError @@ -156,7 +155,7 @@ def test_authorized_only(default_conf, mocker, caplog) -> None: conf = deepcopy(default_conf) conf['telegram']['enabled'] = False - dummy = DummyCls(FreqtradeBot(conf, create_engine('sqlite://'))) + dummy = DummyCls(FreqtradeBot(conf)) dummy.dummy_handler(bot=MagicMock(), update=update) assert dummy.state['called'] is True assert log_has( @@ -187,7 +186,7 @@ def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: conf = deepcopy(default_conf) conf['telegram']['enabled'] = False - dummy = DummyCls(FreqtradeBot(conf, create_engine('sqlite://'))) + dummy = DummyCls(FreqtradeBot(conf)) dummy.dummy_handler(bot=MagicMock(), update=update) assert dummy.state['called'] is False assert not log_has( @@ -217,7 +216,7 @@ def test_authorized_only_exception(default_conf, mocker, caplog) -> None: conf = deepcopy(default_conf) conf['telegram']['enabled'] = False - dummy = DummyCls(FreqtradeBot(conf, create_engine('sqlite://'))) + dummy = DummyCls(FreqtradeBot(conf)) dummy.dummy_exception(bot=MagicMock(), update=update) assert dummy.state['called'] is False assert not log_has( @@ -263,7 +262,7 @@ def test_status(default_conf, update, mocker, fee, ticker) -> None: ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(conf) telegram = Telegram(freqtradebot) # Create some test data @@ -301,7 +300,7 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED @@ -348,7 +347,7 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: conf = deepcopy(default_conf) conf['stake_amount'] = 15.0 - freqtradebot = FreqtradeBot(conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(conf) telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED @@ -402,7 +401,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) # Create some test data @@ -470,7 +469,7 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) # Try invalid data @@ -511,7 +510,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) telegram._profit(bot=MagicMock(), update=update) @@ -608,7 +607,7 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None: send_msg=msg_mock ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) telegram._balance(bot=MagicMock(), update=update) @@ -638,7 +637,7 @@ def test_zero_balance_handle(default_conf, update, mocker) -> None: send_msg=msg_mock ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) telegram._balance(bot=MagicMock(), update=update) @@ -661,7 +660,7 @@ def test_start_handle(default_conf, update, mocker) -> None: ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED @@ -685,7 +684,7 @@ def test_start_handle_already_running(default_conf, update, mocker) -> None: ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) freqtradebot.state = State.RUNNING @@ -710,7 +709,7 @@ def test_stop_handle(default_conf, update, mocker) -> None: ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) freqtradebot.state = State.RUNNING @@ -735,7 +734,7 @@ def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED @@ -762,7 +761,7 @@ def test_forcesell_handle(default_conf, update, ticker, fee, ticker_sell_up, moc get_fee=fee ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) # Create some test data @@ -802,7 +801,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, ticker_sell_do get_fee=fee ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) # Create some test data @@ -847,7 +846,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None get_fee=fee ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) # Create some test data @@ -880,7 +879,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: ) mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) # Trader is not running @@ -927,7 +926,7 @@ def test_performance_handle(default_conf, update, ticker, fee, get_fee=fee ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) # Create some test data @@ -962,7 +961,7 @@ def test_performance_handle_invalid(default_conf, update, mocker) -> None: send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) # Trader is not running @@ -991,7 +990,7 @@ def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: buy=MagicMock(return_value={'id': 'mocked_order_id'}) ) mocker.patch('freqtrade.optimize.backtesting.exchange.get_fee', fee) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED @@ -1027,7 +1026,7 @@ def test_help_handle(default_conf, update, mocker) -> None: _init=MagicMock(), send_msg=msg_mock ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) telegram._help(bot=MagicMock(), update=update) @@ -1047,7 +1046,7 @@ def test_version_handle(default_conf, update, mocker) -> None: _init=MagicMock(), send_msg=msg_mock ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) telegram._version(bot=MagicMock(), update=update) @@ -1064,7 +1063,7 @@ def test_send_msg(default_conf, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) conf = deepcopy(default_conf) bot = MagicMock() - freqtradebot = FreqtradeBot(conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(conf) telegram = Telegram(freqtradebot) telegram._config['telegram']['enabled'] = False @@ -1087,7 +1086,7 @@ def test_send_msg_network_error(default_conf, mocker, caplog) -> None: conf = deepcopy(default_conf) bot = MagicMock() bot.send_message = MagicMock(side_effect=NetworkError('Oh snap')) - freqtradebot = FreqtradeBot(conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(conf) telegram = Telegram(freqtradebot) telegram._config['telegram']['enabled'] = True diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 492fdee70..2f633c021 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -118,7 +118,6 @@ def test_load_config(default_conf, mocker) -> None: assert validated_conf.get('strategy') == 'DefaultStrategy' assert validated_conf.get('strategy_path') is None assert 'dynamic_whitelist' not in validated_conf - assert 'dry_run_db' not in validated_conf def test_load_config_with_params(default_conf, mocker) -> None: @@ -133,7 +132,7 @@ def test_load_config_with_params(default_conf, mocker) -> None: '--dynamic-whitelist', '10', '--strategy', 'TestStrategy', '--strategy-path', '/some/path', - '--dry-run-db', + '--db-url', 'sqlite:///someurl', ] args = Arguments(arglist, '').get_parsed_arg() @@ -143,7 +142,7 @@ def test_load_config_with_params(default_conf, mocker) -> None: assert validated_conf.get('dynamic_whitelist') == 10 assert validated_conf.get('strategy') == 'TestStrategy' assert validated_conf.get('strategy_path') == '/some/path' - assert validated_conf.get('dry_run_db') is True + assert validated_conf.get('db_url') == 'sqlite:///someurl' def test_load_custom_strategy(default_conf, mocker) -> None: @@ -178,7 +177,7 @@ def test_show_info(default_conf, mocker, caplog) -> None: arglist = [ '--dynamic-whitelist', '10', '--strategy', 'TestStrategy', - '--dry-run-db' + '--db-url', 'sqlite:///tmp/testdb', ] args = Arguments(arglist, '').get_parsed_arg() @@ -192,23 +191,8 @@ def test_show_info(default_conf, mocker, caplog) -> None: caplog.record_tuples ) - assert log_has( - 'Parameter --dry-run-db detected ...', - caplog.record_tuples - ) - - assert log_has( - 'Dry_run will use the DB file: "tradesv3.dry_run.sqlite"', - caplog.record_tuples - ) - - # Test the Dry run condition - configuration.config.update({'dry_run': False}) # type: ignore - configuration._load_common_config(configuration.config) # type: ignore - assert log_has( - 'Dry run is disabled. (--dry_run_db ignored)', - caplog.record_tuples - ) + assert log_has('Using DB: "sqlite:///tmp/testdb"', caplog.record_tuples) + assert log_has('Dry run is enabled', caplog.record_tuples) def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None: diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index ebabc0187..8f39c71a8 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -13,7 +13,6 @@ from unittest.mock import MagicMock import arrow import pytest import requests -from sqlalchemy import create_engine from freqtrade import DependencyException, OperationalException, TemporaryError from freqtrade.freqtradebot import FreqtradeBot @@ -36,7 +35,7 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) patch_coinmarketcap(mocker) - return FreqtradeBot(config, create_engine('sqlite://')) + return FreqtradeBot(config) def patch_get_signal(mocker, value=(True, False)) -> None: @@ -237,7 +236,7 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, mocker) -> Non # Save state of current whitelist whitelist = deepcopy(default_conf['exchange']['pair_whitelist']) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) freqtrade.create_trade() trade = Trade.query.first() @@ -274,7 +273,7 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, fee, conf = deepcopy(default_conf) conf['stake_amount'] = 0.0005 - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() rate, amount = buy_mock.call_args[0][1], buy_mock.call_args[0][2] @@ -296,7 +295,7 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, fee get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5), get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) with pytest.raises(DependencyException, match=r'.*stake amount.*'): freqtrade.create_trade() @@ -320,7 +319,7 @@ def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, mocke conf = deepcopy(default_conf) conf['exchange']['pair_whitelist'] = ["ETH/BTC"] conf['exchange']['pair_blacklist'] = ["ETH/BTC"] - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() @@ -347,7 +346,7 @@ def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, conf = deepcopy(default_conf) conf['exchange']['pair_whitelist'] = ["ETH/BTC"] conf['exchange']['pair_blacklist'] = ["ETH/BTC"] - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() @@ -375,7 +374,7 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: conf = deepcopy(default_conf) conf['stake_amount'] = 10 - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) Trade.query = MagicMock() Trade.query.filter = MagicMock() @@ -399,7 +398,7 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, get_order=MagicMock(return_value=limit_buy_order), get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) trades = Trade.query.filter(Trade.is_open.is_(True)).all() assert not trades @@ -440,7 +439,7 @@ def test_process_exchange_failures(default_conf, ticker, markets, mocker) -> Non ) sleep_mock = mocker.patch('time.sleep', side_effect=lambda _: None) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) result = freqtrade._process() assert result is False assert sleep_mock.has_calls() @@ -460,7 +459,7 @@ def test_process_operational_exception(default_conf, ticker, markets, mocker) -> get_markets=markets, buy=MagicMock(side_effect=OperationalException) ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) assert freqtrade.state == State.RUNNING result = freqtrade._process() @@ -486,7 +485,7 @@ def test_process_trade_handling( get_order=MagicMock(return_value=limit_buy_order), get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) trades = Trade.query.filter(Trade.is_open.is_(True)).all() assert not trades @@ -603,7 +602,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mock ) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) freqtrade.create_trade() @@ -646,7 +645,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, fee, get_fee=fee, ) - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() @@ -705,7 +704,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, fee, mocker, ca ) mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() trade = Trade.query.first() @@ -742,7 +741,7 @@ def test_handle_trade_experimental( ) mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() trade = Trade.query.first() @@ -770,7 +769,7 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, fe buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Create trade and sell it freqtrade.create_trade() @@ -801,7 +800,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fe cancel_order=cancel_order_mock, get_fee=fee ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) trade_buy = Trade( pair='ETH/BTC', @@ -841,7 +840,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, get_order=MagicMock(return_value=limit_sell_order_old), cancel_order=cancel_order_mock ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) trade_sell = Trade( pair='ETH/BTC', @@ -881,7 +880,7 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old get_order=MagicMock(return_value=limit_buy_order_old_partial), cancel_order=cancel_order_mock ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) trade_buy = Trade( pair='ETH/BTC', @@ -929,7 +928,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) - get_order=MagicMock(side_effect=requests.exceptions.RequestException('Oh snap')), cancel_order=cancel_order_mock ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) trade_buy = Trade( pair='ETH/BTC', @@ -968,7 +967,7 @@ def test_handle_timedout_limit_buy(mocker, default_conf) -> None: cancel_order=cancel_order_mock ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) Trade.session = MagicMock() trade = MagicMock() @@ -994,7 +993,7 @@ def test_handle_timedout_limit_sell(mocker, default_conf) -> None: cancel_order=cancel_order_mock ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) trade = MagicMock() order = {'remaining': 1, @@ -1021,7 +1020,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N get_fee=fee ) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Create some test data freqtrade.create_trade() @@ -1062,7 +1061,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker) get_ticker=ticker, get_fee=fee ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Create some test data freqtrade.create_trade() @@ -1102,7 +1101,7 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, get_ticker=ticker, get_fee=fee ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Create some test data freqtrade.create_trade() @@ -1143,7 +1142,7 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, get_ticker=ticker, get_fee=fee ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Create some test data freqtrade.create_trade() @@ -1192,7 +1191,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, fee, mock 'use_sell_signal': True, 'sell_profit_only': True, } - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() trade = Trade.query.first() @@ -1225,7 +1224,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, fee, moc 'use_sell_signal': True, 'sell_profit_only': False, } - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() trade = Trade.query.first() @@ -1258,7 +1257,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker 'use_sell_signal': True, 'sell_profit_only': True, } - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() trade = Trade.query.first() @@ -1293,7 +1292,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke 'sell_profit_only': False, } - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() trade = Trade.query.first() @@ -1321,7 +1320,7 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, ca open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' @@ -1348,7 +1347,7 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker): open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' @@ -1375,7 +1374,7 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, ca open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount @@ -1401,7 +1400,7 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, mock open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount @@ -1424,7 +1423,7 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' @@ -1452,7 +1451,7 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, limit_buy_order) == amount - 0.004 assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' @@ -1480,7 +1479,7 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, limit_buy_order) == amount @@ -1505,6 +1504,6 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 3e0f50fbb..edd3d4b30 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -1,5 +1,5 @@ # pragma pylint: disable=missing-docstring, C0103 -import os +from copy import deepcopy import pytest from sqlalchemy import create_engine @@ -21,77 +21,22 @@ def test_init_create_session(default_conf, mocker): assert 'Session' in type(Trade.session).__name__ -def test_init_dry_run_db(default_conf, mocker): - default_conf.update({'dry_run_db': True}) - mocker.patch.dict('freqtrade.persistence._CONF', default_conf) +def test_init_custom_db_url(default_conf, mocker): + conf = deepcopy(default_conf) - # First, protect the existing 'tradesv3.dry_run.sqlite' (Do not delete user data) - dry_run_db = 'tradesv3.dry_run.sqlite' - dry_run_db_swp = dry_run_db + '.swp' - - if os.path.isfile(dry_run_db): - os.rename(dry_run_db, dry_run_db_swp) + # Update path to a value other than default, but still in-memory + conf.update({'db_url': 'sqlite:///'}) + mocker.patch.dict('freqtrade.persistence._CONF', conf) # Check if the new tradesv3.dry_run.sqlite was created - init(default_conf) - assert os.path.isfile(dry_run_db) is True - - # Delete the file made for this unitest and rollback to the previous - # tradesv3.dry_run.sqlite file - - # 1. Delete file from the test - if os.path.isfile(dry_run_db): - os.remove(dry_run_db) - - # 2. Rollback to the initial file - if os.path.isfile(dry_run_db_swp): - os.rename(dry_run_db_swp, dry_run_db) - - -def test_init_dry_run_without_db(default_conf, mocker): - default_conf.update({'dry_run_db': False}) - mocker.patch.dict('freqtrade.persistence._CONF', default_conf) - - # First, protect the existing 'tradesv3.dry_run.sqlite' (Do not delete user data) - dry_run_db = 'tradesv3.dry_run.sqlite' - dry_run_db_swp = dry_run_db + '.swp' - - if os.path.isfile(dry_run_db): - os.rename(dry_run_db, dry_run_db_swp) - - # Check if the new tradesv3.dry_run.sqlite was created - init(default_conf) - assert os.path.isfile(dry_run_db) is False - - # Rollback to the initial 'tradesv3.dry_run.sqlite' file - if os.path.isfile(dry_run_db_swp): - os.rename(dry_run_db_swp, dry_run_db) + init(conf) def test_init_prod_db(default_conf, mocker): default_conf.update({'dry_run': False}) mocker.patch.dict('freqtrade.persistence._CONF', default_conf) - # First, protect the existing 'tradesv3.sqlite' (Do not delete user data) - prod_db = 'tradesv3.sqlite' - prod_db_swp = prod_db + '.swp' - - if os.path.isfile(prod_db): - os.rename(prod_db, prod_db_swp) - - # Check if the new tradesv3.sqlite was created init(default_conf) - assert os.path.isfile(prod_db) is True - - # Delete the file made for this unitest and rollback to the previous tradesv3.sqlite file - - # 1. Delete file from the test - if os.path.isfile(prod_db): - os.remove(prod_db) - - # Rollback to the initial 'tradesv3.sqlite' file - if os.path.isfile(prod_db_swp): - os.rename(prod_db_swp, prod_db) @pytest.mark.usefixtures("init_persistence") @@ -328,7 +273,7 @@ def test_calc_profit_percent(limit_buy_order, limit_sell_order, fee): def test_clean_dry_run_db(default_conf, fee): - init(default_conf, create_engine('sqlite://')) + init(default_conf) # Simulate dry_run entries trade = Trade( @@ -377,7 +322,7 @@ def test_clean_dry_run_db(default_conf, fee): assert len(Trade.query.filter(Trade.open_order_id.isnot(None)).all()) == 1 -def test_migrate_old(default_conf, fee): +def test_migrate_old(mocker, default_conf, fee): """ Test Database migration(starting with old pairformat) """ @@ -409,11 +354,13 @@ def test_migrate_old(default_conf, fee): amount=amount ) engine = create_engine('sqlite://') + mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine) + # Create table using the old format engine.execute(create_table_old) engine.execute(insert_table_old) # Run init to test migration - init(default_conf, engine) + init(default_conf) assert len(Trade.query.filter(Trade.id == 1).all()) == 1 trade = Trade.query.filter(Trade.id == 1).first() @@ -428,7 +375,7 @@ def test_migrate_old(default_conf, fee): assert trade.exchange == "bittrex" -def test_migrate_new(default_conf, fee): +def test_migrate_new(mocker, default_conf, fee): """ Test Database migration (starting with new pairformat) """ @@ -459,12 +406,14 @@ def test_migrate_new(default_conf, fee): stake=default_conf.get("stake_amount"), amount=amount ) + mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine) + engine = create_engine('sqlite://') # Create table using the old format engine.execute(create_table_old) engine.execute(insert_table_old) # Run init to test migration - init(default_conf, engine) + init(default_conf) assert len(Trade.query.filter(Trade.id == 1).all()) == 1 trade = Trade.query.filter(Trade.id == 1).first() From c8a43bad671e2a662d1a79903294dcc5d46e3833 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 7 Jun 2018 05:28:05 +0200 Subject: [PATCH 50/70] add db_url to full example config --- config_full.json.example | 1 + 1 file changed, 1 insertion(+) diff --git a/config_full.json.example b/config_full.json.example index 77ef0faa0..c17d22a15 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -45,6 +45,7 @@ "token": "your_telegram_token", "chat_id": "your_telegram_chat_id" }, + "db_url": "sqlite:///tradesv3.sqlite", "initial_state": "running", "internals": { "process_throttle_secs": 5 From 00b646158cf54d3510ff97ae0b7b41b40e8e58e2 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 7 Jun 2018 05:36:39 +0200 Subject: [PATCH 51/70] update docs --- docs/bot-usage.md | 27 +++++++++++++-------------- docs/configuration.md | 4 +++- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 815fed672..8079d9816 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -9,10 +9,10 @@ it. ## Bot commands ``` -usage: main.py [-h] [-v] [--version] [-c PATH] [-d PATH] [-s NAME] - [--strategy-path PATH] [--dynamic-whitelist [INT]] - [--dry-run-db] - {backtesting,hyperopt} ... +usage: freqtrade [-h] [-v] [--version] [-c PATH] [-d PATH] [-s NAME] + [--strategy-path PATH] [--dynamic-whitelist [INT]] + [--db-url PATH] + {backtesting,hyperopt} ... Simple High Frequency Trading Bot for crypto currencies @@ -28,17 +28,16 @@ optional arguments: -c PATH, --config PATH specify configuration file (default: config.json) -d PATH, --datadir PATH - path to backtest data (default: - freqtrade/tests/testdata + path to backtest data -s NAME, --strategy NAME specify strategy class name (default: DefaultStrategy) --strategy-path PATH specify additional strategy lookup path --dynamic-whitelist [INT] dynamically generate and update whitelist based on 24h - BaseVolume (Default 20 currencies) - --dry-run-db Force dry run to use a local DB - "tradesv3.dry_run.sqlite" instead of memory DB. Work - only if dry_run is enabled. + BaseVolume (default: 20) + --db-url PATH Override trades database URL, this is useful if + dry_run is enabled or in custom deployments (default: + sqlite:///tradesv3.sqlite) ``` ### How to use a different config file? @@ -102,14 +101,14 @@ python3 ./freqtrade/main.py --dynamic-whitelist 30 negative value (e.g -2), `--dynamic-whitelist` will use the default value (20). -### How to use --dry-run-db? +### How to use --db-url? When you run the bot in Dry-run mode, per default no transactions are stored in a database. If you want to store your bot actions in a DB -using `--dry-run-db`. This command will use a separate database file -`tradesv3.dry_run.sqlite` +using `--db-url`. This can also be used to specify a custom database +in production mode. Example command: ```bash -python3 ./freqtrade/main.py -c config.json --dry-run-db +python3 ./freqtrade/main.py -c config.json --db-url sqlite:///tradesv3.dry_run.sqlite ``` diff --git a/docs/configuration.md b/docs/configuration.md index 1c4e2b6e1..fe220403d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -34,6 +34,7 @@ The table below will list all configuration parameters. | `telegram.enabled` | true | Yes | Enable or not the usage of Telegram. | `telegram.token` | token | No | Your Telegram bot token. Only required if `telegram.enabled` is `true`. | `telegram.chat_id` | chat_id | No | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. +| `db_url` | `sqlite:///tradesv3.sqlite` | No | Declares database URL to use. NOTE: This defaults to `sqlite://` if `dry_run` is `True`. | `initial_state` | running | No | Defines the initial application state. More information below. | `strategy` | DefaultStrategy | No | Defines Strategy class to use. | `strategy_path` | null | No | Adds an additional strategy lookup path (must be a folder). @@ -111,9 +112,10 @@ creating trades. ### To switch your bot in Dry-run mode: 1. Edit your `config.json` file -2. Switch dry-run to true +2. Switch dry-run to true and specify db_url for a persistent db ```json "dry_run": true, +"db_url": "sqlite///tradesv3.dryrun.sqlite", ``` 3. Remove your Exchange API key (change them by fake api credentials) From f6ef466876d81e2c967a3d69c8bf54fbd49d1e2c Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 7 Jun 2018 05:47:14 +0200 Subject: [PATCH 52/70] adapt docs --- docs/configuration.md | 2 +- docs/installation.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index fe220403d..d5d53860b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -139,7 +139,7 @@ you run it in production mode. ### To switch your bot in production mode: 1. Edit your `config.json` file -2. Switch dry-run to false +2. Switch dry-run to false and don't forget to adapt your database URL if set ```json "dry_run": false, ``` diff --git a/docs/installation.md b/docs/installation.md index 850b2c255..2fd40e451 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -162,10 +162,10 @@ docker run -d \ -v /etc/localtime:/etc/localtime:ro \ -v ~/.freqtrade/config.json:/freqtrade/config.json \ -v ~/.freqtrade/tradesv3.sqlite:/freqtrade/tradesv3.sqlite \ - freqtrade + freqtrade --db-url sqlite:///tradesv3.sqlite ``` - -If you are using `dry_run=True` it's not necessary to mount `tradesv3.sqlite`, but you can mount `tradesv3.dryrun.sqlite` if you plan to use the dry run mode with the param `--dry-run-db`. +NOTE: db-url defaults to `sqlite:///tradesv3.sqlite` but it defaults to `sqlite://` if `dry_run=True` is being used. +To override this behaviour use a custom db-url value: i.e.: `--db-url sqlite:///tradesv3.dryrun.sqlite` ### 6. Monitor your Docker instance From 4ee5271de77d9a91cbd12855da3e5c7e084786d7 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 7 Jun 2018 05:50:07 +0200 Subject: [PATCH 53/70] fix failing dynamic-whitelist test --- freqtrade/arguments.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 659d39d09..7880cea2a 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -96,10 +96,9 @@ class Arguments(object): self.parser.add_argument( '--dynamic-whitelist', help='dynamically generate and update whitelist' - ' based on 24h BaseVolume (default: %(default)s)', + ' based on 24h BaseVolume (default: %(const)s)', dest='dynamic_whitelist', const=constants.DYNAMIC_WHITELIST, - default=constants.DYNAMIC_WHITELIST, type=int, metavar='INT', nargs='?', From c3d09807636b264660cc46e78cd77ec5607969ff Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 7 Jun 2018 06:06:21 +0200 Subject: [PATCH 54/70] test_persistence: fix reference before assignment --- freqtrade/tests/test_persistence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index edd3d4b30..dae9f3c90 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -406,9 +406,9 @@ def test_migrate_new(mocker, default_conf, fee): stake=default_conf.get("stake_amount"), amount=amount ) + engine = create_engine('sqlite://') mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine) - engine = create_engine('sqlite://') # Create table using the old format engine.execute(create_table_old) engine.execute(insert_table_old) From a2fd70417cd527bd610ab193f3baf728eccd5ee5 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 7 Jun 2018 14:22:07 +0200 Subject: [PATCH 55/70] Update ccxt from 1.14.121 to 1.14.155 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 43043775e..f6e5e2932 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.121 +ccxt==1.14.155 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 34b5203760388f397d6185621e035d0278ff3a7e Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 7 Jun 2018 14:22:08 +0200 Subject: [PATCH 56/70] Update numpy from 1.14.3 to 1.14.4 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f6e5e2932..2dfe15758 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ pandas==0.23.0 scikit-learn==0.19.1 scipy==1.1.0 jsonschema==2.6.0 -numpy==1.14.3 +numpy==1.14.4 TA-Lib==0.4.17 pytest==3.6.0 pytest-mock==1.10.0 From 7b0a5644a3af50318ec90b096e03456535da0f81 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 7 Jun 2018 14:22:10 +0200 Subject: [PATCH 57/70] Update pytest from 3.6.0 to 3.6.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2dfe15758..5f5183321 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.14.4 TA-Lib==0.4.17 -pytest==3.6.0 +pytest==3.6.1 pytest-mock==1.10.0 pytest-cov==2.5.1 hyperopt==0.1 From 7f8e0ba25fe544dbb187e40337044c6b55afadfc Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 6 Jun 2018 13:56:08 +0300 Subject: [PATCH 58/70] use buy/sell signal from previous candle, not current to avoid seeing to the future --- freqtrade/optimize/backtesting.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3dd643561..57282aae9 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -162,6 +162,10 @@ class Backtesting(object): pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run ticker_data = self.populate_sell_trend(self.populate_buy_trend(pair_data))[headers] + + # to avoid using data from future, we buy/sell with signal from previous candle, not current + ticker_data.buy = ticker_data.buy.shift(1) + ticker_data.sell = ticker_data.sell.shift(1) ticker = [x for x in ticker_data.itertuples()] lock_pair_until = None From b4ae5a36a8f69b93a423fed591aab5bbf8995b74 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Thu, 7 Jun 2018 10:21:07 +0300 Subject: [PATCH 59/70] use .copy() to avoid Pandas mistake. drop first row because of shifting --- freqtrade/optimize/backtesting.py | 12 ++++++++---- freqtrade/tests/optimize/test_backtesting.py | 6 +++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 57282aae9..028a4f521 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -161,11 +161,15 @@ class Backtesting(object): for pair, pair_data in processed.items(): pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run - ticker_data = self.populate_sell_trend(self.populate_buy_trend(pair_data))[headers] + ticker_data = self.populate_sell_trend( + self.populate_buy_trend(pair_data))[headers].copy() + + # to avoid using data from future, we buy/sell with signal from previous candle + ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) + ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) + + ticker_data.drop(ticker_data.head(1).index, inplace=True) - # to avoid using data from future, we buy/sell with signal from previous candle, not current - ticker_data.buy = ticker_data.buy.shift(1) - ticker_data.sell = ticker_data.sell.shift(1) ticker = [x for x in ticker_data.itertuples()] lock_pair_until = None diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index efcee3839..f924d21c6 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -30,7 +30,7 @@ def trim_dictlist(dict_list, num): def load_data_test(what): - timerange = TimeRange(None, 'line', 0, -100) + timerange = TimeRange(None, 'line', 0, -101) data = optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC'], timerange=timerange) pair = data['UNITTEST/BTC'] @@ -110,14 +110,14 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals # use for mock freqtrade.exchange.get_ticker_history' def _load_pair_as_ticks(pair, tickfreq): ticks = optimize.load_data(None, ticker_interval=tickfreq, pairs=[pair]) - ticks = trim_dictlist(ticks, -200) + ticks = trim_dictlist(ticks, -201) return ticks[pair] # FIX: fixturize this? def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None): data = optimize.load_data(None, ticker_interval='8m', pairs=[pair]) - data = trim_dictlist(data, -200) + data = trim_dictlist(data, -201) mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) backtesting = Backtesting(conf) return { From 959a03a6b097beecf4a0243b9be8cff7f5694447 Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Thu, 7 Jun 2018 15:13:55 +0000 Subject: [PATCH 60/70] plotting.md update. include an example or plotting a strategy buy/sell output. --- docs/plotting.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/plotting.md b/docs/plotting.md index ae964fb16..87b25e32a 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -48,6 +48,12 @@ To plot trades stored in a database use `--db-url` argument: python scripts/plot_dataframe.py --db-url tradesv3.dry_run.sqlite -p BTC_ETH ``` +To plot a test strategy the strategy should have first be backtested. +The results may then be plotted with the -s argument: +``` +python scripts/plot_dataframe.py -s Strategy_Name -p BTC/ETH --data-dir users_data/data/ +``` + ## Plot profit The profit plotter show a picture with three plots: From 7bcac064c045c418826ca1ca5d2eaf31af39f27d Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Thu, 7 Jun 2018 15:18:19 +0000 Subject: [PATCH 61/70] Update plotting.md typo fixed. --- docs/plotting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plotting.md b/docs/plotting.md index 87b25e32a..242d80005 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -51,7 +51,7 @@ python scripts/plot_dataframe.py --db-url tradesv3.dry_run.sqlite -p BTC_ETH To plot a test strategy the strategy should have first be backtested. The results may then be plotted with the -s argument: ``` -python scripts/plot_dataframe.py -s Strategy_Name -p BTC/ETH --data-dir users_data/data/ +python scripts/plot_dataframe.py -s Strategy_Name -p BTC/ETH --datadir user_data/data// ``` ## Plot profit From 5b1ff6675fc60613298d8d659f2a98348150e419 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 7 Jun 2018 17:29:43 +0200 Subject: [PATCH 62/70] define constants.DEFAULT_DB_DRYRUN_URL and fix StaticPool conditions --- freqtrade/arguments.py | 2 +- freqtrade/configuration.py | 6 +++--- freqtrade/constants.py | 3 ++- freqtrade/persistence.py | 10 +++++----- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 7880cea2a..fc5f11d50 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -108,7 +108,7 @@ class Arguments(object): help='Override trades database URL, this is useful if dry_run is enabled' ' or in custom deployments (default: %(default)s)', dest='db_url', - default=constants.DEFAULT_DB_URL, + default=constants.DEFAULT_DB_PROD_URL, type=str, metavar='PATH', ) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index afabfe225..ce051ecc4 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -103,12 +103,12 @@ class Configuration(object): if config.get('dry_run', False): logger.info('Dry run is enabled') - if config.get('db_url') in [None, constants.DEFAULT_DB_URL]: + if config.get('db_url') in [None, constants.DEFAULT_DB_PROD_URL]: # Default to in-memory db for dry_run if not specified - config['db_url'] = 'sqlite://' + config['db_url'] = constants.DEFAULT_DB_DRYRUN_URL else: if not config.get('db_url', None): - config['db_url'] = constants.DEFAULT_DB_URL + config['db_url'] = constants.DEFAULT_DB_PROD_URL logger.info('Dry run is disabled') logger.info('Using DB: "{}"'.format(config['db_url'])) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 204c6fb36..5be01f977 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -9,7 +9,8 @@ TICKER_INTERVAL = 5 # min HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec DEFAULT_STRATEGY = 'DefaultStrategy' -DEFAULT_DB_URL = 'sqlite:///tradesv3.sqlite' +DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' +DEFAULT_DB_DRYRUN_URL = 'sqlite://' TICKER_INTERVAL_MINUTES = { '1m': 1, diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 63c29dc4f..aa8a978d5 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -16,6 +16,8 @@ from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker from sqlalchemy.pool import StaticPool +from freqtrade import constants + logger = logging.getLogger(__name__) _CONF = {} @@ -35,10 +37,8 @@ def init(config: Dict) -> None: db_url = _CONF.get('db_url', None) kwargs = {} - if not db_url and _CONF.get('dry_run', False): - # Default to in-memory db if not specified - # and take care of thread ownership if in-memory db - db_url = 'sqlite://' + if db_url == constants.DEFAULT_DB_DRYRUN_URL: + # Take care of thread ownership if in-memory db kwargs.update({ 'connect_args': {'check_same_thread': False}, 'poolclass': StaticPool, @@ -53,7 +53,7 @@ def init(config: Dict) -> None: check_migrate(engine) # Clean dry_run DB - if _CONF.get('dry_run', False) and db_url != 'sqlite://': + if _CONF.get('dry_run', False) and db_url != constants.DEFAULT_DB_DRYRUN_URL: clean_dry_run_db() From 01675f50bff286a17d4d0dd67153c47ea634a8d0 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 7 Jun 2018 18:06:27 +0200 Subject: [PATCH 63/70] adapt scripts/plot_dataframe to use freqtrade db_url --- scripts/plot_dataframe.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index e7737a5c7..122c002a8 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -29,16 +29,17 @@ import os import sys from argparse import Namespace from typing import Dict, List, Any -from sqlalchemy import create_engine + +import plotly.graph_objs as go from plotly import tools from plotly.offline import plot -import plotly.graph_objs as go -from freqtrade.arguments import Arguments -from freqtrade.analyze import Analyze -from freqtrade.optimize.backtesting import setup_configuration -from freqtrade import exchange + import freqtrade.optimize as optimize +from freqtrade import exchange from freqtrade import persistence +from freqtrade.analyze import Analyze +from freqtrade.arguments import Arguments +from freqtrade.optimize.backtesting import setup_configuration from freqtrade.persistence import Trade logger = logging.getLogger(__name__) @@ -50,9 +51,10 @@ def plot_analyzed_dataframe(args: Namespace) -> None: Calls analyze() and plots the returned dataframe :return: None """ + global _CONF # Load the configuration - config = setup_configuration(args) + _CONF.update(setup_configuration(args)) # Set the pair to audit pair = args.pair @@ -65,14 +67,13 @@ def plot_analyzed_dataframe(args: Namespace) -> None: logger.critical('--pair format must be XXX/YYY') exit() - # Set timerange to use timerange = Arguments.parse_timerange(args.timerange) # Load the strategy try: - analyze = Analyze(config) - exchange.init(config) + analyze = Analyze(_CONF) + exchange.init(_CONF) except AttributeError: logger.critical( 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', @@ -93,7 +94,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: datadir=args.datadir, pairs=[pair], ticker_interval=tick_interval, - refresh_pairs=config.get('refresh_pairs', False), + refresh_pairs=_CONF.get('refresh_pairs', False), timerange=timerange ) @@ -102,10 +103,9 @@ def plot_analyzed_dataframe(args: Namespace) -> None: exit() # Get trades already made from the DB - trades = [] + trades: List[Trade] = [] if args.db_url: - engine = create_engine('sqlite:///' + args.db_url) - persistence.init(_CONF, engine) + persistence.init(_CONF) trades = Trade.query.filter(Trade.pair.is_(pair)).all() dataframes = analyze.tickerdata_to_dataframe(tickers) From ac602ed5a9e919d38e066108e87c563fb71a34f6 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 7 Jun 2018 19:10:26 +0200 Subject: [PATCH 64/70] persistence: adapt checks to detect in-memory db --- freqtrade/persistence.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index aa8a978d5..ce834bced 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -16,8 +16,6 @@ from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker from sqlalchemy.pool import StaticPool -from freqtrade import constants - logger = logging.getLogger(__name__) _CONF = {} @@ -37,8 +35,8 @@ def init(config: Dict) -> None: db_url = _CONF.get('db_url', None) kwargs = {} - if db_url == constants.DEFAULT_DB_DRYRUN_URL: - # Take care of thread ownership if in-memory db + # Take care of thread ownership if in-memory db + if db_url == 'sqlite://': kwargs.update({ 'connect_args': {'check_same_thread': False}, 'poolclass': StaticPool, @@ -52,8 +50,8 @@ def init(config: Dict) -> None: _DECL_BASE.metadata.create_all(engine) check_migrate(engine) - # Clean dry_run DB - if _CONF.get('dry_run', False) and db_url != constants.DEFAULT_DB_DRYRUN_URL: + # Clean dry_run DB if the db is not in-memory + if _CONF.get('dry_run', False) and db_url != 'sqlite://': clean_dry_run_db() From 526cb1ea2090e570a7f160382a9cf7d6aa0bb9a1 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 7 Jun 2018 20:15:31 +0200 Subject: [PATCH 65/70] fix db-url handling if passed via CLI args --- freqtrade/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index ce051ecc4..2a9e8fbd8 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -97,7 +97,7 @@ class Configuration(object): '(not applicable with Backtesting and Hyperopt)' ) - if self.args.db_url and config.get('db_url', None): + if self.args.db_url != constants.DEFAULT_DB_PROD_URL: config.update({'db_url': self.args.db_url}) logger.info('Parameter --db-url detected ...') From d4f8704a4c908e546e1041bd90facaaf0f1d4685 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 7 Jun 2018 20:30:13 +0200 Subject: [PATCH 66/70] arguments: implement tests for db_url --- freqtrade/tests/test_arguments.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 6c3ecb913..a7237d7c4 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -46,6 +46,11 @@ def test_parse_args_config() -> None: assert args.config == '/dev/null' +def test_parse_args_db_url() -> None: + args = Arguments(['--db-url', 'sqlite:///test.sqlite'], '').get_parsed_arg() + assert args.db_url == 'sqlite:///test.sqlite' + + def test_parse_args_verbose() -> None: args = Arguments(['-v'], '').get_parsed_arg() assert args.loglevel == logging.DEBUG From 3f5efef6e53f73afe0f688d22fe63d2b029bc12d Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 7 Jun 2018 20:41:52 +0200 Subject: [PATCH 67/70] tests: add proper asserts --- freqtrade/tests/test_persistence.py | 33 ++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index dae9f3c90..d8d42461b 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -1,9 +1,11 @@ # pragma pylint: disable=missing-docstring, C0103 from copy import deepcopy +from unittest.mock import MagicMock import pytest from sqlalchemy import create_engine +from freqtrade import constants from freqtrade.persistence import Trade, init, clean_dry_run_db @@ -25,18 +27,39 @@ def test_init_custom_db_url(default_conf, mocker): conf = deepcopy(default_conf) # Update path to a value other than default, but still in-memory - conf.update({'db_url': 'sqlite:///'}) + conf.update({'db_url': 'sqlite:////tmp/freqtrade2_test.sqlite'}) + create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) mocker.patch.dict('freqtrade.persistence._CONF', conf) - # Check if the new tradesv3.dry_run.sqlite was created init(conf) + assert create_engine_mock.call_count == 1 + assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:////tmp/freqtrade2_test.sqlite' def test_init_prod_db(default_conf, mocker): - default_conf.update({'dry_run': False}) - mocker.patch.dict('freqtrade.persistence._CONF', default_conf) + conf = deepcopy(default_conf) + conf.update({'dry_run': False}) + conf.update({'db_url': constants.DEFAULT_DB_PROD_URL}) - init(default_conf) + create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) + mocker.patch.dict('freqtrade.persistence._CONF', conf) + + init(conf) + assert create_engine_mock.call_count == 1 + assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tradesv3.sqlite' + + +def test_init_dryrun_db(default_conf, mocker): + conf = deepcopy(default_conf) + conf.update({'dry_run': True}) + conf.update({'db_url': constants.DEFAULT_DB_DRYRUN_URL}) + + create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) + mocker.patch.dict('freqtrade.persistence._CONF', conf) + + init(conf) + assert create_engine_mock.call_count == 1 + assert create_engine_mock.mock_calls[0][1][0] == 'sqlite://' @pytest.mark.usefixtures("init_persistence") From 0e699b87af3bf3c9097c7774689f0a29f35196a8 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 7 Jun 2018 20:08:46 +0200 Subject: [PATCH 68/70] don't sum percentage, but use mean instead (aligned to backtesting) --- freqtrade/rpc/rpc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index f48666748..c2f097319 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -9,6 +9,7 @@ from typing import Dict, Tuple, Any import arrow import sqlalchemy as sql from pandas import DataFrame +from numpy import mean, nan_to_num from freqtrade import exchange from freqtrade.misc import shorten_date @@ -209,14 +210,14 @@ class RPC(object): fiat = self.freqtrade.fiat_converter # Prepare data to display profit_closed_coin = round(sum(profit_closed_coin), 8) - profit_closed_percent = round(sum(profit_closed_percent) * 100, 2) + profit_closed_percent = round(nan_to_num(mean(profit_closed_percent)) * 100, 2) profit_closed_fiat = fiat.convert_amount( profit_closed_coin, stake_currency, fiat_display_currency ) profit_all_coin = round(sum(profit_all_coin), 8) - profit_all_percent = round(sum(profit_all_percent) * 100, 2) + profit_all_percent = round(nan_to_num(mean(profit_all_percent)) * 100, 2) profit_all_fiat = fiat.convert_amount( profit_all_coin, stake_currency, From f5fe9a4b1c6ef584b52e46a9525fa2fdfbdc1368 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 7 Jun 2018 20:52:03 +0200 Subject: [PATCH 69/70] fix rpc tests (add a test with multiple trades without this, sum/percentage cannot be properly tested. --- freqtrade/tests/rpc/test_rpc.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 1cf374b6b..a6ac45b58 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -206,15 +206,30 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, trade.close_date = datetime.utcnow() trade.is_open = False + freqtradebot.create_trade() + trade = Trade.query.first() + # Simulate fulfilled LIMIT_BUY order for trade + trade.update(limit_buy_order) + + # Update the ticker with a market going up + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_up + ) + trade.update(limit_sell_order) + trade.close_date = datetime.utcnow() + trade.is_open = False + (error, stats) = rpc.rpc_trade_statistics(stake_currency, fiat_display_currency) assert not error assert prec_satoshi(stats['profit_closed_coin'], 6.217e-05) assert prec_satoshi(stats['profit_closed_percent'], 6.2) assert prec_satoshi(stats['profit_closed_fiat'], 0.93255) - assert prec_satoshi(stats['profit_all_coin'], 6.217e-05) - assert prec_satoshi(stats['profit_all_percent'], 6.2) - assert prec_satoshi(stats['profit_all_fiat'], 0.93255) - assert stats['trade_count'] == 1 + assert prec_satoshi(stats['profit_all_coin'], 5.632e-05) + assert prec_satoshi(stats['profit_all_percent'], 2.81) + assert prec_satoshi(stats['profit_all_fiat'], 0.8448) + assert stats['trade_count'] == 2 assert stats['first_trade_date'] == 'just now' assert stats['latest_trade_date'] == 'just now' assert stats['avg_duration'] == '0:00:00' From d41f71bc34c961fe3075cc77071abc645461db7b Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 7 Jun 2018 21:35:57 +0200 Subject: [PATCH 70/70] handle sqlalchemy NoSuchModuleError --- freqtrade/__init__.py | 3 ++- freqtrade/main.py | 4 ++++ freqtrade/persistence.py | 12 +++++++++++- freqtrade/tests/test_persistence.py | 17 ++++++++++++++--- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 37187a404..7cf0fa996 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -12,7 +12,8 @@ class DependencyException(BaseException): class OperationalException(BaseException): """ Requires manual intervention. - This happens when an exchange returns an unexpected error during runtime. + This happens when an exchange returns an unexpected error during runtime + or given configuration is invalid. """ diff --git a/freqtrade/main.py b/freqtrade/main.py index 973ed031d..81e578810 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -7,6 +7,7 @@ import logging import sys from typing import List +from freqtrade import OperationalException from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.freqtradebot import FreqtradeBot @@ -47,6 +48,9 @@ def main(sysargv: List[str]) -> None: except KeyboardInterrupt: logger.info('SIGINT received, aborting ...') return_code = 0 + except OperationalException as e: + logger.error(str(e)) + return_code = 2 except BaseException: logger.exception('Fatal exception!') finally: diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index ce834bced..7fd8fdeb9 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -11,11 +11,14 @@ import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, create_engine) from sqlalchemy import inspect +from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker from sqlalchemy.pool import StaticPool +from freqtrade import OperationalException + logger = logging.getLogger(__name__) _CONF = {} @@ -43,7 +46,14 @@ def init(config: Dict) -> None: 'echo': False, }) - engine = create_engine(db_url, **kwargs) + try: + engine = create_engine(db_url, **kwargs) + except NoSuchModuleError: + error = 'Given value for db_url: \'{}\' is no valid database URL! (See {}).'.format( + db_url, 'http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls' + ) + raise OperationalException(error) + session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) Trade.session = session() Trade.query = session.query_property() diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index d8d42461b..c50ad7d2c 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -5,7 +5,7 @@ from unittest.mock import MagicMock import pytest from sqlalchemy import create_engine -from freqtrade import constants +from freqtrade import constants, OperationalException from freqtrade.persistence import Trade, init, clean_dry_run_db @@ -27,13 +27,24 @@ def test_init_custom_db_url(default_conf, mocker): conf = deepcopy(default_conf) # Update path to a value other than default, but still in-memory - conf.update({'db_url': 'sqlite:////tmp/freqtrade2_test.sqlite'}) + conf.update({'db_url': 'sqlite:///tmp/freqtrade2_test.sqlite'}) create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) mocker.patch.dict('freqtrade.persistence._CONF', conf) init(conf) assert create_engine_mock.call_count == 1 - assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:////tmp/freqtrade2_test.sqlite' + assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tmp/freqtrade2_test.sqlite' + + +def test_init_invalid_db_url(default_conf, mocker): + conf = deepcopy(default_conf) + + # Update path to a value other than default, but still in-memory + conf.update({'db_url': 'unknown:///some.url'}) + mocker.patch.dict('freqtrade.persistence._CONF', conf) + + with pytest.raises(OperationalException, match=r'.*no valid database URL*'): + init(conf) def test_init_prod_db(default_conf, mocker):