diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 2cbbc59cb..2a6d3f18f 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,6 +1,6 @@ ## Step 1: Have you search for this issue before posting it? -If you have discovered a bug in the bot, please [search our issue tracker](https://github.com/gcarq/freqtrade/issues?q=is%3Aissue). +If you have discovered a bug in the bot, please [search our issue tracker](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue). If it hasn't been reported, please create a new issue. ## Step 2: Describe your environment diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index ae79dd8f0..20ef27f0f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,5 +1,5 @@ Thank you for sending your pull request. But first, have you included -unit tests, and is your code PEP8 conformant? [More details](https://github.com/gcarq/freqtrade/blob/develop/CONTRIBUTING.md) +unit tests, and is your code PEP8 conformant? [More details](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) ## Summary Explain in one sentence the goal of this PR diff --git a/.gitignore b/.gitignore index 219a9fb40..b52a31d8e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ logfile.txt hyperopt_trials.pickle user_data/ freqtrade-plot.html +freqtrade-profit-plot.html # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/.pyup.yml b/.pyup.yml new file mode 100644 index 000000000..a0833af39 --- /dev/null +++ b/.pyup.yml @@ -0,0 +1,4 @@ +# autogenerated pyup.io config file +# see https://pyup.io/docs/configuration/ for all available options + +schedule: every day diff --git a/.travis.yml b/.travis.yml index 1cff5c04b..c3c118654 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,10 +21,10 @@ jobs: - script: pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ - script: - cp config.json.example config.json - - python freqtrade/main.py backtesting + - python freqtrade/main.py --datadir freqtrade/tests/testdata backtesting - script: - cp config.json.example config.json - - python freqtrade/main.py hyperopt -e 5 + - python freqtrade/main.py --datadir freqtrade/tests/testdata hyperopt -e 5 - script: flake8 freqtrade - script: mypy freqtrade after_success: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 93089495b..0cbe9167d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,7 @@ Feel like our bot is missing a feature? We welcome your pull requests! Few point conformant (max-line-length = 100). If you are unsure, discuss the feature on our [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE) -or in a [issue](https://github.com/gcarq/freqtrade/issues) before a PR. +or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. **Before sending the PR:** diff --git a/README.md b/README.md index 3ded0099a..24e01531c 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ # freqtrade -[![Build Status](https://travis-ci.org/gcarq/freqtrade.svg?branch=develop)](https://travis-ci.org/gcarq/freqtrade) -[![Coverage Status](https://coveralls.io/repos/github/gcarq/freqtrade/badge.svg?branch=develop&service=github)](https://coveralls.io/github/gcarq/freqtrade?branch=develop) -[![Maintainability](https://api.codeclimate.com/v1/badges/5737e6d668200b7518ff/maintainability)](https://codeclimate.com/github/gcarq/freqtrade/maintainability) +[![Build Status](https://travis-ci.org/freqtrade/freqtrade.svg?branch=develop)](https://travis-ci.org/freqtrade/freqtrade) +[![Coverage Status](https://coveralls.io/repos/github/freqtrade/freqtrade/badge.svg?branch=develop&service=github)](https://coveralls.io/github/freqtrade/freqtrade?branch=develop) +[![Maintainability](https://api.codeclimate.com/v1/badges/5737e6d668200b7518ff/maintainability)](https://codeclimate.com/github/freqtrade/freqtrade/maintainability) Simple High frequency trading bot for crypto currencies designed to support multi exchanges and be controlled via Telegram. -![freqtrade](https://raw.githubusercontent.com/gcarq/freqtrade/develop/docs/assets/freqtrade-screenshot.png) +![freqtrade](https://raw.githubusercontent.com/freqtrade/freqtrade/develop/docs/assets/freqtrade-screenshot.png) ## Disclaimer This software is for educational purposes only. Do not risk money which @@ -25,12 +25,12 @@ hesitate to read the source code and understand the mechanism of this bot. ## Table of Contents - [Features](#features) - [Quick start](#quick-start) -- [Documentations](https://github.com/gcarq/freqtrade/blob/develop/docs/index.md) - - [Installation](https://github.com/gcarq/freqtrade/blob/develop/docs/installation.md) - - [Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md) - - [Strategy Optimization](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md) - - [Backtesting](https://github.com/gcarq/freqtrade/blob/develop/docs/backtesting.md) - - [Hyperopt](https://github.com/gcarq/freqtrade/blob/develop/docs/hyperopt.md) +- [Documentations](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) + - [Installation](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md) + - [Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md) + - [Strategy Optimization](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md) + - [Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) + - [Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Support](#support) - [Help](#help--slack) - [Bugs](#bugs--issues) @@ -73,7 +73,7 @@ strategy parameters with real exchange data. ## Quick start This quick start section is a very short explanation on how to test the bot in dry-run. We invite you to read the -[bot documentation](https://github.com/gcarq/freqtrade/blob/develop/docs/index.md) +[bot documentation](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) to ensure you understand how the bot is working. ### Easy installation @@ -87,7 +87,7 @@ The following steps are made for Linux/MacOS environment **1. Clone the repo** ```bash -git clone git@github.com:gcarq/freqtrade.git +git clone git@github.com:freqtrade/freqtrade.git git checkout develop cd freqtrade ``` @@ -109,26 +109,26 @@ For any questions not covered by the documentation or for further information about the bot, we encourage you to join our slack channel. - [Click here to join Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE). -### [Bugs / Issues](https://github.com/gcarq/freqtrade/issues?q=is%3Aissue) +### [Bugs / Issues](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) If you discover a bug in the bot, please -[search our issue tracker](https://github.com/gcarq/freqtrade/issues?q=is%3Aissue) +[search our issue tracker](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) first. If it hasn't been reported, please -[create a new issue](https://github.com/gcarq/freqtrade/issues/new) and +[create a new issue](https://github.com/freqtrade/freqtrade/issues/new) and ensure you follow the template guide so that our team can assist you as quickly as possible. -### [Feature Requests](https://github.com/gcarq/freqtrade/labels/enhancement) +### [Feature Requests](https://github.com/freqtrade/freqtrade/labels/enhancement) Have you a great idea to improve the bot you want to share? Please, -first search if this feature was not [already discussed](https://github.com/gcarq/freqtrade/labels/enhancement). +first search if this feature was not [already discussed](https://github.com/freqtrade/freqtrade/labels/enhancement). If it hasn't been requested, please -[create a new request](https://github.com/gcarq/freqtrade/issues/new) +[create a new request](https://github.com/freqtrade/freqtrade/issues/new) and ensure you follow the template guide so that it does not get lost in the bug reports. -### [Pull Requests](https://github.com/gcarq/freqtrade/pulls) +### [Pull Requests](https://github.com/freqtrade/freqtrade/pulls) Feel like our bot is missing a feature? We welcome your pull requests! Please read our -[Contributing document](https://github.com/gcarq/freqtrade/blob/develop/CONTRIBUTING.md) +[Contributing document](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) to understand the requirements before sending your pull-requests. **Important:** Always create your PR against the `develop` branch, not @@ -171,14 +171,14 @@ optional arguments: only if dry_run is enabled. ``` More details on: -- [How to run the bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md#bot-commands) -- [How to use Backtesting](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md#backtesting-commands) -- [How to use Hyperopt](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md#hyperopt-commands) +- [How to run the bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#bot-commands) +- [How to use Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#backtesting-commands) +- [How to use Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#hyperopt-commands) ### Telegram RPC commands Telegram is not mandatory. However, this is a great way to control your bot. More details on our -[documentation](https://github.com/gcarq/freqtrade/blob/develop/docs/index.md) +[documentation](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) - `/start`: Starts the trader - `/stop`: Stops the trader diff --git a/config_full.json.example b/config_full.json.example index 77ef0faa0..c17d22a15 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -45,6 +45,7 @@ "token": "your_telegram_token", "chat_id": "your_telegram_chat_id" }, + "db_url": "sqlite:///tradesv3.sqlite", "initial_state": "running", "internals": { "process_throttle_secs": 5 diff --git a/docs/backtesting.md b/docs/backtesting.md index 8c4c4180d..8364d77e4 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -14,7 +14,7 @@ real data. This is what we call Backtesting will use the crypto-currencies (pair) from your config file and load static tickers located in -[/freqtrade/tests/testdata](https://github.com/gcarq/freqtrade/tree/develop/freqtrade/tests/testdata). +[/freqtrade/tests/testdata](https://github.com/freqtrade/freqtrade/tree/develop/freqtrade/tests/testdata). If the 5 min and 1 min ticker for the crypto-currencies to test is not already in the `testdata` folder, backtesting will download them automatically. Testdata files will not be updated until you specify it. @@ -62,6 +62,12 @@ Where `-s TestStrategy` refers to the class name within the strategy file `test_ python3 ./freqtrade/main.py backtesting --export trades ``` +**Exporting trades to file specifying a custom filename** +```bash +python3 ./freqtrade/main.py backtesting --export trades --export-filename=backtest_teststrategy.json +``` + + **Running backtest with smaller testset** Use the `--timerange` argument to change how much of the testset you want to use. The last N ticks/timeframes will be used. @@ -87,22 +93,31 @@ The full timerange specification: `--timerange=1527595200-1527618600` -**Update testdata directory** -To update your testdata directory, or download into another testdata directory: -```bash -mkdir -p user_data/data/testdata-20180113 -cp freqtrade/tests/testdata/pairs.json user_data/data-20180113 -cd user_data/data-20180113 -``` +**Downloading new set of ticker data** +To download new set of backtesting ticker data, you can use a download script. -Possibly edit pairs.json file to include/exclude pairs +If you are using Binance for example: +- create a folder `user_data/data/binance` and copy `pairs.json` in that folder. +- update the `pairs.json` to contain the currency pairs you are interested in. ```bash -python3 freqtrade/tests/testdata/download_backtest_data.py -p pairs.json +mkdir -p user_data/data/binance +cp freqtrade/tests/testdata/pairs.json user_data/data/binance ``` -The script will read your pairs.json file, and download ticker data -into the current working directory. +Then run: + +```bash +python scripts/download_backtest_data --exchange binance +``` + +This will download ticker data for all the currency pairs you defined in `pairs.json`. + +- To use a different folder than the exchange specific default, use `--export user_data/data/some_directory`. +- To change the exchange used to download the tickers, use `--exchange`. Default is `bittrex`. +- To use `pairs.json` from some other folder, use `--pairs-file some_other_dir/pairs.json`. +- To download ticker data for only 10 days, use `--days 10`. +- Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers. For help about backtesting usage, please refer to @@ -160,4 +175,4 @@ strategies, your configuration, and the crypto-currency you have set up. ## Next step Great, your strategy is profitable. What if the bot can give your the optimal parameters to use for your strategy? -Your next step is to learn [how to find optimal parameters with Hyperopt](https://github.com/gcarq/freqtrade/blob/develop/docs/hyperopt.md) +Your next step is to learn [how to find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index b9ff3fe40..bdb21928e 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -49,7 +49,7 @@ If you want to use a strategy from a different folder you can pass `--strategy-p python3 ./freqtrade/main.py --strategy AwesomeStrategy --strategy-path /some/folder ``` -**For the following section we will use the [user_data/strategies/test_strategy.py](https://github.com/gcarq/freqtrade/blob/develop/user_data/strategies/test_strategy.py) +**For the following section we will use the [user_data/strategies/test_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py) file as reference.** ### Buy strategy @@ -138,15 +138,15 @@ def populate_indicators(dataframe: DataFrame) -> DataFrame: ``` **Want more indicators example?** -Look into the [user_data/strategies/test_strategy.py](https://github.com/gcarq/freqtrade/blob/develop/user_data/strategies/test_strategy.py). +Look into the [user_data/strategies/test_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py). Then uncomment indicators you need. ### Where is the default strategy? The default buy strategy is located in the file -[freqtrade/default_strategy.py](https://github.com/gcarq/freqtrade/blob/develop/freqtrade/strategy/default_strategy.py). +[freqtrade/default_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/strategy/default_strategy.py). ## Next step Now you have a perfect strategy you probably want to backtesting it. -Your next step is to learn [How to use the Backtesting](https://github.com/gcarq/freqtrade/blob/develop/docs/backtesting.md). +Your next step is to learn [How to use the Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md). diff --git a/docs/bot-usage.md b/docs/bot-usage.md index e2c18473c..8079d9816 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -9,10 +9,10 @@ it. ## Bot commands ``` -usage: main.py [-h] [-v] [--version] [-c PATH] [-d PATH] [-s NAME] - [--strategy-path PATH] [--dynamic-whitelist [INT]] - [--dry-run-db] - {backtesting,hyperopt} ... +usage: freqtrade [-h] [-v] [--version] [-c PATH] [-d PATH] [-s NAME] + [--strategy-path PATH] [--dynamic-whitelist [INT]] + [--db-url PATH] + {backtesting,hyperopt} ... Simple High Frequency Trading Bot for crypto currencies @@ -28,17 +28,16 @@ optional arguments: -c PATH, --config PATH specify configuration file (default: config.json) -d PATH, --datadir PATH - path to backtest data (default: - freqtrade/tests/testdata + path to backtest data -s NAME, --strategy NAME specify strategy class name (default: DefaultStrategy) --strategy-path PATH specify additional strategy lookup path --dynamic-whitelist [INT] dynamically generate and update whitelist based on 24h - BaseVolume (Default 20 currencies) - --dry-run-db Force dry run to use a local DB - "tradesv3.dry_run.sqlite" instead of memory DB. Work - only if dry_run is enabled. + BaseVolume (default: 20) + --db-url PATH Override trades database URL, this is useful if + dry_run is enabled or in custom deployments (default: + sqlite:///tradesv3.sqlite) ``` ### How to use a different config file? @@ -68,7 +67,7 @@ python3 ./freqtrade/main.py --strategy AwesomeStrategy If the bot does not find your strategy file, it will display in an error message the reason (File not found, or errors in your code). -Learn more about strategy file in [optimize your bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md). +Learn more about strategy file in [optimize your bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md). ### How to use --strategy-path? This parameter allows you to add an additional strategy lookup path, which gets @@ -102,14 +101,14 @@ python3 ./freqtrade/main.py --dynamic-whitelist 30 negative value (e.g -2), `--dynamic-whitelist` will use the default value (20). -### How to use --dry-run-db? +### How to use --db-url? When you run the bot in Dry-run mode, per default no transactions are stored in a database. If you want to store your bot actions in a DB -using `--dry-run-db`. This command will use a separate database file -`tradesv3.dry_run.sqlite` +using `--db-url`. This can also be used to specify a custom database +in production mode. Example command: ```bash -python3 ./freqtrade/main.py -c config.json --dry-run-db +python3 ./freqtrade/main.py -c config.json --db-url sqlite:///tradesv3.dry_run.sqlite ``` @@ -120,6 +119,8 @@ Backtesting also uses the config specified via `-c/--config`. ``` usage: main.py backtesting [-h] [-i TICKER_INTERVAL] [--realistic-simulation] [--timerange TIMERANGE] [-l] [-r] [--export EXPORT] + [--export-filename EXPORTFILENAME] + optional arguments: -h, --help show this help message and exit @@ -137,6 +138,11 @@ optional arguments: run your backtesting with up-to-date data. --export EXPORT export backtest results, argument are: trades Example --export=trades + --export-filename EXPORTFILENAME + Save backtest results to this filename requires + --export to be set as well Example --export- + filename=backtest_today.json (default: backtest- + result.json ``` ### How to use --refresh-pairs-cached parameter? @@ -182,9 +188,9 @@ optional arguments: ## A parameter missing in the configuration? All parameters for `main.py`, `backtesting`, `hyperopt` are referenced -in [misc.py](https://github.com/gcarq/freqtrade/blob/develop/freqtrade/misc.py#L84) +in [misc.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/misc.py#L84) ## Next step The optimal strategy of the bot will change with time depending of the market trends. The next step is to -[optimize your bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md). +[optimize your bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md). diff --git a/docs/configuration.md b/docs/configuration.md index e68651408..9c76faa28 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -34,13 +34,14 @@ The table below will list all configuration parameters. | `telegram.enabled` | true | Yes | Enable or not the usage of Telegram. | `telegram.token` | token | No | Your Telegram bot token. Only required if `telegram.enabled` is `true`. | `telegram.chat_id` | chat_id | No | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. +| `db_url` | `sqlite:///tradesv3.sqlite` | No | Declares database URL to use. NOTE: This defaults to `sqlite://` if `dry_run` is `True`. | `initial_state` | running | No | Defines the initial application state. More information below. | `strategy` | DefaultStrategy | No | Defines Strategy class to use. | `strategy_path` | null | No | Adds an additional strategy lookup path (must be a folder). | `internals.process_throttle_secs` | 5 | Yes | Set the process throttle. Value in second. The definition of each config parameters is in -[misc.py](https://github.com/gcarq/freqtrade/blob/develop/freqtrade/misc.py#L205). +[misc.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/misc.py#L205). ### Understand stake_amount `stake_amount` is an amount of crypto-currency your bot will use for each trade. @@ -80,6 +81,12 @@ value. This parameter is optional. If you use it, it will take over the Possible values are `running` or `stopped`. (default=`running`) If the value is `stopped` the bot has to be started with `/start` first. +### Understand process_throttle_secs +`process_throttle_secs` is an optional field that defines in seconds how long the bot should wait +before asking the strategy if we should buy or a sell an asset. After each wait period, the strategy is asked again for +every opened trade wether or not we should sell, and for all the remaining pairs (either the dynamic list of pairs or +the static list of pairs) if we should buy. + ### Understand ask_last_balance `ask_last_balance` sets the bidding price. Value `0.0` will use `ask` price, `1.0` will use the `last` price and values between those interpolate between ask and last @@ -99,8 +106,10 @@ The bot was tested with the following exchanges: Feel free to test other exchanges and submit your PR to improve the bot. ### What values for fiat_display_currency? -`fiat_display_currency` set the fiat to use for the conversion form coin to fiat in Telegram. -The valid value are: "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD". +`fiat_display_currency` set the base currency to use for the conversion from coin to fiat in Telegram. +The valid values are: "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD". +In addition to central bank currencies, a range of cryto currencies are supported. +The valid values are: "BTC", "ETH", "XRP", "LTC", "BCH", "USDT". ## Switch to dry-run mode We recommend starting the bot in dry-run mode to see how your bot will @@ -110,9 +119,10 @@ creating trades. ### To switch your bot in Dry-run mode: 1. Edit your `config.json` file -2. Switch dry-run to true +2. Switch dry-run to true and specify db_url for a persistent db ```json "dry_run": true, +"db_url": "sqlite///tradesv3.dryrun.sqlite", ``` 3. Remove your Exchange API key (change them by fake api credentials) @@ -136,7 +146,7 @@ you run it in production mode. ### To switch your bot in production mode: 1. Edit your `config.json` file -2. Switch dry-run to false +2. Switch dry-run to false and don't forget to adapt your database URL if set ```json "dry_run": false, ``` @@ -148,12 +158,11 @@ you run it in production mode. "key": "af8ddd35195e9dc500b9a6f799f6f5c93d89193b", "secret": "08a9dc6db3d7b53e1acebd9275677f4b0a04f1a5", ... -} +} ``` If you have not your Bittrex API key yet, -[see our tutorial](https://github.com/gcarq/freqtrade/blob/develop/docs/pre-requisite.md). - +[see our tutorial](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md). ## Next step Now you have configured your config.json, the next step is to -[start your bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md). +[start your bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md). diff --git a/docs/faq.md b/docs/faq.md index b3f15613a..31a302067 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -27,7 +27,7 @@ like pauses. You can stop your bot, adjust settings and start it again. #### I want to improve the bot with a new strategy That's great. We have a nice backtesting and hyperoptimizing setup. See -the tutorial [here|Testing-new-strategies-with-Hyperopt](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md#hyperopt-commands). +the tutorial [here|Testing-new-strategies-with-Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#hyperopt-commands). #### Is there a setting to only SELL the coins being held and not perform anymore BUYS? diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 0bfa9ae2d..a079e34df 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -14,13 +14,13 @@ parameters with Hyperopt. ## Prepare Hyperopt Before we start digging in Hyperopt, we recommend you to take a look at -your strategy file located into [user_data/strategies/](https://github.com/gcarq/freqtrade/blob/develop/user_data/strategies/test_strategy.py) +your strategy file located into [user_data/strategies/](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py) ### 1. Configure your Guards and Triggers There are two places you need to change in your strategy file to add a new buy strategy for testing: -- Inside [populate_buy_trend()](https://github.com/gcarq/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L278-L294). -- Inside [hyperopt_space()](https://github.com/gcarq/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L244-L297) known as `SPACE`. +- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L278-L294). +- Inside [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L244-L297) known as `SPACE`. There you have two different type of indicators: 1. `guards` and 2. `triggers`. @@ -110,13 +110,13 @@ cannot use your config file. It is also made on purpose to allow you testing your strategy with different configurations. The Hyperopt configuration is located in -[user_data/hyperopt_conf.py](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopt_conf.py). +[user_data/hyperopt_conf.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopt_conf.py). ## Advanced notions ### Understand the Guards and Triggers When you need to add the new guards and triggers to be hyperopt -parameters, you do this by adding them into the [hyperopt_space()](https://github.com/gcarq/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L244-L297). +parameters, you do this by adding them into the [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L244-L297). If it's a trigger, you add one line to the 'trigger' choice group and that's it. @@ -312,4 +312,4 @@ def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: ## Next step Now you have a perfect bot and want to control it from Telegram. Your -next step is to learn the [Telegram usage](https://github.com/gcarq/freqtrade/blob/develop/docs/telegram-usage.md). +next step is to learn the [Telegram usage](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md). diff --git a/docs/index.md b/docs/index.md index ed940d132..afde2d5eb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,27 +6,27 @@ Pull-request. Do not hesitate to reach us on if you do not find the answer to your questions. ## Table of Contents -- [Pre-requisite](https://github.com/gcarq/freqtrade/blob/develop/docs/pre-requisite.md) - - [Setup your Bittrex account](https://github.com/gcarq/freqtrade/blob/develop/docs/pre-requisite.md#setup-your-bittrex-account) - - [Setup your Telegram bot](https://github.com/gcarq/freqtrade/blob/develop/docs/pre-requisite.md#setup-your-telegram-bot) -- [Bot Installation](https://github.com/gcarq/freqtrade/blob/develop/docs/installation.md) - - [Install with Docker (all platforms)](https://github.com/gcarq/freqtrade/blob/develop/docs/installation.md#docker) - - [Install on Linux Ubuntu](https://github.com/gcarq/freqtrade/blob/develop/docs/installation.md#21-linux---ubuntu-1604) - - [Install on MacOS](https://github.com/gcarq/freqtrade/blob/develop/docs/installation.md#23-macos-installation) - - [Install on Windows](https://github.com/gcarq/freqtrade/blob/develop/docs/installation.md#windows) -- [Bot Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md) -- [Bot usage (Start your bot)](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md) - - [Bot commands](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md#bot-commands) - - [Backtesting commands](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md#backtesting-commands) - - [Hyperopt commands](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md#hyperopt-commands) -- [Bot Optimization](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md) - - [Change your strategy](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md#change-your-strategy) - - [Add more Indicator](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md#add-more-indicator) - - [Test your strategy with Backtesting](https://github.com/gcarq/freqtrade/blob/develop/docs/backtesting.md) - - [Find optimal parameters with Hyperopt](https://github.com/gcarq/freqtrade/blob/develop/docs/hyperopt.md) -- [Control the bot with telegram](https://github.com/gcarq/freqtrade/blob/develop/docs/telegram-usage.md) -- [Contribute to the project](https://github.com/gcarq/freqtrade/blob/develop/CONTRIBUTING.md) - - [How to contribute](https://github.com/gcarq/freqtrade/blob/develop/CONTRIBUTING.md) - - [Run tests & Check PEP8 compliance](https://github.com/gcarq/freqtrade/blob/develop/CONTRIBUTING.md) -- [FAQ](https://github.com/gcarq/freqtrade/blob/develop/docs/faq.md) - - [SQL cheatsheet](https://github.com/gcarq/freqtrade/blob/develop/docs/sql_cheatsheet.md) \ No newline at end of file +- [Pre-requisite](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md) + - [Setup your Bittrex account](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md#setup-your-bittrex-account) + - [Setup your Telegram bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md#setup-your-telegram-bot) +- [Bot Installation](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md) + - [Install with Docker (all platforms)](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md#docker) + - [Install on Linux Ubuntu](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md#21-linux---ubuntu-1604) + - [Install on MacOS](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md#23-macos-installation) + - [Install on Windows](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md#windows) +- [Bot Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md) +- [Bot usage (Start your bot)](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md) + - [Bot commands](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#bot-commands) + - [Backtesting commands](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#backtesting-commands) + - [Hyperopt commands](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#hyperopt-commands) +- [Bot Optimization](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md) + - [Change your strategy](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#change-your-strategy) + - [Add more Indicator](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#add-more-indicator) + - [Test your strategy with Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) + - [Find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) +- [Control the bot with telegram](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md) +- [Contribute to the project](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) + - [How to contribute](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) + - [Run tests & Check PEP8 compliance](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) +- [FAQ](https://github.com/freqtrade/freqtrade/blob/develop/docs/faq.md) + - [SQL cheatsheet](https://github.com/freqtrade/freqtrade/blob/develop/docs/sql_cheatsheet.md) diff --git a/docs/installation.md b/docs/installation.md index be8e2e501..2fd40e451 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -2,7 +2,7 @@ This page explains how to prepare your environment for running the bot. -To understand how to set up the bot please read the [Bot Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md) page. +To understand how to set up the bot please read the [Bot Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md) page. ## Table of Contents @@ -69,7 +69,7 @@ Once you have Docker installed, simply create the config file (e.g. `config.json #### 1.1. Clone the git repository ```bash -git clone https://github.com/gcarq/freqtrade.git +git clone https://github.com/freqtrade/freqtrade.git ``` #### 1.2. (Optional) Checkout the develop branch @@ -90,7 +90,7 @@ cd freqtrade cp -n config.json.example config.json ``` -> To edit the config please refer to the [Bot Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md) page. +> To edit the config please refer to the [Bot Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md) page. #### 1.5. Create your database file *(optional - the bot will create it if it is missing)* @@ -162,10 +162,10 @@ docker run -d \ -v /etc/localtime:/etc/localtime:ro \ -v ~/.freqtrade/config.json:/freqtrade/config.json \ -v ~/.freqtrade/tradesv3.sqlite:/freqtrade/tradesv3.sqlite \ - freqtrade + freqtrade --db-url sqlite:///tradesv3.sqlite ``` - -If you are using `dry_run=True` it's not necessary to mount `tradesv3.sqlite`, but you can mount `tradesv3.dryrun.sqlite` if you plan to use the dry run mode with the param `--dry-run-db`. +NOTE: db-url defaults to `sqlite:///tradesv3.sqlite` but it defaults to `sqlite://` if `dry_run=True` is being used. +To override this behaviour use a custom db-url value: i.e.: `--db-url sqlite:///tradesv3.dryrun.sqlite` ### 6. Monitor your Docker instance @@ -237,7 +237,7 @@ sudo apt-get install mongodb-org Clone the git repository: ```bash -git clone https://github.com/gcarq/freqtrade.git +git clone https://github.com/freqtrade/freqtrade.git ``` Optionally checkout the develop branch: @@ -287,7 +287,7 @@ export PATH=/env/mongodb/bin:$PATH Clone the git repository: ```bash -git clone https://github.com/gcarq/freqtrade.git +git clone https://github.com/freqtrade/freqtrade.git ``` Optionally checkout the develop branch: @@ -306,7 +306,7 @@ cd freqtrade cp config.json.example config.json ``` -> *To edit the config please refer to [Bot Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md).* +> *To edit the config please refer to [Bot Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md).* #### 2. Setup your Python virtual environment (virtualenv) @@ -350,8 +350,8 @@ copy paste `config.json` to ``\path\freqtrade-develop\freqtrade` >python main.py ``` -> Thanks [Owdr](https://github.com/Owdr) for the commands. Source: [Issue #222](https://github.com/gcarq/freqtrade/issues/222) +> Thanks [Owdr](https://github.com/Owdr) for the commands. Source: [Issue #222](https://github.com/freqtrade/freqtrade/issues/222) Now you have an environment ready, the next step is -[Bot Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md)... +[Bot Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md)... diff --git a/docs/plotting.md b/docs/plotting.md index ae964fb16..242d80005 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -48,6 +48,12 @@ To plot trades stored in a database use `--db-url` argument: python scripts/plot_dataframe.py --db-url tradesv3.dry_run.sqlite -p BTC_ETH ``` +To plot a test strategy the strategy should have first be backtested. +The results may then be plotted with the -s argument: +``` +python scripts/plot_dataframe.py -s Strategy_Name -p BTC/ETH --datadir user_data/data// +``` + ## Plot profit The profit plotter show a picture with three plots: diff --git a/docs/sql_cheatsheet.md b/docs/sql_cheatsheet.md index 141eaeafe..ba26f1707 100644 --- a/docs/sql_cheatsheet.md +++ b/docs/sql_cheatsheet.md @@ -85,7 +85,7 @@ INSERT INTO trades (exchange, pair, is_open, fee_open, fee_close, open_rate, sta ## Fix wrong fees in the table If your DB was created before -[PR#200](https://github.com/gcarq/freqtrade/pull/200) was merged +[PR#200](https://github.com/freqtrade/freqtrade/pull/200) was merged (before 12/23/17). ```sql diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 20e80269b..4f878dbd1 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -4,7 +4,7 @@ This page explains how to command your bot with Telegram. ## Pre-requisite To control your bot with Telegram, you need first to -[set up a Telegram bot](https://github.com/gcarq/freqtrade/blob/develop/docs/pre-requisite.md) +[set up a Telegram bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md) and add your Telegram API keys into your config file. ## Telegram commands diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 37187a404..7cf0fa996 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -12,7 +12,8 @@ class DependencyException(BaseException): class OperationalException(BaseException): """ Requires manual intervention. - This happens when an exchange returns an unexpected error during runtime. + This happens when an exchange returns an unexpected error during runtime + or given configuration is invalid. """ diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 1ae805ca5..cd9837d43 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -2,16 +2,28 @@ This module contains the argument manager class """ +import os import argparse import logging -import os import re import arrow -from typing import List, Tuple, Optional +from typing import List, Optional, NamedTuple from freqtrade import __version__, constants +class TimeRange(NamedTuple): + """ + NamedTuple Defining timerange inputs. + [start/stop]type defines if [start/stop]ts shall be used. + if *type is none, don't use corresponding startvalue. + """ + starttype: Optional[str] = None + stoptype: Optional[str] = None + startts: int = 0 + stopts: int = 0 + + class Arguments(object): """ Arguments Class. Manage the arguments received by the cli @@ -72,9 +84,9 @@ class Arguments(object): ) self.parser.add_argument( '-d', '--datadir', - help='path to backtest data (default: %(default)s', + help='path to backtest data', dest='datadir', - default=os.path.join('freqtrade', 'tests', 'testdata'), + default=None, type=str, metavar='PATH', ) @@ -95,8 +107,8 @@ class Arguments(object): ) self.parser.add_argument( '--dynamic-whitelist', - help='dynamically generate and update whitelist \ - based on 24h BaseVolume (Default 20 currencies)', # noqa + help='dynamically generate and update whitelist' + ' based on 24h BaseVolume (default: %(const)s)', dest='dynamic_whitelist', const=constants.DYNAMIC_WHITELIST, type=int, @@ -104,11 +116,13 @@ class Arguments(object): nargs='?', ) self.parser.add_argument( - '--dry-run-db', - help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" \ - instead of memory DB. Work only if dry_run is enabled.', - action='store_true', - dest='dry_run_db', + '--db-url', + help='Override trades database URL, this is useful if dry_run is enabled' + ' or in custom deployments (default: %(default)s)', + dest='db_url', + default=constants.DEFAULT_DB_PROD_URL, + type=str, + metavar='PATH', ) @staticmethod @@ -124,8 +138,8 @@ class Arguments(object): ) parser.add_argument( '-r', '--refresh-pairs-cached', - help='refresh the pairs files in tests/testdata with the latest data from the exchange. \ - Use it if you want to run your backtesting with up-to-date data.', + help='refresh the pairs files in tests/testdata with the latest data from the ' + 'exchange. Use it if you want to run your backtesting with up-to-date data.', action='store_true', dest='refresh_pairs', ) @@ -137,6 +151,17 @@ class Arguments(object): default=None, dest='export', ) + parser.add_argument( + '--export-filename', + help='Save backtest results to this filename \ + requires --export to be set as well\ + Example --export-filename=user_data/backtest_data/backtest_today.json\ + (default: %(default)s)', + type=str, + default=os.path.join('user_data', 'backtest_data', 'backtest-result.json'), + dest='exportfilename', + metavar='PATH', + ) @staticmethod def optimizer_shared_options(parser: argparse.ArgumentParser) -> None: @@ -211,15 +236,14 @@ class Arguments(object): self.hyperopt_options(hyperopt_cmd) @staticmethod - def parse_timerange(text: Optional[str]) -> Optional[Tuple[Tuple, - Optional[int], Optional[int]]]: + def parse_timerange(text: Optional[str]) -> TimeRange: """ Parse the value of the argument --timerange to determine what is the range desired :param text: value from --timerange :return: Start and End range period """ if text is None: - return None + return TimeRange() syntax = [(r'^-(\d{8})$', (None, 'date')), (r'^(\d{8})-$', ('date', None)), (r'^(\d{8})-(\d{8})$', ('date', 'date')), @@ -235,8 +259,8 @@ class Arguments(object): if match: # Regex has matched rvals = match.groups() index = 0 - start: Optional[int] = None - stop: Optional[int] = None + start: int = 0 + stop: int = 0 if stype[0]: starts = rvals[index] if stype[0] == 'date' and len(starts) == 8: @@ -250,7 +274,7 @@ class Arguments(object): stop = arrow.get(stops, 'YYYYMMDD').timestamp else: stop = int(stops) - return stype, start, stop + return TimeRange(stype[0], stype[1], start, stop) raise Exception('Incorrect syntax for timerange "%s"' % text) def scripts_options(self) -> None: @@ -264,13 +288,6 @@ class Arguments(object): default=None ) - self.parser.add_argument( - '-db', '--db-url', - help='Show trades stored in database.', - dest='db_url', - default=None - ) - def testdata_dl_options(self) -> None: """ Parses given arguments for testdata download @@ -297,7 +314,18 @@ class Arguments(object): self.parser.add_argument( '--exchange', - help='Exchange name', + help='Exchange name (default: %(default)s)', dest='exchange', type=str, default='bittrex') + + self.parser.add_argument( + '-t', '--timeframes', + help='Specify which tickers to download. Space separated list. \ + Default: %(default)s', + choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', + '6h', '8h', '12h', '1d', '3d', '1w'], + default=['1m', '5m'], + nargs='+', + dest='timeframes', + ) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index a800bde78..2a9e8fbd8 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -1,7 +1,7 @@ """ This module contains the configuration class """ - +import os import json import logging from argparse import Namespace @@ -97,22 +97,35 @@ class Configuration(object): '(not applicable with Backtesting and Hyperopt)' ) - # Add dry_run_db if found and the bot in dry run - if self.args.dry_run_db and config.get('dry_run', False): - config.update({'dry_run_db': True}) - logger.info('Parameter --dry-run-db detected ...') + if self.args.db_url != constants.DEFAULT_DB_PROD_URL: + config.update({'db_url': self.args.db_url}) + logger.info('Parameter --db-url detected ...') - if config.get('dry_run_db', False): - if config.get('dry_run', False): - logger.info('Dry_run will use the DB file: "tradesv3.dry_run.sqlite"') - else: - logger.info('Dry run is disabled. (--dry_run_db ignored)') + if config.get('dry_run', False): + logger.info('Dry run is enabled') + if config.get('db_url') in [None, constants.DEFAULT_DB_PROD_URL]: + # Default to in-memory db for dry_run if not specified + config['db_url'] = constants.DEFAULT_DB_DRYRUN_URL + else: + if not config.get('db_url', None): + config['db_url'] = constants.DEFAULT_DB_PROD_URL + logger.info('Dry run is disabled') + + logger.info('Using DB: "{}"'.format(config['db_url'])) # Check if the exchange set by the user is supported self.check_exchange(config) return config + def _create_default_datadir(self, config: Dict[str, Any]) -> str: + exchange_name = config.get('exchange', {}).get('name').lower() + default_path = os.path.join('user_data', 'data', exchange_name) + if not os.path.isdir(default_path): + os.makedirs(default_path) + logger.info(f'Created data directory: {default_path}') + return default_path + def _load_backtesting_config(self, config: Dict[str, Any]) -> Dict[str, Any]: """ Extract information for sys.argv and load Backtesting configuration @@ -145,7 +158,9 @@ class Configuration(object): # If --datadir is used we add it to the configuration if 'datadir' in self.args and self.args.datadir: config.update({'datadir': self.args.datadir}) - logger.info('Using data folder: %s ...', self.args.datadir) + else: + config.update({'datadir': self._create_default_datadir(config)}) + logger.info('Using data folder: %s ...', config.get('datadir')) # If -r/--refresh-pairs-cached is used we add it to the configuration if 'refresh_pairs' in self.args and self.args.refresh_pairs: @@ -157,6 +172,11 @@ class Configuration(object): config.update({'export': self.args.export}) logger.info('Parameter --export detected: %s ...', self.args.export) + # If --export-filename is used we add it to the configuration + if 'export' in config and 'exportfilename' in self.args and self.args.exportfilename: + config.update({'exportfilename': self.args.exportfilename}) + logger.info('Storing backtest results to %s ...', self.args.exportfilename) + return config def _load_hyperopt_config(self, config: Dict[str, Any]) -> Dict[str, Any]: diff --git a/freqtrade/constants.py b/freqtrade/constants.py index cb32cf930..30d7111dc 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -9,10 +9,14 @@ TICKER_INTERVAL = 5 # min HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec DEFAULT_STRATEGY = 'DefaultStrategy' +DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' +DEFAULT_DB_DRYRUN_URL = 'sqlite://' UNLIMITED_STAKE_AMOUNT = 'unlimited' + TICKER_INTERVAL_MINUTES = { '1m': 1, + '3m': 3, '5m': 5, '15m': 15, '30m': 30, @@ -20,8 +24,10 @@ TICKER_INTERVAL_MINUTES = { '2h': 120, '4h': 240, '6h': 360, + '8h': 480, '12h': 720, '1d': 1440, + '3d': 4320, '1w': 10080, } @@ -29,7 +35,8 @@ SUPPORTED_FIAT = [ "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", - "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD" + "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD", + "BTC", "ETH", "XRP", "LTC", "BCH", "USDT" ] # Required json-schema for user specified config @@ -84,6 +91,7 @@ CONF_SCHEMA = { }, 'required': ['enabled', 'token', 'chat_id'] }, + 'db_url': {'type': 'string'}, 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, 'internals': { 'type': 'object', diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 3c67db71f..5e768518b 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -18,6 +18,8 @@ _API: ccxt.Exchange = None _CONF: Dict = {} API_RETRY_COUNT = 4 +_CACHED_TICKER: Dict[str, Any] = {} + # Holds all open sell orders for dry_run _DRY_RUN_OPEN_ORDERS: Dict[str, Any] = {} @@ -264,17 +266,29 @@ def get_tickers() -> Dict: raise OperationalException(e) -# TODO: remove refresh argument, keeping it to keep track of where it was intended to be used @retrier def get_ticker(pair: str, refresh: Optional[bool] = True) -> dict: - try: - return _API.fetch_ticker(pair) - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - 'Could not load ticker history due to {}. Message: {}'.format( - e.__class__.__name__, e)) - except ccxt.BaseError as e: - raise OperationalException(e) + global _CACHED_TICKER + if refresh or pair not in _CACHED_TICKER.keys(): + try: + data = _API.fetch_ticker(pair) + try: + _CACHED_TICKER[pair] = { + 'bid': float(data['bid']), + 'ask': float(data['ask']), + } + except KeyError as e: + logger.debug("Could not cache ticker data for %s", pair) + return data + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + 'Could not load ticker history due to {}. Message: {}'.format( + e.__class__.__name__, e)) + except ccxt.BaseError as e: + raise OperationalException(e) + else: + logger.info("returning cached ticker-data for %s", pair) + return _CACHED_TICKER[pair] @retrier diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 49c179907..9392f5b39 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -33,12 +33,11 @@ class FreqtradeBot(object): This is from here the bot start its logic. """ - def __init__(self, config: Dict[str, Any], db_url: Optional[str] = None)-> None: + def __init__(self, config: Dict[str, Any])-> None: """ Init all variables and object the bot need to work :param config: configuration dict, you can use the Configuration.get_config() method to get the config dict. - :param db_url: database connector string for sqlalchemy (Optional) """ logger.info( @@ -57,17 +56,16 @@ class FreqtradeBot(object): self.persistence = None self.exchange = None - self._init_modules(db_url=db_url) + self._init_modules() - def _init_modules(self, db_url: Optional[str] = None) -> None: + def _init_modules(self) -> None: """ Initializes all modules and updates the config - :param db_url: database connector string for sqlalchemy (Optional) :return: None """ # Initialize all modules - persistence.init(self.config, db_url) + persistence.init(self.config) exchange.init(self.config) # Set initial application state diff --git a/freqtrade/main.py b/freqtrade/main.py index 973ed031d..81e578810 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -7,6 +7,7 @@ import logging import sys from typing import List +from freqtrade import OperationalException from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.freqtradebot import FreqtradeBot @@ -47,6 +48,9 @@ def main(sysargv: List[str]) -> None: except KeyboardInterrupt: logger.info('SIGINT received, aborting ...') return_code = 0 + except OperationalException as e: + logger.error(str(e)) + return_code = 2 except BaseException: logger.exception('Fatal exception!') finally: diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 711adfd28..00f05cc46 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -9,39 +9,40 @@ import arrow from freqtrade import misc, constants from freqtrade.exchange import get_ticker_history +from freqtrade.arguments import TimeRange from user_data.hyperopt_conf import hyperopt_optimize_conf logger = logging.getLogger(__name__) -def trim_tickerlist(tickerlist: List[Dict], timerange: Tuple[Tuple, int, int]) -> List[Dict]: +def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: if not tickerlist: return tickerlist - stype, start, stop = timerange - start_index = 0 stop_index = len(tickerlist) - if stype[0] == 'line': - stop_index = start - if stype[0] == 'index': - start_index = start - elif stype[0] == 'date': - while start_index < len(tickerlist) and tickerlist[start_index][0] < start * 1000: + if timerange.starttype == 'line': + stop_index = timerange.startts + if timerange.starttype == 'index': + start_index = timerange.startts + elif timerange.starttype == 'date': + while (start_index < len(tickerlist) and + tickerlist[start_index][0] < timerange.startts * 1000): start_index += 1 - if stype[1] == 'line': - start_index = len(tickerlist) + stop - if stype[1] == 'index': - stop_index = stop - elif stype[1] == 'date': - while stop_index > 0 and tickerlist[stop_index-1][0] > stop * 1000: + if timerange.stoptype == 'line': + start_index = len(tickerlist) + timerange.stopts + if timerange.stoptype == 'index': + stop_index = timerange.stopts + elif timerange.stoptype == 'date': + while (stop_index > 0 and + tickerlist[stop_index-1][0] > timerange.stopts * 1000): stop_index -= 1 if start_index > stop_index: - raise ValueError(f'The timerange [{start},{stop}] is incorrect') + raise ValueError(f'The timerange [{timerange.startts},{timerange.stopts}] is incorrect') return tickerlist[start_index:stop_index] @@ -49,7 +50,7 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: Tuple[Tuple, int, int]) - def load_tickerdata_file( datadir: str, pair: str, ticker_interval: str, - timerange: Optional[Tuple[Tuple, int, int]] = None) -> Optional[List[Dict]]: + timerange: Optional[TimeRange] = None) -> Optional[List[Dict]]: """ Load a pair from file, :return dict OR empty if unsuccesful @@ -84,7 +85,7 @@ def load_data(datadir: str, ticker_interval: str, pairs: Optional[List[str]] = None, refresh_pairs: Optional[bool] = False, - timerange: Optional[Tuple[Tuple, int, int]] = None) -> Dict[str, List]: + timerange: TimeRange = TimeRange()) -> Dict[str, List]: """ Loads ticker history data for the given parameters :return: dict @@ -124,7 +125,7 @@ def make_testdata_path(datadir: str) -> str: def download_pairs(datadir, pairs: List[str], ticker_interval: str, - timerange: Optional[Tuple[Tuple, int, int]] = None) -> bool: + timerange: TimeRange = TimeRange()) -> bool: """For each pairs passed in parameters, download the ticker intervals""" for pair in pairs: try: @@ -144,7 +145,7 @@ def download_pairs(datadir, pairs: List[str], def load_cached_data_for_updating(filename: str, tick_interval: str, - timerange: Optional[Tuple[Tuple, int, int]]) -> Tuple[ + timerange: Optional[TimeRange]) -> Tuple[ List[Any], Optional[int]]: """ @@ -155,10 +156,10 @@ def load_cached_data_for_updating(filename: str, # user sets timerange, so find the start time if timerange: - if timerange[0][0] == 'date': - since_ms = timerange[1] * 1000 - elif timerange[0][1] == 'line': - num_minutes = timerange[2] * constants.TICKER_INTERVAL_MINUTES[tick_interval] + if timerange.starttype == 'date': + since_ms = timerange.startts * 1000 + elif timerange.stoptype == 'line': + num_minutes = timerange.stopts * constants.TICKER_INTERVAL_MINUTES[tick_interval] since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000 # read the cached file @@ -188,7 +189,7 @@ def load_cached_data_for_updating(filename: str, def download_backtesting_testdata(datadir: str, pair: str, tick_interval: str = '5m', - timerange: Optional[Tuple[Tuple, int, int]] = None) -> None: + timerange: Optional[TimeRange] = None) -> None: """ Download the latest ticker intervals from the exchange for the pairs passed in parameters diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d34430a5c..095f214f5 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -154,13 +154,22 @@ class Backtesting(object): max_open_trades = args.get('max_open_trades', 0) realistic = args.get('realistic', False) record = args.get('record', None) + recordfilename = args.get('recordfn', 'backtest-result.json') records = [] trades = [] trade_count_lock: Dict = {} for pair, pair_data in processed.items(): pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run - ticker_data = self.populate_sell_trend(self.populate_buy_trend(pair_data))[headers] + ticker_data = self.populate_sell_trend( + self.populate_buy_trend(pair_data))[headers].copy() + + # to avoid using data from future, we buy/sell with signal from previous candle + ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) + ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) + + ticker_data.drop(ticker_data.head(1).index, inplace=True) + ticker = [x for x in ticker_data.itertuples()] lock_pair_until = None @@ -196,8 +205,8 @@ class Backtesting(object): # For now export inside backtest(), maybe change so that backtest() # returns a tuple like: (dataframe, records, logs, etc) if record and record.find('trades') >= 0: - logger.info('Dumping backtest results') - file_dump_json('backtest-result.json', records) + logger.info('Dumping backtest results to %s', recordfilename) + file_dump_json(recordfilename, records) labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] return DataFrame.from_records(trades, columns=labels) @@ -220,7 +229,7 @@ class Backtesting(object): timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - data = optimize.load_data( # type: ignore # timerange will be refactored + data = optimize.load_data( self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, @@ -257,7 +266,8 @@ class Backtesting(object): 'realistic': self.config.get('realistic_simulation', False), 'sell_profit_only': sell_profit_only, 'use_sell_signal': use_sell_signal, - 'record': self.config.get('export') + 'record': self.config.get('export'), + 'recordfn': self.config.get('exportfilename'), } ) logger.info( diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 74b39b445..878acc2dc 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -497,7 +497,7 @@ class Hyperopt(Backtesting): def start(self) -> None: timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - data = load_data( # type: ignore # timerange will be refactored + data = load_data( datadir=str(self.config.get('datadir')), pairs=self.config['exchange']['pair_whitelist'], ticker_interval=self.ticker_interval, diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index f9a7d1e3c..7fd8fdeb9 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -10,13 +10,14 @@ from typing import Dict, Optional, Any import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, create_engine) -from sqlalchemy.engine import Engine +from sqlalchemy import inspect +from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker from sqlalchemy.pool import StaticPool -from sqlalchemy import inspect +from freqtrade import OperationalException logger = logging.getLogger(__name__) @@ -24,29 +25,34 @@ _CONF = {} _DECL_BASE: Any = declarative_base() -def init(config: dict, engine: Optional[Engine] = None) -> None: +def init(config: Dict) -> None: """ Initializes this module with the given config, registers all known command handlers and starts polling for message updates :param config: config to use - :param engine: database engine for sqlalchemy (Optional) :return: None """ _CONF.update(config) - if not engine: - if _CONF.get('dry_run', False): - # the user wants dry run to use a DB - if _CONF.get('dry_run_db', False): - engine = create_engine('sqlite:///tradesv3.dry_run.sqlite') - # Otherwise dry run will store in memory - else: - engine = create_engine('sqlite://', - connect_args={'check_same_thread': False}, - poolclass=StaticPool, - echo=False) - else: - engine = create_engine('sqlite:///tradesv3.sqlite') + + db_url = _CONF.get('db_url', None) + kwargs = {} + + # Take care of thread ownership if in-memory db + if db_url == 'sqlite://': + kwargs.update({ + 'connect_args': {'check_same_thread': False}, + 'poolclass': StaticPool, + 'echo': False, + }) + + try: + engine = create_engine(db_url, **kwargs) + except NoSuchModuleError: + error = 'Given value for db_url: \'{}\' is no valid database URL! (See {}).'.format( + db_url, 'http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls' + ) + raise OperationalException(error) session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) Trade.session = session() @@ -54,8 +60,8 @@ def init(config: dict, engine: Optional[Engine] = None) -> None: _DECL_BASE.metadata.create_all(engine) check_migrate(engine) - # Clean dry_run DB - if _CONF.get('dry_run', False) and _CONF.get('dry_run_db', False): + # Clean dry_run DB if the db is not in-memory + if _CONF.get('dry_run', False) and db_url != 'sqlite://': clean_dry_run_db() diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index f48666748..c2f097319 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -9,6 +9,7 @@ from typing import Dict, Tuple, Any import arrow import sqlalchemy as sql from pandas import DataFrame +from numpy import mean, nan_to_num from freqtrade import exchange from freqtrade.misc import shorten_date @@ -209,14 +210,14 @@ class RPC(object): fiat = self.freqtrade.fiat_converter # Prepare data to display profit_closed_coin = round(sum(profit_closed_coin), 8) - profit_closed_percent = round(sum(profit_closed_percent) * 100, 2) + profit_closed_percent = round(nan_to_num(mean(profit_closed_percent)) * 100, 2) profit_closed_fiat = fiat.convert_amount( profit_closed_coin, stake_currency, fiat_display_currency ) profit_all_coin = round(sum(profit_all_coin), 8) - profit_all_percent = round(sum(profit_all_percent) * 100, 2) + profit_all_percent = round(nan_to_num(mean(profit_all_percent)) * 100, 2) profit_all_fiat = fiat.convert_amount( profit_all_coin, stake_currency, diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 5d6195a2f..1311687b7 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -9,7 +9,6 @@ from unittest.mock import MagicMock import arrow import pytest from jsonschema import validate -from sqlalchemy import create_engine from telegram import Chat, Message, Update from freqtrade.analyze import Analyze @@ -45,7 +44,7 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) mocker.patch('freqtrade.freqtradebot.Analyze.get_signal', MagicMock()) - return FreqtradeBot(config, create_engine('sqlite://')) + return FreqtradeBot(config) def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]] = None) -> None: @@ -108,7 +107,8 @@ def default_conf(): "chat_id": "0" }, "initial_state": "running", - "loglevel": logging.DEBUG + "db_url": "sqlite://", + "loglevel": logging.DEBUG, } validate(configuration, constants.CONF_SCHEMA) return configuration diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 56812c75e..97a723929 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -310,9 +310,19 @@ def test_get_ticker(default_conf, mocker): # if not fetching a new result we should get the cached ticker ticker = get_ticker(pair='ETH/BTC') + assert api_mock.fetch_ticker.call_count == 1 assert ticker['bid'] == 0.5 assert ticker['ask'] == 1 + assert 'ETH/BTC' in exchange._CACHED_TICKER + assert exchange._CACHED_TICKER['ETH/BTC']['bid'] == 0.5 + assert exchange._CACHED_TICKER['ETH/BTC']['ask'] == 1 + + # Test caching + api_mock.fetch_ticker = MagicMock() + get_ticker(pair='ETH/BTC', refresh=False) + assert api_mock.fetch_ticker.call_count == 0 + with pytest.raises(TemporaryError): # test retrier api_mock.fetch_ticker = MagicMock(side_effect=ccxt.NetworkError) mocker.patch('freqtrade.exchange._API', api_mock) @@ -323,6 +333,10 @@ def test_get_ticker(default_conf, mocker): mocker.patch('freqtrade.exchange._API', api_mock) get_ticker(pair='ETH/BTC', refresh=True) + api_mock.fetch_ticker = MagicMock(return_value={}) + mocker.patch('freqtrade.exchange._API', api_mock) + get_ticker(pair='ETH/BTC', refresh=True) + def make_fetch_ohlcv_mock(data): def fetch_ohlcv_mock(pair, timeframe, since): diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 37c15f438..283e88812 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -14,7 +14,7 @@ from arrow import Arrow from freqtrade import optimize, constants, DependencyException from freqtrade.analyze import Analyze -from freqtrade.arguments import Arguments +from freqtrade.arguments import Arguments, TimeRange from freqtrade.optimize.backtesting import Backtesting, start, setup_configuration from freqtrade.tests.conftest import log_has @@ -31,7 +31,7 @@ def trim_dictlist(dict_list, num): def load_data_test(what): - timerange = ((None, 'line'), None, -100) + timerange = TimeRange(None, 'line', 0, -101) data = optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC'], timerange=timerange) pair = data['UNITTEST/BTC'] @@ -111,14 +111,14 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals # use for mock freqtrade.exchange.get_ticker_history' def _load_pair_as_ticks(pair, tickfreq): ticks = optimize.load_data(None, ticker_interval=tickfreq, pairs=[pair]) - ticks = trim_dictlist(ticks, -200) + ticks = trim_dictlist(ticks, -201) return ticks[pair] # FIX: fixturize this? def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None): data = optimize.load_data(None, ticker_interval='8m', pairs=[pair]) - data = trim_dictlist(data, -200) + data = trim_dictlist(data, -201) mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) backtesting = Backtesting(conf) return { @@ -219,7 +219,8 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non '--realistic-simulation', '--refresh-pairs-cached', '--timerange', ':100', - '--export', '/bar/foo' + '--export', '/bar/foo', + '--export-filename', 'foo_bar.json' ] config = setup_configuration(get_args(args)) @@ -260,6 +261,11 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non 'Parameter --export detected: {} ...'.format(config['export']), caplog.record_tuples ) + assert 'exportfilename' in config + assert log_has( + 'Storing backtest results to {} ...'.format(config['exportfilename']), + caplog.record_tuples + ) def test_setup_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None: @@ -328,7 +334,7 @@ def test_tickerdata_to_dataframe(default_conf, mocker) -> None: Test Backtesting.tickerdata_to_dataframe() method """ mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True)) - timerange = ((None, 'line'), None, -100) + timerange = TimeRange(None, 'line', 0, -100) tick = optimize.load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) tickerlist = {'UNITTEST/BTC': tick} diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 349fa3be3..3f358cfb8 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -11,6 +11,7 @@ from freqtrade.misc import file_dump_json from freqtrade.optimize.__init__ import make_testdata_path, download_pairs, \ download_backtesting_testdata, load_tickerdata_file, trim_tickerlist, \ load_cached_data_for_updating +from freqtrade.arguments import TimeRange from freqtrade.tests.conftest import log_has # Change this if modifying UNITTEST/BTC testdatafile @@ -176,7 +177,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # timeframe starts earlier than the cached data # should fully update data - timerange = (('date', None), test_data[0][0] / 1000 - 1, None) + timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0) data, start_ts = load_cached_data_for_updating(test_filename, '1m', timerange) @@ -187,13 +188,13 @@ def test_load_cached_data_for_updating(mocker) -> None: num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 120 data, start_ts = load_cached_data_for_updating(test_filename, '1m', - ((None, 'line'), None, -num_lines)) + TimeRange(None, 'line', 0, -num_lines)) assert data == [] assert start_ts < test_data[0][0] - 1 # timeframe starts in the center of the cached data # should return the chached data w/o the last item - timerange = (('date', None), test_data[0][0] / 1000 + 1, None) + timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0) data, start_ts = load_cached_data_for_updating(test_filename, '1m', timerange) @@ -202,7 +203,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # same with 'line' timeframe num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 30 - timerange = ((None, 'line'), None, -num_lines) + timerange = TimeRange(None, 'line', 0, -num_lines) data, start_ts = load_cached_data_for_updating(test_filename, '1m', timerange) @@ -211,7 +212,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # timeframe starts after the chached data # should return the chached data w/o the last item - timerange = (('date', None), test_data[-1][0] / 1000 + 1, None) + timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 1, 0) data, start_ts = load_cached_data_for_updating(test_filename, '1m', timerange) @@ -220,7 +221,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # same with 'line' timeframe num_lines = 30 - timerange = ((None, 'line'), None, -num_lines) + timerange = TimeRange(None, 'line', 0, -num_lines) data, start_ts = load_cached_data_for_updating(test_filename, '1m', timerange) @@ -230,7 +231,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # no timeframe is set # should return the chached data w/o the last item num_lines = 30 - timerange = ((None, 'line'), None, -num_lines) + timerange = TimeRange(None, 'line', 0, -num_lines) data, start_ts = load_cached_data_for_updating(test_filename, '1m', timerange) @@ -239,7 +240,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # no datafile exist # should return timestamp start time - timerange = (('date', None), now_ts - 10000, None) + timerange = TimeRange('date', None, now_ts - 10000, 0) data, start_ts = load_cached_data_for_updating(test_filename + 'unexist', '1m', timerange) @@ -248,7 +249,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # same with 'line' timeframe num_lines = 30 - timerange = ((None, 'line'), None, -num_lines) + timerange = TimeRange(None, 'line', 0, -num_lines) data, start_ts = load_cached_data_for_updating(test_filename + 'unexist', '1m', timerange) @@ -343,7 +344,7 @@ def test_trim_tickerlist() -> None: # Test the pattern ^(-\d+)$ # This pattern uses the latest N elements - timerange = ((None, 'line'), None, -5) + timerange = TimeRange(None, 'line', 0, -5) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) @@ -353,7 +354,7 @@ def test_trim_tickerlist() -> None: # Test the pattern ^(\d+)-$ # This pattern keep X element from the end - timerange = (('line', None), 5, None) + timerange = TimeRange('line', None, 5, 0) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) @@ -363,7 +364,7 @@ def test_trim_tickerlist() -> None: # Test the pattern ^(\d+)-(\d+)$ # This pattern extract a window - timerange = (('index', 'index'), 5, 10) + timerange = TimeRange('index', 'index', 5, 10) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) @@ -374,7 +375,7 @@ def test_trim_tickerlist() -> None: # Test the pattern ^(\d{8})-(\d{8})$ # This pattern extract a window between the dates - timerange = (('date', 'date'), ticker_list[5][0] / 1000, ticker_list[10][0] / 1000 - 1) + timerange = TimeRange('date', 'date', ticker_list[5][0] / 1000, ticker_list[10][0] / 1000 - 1) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) @@ -385,7 +386,7 @@ def test_trim_tickerlist() -> None: # Test the pattern ^-(\d{8})$ # This pattern extracts elements from the start to the date - timerange = ((None, 'date'), None, ticker_list[10][0] / 1000 - 1) + timerange = TimeRange(None, 'date', 0, ticker_list[10][0] / 1000 - 1) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) @@ -395,7 +396,7 @@ def test_trim_tickerlist() -> None: # Test the pattern ^(\d{8})-$ # This pattern extracts elements from the date to now - timerange = (('date', None), ticker_list[10][0] / 1000 - 1, None) + timerange = TimeRange('date', None, ticker_list[10][0] / 1000 - 1, None) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) @@ -405,7 +406,7 @@ def test_trim_tickerlist() -> None: # Test a wrong pattern # This pattern must return the list unchanged - timerange = ((None, None), None, 5) + timerange = TimeRange(None, None, None, 5) ticker = trim_tickerlist(ticker_list, timerange) ticker_len = len(ticker) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 1cf374b6b..5cdd22c7a 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -7,8 +7,6 @@ Unit test file for rpc/rpc.py from datetime import datetime from unittest.mock import MagicMock -from sqlalchemy import create_engine - from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc.rpc import RPC @@ -39,7 +37,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: get_fee=fee ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED @@ -88,7 +86,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: get_fee=fee ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED @@ -123,7 +121,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, get_fee=fee ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] @@ -180,7 +178,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, get_fee=fee ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] @@ -206,15 +204,30 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, trade.close_date = datetime.utcnow() trade.is_open = False + freqtradebot.create_trade() + trade = Trade.query.first() + # Simulate fulfilled LIMIT_BUY order for trade + trade.update(limit_buy_order) + + # Update the ticker with a market going up + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker_sell_up + ) + trade.update(limit_sell_order) + trade.close_date = datetime.utcnow() + trade.is_open = False + (error, stats) = rpc.rpc_trade_statistics(stake_currency, fiat_display_currency) assert not error assert prec_satoshi(stats['profit_closed_coin'], 6.217e-05) assert prec_satoshi(stats['profit_closed_percent'], 6.2) assert prec_satoshi(stats['profit_closed_fiat'], 0.93255) - assert prec_satoshi(stats['profit_all_coin'], 6.217e-05) - assert prec_satoshi(stats['profit_all_percent'], 6.2) - assert prec_satoshi(stats['profit_all_fiat'], 0.93255) - assert stats['trade_count'] == 1 + assert prec_satoshi(stats['profit_all_coin'], 5.632e-05) + assert prec_satoshi(stats['profit_all_percent'], 2.81) + assert prec_satoshi(stats['profit_all_fiat'], 0.8448) + assert stats['trade_count'] == 2 assert stats['first_trade_date'] == 'just now' assert stats['latest_trade_date'] == 'just now' assert stats['avg_duration'] == '0:00:00' @@ -243,7 +256,7 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, get_fee=fee ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] @@ -314,7 +327,7 @@ def test_rpc_balance_handle(default_conf, mocker): get_balances=MagicMock(return_value=mock_balance) ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) (error, res) = rpc.rpc_balance(default_conf['fiat_display_currency']) @@ -344,7 +357,7 @@ def test_rpc_start(mocker, default_conf) -> None: get_ticker=MagicMock() ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED @@ -372,7 +385,7 @@ def test_rpc_stop(mocker, default_conf) -> None: get_ticker=MagicMock() ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) freqtradebot.state = State.RUNNING @@ -411,7 +424,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: get_fee=fee, ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED @@ -521,7 +534,7 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, get_fee=fee ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) # Create some test data @@ -560,7 +573,7 @@ def test_rpc_count(mocker, default_conf, ticker, fee) -> None: get_fee=fee, ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) (error, trades) = rpc.rpc_count() diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 28fdc7902..2e6e3b285 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -11,7 +11,6 @@ from datetime import datetime from random import randint from unittest.mock import MagicMock -from sqlalchemy import create_engine from telegram import Update, Message, Chat from telegram.error import NetworkError @@ -156,7 +155,7 @@ def test_authorized_only(default_conf, mocker, caplog) -> None: conf = deepcopy(default_conf) conf['telegram']['enabled'] = False - dummy = DummyCls(FreqtradeBot(conf, create_engine('sqlite://'))) + dummy = DummyCls(FreqtradeBot(conf)) dummy.dummy_handler(bot=MagicMock(), update=update) assert dummy.state['called'] is True assert log_has( @@ -187,7 +186,7 @@ def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: conf = deepcopy(default_conf) conf['telegram']['enabled'] = False - dummy = DummyCls(FreqtradeBot(conf, create_engine('sqlite://'))) + dummy = DummyCls(FreqtradeBot(conf)) dummy.dummy_handler(bot=MagicMock(), update=update) assert dummy.state['called'] is False assert not log_has( @@ -217,7 +216,7 @@ def test_authorized_only_exception(default_conf, mocker, caplog) -> None: conf = deepcopy(default_conf) conf['telegram']['enabled'] = False - dummy = DummyCls(FreqtradeBot(conf, create_engine('sqlite://'))) + dummy = DummyCls(FreqtradeBot(conf)) dummy.dummy_exception(bot=MagicMock(), update=update) assert dummy.state['called'] is False assert not log_has( @@ -263,7 +262,7 @@ def test_status(default_conf, update, mocker, fee, ticker) -> None: ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(conf) telegram = Telegram(freqtradebot) # Create some test data @@ -301,7 +300,7 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED @@ -348,7 +347,7 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: conf = deepcopy(default_conf) conf['stake_amount'] = 15.0 - freqtradebot = FreqtradeBot(conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(conf) telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED @@ -402,7 +401,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) # Create some test data @@ -470,7 +469,7 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) # Try invalid data @@ -511,7 +510,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) telegram._profit(bot=MagicMock(), update=update) @@ -608,7 +607,7 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None: send_msg=msg_mock ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) telegram._balance(bot=MagicMock(), update=update) @@ -638,7 +637,7 @@ def test_zero_balance_handle(default_conf, update, mocker) -> None: send_msg=msg_mock ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) telegram._balance(bot=MagicMock(), update=update) @@ -661,7 +660,7 @@ def test_start_handle(default_conf, update, mocker) -> None: ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED @@ -685,7 +684,7 @@ def test_start_handle_already_running(default_conf, update, mocker) -> None: ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) freqtradebot.state = State.RUNNING @@ -710,7 +709,7 @@ def test_stop_handle(default_conf, update, mocker) -> None: ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) freqtradebot.state = State.RUNNING @@ -735,7 +734,7 @@ def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED @@ -762,7 +761,7 @@ def test_forcesell_handle(default_conf, update, ticker, fee, ticker_sell_up, moc get_fee=fee ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) # Create some test data @@ -802,7 +801,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, ticker_sell_do get_fee=fee ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) # Create some test data @@ -847,7 +846,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None get_fee=fee ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) # Create some test data @@ -880,7 +879,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: ) mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) # Trader is not running @@ -927,7 +926,7 @@ def test_performance_handle(default_conf, update, ticker, fee, get_fee=fee ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) # Create some test data @@ -962,7 +961,7 @@ def test_performance_handle_invalid(default_conf, update, mocker) -> None: send_msg=msg_mock ) mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock()) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) # Trader is not running @@ -991,7 +990,7 @@ def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: buy=MagicMock(return_value={'id': 'mocked_order_id'}) ) mocker.patch('freqtrade.optimize.backtesting.exchange.get_fee', fee) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED @@ -1027,7 +1026,7 @@ def test_help_handle(default_conf, update, mocker) -> None: _init=MagicMock(), send_msg=msg_mock ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) telegram._help(bot=MagicMock(), update=update) @@ -1047,7 +1046,7 @@ def test_version_handle(default_conf, update, mocker) -> None: _init=MagicMock(), send_msg=msg_mock ) - freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(default_conf) telegram = Telegram(freqtradebot) telegram._version(bot=MagicMock(), update=update) @@ -1064,7 +1063,7 @@ def test_send_msg(default_conf, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) conf = deepcopy(default_conf) bot = MagicMock() - freqtradebot = FreqtradeBot(conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(conf) telegram = Telegram(freqtradebot) telegram._config['telegram']['enabled'] = False @@ -1087,7 +1086,7 @@ def test_send_msg_network_error(default_conf, mocker, caplog) -> None: conf = deepcopy(default_conf) bot = MagicMock() bot.send_message = MagicMock(side_effect=NetworkError('Oh snap')) - freqtradebot = FreqtradeBot(conf, create_engine('sqlite://')) + freqtradebot = FreqtradeBot(conf) telegram = Telegram(freqtradebot) telegram._config['telegram']['enabled'] = True diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 01033ce7d..418f31851 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -13,6 +13,7 @@ from pandas import DataFrame from freqtrade.analyze import Analyze, SignalType from freqtrade.optimize.__init__ import load_tickerdata_file +from freqtrade.arguments import TimeRange from freqtrade.tests.conftest import log_has # Avoid to reinit the same object again and again @@ -183,7 +184,7 @@ def test_tickerdata_to_dataframe(default_conf) -> None: """ analyze = Analyze(default_conf) - timerange = ((None, 'line'), None, -100) + timerange = TimeRange(None, 'line', 0, -100) tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) tickerlist = {'UNITTEST/BTC': tick} data = analyze.tickerdata_to_dataframe(tickerlist) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 474aa2507..a7237d7c4 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -9,7 +9,7 @@ import logging import pytest -from freqtrade.arguments import Arguments +from freqtrade.arguments import Arguments, TimeRange def test_arguments_object() -> None: @@ -46,6 +46,11 @@ def test_parse_args_config() -> None: assert args.config == '/dev/null' +def test_parse_args_db_url() -> None: + args = Arguments(['--db-url', 'sqlite:///test.sqlite'], '').get_parsed_arg() + assert args.db_url == 'sqlite:///test.sqlite' + + def test_parse_args_verbose() -> None: args = Arguments(['-v'], '').get_parsed_arg() assert args.loglevel == logging.DEBUG @@ -107,20 +112,24 @@ def test_parse_args_dynamic_whitelist_invalid_values() -> None: def test_parse_timerange_incorrect() -> None: - assert ((None, 'line'), None, -200) == Arguments.parse_timerange('-200') - assert (('line', None), 200, None) == Arguments.parse_timerange('200-') - assert (('index', 'index'), 200, 500) == Arguments.parse_timerange('200-500') + assert TimeRange(None, 'line', 0, -200) == Arguments.parse_timerange('-200') + assert TimeRange('line', None, 200, 0) == Arguments.parse_timerange('200-') + assert TimeRange('index', 'index', 200, 500) == Arguments.parse_timerange('200-500') - assert (('date', None), 1274486400, None) == Arguments.parse_timerange('20100522-') - assert ((None, 'date'), None, 1274486400) == Arguments.parse_timerange('-20100522') + assert TimeRange('date', None, 1274486400, 0) == Arguments.parse_timerange('20100522-') + assert TimeRange(None, 'date', 0, 1274486400) == Arguments.parse_timerange('-20100522') timerange = Arguments.parse_timerange('20100522-20150730') - assert timerange == (('date', 'date'), 1274486400, 1438214400) + assert timerange == TimeRange('date', 'date', 1274486400, 1438214400) # Added test for unix timestamp - BTC genesis date - assert (('date', None), 1231006505, None) == Arguments.parse_timerange('1231006505-') - assert ((None, 'date'), None, 1233360000) == Arguments.parse_timerange('-1233360000') + assert TimeRange('date', None, 1231006505, 0) == Arguments.parse_timerange('1231006505-') + assert TimeRange(None, 'date', 0, 1233360000) == Arguments.parse_timerange('-1233360000') timerange = Arguments.parse_timerange('1231006505-1233360000') - assert timerange == (('date', 'date'), 1231006505, 1233360000) + assert TimeRange('date', 'date', 1231006505, 1233360000) == timerange + + # TODO: Find solution for the following case (passing timestamp in ms) + timerange = Arguments.parse_timerange('1231006505000-1233360000000') + assert TimeRange('date', 'date', 1231006505, 1233360000) != timerange with pytest.raises(Exception, match=r'Incorrect syntax.*'): Arguments.parse_timerange('-') diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 98adf19e9..8694020d5 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -130,7 +130,6 @@ def test_load_config(default_conf, mocker) -> None: assert validated_conf.get('strategy') == 'DefaultStrategy' assert validated_conf.get('strategy_path') is None assert 'dynamic_whitelist' not in validated_conf - assert 'dry_run_db' not in validated_conf def test_load_config_with_params(default_conf, mocker) -> None: @@ -145,7 +144,7 @@ def test_load_config_with_params(default_conf, mocker) -> None: '--dynamic-whitelist', '10', '--strategy', 'TestStrategy', '--strategy-path', '/some/path', - '--dry-run-db', + '--db-url', 'sqlite:///someurl', ] args = Arguments(arglist, '').get_parsed_arg() @@ -155,7 +154,7 @@ def test_load_config_with_params(default_conf, mocker) -> None: assert validated_conf.get('dynamic_whitelist') == 10 assert validated_conf.get('strategy') == 'TestStrategy' assert validated_conf.get('strategy_path') == '/some/path' - assert validated_conf.get('dry_run_db') is True + assert validated_conf.get('db_url') == 'sqlite:///someurl' def test_load_custom_strategy(default_conf, mocker) -> None: @@ -190,7 +189,7 @@ def test_show_info(default_conf, mocker, caplog) -> None: arglist = [ '--dynamic-whitelist', '10', '--strategy', 'TestStrategy', - '--dry-run-db' + '--db-url', 'sqlite:///tmp/testdb', ] args = Arguments(arglist, '').get_parsed_arg() @@ -204,23 +203,8 @@ def test_show_info(default_conf, mocker, caplog) -> None: caplog.record_tuples ) - assert log_has( - 'Parameter --dry-run-db detected ...', - caplog.record_tuples - ) - - assert log_has( - 'Dry_run will use the DB file: "tradesv3.dry_run.sqlite"', - caplog.record_tuples - ) - - # Test the Dry run condition - configuration.config.update({'dry_run': False}) # type: ignore - configuration._load_common_config(configuration.config) # type: ignore - assert log_has( - 'Dry run is disabled. (--dry_run_db ignored)', - caplog.record_tuples - ) + assert log_has('Using DB: "sqlite:///tmp/testdb"', caplog.record_tuples) + assert log_has('Dry run is enabled', caplog.record_tuples) def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None: diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 581e76406..f7e84ebdd 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -13,7 +13,6 @@ from unittest.mock import MagicMock import arrow import pytest import requests -from sqlalchemy import create_engine from freqtrade import constants, DependencyException, OperationalException, TemporaryError from freqtrade.freqtradebot import FreqtradeBot @@ -36,7 +35,7 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock()) patch_coinmarketcap(mocker) - return FreqtradeBot(config, create_engine('sqlite://')) + return FreqtradeBot(config) def patch_get_signal(mocker, value=(True, False)) -> None: @@ -346,7 +345,7 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, mocker) -> Non # Save state of current whitelist whitelist = deepcopy(default_conf['exchange']['pair_whitelist']) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) freqtrade.create_trade() trade = Trade.query.first() @@ -383,13 +382,34 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, fee, conf = deepcopy(default_conf) conf['stake_amount'] = 0.0005 - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() rate, amount = buy_mock.call_args[0][1], buy_mock.call_args[0][2] assert rate * amount >= conf['stake_amount'] +def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, fee, mocker) -> None: + """ + Test create_trade() method + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch.multiple( + 'freqtrade.freqtradebot.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5), + get_fee=fee, + ) + freqtrade = FreqtradeBot(default_conf) + + with pytest.raises(DependencyException, match=r'.*stake amount.*'): + freqtrade.create_trade() + + def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, mocker) -> None: """ Test create_trade() method @@ -408,7 +428,7 @@ def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, mocke conf = deepcopy(default_conf) conf['exchange']['pair_whitelist'] = ["ETH/BTC"] conf['exchange']['pair_blacklist'] = ["ETH/BTC"] - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() @@ -435,7 +455,7 @@ def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, conf = deepcopy(default_conf) conf['exchange']['pair_whitelist'] = ["ETH/BTC"] conf['exchange']['pair_blacklist'] = ["ETH/BTC"] - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() @@ -463,7 +483,7 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: conf = deepcopy(default_conf) conf['stake_amount'] = 10 - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) Trade.query = MagicMock() Trade.query.filter = MagicMock() @@ -487,7 +507,7 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, get_order=MagicMock(return_value=limit_buy_order), get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) trades = Trade.query.filter(Trade.is_open.is_(True)).all() assert not trades @@ -528,7 +548,7 @@ def test_process_exchange_failures(default_conf, ticker, markets, mocker) -> Non ) sleep_mock = mocker.patch('time.sleep', side_effect=lambda _: None) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) result = freqtrade._process() assert result is False assert sleep_mock.has_calls() @@ -548,7 +568,7 @@ def test_process_operational_exception(default_conf, ticker, markets, mocker) -> get_markets=markets, buy=MagicMock(side_effect=OperationalException) ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) assert freqtrade.state == State.RUNNING result = freqtrade._process() @@ -574,7 +594,7 @@ def test_process_trade_handling( get_order=MagicMock(return_value=limit_buy_order), get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) trades = Trade.query.filter(Trade.is_open.is_(True)).all() assert not trades @@ -691,7 +711,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mock ) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) freqtrade.create_trade() @@ -734,7 +754,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, fee, get_fee=fee, ) - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() @@ -793,7 +813,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, fee, mocker, ca ) mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() trade = Trade.query.first() @@ -830,7 +850,7 @@ def test_handle_trade_experimental( ) mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() trade = Trade.query.first() @@ -858,7 +878,7 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, fe buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Create trade and sell it freqtrade.create_trade() @@ -889,7 +909,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fe cancel_order=cancel_order_mock, get_fee=fee ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) trade_buy = Trade( pair='ETH/BTC', @@ -929,7 +949,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, get_order=MagicMock(return_value=limit_sell_order_old), cancel_order=cancel_order_mock ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) trade_sell = Trade( pair='ETH/BTC', @@ -969,7 +989,7 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old get_order=MagicMock(return_value=limit_buy_order_old_partial), cancel_order=cancel_order_mock ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) trade_buy = Trade( pair='ETH/BTC', @@ -1017,7 +1037,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) - get_order=MagicMock(side_effect=requests.exceptions.RequestException('Oh snap')), cancel_order=cancel_order_mock ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) trade_buy = Trade( pair='ETH/BTC', @@ -1056,7 +1076,7 @@ def test_handle_timedout_limit_buy(mocker, default_conf) -> None: cancel_order=cancel_order_mock ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) Trade.session = MagicMock() trade = MagicMock() @@ -1082,7 +1102,7 @@ def test_handle_timedout_limit_sell(mocker, default_conf) -> None: cancel_order=cancel_order_mock ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) trade = MagicMock() order = {'remaining': 1, @@ -1109,7 +1129,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N get_fee=fee ) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Create some test data freqtrade.create_trade() @@ -1150,7 +1170,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker) get_ticker=ticker, get_fee=fee ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Create some test data freqtrade.create_trade() @@ -1190,7 +1210,7 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, get_ticker=ticker, get_fee=fee ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Create some test data freqtrade.create_trade() @@ -1231,7 +1251,7 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, get_ticker=ticker, get_fee=fee ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Create some test data freqtrade.create_trade() @@ -1280,7 +1300,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, fee, mock 'use_sell_signal': True, 'sell_profit_only': True, } - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() trade = Trade.query.first() @@ -1313,7 +1333,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, fee, moc 'use_sell_signal': True, 'sell_profit_only': False, } - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() trade = Trade.query.first() @@ -1346,7 +1366,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker 'use_sell_signal': True, 'sell_profit_only': True, } - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() trade = Trade.query.first() @@ -1381,7 +1401,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke 'sell_profit_only': False, } - freqtrade = FreqtradeBot(conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(conf) freqtrade.create_trade() trade = Trade.query.first() @@ -1409,7 +1429,7 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, ca open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' @@ -1436,7 +1456,7 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker): open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' @@ -1463,7 +1483,7 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, ca open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount @@ -1489,7 +1509,7 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, mock open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount @@ -1512,7 +1532,7 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' @@ -1540,7 +1560,7 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, limit_buy_order) == amount - 0.004 assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' @@ -1568,7 +1588,7 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, limit_buy_order) == amount @@ -1593,6 +1613,6 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) + freqtrade = FreqtradeBot(default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 3e0f50fbb..c50ad7d2c 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -1,9 +1,11 @@ # pragma pylint: disable=missing-docstring, C0103 -import os +from copy import deepcopy +from unittest.mock import MagicMock import pytest from sqlalchemy import create_engine +from freqtrade import constants, OperationalException from freqtrade.persistence import Trade, init, clean_dry_run_db @@ -21,77 +23,54 @@ def test_init_create_session(default_conf, mocker): assert 'Session' in type(Trade.session).__name__ -def test_init_dry_run_db(default_conf, mocker): - default_conf.update({'dry_run_db': True}) - mocker.patch.dict('freqtrade.persistence._CONF', default_conf) +def test_init_custom_db_url(default_conf, mocker): + conf = deepcopy(default_conf) - # First, protect the existing 'tradesv3.dry_run.sqlite' (Do not delete user data) - dry_run_db = 'tradesv3.dry_run.sqlite' - dry_run_db_swp = dry_run_db + '.swp' + # Update path to a value other than default, but still in-memory + conf.update({'db_url': 'sqlite:///tmp/freqtrade2_test.sqlite'}) + create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) + mocker.patch.dict('freqtrade.persistence._CONF', conf) - if os.path.isfile(dry_run_db): - os.rename(dry_run_db, dry_run_db_swp) - - # Check if the new tradesv3.dry_run.sqlite was created - init(default_conf) - assert os.path.isfile(dry_run_db) is True - - # Delete the file made for this unitest and rollback to the previous - # tradesv3.dry_run.sqlite file - - # 1. Delete file from the test - if os.path.isfile(dry_run_db): - os.remove(dry_run_db) - - # 2. Rollback to the initial file - if os.path.isfile(dry_run_db_swp): - os.rename(dry_run_db_swp, dry_run_db) + init(conf) + assert create_engine_mock.call_count == 1 + assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tmp/freqtrade2_test.sqlite' -def test_init_dry_run_without_db(default_conf, mocker): - default_conf.update({'dry_run_db': False}) - mocker.patch.dict('freqtrade.persistence._CONF', default_conf) +def test_init_invalid_db_url(default_conf, mocker): + conf = deepcopy(default_conf) - # First, protect the existing 'tradesv3.dry_run.sqlite' (Do not delete user data) - dry_run_db = 'tradesv3.dry_run.sqlite' - dry_run_db_swp = dry_run_db + '.swp' + # Update path to a value other than default, but still in-memory + conf.update({'db_url': 'unknown:///some.url'}) + mocker.patch.dict('freqtrade.persistence._CONF', conf) - if os.path.isfile(dry_run_db): - os.rename(dry_run_db, dry_run_db_swp) - - # Check if the new tradesv3.dry_run.sqlite was created - init(default_conf) - assert os.path.isfile(dry_run_db) is False - - # Rollback to the initial 'tradesv3.dry_run.sqlite' file - if os.path.isfile(dry_run_db_swp): - os.rename(dry_run_db_swp, dry_run_db) + with pytest.raises(OperationalException, match=r'.*no valid database URL*'): + init(conf) def test_init_prod_db(default_conf, mocker): - default_conf.update({'dry_run': False}) - mocker.patch.dict('freqtrade.persistence._CONF', default_conf) + conf = deepcopy(default_conf) + conf.update({'dry_run': False}) + conf.update({'db_url': constants.DEFAULT_DB_PROD_URL}) - # First, protect the existing 'tradesv3.sqlite' (Do not delete user data) - prod_db = 'tradesv3.sqlite' - prod_db_swp = prod_db + '.swp' + create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) + mocker.patch.dict('freqtrade.persistence._CONF', conf) - if os.path.isfile(prod_db): - os.rename(prod_db, prod_db_swp) + init(conf) + assert create_engine_mock.call_count == 1 + assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tradesv3.sqlite' - # Check if the new tradesv3.sqlite was created - init(default_conf) - assert os.path.isfile(prod_db) is True - # Delete the file made for this unitest and rollback to the previous tradesv3.sqlite file +def test_init_dryrun_db(default_conf, mocker): + conf = deepcopy(default_conf) + conf.update({'dry_run': True}) + conf.update({'db_url': constants.DEFAULT_DB_DRYRUN_URL}) - # 1. Delete file from the test - if os.path.isfile(prod_db): - os.remove(prod_db) + create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) + mocker.patch.dict('freqtrade.persistence._CONF', conf) - # Rollback to the initial 'tradesv3.sqlite' file - if os.path.isfile(prod_db_swp): - os.rename(prod_db_swp, prod_db) + init(conf) + assert create_engine_mock.call_count == 1 + assert create_engine_mock.mock_calls[0][1][0] == 'sqlite://' @pytest.mark.usefixtures("init_persistence") @@ -328,7 +307,7 @@ def test_calc_profit_percent(limit_buy_order, limit_sell_order, fee): def test_clean_dry_run_db(default_conf, fee): - init(default_conf, create_engine('sqlite://')) + init(default_conf) # Simulate dry_run entries trade = Trade( @@ -377,7 +356,7 @@ def test_clean_dry_run_db(default_conf, fee): assert len(Trade.query.filter(Trade.open_order_id.isnot(None)).all()) == 1 -def test_migrate_old(default_conf, fee): +def test_migrate_old(mocker, default_conf, fee): """ Test Database migration(starting with old pairformat) """ @@ -409,11 +388,13 @@ def test_migrate_old(default_conf, fee): amount=amount ) engine = create_engine('sqlite://') + mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine) + # Create table using the old format engine.execute(create_table_old) engine.execute(insert_table_old) # Run init to test migration - init(default_conf, engine) + init(default_conf) assert len(Trade.query.filter(Trade.id == 1).all()) == 1 trade = Trade.query.filter(Trade.id == 1).first() @@ -428,7 +409,7 @@ def test_migrate_old(default_conf, fee): assert trade.exchange == "bittrex" -def test_migrate_new(default_conf, fee): +def test_migrate_new(mocker, default_conf, fee): """ Test Database migration (starting with new pairformat) """ @@ -460,11 +441,13 @@ def test_migrate_new(default_conf, fee): amount=amount ) engine = create_engine('sqlite://') + mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine) + # Create table using the old format engine.execute(create_table_old) engine.execute(insert_table_old) # Run init to test migration - init(default_conf, engine) + init(default_conf) assert len(Trade.query.filter(Trade.id == 1).all()) == 1 trade = Trade.query.filter(Trade.id == 1).first() diff --git a/requirements.txt b/requirements.txt index 6e7550515..5f5183321 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.121 +ccxt==1.14.155 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 @@ -10,14 +10,14 @@ pandas==0.23.0 scikit-learn==0.19.1 scipy==1.1.0 jsonschema==2.6.0 -numpy==1.14.3 +numpy==1.14.4 TA-Lib==0.4.17 -pytest==3.6.0 +pytest==3.6.1 pytest-mock==1.10.0 pytest-cov==2.5.1 hyperopt==0.1 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 -networkx==1.11 +networkx==1.11 # pyup: ignore tabulate==0.8.2 coinmarketcap==5.0.3 diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 1c73eae03..6185edaf7 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -8,24 +8,28 @@ import arrow from freqtrade import (exchange, arguments, misc) -DEFAULT_DL_PATH = 'freqtrade/tests/testdata' +DEFAULT_DL_PATH = 'user_data/data' arguments = arguments.Arguments(sys.argv[1:], 'download utility') arguments.testdata_dl_options() args = arguments.parse_args() -TICKER_INTERVALS = ['1m', '5m'] -PAIRS = [] +timeframes = args.timeframes -if args.pairs_file: - with open(args.pairs_file) as file: - PAIRS = json.load(file) -PAIRS = list(set(PAIRS)) - -dl_path = DEFAULT_DL_PATH -if args.export and os.path.exists(args.export): +dl_path = os.path.join(DEFAULT_DL_PATH, args.exchange) +if args.export: dl_path = args.export +if not os.path.isdir(dl_path): + sys.exit(f'Directory {dl_path} does not exist.') + +pairs_file = args.pairs_file if args.pairs_file else os.path.join(dl_path, 'pairs.json') +if not os.path.isfile(pairs_file): + sys.exit(f'No pairs file found with path {pairs_file}.') + +with open(pairs_file) as file: + PAIRS = list(set(json.load(file))) + since_time = None if args.days: since_time = arrow.utcnow().shift(days=-args.days).timestamp * 1000 @@ -40,7 +44,7 @@ exchange._API = exchange.init_ccxt({'key': '', for pair in PAIRS: - for tick_interval in TICKER_INTERVALS: + for tick_interval in timeframes: print(f'downloading pair {pair}, interval {tick_interval}') data = exchange.get_ticker_history(pair, tick_interval, since_ms=since_time) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 108c0b609..122c002a8 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -5,47 +5,75 @@ Script to display when the bot will buy a specific pair Mandatory Cli parameters: -p / --pair: pair to examine -Optional Cli parameters +Option but recommended -s / --strategy: strategy to use + + +Optional Cli parameters -d / --datadir: path to pair backtest data --timerange: specify what timerange of data to use. -l / --live: Live, to download the latest ticker for the pair -db / --db-url: Show trades stored in database + + +Indicators recommended +Row 1: sma, ema3, ema5, ema10, ema50 +Row 3: macd, rsi, fisher_rsi, mfi, slowd, slowk, fastd, fastk + +Example of usage: +> python3 scripts/plot_dataframe.py --pair BTC/EUR -d user_data/data/ --indicators1 sma,ema3 +--indicators2 fastk,fastd """ import logging +import os import sys from argparse import Namespace +from typing import Dict, List, Any -from typing import List - +import plotly.graph_objs as go from plotly import tools from plotly.offline import plot -import plotly.graph_objs as go -from typing import Dict, List, Any -from sqlalchemy import create_engine - -from freqtrade.arguments import Arguments -from freqtrade.analyze import Analyze -from freqtrade import exchange import freqtrade.optimize as optimize +from freqtrade import exchange from freqtrade import persistence +from freqtrade.analyze import Analyze +from freqtrade.arguments import Arguments +from freqtrade.optimize.backtesting import setup_configuration from freqtrade.persistence import Trade logger = logging.getLogger(__name__) _CONF: Dict[str, Any] = {} + def plot_analyzed_dataframe(args: Namespace) -> None: """ Calls analyze() and plots the returned dataframe :return: None """ - pair = args.pair.replace('-', '_') + global _CONF + + # Load the configuration + _CONF.update(setup_configuration(args)) + + # Set the pair to audit + pair = args.pair + + if pair is None: + logger.critical('Parameter --pair mandatory;. E.g --pair ETH/BTC') + exit() + + if '/' not in pair: + logger.critical('--pair format must be XXX/YYY') + exit() + + # Set timerange to use timerange = Arguments.parse_timerange(args.timerange) - # Init strategy + # Load the strategy try: - analyze = Analyze({'strategy': args.strategy}) + analyze = Analyze(_CONF) + exchange.init(_CONF) except AttributeError: logger.critical( 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', @@ -53,37 +81,75 @@ def plot_analyzed_dataframe(args: Namespace) -> None: ) exit() - tick_interval = analyze.strategy.ticker_interval + # Set the ticker to use + tick_interval = analyze.get_ticker_interval() + # Load pair tickers tickers = {} if args.live: logger.info('Downloading pair.') - # Init Bittrex to use public API - exchange.init({'key': '', 'secret': ''}) tickers[pair] = exchange.get_ticker_history(pair, tick_interval) else: tickers = optimize.load_data( datadir=args.datadir, pairs=[pair], ticker_interval=tick_interval, - refresh_pairs=False, + refresh_pairs=_CONF.get('refresh_pairs', False), timerange=timerange ) + + # No ticker found, or impossible to download + if tickers == {}: + exit() + + # Get trades already made from the DB + trades: List[Trade] = [] + if args.db_url: + persistence.init(_CONF) + trades = Trade.query.filter(Trade.pair.is_(pair)).all() + dataframes = analyze.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] dataframe = analyze.populate_buy_trend(dataframe) dataframe = analyze.populate_sell_trend(dataframe) - trades = [] - if args.db_url: - engine = create_engine('sqlite:///' + args.db_url) - persistence.init(_CONF, engine) - trades = Trade.query.filter(Trade.pair.is_(pair)).all() - if len(dataframe.index) > 750: logger.warning('Ticker contained more than 750 candles, clipping.') - data = dataframe.tail(750) + fig = generate_graph( + pair=pair, + trades=trades, + data=dataframe.tail(750), + args=args + ) + + plot(fig, filename=os.path.join('user_data', 'freqtrade-plot.html')) + + +def generate_graph(pair, trades, data, args) -> tools.make_subplots: + """ + Generate the graph from the data generated by Backtesting or from DB + :param pair: Pair to Display on the graph + :param trades: All trades created + :param data: Dataframe + :param args: sys.argv that contrains the two params indicators1, and indicators2 + :return: None + """ + + # Define the graph + fig = tools.make_subplots( + rows=3, + cols=1, + shared_xaxes=True, + row_width=[1, 1, 4], + vertical_spacing=0.0001, + ) + fig['layout'].update(title=pair) + fig['layout']['yaxis1'].update(title='Price') + fig['layout']['yaxis2'].update(title='Volume') + fig['layout']['yaxis3'].update(title='Other') + + # Common information candles = go.Candlestick( x=data.date, open=data.open, @@ -145,49 +211,67 @@ def plot_analyzed_dataframe(args: Namespace) -> None: ) ) - bb_lower = go.Scatter( - x=data.date, - y=data.bb_lowerband, - name='BB lower', - line={'color': "transparent"}, - ) - bb_upper = go.Scatter( - x=data.date, - y=data.bb_upperband, - name='BB upper', - fill="tonexty", - fillcolor="rgba(0,176,246,0.2)", - line={'color': "transparent"}, - ) - macd = go.Scattergl(x=data['date'], y=data['macd'], name='MACD') - macdsignal = go.Scattergl(x=data['date'], y=data['macdsignal'], name='MACD signal') - volume = go.Bar(x=data['date'], y=data['volume'], name='Volume') - - fig = tools.make_subplots( - rows=3, - cols=1, - shared_xaxes=True, - row_width=[1, 1, 4], - vertical_spacing=0.0001, - ) - + # Row 1 fig.append_trace(candles, 1, 1) - fig.append_trace(bb_lower, 1, 1) - fig.append_trace(bb_upper, 1, 1) + + if 'bb_lowerband' in data and 'bb_upperband' in data: + bb_lower = go.Scatter( + x=data.date, + y=data.bb_lowerband, + name='BB lower', + line={'color': "transparent"}, + ) + bb_upper = go.Scatter( + x=data.date, + y=data.bb_upperband, + name='BB upper', + fill="tonexty", + fillcolor="rgba(0,176,246,0.2)", + line={'color': "transparent"}, + ) + fig.append_trace(bb_lower, 1, 1) + fig.append_trace(bb_upper, 1, 1) + + fig = generate_row(fig=fig, row=1, raw_indicators=args.indicators1, data=data) fig.append_trace(buys, 1, 1) fig.append_trace(sells, 1, 1) - fig.append_trace(volume, 2, 1) - fig.append_trace(macd, 3, 1) - fig.append_trace(macdsignal, 3, 1) fig.append_trace(trade_buys, 1, 1) fig.append_trace(trade_sells, 1, 1) - fig['layout'].update(title=args.pair) - fig['layout']['yaxis1'].update(title='Price') - fig['layout']['yaxis2'].update(title='Volume') - fig['layout']['yaxis3'].update(title='MACD') + # Row 2 + volume = go.Bar( + x=data['date'], + y=data['volume'], + name='Volume' + ) + fig.append_trace(volume, 2, 1) - plot(fig, filename='freqtrade-plot.html') + # Row 3 + fig = generate_row(fig=fig, row=3, raw_indicators=args.indicators2, data=data) + + return fig + + +def generate_row(fig, row, raw_indicators, data) -> tools.make_subplots: + """ + Generator all the indicator selected by the user for a specific row + """ + for indicator in raw_indicators.split(','): + if indicator in data: + scattergl = go.Scattergl( + x=data['date'], + y=data[indicator], + name=indicator + ) + fig.append_trace(scattergl, row, 1) + else: + logger.info( + 'Indicator "%s" ignored. Reason: This indicator is not found ' + 'in your strategy.', + indicator + ) + + return fig def plot_parse_args(args: List[str]) -> Namespace: @@ -198,6 +282,24 @@ def plot_parse_args(args: List[str]) -> Namespace: """ arguments = Arguments(args, 'Graph dataframe') arguments.scripts_options() + arguments.parser.add_argument( + '--indicators1', + help='Set indicators from your strategy you want in the first row of the graph. Separate ' + 'them with a coma. E.g: ema3,ema5 (default: %(default)s)', + type=str, + default='sma,ema3,ema5', + dest='indicators1', + ) + + arguments.parser.add_argument( + '--indicators2', + help='Set indicators from your strategy you want in the third row of the graph. Separate ' + 'them with a coma. E.g: fastd,fastk (default: %(default)s)', + type=str, + default='macd', + dest='indicators2', + ) + arguments.common_args_parser() arguments.optimizer_shared_options(arguments.parser) arguments.backtesting_options(arguments.parser) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index daa16ddc9..a5ac00169 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -8,9 +8,12 @@ Mandatory Cli parameters: Optional Cli parameters -c / --config: specify configuration file -s / --strategy: strategy to use ---timerange: specify what timerange of data to use. +-d / --datadir: path to pair backtest data +--timerange: specify what timerange of data to use +--export-filename: Specify where the backtest export is located. """ import logging +import os import sys import json from argparse import Namespace @@ -90,7 +93,18 @@ def plot_profit(args: Namespace) -> None: 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', config.get('strategy') ) - exit() + exit(0) + + # Load the profits results + try: + filename = args.exportfilename + with open(filename) as file: + data = json.load(file) + except FileNotFoundError: + logger.critical( + 'File "backtest-result.json" not found. This script require backtesting ' + 'results to run.\nPlease run a backtesting with the parameter --export.') + exit(0) # Take pairs from the cli otherwise switch to the pair in the config file if args.pair: @@ -140,18 +154,7 @@ def plot_profit(args: Namespace) -> None: num += 1 avgclose /= num - # Load the profits results - # And make an profits-growth array - - try: - filename = 'backtest-result.json' - with open(filename) as file: - data = json.load(file) - except FileNotFoundError: - logger.critical('File "backtest-result.json" not found. This script require backtesting ' - 'results to run.\nPlease run a backtesting with the parameter --export.') - exit(0) - + # make an profits-growth array pg = make_profit_array(data, num_iterations, min_date, tick_interval, filter_pairs) # @@ -184,7 +187,7 @@ def plot_profit(args: Namespace) -> None: ) fig.append_trace(pair_profit, 3, 1) - plot(fig, filename='freqtrade-profit-plot.html') + plot(fig, filename=os.path.join('user_data', 'freqtrade-profit-plot.html')) def define_index(min_date: int, max_date: int, interval: str) -> int: diff --git a/setup.py b/setup.py index 856a47181..ee6b7ae38 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ from freqtrade import __version__ setup(name='freqtrade', version=__version__, description='Simple High Frequency Trading Bot for crypto currencies', - url='https://github.com/gcarq/freqtrade', + url='https://github.com/freqtrade/freqtrade', author='gcarq and contributors', author_email='michael.egger@tsn.at', license='GPLv3', diff --git a/docs/.gitkeep b/user_data/backtest_data/.gitkeep similarity index 100% rename from docs/.gitkeep rename to user_data/backtest_data/.gitkeep diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index af28388be..34f496e38 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -14,7 +14,7 @@ import numpy # noqa class TestStrategy(IStrategy): """ This is a test strategy to inspire you. - More information in https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md + More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md You can: - Rename the class name (Do not forget to update class_name)