diff --git a/Dockerfile b/Dockerfile index d986f20ae..b6333fb13 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8.2-slim-buster +FROM python:3.8.3-slim-buster RUN apt-get update \ && apt-get -y install curl build-essential libssl-dev \ diff --git a/build_helpers/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl b/build_helpers/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl deleted file mode 100644 index 87469a199..000000000 Binary files a/build_helpers/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.17-cp38-cp38-win_amd64.whl b/build_helpers/TA_Lib-0.4.17-cp38-cp38-win_amd64.whl deleted file mode 100644 index 90626b183..000000000 Binary files a/build_helpers/TA_Lib-0.4.17-cp38-cp38-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.18-cp37-cp37m-win_amd64.whl b/build_helpers/TA_Lib-0.4.18-cp37-cp37m-win_amd64.whl new file mode 100644 index 000000000..bd61e812b Binary files /dev/null and b/build_helpers/TA_Lib-0.4.18-cp37-cp37m-win_amd64.whl differ diff --git a/build_helpers/TA_Lib-0.4.18-cp38-cp38-win_amd64.whl b/build_helpers/TA_Lib-0.4.18-cp38-cp38-win_amd64.whl new file mode 100644 index 000000000..f81addb44 Binary files /dev/null and b/build_helpers/TA_Lib-0.4.18-cp38-cp38-win_amd64.whl differ diff --git a/build_helpers/install_windows.ps1 b/build_helpers/install_windows.ps1 index 7dbdd77dd..0a55b6ddd 100644 --- a/build_helpers/install_windows.ps1 +++ b/build_helpers/install_windows.ps1 @@ -7,10 +7,10 @@ python -m pip install --upgrade pip $pyv = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" if ($pyv -eq '3.7') { - pip install build_helpers\TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl + pip install build_helpers\TA_Lib-0.4.18-cp37-cp37m-win_amd64.whl } if ($pyv -eq '3.8') { - pip install build_helpers\TA_Lib-0.4.17-cp38-cp38-win_amd64.whl + pip install build_helpers\TA_Lib-0.4.18-cp38-cp38-win_amd64.whl } pip install -r requirements-dev.txt diff --git a/config_full.json.example b/config_full.json.example index 79154b0d4..0cd265cbe 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -121,6 +121,7 @@ "enabled": false, "listen_ip_address": "127.0.0.1", "listen_port": 8080, + "jwt_secret_key": "somethingrandom", "username": "freqtrader", "password": "SuperSecurePassword" }, diff --git a/docs/configuration.md b/docs/configuration.md index 59d8ba1eb..b0b840fcc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -109,7 +109,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `forcebuy_enable` | Enables the RPC Commands to force a buy. More information below.
**Datatype:** Boolean | `strategy` | **Required** Defines Strategy class to use. Recommended to be set via `--strategy NAME`.
**Datatype:** ClassName | `strategy_path` | Adds an additional strategy lookup path (must be a directory).
**Datatype:** String -| `internals.process_throttle_secs` | Set the process throttle. Value in second.
*Defaults to `5` seconds.*
**Datatype:** Positive Intege +| `internals.process_throttle_secs` | Set the process throttle. Value in second.
*Defaults to `5` seconds.*
**Datatype:** Positive Integer | `internals.heartbeat_interval` | Print heartbeat message every N seconds. Set to 0 to disable heartbeat messages.
*Defaults to `60` seconds.*
**Datatype:** Positive Integer or 0 | `internals.sd_notify` | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details.
**Datatype:** Boolean | `logfile` | Specifies logfile name. Uses a rolling strategy for log file rotation for 10 files with the 1MB limit per file.
**Datatype:** String diff --git a/docs/docker.md b/docs/docker.md index cd24994bc..92478088a 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -22,6 +22,9 @@ Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.co !!! Note All below comands use relative directories and will have to be executed from the directory containing the `docker-compose.yml` file. +!!! Note "Docker on Raspberry" + If you're running freqtrade on a Raspberry PI, you must change the image from `freqtradeorg/freqtrade:master` to `freqtradeorg/freqtrade:master_pi` or `freqtradeorg/freqtrade:develop_pi`, otherwise the image will not work. + ### Docker quick start Create a new directory and place the [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) in this directory. @@ -65,7 +68,7 @@ docker-compose up -d #### Docker-compose logs -Logs will be written to `user_data/freqtrade.log`. +Logs will be written to `user_data/logs/freqtrade.log`. Alternatively, you can check the latest logs using `docker-compose logs -f`. #### Database diff --git a/docs/installation.md b/docs/installation.md index 88e2ef6eb..f017bef96 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -248,14 +248,14 @@ git clone https://github.com/freqtrade/freqtrade.git Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows). -As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial precompiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.17‑cp36‑cp36m‑win32.whl` (make sure to use the version matching your python version) +As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial precompiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl` (make sure to use the version matching your python version) ```cmd >cd \path\freqtrade-develop >python -m venv .env >.env\Scripts\activate.bat REM optionally install ta-lib from wheel -REM >pip install TA_Lib‑0.4.17‑cp36‑cp36m‑win32.whl +REM >pip install TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl >pip install -r requirements.txt >pip install -e . >freqtrade diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 1c0e280ae..c121dec64 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,2 +1,2 @@ -mkdocs-material==5.1.3 +mkdocs-material==5.1.6 mdx_truly_sane_lists==1.2 diff --git a/docs/rest-api.md b/docs/rest-api.md index b68364f39..7f1a95b12 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -11,6 +11,7 @@ Sample configuration: "enabled": true, "listen_ip_address": "127.0.0.1", "listen_port": 8080, + "jwt_secret_key": "somethingrandom", "username": "Freqtrader", "password": "SuperSecret1!" }, @@ -29,7 +30,7 @@ This should return the response: {"status":"pong"} ``` -All other endpoints return sensitive info and require authentication, so are not available through a web browser. +All other endpoints return sensitive info and require authentication and are therefore not available through a web browser. To generate a secure password, either use a password manager, or use the below code snipped. @@ -38,6 +39,9 @@ import secrets secrets.token_hex() ``` +!!! Hint + Use the same method to also generate a JWT secret key (`jwt_secret_key`). + ### Configuration with docker If you run your bot using docker, you'll need to have the bot listen to incomming connections. The security is then handled by docker. @@ -202,3 +206,28 @@ whitelist Show the current whitelist :returns: json object ``` + +## Advanced API usage using JWT tokens + +!!! Note + The below should be done in an application (a Freqtrade REST API client, which fetches info via API), and is not intended to be used on a regular basis. + +Freqtrade's REST API also offers JWT (JSON Web Tokens). +You can login using the following command, and subsequently use the resulting access_token. + +``` bash +> curl -X POST --user Freqtrader http://localhost:8080/api/v1/token/login +{"access_token":"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE1ODkxMTk2ODEsIm5iZiI6MTU4OTExOTY4MSwianRpIjoiMmEwYmY0NWUtMjhmOS00YTUzLTlmNzItMmM5ZWVlYThkNzc2IiwiZXhwIjoxNTg5MTIwNTgxLCJpZGVudGl0eSI6eyJ1IjoiRnJlcXRyYWRlciJ9LCJmcmVzaCI6ZmFsc2UsInR5cGUiOiJhY2Nlc3MifQ.qt6MAXYIa-l556OM7arBvYJ0SDI9J8bIk3_glDujF5g","refresh_token":"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE1ODkxMTk2ODEsIm5iZiI6MTU4OTExOTY4MSwianRpIjoiZWQ1ZWI3YjAtYjMwMy00YzAyLTg2N2MtNWViMjIxNWQ2YTMxIiwiZXhwIjoxNTkxNzExNjgxLCJpZGVudGl0eSI6eyJ1IjoiRnJlcXRyYWRlciJ9LCJ0eXBlIjoicmVmcmVzaCJ9.d1AT_jYICyTAjD0fiQAr52rkRqtxCjUGEMwlNuuzgNQ"} + +> access_token="eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE1ODkxMTk2ODEsIm5iZiI6MTU4OTExOTY4MSwianRpIjoiMmEwYmY0NWUtMjhmOS00YTUzLTlmNzItMmM5ZWVlYThkNzc2IiwiZXhwIjoxNTg5MTIwNTgxLCJpZGVudGl0eSI6eyJ1IjoiRnJlcXRyYWRlciJ9LCJmcmVzaCI6ZmFsc2UsInR5cGUiOiJhY2Nlc3MifQ.qt6MAXYIa-l556OM7arBvYJ0SDI9J8bIk3_glDujF5g" +# Use access_token for authentication +> curl -X GET --header "Authorization: Bearer ${access_token}" http://localhost:8080/api/v1/count + +``` + +Since the access token has a short timeout (15 min) - the `token/refresh` request should be used periodically to get a fresh access token: + +``` bash +> curl -X POST --header "Authorization: Bearer ${refresh_token}"http://localhost:8080/api/v1/token/refresh +{"access_token":"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE1ODkxMTk5NzQsIm5iZiI6MTU4OTExOTk3NCwianRpIjoiMDBjNTlhMWUtMjBmYS00ZTk0LTliZjAtNWQwNTg2MTdiZDIyIiwiZXhwIjoxNTg5MTIwODc0LCJpZGVudGl0eSI6eyJ1IjoiRnJlcXRyYWRlciJ9LCJmcmVzaCI6ZmFsc2UsInR5cGUiOiJhY2Nlc3MifQ.1seHlII3WprjjclY6DpRhen0rqdF4j6jbvxIhUFaSbs"} +``` diff --git a/docs/sql_cheatsheet.md b/docs/sql_cheatsheet.md index b7b38c3dc..895a0536a 100644 --- a/docs/sql_cheatsheet.md +++ b/docs/sql_cheatsheet.md @@ -1,13 +1,20 @@ # SQL Helper + This page contains some help if you want to edit your sqlite db. ## Install sqlite3 -**Ubuntu/Debian installation** + +Sqlite3 is a terminal based sqlite application. +Feel free to use a visual Database editor like SqliteBrowser if you feel more comfortable with that. + +### Ubuntu/Debian installation + ```bash sudo apt-get install sqlite3 ``` ## Open the DB + ```bash sqlite3 .open @@ -16,45 +23,61 @@ sqlite3 ## Table structure ### List tables + ```bash .tables ``` ### Display table structure + ```bash .schema ``` ### Trade table structure + ```sql -CREATE TABLE trades ( - id INTEGER NOT NULL, - exchange VARCHAR NOT NULL, - pair VARCHAR NOT NULL, - is_open BOOLEAN NOT NULL, - fee_open FLOAT NOT NULL, - fee_close FLOAT NOT NULL, - open_rate FLOAT, - open_rate_requested FLOAT, - close_rate FLOAT, - close_rate_requested FLOAT, - close_profit FLOAT, - stake_amount FLOAT NOT NULL, - amount FLOAT, - open_date DATETIME NOT NULL, - close_date DATETIME, - open_order_id VARCHAR, - stop_loss FLOAT, - initial_stop_loss FLOAT, - stoploss_order_id VARCHAR, - stoploss_last_update DATETIME, - max_rate FLOAT, - sell_reason VARCHAR, - strategy VARCHAR, - ticker_interval INTEGER, - PRIMARY KEY (id), - CHECK (is_open IN (0, 1)) +CREATE TABLE trades + id INTEGER NOT NULL, + exchange VARCHAR NOT NULL, + pair VARCHAR NOT NULL, + is_open BOOLEAN NOT NULL, + fee_open FLOAT NOT NULL, + fee_open_cost FLOAT, + fee_open_currency VARCHAR, + fee_close FLOAT NOT NULL, + fee_close_cost FLOAT, + fee_close_currency VARCHAR, + open_rate FLOAT, + open_rate_requested FLOAT, + open_trade_price FLOAT, + close_rate FLOAT, + close_rate_requested FLOAT, + close_profit FLOAT, + close_profit_abs FLOAT, + stake_amount FLOAT NOT NULL, + amount FLOAT, + open_date DATETIME NOT NULL, + close_date DATETIME, + open_order_id VARCHAR, + stop_loss FLOAT, + stop_loss_pct FLOAT, + initial_stop_loss FLOAT, + initial_stop_loss_pct FLOAT, + stoploss_order_id VARCHAR, + stoploss_last_update DATETIME, + max_rate FLOAT, + min_rate FLOAT, + sell_reason VARCHAR, + strategy VARCHAR, + ticker_interval INTEGER, + PRIMARY KEY (id), + CHECK (is_open IN (0, 1)) ); +CREATE INDEX ix_trades_stoploss_order_id ON trades (stoploss_order_id); +CREATE INDEX ix_trades_pair ON trades (pair); +CREATE INDEX ix_trades_is_open ON trades (is_open); + ``` ## Get all trades in the table diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 39e92d651..69e2256a1 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -20,7 +20,7 @@ It applies a tight timeout for higher priced assets, while allowing more time to The function must return either `True` (cancel order) or `False` (keep order alive). ``` python -from datetime import datetime, timestamp +from datetime import datetime, timedelta from freqtrade.persistence import Trade class Awesomestrategy(IStrategy): @@ -59,7 +59,7 @@ class Awesomestrategy(IStrategy): ### Custom order timeout example (using additional data) ``` python -from datetime import datetime, timestamp +from datetime import datetime from freqtrade.persistence import Trade class Awesomestrategy(IStrategy): diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index c4fc55811..7197b0fba 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -324,67 +324,14 @@ class Awesomestrategy(IStrategy): !!! Note If the data is pair-specific, make sure to use pair as one of the keys in the dictionary. -### Additional data (DataProvider) +*** -The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy. - -All methods return `None` in case of failure (do not raise an exception). - -Please always check the mode of operation to select the correct method to get data (samples see below). - -#### Possible options for DataProvider - -- `available_pairs` - Property with tuples listing cached pairs with their intervals (pair, interval). -- `ohlcv(pair, timeframe)` - Currently cached candle (OHLCV) data for the pair, returns DataFrame or empty DataFrame. -- `historic_ohlcv(pair, timeframe)` - Returns historical data stored on disk. -- `get_pair_dataframe(pair, timeframe)` - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes). -- `orderbook(pair, maximum)` - Returns latest orderbook data for the pair, a dict with bids/asks with a total of `maximum` entries. -- `market(pair)` - Returns market data for the pair: fees, limits, precisions, activity flag, etc. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#markets) for more details on Market data structure. -- `runmode` - Property containing the current runmode. - -#### Example: fetch live / historical candle (OHLCV) data for the first informative pair - -``` python -if self.dp: - inf_pair, inf_timeframe = self.informative_pairs()[0] - informative = self.dp.get_pair_dataframe(pair=inf_pair, - timeframe=inf_timeframe) -``` - -!!! Warning "Warning about backtesting" - Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()` - for the backtesting runmode) provides the full time-range in one go, - so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode). - -!!! Warning "Warning in hyperopt" - This option cannot currently be used during hyperopt. - -#### Orderbook - -``` python -if self.dp: - if self.dp.runmode.value in ('live', 'dry_run'): - ob = self.dp.orderbook(metadata['pair'], 1) - dataframe['best_bid'] = ob['bids'][0][0] - dataframe['best_ask'] = ob['asks'][0][0] -``` - -!!! Warning - The order book is not part of the historic data which means backtesting and hyperopt will not work if this - method is used. - -#### Available Pairs - -``` python -if self.dp: - for pair, timeframe in self.dp.available_pairs: - print(f"available {pair}, {timeframe}") -``` +### Additional data (informative_pairs) #### Get data for non-tradeable pairs Data for additional, informative pairs (reference pairs) can be beneficial for some strategies. -Ohlcv data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see above). +Ohlcv data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see below). These parts will **not** be traded unless they are also specified in the pair whitelist, or have been selected by Dynamic Whitelisting. The pairs need to be specified as tuples in the format `("pair", "interval")`, with pair as the first and time interval as the second argument. @@ -404,6 +351,125 @@ def informative_pairs(self): It is however better to use resampling to longer time-intervals when possible to avoid hammering the exchange with too many requests and risk being blocked. +*** + +### Additional data (DataProvider) + +The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy. + +All methods return `None` in case of failure (do not raise an exception). + +Please always check the mode of operation to select the correct method to get data (samples see below). + +#### Possible options for DataProvider + +- [`available_pairs`](#available_pairs) - Property with tuples listing cached pairs with their intervals (pair, interval). +- [`current_whitelist()`](#current_whitelist) - Returns a current list of whitelisted pairs. Useful for accessing dynamic whitelists (ie. VolumePairlist) +- [`get_pair_dataframe(pair, timeframe)`](#get_pair_dataframepair-timeframe) - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes). +- `historic_ohlcv(pair, timeframe)` - Returns historical data stored on disk. +- `market(pair)` - Returns market data for the pair: fees, limits, precisions, activity flag, etc. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#markets) for more details on the Market data structure. +- `ohlcv(pair, timeframe)` - Currently cached candle (OHLCV) data for the pair, returns DataFrame or empty DataFrame. +- [`orderbook(pair, maximum)`](#orderbookpair-maximum) - Returns latest orderbook data for the pair, a dict with bids/asks with a total of `maximum` entries. +- [`ticker(pair)`](#tickerpair) - Returns current ticker data for the pair. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#price-tickers) for more details on the Ticker data structure. +- `runmode` - Property containing the current runmode. + +#### Example Usages: + +#### *available_pairs* + +``` python +if self.dp: + for pair, timeframe in self.dp.available_pairs: + print(f"available {pair}, {timeframe}") +``` + +#### *current_whitelist()* +Imagine you've developed a strategy that trades the `5m` timeframe using signals generated from a `1d` timeframe on the top 10 volume pairs by volume. + +The strategy might look something like this: + +*Scan through the top 10 pairs by volume using the `VolumePairList` every 5 minutes and use a 14 day ATR to buy and sell.* + +Due to the limited available data, it's very difficult to resample our `5m` candles into daily candles for use in a 14 day ATR. Most exchanges limit us to just 500 candles which effectively gives us around 1.74 daily candles. We need 14 days at least! + +Since we can't resample our data we will have to use an informative pair; and since our whitelist will be dynamic we don't know which pair(s) to use. + +This is where calling `self.dp.current_whitelist()` comes in handy. + +```python +class SampleStrategy(IStrategy): + # strategy init stuff... + + ticker_interval = '5m' + + # more strategy init stuff.. + + def informative_pairs(self): + + # get access to all pairs available in whitelist. + pairs = self.dp.current_whitelist() + # Assign tf to each pair so they can be downloaded and cached for strategy. + informative_pairs = [(pair, '1d') for pair in pairs] + return informative_pairs + + def populate_indicators(self, dataframe, metadata): + # Get the informative pair + informative = self.dp.get_pair_dataframe(pair=metadata['pair'], timeframe='1d') + # Get the 14 day ATR. + atr = ta.ATR(informative, timeperiod=14) + # Do other stuff +``` + +#### *get_pair_dataframe(pair, timeframe)* + +``` python +# fetch live / historical candle (OHLCV) data for the first informative pair +if self.dp: + inf_pair, inf_timeframe = self.informative_pairs()[0] + informative = self.dp.get_pair_dataframe(pair=inf_pair, + timeframe=inf_timeframe) +``` + +!!! Warning "Warning about backtesting" + Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()` + for the backtesting runmode) provides the full time-range in one go, + so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode). + +!!! Warning "Warning in hyperopt" + This option cannot currently be used during hyperopt. + +#### *orderbook(pair, maximum)* + +``` python +if self.dp: + if self.dp.runmode.value in ('live', 'dry_run'): + ob = self.dp.orderbook(metadata['pair'], 1) + dataframe['best_bid'] = ob['bids'][0][0] + dataframe['best_ask'] = ob['asks'][0][0] +``` + +!!! Warning + The order book is not part of the historic data which means backtesting and hyperopt will not work if this + method is used. + +#### *ticker(pair)* + +``` python +if self.dp: + if self.dp.runmode.value in ('live', 'dry_run'): + ticker = self.dp.ticker(metadata['pair']) + dataframe['last_price'] = ticker['last'] + dataframe['volume24h'] = ticker['quoteVolume'] + dataframe['vwap'] = ticker['vwap'] +``` + +!!! Warning + Although the ticker data structure is a part of the ccxt Unified Interface, the values returned by this method can + vary for different exchanges. For instance, many exchanges do not return `vwap` values, the FTX exchange + does not always fills in the `last` field (so it can be None), etc. So you need to carefully verify the ticker + data returned from the exchange and add appropriate error handling / defaults. + +*** ### Additional data (Wallets) The strategy provides access to the `Wallets` object. This contains the current balances on the exchange. @@ -426,6 +492,7 @@ if self.wallets: - `get_used(asset)` - currently tied up balance (open orders) - `get_total(asset)` - total available balance - sum of the 2 above +*** ### Additional data (Trades) A history of Trades can be retrieved in the strategy by querying the database. diff --git a/docs/utils.md b/docs/utils.md index 57210ac7e..7ed31376f 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -521,3 +521,48 @@ Prints JSON data with details for the last best epoch (i.e., the best of all epo ``` freqtrade hyperopt-show --best -n -1 --print-json --no-header ``` + +## Show trades + +Print selected (or all) trades from database to screen. + +``` +usage: freqtrade show-trades [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] [--db-url PATH] + [--trade-ids TRADE_IDS [TRADE_IDS ...]] + [--print-json] + +optional arguments: + -h, --help show this help message and exit + --db-url PATH Override trades database URL, this is useful in custom + deployments (default: `sqlite:///tradesv3.sqlite` for + Live Run mode, `sqlite:///tradesv3.dryrun.sqlite` for + Dry Run). + --trade-ids TRADE_IDS [TRADE_IDS ...] + Specify the list of trade ids. + --print-json Print output in JSON format. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. +``` + +### Examples + +Print trades with id 2 and 3 as json + +``` bash +freqtrade show-trades --db-url sqlite:///tradesv3.sqlite --trade-ids 2 3 --print-json +``` diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index f80c74e05..2d0c7733c 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -19,7 +19,8 @@ from freqtrade.commands.list_commands import (start_list_exchanges, start_list_hyperopts, start_list_markets, start_list_strategies, - start_list_timeframes) + start_list_timeframes, + start_show_trades) from freqtrade.commands.optimize_commands import (start_backtesting, start_edge, start_hyperopt) from freqtrade.commands.pairlist_commands import start_test_pairlist diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index be03bb906..234b719f7 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -64,6 +64,8 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source", "ticker_interval"] +ARGS_SHOW_TRADES = ["db_url", "trade_ids", "print_json"] + ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_list_min_trades", "hyperopt_list_max_trades", "hyperopt_list_min_avg_time", "hyperopt_list_max_avg_time", @@ -78,7 +80,7 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", "list-markets", "list-pairs", "list-strategies", "list-hyperopts", "hyperopt-list", "hyperopt-show", - "plot-dataframe", "plot-profit"] + "plot-dataframe", "plot-profit", "show-trades"] NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"] @@ -163,7 +165,7 @@ class Arguments: start_list_markets, start_list_strategies, start_list_timeframes, start_new_config, start_new_hyperopt, start_new_strategy, - start_plot_dataframe, start_plot_profit, + start_plot_dataframe, start_plot_profit, start_show_trades, start_backtesting, start_hyperopt, start_edge, start_test_pairlist, start_trading) @@ -330,6 +332,15 @@ class Arguments: plot_profit_cmd.set_defaults(func=start_plot_profit) self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd) + # Add show-trades subcommand + show_trades = subparsers.add_parser( + 'show-trades', + help='Show trades.', + parents=[_common_parser], + ) + show_trades.set_defaults(func=start_show_trades) + self._build_args(optionlist=ARGS_SHOW_TRADES, parser=show_trades) + # Add hyperopt-list subcommand hyperopt_list_cmd = subparsers.add_parser( 'hyperopt-list', diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index adb28399a..cca9d7048 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -222,7 +222,7 @@ AVAILABLE_CLI_OPTIONS = { ), "print_json": Arg( '--print-json', - help='Print best result detailization in JSON format.', + help='Print output in JSON format.', action='store_true', default=False, ), @@ -430,6 +430,11 @@ AVAILABLE_CLI_OPTIONS = { choices=["DB", "file"], default="file", ), + "trade_ids": Arg( + '--trade-ids', + help='Specify the list of trade ids.', + nargs='+', + ), # hyperopt-list, hyperopt-show "hyperopt_list_profitable": Arg( '--profitable', diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 327901dc0..e5131f9b2 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -197,3 +197,30 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: args.get('list_pairs_print_json', False) or args.get('print_csv', False)): print(f"{summary_str}.") + + +def start_show_trades(args: Dict[str, Any]) -> None: + """ + Show trades + """ + from freqtrade.persistence import init, Trade + import json + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + if 'db_url' not in config: + raise OperationalException("--db-url is required for this command.") + + logger.info(f'Using DB: "{config["db_url"]}"') + init(config['db_url'], clean_open_orders=False) + tfilter = [] + + if config.get('trade_ids'): + tfilter.append(Trade.id.in_(config['trade_ids'])) + + trades = Trade.get_trades(tfilter).all() + logger.info(f"Printing {len(trades)} Trades: ") + if config.get('print_json', False): + print(json.dumps([trade.to_json() for trade in trades], indent=4)) + else: + for trade in trades: + print(trade) diff --git a/freqtrade/commands/trade_commands.py b/freqtrade/commands/trade_commands.py index 352fac26d..c058e4f9d 100644 --- a/freqtrade/commands/trade_commands.py +++ b/freqtrade/commands/trade_commands.py @@ -18,6 +18,9 @@ def start_trading(args: Dict[str, Any]) -> int: try: worker = Worker(args) worker.run() + except Exception as e: + logger.error(str(e)) + logger.exception("Fatal exception!") except KeyboardInterrupt: logger.info('SIGINT received, aborting ...') finally: diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index cfbba9987..188d79f90 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -356,8 +356,12 @@ class Configuration: self._args_to_config(config, argname='indicators2', logstring='Using indicators2: {}') + self._args_to_config(config, argname='trade_ids', + logstring='Filtering on trade_ids: {}') + self._args_to_config(config, argname='plot_limit', logstring='Limiting plot to: {}') + self._args_to_config(config, argname='trade_source', logstring='Using trades from: {}') diff --git a/freqtrade/constants.py b/freqtrade/constants.py index c238f227b..0c8dde8c4 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -24,6 +24,9 @@ AVAILABLE_DATAHANDLERS = ['json', 'jsongz'] DRY_RUN_WALLET = 1000 MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume'] +# Don't modify sequence of DEFAULT_TRADES_COLUMNS +# it has wide consequences for stored trades files +DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost'] USERPATH_HYPEROPTS = 'hyperopts' USERPATH_STRATEGIES = 'strategies' diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 77371bf27..0ef7955a4 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -1,14 +1,17 @@ """ Functions to convert data from one format to another """ +import itertools import logging from datetime import datetime, timezone -from typing import Any, Dict +from operator import itemgetter +from typing import Any, Dict, List import pandas as pd from pandas import DataFrame, to_datetime -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS +from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, + DEFAULT_TRADES_COLUMNS) logger = logging.getLogger(__name__) @@ -154,7 +157,27 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame: return frame -def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame: +def trades_remove_duplicates(trades: List[List]) -> List[List]: + """ + Removes duplicates from the trades list. + Uses itertools.groupby to avoid converting to pandas. + Tests show it as being pretty efficient on lists of 4M Lists. + :param trades: List of Lists with constants.DEFAULT_TRADES_COLUMNS as columns + :return: same format as above, but with duplicates removed + """ + return [i for i, _ in itertools.groupby(sorted(trades, key=itemgetter(0)))] + + +def trades_dict_to_list(trades: List[Dict]) -> List[List]: + """ + Convert fetch_trades result into a List (to be more memory efficient). + :param trades: List of trades, as returned by ccxt.fetch_trades. + :return: List of Lists, with constants.DEFAULT_TRADES_COLUMNS as columns + """ + return [[t[col] for col in DEFAULT_TRADES_COLUMNS] for t in trades] + + +def trades_to_ohlcv(trades: List, timeframe: str) -> DataFrame: """ Converts trades list to OHLCV list TODO: This should get a dedicated test @@ -164,9 +187,10 @@ def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame: """ from freqtrade.exchange import timeframe_to_minutes timeframe_minutes = timeframe_to_minutes(timeframe) - df = pd.DataFrame(trades) - df['datetime'] = pd.to_datetime(df['datetime']) - df = df.set_index('datetime') + df = pd.DataFrame(trades, columns=DEFAULT_TRADES_COLUMNS) + df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', + utc=True,) + df = df.set_index('timestamp') df_new = df['price'].resample(f'{timeframe_minutes}min').ohlc() df_new['volume'] = df['amount'].resample(f'{timeframe_minutes}min').sum() diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 1df710152..7ada4f642 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -10,6 +10,7 @@ from typing import Any, Dict, List, Optional, Tuple from pandas import DataFrame from freqtrade.data.history import load_pair_history +from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import Exchange from freqtrade.state import RunMode @@ -18,9 +19,10 @@ logger = logging.getLogger(__name__) class DataProvider: - def __init__(self, config: dict, exchange: Exchange) -> None: + def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None: self._config = config self._exchange = exchange + self._pairlists = pairlists def refresh(self, pairlist: List[Tuple[str, str]], @@ -95,10 +97,14 @@ class DataProvider: def ticker(self, pair: str): """ - Return last ticker data + Return last ticker data from exchange + :param pair: Pair to get the data for + :return: Ticker dict from exchange or empty dict if ticker is not available for the pair """ - # TODO: Implement me - pass + try: + return self._exchange.fetch_ticker(pair) + except DependencyException: + return {} def orderbook(self, pair: str, maximum: int) -> Dict[str, List]: """ @@ -116,3 +122,17 @@ class DataProvider: can be "live", "dry-run", "backtest", "edgecli", "hyperopt" or "other". """ return RunMode(self._config.get('runmode', RunMode.OTHER)) + + def current_whitelist(self) -> List[str]: + """ + fetch latest available whitelist. + + Useful when you have a large whitelist and need to call each pair as an informative pair. + As available pairs does not show whitelist until after informative pairs have been cached. + :return: list of pairs in whitelist + """ + + if self._pairlists: + return self._pairlists.whitelist + else: + raise OperationalException("Dataprovider was not initialized with a pairlist provider.") diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 89d29d33b..4f3f75a87 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -9,10 +9,13 @@ from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS -from freqtrade.data.converter import ohlcv_to_dataframe, trades_to_ohlcv +from freqtrade.data.converter import (ohlcv_to_dataframe, + trades_remove_duplicates, + trades_to_ohlcv) from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange +from freqtrade.misc import format_ms_time logger = logging.getLogger(__name__) @@ -257,27 +260,40 @@ def _download_trades_history(exchange: Exchange, """ try: - since = timerange.startts * 1000 if timerange and timerange.starttype == 'date' else None + since = timerange.startts * 1000 if \ + (timerange and timerange.starttype == 'date') else int(arrow.utcnow().shift( + days=-30).float_timestamp) * 1000 trades = data_handler.trades_load(pair) - from_id = trades[-1]['id'] if trades else None + # TradesList columns are defined in constants.DEFAULT_TRADES_COLUMNS + # DEFAULT_TRADES_COLUMNS: 0 -> timestamp + # DEFAULT_TRADES_COLUMNS: 1 -> id - logger.debug("Current Start: %s", trades[0]['datetime'] if trades else 'None') - logger.debug("Current End: %s", trades[-1]['datetime'] if trades else 'None') + from_id = trades[-1][1] if trades else None + if trades and since < trades[-1][0]: + # Reset since to the last available point + # - 5 seconds (to ensure we're getting all trades) + since = trades[-1][0] - (5 * 1000) + logger.info(f"Using last trade date -5s - Downloading trades for {pair} " + f"since: {format_ms_time(since)}.") + + logger.debug(f"Current Start: {format_ms_time(trades[0][0]) if trades else 'None'}") + logger.debug(f"Current End: {format_ms_time(trades[-1][0]) if trades else 'None'}") + logger.info(f"Current Amount of trades: {len(trades)}") # Default since_ms to 30 days if nothing is given new_trades = exchange.get_historic_trades(pair=pair, - since=since if since else - int(arrow.utcnow().shift( - days=-30).float_timestamp) * 1000, + since=since, from_id=from_id, ) trades.extend(new_trades[1]) + # Remove duplicates to make sure we're not storing data we don't need + trades = trades_remove_duplicates(trades) data_handler.trades_store(pair, data=trades) - logger.debug("New Start: %s", trades[0]['datetime']) - logger.debug("New End: %s", trades[-1]['datetime']) + logger.debug(f"New Start: {format_ms_time(trades[0][0])}") + logger.debug(f"New End: {format_ms_time(trades[-1][0])}") logger.info(f"New Amount of trades: {len(trades)}") return True diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 1bb4d5971..d5d7c16db 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -8,16 +8,20 @@ from abc import ABC, abstractclassmethod, abstractmethod from copy import deepcopy from datetime import datetime, timezone from pathlib import Path -from typing import Dict, List, Optional, Type +from typing import List, Optional, Type from pandas import DataFrame from freqtrade.configuration import TimeRange -from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe +from freqtrade.data.converter import (clean_ohlcv_dataframe, + trades_remove_duplicates, trim_dataframe) from freqtrade.exchange import timeframe_to_seconds logger = logging.getLogger(__name__) +# Type for trades list +TradeList = List[List] + class IDataHandler(ABC): @@ -89,23 +93,25 @@ class IDataHandler(ABC): """ @abstractmethod - def trades_store(self, pair: str, data: List[Dict]) -> None: + def trades_store(self, pair: str, data: TradeList) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename - :param data: List of Dicts containing trade data + :param data: List of Lists containing trade data, + column sequence as in DEFAULT_TRADES_COLUMNS """ @abstractmethod - def trades_append(self, pair: str, data: List[Dict]): + def trades_append(self, pair: str, data: TradeList): """ Append data to existing files :param pair: Pair - used for filename - :param data: List of Dicts containing trade data + :param data: List of Lists containing trade data, + column sequence as in DEFAULT_TRADES_COLUMNS """ @abstractmethod - def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> List[Dict]: + def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: """ Load a pair from file, either .json.gz or .json :param pair: Load trades for this pair @@ -121,6 +127,16 @@ class IDataHandler(ABC): :return: True when deleted, false if file did not exist. """ + def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + """ + Load a pair from file, either .json.gz or .json + Removes duplicates in the process. + :param pair: Load trades for this pair + :param timerange: Timerange to load trades for - currently not implemented + :return: List of trades + """ + return trades_remove_duplicates(self._trades_load(pair, timerange=timerange)) + def ohlcv_load(self, pair, timeframe: str, timerange: Optional[TimeRange] = None, fill_missing: bool = True, diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 363b03958..01320f129 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -1,6 +1,7 @@ +import logging import re from pathlib import Path -from typing import Dict, List, Optional +from typing import List, Optional import numpy as np from pandas import DataFrame, read_json, to_datetime @@ -8,8 +9,11 @@ from pandas import DataFrame, read_json, to_datetime from freqtrade import misc from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS +from freqtrade.data.converter import trades_dict_to_list -from .idatahandler import IDataHandler +from .idatahandler import IDataHandler, TradeList + +logger = logging.getLogger(__name__) class JsonDataHandler(IDataHandler): @@ -113,24 +117,26 @@ class JsonDataHandler(IDataHandler): # Check if regex found something and only return these results to avoid exceptions. return [match[0].replace('_', '/') for match in _tmp if match] - def trades_store(self, pair: str, data: List[Dict]) -> None: + def trades_store(self, pair: str, data: TradeList) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename - :param data: List of Dicts containing trade data + :param data: List of Lists containing trade data, + column sequence as in DEFAULT_TRADES_COLUMNS """ filename = self._pair_trades_filename(self._datadir, pair) misc.file_dump_json(filename, data, is_zip=self._use_zip) - def trades_append(self, pair: str, data: List[Dict]): + def trades_append(self, pair: str, data: TradeList): """ Append data to existing files :param pair: Pair - used for filename - :param data: List of Dicts containing trade data + :param data: List of Lists containing trade data, + column sequence as in DEFAULT_TRADES_COLUMNS """ raise NotImplementedError() - def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> List[Dict]: + def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: """ Load a pair from file, either .json.gz or .json # TODO: respect timerange ... @@ -140,9 +146,15 @@ class JsonDataHandler(IDataHandler): """ filename = self._pair_trades_filename(self._datadir, pair) tradesdata = misc.file_load_json(filename) + if not tradesdata: return [] + if isinstance(tradesdata[0], dict): + # Convert trades dict to list + logger.info("Old trades format detected - converting") + tradesdata = trades_dict_to_list(tradesdata) + pass return tradesdata def trades_purge(self, pair: str) -> bool: diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index 5305e23cf..c19d4552a 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -238,20 +238,9 @@ class Edge: :param result Dataframe :return: result Dataframe """ - - # stake and fees - # stake = 0.015 - # 0.05% is 0.0005 - # fee = 0.001 - - # we set stake amount to an arbitrary amount. - # as it doesn't change the calculation. - # all returned values are relative. - # they are defined as ratios. + # We set stake amount to an arbitrary amount, as it doesn't change the calculation. + # All returned values are relative, they are defined as ratios. stake = 0.015 - fee = self.fee - open_fee = fee / 2 - close_fee = fee / 2 result['trade_duration'] = result['close_time'] - result['open_time'] @@ -262,12 +251,12 @@ class Edge: # Buy Price result['buy_vol'] = stake / result['open_rate'] # How many target are we buying - result['buy_fee'] = stake * open_fee + result['buy_fee'] = stake * self.fee result['buy_spend'] = stake + result['buy_fee'] # How much we're spending # Sell price result['sell_sum'] = result['buy_vol'] * result['close_rate'] - result['sell_fee'] = result['sell_sum'] * close_fee + result['sell_fee'] = result['sell_sum'] * self.fee result['sell_take'] = result['sell_sum'] - result['sell_fee'] # profit_ratio diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 875628af9..37183dc2c 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -72,7 +72,7 @@ class Binance(Exchange): rate = self.price_to_precision(pair, rate) order = self._api.create_order(symbol=pair, type=ordertype, side='sell', - amount=amount, price=stop_price, params=params) + amount=amount, price=rate, params=params) logger.info('stoploss limit order added for %s. ' 'stop price: %s. limit: %s', pair, stop_price, rate) return order diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 1a0565959..6ad7ad582 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -18,12 +18,11 @@ from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, decimal_to_precision) from pandas import DataFrame -from freqtrade.data.converter import ohlcv_to_dataframe +from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list from freqtrade.exceptions import (DependencyException, InvalidOrderException, OperationalException, TemporaryError) from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async -from freqtrade.misc import deep_merge_dicts - +from freqtrade.misc import deep_merge_dicts, safe_value_fallback CcxtModuleType = Any @@ -472,26 +471,31 @@ class Exchange: 'pair': pair, 'price': rate, 'amount': _amount, - "cost": _amount * rate, + 'cost': _amount * rate, 'type': ordertype, 'side': side, 'remaining': _amount, 'datetime': arrow.utcnow().isoformat(), 'status': "closed" if ordertype == "market" else "open", 'fee': None, - "info": {} + 'info': {} } - self._store_dry_order(dry_order) + self._store_dry_order(dry_order, pair) # Copy order and close it - so the returned order is open unless it's a market order return dry_order - def _store_dry_order(self, dry_order: Dict) -> None: + def _store_dry_order(self, dry_order: Dict, pair: str) -> None: closed_order = dry_order.copy() - if closed_order["type"] in ["market", "limit"]: + if closed_order['type'] in ["market", "limit"]: closed_order.update({ - "status": "closed", - "filled": closed_order["amount"], - "remaining": 0 + 'status': 'closed', + 'filled': closed_order['amount'], + 'remaining': 0, + 'fee': { + 'currency': self.get_pair_quote_currency(pair), + 'cost': dry_order['cost'] * self.get_fee(pair), + 'rate': self.get_fee(pair) + } }) if closed_order["type"] in ["stop_loss_limit"]: closed_order["info"].update({"stopPrice": closed_order["price"]}) @@ -769,7 +773,7 @@ class Exchange: @retrier_async async def _async_fetch_trades(self, pair: str, since: Optional[int] = None, - params: Optional[dict] = None) -> List[Dict]: + params: Optional[dict] = None) -> List[List]: """ Asyncronously gets trade history using fetch_trades. Handles exchange errors, does one call to the exchange. @@ -789,7 +793,7 @@ class Exchange: '(' + arrow.get(since // 1000).isoformat() + ') ' if since is not None else '' ) trades = await self._api_async.fetch_trades(pair, since=since, limit=1000) - return trades + return trades_dict_to_list(trades) except ccxt.NotSupported as e: raise OperationalException( f'Exchange {self._api.name} does not support fetching historical trade data.' @@ -803,7 +807,7 @@ class Exchange: async def _async_get_trade_history_id(self, pair: str, until: int, since: Optional[int] = None, - from_id: Optional[str] = None) -> Tuple[str, List[Dict]]: + from_id: Optional[str] = None) -> Tuple[str, List[List]]: """ Asyncronously gets trade history using fetch_trades use this when exchange uses id-based iteration (check `self._trades_pagination`) @@ -814,7 +818,7 @@ class Exchange: returns tuple: (pair, trades-list) """ - trades: List[Dict] = [] + trades: List[List] = [] if not from_id: # Fetch first elements using timebased method to get an ID to paginate on @@ -823,7 +827,9 @@ class Exchange: # e.g. Binance returns the "last 1000" candles within a 1h time interval # - so we will miss the first trades. t = await self._async_fetch_trades(pair, since=since) - from_id = t[-1]['id'] + # DEFAULT_TRADES_COLUMNS: 0 -> timestamp + # DEFAULT_TRADES_COLUMNS: 1 -> id + from_id = t[-1][1] trades.extend(t[:-1]) while True: t = await self._async_fetch_trades(pair, @@ -831,21 +837,21 @@ class Exchange: if len(t): # Skip last id since its the key for the next call trades.extend(t[:-1]) - if from_id == t[-1]['id'] or t[-1]['timestamp'] > until: + if from_id == t[-1][1] or t[-1][0] > until: logger.debug(f"Stopping because from_id did not change. " - f"Reached {t[-1]['timestamp']} > {until}") + f"Reached {t[-1][0]} > {until}") # Reached the end of the defined-download period - add last trade as well. trades.extend(t[-1:]) break - from_id = t[-1]['id'] + from_id = t[-1][1] else: break return (pair, trades) async def _async_get_trade_history_time(self, pair: str, until: int, - since: Optional[int] = None) -> Tuple[str, List]: + since: Optional[int] = None) -> Tuple[str, List[List]]: """ Asyncronously gets trade history using fetch_trades, when the exchange uses time-based iteration (check `self._trades_pagination`) @@ -855,16 +861,18 @@ class Exchange: returns tuple: (pair, trades-list) """ - trades: List[Dict] = [] + trades: List[List] = [] + # DEFAULT_TRADES_COLUMNS: 0 -> timestamp + # DEFAULT_TRADES_COLUMNS: 1 -> id while True: t = await self._async_fetch_trades(pair, since=since) if len(t): - since = t[-1]['timestamp'] + since = t[-1][1] trades.extend(t) # Reached the end of the defined-download period - if until and t[-1]['timestamp'] > until: + if until and t[-1][0] > until: logger.debug( - f"Stopping because until was reached. {t[-1]['timestamp']} > {until}") + f"Stopping because until was reached. {t[-1][0]} > {until}") break else: break @@ -874,7 +882,7 @@ class Exchange: async def _async_get_trade_history(self, pair: str, since: Optional[int] = None, until: Optional[int] = None, - from_id: Optional[str] = None) -> Tuple[str, List[Dict]]: + from_id: Optional[str] = None) -> Tuple[str, List[List]]: """ Async wrapper handling downloading trades using either time or id based methods. """ @@ -1041,9 +1049,9 @@ class Exchange: return matched_trades - except ccxt.NetworkError as e: + except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( - f'Could not get trades due to networking error. Message: {e}') from e + f'Could not get trades due to {e.__class__.__name__}. Message: {e}') from e except ccxt.BaseError as e: raise OperationalException(e) from e @@ -1063,6 +1071,61 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + @staticmethod + def order_has_fee(order: Dict) -> bool: + """ + Verifies if the passed in order dict has the needed keys to extract fees, + and that these keys (currency, cost) are not empty. + :param order: Order or trade (one trade) dict + :return: True if the fee substructure contains currency and cost, false otherwise + """ + if not isinstance(order, dict): + return False + return ('fee' in order and order['fee'] is not None + and (order['fee'].keys() >= {'currency', 'cost'}) + and order['fee']['currency'] is not None + and order['fee']['cost'] is not None + ) + + def calculate_fee_rate(self, order: Dict) -> Optional[float]: + """ + Calculate fee rate if it's not given by the exchange. + :param order: Order or trade (one trade) dict + """ + if order['fee'].get('rate') is not None: + return order['fee'].get('rate') + fee_curr = order['fee']['currency'] + # Calculate fee based on order details + if fee_curr in self.get_pair_base_currency(order['symbol']): + # Base currency - divide by amount + return round( + order['fee']['cost'] / safe_value_fallback(order, order, 'filled', 'amount'), 8) + elif fee_curr in self.get_pair_quote_currency(order['symbol']): + # Quote currency - divide by cost + return round(order['fee']['cost'] / order['cost'], 8) + else: + # If Fee currency is a different currency + try: + comb = self.get_valid_pair_combination(fee_curr, self._config['stake_currency']) + tick = self.fetch_ticker(comb) + + fee_to_quote_rate = safe_value_fallback(tick, tick, 'last', 'ask') + return round((order['fee']['cost'] * fee_to_quote_rate) / order['cost'], 8) + except DependencyException: + return None + + def extract_cost_curr_rate(self, order: Dict) -> Tuple[float, str, Optional[float]]: + """ + Extract tuple of cost, currency, rate. + Requires order_has_fee to run first! + :param order: Order or trade (one trade) dict + :return: Tuple with cost, currency, rate of the given fee dict + """ + return (order['fee']['cost'], + order['fee']['currency'], + self.calculate_fee_rate(order)) + # calculate rate ? (order['fee']['cost'] / (order['amount'] * order['price'])) + def is_exchange_bad(exchange_name: str) -> bool: return exchange_name in BAD_EXCHANGES diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5b8d59ed2..cad8b4aea 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -54,8 +54,11 @@ class FreqtradeBot: # Init objects self.config = config - self._sell_rate_cache = TTLCache(maxsize=100, ttl=5) - self._buy_rate_cache = TTLCache(maxsize=100, ttl=5) + # Cache values for 1800 to avoid frequent polling of the exchange for prices + # Caching only applies to RPC methods, so prices for open trades are still + # refreshed once every iteration. + self._sell_rate_cache = TTLCache(maxsize=100, ttl=1800) + self._buy_rate_cache = TTLCache(maxsize=100, ttl=1800) self.strategy: IStrategy = StrategyResolver.load_strategy(self.config) @@ -68,15 +71,15 @@ class FreqtradeBot: self.wallets = Wallets(self.config, self.exchange) - self.dataprovider = DataProvider(self.config, self.exchange) + self.pairlists = PairListManager(self.exchange, self.config) + + self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists) # Attach Dataprovider to Strategy baseclass IStrategy.dp = self.dataprovider # Attach Wallets to Strategy baseclass IStrategy.wallets = self.wallets - self.pairlists = PairListManager(self.exchange, self.config) - # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ self.config.get('edge', {}).get('enabled', False) else None @@ -620,7 +623,7 @@ class FreqtradeBot: trades_closed += 1 continue # Check if we can sell our current pair - if trade.open_order_id is None and self.handle_trade(trade): + if trade.open_order_id is None and trade.is_open and self.handle_trade(trade): trades_closed += 1 except DependencyException as exception: @@ -762,7 +765,7 @@ class FreqtradeBot: # We check if stoploss order is fulfilled if stoploss_order and stoploss_order['status'] == 'closed': trade.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value - trade.update(stoploss_order) + self.update_trade_state(trade, stoploss_order, sl_order=True) # Lock pair for one candle to prevent immediate rebuys self.strategy.lock_pair(trade.pair, timeframe_to_next_date(self.config['ticker_interval'])) @@ -922,7 +925,8 @@ class FreqtradeBot: """ was_trade_fully_canceled = False - if order['status'] != 'canceled': + # Cancelled orders may have the status of 'canceled' or 'closed' + if order['status'] not in ('canceled', 'closed'): reason = constants.CANCEL_REASON['TIMEOUT'] corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair, trade.amount) @@ -933,7 +937,10 @@ class FreqtradeBot: logger.info('Buy order %s for %s.', reason, trade) - if safe_value_fallback(corder, order, 'remaining', 'remaining') == order['amount']: + # Using filled to determine the filled amount + filled_amount = safe_value_fallback(corder, order, 'filled', 'filled') + + if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC): logger.info('Buy order fully cancelled. Removing %s from database.', trade) # if trade is not partially completed, just delete the trade Trade.session.delete(trade) @@ -945,8 +952,7 @@ class FreqtradeBot: # cancel_order may not contain the full order dict, so we need to fallback # to the order dict aquired before cancelling. # we need to fall back to the values from order if corder does not contain these keys. - trade.amount = order['amount'] - safe_value_fallback(corder, order, - 'remaining', 'remaining') + trade.amount = filled_amount trade.stake_amount = trade.amount * trade.open_rate self.update_trade_state(trade, corder, trade.amount) @@ -966,11 +972,15 @@ class FreqtradeBot: Sell cancel - cancel order and update trade :return: Reason for cancel """ - # if trade is not partially completed, just cancel the trade + # if trade is not partially completed, just cancel the order if order['remaining'] == order['amount'] or order.get('filled') == 0.0: if not self.exchange.check_order_canceled_empty(order): - # if trade is not partially completed, just delete the trade - self.exchange.cancel_order(trade.open_order_id, trade.pair) + try: + # if trade is not partially completed, just delete the order + self.exchange.cancel_order(trade.open_order_id, trade.pair) + except InvalidOrderException: + logger.exception(f"Could not cancel sell order {trade.open_order_id}") + return 'error cancelling order' logger.info('Sell order %s for %s.', reason, trade) else: reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE'] @@ -1014,7 +1024,7 @@ class FreqtradeBot: if wallet_amount >= amount: return amount elif wallet_amount > amount * 0.98: - logger.info(f"{pair} - Falling back to wallet-amount.") + logger.info(f"{pair} - Falling back to wallet-amount {wallet_amount} -> {amount}.") return wallet_amount else: raise DependencyException( @@ -1064,7 +1074,7 @@ class FreqtradeBot: trade.sell_reason = sell_reason.value # In case of market sell orders the order can be closed immediately if order.get('status', 'unknown') == 'closed': - trade.update(order) + self.update_trade_state(trade, order) Trade.session.flush() # Lock pair for one candle to prevent immediate rebuys @@ -1155,7 +1165,7 @@ class FreqtradeBot: # def update_trade_state(self, trade: Trade, action_order: dict = None, - order_amount: float = None) -> bool: + order_amount: float = None, sl_order: bool = False) -> bool: """ Checks trades with open orders and updates the amount if necessary Handles closing both buy and sell orders. @@ -1163,84 +1173,125 @@ class FreqtradeBot: """ # Get order details for actual price per unit if trade.open_order_id: - # Update trade with order values - logger.info('Found open order for %s', trade) - try: - order = action_order or self.exchange.get_order(trade.open_order_id, trade.pair) - except InvalidOrderException as exception: - logger.warning('Unable to fetch order %s: %s', trade.open_order_id, exception) - return False - # Try update amount (binance-fix) - try: - new_amount = self.get_real_amount(trade, order, order_amount) - if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC): - order['amount'] = new_amount - order.pop('filled', None) - # Fee was applied, so set to 0 - trade.fee_open = 0 - trade.recalc_open_trade_price() - except DependencyException as exception: - logger.warning("Could not update trade amount: %s", exception) + order_id = trade.open_order_id + elif trade.stoploss_order_id and sl_order: + order_id = trade.stoploss_order_id + else: + return False + # Update trade with order values + logger.info('Found open order for %s', trade) + try: + order = action_order or self.exchange.get_order(order_id, trade.pair) + except InvalidOrderException as exception: + logger.warning('Unable to fetch order %s: %s', order_id, exception) + return False + # Try update amount (binance-fix) + try: + new_amount = self.get_real_amount(trade, order, order_amount) + if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC): + order['amount'] = new_amount + order.pop('filled', None) + trade.recalc_open_trade_price() + except DependencyException as exception: + logger.warning("Could not update trade amount: %s", exception) - if self.exchange.check_order_canceled_empty(order): - # Trade has been cancelled on exchange - # Handling of this will happen in check_handle_timeout. - return True - trade.update(order) - - # Updating wallets when order is closed - if not trade.is_open: - self.wallets.update() + if self.exchange.check_order_canceled_empty(order): + # Trade has been cancelled on exchange + # Handling of this will happen in check_handle_timeout. + return True + trade.update(order) + # Updating wallets when order is closed + if not trade.is_open: + self.wallets.update() return False + def apply_fee_conditional(self, trade: Trade, trade_base_currency: str, + amount: float, fee_abs: float) -> float: + """ + Applies the fee to amount (either from Order or from Trades). + Can eat into dust if more than the required asset is available. + """ + self.wallets.update() + if fee_abs != 0 and self.wallets.get_free(trade_base_currency) >= amount: + # Eat into dust if we own more than base currency + logger.info(f"Fee amount for {trade} was in base currency - " + f"Eating Fee {fee_abs} into dust.") + elif fee_abs != 0: + real_amount = self.exchange.amount_to_precision(trade.pair, amount - fee_abs) + logger.info(f"Applying fee on amount for {trade} " + f"(from {amount} to {real_amount}).") + return real_amount + return amount + def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float: """ - Get real amount for the trade + Detect and update trade fee. + Calls trade.update_fee() uppon correct detection. + Returns modified amount if the fee was taken from the destination currency. Necessary for exchanges which charge fees in base currency (e.g. binance) + :return: identical (or new) amount for the trade """ + # Init variables if order_amount is None: order_amount = order['amount'] # Only run for closed orders - if trade.fee_open == 0 or order['status'] == 'open': + if trade.fee_updated(order.get('side', '')) or order['status'] == 'open': return order_amount trade_base_currency = self.exchange.get_pair_base_currency(trade.pair) # use fee from order-dict if possible - if ('fee' in order and order['fee'] is not None and - (order['fee'].keys() >= {'currency', 'cost'})): - if (order['fee']['currency'] is not None and - order['fee']['cost'] is not None and - trade_base_currency == order['fee']['currency']): - new_amount = order_amount - order['fee']['cost'] - logger.info("Applying fee on amount for %s (from %s to %s) from Order", - trade, order['amount'], new_amount) - return new_amount + if self.exchange.order_has_fee(order): + fee_cost, fee_currency, fee_rate = self.exchange.extract_cost_curr_rate(order) + logger.info(f"Fee for Trade {trade} [{order.get('side')}]: " + f"{fee_cost:.8g} {fee_currency} - rate: {fee_rate}") - # Fallback to Trades + trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', '')) + if trade_base_currency == fee_currency: + # Apply fee to amount + return self.apply_fee_conditional(trade, trade_base_currency, + amount=order_amount, fee_abs=fee_cost) + return order_amount + return self.fee_detection_from_trades(trade, order, order_amount) + + def fee_detection_from_trades(self, trade: Trade, order: Dict, order_amount: float) -> float: + """ + fee-detection fallback to Trades. Parses result of fetch_my_trades to get correct fee. + """ trades = self.exchange.get_trades_for_order(trade.open_order_id, trade.pair, trade.open_date) if len(trades) == 0: logger.info("Applying fee on amount for %s failed: myTrade-Dict empty found", trade) return order_amount + fee_currency = None amount = 0 - fee_abs = 0 + fee_abs = 0.0 + fee_cost = 0.0 + trade_base_currency = self.exchange.get_pair_base_currency(trade.pair) + fee_rate_array: List[float] = [] for exectrade in trades: amount += exectrade['amount'] - if ("fee" in exectrade and exectrade['fee'] is not None and - (exectrade['fee'].keys() >= {'currency', 'cost'})): + if self.exchange.order_has_fee(exectrade): + fee_cost_, fee_currency, fee_rate_ = self.exchange.extract_cost_curr_rate(exectrade) + fee_cost += fee_cost_ + if fee_rate_ is not None: + fee_rate_array.append(fee_rate_) # only applies if fee is in quote currency! - if (exectrade['fee']['currency'] is not None and - exectrade['fee']['cost'] is not None and - trade_base_currency == exectrade['fee']['currency']): - fee_abs += exectrade['fee']['cost'] + if trade_base_currency == fee_currency: + fee_abs += fee_cost_ + # Ensure at least one trade was found: + if fee_currency: + # fee_rate should use mean + fee_rate = sum(fee_rate_array) / float(len(fee_rate_array)) if fee_rate_array else None + trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', '')) if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC): logger.warning(f"Amount {amount} does not match amount {trade.amount}") raise DependencyException("Half bought? Amounts don't match") - real_amount = amount - fee_abs + if fee_abs != 0: - logger.info(f"Applying fee on amount for {trade} " - f"(from {order_amount} to {real_amount}) from Trades") - return real_amount + return self.apply_fee_conditional(trade, trade_base_currency, + amount=amount, fee_abs=fee_abs) + else: + return amount diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 02695d1aa..3a28de785 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -387,12 +387,19 @@ class Hyperopt: trials = json_normalize(results, max_level=1) trials['Best'] = '' trials['Stake currency'] = config['stake_currency'] - trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count', - 'results_metrics.avg_profit', 'results_metrics.total_profit', - 'Stake currency', 'results_metrics.profit', 'results_metrics.duration', - 'loss', 'is_initial_point', 'is_best']] - trials.columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit', 'Stake currency', - 'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best'] + + base_metrics = ['Best', 'current_epoch', 'results_metrics.trade_count', + 'results_metrics.avg_profit', 'results_metrics.total_profit', + 'Stake currency', 'results_metrics.profit', 'results_metrics.duration', + 'loss', 'is_initial_point', 'is_best'] + param_metrics = [("params_dict."+param) for param in results[0]['params_dict'].keys()] + trials = trials[base_metrics + param_metrics] + + base_columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit', 'Stake currency', + 'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best'] + param_columns = list(results[0]['params_dict'].keys()) + trials.columns = base_columns + param_columns + trials['is_profit'] = False trials.loc[trials['is_initial_point'], 'Best'] = '*' trials.loc[trials['is_best'], 'Best'] = 'Best' diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index fb314f439..ea34fd5bf 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -86,11 +86,15 @@ def check_migrate(engine) -> None: logger.debug(f'trying {table_back_name}') # Check for latest column - if not has_column(cols, 'close_profit_abs'): + if not has_column(cols, 'fee_close_cost'): logger.info(f'Running database migration - backup available as {table_back_name}') fee_open = get_column_def(cols, 'fee_open', 'fee') + fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null') + fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null') fee_close = get_column_def(cols, 'fee_close', 'fee') + fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null') + fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null') open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null') stop_loss = get_column_def(cols, 'stop_loss', '0.0') @@ -120,7 +124,9 @@ def check_migrate(engine) -> None: # Copy data back - following the correct schema engine.execute(f"""insert into trades - (id, exchange, pair, is_open, fee_open, fee_close, open_rate, + (id, exchange, pair, is_open, + fee_open, fee_open_cost, fee_open_currency, + fee_close, fee_close_cost, fee_open_currency, open_rate, open_rate_requested, close_rate, close_rate_requested, close_profit, stake_amount, amount, open_date, close_date, open_order_id, stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, @@ -136,7 +142,9 @@ def check_migrate(engine) -> None: else pair end pair, - is_open, {fee_open} fee_open, {fee_close} fee_close, + is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost, + {fee_open_currency} fee_open_currency, {fee_close} fee_close, + {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency, open_rate, {open_rate_requested} open_rate_requested, close_rate, {close_rate_requested} close_rate_requested, close_profit, stake_amount, amount, open_date, close_date, open_order_id, @@ -185,7 +193,11 @@ class Trade(_DECL_BASE): pair = Column(String, nullable=False, index=True) is_open = Column(Boolean, nullable=False, default=True, index=True) fee_open = Column(Float, nullable=False, default=0.0) + fee_open_cost = Column(Float, nullable=True) + fee_open_currency = Column(String, nullable=True) fee_close = Column(Float, nullable=False, default=0.0) + fee_close_cost = Column(Float, nullable=True) + fee_close_currency = Column(String, nullable=True) open_rate = Column(Float) open_rate_requested = Column(Float) # open_trade_price - calculated via _calc_open_trade_price @@ -235,7 +247,11 @@ class Trade(_DECL_BASE): 'pair': self.pair, 'is_open': self.is_open, 'fee_open': self.fee_open, + 'fee_open_cost': self.fee_open_cost, + 'fee_open_currency': self.fee_open_currency, 'fee_close': self.fee_close, + 'fee_close_cost': self.fee_close_cost, + 'fee_close_currency': self.fee_close_currency, 'open_date_hum': arrow.get(self.open_date).humanize(), 'open_date': self.open_date.strftime("%Y-%m-%d %H:%M:%S"), 'close_date_hum': (arrow.get(self.close_date).humanize() @@ -360,6 +376,35 @@ class Trade(_DECL_BASE): self ) + def update_fee(self, fee_cost: float, fee_currency: Optional[str], fee_rate: Optional[float], + side: str) -> None: + """ + Update Fee parameters. Only acts once per side + """ + if side == 'buy' and self.fee_open_currency is None: + self.fee_open_cost = fee_cost + self.fee_open_currency = fee_currency + if fee_rate is not None: + self.fee_open = fee_rate + # Assume close-fee will fall into the same fee category and take an educated guess + self.fee_close = fee_rate + elif side == 'sell' and self.fee_close_currency is None: + self.fee_close_cost = fee_cost + self.fee_close_currency = fee_currency + if fee_rate is not None: + self.fee_close = fee_rate + + def fee_updated(self, side: str) -> bool: + """ + Verify if this side (buy / sell) has already been updated + """ + if side == 'buy': + return self.fee_open_currency is not None + elif side == 'sell': + return self.fee_close_currency is not None + else: + return False + def _calc_open_trade_price(self) -> float: """ Calculate the open_rate including open_fee. diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index 0335bb151..61eacf639 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -2,11 +2,17 @@ import logging import threading from datetime import date, datetime from ipaddress import IPv4Address -from typing import Dict, Callable, Any +from typing import Any, Callable, Dict from arrow import Arrow from flask import Flask, jsonify, request from flask.json import JSONEncoder +from flask_cors import CORS +from flask_jwt_extended import (JWTManager, create_access_token, + create_refresh_token, get_jwt_identity, + jwt_refresh_token_required, + verify_jwt_in_request_optional) +from werkzeug.security import safe_str_cmp from werkzeug.serving import make_server from freqtrade.__init__ import __version__ @@ -38,9 +44,9 @@ class ArrowJSONEncoder(JSONEncoder): def require_login(func: Callable[[Any, Any], Any]): def func_wrapper(obj, *args, **kwargs): - + verify_jwt_in_request_optional() auth = request.authorization - if auth and obj.check_auth(auth.username, auth.password): + if get_jwt_identity() or auth and obj.check_auth(auth.username, auth.password): return func(obj, *args, **kwargs) else: return jsonify({"error": "Unauthorized"}), 401 @@ -70,8 +76,8 @@ class ApiServer(RPC): """ def check_auth(self, username, password): - return (username == self._config['api_server'].get('username') and - password == self._config['api_server'].get('password')) + return (safe_str_cmp(username, self._config['api_server'].get('username')) and + safe_str_cmp(password, self._config['api_server'].get('password'))) def __init__(self, freqtrade) -> None: """ @@ -83,6 +89,13 @@ class ApiServer(RPC): self._config = freqtrade.config self.app = Flask(__name__) + self._cors = CORS(self.app, resources={r"/api/*": {"origins": "*"}}) + + # Setup the Flask-JWT-Extended extension + self.app.config['JWT_SECRET_KEY'] = self._config['api_server'].get( + 'jwt_secret_key', 'super-secret') + + self.jwt = JWTManager(self.app) self.app.json_encoder = ArrowJSONEncoder # Register application handling @@ -148,6 +161,10 @@ class ApiServer(RPC): self.app.register_error_handler(404, self.page_not_found) # Actions to control the bot + self.app.add_url_rule(f'{BASE_URI}/token/login', 'login', + view_func=self._token_login, methods=['POST']) + self.app.add_url_rule(f'{BASE_URI}/token/refresh', 'token_refresh', + view_func=self._token_refresh, methods=['POST']) self.app.add_url_rule(f'{BASE_URI}/start', 'start', view_func=self._start, methods=['POST']) self.app.add_url_rule(f'{BASE_URI}/stop', 'stop', view_func=self._stop, methods=['POST']) @@ -199,6 +216,37 @@ class ApiServer(RPC): 'code': 404 }), 404 + @require_login + @rpc_catch_errors + def _token_login(self): + """ + Handler for /token/login + Returns a JWT token + """ + auth = request.authorization + if auth and self.check_auth(auth.username, auth.password): + keystuff = {'u': auth.username} + ret = { + 'access_token': create_access_token(identity=keystuff), + 'refresh_token': create_refresh_token(identity=keystuff), + } + return self.rest_dump(ret) + + return jsonify({"error": "Unauthorized"}), 401 + + @jwt_refresh_token_required + @rpc_catch_errors + def _token_refresh(self): + """ + Handler for /token/refresh + Returns a JWT token based on a JWT refresh token + """ + current_user = get_jwt_identity() + new_token = create_access_token(identity=current_user, fresh=False) + + ret = {'access_token': new_token} + return self.rest_dump(ret) + @require_login @rpc_catch_errors def _start(self): diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 8645e466e..d3b6b9639 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -94,6 +94,7 @@ class RPC: 'dry_run': config['dry_run'], 'stake_currency': config['stake_currency'], 'stake_amount': config['stake_amount'], + 'max_open_trades': config['max_open_trades'], 'minimal_roi': config['minimal_roi'].copy(), 'stoploss': config['stoploss'], 'trailing_stop': config['trailing_stop'], @@ -103,6 +104,8 @@ class RPC: 'ticker_interval': config['ticker_interval'], 'exchange': config['exchange']['name'], 'strategy': config['strategy'], + 'forcebuy_enabled': config.get('forcebuy_enable', False), + 'state': str(self._freqtrade.state) } return val diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index a21f7556c..856b8f138 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -579,7 +579,7 @@ class Telegram(RPC): "*/whitelist:* `Show current whitelist` \n" \ "*/blacklist [pair]:* `Show current blacklist, or adds one or more pairs " \ "to the blacklist.` \n" \ - "*/edge:* `Shows validated pairs by Edge if it is enabeld` \n" \ + "*/edge:* `Shows validated pairs by Edge if it is enabled` \n" \ "*/help:* `This help message`\n" \ "*/version:* `Show version`" @@ -621,10 +621,12 @@ class Telegram(RPC): f"*Mode:* `{'Dry-run' if val['dry_run'] else 'Live'}`\n" f"*Exchange:* `{val['exchange']}`\n" f"*Stake per trade:* `{val['stake_amount']} {val['stake_currency']}`\n" + f"*Max open Trades:* `{val['max_open_trades']}`\n" f"*Minimum ROI:* `{val['minimal_roi']}`\n" f"{sl_info}" f"*Ticker Interval:* `{val['ticker_interval']}`\n" - f"*Strategy:* `{val['strategy']}`" + f"*Strategy:* `{val['strategy']}`\n" + f"*Current state:* `{val['state']}`" ) def _send_msg(self, msg: str, parse_mode: ParseMode = ParseMode.MARKDOWN) -> None: diff --git a/freqtrade/state.py b/freqtrade/state.py index 415f6f5f2..38784c6a4 100644 --- a/freqtrade/state.py +++ b/freqtrade/state.py @@ -14,6 +14,9 @@ class State(Enum): STOPPED = 2 RELOAD_CONF = 3 + def __str__(self): + return f"{self.name.lower()}" + class RunMode(Enum): """ diff --git a/requirements-common.txt b/requirements-common.txt index a53fc3999..017974c9e 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,15 +1,15 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.27.1 +ccxt==1.27.49 SQLAlchemy==1.3.16 -python-telegram-bot==12.6.1 -arrow==0.15.5 +python-telegram-bot==12.7 +arrow==0.15.6 cachetools==4.1.0 requests==2.23.0 urllib3==1.25.9 wrapt==1.12.1 jsonschema==3.2.0 -TA-Lib==0.4.17 +TA-Lib==0.4.18 tabulate==0.8.7 pycoingecko==1.2.0 jinja2==2.11.2 @@ -25,6 +25,8 @@ sdnotify==0.3.2 # Api server flask==1.1.2 +flask-jwt-extended==3.24.1 +flask-cors==3.0.8 # Support for colorized terminal output colorama==0.4.3 diff --git a/requirements-dev.txt b/requirements-dev.txt index 508716bde..616ca20f9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,8 +8,8 @@ flake8==3.7.9 flake8-type-annotations==0.1.0 flake8-tidy-imports==4.1.0 mypy==0.770 -pytest==5.4.1 -pytest-asyncio==0.11.0 +pytest==5.4.2 +pytest-asyncio==0.12.0 pytest-cov==2.8.1 pytest-mock==3.1.0 pytest-random-order==1.0.4 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index b0e18867d..9afd07357 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -7,4 +7,4 @@ scikit-learn==0.22.2.post1 scikit-optimize==0.7.4 filelock==3.0.12 joblib==0.14.1 -progressbar2==3.51.0 +progressbar2==3.51.3 diff --git a/requirements-plot.txt b/requirements-plot.txt index 3db48a201..d81239053 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==4.6.0 +plotly==4.7.1 diff --git a/requirements.txt b/requirements.txt index 967f8df10..18cab206b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Load common requirements -r requirements-common.txt -numpy==1.18.3 +numpy==1.18.4 pandas==1.0.3 diff --git a/setup.py b/setup.py index 9c253ea4e..20963a15f 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ if readme_file.is_file(): readme_long = (Path(__file__).parent / "README.md").read_text() # Requirements used for submodules -api = ['flask'] +api = ['flask', 'flask-jwt-extended', 'flask-cors'] plot = ['plotly>=4.0'] hyperopt = [ 'scipy', diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 264ae9a63..46350beff 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -10,11 +10,13 @@ from freqtrade.commands import (start_convert_data, start_create_userdir, start_list_hyperopts, start_list_markets, start_list_strategies, start_list_timeframes, start_new_hyperopt, start_new_strategy, - start_test_pairlist, start_trading) + start_show_trades, start_test_pairlist, + start_trading) from freqtrade.configuration import setup_utils_configuration from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode -from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, +from tests.conftest import (create_mock_trades, get_args, log_has, log_has_re, + patch_exchange, patched_configuration_load_config_file) @@ -30,7 +32,7 @@ def test_setup_utils_configuration(): assert config['exchange']['secret'] == '' -def test_start_trading_fail(mocker): +def test_start_trading_fail(mocker, caplog): mocker.patch("freqtrade.worker.Worker.run", MagicMock(side_effect=OperationalException)) @@ -41,16 +43,15 @@ def test_start_trading_fail(mocker): 'trade', '-c', 'config.json.example' ] - with pytest.raises(OperationalException): - start_trading(get_args(args)) + start_trading(get_args(args)) assert exitmock.call_count == 1 exitmock.reset_mock() - + caplog.clear() mocker.patch("freqtrade.worker.Worker.__init__", MagicMock(side_effect=OperationalException)) - with pytest.raises(OperationalException): - start_trading(get_args(args)) + start_trading(get_args(args)) assert exitmock.call_count == 0 + assert log_has('Fatal exception!', caplog) def test_list_exchanges(capsys): @@ -1040,3 +1041,46 @@ def test_convert_data_trades(mocker, testdatadir): assert trades_mock.call_args[1]['convert_from'] == 'jsongz' assert trades_mock.call_args[1]['convert_to'] == 'json' assert trades_mock.call_args[1]['erase'] is False + + +@pytest.mark.usefixtures("init_persistence") +def test_show_trades(mocker, fee, capsys, caplog): + mocker.patch("freqtrade.persistence.init") + create_mock_trades(fee) + args = [ + "show-trades", + "--db-url", + "sqlite:///" + ] + pargs = get_args(args) + pargs['config'] = None + start_show_trades(pargs) + assert log_has("Printing 3 Trades: ", caplog) + captured = capsys.readouterr() + assert "Trade(id=1" in captured.out + assert "Trade(id=2" in captured.out + assert "Trade(id=3" in captured.out + args = [ + "show-trades", + "--db-url", + "sqlite:///", + "--print-json", + "--trade-ids", "1", "2" + ] + pargs = get_args(args) + pargs['config'] = None + start_show_trades(pargs) + + captured = capsys.readouterr() + assert log_has("Printing 2 Trades: ", caplog) + assert '"trade_id": 1' in captured.out + assert '"trade_id": 2' in captured.out + assert '"trade_id": 3' not in captured.out + args = [ + "show-trades", + ] + pargs = get_args(args) + pargs['config'] = None + + with pytest.raises(OperationalException, match=r"--db-url is required for this command."): + start_show_trades(pargs) diff --git a/tests/conftest.py b/tests/conftest.py index 2f5f468f6..36c60e27e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -305,7 +305,8 @@ def default_conf(testdatadir): "user_data_dir": Path("user_data"), "verbosity": 3, "strategy_path": str(Path(__file__).parent / "strategy" / "strats"), - "strategy": "DefaultStrategy" + "strategy": "DefaultStrategy", + "internals": {}, } return configuration @@ -780,7 +781,7 @@ def limit_buy_order(): 'id': 'mocked_limit_buy', 'type': 'limit', 'side': 'buy', - 'pair': 'mocked', + 'symbol': 'mocked', 'datetime': arrow.utcnow().isoformat(), 'price': 0.00001099, 'amount': 90.99181073, @@ -796,7 +797,7 @@ def market_buy_order(): 'id': 'mocked_market_buy', 'type': 'market', 'side': 'buy', - 'pair': 'mocked', + 'symbol': 'mocked', 'datetime': arrow.utcnow().isoformat(), 'price': 0.00004099, 'amount': 91.99181073, @@ -812,7 +813,7 @@ def market_sell_order(): 'id': 'mocked_limit_sell', 'type': 'market', 'side': 'sell', - 'pair': 'mocked', + 'symbol': 'mocked', 'datetime': arrow.utcnow().isoformat(), 'price': 0.00004173, 'amount': 91.99181073, @@ -828,7 +829,7 @@ def limit_buy_order_old(): 'id': 'mocked_limit_buy_old', 'type': 'limit', 'side': 'buy', - 'pair': 'mocked', + 'symbol': 'mocked', 'datetime': str(arrow.utcnow().shift(minutes=-601).datetime), 'price': 0.00001099, 'amount': 90.99181073, @@ -844,7 +845,7 @@ def limit_sell_order_old(): 'id': 'mocked_limit_sell_old', 'type': 'limit', 'side': 'sell', - 'pair': 'ETH/BTC', + 'symbol': 'ETH/BTC', 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), 'price': 0.00001099, 'amount': 90.99181073, @@ -860,7 +861,7 @@ def limit_buy_order_old_partial(): 'id': 'mocked_limit_buy_old_partial', 'type': 'limit', 'side': 'buy', - 'pair': 'ETH/BTC', + 'symbol': 'ETH/BTC', 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), 'price': 0.00001099, 'amount': 90.99181073, @@ -874,10 +875,103 @@ def limit_buy_order_old_partial(): def limit_buy_order_old_partial_canceled(limit_buy_order_old_partial): res = deepcopy(limit_buy_order_old_partial) res['status'] = 'canceled' - res['fee'] = {'cost': 0.0001, 'currency': 'ETH'} + res['fee'] = {'cost': 0.023, 'currency': 'ETH'} return res +@pytest.fixture(scope='function') +def limit_buy_order_canceled_empty(request): + # Indirect fixture + # Documentation: + # https://docs.pytest.org/en/latest/example/parametrize.html#apply-indirect-on-particular-arguments + + exchange_name = request.param + if exchange_name == 'ftx': + return { + 'info': {}, + 'id': '1234512345', + 'clientOrderId': None, + 'timestamp': arrow.utcnow().shift(minutes=-601).timestamp, + 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'lastTradeTimestamp': None, + 'symbol': 'LTC/USDT', + 'type': 'limit', + 'side': 'buy', + 'price': 34.3225, + 'amount': 0.55, + 'cost': 0.0, + 'average': None, + 'filled': 0.0, + 'remaining': 0.0, + 'status': 'closed', + 'fee': None, + 'trades': None + } + elif exchange_name == 'kraken': + return { + 'info': {}, + 'id': 'AZNPFF-4AC4N-7MKTAT', + 'clientOrderId': None, + 'timestamp': arrow.utcnow().shift(minutes=-601).timestamp, + 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'lastTradeTimestamp': None, + 'status': 'canceled', + 'symbol': 'LTC/USDT', + 'type': 'limit', + 'side': 'buy', + 'price': 34.3225, + 'cost': 0.0, + 'amount': 0.55, + 'filled': 0.0, + 'average': 0.0, + 'remaining': 0.55, + 'fee': {'cost': 0.0, 'rate': None, 'currency': 'USDT'}, + 'trades': [] + } + elif exchange_name == 'binance': + return { + 'info': {}, + 'id': '1234512345', + 'clientOrderId': 'alb1234123', + 'timestamp': arrow.utcnow().shift(minutes=-601).timestamp, + 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'lastTradeTimestamp': None, + 'symbol': 'LTC/USDT', + 'type': 'limit', + 'side': 'buy', + 'price': 0.016804, + 'amount': 0.55, + 'cost': 0.0, + 'average': None, + 'filled': 0.0, + 'remaining': 0.55, + 'status': 'canceled', + 'fee': None, + 'trades': None + } + else: + return { + 'info': {}, + 'id': '1234512345', + 'clientOrderId': 'alb1234123', + 'timestamp': arrow.utcnow().shift(minutes=-601).timestamp, + 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'lastTradeTimestamp': None, + 'symbol': 'LTC/USDT', + 'type': 'limit', + 'side': 'buy', + 'price': 0.016804, + 'amount': 0.55, + 'cost': 0.0, + 'average': None, + 'filled': 0.0, + 'remaining': 0.55, + 'status': 'canceled', + 'fee': None, + 'trades': None + } + + @pytest.fixture def limit_sell_order(): return { @@ -1329,6 +1423,15 @@ def trades_for_order(): @pytest.fixture(scope="function") def trades_history(): + return [[1565798399463, '126181329', None, 'buy', 0.019627, 0.04, 0.00078508], + [1565798399629, '126181330', None, 'buy', 0.019627, 0.244, 0.004788987999999999], + [1565798399752, '126181331', None, 'sell', 0.019626, 0.011, 0.00021588599999999999], + [1565798399862, '126181332', None, 'sell', 0.019626, 0.011, 0.00021588599999999999], + [1565798399872, '126181333', None, 'sell', 0.019626, 0.011, 0.00021588599999999999]] + + +@pytest.fixture(scope="function") +def fetch_trades_result(): return [{'info': {'a': 126181329, 'p': '0.01962700', 'q': '0.04000000', @@ -1483,7 +1586,7 @@ def buy_order_fee(): 'id': 'mocked_limit_buy_old', 'type': 'limit', 'side': 'buy', - 'pair': 'mocked', + 'symbol': 'mocked', 'datetime': str(arrow.utcnow().shift(minutes=-601).datetime), 'price': 0.245441, 'amount': 8.0, diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 7dff520e0..4a580366f 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -5,12 +5,10 @@ from freqtrade.configuration.timerange import TimeRange from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_format, ohlcv_fill_up_missing_data, - ohlcv_to_dataframe, - trim_dataframe) -from freqtrade.data.history import (get_timerange, - load_data, - load_pair_history, - validate_backtest_data) + ohlcv_to_dataframe, trades_dict_to_list, + trades_remove_duplicates, trim_dataframe) +from freqtrade.data.history import (get_timerange, load_data, + load_pair_history, validate_backtest_data) from tests.conftest import log_has from tests.data.test_history import _backup_file, _clean_test_file @@ -197,32 +195,60 @@ def test_trim_dataframe(testdatadir) -> None: assert all(data_modify.iloc[0] == data.iloc[25]) -def test_convert_trades_format(mocker, default_conf, testdatadir): - file = testdatadir / "XRP_ETH-trades.json.gz" - file_new = testdatadir / "XRP_ETH-trades.json" - _backup_file(file, copy_file=True) - default_conf['datadir'] = testdatadir +def test_trades_remove_duplicates(trades_history): + trades_history1 = trades_history * 3 + assert len(trades_history1) == len(trades_history) * 3 + res = trades_remove_duplicates(trades_history1) + assert len(res) == len(trades_history) + for i, t in enumerate(res): + assert t == trades_history[i] - assert not file_new.exists() + +def test_trades_dict_to_list(fetch_trades_result): + res = trades_dict_to_list(fetch_trades_result) + assert isinstance(res, list) + assert isinstance(res[0], list) + for i, t in enumerate(res): + assert t[0] == fetch_trades_result[i]['timestamp'] + assert t[1] == fetch_trades_result[i]['id'] + assert t[2] == fetch_trades_result[i]['type'] + assert t[3] == fetch_trades_result[i]['side'] + assert t[4] == fetch_trades_result[i]['price'] + assert t[5] == fetch_trades_result[i]['amount'] + assert t[6] == fetch_trades_result[i]['cost'] + + +def test_convert_trades_format(mocker, default_conf, testdatadir): + files = [{'old': testdatadir / "XRP_ETH-trades.json.gz", + 'new': testdatadir / "XRP_ETH-trades.json"}, + {'old': testdatadir / "XRP_OLD-trades.json.gz", + 'new': testdatadir / "XRP_OLD-trades.json"}, + ] + for file in files: + _backup_file(file['old'], copy_file=True) + assert not file['new'].exists() + + default_conf['datadir'] = testdatadir convert_trades_format(default_conf, convert_from='jsongz', convert_to='json', erase=False) - assert file_new.exists() - assert file.exists() + for file in files: + assert file['new'].exists() + assert file['old'].exists() - # Remove original file - file.unlink() + # Remove original file + file['old'].unlink() # Convert back convert_trades_format(default_conf, convert_from='json', convert_to='jsongz', erase=True) + for file in files: + assert file['old'].exists() + assert not file['new'].exists() - assert file.exists() - assert not file_new.exists() - - _clean_test_file(file) - if file_new.exists(): - file_new.unlink() + _clean_test_file(file['old']) + if file['new'].exists(): + file['new'].unlink() def test_convert_ohlcv_format(mocker, default_conf, testdatadir): diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index 2b3dda188..c2d6e82f1 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -1,8 +1,11 @@ from unittest.mock import MagicMock from pandas import DataFrame +import pytest from freqtrade.data.dataprovider import DataProvider +from freqtrade.pairlist.pairlistmanager import PairListManager +from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.state import RunMode from tests.conftest import get_patched_exchange @@ -64,8 +67,8 @@ def test_get_pair_dataframe(mocker, default_conf, ohlcv_history): assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty # Test with and without parameter - assert dp.get_pair_dataframe("UNITTEST/BTC", - ticker_interval).equals(dp.get_pair_dataframe("UNITTEST/BTC")) + assert dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval)\ + .equals(dp.get_pair_dataframe("UNITTEST/BTC")) default_conf["runmode"] = RunMode.LIVE dp = DataProvider(default_conf, exchange) @@ -90,10 +93,7 @@ def test_available_pairs(mocker, default_conf, ohlcv_history): dp = DataProvider(default_conf, exchange) assert len(dp.available_pairs) == 2 - assert dp.available_pairs == [ - ("XRP/BTC", ticker_interval), - ("UNITTEST/BTC", ticker_interval), - ] + assert dp.available_pairs == [("XRP/BTC", ticker_interval), ("UNITTEST/BTC", ticker_interval), ] def test_refresh(mocker, default_conf, ohlcv_history): @@ -152,3 +152,45 @@ def test_market(mocker, default_conf, markets): res = dp.market('UNITTEST/BTC') assert res is None + + +def test_ticker(mocker, default_conf, tickers): + ticker_mock = MagicMock(return_value=tickers()['ETH/BTC']) + mocker.patch("freqtrade.exchange.Exchange.fetch_ticker", ticker_mock) + exchange = get_patched_exchange(mocker, default_conf) + dp = DataProvider(default_conf, exchange) + res = dp.ticker('ETH/BTC') + assert type(res) is dict + assert 'symbol' in res + assert res['symbol'] == 'ETH/BTC' + + ticker_mock = MagicMock(side_effect=DependencyException('Pair not found')) + mocker.patch("freqtrade.exchange.Exchange.fetch_ticker", ticker_mock) + exchange = get_patched_exchange(mocker, default_conf) + dp = DataProvider(default_conf, exchange) + res = dp.ticker('UNITTEST/BTC') + assert res == {} + + +def test_current_whitelist(mocker, default_conf, tickers): + # patch default conf to volumepairlist + default_conf['pairlists'][0] = {'method': 'VolumePairList', "number_assets": 5} + + mocker.patch.multiple('freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + get_tickers=tickers) + exchange = get_patched_exchange(mocker, default_conf) + + pairlist = PairListManager(exchange, default_conf) + dp = DataProvider(default_conf, exchange, pairlist) + + # Simulate volumepairs from exchange. + pairlist.refresh_pairlist() + + assert dp.current_whitelist() == pairlist._whitelist + # The identity of the 2 lists should be identical + assert dp.current_whitelist() is pairlist._whitelist + + with pytest.raises(OperationalException): + dp = DataProvider(default_conf, exchange) + dp.current_whitelist() diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 12390538a..6fd4d9569 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -547,6 +547,17 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad assert log_has("New Amount of trades: 5", caplog) assert file1.is_file() + ght_mock.reset_mock() + since_time = int(trades_history[-3][0] // 1000) + since_time2 = int(trades_history[-1][0] // 1000) + timerange = TimeRange('date', None, since_time, 0) + assert _download_trades_history(data_handler=data_handler, exchange=exchange, + pair='ETH/BTC', timerange=timerange) + + assert ght_mock.call_count == 1 + # Check this in seconds - since we had to convert to seconds above too. + assert int(ght_mock.call_args_list[0][1]['since'] // 1000) == since_time2 - 5 + # clean files freshly downloaded _clean_test_file(file1) @@ -601,7 +612,7 @@ def test_jsondatahandler_ohlcv_get_pairs(testdatadir): def test_jsondatahandler_trades_get_pairs(testdatadir): pairs = JsonGzDataHandler.trades_get_pairs(testdatadir) # Convert to set to avoid failures due to sorting - assert set(pairs) == {'XRP/ETH'} + assert set(pairs) == {'XRP/ETH', 'XRP/OLD'} def test_jsondatahandler_ohlcv_purge(mocker, testdatadir): @@ -614,6 +625,17 @@ def test_jsondatahandler_ohlcv_purge(mocker, testdatadir): assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m') +def test_jsondatahandler_trades_load(mocker, testdatadir, caplog): + dh = JsonGzDataHandler(testdatadir) + logmsg = "Old trades format detected - converting" + dh.trades_load('XRP/ETH') + assert not log_has(logmsg, caplog) + + # Test conversation is happening + dh.trades_load('XRP/OLD') + assert log_has(logmsg, caplog) + + def test_jsondatahandler_trades_purge(mocker, testdatadir): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) mocker.patch.object(Path, "unlink", MagicMock()) diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index 2304c53c2..163ceff4b 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -335,12 +335,16 @@ def test_edge_init_error(mocker, edge_conf,): get_patched_freqtradebot(mocker, edge_conf) -def test_process_expectancy(mocker, edge_conf): +@pytest.mark.parametrize("fee,risk_reward_ratio,expectancy", [ + (0.0005, 306.5384615384, 101.5128205128), + (0.001, 152.6923076923, 50.2307692308), +]) +def test_process_expectancy(mocker, edge_conf, fee, risk_reward_ratio, expectancy): edge_conf['edge']['min_trade_number'] = 2 freqtrade = get_patched_freqtradebot(mocker, edge_conf) def get_fee(*args, **kwargs): - return 0.001 + return fee freqtrade.exchange.get_fee = get_fee edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) @@ -394,9 +398,9 @@ def test_process_expectancy(mocker, edge_conf): assert 'TEST/BTC' in final assert final['TEST/BTC'].stoploss == -0.9 assert round(final['TEST/BTC'].winrate, 10) == 0.3333333333 - assert round(final['TEST/BTC'].risk_reward_ratio, 10) == 306.5384615384 + assert round(final['TEST/BTC'].risk_reward_ratio, 10) == risk_reward_ratio assert round(final['TEST/BTC'].required_risk_reward, 10) == 2.0 - assert round(final['TEST/BTC'].expectancy, 10) == 101.5128205128 + assert round(final['TEST/BTC'].expectancy, 10) == expectancy # Pop last item so no trade is profitable trades.pop() diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index e4599dcd7..52faa284b 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -9,7 +9,12 @@ from freqtrade.exceptions import (DependencyException, InvalidOrderException, from tests.conftest import get_patched_exchange -def test_stoploss_order_binance(default_conf, mocker): +@pytest.mark.parametrize('limitratio,expected', [ + (None, 220 * 0.99), + (0.99, 220 * 0.99), + (0.98, 220 * 0.98), +]) +def test_stoploss_order_binance(default_conf, mocker, limitratio, expected): api_mock = MagicMock() order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) order_type = 'stop_loss_limit' @@ -20,7 +25,6 @@ def test_stoploss_order_binance(default_conf, mocker): 'foo': 'bar' } }) - default_conf['dry_run'] = False mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y) mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) @@ -32,8 +36,8 @@ def test_stoploss_order_binance(default_conf, mocker): order_types={'stoploss_on_exchange_limit_ratio': 1.05}) api_mock.create_order.reset_mock() - - order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + order_types = {} if limitratio is None else {'stoploss_on_exchange_limit_ratio': limitratio} + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types=order_types) assert 'id' in order assert 'info' in order @@ -42,7 +46,8 @@ def test_stoploss_order_binance(default_conf, mocker): assert api_mock.create_order.call_args_list[0][1]['type'] == order_type assert api_mock.create_order.call_args_list[0][1]['side'] == 'sell' assert api_mock.create_order.call_args_list[0][1]['amount'] == 1 - assert api_mock.create_order.call_args_list[0][1]['price'] == 220 + # Price should be 1% below stopprice + assert api_mock.create_order.call_args_list[0][1]['price'] == expected assert api_mock.create_order.call_args_list[0][1]['params'] == {'stopPrice': 220} # test exception handling diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 3c92612a0..aa42950e2 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1537,18 +1537,18 @@ async def test___async_get_candle_history_sort(default_conf, mocker, exchange_na @pytest.mark.asyncio @pytest.mark.parametrize("exchange_name", EXCHANGES) async def test__async_fetch_trades(default_conf, mocker, caplog, exchange_name, - trades_history): + fetch_trades_result): caplog.set_level(logging.DEBUG) exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) # Monkey-patch async function - exchange._api_async.fetch_trades = get_mock_coro(trades_history) + exchange._api_async.fetch_trades = get_mock_coro(fetch_trades_result) pair = 'ETH/BTC' res = await exchange._async_fetch_trades(pair, since=None, params=None) assert type(res) is list - assert isinstance(res[0], dict) - assert isinstance(res[1], dict) + assert isinstance(res[0], list) + assert isinstance(res[1], list) assert exchange._api_async.fetch_trades.call_count == 1 assert exchange._api_async.fetch_trades.call_args[0][0] == pair @@ -1594,7 +1594,7 @@ async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchang if 'since' in kwargs: # Return first 3 return trades_history[:-2] - elif kwargs.get('params', {}).get(pagination_arg) == trades_history[-3]['id']: + elif kwargs.get('params', {}).get(pagination_arg) == trades_history[-3][1]: # Return 2 return trades_history[-3:-1] else: @@ -1604,8 +1604,8 @@ async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchang exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist) pair = 'ETH/BTC' - ret = await exchange._async_get_trade_history_id(pair, since=trades_history[0]["timestamp"], - until=trades_history[-1]["timestamp"]-1) + ret = await exchange._async_get_trade_history_id(pair, since=trades_history[0][0], + until=trades_history[-1][0]-1) assert type(ret) is tuple assert ret[0] == pair assert type(ret[1]) is list @@ -1614,7 +1614,7 @@ async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchang fetch_trades_cal = exchange._async_fetch_trades.call_args_list # first call (using since, not fromId) assert fetch_trades_cal[0][0][0] == pair - assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"] + assert fetch_trades_cal[0][1]['since'] == trades_history[0][0] # 2nd call assert fetch_trades_cal[1][0][0] == pair @@ -1630,7 +1630,7 @@ async def test__async_get_trade_history_time(default_conf, mocker, caplog, excha caplog.set_level(logging.DEBUG) async def mock_get_trade_hist(pair, *args, **kwargs): - if kwargs['since'] == trades_history[0]["timestamp"]: + if kwargs['since'] == trades_history[0][0]: return trades_history[:-1] else: return trades_history[-1:] @@ -1640,8 +1640,8 @@ async def test__async_get_trade_history_time(default_conf, mocker, caplog, excha # Monkey-patch async function exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist) pair = 'ETH/BTC' - ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0]["timestamp"], - until=trades_history[-1]["timestamp"]-1) + ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0][0], + until=trades_history[-1][0]-1) assert type(ret) is tuple assert ret[0] == pair assert type(ret[1]) is list @@ -1650,11 +1650,11 @@ async def test__async_get_trade_history_time(default_conf, mocker, caplog, excha fetch_trades_cal = exchange._async_fetch_trades.call_args_list # first call (using since, not fromId) assert fetch_trades_cal[0][0][0] == pair - assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"] + assert fetch_trades_cal[0][1]['since'] == trades_history[0][0] # 2nd call assert fetch_trades_cal[1][0][0] == pair - assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"] + assert fetch_trades_cal[0][1]['since'] == trades_history[0][0] assert log_has_re(r"Stopping because until was reached.*", caplog) @@ -1666,7 +1666,7 @@ async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog, caplog.set_level(logging.DEBUG) async def mock_get_trade_hist(pair, *args, **kwargs): - if kwargs['since'] == trades_history[0]["timestamp"]: + if kwargs['since'] == trades_history[0][0]: return trades_history[:-1] else: return [] @@ -1676,8 +1676,8 @@ async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog, # Monkey-patch async function exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist) pair = 'ETH/BTC' - ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0]["timestamp"], - until=trades_history[-1]["timestamp"]-1) + ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0][0], + until=trades_history[-1][0]-1) assert type(ret) is tuple assert ret[0] == pair assert type(ret[1]) is list @@ -1686,7 +1686,7 @@ async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog, fetch_trades_cal = exchange._async_fetch_trades.call_args_list # first call (using since, not fromId) assert fetch_trades_cal[0][0][0] == pair - assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"] + assert fetch_trades_cal[0][1]['since'] == trades_history[0][0] @pytest.mark.parametrize("exchange_name", EXCHANGES) @@ -1698,8 +1698,8 @@ def test_get_historic_trades(default_conf, mocker, caplog, exchange_name, trades exchange._async_get_trade_history_id = get_mock_coro((pair, trades_history)) exchange._async_get_trade_history_time = get_mock_coro((pair, trades_history)) - ret = exchange.get_historic_trades(pair, since=trades_history[0]["timestamp"], - until=trades_history[-1]["timestamp"]) + ret = exchange.get_historic_trades(pair, since=trades_history[0][0], + until=trades_history[-1][0]) # Depending on the exchange, one or the other method should be called assert sum([exchange._async_get_trade_history_id.call_count, @@ -1720,8 +1720,8 @@ def test_get_historic_trades_notsupported(default_conf, mocker, caplog, exchange with pytest.raises(OperationalException, match="This exchange does not suport downloading Trades."): - exchange.get_historic_trades(pair, since=trades_history[0]["timestamp"], - until=trades_history[-1]["timestamp"]) + exchange.get_historic_trades(pair, since=trades_history[0][0], + until=trades_history[-1][0]) @pytest.mark.parametrize("exchange_name", EXCHANGES) @@ -2145,3 +2145,58 @@ def test_symbol_is_pair(market_symbol, base_currency, quote_currency, expected_r ]) def test_market_is_active(market, expected_result) -> None: assert market_is_active(market) == expected_result + + +@pytest.mark.parametrize("order,expected", [ + ([{'fee'}], False), + ({'fee': None}, False), + ({'fee': {'currency': 'ETH/BTC'}}, False), + ({'fee': {'currency': 'ETH/BTC', 'cost': None}}, False), + ({'fee': {'currency': 'ETH/BTC', 'cost': 0.01}}, True), +]) +def test_order_has_fee(order, expected) -> None: + assert Exchange.order_has_fee(order) == expected + + +@pytest.mark.parametrize("order,expected", [ + ({'symbol': 'ETH/BTC', 'fee': {'currency': 'ETH', 'cost': 0.43}}, + (0.43, 'ETH', 0.01)), + ({'symbol': 'ETH/USDT', 'fee': {'currency': 'USDT', 'cost': 0.01}}, + (0.01, 'USDT', 0.01)), + ({'symbol': 'BTC/USDT', 'fee': {'currency': 'USDT', 'cost': 0.34, 'rate': 0.01}}, + (0.34, 'USDT', 0.01)), +]) +def test_extract_cost_curr_rate(mocker, default_conf, order, expected) -> None: + mocker.patch('freqtrade.exchange.Exchange.calculate_fee_rate', MagicMock(return_value=0.01)) + ex = get_patched_exchange(mocker, default_conf) + assert ex.extract_cost_curr_rate(order) == expected + + +@pytest.mark.parametrize("order,expected", [ + # Using base-currency + ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, + 'fee': {'currency': 'ETH', 'cost': 0.004, 'rate': None}}, 0.1), + ({'symbol': 'ETH/BTC', 'amount': 0.05, 'cost': 0.05, + 'fee': {'currency': 'ETH', 'cost': 0.004, 'rate': None}}, 0.08), + # Using quote currency + ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, + 'fee': {'currency': 'BTC', 'cost': 0.005}}, 0.1), + ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, + 'fee': {'currency': 'BTC', 'cost': 0.002, 'rate': None}}, 0.04), + # Using foreign currency + ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, + 'fee': {'currency': 'NEO', 'cost': 0.0012}}, 0.001944), + ({'symbol': 'ETH/BTC', 'amount': 2.21, 'cost': 0.02992561, + 'fee': {'currency': 'NEO', 'cost': 0.00027452}}, 0.00074305), + # TODO: More tests here! + # Rate included in return - return as is + ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, + 'fee': {'currency': 'USDT', 'cost': 0.34, 'rate': 0.01}}, 0.01), + ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, + 'fee': {'currency': 'USDT', 'cost': 0.34, 'rate': 0.005}}, 0.005), +]) +def test_calculate_fee_rate(mocker, default_conf, order, expected) -> None: + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', return_value={'last': 0.081}) + + ex = get_patched_exchange(mocker, default_conf) + assert ex.calculate_fee_rate(order) == expected diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 6c2d6c9dd..093cbf966 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -649,6 +649,7 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): assert log_has(line, caplog) +@pytest.mark.filterwarnings("ignore:deprecated") def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): patch_exchange(mocker) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index d2af4bd87..a1e6d9f26 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -51,7 +51,11 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'open_date_hum': ANY, 'is_open': ANY, 'fee_open': ANY, + 'fee_open_cost': ANY, + 'fee_open_currency': ANY, 'fee_close': ANY, + 'fee_close_cost': ANY, + 'fee_close_currency': ANY, 'open_rate_requested': ANY, 'open_trade_price': ANY, 'close_rate_requested': ANY, @@ -90,7 +94,11 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'open_date_hum': ANY, 'is_open': ANY, 'fee_open': ANY, + 'fee_open_cost': ANY, + 'fee_open_currency': ANY, 'fee_close': ANY, + 'fee_close_cost': ANY, + 'fee_close_currency': ANY, 'open_rate_requested': ANY, 'open_trade_price': ANY, 'close_rate_requested': ANY, diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 6548790cb..b953097d5 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -49,6 +49,7 @@ def client_get(client, url): def assert_response(response, expected_code=200): assert response.status_code == expected_code assert response.content_type == "application/json" + assert ('Access-Control-Allow-Origin', '*') in response.headers._list def test_api_not_found(botclient): @@ -94,6 +95,33 @@ def test_api_unauthorized(botclient): assert rc.json == {'error': 'Unauthorized'} +def test_api_token_login(botclient): + ftbot, client = botclient + rc = client_post(client, f"{BASE_URI}/token/login") + assert_response(rc) + assert 'access_token' in rc.json + assert 'refresh_token' in rc.json + + # test Authentication is working with JWT tokens too + rc = client.get(f"{BASE_URI}/count", + content_type="application/json", + headers={'Authorization': f'Bearer {rc.json["access_token"]}'}) + assert_response(rc) + + +def test_api_token_refresh(botclient): + ftbot, client = botclient + rc = client_post(client, f"{BASE_URI}/token/login") + assert_response(rc) + rc = client.post(f"{BASE_URI}/token/refresh", + content_type="application/json", + data=None, + headers={'Authorization': f'Bearer {rc.json["refresh_token"]}'}) + assert_response(rc) + assert 'access_token' in rc.json + assert 'refresh_token' not in rc.json + + def test_api_stop_workflow(botclient): ftbot, client = botclient assert ftbot.state == State.RUNNING @@ -123,6 +151,12 @@ def test_api__init__(default_conf, mocker): """ Test __init__() method """ + default_conf.update({"api_server": {"enabled": True, + "listen_ip_address": "127.0.0.1", + "listen_port": 8080, + "username": "TestUser", + "password": "testPass", + }}) mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) mocker.patch('freqtrade.rpc.api_server.ApiServer.run', MagicMock()) @@ -283,6 +317,7 @@ def test_api_show_config(botclient, mocker): assert 'dry_run' in rc.json assert rc.json['exchange'] == 'bittrex' assert rc.json['ticker_interval'] == '5m' + assert rc.json['state'] == 'running' assert not rc.json['trailing_stop'] @@ -472,7 +507,11 @@ def test_api_status(botclient, mocker, ticker, fee, markets): 'close_rate_requested': None, 'current_rate': 1.099e-05, 'fee_close': 0.0025, + 'fee_close_cost': None, + 'fee_close_currency': None, 'fee_open': 0.0025, + 'fee_open_cost': None, + 'fee_open_currency': None, 'open_date': ANY, 'is_open': True, 'max_rate': 0.0, @@ -575,7 +614,11 @@ def test_api_forcebuy(botclient, mocker, fee): 'close_profit': None, 'close_rate_requested': None, 'fee_close': 0.0025, + 'fee_close_cost': None, + 'fee_close_currency': None, 'fee_open': 0.0025, + 'fee_open_cost': None, + 'fee_open_currency': None, 'is_open': False, 'max_rate': None, 'min_rate': None, diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 940ae8e77..5c5785ca3 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1158,7 +1158,8 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, 'status': 'closed', 'type': 'stop_loss_limit', 'price': 3, - 'average': 2 + 'average': 2, + 'amount': limit_buy_order['amount'], }) mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_order_hit) assert freqtrade.handle_stoploss_on_exchange(trade) is True @@ -2220,6 +2221,7 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap limit_buy_order_old_partial_canceled, mocker) -> None: rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock(return_value=limit_buy_order_old_partial_canceled) + mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=0)) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -2239,7 +2241,7 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap # and apply fees if necessary. freqtrade.check_handle_timedout() - assert log_has_re(r"Applying fee on amount for Trade.* Order", caplog) + assert log_has_re(r"Applying fee on amount for Trade.*", caplog) assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 2 @@ -2247,9 +2249,10 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap assert len(trades) == 1 # Verify that trade has been updated assert trades[0].amount == (limit_buy_order_old_partial['amount'] - - limit_buy_order_old_partial['remaining']) - 0.0001 + limit_buy_order_old_partial['remaining']) - 0.023 assert trades[0].open_order_id is None - assert trades[0].fee_open == 0 + assert trades[0].fee_updated('buy') + assert pytest.approx(trades[0].fee_open) == 0.001 def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, caplog, fee, @@ -2320,7 +2323,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke caplog) -def test_handle_timedout_limit_buy(mocker, caplog, default_conf, limit_buy_order) -> None: +def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> None: patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = MagicMock(return_value=limit_buy_order) @@ -2332,28 +2335,52 @@ def test_handle_timedout_limit_buy(mocker, caplog, default_conf, limit_buy_order Trade.session = MagicMock() trade = MagicMock() trade.pair = 'LTC/ETH' - limit_buy_order['remaining'] = limit_buy_order['amount'] + limit_buy_order['filled'] = 0.0 + limit_buy_order['status'] = 'open' reason = CANCEL_REASON['TIMEOUT'] assert freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 cancel_order_mock.reset_mock() - limit_buy_order['amount'] = 2 + limit_buy_order['filled'] = 2 assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 + limit_buy_order['filled'] = 2 mocker.patch('freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException) assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) +@pytest.mark.parametrize("limit_buy_order_canceled_empty", ['binance', 'ftx', 'kraken', 'bittrex'], + indirect=['limit_buy_order_canceled_empty']) +def test_handle_cancel_buy_exchanges(mocker, caplog, default_conf, + limit_buy_order_canceled_empty) -> None: + patch_RPCManager(mocker) + patch_exchange(mocker) + cancel_order_mock = mocker.patch( + 'freqtrade.exchange.Exchange.cancel_order_with_result', + return_value=limit_buy_order_canceled_empty) + nofiy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot._notify_buy_cancel') + freqtrade = FreqtradeBot(default_conf) + + Trade.session = MagicMock() + reason = CANCEL_REASON['TIMEOUT'] + trade = MagicMock() + trade.pair = 'LTC/ETH' + assert freqtrade.handle_cancel_buy(trade, limit_buy_order_canceled_empty, reason) + assert cancel_order_mock.call_count == 0 + assert log_has_re(r'Buy order fully cancelled. Removing .* from database\.', caplog) + assert nofiy_mock.call_count == 1 + + @pytest.mark.parametrize('cancelorder', [ {}, {'remaining': None}, 'String Return value', 123 ]) -def test_handle_timedout_limit_buy_corder_empty(mocker, default_conf, limit_buy_order, - cancelorder) -> None: +def test_handle_cancel_buy_corder_empty(mocker, default_conf, limit_buy_order, + cancelorder) -> None: patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = MagicMock(return_value=cancelorder) @@ -2368,18 +2395,19 @@ def test_handle_timedout_limit_buy_corder_empty(mocker, default_conf, limit_buy_ Trade.session = MagicMock() trade = MagicMock() trade.pair = 'LTC/ETH' - limit_buy_order['remaining'] = limit_buy_order['amount'] + limit_buy_order['filled'] = 0.0 + limit_buy_order['status'] = 'open' reason = CANCEL_REASON['TIMEOUT'] assert freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 cancel_order_mock.reset_mock() - limit_buy_order['amount'] = 2 + limit_buy_order['filled'] = 1.0 assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 -def test_handle_timedout_limit_sell(mocker, default_conf) -> None: +def test_handle_cancel_sell_limit(mocker, default_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = MagicMock() @@ -2405,6 +2433,22 @@ def test_handle_timedout_limit_sell(mocker, default_conf) -> None: assert cancel_order_mock.call_count == 1 +def test_handle_cancel_sell_cancel_exception(mocker, default_conf) -> None: + patch_RPCManager(mocker) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException()) + + freqtrade = FreqtradeBot(default_conf) + + trade = MagicMock() + reason = CANCEL_REASON['TIMEOUT'] + order = {'remaining': 1, + 'amount': 1, + 'status': "open"} + assert freqtrade.handle_cancel_sell(trade, order, reason) == 'error cancelling order' + + def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> None: rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) @@ -3309,8 +3353,6 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fee, caplog, mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) - patch_RPCManager(mocker) - patch_exchange(mocker) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( pair='LTC/ETH', @@ -3321,21 +3363,43 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fe fee_close=fee.return_value, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992) from Trades', + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).', caplog) +def test_get_real_amount_quote_dust(default_conf, trades_for_order, buy_order_fee, fee, + caplog, mocker): + mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) + walletmock = mocker.patch('freqtrade.wallets.Wallets.update') + mocker.patch('freqtrade.wallets.Wallets.get_free', return_value=8.1122) + amount = sum(x['amount'] for x in trades_for_order) + trade = Trade( + pair='LTC/ETH', + amount=amount, + exchange='binance', + open_rate=0.245441, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_order_id="123456" + ) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + walletmock.reset_mock() + # Amount is kept as is + assert freqtrade.get_real_amount(trade, buy_order_fee) == amount + assert walletmock.call_count == 1 + assert log_has_re(r'Fee amount for Trade.* was in base currency ' + '- Eating Fee 0.008 into dust', caplog) + + def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, fee): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) - patch_RPCManager(mocker) - patch_exchange(mocker) amount = buy_order_fee['amount'] trade = Trade( pair='LTC/ETH', @@ -3346,8 +3410,7 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f fee_close=fee.return_value, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount @@ -3359,8 +3422,6 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fee, mocker): trades_for_order[0]['fee']['currency'] = 'ETH' - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( @@ -3372,8 +3433,7 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fe open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount @@ -3386,8 +3446,6 @@ def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_ limit_buy_order['fee'] = {'cost': 0.004, 'currency': None} trades_for_order[0]['fee']['currency'] = None - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( @@ -3399,8 +3457,7 @@ def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_ open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, limit_buy_order) == amount @@ -3410,8 +3467,6 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, fee, trades_for_order[0]['fee']['currency'] = 'BNB' trades_for_order[0]['fee']['cost'] = 0.00094518 - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( @@ -3423,16 +3478,13 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, fee, open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, caplog, fee, mocker): - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order2) amount = float(sum(x['amount'] for x in trades_for_order2)) trade = Trade( @@ -3444,13 +3496,12 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992) from Trades', + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).', caplog) @@ -3459,8 +3510,6 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['fee'] = {'cost': 0.004, 'currency': 'LTC'} - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[trades_for_order]) amount = float(sum(x['amount'] for x in trades_for_order)) @@ -3473,13 +3522,12 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, limit_buy_order) == amount - 0.004 assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996) from Order', + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).', caplog) @@ -3487,8 +3535,6 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['fee'] = {'cost': 0.004} - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) amount = float(sum(x['amount'] for x in trades_for_order)) trade = Trade( @@ -3500,8 +3546,7 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, limit_buy_order) == amount @@ -3511,8 +3556,6 @@ def test_get_real_amount_wrong_amount(default_conf, trades_for_order, buy_order_ limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['amount'] = limit_buy_order['amount'] - 0.001 - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = float(sum(x['amount'] for x in trades_for_order)) trade = Trade( @@ -3524,8 +3567,7 @@ def test_get_real_amount_wrong_amount(default_conf, trades_for_order, buy_order_ fee_close=fee.return_value, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount does not change with pytest.raises(DependencyException, match=r"Half bought\? Amounts don't match"): @@ -3538,8 +3580,6 @@ def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, b limit_buy_order = deepcopy(buy_order_fee) trades_for_order[0]['amount'] = trades_for_order[0]['amount'] + 1e-15 - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = float(sum(x['amount'] for x in trades_for_order)) trade = Trade( @@ -3551,8 +3591,7 @@ def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, b open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount changes by fee amount. assert isclose(freqtrade.get_real_amount(trade, limit_buy_order), amount - (amount * 0.001), @@ -3563,8 +3602,6 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, # Remove "Currency" from fee dict trades_for_order[0]['fee'] = {'cost': 0.008} - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( @@ -3577,15 +3614,12 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount def test_get_real_amount_open_trade(default_conf, fee, mocker): - patch_RPCManager(mocker) - patch_exchange(mocker) amount = 12345 trade = Trade( pair='LTC/ETH', @@ -3600,12 +3634,41 @@ def test_get_real_amount_open_trade(default_conf, fee, mocker): 'id': 'mocked_order', 'amount': amount, 'status': 'open', + 'side': 'buy', } - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) assert freqtrade.get_real_amount(trade, order) == amount +@pytest.mark.parametrize('amount,fee_abs,wallet,amount_exp', [ + (8.0, 0.0, 10, 8), + (8.0, 0.0, 0, 8), + (8.0, 0.1, 0, 7.9), + (8.0, 0.1, 10, 8), + (8.0, 0.1, 8.0, 8.0), + (8.0, 0.1, 7.9, 7.9), +]) +def test_apply_fee_conditional(default_conf, fee, caplog, mocker, + amount, fee_abs, wallet, amount_exp): + walletmock = mocker.patch('freqtrade.wallets.Wallets.update') + mocker.patch('freqtrade.wallets.Wallets.get_free', return_value=wallet) + trade = Trade( + pair='LTC/ETH', + amount=amount, + exchange='binance', + open_rate=0.245441, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_order_id="123456" + ) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + walletmock.reset_mock() + # Amount is kept as is + assert freqtrade.apply_fee_conditional(trade, 'LTC', amount, fee_abs) == amount_exp + assert walletmock.call_count == 1 + + def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, mocker, order_book_l2): default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True @@ -3899,4 +3962,3 @@ def test_cancel_all_open_orders(mocker, default_conf, fee, limit_buy_order, limi freqtrade.cancel_all_open_orders() assert buy_mock.call_count == 1 assert sell_mock.call_count == 1 - diff --git a/tests/test_integration.py b/tests/test_integration.py index c40da7e9d..1396e86f5 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -44,6 +44,8 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, } stoploss_order_closed = stoploss_order_open.copy() stoploss_order_closed['status'] = 'closed' + stoploss_order_closed['filled'] = stoploss_order_closed['amount'] + # Sell first trade based on stoploss, keep 2nd and 3rd trade open stoploss_order_mock = MagicMock( side_effect=[stoploss_order_closed, stoploss_order_open, stoploss_order_open]) @@ -67,7 +69,6 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', create_stoploss_order=MagicMock(return_value=True), - update_trade_state=MagicMock(), _notify_sell=MagicMock(), ) mocker.patch("freqtrade.strategy.interface.IStrategy.should_sell", should_sell_mock) @@ -97,8 +98,9 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, # Only order for 3rd trade needs to be cancelled assert cancel_order_mock.call_count == 1 - # Wallets must be updated between stoploss cancellation and selling. - assert wallets_mock.call_count == 2 + # Wallets must be updated between stoploss cancellation and selling, and will be updated again + # during update_trade_state + assert wallets_mock.call_count == 4 trade = trades[0] assert trade.sell_reason == SellType.STOPLOSS_ON_EXCHANGE.value @@ -144,7 +146,6 @@ def test_forcebuy_last_unlimited(default_conf, ticker, fee, limit_buy_order, moc mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', create_stoploss_order=MagicMock(return_value=True), - update_trade_state=MagicMock(), _notify_sell=MagicMock(), ) should_sell_mock = MagicMock(side_effect=[ diff --git a/tests/test_main.py b/tests/test_main.py index 70b784002..11d0ede3a 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -115,6 +115,32 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None: assert log_has('Oh snap!', caplog) +def test_main_operational_exception1(mocker, default_conf, caplog) -> None: + patch_exchange(mocker) + mocker.patch( + 'freqtrade.commands.list_commands.available_exchanges', + MagicMock(side_effect=ValueError('Oh snap!')) + ) + patched_configuration_load_config_file(mocker, default_conf) + + args = ['list-exchanges'] + + # Test Main + the KeyboardInterrupt exception + with pytest.raises(SystemExit): + main(args) + + assert log_has('Fatal exception!', caplog) + assert not log_has_re(r'SIGINT.*', caplog) + mocker.patch( + 'freqtrade.commands.list_commands.available_exchanges', + MagicMock(side_effect=KeyboardInterrupt) + ) + with pytest.raises(SystemExit): + main(args) + + assert log_has_re(r'SIGINT.*', caplog) + + def test_main_reload_conf(mocker, default_conf, caplog) -> None: patch_exchange(mocker) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.cleanup', MagicMock()) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index ceac24356..5c7686e28 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -465,6 +465,10 @@ def test_migrate_old(mocker, default_conf, fee): assert trade.initial_stop_loss == 0.0 assert trade.open_trade_price == trade._calc_open_trade_price() assert trade.close_profit_abs is None + assert trade.fee_open_cost is None + assert trade.fee_open_currency is None + assert trade.fee_close_cost is None + assert trade.fee_close_currency is None trade = Trade.query.filter(Trade.id == 2).first() assert trade.close_rate is not None @@ -741,7 +745,11 @@ def test_to_json(default_conf, fee): 'open_rate_requested': None, 'open_trade_price': 15.1668225, 'fee_close': 0.0025, + 'fee_close_cost': None, + 'fee_close_currency': None, 'fee_open': 0.0025, + 'fee_open_cost': None, + 'fee_open_currency': None, 'close_rate': None, 'close_rate_requested': None, 'amount': 123.0, @@ -790,7 +798,11 @@ def test_to_json(default_conf, fee): 'close_profit': None, 'close_rate_requested': None, 'fee_close': 0.0025, + 'fee_close_cost': None, + 'fee_close_currency': None, 'fee_open': 0.0025, + 'fee_open_cost': None, + 'fee_open_currency': None, 'is_open': None, 'max_rate': None, 'min_rate': None, @@ -862,6 +874,75 @@ def test_stoploss_reinitialization(default_conf, fee): assert trade_adj.initial_stop_loss_pct == -0.04 +def test_update_fee(fee): + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + fee_open=fee.return_value, + open_date=arrow.utcnow().shift(hours=-2).datetime, + amount=10, + fee_close=fee.return_value, + exchange='bittrex', + open_rate=1, + max_rate=1, + ) + fee_cost = 0.15 + fee_currency = 'BTC' + fee_rate = 0.0075 + assert trade.fee_open_currency is None + assert not trade.fee_updated('buy') + assert not trade.fee_updated('sell') + + trade.update_fee(fee_cost, fee_currency, fee_rate, 'buy') + assert trade.fee_updated('buy') + assert not trade.fee_updated('sell') + assert trade.fee_open_currency == fee_currency + assert trade.fee_open_cost == fee_cost + assert trade.fee_open == fee_rate + # Setting buy rate should "guess" close rate + assert trade.fee_close == fee_rate + assert trade.fee_close_currency is None + assert trade.fee_close_cost is None + + fee_rate = 0.0076 + trade.update_fee(fee_cost, fee_currency, fee_rate, 'sell') + assert trade.fee_updated('buy') + assert trade.fee_updated('sell') + assert trade.fee_close == 0.0076 + assert trade.fee_close_cost == fee_cost + assert trade.fee_close == fee_rate + + +def test_fee_updated(fee): + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + fee_open=fee.return_value, + open_date=arrow.utcnow().shift(hours=-2).datetime, + amount=10, + fee_close=fee.return_value, + exchange='bittrex', + open_rate=1, + max_rate=1, + ) + + assert trade.fee_open_currency is None + assert not trade.fee_updated('buy') + assert not trade.fee_updated('sell') + assert not trade.fee_updated('asdf') + + trade.update_fee(0.15, 'BTC', 0.0075, 'buy') + assert trade.fee_updated('buy') + assert not trade.fee_updated('sell') + assert trade.fee_open_currency is not None + assert trade.fee_close_currency is None + + trade.update_fee(0.15, 'ABC', 0.0075, 'sell') + assert trade.fee_updated('buy') + assert trade.fee_updated('sell') + assert not trade.fee_updated('asfd') + + @pytest.mark.usefixtures("init_persistence") def test_total_open_trades_stakes(fee): diff --git a/tests/testdata/XRP_ETH-trades.json.gz b/tests/testdata/XRP_ETH-trades.json.gz index 69b92cac8..dad822005 100644 Binary files a/tests/testdata/XRP_ETH-trades.json.gz and b/tests/testdata/XRP_ETH-trades.json.gz differ diff --git a/tests/testdata/XRP_OLD-trades.json.gz b/tests/testdata/XRP_OLD-trades.json.gz new file mode 100644 index 000000000..69b92cac8 Binary files /dev/null and b/tests/testdata/XRP_OLD-trades.json.gz differ