diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4d1f28a0d..dfbc0cee7 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,7 +10,7 @@ updates: directory: "/" schedule: interval: weekly - open-pull-requests-limit: 10 + open-pull-requests-limit: 15 target-branch: develop - package-ecosystem: "github-actions" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4b550855..d187e650b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -461,7 +461,7 @@ jobs: python setup.py sdist bdist_wheel - name: Publish to PyPI (Test) - uses: pypa/gh-action-pypi-publish@v1.8.8 + uses: pypa/gh-action-pypi-publish@v1.8.10 if: (github.event_name == 'release') with: user: __token__ @@ -469,7 +469,7 @@ jobs: repository_url: https://test.pypi.org/legacy/ - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.8.8 + uses: pypa/gh-action-pypi-publish@v1.8.10 if: (github.event_name == 'release') with: user: __token__ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6e4919763..5bbf75965 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ repos: # stages: [push] - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.3.0" + rev: "v1.5.0" hooks: - id: mypy exclude: build_helpers @@ -18,7 +18,7 @@ repos: - types-requests==2.31.0.2 - types-tabulate==0.9.0.3 - types-python-dateutil==2.8.19.14 - - SQLAlchemy==2.0.19 + - SQLAlchemy==2.0.20 # stages: [push] - repo: https://github.com/pycqa/isort diff --git a/.readthedocs.yml b/.readthedocs.yml index 446181452..6d0d1fe64 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,8 +1,14 @@ # .readthedocs.yml +version: 2 build: - image: latest + os: "ubuntu-22.04" + tools: + python: "3.11" python: - version: 3.8 - setup_py_install: false + install: + - requirements: docs/requirements-docs.txt + +mkdocs: + configuration: mkdocs.yml diff --git a/build_helpers/TA_Lib-0.4.27-cp310-cp310-win_amd64.whl b/build_helpers/TA_Lib-0.4.27-cp310-cp310-win_amd64.whl deleted file mode 100644 index fba8f932a..000000000 Binary files a/build_helpers/TA_Lib-0.4.27-cp310-cp310-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.27-cp311-cp311-win_amd64.whl b/build_helpers/TA_Lib-0.4.27-cp311-cp311-win_amd64.whl deleted file mode 100644 index 63e6eacce..000000000 Binary files a/build_helpers/TA_Lib-0.4.27-cp311-cp311-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.27-cp38-cp38-win_amd64.whl b/build_helpers/TA_Lib-0.4.27-cp38-cp38-win_amd64.whl deleted file mode 100644 index 80a81af98..000000000 Binary files a/build_helpers/TA_Lib-0.4.27-cp38-cp38-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.27-cp39-cp39-win_amd64.whl b/build_helpers/TA_Lib-0.4.27-cp39-cp39-win_amd64.whl deleted file mode 100644 index 7b3fb91f9..000000000 Binary files a/build_helpers/TA_Lib-0.4.27-cp39-cp39-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.28-cp310-cp310-win_amd64.whl b/build_helpers/TA_Lib-0.4.28-cp310-cp310-win_amd64.whl new file mode 100644 index 000000000..cb7fdf907 Binary files /dev/null and b/build_helpers/TA_Lib-0.4.28-cp310-cp310-win_amd64.whl differ diff --git a/build_helpers/TA_Lib-0.4.28-cp311-cp311-win_amd64.whl b/build_helpers/TA_Lib-0.4.28-cp311-cp311-win_amd64.whl new file mode 100644 index 000000000..4cb902191 Binary files /dev/null and b/build_helpers/TA_Lib-0.4.28-cp311-cp311-win_amd64.whl differ diff --git a/build_helpers/TA_Lib-0.4.28-cp38-cp38-win_amd64.whl b/build_helpers/TA_Lib-0.4.28-cp38-cp38-win_amd64.whl new file mode 100644 index 000000000..34fecd677 Binary files /dev/null and b/build_helpers/TA_Lib-0.4.28-cp38-cp38-win_amd64.whl differ diff --git a/build_helpers/TA_Lib-0.4.28-cp39-cp39-win_amd64.whl b/build_helpers/TA_Lib-0.4.28-cp39-cp39-win_amd64.whl new file mode 100644 index 000000000..64a61ff0d Binary files /dev/null and b/build_helpers/TA_Lib-0.4.28-cp39-cp39-win_amd64.whl differ diff --git a/build_helpers/install_ta-lib.sh b/build_helpers/install_ta-lib.sh index 005d9abca..0315c7025 100755 --- a/build_helpers/install_ta-lib.sh +++ b/build_helpers/install_ta-lib.sh @@ -8,8 +8,9 @@ if [ -n "$2" ] || [ ! -f "${INSTALL_LOC}/lib/libta_lib.a" ]; then tar zxvf ta-lib-0.4.0-src.tar.gz cd ta-lib \ && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \ - && curl 'https://raw.githubusercontent.com/gcc-mirror/gcc/master/config.guess' -o config.guess \ - && curl 'https://raw.githubusercontent.com/gcc-mirror/gcc/master/config.sub' -o config.sub \ + && echo "Downloading gcc config.guess and config.sub" \ + && curl -s 'https://raw.githubusercontent.com/gcc-mirror/gcc/master/config.guess' -o config.guess \ + && curl -s 'https://raw.githubusercontent.com/gcc-mirror/gcc/master/config.sub' -o config.sub \ && ./configure --prefix=${INSTALL_LOC}/ \ && make if [ $? -ne 0 ]; then diff --git a/build_helpers/install_windows.ps1 b/build_helpers/install_windows.ps1 index 73a6eea06..4aa070992 100644 --- a/build_helpers/install_windows.ps1 +++ b/build_helpers/install_windows.ps1 @@ -5,7 +5,7 @@ python -m pip install --upgrade pip wheel $pyv = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" -pip install --find-links=build_helpers\ TA-Lib +pip install --find-links=build_helpers\ --prefer-binary TA-Lib pip install -r requirements-dev.txt pip install -e . diff --git a/config_examples/config_full.example.json b/config_examples/config_full.example.json index 42c05eaf5..4681ec7df 100644 --- a/config_examples/config_full.example.json +++ b/config_examples/config_full.example.json @@ -89,7 +89,6 @@ ], "exchange": { "name": "binance", - "sandbox": false, "key": "your_exchange_key", "secret": "your_exchange_secret", "password": "", diff --git a/docs/assets/pycharm_debug.png b/docs/assets/pycharm_debug.png new file mode 100644 index 000000000..b1c19ca93 Binary files /dev/null and b/docs/assets/pycharm_debug.png differ diff --git a/docs/bot-basics.md b/docs/bot-basics.md index ef5e6900b..a19f005db 100644 --- a/docs/bot-basics.md +++ b/docs/bot-basics.md @@ -7,7 +7,7 @@ This page provides you some basic concepts on how Freqtrade works and operates. * **Strategy**: Your trading strategy, telling the bot what to do. * **Trade**: Open position. * **Open Order**: Order which is currently placed on the exchange, and is not yet complete. -* **Pair**: Tradable pair, usually in the format of Base/Quote (e.g. XRP/USDT). +* **Pair**: Tradable pair, usually in the format of Base/Quote (e.g. `XRP/USDT` for spot, `XRP/USDT:USDT` for futures). * **Timeframe**: Candle length to use (e.g. `"5m"`, `"1h"`, ...). * **Indicators**: Technical indicators (SMA, EMA, RSI, ...). * **Limit order**: Limit orders which execute at the defined limit price or better. @@ -20,6 +20,20 @@ This page provides you some basic concepts on how Freqtrade works and operates. All profit calculations of Freqtrade include fees. For Backtesting / Hyperopt / Dry-run modes, the exchange default fee is used (lowest tier on the exchange). For live operations, fees are used as applied by the exchange (this includes BNB rebates etc.). +## Pair naming + +Freqtrade follows the [ccxt naming convention](https://docs.ccxt.com/#/README?id=consistency-of-base-and-quote-currencies) for currencies. +Using the wrong naming convention in the wrong market will usually result in the bot not recognizing the pair, usually resulting in errors like "this pair is not available". + +### Spot pair naming + +For spot pairs, naming will be `base/quote` (e.g. `ETH/USDT`). + +### Futures pair naming + +For futures pairs, naming will be `base/quote:settle` (e.g. `ETH/USDT:USDT`). + + ## Bot execution logic Starting freqtrade in dry-run or live mode (using `freqtrade trade`) will start the bot and start the bot iteration loop. diff --git a/docs/bot-usage.md b/docs/bot-usage.md index c6a7f6103..7aeda0c42 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -3,7 +3,7 @@ This page explains the different parameters of the bot and how to run it. !!! Note - If you've used `setup.sh`, don't forget to activate your virtual environment (`source .env/bin/activate`) before running freqtrade commands. + If you've used `setup.sh`, don't forget to activate your virtual environment (`source .venv/bin/activate`) before running freqtrade commands. !!! Warning "Up-to-date clock" The clock on the system running the bot must be accurate, synchronized to a NTP server frequently enough to avoid problems with communication to the exchanges. diff --git a/docs/configuration.md b/docs/configuration.md index e05d1b450..6c0795c67 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -188,7 +188,6 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `max_entry_position_adjustment` | Maximum additional order(s) for each open trade on top of the first entry Order. Set it to `-1` for unlimited additional orders. [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `-1`.*
**Datatype:** Positive Integer or -1 | | **Exchange** | `exchange.name` | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename).
**Datatype:** String -| `exchange.sandbox` | Use the 'sandbox' version of the exchange, where the exchange provides a sandbox for risk-free integration. See [here](sandbox-testing.md) in more details.
**Datatype:** Boolean | `exchange.key` | API key to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `exchange.secret` | API secret to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `exchange.password` | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String diff --git a/docs/data-analysis.md b/docs/data-analysis.md index afee9049f..7f5637dd0 100644 --- a/docs/data-analysis.md +++ b/docs/data-analysis.md @@ -27,7 +27,7 @@ For this to work, first activate your virtual environment and run the following ``` bash # Activate virtual environment -source .env/bin/activate +source .venv/bin/activate pip install ipykernel ipython kernel install --user --name=freqtrade diff --git a/docs/developer.md b/docs/developer.md index 4784e5352..23f4a9b26 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -77,7 +77,7 @@ def test_method_to_test(caplog): ### Debug configuration -To debug freqtrade, we recommend VSCode with the following launch configuration (located in `.vscode/launch.json`). +To debug freqtrade, we recommend VSCode (with the Python extension) with the following launch configuration (located in `.vscode/launch.json`). Details will obviously vary between setups - but this should work to get you started. ``` json @@ -102,6 +102,19 @@ This method can also be used to debug a strategy, by setting the breakpoints wit A similar setup can also be taken for Pycharm - using `freqtrade` as module name, and setting the command line arguments as "parameters". +??? Tip "Correct venv usage" + When using a virtual environment (which you should), make sure that your Editor is using the correct virtual environment to avoid problems or "unknown import" errors. + + #### Vscode + + You can select the correct environment in VSCode with the command "Python: Select Interpreter" - which will show you environments the extension detected. + If your environment has not been detected, you can also pick a path manually. + + #### Pycharm + + In pycharm, you can select the appropriate Environment in the "Run/Debug Configurations" window. + ![Pycharm debug configuration](assets/pycharm_debug.png) + !!! Note "Startup directory" This assumes that you have the repository checked out, and the editor is started at the repository root level (so setup.py is at the top level of your repository). diff --git a/docs/faq.md b/docs/faq.md index 0a3b152cb..50aaa03a3 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -36,7 +36,7 @@ Running the bot with `freqtrade trade --config config.json` shows the output `fr This could be caused by the following reasons: * The virtual environment is not active. - * Run `source .env/bin/activate` to activate the virtual environment. + * Run `source .venv/bin/activate` to activate the virtual environment. * The installation did not complete successfully. * Please check the [Installation documentation](installation.md). diff --git a/docs/freqai-parameter-table.md b/docs/freqai-parameter-table.md index 5e60d2a07..95687c7ab 100644 --- a/docs/freqai-parameter-table.md +++ b/docs/freqai-parameter-table.md @@ -100,12 +100,12 @@ Mandatory parameters are marked as **Required** and have to be set in one of the #### trainer_kwargs -| Parameter | Description | -|------------|-------------| -| | **Model training parameters within the `freqai.model_training_parameters.model_kwargs` sub dictionary** -| `max_iters` | The number of training iterations to run. iteration here refers to the number of times we call self.optimizer.step(). used to calculate n_epochs.
**Datatype:** int.
Default: `100`. -| `batch_size` | The size of the batches to use during training..
**Datatype:** int.
Default: `64`. -| `max_n_eval_batches` | The maximum number batches to use for evaluation..
**Datatype:** int, optional.
Default: `None`. +| Parameter | Description | +|--------------|-------------| +| | **Model training parameters within the `freqai.model_training_parameters.model_kwargs` sub dictionary** +| `n_epochs` | The `n_epochs` parameter is a crucial setting in the PyTorch training loop that determines the number of times the entire training dataset will be used to update the model's parameters. An epoch represents one full pass through the entire training dataset. Overrides `n_steps`. Either `n_epochs` or `n_steps` must be set.

**Datatype:** int. optional.
Default: `10`. +| `n_steps` | An alternative way of setting `n_epochs` - the number of training iterations to run. Iteration here refer to the number of times we call `optimizer.step()`. Ignored if `n_epochs` is set. A simplified version of the function:

n_epochs = n_steps / (n_obs / batch_size)

The motivation here is that `n_steps` is easier to optimize and keep stable across different n_obs - the number of data points.

**Datatype:** int. optional.
Default: `None`. +| `batch_size` | The size of the batches to use during training.

**Datatype:** int.
Default: `64`. ### Additional parameters diff --git a/docs/freqai-reinforcement-learning.md b/docs/freqai-reinforcement-learning.md index 1c95409ae..df4508c86 100644 --- a/docs/freqai-reinforcement-learning.md +++ b/docs/freqai-reinforcement-learning.md @@ -20,7 +20,7 @@ With the current framework, we aim to expose the training environment via the co We envision the majority of users focusing their effort on creative design of the `calculate_reward()` function [details here](#creating-a-custom-reward-function), while leaving the rest of the environment untouched. Other users may not touch the environment at all, and they will only play with the configuration settings and the powerful feature engineering that already exists in FreqAI. Meanwhile, we enable advanced users to create their own model classes entirely. -The framework is built on stable_baselines3 (torch) and OpenAI gym for the base environment class. But generally speaking, the model class is well isolated. Thus, the addition of competing libraries can be easily integrated into the existing framework. For the environment, it is inheriting from `gym.env` which means that it is necessary to write an entirely new environment in order to switch to a different library. +The framework is built on stable_baselines3 (torch) and OpenAI gym for the base environment class. But generally speaking, the model class is well isolated. Thus, the addition of competing libraries can be easily integrated into the existing framework. For the environment, it is inheriting from `gym.Env` which means that it is necessary to write an entirely new environment in order to switch to a different library. ### Important considerations @@ -173,7 +173,7 @@ class MyCoolRLModel(ReinforcementLearner): """ class MyRLEnv(Base5ActionRLEnv): """ - User made custom environment. This class inherits from BaseEnvironment and gym.env. + User made custom environment. This class inherits from BaseEnvironment and gym.Env. Users can override any functions from those parent classes. Here is an example of a user customized `calculate_reward()` function. @@ -254,7 +254,7 @@ FreqAI also provides a built in episodic summary logger called `self.tensorboard ```python class MyRLEnv(Base5ActionRLEnv): """ - User made custom environment. This class inherits from BaseEnvironment and gym.env. + User made custom environment. This class inherits from BaseEnvironment and gym.Env. Users can override any functions from those parent classes. Here is an example of a user customized `calculate_reward()` function. """ diff --git a/docs/hyperopt.md b/docs/hyperopt.md index b0920f9c4..3fbbee7f6 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -31,7 +31,7 @@ The docker-image includes hyperopt dependencies, no further action needed. ### Easy installation script (setup.sh) / Manual installation ```bash -source .env/bin/activate +source .venv/bin/activate pip install -r requirements-hyperopt.txt ``` diff --git a/docs/installation.md b/docs/installation.md index a06968dba..a6bfaccf0 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -143,11 +143,11 @@ If you are on Debian, Ubuntu or MacOS, freqtrade provides the script to install ### Activate your virtual environment -Each time you open a new terminal, you must run `source .env/bin/activate` to activate your virtual environment. +Each time you open a new terminal, you must run `source .venv/bin/activate` to activate your virtual environment. ```bash -# then activate your .env -source ./.env/bin/activate +# activate virtual environment +source ./.venv/bin/activate ``` ### Congratulations @@ -172,7 +172,7 @@ With this option, the script will install the bot and most dependencies: You will need to have git and python3.8+ installed beforehand for this to work. * Mandatory software as: `ta-lib` -* Setup your virtualenv under `.env/` +* Setup your virtualenv under `.venv/` This option is a combination of installation tasks and `--reset` @@ -225,11 +225,11 @@ rm -rf ./ta-lib* You will run freqtrade in separated `virtual environment` ```bash -# create virtualenv in directory /freqtrade/.env -python3 -m venv .env +# create virtualenv in directory /freqtrade/.venv +python3 -m venv .venv # run virtualenv -source .env/bin/activate +source .venv/bin/activate ``` #### Install python dependencies @@ -383,7 +383,7 @@ You've made it this far, so you have successfully installed freqtrade. freqtrade create-userdir --userdir user_data # Step 2 - Create a new configuration file -freqtrade new-config --config config.json +freqtrade new-config --config user_data/config.json ``` You are ready to run, read [Bot Configuration](configuration.md), remember to start with `dry_run: True` and verify that everything is working. @@ -393,7 +393,7 @@ To learn how to setup your configuration, please refer to the [Bot Configuration ### Start the Bot ```bash -freqtrade trade --config config.json --strategy SampleStrategy +freqtrade trade --config user_data/config.json --strategy SampleStrategy ``` !!! Warning @@ -411,8 +411,8 @@ If you used (1)`Script` or (2)`Manual` installation, you need to run the bot in # if: bash: freqtrade: command not found -# then activate your .env -source ./.env/bin/activate +# then activate your virtual environment +source ./.venv/bin/activate ``` ### MacOS installation error diff --git a/docs/leverage.md b/docs/leverage.md index deaa65896..09ebf1075 100644 --- a/docs/leverage.md +++ b/docs/leverage.md @@ -64,7 +64,7 @@ You will also have to pick a "margin mode" (explanation below) - with freqtrade ##### Pair namings -Freqtrade follows the [ccxt naming conventions for futures](https://docs.ccxt.com/en/latest/manual.html?#perpetual-swap-perpetual-future). +Freqtrade follows the [ccxt naming conventions for futures](https://docs.ccxt.com/#/README?id=perpetual-swap-perpetual-future). A futures pair will therefore have the naming of `base/quote:settle` (e.g. `ETH/USDT:USDT`). ### Margin mode diff --git a/docs/lookahead-analysis.md b/docs/lookahead-analysis.md index 9d57de779..e998b1b77 100644 --- a/docs/lookahead-analysis.md +++ b/docs/lookahead-analysis.md @@ -21,7 +21,10 @@ It also supports the lookahead-analysis of freqai strategies. - `--cache` is forced to "none". - `--max-open-trades` is forced to be at least equal to the number of pairs. -- `--dry-run-wallet` is forced to be basically infinite. +- `--dry-run-wallet` is forced to be basically infinite (1 billion). +- `--stake-amount` is forced to be a static 10000 (10k). + +Those are set to avoid users accidentally generating false positives. ## Lookahead-analysis command reference diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index cbf648dac..96e3dbf1d 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.4.4 mkdocs==1.5.2 -mkdocs-material==9.1.21 +mkdocs-material==9.2.1 mdx_truly_sane_lists==1.3 pymdown-extensions==10.1 jinja2==3.1.2 diff --git a/docs/sandbox-testing.md b/docs/sandbox-testing.md deleted file mode 100644 index 2c0f306cf..000000000 --- a/docs/sandbox-testing.md +++ /dev/null @@ -1,121 +0,0 @@ -# Sandbox API testing - -Some exchanges provide sandboxes or testbeds for risk-free testing, while running the bot against a real exchange. -With some configuration, freqtrade (in combination with ccxt) provides access to these. - -This document is an overview to configure Freqtrade to be used with sandboxes. -This can be useful to developers and trader alike. - -!!! Warning - Sandboxes usually have very low volume, and either a very wide spread, or no orders available at all. - Therefore, sandboxes will usually not do a good job of showing you how a strategy would work in real trading. - -## Exchanges known to have a sandbox / testnet - -* [binance](https://testnet.binance.vision/) -* [coinbasepro](https://public.sandbox.pro.coinbase.com) -* [gemini](https://exchange.sandbox.gemini.com/) -* [huobipro](https://www.testnet.huobi.pro/) -* [kucoin](https://sandbox.kucoin.com/) -* [phemex](https://testnet.phemex.com/) - -!!! Note - We did not test correct functioning of all of the above testnets. Please report your experiences with each sandbox. - ---- - -## Configure a Sandbox account - -When testing your API connectivity, make sure to use the appropriate sandbox / testnet URL. - -In general, you should follow these steps to enable an exchange's sandbox: - -* Figure out if an exchange has a sandbox (most likely by using google or the exchange's support documents) -* Create a sandbox account (often the sandbox-account requires separate registration) -* [Add some test assets to account](#add-test-funds) -* Create API keys - -### Add test funds - -Usually, sandbox exchanges allow depositing funds directly via web-interface. -You should make sure to have a realistic amount of funds available to your test-account, so results are representable of your real account funds. - -!!! Warning - Test exchanges will **NEVER** require your real credit card or banking details! - -## Configure freqtrade to use a exchange's sandbox - -### Sandbox URLs - -Freqtrade makes use of CCXT which in turn provides a list of URLs to Freqtrade. -These include `['test']` and `['api']`. - -* `[Test]` if available will point to an Exchanges sandbox. -* `[Api]` normally used, and resolves to live API target on the exchange. - -To make use of sandbox / test add "sandbox": true, to your config.json - -```json - "exchange": { - "name": "coinbasepro", - "sandbox": true, - "key": "5wowfxemogxeowo;heiohgmd", - "secret": "/ZMH1P62rCVmwefewrgcewX8nh4gob+lywxfwfxwwfxwfNsH1ySgvWCUR/w==", - "password": "1bkjfkhfhfu6sr", - "outdated_offset": 5 - "pair_whitelist": [ - "BTC/USD" - ] - }, - "datadir": "user_data/data/coinbasepro_sandbox" -``` - -Also the following information: - -* api-key (created for the sandbox webpage) -* api-secret (noted earlier) -* password (the passphrase - noted earlier) - -!!! Tip "Different data directory" - We also recommend to set `datadir` to something identifying downloaded data as sandbox data, to avoid having sandbox data mixed with data from the real exchange. - This can be done by adding the `"datadir"` key to the configuration. - Now, whenever you use this configuration, your data directory will be set to this directory. - ---- - -## You should now be ready to test your sandbox - -Ensure Freqtrade logs show the sandbox URL, and trades made are shown in sandbox. Also make sure to select a pair which shows at least some decent value (which very often is BTC/). - -## Common problems with sandbox exchanges - -Sandbox exchange instances often have very low volume, which can cause some problems which usually are not seen on a real exchange instance. - -### Old Candles problem - -Since Sandboxes often have low volume, candles can be quite old and show no volume. -To disable the error "Outdated history for pair ...", best increase the parameter `"outdated_offset"` to a number that seems realistic for the sandbox you're using. - -### Unfilled orders - -Sandboxes often have very low volumes - which means that many trades can go unfilled, or can go unfilled for a very long time. - -To mitigate this, you can try to match the first order on the opposite orderbook side using the following configuration: - -``` jsonc - "order_types": { - "entry": "limit", - "exit": "limit" - // ... - }, - "entry_pricing": { - "price_side": "other", - // ... - }, - "exit_pricing":{ - "price_side": "other", - // ... - }, - ``` - - The configuration is similar to the suggested configuration for market orders - however by using limit-orders you can avoid moving the price too much, and you can set the worst price you might get. diff --git a/docs/windows_installation.md b/docs/windows_installation.md index 0327f21e5..db785a1fc 100644 --- a/docs/windows_installation.md +++ b/docs/windows_installation.md @@ -31,8 +31,8 @@ Other versions must be downloaded from the above link. ``` powershell cd \path\freqtrade -python -m venv .env -.env\Scripts\activate.ps1 +python -m venv .venv +.venv\Scripts\activate.ps1 # optionally install ta-lib from wheel # Eventually adjust the below filename to match the downloaded wheel pip install --find-links build_helpers\ TA-Lib -U diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 311622458..6692ad59a 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -10,7 +10,7 @@ from freqtrade.configuration.directory_operations import chown_user_directory from freqtrade.constants import UNLIMITED_STAKE_AMOUNT from freqtrade.exceptions import OperationalException from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS, available_exchanges -from freqtrade.misc import render_template +from freqtrade.util import render_template logger = logging.getLogger(__name__) @@ -105,7 +105,7 @@ def ask_user_config() -> Dict[str, Any]: "type": "select", "name": "exchange_name", "message": "Select exchange", - "choices": lambda x: [ + "choices": [ "binance", "binanceus", "bittrex", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 54471917d..586318e30 100755 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -441,7 +441,7 @@ AVAILABLE_CLI_OPTIONS = { "dataformat_trades": Arg( '--data-format-trades', help='Storage format for downloaded trades data. (default: `feather`).', - choices=constants.AVAILABLE_DATAHANDLERS_TRADES, + choices=constants.AVAILABLE_DATAHANDLERS, ), "show_timerange": Arg( '--show-timerange', diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index 9ec33eac4..75da2552e 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -10,7 +10,7 @@ from freqtrade.configuration.directory_operations import copy_sample_files, crea from freqtrade.constants import USERPATH_STRATEGIES from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.misc import render_template, render_template_with_fallback +from freqtrade.util import render_template, render_template_with_fallback logger = logging.getLogger(__name__) @@ -35,6 +35,10 @@ def deploy_new_strategy(strategy_name: str, strategy_path: Path, subtemplate: st Deploy new strategy from template to strategy_path """ fallback = 'full' + attributes = render_template_with_fallback( + templatefile=f"strategy_subtemplates/strategy_attributes_{subtemplate}.j2", + templatefallbackfile=f"strategy_subtemplates/strategy_attributes_{fallback}.j2", + ) indicators = render_template_with_fallback( templatefile=f"strategy_subtemplates/indicators_{subtemplate}.j2", templatefallbackfile=f"strategy_subtemplates/indicators_{fallback}.j2", @@ -58,6 +62,7 @@ def deploy_new_strategy(strategy_name: str, strategy_path: Path, subtemplate: st strategy_text = render_template(templatefile='base_strategy.py.j2', arguments={"strategy": strategy_name, + "attributes": attributes, "indicators": indicators, "buy_trend": buy_trend, "sell_trend": sell_trend, diff --git a/freqtrade/commands/webserver_commands.py b/freqtrade/commands/webserver_commands.py index 9a5975227..2fd7fe75c 100644 --- a/freqtrade/commands/webserver_commands.py +++ b/freqtrade/commands/webserver_commands.py @@ -7,9 +7,10 @@ def start_webserver(args: Dict[str, Any]) -> None: """ Main entry point for webserver mode """ - from freqtrade.configuration import Configuration + from freqtrade.configuration import setup_utils_configuration from freqtrade.rpc.api_server import ApiServer # Initialize configuration - config = Configuration(args, RunMode.WEBSERVER).get_config() + + config = setup_utils_configuration(args, RunMode.WEBSERVER) ApiServer(config, standalone=True) diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index f1745df61..395826557 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -51,6 +51,8 @@ def validate_config_schema(conf: Dict[str, Any], preliminary: bool = False) -> D conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED else: conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED_FINAL + elif conf.get('runmode', RunMode.OTHER) == RunMode.WEBSERVER: + conf_schema['required'] = constants.SCHEMA_MINIMAL_WEBSERVER else: conf_schema['required'] = constants.SCHEMA_MINIMAL_REQUIRED try: diff --git a/freqtrade/configuration/environment_vars.py b/freqtrade/configuration/environment_vars.py index d59d4bd23..c5efd45b7 100644 --- a/freqtrade/configuration/environment_vars.py +++ b/freqtrade/configuration/environment_vars.py @@ -41,7 +41,7 @@ def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, key = env_var.replace(prefix, '') for k in reversed(key.split('__')): val = {k.lower(): get_var_typed(val) - if type(val) != dict and k not in no_convert else val} + if not isinstance(val, dict) and k not in no_convert else val} relevant_vars = deep_merge_dicts(val, relevant_vars) return relevant_vars diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 4b3cff7a1..733fa11b3 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -38,8 +38,7 @@ AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', ' 'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter'] AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard'] -AVAILABLE_DATAHANDLERS_TRADES = ['json', 'jsongz', 'hdf5', 'feather'] -AVAILABLE_DATAHANDLERS = AVAILABLE_DATAHANDLERS_TRADES + ['parquet'] +AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5', 'feather', 'parquet'] BACKTEST_BREAKDOWNS = ['day', 'week', 'month'] BACKTEST_CACHE_AGE = ['none', 'day', 'week', 'month'] BACKTEST_CACHE_DEFAULT = 'day' @@ -50,6 +49,15 @@ DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume'] # Don't modify sequence of DEFAULT_TRADES_COLUMNS # it has wide consequences for stored trades files DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost'] +TRADES_DTYPES = { + 'timestamp': 'int64', + 'id': 'str', + 'type': 'str', + 'side': 'str', + 'price': 'float64', + 'amount': 'float64', + 'cost': 'float64', +} TRADING_MODES = ['spot', 'margin', 'futures'] MARGIN_MODES = ['cross', 'isolated', ''] @@ -450,7 +458,7 @@ CONF_SCHEMA = { }, 'dataformat_trades': { 'type': 'string', - 'enum': AVAILABLE_DATAHANDLERS_TRADES, + 'enum': AVAILABLE_DATAHANDLERS, 'default': 'feather' }, 'position_adjustment_enable': {'type': 'boolean'}, @@ -461,7 +469,6 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'name': {'type': 'string'}, - 'sandbox': {'type': 'boolean', 'default': False}, 'key': {'type': 'string', 'default': ''}, 'secret': {'type': 'string', 'default': ''}, 'password': {'type': 'string', 'default': ''}, @@ -668,6 +675,9 @@ SCHEMA_MINIMAL_REQUIRED = [ 'dataformat_ohlcv', 'dataformat_trades', ] +SCHEMA_MINIMAL_WEBSERVER = SCHEMA_MINIMAL_REQUIRED + [ + 'api_server', +] CANCEL_REASON = { "TIMEOUT": "cancelled due to timeout", diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 0eb6faf77..324ce5def 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -1,16 +1,15 @@ """ Functions to convert data from one format to another """ -import itertools import logging -from operator import itemgetter from typing import Dict, List import numpy as np import pandas as pd from pandas import DataFrame, to_datetime -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, Config, TradeList +from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TRADES_DTYPES, + Config, TradeList) from freqtrade.enums import CandleType, TradingMode @@ -195,15 +194,14 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame: return frame -def trades_remove_duplicates(trades: List[List]) -> List[List]: +def trades_df_remove_duplicates(trades: pd.DataFrame) -> pd.DataFrame: """ - Removes duplicates from the trades list. - Uses itertools.groupby to avoid converting to pandas. - Tests show it as being pretty efficient on lists of 4M Lists. - :param trades: List of Lists with constants.DEFAULT_TRADES_COLUMNS as columns - :return: same format as above, but with duplicates removed + Removes duplicates from the trades DataFrame. + Uses pandas.DataFrame.drop_duplicates to remove duplicates based on the 'timestamp' column. + :param trades: DataFrame with the columns constants.DEFAULT_TRADES_COLUMNS + :return: DataFrame with duplicates removed based on the 'timestamp' column """ - return [i for i, _ in itertools.groupby(sorted(trades, key=itemgetter(0)))] + return trades.drop_duplicates(subset=['timestamp', 'id']) def trades_dict_to_list(trades: List[Dict]) -> TradeList: @@ -215,7 +213,32 @@ def trades_dict_to_list(trades: List[Dict]) -> TradeList: return [[t[col] for col in DEFAULT_TRADES_COLUMNS] for t in trades] -def trades_to_ohlcv(trades: TradeList, timeframe: str) -> DataFrame: +def trades_convert_types(trades: DataFrame) -> DataFrame: + """ + Convert Trades dtypes and add 'date' column + """ + trades = trades.astype(TRADES_DTYPES) + trades['date'] = to_datetime(trades['timestamp'], unit='ms', utc=True) + return trades + + +def trades_list_to_df(trades: TradeList, convert: bool = True): + """ + convert trades list to dataframe + :param trades: List of Lists with constants.DEFAULT_TRADES_COLUMNS as columns + """ + if not trades: + df = DataFrame(columns=DEFAULT_TRADES_COLUMNS) + else: + df = DataFrame(trades, columns=DEFAULT_TRADES_COLUMNS) + + if convert: + df = trades_convert_types(df) + + return df + + +def trades_to_ohlcv(trades: DataFrame, timeframe: str) -> DataFrame: """ Converts trades list to OHLCV list :param trades: List of trades, as returned by ccxt.fetch_trades. @@ -225,12 +248,9 @@ def trades_to_ohlcv(trades: TradeList, timeframe: str) -> DataFrame: """ from freqtrade.exchange import timeframe_to_minutes timeframe_minutes = timeframe_to_minutes(timeframe) - if not trades: + if trades.empty: raise ValueError('Trade-list empty.') - df = pd.DataFrame(trades, columns=DEFAULT_TRADES_COLUMNS) - df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', - utc=True,) - df = df.set_index('timestamp') + df = trades.set_index('date', drop=True) df_new = df['price'].resample(f'{timeframe_minutes}min').ohlc() df_new['volume'] = df['amount'].resample(f'{timeframe_minutes}min').sum() diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index e7ba20c44..11cbd7934 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -17,7 +17,7 @@ from freqtrade.constants import (FULL_DATAFRAME_THRESHOLD, Config, ListPairsWith from freqtrade.data.history import load_pair_history from freqtrade.enums import CandleType, RPCMessageType, RunMode from freqtrade.exceptions import ExchangeError, OperationalException -from freqtrade.exchange import Exchange, timeframe_to_seconds +from freqtrade.exchange import Exchange, timeframe_to_prev_date, timeframe_to_seconds from freqtrade.exchange.types import OrderBook from freqtrade.misc import append_candles_to_dataframe from freqtrade.rpc import RPCManager @@ -46,6 +46,8 @@ class DataProvider: self.__rpc = rpc self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {} self.__slice_index: Optional[int] = None + self.__slice_date: Optional[datetime] = None + self.__cached_pairs_backtesting: Dict[PairWithTimeframe, DataFrame] = {} self.__producer_pairs_df: Dict[str, Dict[PairWithTimeframe, Tuple[DataFrame, datetime]]] = {} @@ -64,10 +66,19 @@ class DataProvider: def _set_dataframe_max_index(self, limit_index: int): """ Limit analyzed dataframe to max specified index. + Only relevant in backtesting. :param limit_index: dataframe index. """ self.__slice_index = limit_index + def _set_dataframe_max_date(self, limit_date: datetime): + """ + Limit infomrative dataframe to max specified index. + Only relevant in backtesting. + :param limit_date: "current date" + """ + self.__slice_date = limit_date + def _set_cached_df( self, pair: str, @@ -284,7 +295,7 @@ class DataProvider: def historic_ohlcv( self, pair: str, - timeframe: Optional[str] = None, + timeframe: str, candle_type: str = '' ) -> DataFrame: """ @@ -307,7 +318,7 @@ class DataProvider: timerange.subtract_start(tf_seconds * startup_candles) self.__cached_pairs_backtesting[saved_pair] = load_pair_history( pair=pair, - timeframe=timeframe or self._config['timeframe'], + timeframe=timeframe, datadir=self._config['datadir'], timerange=timerange, data_format=self._config['dataformat_ohlcv'], @@ -354,7 +365,13 @@ class DataProvider: data = self.ohlcv(pair=pair, timeframe=timeframe, candle_type=candle_type) else: # Get historical OHLCV data (cached on disk). + timeframe = timeframe or self._config['timeframe'] data = self.historic_ohlcv(pair=pair, timeframe=timeframe, candle_type=candle_type) + # Cut date to timeframe-specific date. + # This is necessary to prevent lookahead bias in callbacks through informative pairs. + if self.__slice_date: + cutoff_date = timeframe_to_prev_date(timeframe, self.__slice_date) + data = data.loc[data['date'] < cutoff_date] if len(data) == 0: logger.warning(f"No data found for ({pair}, {timeframe}, {candle_type}).") return data diff --git a/freqtrade/data/history/featherdatahandler.py b/freqtrade/data/history/featherdatahandler.py index 28a12fb29..44d337836 100644 --- a/freqtrade/data/history/featherdatahandler.py +++ b/freqtrade/data/history/featherdatahandler.py @@ -4,7 +4,7 @@ from typing import Optional from pandas import DataFrame, read_feather, to_datetime from freqtrade.configuration import TimeRange -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS from freqtrade.enums import CandleType from .idatahandler import IDataHandler @@ -82,43 +82,41 @@ class FeatherDataHandler(IDataHandler): """ raise NotImplementedError() - def trades_store(self, pair: str, data: TradeList) -> None: + def _trades_store(self, pair: str, data: DataFrame) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename - :param data: List of Lists containing trade data, + :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS """ filename = self._pair_trades_filename(self._datadir, pair) self.create_dir_if_needed(filename) + data.reset_index(drop=True).to_feather(filename, compression_level=9, compression='lz4') - tradesdata = DataFrame(data, columns=DEFAULT_TRADES_COLUMNS) - tradesdata.to_feather(filename, compression_level=9, compression='lz4') - - def trades_append(self, pair: str, data: TradeList): + def trades_append(self, pair: str, data: DataFrame): """ Append data to existing files :param pair: Pair - used for filename - :param data: List of Lists containing trade data, + :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS """ raise NotImplementedError() - def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame: """ Load a pair from file, either .json.gz or .json # TODO: respect timerange ... :param pair: Load trades for this pair :param timerange: Timerange to load trades for - currently not implemented - :return: List of trades + :return: Dataframe containing trades """ filename = self._pair_trades_filename(self._datadir, pair) if not filename.exists(): - return [] + return DataFrame(columns=DEFAULT_TRADES_COLUMNS) tradesdata = read_feather(filename) - return tradesdata.values.tolist() + return tradesdata @classmethod def _get_file_extension(cls): diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index fd46115de..d22fd9e31 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -5,7 +5,7 @@ import numpy as np import pandas as pd from freqtrade.configuration import TimeRange -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS from freqtrade.enums import CandleType from .idatahandler import IDataHandler @@ -100,42 +100,42 @@ class HDF5DataHandler(IDataHandler): """ raise NotImplementedError() - def trades_store(self, pair: str, data: TradeList) -> None: + def _trades_store(self, pair: str, data: pd.DataFrame) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename - :param data: List of Lists containing trade data, + :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS """ key = self._pair_trades_key(pair) - pd.DataFrame(data, columns=DEFAULT_TRADES_COLUMNS).to_hdf( + data.to_hdf( self._pair_trades_filename(self._datadir, pair), key, mode='a', complevel=9, complib='blosc', format='table', data_columns=['timestamp'] ) - def trades_append(self, pair: str, data: TradeList): + def trades_append(self, pair: str, data: pd.DataFrame): """ Append data to existing files :param pair: Pair - used for filename - :param data: List of Lists containing trade data, + :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS """ raise NotImplementedError() - def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> pd.DataFrame: """ Load a pair from h5 file. :param pair: Load trades for this pair :param timerange: Timerange to load trades for - currently not implemented - :return: List of trades + :return: Dataframe containing trades """ key = self._pair_trades_key(pair) filename = self._pair_trades_filename(self._datadir, pair) if not filename.exists(): - return [] + return pd.DataFrame(columns=DEFAULT_TRADES_COLUMNS) where = [] if timerange: if timerange.starttype == 'date': @@ -145,7 +145,7 @@ class HDF5DataHandler(IDataHandler): trades: pd.DataFrame = pd.read_hdf(filename, key=key, mode="r", where=where) trades[['id', 'type']] = trades[['id', 'type']].replace({np.nan: None}) - return trades.values.tolist() + return trades @classmethod def _get_file_extension(cls): diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 1a500d65c..5175195c5 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -10,14 +10,16 @@ from freqtrade.configuration import TimeRange from freqtrade.constants import (DATETIME_PRINT_FORMAT, DEFAULT_DATAFRAME_COLUMNS, DL_DATA_TIMEFRAMES, Config) from freqtrade.data.converter import (clean_ohlcv_dataframe, ohlcv_to_dataframe, - trades_remove_duplicates, trades_to_ohlcv) + trades_df_remove_duplicates, trades_list_to_df, + trades_to_ohlcv) from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist -from freqtrade.util import format_ms_time +from freqtrade.util import dt_ts, format_ms_time from freqtrade.util.binance_mig import migrate_binance_futures_data +from freqtrade.util.datetime_helpers import dt_now logger = logging.getLogger(__name__) @@ -349,24 +351,27 @@ def _download_trades_history(exchange: Exchange, # DEFAULT_TRADES_COLUMNS: 0 -> timestamp # DEFAULT_TRADES_COLUMNS: 1 -> id - if trades and since < trades[0][0]: + if not trades.empty and since > 0 and since < trades.iloc[0]['timestamp']: # since is before the first trade - logger.info(f"Start earlier than available data. Redownloading trades for {pair}...") - trades = [] + logger.info(f"Start ({trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}) earlier than " + f"available data. Redownloading trades for {pair}...") + trades = trades_list_to_df([]) - if not since: - since = int((datetime.now() - timedelta(days=new_pairs_days)).timestamp()) * 1000 - - from_id = trades[-1][1] if trades else None - if trades and since < trades[-1][0]: + from_id = trades.iloc[-1]['id'] if not trades.empty else None + if not trades.empty and since < trades.iloc[-1]['timestamp']: # Reset since to the last available point # - 5 seconds (to ensure we're getting all trades) - since = trades[-1][0] - (5 * 1000) + since = trades.iloc[-1]['timestamp'] - (5 * 1000) logger.info(f"Using last trade date -5s - Downloading trades for {pair} " f"since: {format_ms_time(since)}.") - logger.debug(f"Current Start: {format_ms_time(trades[0][0]) if trades else 'None'}") - logger.debug(f"Current End: {format_ms_time(trades[-1][0]) if trades else 'None'}") + if not since: + since = dt_ts(dt_now() - timedelta(days=new_pairs_days)) + + logger.debug("Current Start: %s", 'None' if trades.empty else + f"{trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}") + logger.debug("Current End: %s", 'None' if trades.empty else + f"{trades.iloc[-1]['date']:{DATETIME_PRINT_FORMAT}}") logger.info(f"Current Amount of trades: {len(trades)}") # Default since_ms to 30 days if nothing is given @@ -375,13 +380,16 @@ def _download_trades_history(exchange: Exchange, until=until, from_id=from_id, ) - trades.extend(new_trades[1]) + new_trades_df = trades_list_to_df(new_trades[1]) + trades = concat([trades, new_trades_df], axis=0) # Remove duplicates to make sure we're not storing data we don't need - trades = trades_remove_duplicates(trades) + trades = trades_df_remove_duplicates(trades) data_handler.trades_store(pair, data=trades) - logger.debug(f"New Start: {format_ms_time(trades[0][0])}") - logger.debug(f"New End: {format_ms_time(trades[-1][0])}") + logger.debug("New Start: %s", 'None' if trades.empty else + f"{trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}") + logger.debug("New End: %s", 'None' if trades.empty else + f"{trades.iloc[-1]['date']:{DATETIME_PRINT_FORMAT}}") logger.info(f"New Amount of trades: {len(trades)}") return True diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 698c7e267..d8c063f2a 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -15,8 +15,9 @@ from pandas import DataFrame from freqtrade import misc from freqtrade.configuration import TimeRange -from freqtrade.constants import ListPairsWithTimeframes, TradeList -from freqtrade.data.converter import clean_ohlcv_dataframe, trades_remove_duplicates, trim_dataframe +from freqtrade.constants import DEFAULT_TRADES_COLUMNS, ListPairsWithTimeframes +from freqtrade.data.converter import (clean_ohlcv_dataframe, trades_convert_types, + trades_df_remove_duplicates, trim_dataframe) from freqtrade.enums import CandleType, TradingMode from freqtrade.exchange import timeframe_to_seconds @@ -170,32 +171,42 @@ class IDataHandler(ABC): return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match] @abstractmethod - def trades_store(self, pair: str, data: TradeList) -> None: + def _trades_store(self, pair: str, data: DataFrame) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename - :param data: List of Lists containing trade data, + :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS """ @abstractmethod - def trades_append(self, pair: str, data: TradeList): + def trades_append(self, pair: str, data: DataFrame): """ Append data to existing files :param pair: Pair - used for filename - :param data: List of Lists containing trade data, + :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS """ @abstractmethod - def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame: """ Load a pair from file, either .json.gz or .json :param pair: Load trades for this pair :param timerange: Timerange to load trades for - currently not implemented - :return: List of trades + :return: Dataframe containing trades """ + def trades_store(self, pair: str, data: DataFrame) -> None: + """ + Store trades data (list of Dicts) to file + :param pair: Pair - used for filename + :param data: Dataframe containing trades + column sequence as in DEFAULT_TRADES_COLUMNS + """ + # Filter on expected columns (will remove the actual date column). + self._trades_store(pair, data[DEFAULT_TRADES_COLUMNS]) + def trades_purge(self, pair: str) -> bool: """ Remove data for this pair @@ -208,7 +219,7 @@ class IDataHandler(ABC): return True return False - def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame: """ Load a pair from file, either .json.gz or .json Removes duplicates in the process. @@ -216,7 +227,10 @@ class IDataHandler(ABC): :param timerange: Timerange to load trades for - currently not implemented :return: List of trades """ - return trades_remove_duplicates(self._trades_load(pair, timerange=timerange)) + trades = trades_df_remove_duplicates(self._trades_load(pair, timerange=timerange)) + + trades = trades_convert_types(trades) + return trades @classmethod def create_dir_if_needed(cls, datadir: Path): diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index ed7a33f8e..9a02a7769 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -6,8 +6,8 @@ from pandas import DataFrame, read_json, to_datetime from freqtrade import misc from freqtrade.configuration import TimeRange -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, TradeList -from freqtrade.data.converter import trades_dict_to_list +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS +from freqtrade.data.converter import trades_dict_to_list, trades_list_to_df from freqtrade.enums import CandleType from .idatahandler import IDataHandler @@ -94,45 +94,46 @@ class JsonDataHandler(IDataHandler): """ raise NotImplementedError() - def trades_store(self, pair: str, data: TradeList) -> None: + def _trades_store(self, pair: str, data: DataFrame) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename - :param data: List of Lists containing trade data, + :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS """ filename = self._pair_trades_filename(self._datadir, pair) - misc.file_dump_json(filename, data, is_zip=self._use_zip) + trades = data.values.tolist() + misc.file_dump_json(filename, trades, is_zip=self._use_zip) - def trades_append(self, pair: str, data: TradeList): + def trades_append(self, pair: str, data: DataFrame): """ Append data to existing files :param pair: Pair - used for filename - :param data: List of Lists containing trade data, + :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS """ raise NotImplementedError() - def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame: """ Load a pair from file, either .json.gz or .json # TODO: respect timerange ... :param pair: Load trades for this pair :param timerange: Timerange to load trades for - currently not implemented - :return: List of trades + :return: Dataframe containing trades """ filename = self._pair_trades_filename(self._datadir, pair) tradesdata = misc.file_load_json(filename) if not tradesdata: - return [] + return DataFrame(columns=DEFAULT_TRADES_COLUMNS) if isinstance(tradesdata[0], dict): # Convert trades dict to list logger.info("Old trades format detected - converting") tradesdata = trades_dict_to_list(tradesdata) pass - return tradesdata + return trades_list_to_df(tradesdata, convert=False) @classmethod def _get_file_extension(cls): diff --git a/freqtrade/data/history/parquetdatahandler.py b/freqtrade/data/history/parquetdatahandler.py index e6b2481d2..c0b0cad63 100644 --- a/freqtrade/data/history/parquetdatahandler.py +++ b/freqtrade/data/history/parquetdatahandler.py @@ -4,7 +4,7 @@ from typing import Optional from pandas import DataFrame, read_parquet, to_datetime from freqtrade.configuration import TimeRange -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, TradeList +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList from freqtrade.enums import CandleType from .idatahandler import IDataHandler @@ -81,25 +81,22 @@ class ParquetDataHandler(IDataHandler): """ raise NotImplementedError() - def trades_store(self, pair: str, data: TradeList) -> None: + def _trades_store(self, pair: str, data: DataFrame) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename - :param data: List of Lists containing trade data, + :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS """ - # filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair) + self.create_dir_if_needed(filename) + data.reset_index(drop=True).to_parquet(filename) - raise NotImplementedError() - # array = pa.array(data) - # array - # feather.write_feather(data, filename) - - def trades_append(self, pair: str, data: TradeList): + def trades_append(self, pair: str, data: DataFrame): """ Append data to existing files :param pair: Pair - used for filename - :param data: List of Lists containing trade data, + :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS """ raise NotImplementedError() @@ -112,14 +109,13 @@ class ParquetDataHandler(IDataHandler): :param timerange: Timerange to load trades for - currently not implemented :return: List of trades """ - raise NotImplementedError() - # filename = self._pair_trades_filename(self._datadir, pair) - # tradesdata = misc.file_load_json(filename) + filename = self._pair_trades_filename(self._datadir, pair) + if not filename.exists(): + return DataFrame(columns=DEFAULT_TRADES_COLUMNS) - # if not tradesdata: - # return [] + tradesdata = read_parquet(filename) - # return tradesdata + return tradesdata @classmethod def _get_file_extension(cls): diff --git a/freqtrade/exchange/binance_leverage_tiers.json b/freqtrade/exchange/binance_leverage_tiers.json index 7be7a9d29..a4a81abb3 100644 --- a/freqtrade/exchange/binance_leverage_tiers.json +++ b/freqtrade/exchange/binance_leverage_tiers.json @@ -590,13 +590,13 @@ "tier": 3.0, "currency": "USDT", "minNotional": 25000.0, - "maxNotional": 100000.0, + "maxNotional": 150000.0, "maintenanceMarginRate": 0.01, "maxLeverage": 25.0, "info": { "bracket": "3", "initialLeverage": "25", - "notionalCap": "100000", + "notionalCap": "150000", "notionalFloor": "25000", "maintMarginRatio": "0.01", "cum": "67.5" @@ -605,87 +605,87 @@ { "tier": 4.0, "currency": "USDT", - "minNotional": 100000.0, - "maxNotional": 300000.0, + "minNotional": 150000.0, + "maxNotional": 600000.0, "maintenanceMarginRate": 0.025, "maxLeverage": 20.0, "info": { "bracket": "4", "initialLeverage": "20", - "notionalCap": "300000", - "notionalFloor": "100000", + "notionalCap": "600000", + "notionalFloor": "150000", "maintMarginRatio": "0.025", - "cum": "1567.5" + "cum": "2317.5" } }, { "tier": 5.0, "currency": "USDT", - "minNotional": 300000.0, - "maxNotional": 750000.0, + "minNotional": 600000.0, + "maxNotional": 1500000.0, "maintenanceMarginRate": 0.05, "maxLeverage": 10.0, "info": { "bracket": "5", "initialLeverage": "10", - "notionalCap": "750000", - "notionalFloor": "300000", + "notionalCap": "1500000", + "notionalFloor": "600000", "maintMarginRatio": "0.05", - "cum": "9067.5" + "cum": "17317.5" } }, { "tier": 6.0, "currency": "USDT", - "minNotional": 750000.0, - "maxNotional": 1500000.0, + "minNotional": 1500000.0, + "maxNotional": 3000000.0, "maintenanceMarginRate": 0.1, "maxLeverage": 5.0, "info": { "bracket": "6", "initialLeverage": "5", - "notionalCap": "1500000", - "notionalFloor": "750000", + "notionalCap": "3000000", + "notionalFloor": "1500000", "maintMarginRatio": "0.1", - "cum": "46567.5" + "cum": "92317.5" } }, { "tier": 7.0, "currency": "USDT", - "minNotional": 1500000.0, - "maxNotional": 3000000.0, + "minNotional": 3000000.0, + "maxNotional": 5000000.0, "maintenanceMarginRate": 0.125, "maxLeverage": 4.0, "info": { "bracket": "7", "initialLeverage": "4", - "notionalCap": "3000000", - "notionalFloor": "1500000", + "notionalCap": "5000000", + "notionalFloor": "3000000", "maintMarginRatio": "0.125", - "cum": "84067.5" + "cum": "167317.5" } }, { "tier": 8.0, "currency": "USDT", - "minNotional": 3000000.0, - "maxNotional": 6000000.0, + "minNotional": 5000000.0, + "maxNotional": 9000000.0, "maintenanceMarginRate": 0.25, "maxLeverage": 2.0, "info": { "bracket": "8", "initialLeverage": "2", - "notionalCap": "6000000", - "notionalFloor": "3000000", + "notionalCap": "9000000", + "notionalFloor": "5000000", "maintMarginRatio": "0.25", - "cum": "459067.5" + "cum": "792317.5" } }, { "tier": 9.0, "currency": "USDT", - "minNotional": 6000000.0, + "minNotional": 9000000.0, "maxNotional": 30000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, @@ -693,9 +693,9 @@ "bracket": "9", "initialLeverage": "1", "notionalCap": "30000000", - "notionalFloor": "6000000", + "notionalFloor": "9000000", "maintMarginRatio": "0.5", - "cum": "1959067.5" + "cum": "3042317.5" } } ], @@ -1114,10 +1114,10 @@ "minNotional": 0.0, "maxNotional": 100000.0, "maintenanceMarginRate": 0.025, - "maxLeverage": 20.0, + "maxLeverage": 10.0, "info": { "bracket": "1", - "initialLeverage": "20", + "initialLeverage": "10", "notionalCap": "100000", "notionalFloor": "0", "maintMarginRatio": "0.025", @@ -1130,10 +1130,10 @@ "minNotional": 100000.0, "maxNotional": 500000.0, "maintenanceMarginRate": 0.05, - "maxLeverage": 10.0, + "maxLeverage": 8.0, "info": { "bracket": "2", - "initialLeverage": "10", + "initialLeverage": "8", "notionalCap": "500000", "notionalFloor": "100000", "maintMarginRatio": "0.05", @@ -1192,13 +1192,13 @@ "tier": 6.0, "currency": "BUSD", "minNotional": 5000000.0, - "maxNotional": 8000000.0, + "maxNotional": 5500000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { "bracket": "6", "initialLeverage": "1", - "notionalCap": "8000000", + "notionalCap": "5500000", "notionalFloor": "5000000", "maintMarginRatio": "0.5", "cum": "1527500.0" @@ -2759,14 +2759,14 @@ "currency": "USDT", "minNotional": 0.0, "maxNotional": 5000.0, - "maintenanceMarginRate": 0.02, - "maxLeverage": 25.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, "info": { "bracket": "1", - "initialLeverage": "25", + "initialLeverage": "20", "notionalCap": "5000", "notionalFloor": "0", - "maintMarginRatio": "0.02", + "maintMarginRatio": "0.025", "cum": "0.0" } }, @@ -2775,37 +2775,37 @@ "currency": "USDT", "minNotional": 5000.0, "maxNotional": 25000.0, - "maintenanceMarginRate": 0.025, - "maxLeverage": 20.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, "info": { "bracket": "2", - "initialLeverage": "20", + "initialLeverage": "10", "notionalCap": "25000", "notionalFloor": "5000", - "maintMarginRatio": "0.025", - "cum": "25.0" + "maintMarginRatio": "0.05", + "cum": "125.0" } }, { "tier": 3.0, "currency": "USDT", "minNotional": 25000.0, - "maxNotional": 600000.0, - "maintenanceMarginRate": 0.05, - "maxLeverage": 10.0, + "maxNotional": 500000.0, + "maintenanceMarginRate": 0.06, + "maxLeverage": 8.0, "info": { "bracket": "3", - "initialLeverage": "10", - "notionalCap": "600000", + "initialLeverage": "8", + "notionalCap": "500000", "notionalFloor": "25000", - "maintMarginRatio": "0.05", - "cum": "650.0" + "maintMarginRatio": "0.06", + "cum": "375.0" } }, { "tier": 4.0, "currency": "USDT", - "minNotional": 600000.0, + "minNotional": 500000.0, "maxNotional": 1600000.0, "maintenanceMarginRate": 0.1, "maxLeverage": 5.0, @@ -2813,9 +2813,9 @@ "bracket": "4", "initialLeverage": "5", "notionalCap": "1600000", - "notionalFloor": "600000", + "notionalFloor": "500000", "maintMarginRatio": "0.1", - "cum": "30650.0" + "cum": "20375.0" } }, { @@ -2831,39 +2831,39 @@ "notionalCap": "2000000", "notionalFloor": "1600000", "maintMarginRatio": "0.125", - "cum": "70650.0" + "cum": "60375.0" } }, { "tier": 6.0, "currency": "USDT", "minNotional": 2000000.0, - "maxNotional": 6000000.0, + "maxNotional": 3000000.0, "maintenanceMarginRate": 0.25, "maxLeverage": 2.0, "info": { "bracket": "6", "initialLeverage": "2", - "notionalCap": "6000000", + "notionalCap": "3000000", "notionalFloor": "2000000", "maintMarginRatio": "0.25", - "cum": "320650.0" + "cum": "310375.0" } }, { "tier": 7.0, "currency": "USDT", - "minNotional": 6000000.0, - "maxNotional": 10000000.0, + "minNotional": 3000000.0, + "maxNotional": 4000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { "bracket": "7", "initialLeverage": "1", - "notionalCap": "10000000", - "notionalFloor": "6000000", + "notionalCap": "4000000", + "notionalFloor": "3000000", "maintMarginRatio": "0.5", - "cum": "1820650.0" + "cum": "1060375.0" } } ], @@ -5413,14 +5413,14 @@ "currency": "USDT", "minNotional": 0.0, "maxNotional": 5000.0, - "maintenanceMarginRate": 0.02, - "maxLeverage": 20.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 8.0, "info": { "bracket": "1", - "initialLeverage": "20", + "initialLeverage": "8", "notionalCap": "5000", "notionalFloor": "0", - "maintMarginRatio": "0.02", + "maintMarginRatio": "0.025", "cum": "0.0" } }, @@ -5428,80 +5428,64 @@ "tier": 2.0, "currency": "USDT", "minNotional": 5000.0, - "maxNotional": 25000.0, - "maintenanceMarginRate": 0.025, - "maxLeverage": 10.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 6.0, "info": { "bracket": "2", - "initialLeverage": "10", - "notionalCap": "25000", + "initialLeverage": "6", + "notionalCap": "100000", "notionalFloor": "5000", - "maintMarginRatio": "0.025", - "cum": "25.0" + "maintMarginRatio": "0.05", + "cum": "125.0" } }, { "tier": 3.0, "currency": "USDT", - "minNotional": 25000.0, - "maxNotional": 100000.0, - "maintenanceMarginRate": 0.05, - "maxLeverage": 8.0, - "info": { - "bracket": "3", - "initialLeverage": "8", - "notionalCap": "100000", - "notionalFloor": "25000", - "maintMarginRatio": "0.05", - "cum": "650.0" - } - }, - { - "tier": 4.0, - "currency": "USDT", "minNotional": 100000.0, "maxNotional": 250000.0, "maintenanceMarginRate": 0.1, "maxLeverage": 5.0, "info": { - "bracket": "4", + "bracket": "3", "initialLeverage": "5", "notionalCap": "250000", "notionalFloor": "100000", "maintMarginRatio": "0.1", - "cum": "5650.0" + "cum": "5125.0" } }, { - "tier": 5.0, + "tier": 4.0, "currency": "USDT", "minNotional": 250000.0, "maxNotional": 1000000.0, "maintenanceMarginRate": 0.125, "maxLeverage": 2.0, "info": { - "bracket": "5", + "bracket": "4", "initialLeverage": "2", "notionalCap": "1000000", "notionalFloor": "250000", "maintMarginRatio": "0.125", - "cum": "11900.0" + "cum": "11375.0" } }, { - "tier": 6.0, + "tier": 5.0, "currency": "USDT", "minNotional": 1000000.0, - "maxNotional": 2000000.0, + "maxNotional": 1200000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { - "bracket": "6", + "bracket": "5", "initialLeverage": "1", - "notionalCap": "2000000", + "notionalCap": "1200000", "notionalFloor": "1000000", "maintMarginRatio": "0.5", - "cum": "386900.0" + "cum": "386375.0" } } ], @@ -5765,6 +5749,120 @@ } } ], + "BNT/USDT:USDT": [ + { + "tier": 1.0, + "currency": "USDT", + "minNotional": 0.0, + "maxNotional": 5000.0, + "maintenanceMarginRate": 0.02, + "maxLeverage": 20.0, + "info": { + "bracket": "1", + "initialLeverage": "20", + "notionalCap": "5000", + "notionalFloor": "0", + "maintMarginRatio": "0.02", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "USDT", + "minNotional": 5000.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 15.0, + "info": { + "bracket": "2", + "initialLeverage": "15", + "notionalCap": "25000", + "notionalFloor": "5000", + "maintMarginRatio": "0.025", + "cum": "25.0" + } + }, + { + "tier": 3.0, + "currency": "USDT", + "minNotional": 25000.0, + "maxNotional": 200000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "3", + "initialLeverage": "10", + "notionalCap": "200000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "650.0" + } + }, + { + "tier": 4.0, + "currency": "USDT", + "minNotional": 200000.0, + "maxNotional": 500000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "4", + "initialLeverage": "5", + "notionalCap": "500000", + "notionalFloor": "200000", + "maintMarginRatio": "0.1", + "cum": "10650.0" + } + }, + { + "tier": 5.0, + "currency": "USDT", + "minNotional": 500000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 4.0, + "info": { + "bracket": "5", + "initialLeverage": "4", + "notionalCap": "1000000", + "notionalFloor": "500000", + "maintMarginRatio": "0.125", + "cum": "23150.0" + } + }, + { + "tier": 6.0, + "currency": "USDT", + "minNotional": 1000000.0, + "maxNotional": 3000000.0, + "maintenanceMarginRate": 0.25, + "maxLeverage": 2.0, + "info": { + "bracket": "6", + "initialLeverage": "2", + "notionalCap": "3000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.25", + "cum": "148150.0" + } + }, + { + "tier": 7.0, + "currency": "USDT", + "minNotional": 3000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "7", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "3000000", + "maintMarginRatio": "0.5", + "cum": "898150.0" + } + } + ], "BNX/USDT:USDT": [ { "tier": 1.0, @@ -7626,13 +7724,13 @@ "tier": 3.0, "currency": "USDT", "minNotional": 50000.0, - "maxNotional": 200000.0, + "maxNotional": 400000.0, "maintenanceMarginRate": 0.05, "maxLeverage": 10.0, "info": { "bracket": "3", "initialLeverage": "10", - "notionalCap": "200000", + "notionalCap": "400000", "notionalFloor": "50000", "maintMarginRatio": "0.05", "cum": "1325.0" @@ -7641,49 +7739,65 @@ { "tier": 4.0, "currency": "USDT", - "minNotional": 200000.0, - "maxNotional": 500000.0, + "minNotional": 400000.0, + "maxNotional": 1000000.0, "maintenanceMarginRate": 0.1, "maxLeverage": 5.0, "info": { "bracket": "4", "initialLeverage": "5", - "notionalCap": "500000", - "notionalFloor": "200000", + "notionalCap": "1000000", + "notionalFloor": "400000", "maintMarginRatio": "0.1", - "cum": "11325.0" + "cum": "21325.0" } }, { "tier": 5.0, "currency": "USDT", - "minNotional": 500000.0, - "maxNotional": 1000000.0, + "minNotional": 1000000.0, + "maxNotional": 1200000.0, "maintenanceMarginRate": 0.125, - "maxLeverage": 2.0, + "maxLeverage": 4.0, "info": { "bracket": "5", - "initialLeverage": "2", - "notionalCap": "1000000", - "notionalFloor": "500000", + "initialLeverage": "4", + "notionalCap": "1200000", + "notionalFloor": "1000000", "maintMarginRatio": "0.125", - "cum": "23825.0" + "cum": "46325.0" } }, { "tier": 6.0, "currency": "USDT", - "minNotional": 1000000.0, + "minNotional": 1200000.0, + "maxNotional": 2000000.0, + "maintenanceMarginRate": 0.25, + "maxLeverage": 2.0, + "info": { + "bracket": "6", + "initialLeverage": "2", + "notionalCap": "2000000", + "notionalFloor": "1200000", + "maintMarginRatio": "0.25", + "cum": "196325.0" + } + }, + { + "tier": 7.0, + "currency": "USDT", + "minNotional": 2000000.0, "maxNotional": 5000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { - "bracket": "6", + "bracket": "7", "initialLeverage": "1", "notionalCap": "5000000", - "notionalFloor": "1000000", + "notionalFloor": "2000000", "maintMarginRatio": "0.5", - "cum": "398825.0" + "cum": "696325.0" } } ], @@ -8900,6 +9014,88 @@ "tier": 1.0, "currency": "BUSD", "minNotional": 0.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 8.0, + "info": { + "bracket": "1", + "initialLeverage": "8", + "notionalCap": "25000", + "notionalFloor": "0", + "maintMarginRatio": "0.025", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "BUSD", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 6.0, + "info": { + "bracket": "2", + "initialLeverage": "6", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "625.0" + } + }, + { + "tier": 3.0, + "currency": "BUSD", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "3", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5625.0" + } + }, + { + "tier": 4.0, + "currency": "BUSD", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "4", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11875.0" + } + }, + { + "tier": 5.0, + "currency": "BUSD", + "minNotional": 1000000.0, + "maxNotional": 1200000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "5", + "initialLeverage": "1", + "notionalCap": "1200000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386875.0" + } + } + ], + "DODOX/USDT:USDT": [ + { + "tier": 1.0, + "currency": "USDT", + "minNotional": 0.0, "maxNotional": 5000.0, "maintenanceMarginRate": 0.02, "maxLeverage": 20.0, @@ -8914,14 +9110,14 @@ }, { "tier": 2.0, - "currency": "BUSD", + "currency": "USDT", "minNotional": 5000.0, "maxNotional": 25000.0, "maintenanceMarginRate": 0.025, - "maxLeverage": 10.0, + "maxLeverage": 15.0, "info": { "bracket": "2", - "initialLeverage": "10", + "initialLeverage": "15", "notionalCap": "25000", "notionalFloor": "5000", "maintMarginRatio": "0.025", @@ -8930,15 +9126,15 @@ }, { "tier": 3.0, - "currency": "BUSD", + "currency": "USDT", "minNotional": 25000.0, - "maxNotional": 100000.0, + "maxNotional": 200000.0, "maintenanceMarginRate": 0.05, - "maxLeverage": 8.0, + "maxLeverage": 10.0, "info": { "bracket": "3", - "initialLeverage": "8", - "notionalCap": "100000", + "initialLeverage": "10", + "notionalCap": "200000", "notionalFloor": "25000", "maintMarginRatio": "0.05", "cum": "650.0" @@ -8946,50 +9142,66 @@ }, { "tier": 4.0, - "currency": "BUSD", - "minNotional": 100000.0, - "maxNotional": 250000.0, + "currency": "USDT", + "minNotional": 200000.0, + "maxNotional": 500000.0, "maintenanceMarginRate": 0.1, "maxLeverage": 5.0, "info": { "bracket": "4", "initialLeverage": "5", - "notionalCap": "250000", - "notionalFloor": "100000", + "notionalCap": "500000", + "notionalFloor": "200000", "maintMarginRatio": "0.1", - "cum": "5650.0" + "cum": "10650.0" } }, { "tier": 5.0, - "currency": "BUSD", - "minNotional": 250000.0, + "currency": "USDT", + "minNotional": 500000.0, "maxNotional": 1000000.0, "maintenanceMarginRate": 0.125, - "maxLeverage": 2.0, + "maxLeverage": 4.0, "info": { "bracket": "5", - "initialLeverage": "2", + "initialLeverage": "4", "notionalCap": "1000000", - "notionalFloor": "250000", + "notionalFloor": "500000", "maintMarginRatio": "0.125", - "cum": "11900.0" + "cum": "23150.0" } }, { "tier": 6.0, - "currency": "BUSD", + "currency": "USDT", "minNotional": 1000000.0, "maxNotional": 3000000.0, + "maintenanceMarginRate": 0.25, + "maxLeverage": 2.0, + "info": { + "bracket": "6", + "initialLeverage": "2", + "notionalCap": "3000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.25", + "cum": "148150.0" + } + }, + { + "tier": 7.0, + "currency": "USDT", + "minNotional": 3000000.0, + "maxNotional": 5000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { - "bracket": "6", + "bracket": "7", "initialLeverage": "1", - "notionalCap": "3000000", - "notionalFloor": "1000000", + "notionalCap": "5000000", + "notionalFloor": "3000000", "maintMarginRatio": "0.5", - "cum": "386900.0" + "cum": "898150.0" } } ], @@ -16652,96 +16864,80 @@ "tier": 1.0, "currency": "USDT", "minNotional": 0.0, - "maxNotional": 5000.0, - "maintenanceMarginRate": 0.01, - "maxLeverage": 20.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 10.0, "info": { "bracket": "1", - "initialLeverage": "20", - "notionalCap": "5000", + "initialLeverage": "10", + "notionalCap": "25000", "notionalFloor": "0", - "maintMarginRatio": "0.01", + "maintMarginRatio": "0.025", "cum": "0.0" } }, { "tier": 2.0, "currency": "USDT", - "minNotional": 5000.0, - "maxNotional": 25000.0, - "maintenanceMarginRate": 0.025, - "maxLeverage": 10.0, - "info": { - "bracket": "2", - "initialLeverage": "10", - "notionalCap": "25000", - "notionalFloor": "5000", - "maintMarginRatio": "0.025", - "cum": "75.0" - } - }, - { - "tier": 3.0, - "currency": "USDT", "minNotional": 25000.0, "maxNotional": 100000.0, "maintenanceMarginRate": 0.05, "maxLeverage": 8.0, "info": { - "bracket": "3", + "bracket": "2", "initialLeverage": "8", "notionalCap": "100000", "notionalFloor": "25000", "maintMarginRatio": "0.05", - "cum": "700.0" + "cum": "625.0" } }, { - "tier": 4.0, + "tier": 3.0, "currency": "USDT", "minNotional": 100000.0, "maxNotional": 250000.0, "maintenanceMarginRate": 0.1, "maxLeverage": 5.0, "info": { - "bracket": "4", + "bracket": "3", "initialLeverage": "5", "notionalCap": "250000", "notionalFloor": "100000", "maintMarginRatio": "0.1", - "cum": "5700.0" + "cum": "5625.0" } }, { - "tier": 5.0, + "tier": 4.0, "currency": "USDT", "minNotional": 250000.0, "maxNotional": 1000000.0, "maintenanceMarginRate": 0.125, "maxLeverage": 2.0, "info": { - "bracket": "5", + "bracket": "4", "initialLeverage": "2", "notionalCap": "1000000", "notionalFloor": "250000", "maintMarginRatio": "0.125", - "cum": "11950.0" + "cum": "11875.0" } }, { - "tier": 6.0, + "tier": 5.0, "currency": "USDT", "minNotional": 1000000.0, - "maxNotional": 5000000.0, + "maxNotional": 3000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { - "bracket": "6", + "bracket": "5", "initialLeverage": "1", - "notionalCap": "5000000", + "notionalCap": "3000000", "notionalFloor": "1000000", "maintMarginRatio": "0.5", - "cum": "386950.0" + "cum": "386875.0" } } ], @@ -17712,10 +17908,10 @@ "minNotional": 0.0, "maxNotional": 5000.0, "maintenanceMarginRate": 0.02, - "maxLeverage": 25.0, + "maxLeverage": 10.0, "info": { "bracket": "1", - "initialLeverage": "25", + "initialLeverage": "10", "notionalCap": "5000", "notionalFloor": "0", "maintMarginRatio": "0.02", @@ -17728,10 +17924,10 @@ "minNotional": 5000.0, "maxNotional": 25000.0, "maintenanceMarginRate": 0.025, - "maxLeverage": 15.0, + "maxLeverage": 8.0, "info": { "bracket": "2", - "initialLeverage": "15", + "initialLeverage": "8", "notionalCap": "25000", "notionalFloor": "5000", "maintMarginRatio": "0.025", @@ -17744,10 +17940,10 @@ "minNotional": 25000.0, "maxNotional": 100000.0, "maintenanceMarginRate": 0.05, - "maxLeverage": 10.0, + "maxLeverage": 6.0, "info": { "bracket": "3", - "initialLeverage": "10", + "initialLeverage": "6", "notionalCap": "100000", "notionalFloor": "25000", "maintMarginRatio": "0.05", @@ -17806,13 +18002,13 @@ "tier": 7.0, "currency": "BUSD", "minNotional": 3000000.0, - "maxNotional": 8000000.0, + "maxNotional": 3500000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { "bracket": "7", "initialLeverage": "1", - "notionalCap": "8000000", + "notionalCap": "3500000", "notionalFloor": "3000000", "maintMarginRatio": "0.5", "cum": "949400.0" @@ -19757,6 +19953,120 @@ } } ], + "OXT/USDT:USDT": [ + { + "tier": 1.0, + "currency": "USDT", + "minNotional": 0.0, + "maxNotional": 5000.0, + "maintenanceMarginRate": 0.02, + "maxLeverage": 20.0, + "info": { + "bracket": "1", + "initialLeverage": "20", + "notionalCap": "5000", + "notionalFloor": "0", + "maintMarginRatio": "0.02", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "USDT", + "minNotional": 5000.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 15.0, + "info": { + "bracket": "2", + "initialLeverage": "15", + "notionalCap": "25000", + "notionalFloor": "5000", + "maintMarginRatio": "0.025", + "cum": "25.0" + } + }, + { + "tier": 3.0, + "currency": "USDT", + "minNotional": 25000.0, + "maxNotional": 200000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "3", + "initialLeverage": "10", + "notionalCap": "200000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "650.0" + } + }, + { + "tier": 4.0, + "currency": "USDT", + "minNotional": 200000.0, + "maxNotional": 500000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "4", + "initialLeverage": "5", + "notionalCap": "500000", + "notionalFloor": "200000", + "maintMarginRatio": "0.1", + "cum": "10650.0" + } + }, + { + "tier": 5.0, + "currency": "USDT", + "minNotional": 500000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 4.0, + "info": { + "bracket": "5", + "initialLeverage": "4", + "notionalCap": "1000000", + "notionalFloor": "500000", + "maintMarginRatio": "0.125", + "cum": "23150.0" + } + }, + { + "tier": 6.0, + "currency": "USDT", + "minNotional": 1000000.0, + "maxNotional": 3000000.0, + "maintenanceMarginRate": 0.25, + "maxLeverage": 2.0, + "info": { + "bracket": "6", + "initialLeverage": "2", + "notionalCap": "3000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.25", + "cum": "148150.0" + } + }, + { + "tier": 7.0, + "currency": "USDT", + "minNotional": 3000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "7", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "3000000", + "maintMarginRatio": "0.5", + "cum": "898150.0" + } + } + ], "PENDLE/USDT:USDT": [ { "tier": 1.0, @@ -24442,10 +24752,10 @@ "minNotional": 0.0, "maxNotional": 5000.0, "maintenanceMarginRate": 0.02, - "maxLeverage": 25.0, + "maxLeverage": 10.0, "info": { "bracket": "1", - "initialLeverage": "25", + "initialLeverage": "10", "notionalCap": "5000", "notionalFloor": "0", "maintMarginRatio": "0.02", @@ -24458,10 +24768,10 @@ "minNotional": 5000.0, "maxNotional": 25000.0, "maintenanceMarginRate": 0.025, - "maxLeverage": 15.0, + "maxLeverage": 8.0, "info": { "bracket": "2", - "initialLeverage": "15", + "initialLeverage": "8", "notionalCap": "25000", "notionalFloor": "5000", "maintMarginRatio": "0.025", @@ -24474,10 +24784,10 @@ "minNotional": 25000.0, "maxNotional": 100000.0, "maintenanceMarginRate": 0.05, - "maxLeverage": 10.0, + "maxLeverage": 6.0, "info": { "bracket": "3", - "initialLeverage": "10", + "initialLeverage": "6", "notionalCap": "100000", "notionalFloor": "25000", "maintMarginRatio": "0.05", @@ -24536,13 +24846,13 @@ "tier": 7.0, "currency": "BUSD", "minNotional": 3000000.0, - "maxNotional": 8000000.0, + "maxNotional": 4000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { "bracket": "7", "initialLeverage": "1", - "notionalCap": "8000000", + "notionalCap": "4000000", "notionalFloor": "3000000", "maintMarginRatio": "0.5", "cum": "949400.0" @@ -25613,14 +25923,14 @@ "currency": "USDT", "minNotional": 0.0, "maxNotional": 5000.0, - "maintenanceMarginRate": 0.02, - "maxLeverage": 20.0, + "maintenanceMarginRate": 0.01, + "maxLeverage": 50.0, "info": { "bracket": "1", - "initialLeverage": "20", + "initialLeverage": "50", "notionalCap": "5000", "notionalFloor": "0", - "maintMarginRatio": "0.02", + "maintMarginRatio": "0.01", "cum": "0.0" } }, @@ -25629,95 +25939,111 @@ "currency": "USDT", "minNotional": 5000.0, "maxNotional": 25000.0, - "maintenanceMarginRate": 0.025, - "maxLeverage": 15.0, + "maintenanceMarginRate": 0.02, + "maxLeverage": 25.0, "info": { "bracket": "2", - "initialLeverage": "15", + "initialLeverage": "25", "notionalCap": "25000", "notionalFloor": "5000", - "maintMarginRatio": "0.025", - "cum": "25.0" + "maintMarginRatio": "0.02", + "cum": "50.0" } }, { "tier": 3.0, "currency": "USDT", "minNotional": 25000.0, - "maxNotional": 200000.0, - "maintenanceMarginRate": 0.05, - "maxLeverage": 10.0, + "maxNotional": 50000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, "info": { "bracket": "3", - "initialLeverage": "10", - "notionalCap": "200000", + "initialLeverage": "20", + "notionalCap": "50000", "notionalFloor": "25000", - "maintMarginRatio": "0.05", - "cum": "650.0" + "maintMarginRatio": "0.025", + "cum": "175.0" } }, { "tier": 4.0, "currency": "USDT", - "minNotional": 200000.0, - "maxNotional": 500000.0, - "maintenanceMarginRate": 0.1, - "maxLeverage": 5.0, + "minNotional": 50000.0, + "maxNotional": 400000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, "info": { "bracket": "4", - "initialLeverage": "5", - "notionalCap": "500000", - "notionalFloor": "200000", - "maintMarginRatio": "0.1", - "cum": "10650.0" + "initialLeverage": "10", + "notionalCap": "400000", + "notionalFloor": "50000", + "maintMarginRatio": "0.05", + "cum": "1425.0" } }, { "tier": 5.0, "currency": "USDT", - "minNotional": 500000.0, + "minNotional": 400000.0, "maxNotional": 1000000.0, - "maintenanceMarginRate": 0.125, - "maxLeverage": 4.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, "info": { "bracket": "5", - "initialLeverage": "4", + "initialLeverage": "5", "notionalCap": "1000000", - "notionalFloor": "500000", - "maintMarginRatio": "0.125", - "cum": "23150.0" + "notionalFloor": "400000", + "maintMarginRatio": "0.1", + "cum": "21425.0" } }, { "tier": 6.0, "currency": "USDT", "minNotional": 1000000.0, - "maxNotional": 3000000.0, - "maintenanceMarginRate": 0.25, - "maxLeverage": 2.0, + "maxNotional": 2000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 4.0, "info": { "bracket": "6", - "initialLeverage": "2", - "notionalCap": "3000000", + "initialLeverage": "4", + "notionalCap": "2000000", "notionalFloor": "1000000", - "maintMarginRatio": "0.25", - "cum": "148150.0" + "maintMarginRatio": "0.125", + "cum": "46425.0" } }, { "tier": 7.0, "currency": "USDT", - "minNotional": 3000000.0, + "minNotional": 2000000.0, "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.25, + "maxLeverage": 2.0, + "info": { + "bracket": "7", + "initialLeverage": "2", + "notionalCap": "5000000", + "notionalFloor": "2000000", + "maintMarginRatio": "0.25", + "cum": "296425.0" + } + }, + { + "tier": 8.0, + "currency": "USDT", + "minNotional": 5000000.0, + "maxNotional": 10000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { - "bracket": "7", + "bracket": "8", "initialLeverage": "1", - "notionalCap": "5000000", - "notionalFloor": "3000000", + "notionalCap": "10000000", + "notionalFloor": "5000000", "maintMarginRatio": "0.5", - "cum": "898150.0" + "cum": "1546425.0" } } ], @@ -26861,6 +27187,120 @@ } } ], + "YGG/USDT:USDT": [ + { + "tier": 1.0, + "currency": "USDT", + "minNotional": 0.0, + "maxNotional": 5000.0, + "maintenanceMarginRate": 0.02, + "maxLeverage": 20.0, + "info": { + "bracket": "1", + "initialLeverage": "20", + "notionalCap": "5000", + "notionalFloor": "0", + "maintMarginRatio": "0.02", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "USDT", + "minNotional": 5000.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 15.0, + "info": { + "bracket": "2", + "initialLeverage": "15", + "notionalCap": "25000", + "notionalFloor": "5000", + "maintMarginRatio": "0.025", + "cum": "25.0" + } + }, + { + "tier": 3.0, + "currency": "USDT", + "minNotional": 25000.0, + "maxNotional": 200000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "3", + "initialLeverage": "10", + "notionalCap": "200000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "650.0" + } + }, + { + "tier": 4.0, + "currency": "USDT", + "minNotional": 200000.0, + "maxNotional": 500000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "4", + "initialLeverage": "5", + "notionalCap": "500000", + "notionalFloor": "200000", + "maintMarginRatio": "0.1", + "cum": "10650.0" + } + }, + { + "tier": 5.0, + "currency": "USDT", + "minNotional": 500000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 4.0, + "info": { + "bracket": "5", + "initialLeverage": "4", + "notionalCap": "1000000", + "notionalFloor": "500000", + "maintMarginRatio": "0.125", + "cum": "23150.0" + } + }, + { + "tier": 6.0, + "currency": "USDT", + "minNotional": 1000000.0, + "maxNotional": 3000000.0, + "maintenanceMarginRate": 0.25, + "maxLeverage": 2.0, + "info": { + "bracket": "6", + "initialLeverage": "2", + "notionalCap": "3000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.25", + "cum": "148150.0" + } + }, + { + "tier": 7.0, + "currency": "USDT", + "minNotional": 3000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "7", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "3000000", + "maintMarginRatio": "0.5", + "cum": "898150.0" + } + } + ], "ZEC/USDT:USDT": [ { "tier": 1.0, diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 5f2530431..9e168dcb7 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -5,6 +5,7 @@ Cryptocurrency Exchanges support import asyncio import inspect import logging +import signal from copy import deepcopy from datetime import datetime, timedelta, timezone from math import floor @@ -263,8 +264,6 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(f"Initialization of ccxt failed. Reason: {e}") from e - self.set_sandbox(api, exchange_config, name) - return api @property @@ -465,16 +464,6 @@ class Exchange: return amount_to_contract_precision(amount, self.get_precision_amount(pair), self.precisionMode, contract_size) - def set_sandbox(self, api: ccxt.Exchange, exchange_config: dict, name: str) -> None: - if exchange_config.get('sandbox'): - if api.urls.get('test'): - api.urls['api'] = api.urls['test'] - logger.info("Enabled Sandbox API on %s", name) - else: - logger.warning( - f"No Sandbox URL in CCXT for {name}, exiting. Please check your config.json") - raise OperationalException(f'Exchange {name} does not provide a sandbox api') - def _load_async_markets(self, reload: bool = False) -> None: try: if self._api_async: @@ -580,7 +569,7 @@ class Exchange: for pair in [f"{curr_1}/{curr_2}", f"{curr_2}/{curr_1}"]: if pair in self.markets and self.markets[pair].get('active'): return pair - raise ExchangeError(f"Could not combine {curr_1} and {curr_2} to get a valid pair.") + raise ValueError(f"Could not combine {curr_1} and {curr_2} to get a valid pair.") def validate_timeframes(self, timeframe: Optional[str]) -> None: """ @@ -1876,7 +1865,7 @@ class Exchange: tick = self.fetch_ticker(comb) fee_to_quote_rate = safe_value_fallback2(tick, tick, 'last', 'ask') - except ExchangeError: + except (ValueError, ExchangeError): fee_to_quote_rate = self._config['exchange'].get('unknown_fee_rate', None) if not fee_to_quote_rate: return None @@ -2265,20 +2254,24 @@ class Exchange: from_id = t[-1][1] trades.extend(t[:-1]) while True: - t = await self._async_fetch_trades(pair, - params={self._trades_pagination_arg: from_id}) - if t: - # Skip last id since its the key for the next call - trades.extend(t[:-1]) - if from_id == t[-1][1] or t[-1][0] > until: - logger.debug(f"Stopping because from_id did not change. " - f"Reached {t[-1][0]} > {until}") - # Reached the end of the defined-download period - add last trade as well. - trades.extend(t[-1:]) - break + try: + t = await self._async_fetch_trades(pair, + params={self._trades_pagination_arg: from_id}) + if t: + # Skip last id since its the key for the next call + trades.extend(t[:-1]) + if from_id == t[-1][1] or t[-1][0] > until: + logger.debug(f"Stopping because from_id did not change. " + f"Reached {t[-1][0]} > {until}") + # Reached the end of the defined-download period - add last trade as well. + trades.extend(t[-1:]) + break - from_id = t[-1][1] - else: + from_id = t[-1][1] + else: + break + except asyncio.CancelledError: + logger.debug("Async operation Interrupted, breaking trades DL loop.") break return (pair, trades) @@ -2298,16 +2291,20 @@ class Exchange: # DEFAULT_TRADES_COLUMNS: 0 -> timestamp # DEFAULT_TRADES_COLUMNS: 1 -> id while True: - t = await self._async_fetch_trades(pair, since=since) - if t: - since = t[-1][0] - trades.extend(t) - # Reached the end of the defined-download period - if until and t[-1][0] > until: - logger.debug( - f"Stopping because until was reached. {t[-1][0]} > {until}") + try: + t = await self._async_fetch_trades(pair, since=since) + if t: + since = t[-1][0] + trades.extend(t) + # Reached the end of the defined-download period + if until and t[-1][0] > until: + logger.debug( + f"Stopping because until was reached. {t[-1][0]} > {until}") + break + else: break - else: + except asyncio.CancelledError: + logger.debug("Async operation Interrupted, breaking trades DL loop.") break return (pair, trades) @@ -2356,9 +2353,16 @@ class Exchange: raise OperationalException("This exchange does not support downloading Trades.") with self._loop_lock: - return self.loop.run_until_complete( - self._async_get_trade_history(pair=pair, since=since, - until=until, from_id=from_id)) + task = asyncio.ensure_future(self._async_get_trade_history( + pair=pair, since=since, until=until, from_id=from_id)) + + for sig in [signal.SIGINT, signal.SIGTERM]: + try: + self.loop.add_signal_handler(sig, task.cancel) + except NotImplementedError: + # Not all platforms implement signals (e.g. windows) + pass + return self.loop.run_until_complete(task) @retrier def _get_funding_fees_from_exchange(self, pair: str, since: Union[datetime, int]) -> float: diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index b9854919d..b6ded83b1 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -375,7 +375,7 @@ class FreqaiDataDrawer: num_keep = self.freqai_info["purge_old_models"] if not num_keep: return - elif type(num_keep) == bool: + elif isinstance(num_keep, bool): num_keep = 2 model_folders = [x for x in self.full_path.iterdir() if x.is_dir()] diff --git a/freqtrade/freqai/prediction_models/PyTorchMLPClassifier.py b/freqtrade/freqai/prediction_models/PyTorchMLPClassifier.py index 71279dba9..9aabdf7ad 100644 --- a/freqtrade/freqai/prediction_models/PyTorchMLPClassifier.py +++ b/freqtrade/freqai/prediction_models/PyTorchMLPClassifier.py @@ -26,9 +26,9 @@ class PyTorchMLPClassifier(BasePyTorchClassifier): "model_training_parameters" : { "learning_rate": 3e-4, "trainer_kwargs": { - "max_iters": 5000, + "n_steps": 5000, "batch_size": 64, - "max_n_eval_batches": null, + "n_epochs": null, }, "model_kwargs": { "hidden_dim": 512, diff --git a/freqtrade/freqai/prediction_models/PyTorchMLPRegressor.py b/freqtrade/freqai/prediction_models/PyTorchMLPRegressor.py index 9f4534487..dc8dc4b61 100644 --- a/freqtrade/freqai/prediction_models/PyTorchMLPRegressor.py +++ b/freqtrade/freqai/prediction_models/PyTorchMLPRegressor.py @@ -27,9 +27,9 @@ class PyTorchMLPRegressor(BasePyTorchRegressor): "model_training_parameters" : { "learning_rate": 3e-4, "trainer_kwargs": { - "max_iters": 5000, + "n_steps": 5000, "batch_size": 64, - "max_n_eval_batches": null, + "n_epochs": null, }, "model_kwargs": { "hidden_dim": 512, diff --git a/freqtrade/freqai/prediction_models/PyTorchTransformerRegressor.py b/freqtrade/freqai/prediction_models/PyTorchTransformerRegressor.py index a76bab05c..846d6df2e 100644 --- a/freqtrade/freqai/prediction_models/PyTorchTransformerRegressor.py +++ b/freqtrade/freqai/prediction_models/PyTorchTransformerRegressor.py @@ -30,9 +30,9 @@ class PyTorchTransformerRegressor(BasePyTorchRegressor): "model_training_parameters" : { "learning_rate": 3e-4, "trainer_kwargs": { - "max_iters": 5000, + "n_steps": 5000, "batch_size": 64, - "max_n_eval_batches": null + "n_epochs": null }, "model_kwargs": { "hidden_dim": 512, diff --git a/freqtrade/freqai/torch/PyTorchDataConvertor.py b/freqtrade/freqai/torch/PyTorchDataConvertor.py index e6b815373..0af14dd14 100644 --- a/freqtrade/freqai/torch/PyTorchDataConvertor.py +++ b/freqtrade/freqai/torch/PyTorchDataConvertor.py @@ -1,5 +1,4 @@ from abc import ABC, abstractmethod -from typing import Optional import pandas as pd import torch @@ -12,14 +11,14 @@ class PyTorchDataConvertor(ABC): """ @abstractmethod - def convert_x(self, df: pd.DataFrame, device: Optional[str] = None) -> torch.Tensor: + def convert_x(self, df: pd.DataFrame, device: str) -> torch.Tensor: """ :param df: "*_features" dataframe. :param device: The device to use for training (e.g. 'cpu', 'cuda'). """ @abstractmethod - def convert_y(self, df: pd.DataFrame, device: Optional[str] = None) -> torch.Tensor: + def convert_y(self, df: pd.DataFrame, device: str) -> torch.Tensor: """ :param df: "*_labels" dataframe. :param device: The device to use for training (e.g. 'cpu', 'cuda'). @@ -33,8 +32,8 @@ class DefaultPyTorchDataConvertor(PyTorchDataConvertor): def __init__( self, - target_tensor_type: Optional[torch.dtype] = None, - squeeze_target_tensor: bool = False + target_tensor_type: torch.dtype = torch.float32, + squeeze_target_tensor: bool = False, ): """ :param target_tensor_type: type of target tensor, for classification use @@ -45,23 +44,14 @@ class DefaultPyTorchDataConvertor(PyTorchDataConvertor): self._target_tensor_type = target_tensor_type self._squeeze_target_tensor = squeeze_target_tensor - def convert_x(self, df: pd.DataFrame, device: Optional[str] = None) -> torch.Tensor: - x = torch.from_numpy(df.values).float() - if device: - x = x.to(device) - + def convert_x(self, df: pd.DataFrame, device: str) -> torch.Tensor: + numpy_arrays = df.values + x = torch.tensor(numpy_arrays, device=device, dtype=torch.float32) return x - def convert_y(self, df: pd.DataFrame, device: Optional[str] = None) -> torch.Tensor: - y = torch.from_numpy(df.values) - - if self._target_tensor_type: - y = y.to(self._target_tensor_type) - + def convert_y(self, df: pd.DataFrame, device: str) -> torch.Tensor: + numpy_arrays = df.values + y = torch.tensor(numpy_arrays, device=device, dtype=self._target_tensor_type) if self._squeeze_target_tensor: y = y.squeeze() - - if device: - y = y.to(device) - return y diff --git a/freqtrade/freqai/torch/PyTorchModelTrainer.py b/freqtrade/freqai/torch/PyTorchModelTrainer.py index 603e7ac12..371a953e7 100644 --- a/freqtrade/freqai/torch/PyTorchModelTrainer.py +++ b/freqtrade/freqai/torch/PyTorchModelTrainer.py @@ -1,5 +1,4 @@ import logging -import math from pathlib import Path from typing import Any, Dict, List, Optional @@ -40,23 +39,27 @@ class PyTorchModelTrainer(PyTorchTrainerInterface): state_dict and model_meta_data saved by self.save() method. :param model_meta_data: Additional metadata about the model (optional). :param data_convertor: convertor from pd.DataFrame to torch.tensor. - :param max_iters: The number of training iterations to run. - iteration here refers to the number of times we call - self.optimizer.step(). used to calculate n_epochs. + :param n_steps: used to calculate n_epochs. The number of training iterations to run. + iteration here refers to the number of times optimizer.step() is called. + ignored if n_epochs is set. + :param n_epochs: The maximum number batches to use for evaluation. :param batch_size: The size of the batches to use during training. - :param max_n_eval_batches: The maximum number batches to use for evaluation. """ self.model = model self.optimizer = optimizer self.criterion = criterion self.model_meta_data = model_meta_data self.device = device - self.max_iters: int = kwargs.get("max_iters", 100) + self.n_epochs: Optional[int] = kwargs.get("n_epochs", 10) + self.n_steps: Optional[int] = kwargs.get("n_steps", None) + if self.n_steps is None and not self.n_epochs: + raise Exception("Either `n_steps` or `n_epochs` should be set.") + self.batch_size: int = kwargs.get("batch_size", 64) - self.max_n_eval_batches: Optional[int] = kwargs.get("max_n_eval_batches", None) self.data_convertor = data_convertor self.window_size: int = window_size self.tb_logger = tb_logger + self.test_batch_counter = 0 def fit(self, data_dictionary: Dict[str, pd.DataFrame], splits: List[str]): """ @@ -72,55 +75,46 @@ class PyTorchModelTrainer(PyTorchTrainerInterface): backpropagation. - Updates the model's parameters using an optimizer. """ - data_loaders_dictionary = self.create_data_loaders_dictionary(data_dictionary, splits) - epochs = self.calc_n_epochs( - n_obs=len(data_dictionary["train_features"]), - batch_size=self.batch_size, - n_iters=self.max_iters - ) self.model.train() - for epoch in range(1, epochs + 1): - for i, batch_data in enumerate(data_loaders_dictionary["train"]): + data_loaders_dictionary = self.create_data_loaders_dictionary(data_dictionary, splits) + n_obs = len(data_dictionary["train_features"]) + n_epochs = self.n_epochs or self.calc_n_epochs(n_obs=n_obs) + batch_counter = 0 + for _ in range(n_epochs): + for _, batch_data in enumerate(data_loaders_dictionary["train"]): xb, yb = batch_data - xb.to(self.device) - yb.to(self.device) + xb = xb.to(self.device) + yb = yb.to(self.device) yb_pred = self.model(xb) loss = self.criterion(yb_pred, yb) self.optimizer.zero_grad(set_to_none=True) loss.backward() self.optimizer.step() - self.tb_logger.log_scalar("train_loss", loss.item(), i) + self.tb_logger.log_scalar("train_loss", loss.item(), batch_counter) + batch_counter += 1 # evaluation if "test" in splits: - self.estimate_loss( - data_loaders_dictionary, - self.max_n_eval_batches, - "test" - ) + self.estimate_loss(data_loaders_dictionary, "test") @torch.no_grad() def estimate_loss( self, data_loader_dictionary: Dict[str, DataLoader], - max_n_eval_batches: Optional[int], split: str, ) -> None: self.model.eval() - n_batches = 0 - for i, batch_data in enumerate(data_loader_dictionary[split]): - if max_n_eval_batches and i > max_n_eval_batches: - n_batches += 1 - break + for _, batch_data in enumerate(data_loader_dictionary[split]): xb, yb = batch_data - xb.to(self.device) - yb.to(self.device) + xb = xb.to(self.device) + yb = yb.to(self.device) yb_pred = self.model(xb) loss = self.criterion(yb_pred, yb) - self.tb_logger.log_scalar(f"{split}_loss", loss.item(), i) + self.tb_logger.log_scalar(f"{split}_loss", loss.item(), self.test_batch_counter) + self.test_batch_counter += 1 self.model.train() @@ -148,31 +142,30 @@ class PyTorchModelTrainer(PyTorchTrainerInterface): return data_loader_dictionary - @staticmethod - def calc_n_epochs(n_obs: int, batch_size: int, n_iters: int) -> int: + def calc_n_epochs(self, n_obs: int) -> int: """ Calculates the number of epochs required to reach the maximum number of iterations specified in the model training parameters. - the motivation here is that `max_iters` is easier to optimize and keep stable, + the motivation here is that `n_steps` is easier to optimize and keep stable, across different n_obs - the number of data points. """ + assert isinstance(self.n_steps, int), "Either `n_steps` or `n_epochs` should be set." + n_batches = n_obs // self.batch_size + n_epochs = min(self.n_steps // n_batches, 1) + if n_epochs <= 10: + logger.warning( + f"Setting low n_epochs: {n_epochs}. " + f"Please consider increasing `n_steps` hyper-parameter." + ) - n_batches = math.ceil(n_obs // batch_size) - epochs = math.ceil(n_iters // n_batches) - if epochs <= 10: - logger.warning("User set `max_iters` in such a way that the trainer will only perform " - f" {epochs} epochs. Please consider increasing this value accordingly") - if epochs <= 1: - logger.warning("Epochs set to 1. Please review your `max_iters` value") - epochs = 1 - return epochs + return n_epochs def save(self, path: Path): """ - Saving any nn.Module state_dict - Saving model_meta_data, this dict should contain any additional data that the - user needs to store. e.g class_names for classification models. + user needs to store. e.g. class_names for classification models. """ torch.save({ diff --git a/freqtrade/misc.py b/freqtrade/misc.py index e715c280a..f8d730fae 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -192,30 +192,6 @@ def plural(num: float, singular: str, plural: Optional[str] = None) -> str: return singular if (num == 1 or num == -1) else plural or singular + 's' -def render_template(templatefile: str, arguments: dict = {}) -> str: - - from jinja2 import Environment, PackageLoader, select_autoescape - - env = Environment( - loader=PackageLoader('freqtrade', 'templates'), - autoescape=select_autoescape(['html', 'xml']) - ) - template = env.get_template(templatefile) - return template.render(**arguments) - - -def render_template_with_fallback(templatefile: str, templatefallbackfile: str, - arguments: dict = {}) -> str: - """ - Use templatefile if possible, otherwise fall back to templatefallbackfile - """ - from jinja2.exceptions import TemplateNotFound - try: - return render_template(templatefile, arguments) - except TemplateNotFound: - return render_template(templatefallbackfile, arguments) - - def chunks(lst: List[Any], n: int) -> Iterator[List[Any]]: """ Split lst into chunks of the size n. diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ce20d2178..21390489e 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -369,13 +369,14 @@ class Backtesting: # Cleanup from prior runs pair_data.drop(HEADERS[5:] + ['buy', 'sell'], axis=1, errors='ignore') df_analyzed = self.strategy.ft_advise_signals(pair_data, {'pair': pair}) - # Trim startup period from analyzed dataframe - df_analyzed = processed[pair] = pair_data = trim_dataframe( - df_analyzed, self.timerange, startup_candles=self.required_startup) # Update dataprovider cache self.dataprovider._set_cached_df( pair, self.timeframe, df_analyzed, self.config['candle_type_def']) + # Trim startup period from analyzed dataframe + df_analyzed = processed[pair] = pair_data = trim_dataframe( + df_analyzed, self.timerange, startup_candles=self.required_startup) + # Create a copy of the dataframe before shifting, that way the entry signal/tag # remains on the correct candle for callbacks. df_analyzed = df_analyzed.copy() @@ -567,8 +568,7 @@ class Backtesting: pos_trade = self._get_exit_for_signal(trade, row, exit_, amount) if pos_trade is not None: order = pos_trade.orders[-1] - if self._get_order_filled(order.ft_price, row): - order.close_bt_order(current_date, trade) + if self._try_close_open_order(order, trade, current_date, row): trade.recalc_trade_from_orders() self.wallets.update() return pos_trade @@ -579,6 +579,19 @@ class Backtesting: """ Rate is within candle, therefore filled""" return row[LOW_IDX] <= rate <= row[HIGH_IDX] + def _try_close_open_order( + self, order: Optional[Order], trade: LocalTrade, current_date: datetime, + row: Tuple) -> bool: + """ + Check if an order is open and if it should've filled. + :return: True if the order filled. + """ + if order and self._get_order_filled(order.ft_price, row): + order.close_bt_order(current_date, trade) + trade.open_order_id = None + return True + return False + def _get_exit_for_signal( self, trade: LocalTrade, row: Tuple, exit_: ExitCheckTuple, amount: Optional[float] = None) -> Optional[LocalTrade]: @@ -903,9 +916,7 @@ class Backtesting: ) order._trade_bt = trade trade.orders.append(order) - if pos_adjust and self._get_order_filled(order.ft_price, row): - order.close_bt_order(current_time, trade) - else: + if not self._try_close_open_order(order, trade, current_time, row): trade.open_order_id = str(self.order_id_counter) trade.recalc_trade_from_orders() @@ -1121,23 +1132,18 @@ class Backtesting: for trade in list(LocalTrade.bt_trades_open_pp[pair]): # 3. Process entry orders. order = trade.select_order(trade.entry_side, is_open=True) - if order and self._get_order_filled(order.ft_price, row): - order.close_bt_order(current_time, trade) - trade.open_order_id = None + if self._try_close_open_order(order, trade, current_time, row): self.wallets.update() - # 4. Create exit orders (if any) + # 4. Create exit orders (if any) if not trade.open_order_id: self._check_trade_exit(trade, row) # Place exit order if necessary - # 5. Process exit orders. + # 5. Process exit orders. order = trade.select_order(trade.exit_side, is_open=True) - if order and self._get_order_filled(order.ft_price, row): - order.close_bt_order(current_time, trade) - trade.open_order_id = None + if order and self._try_close_open_order(order, trade, current_time, row): sub_trade = order.safe_amount_after_fee != trade.amount if sub_trade: - order.close_bt_order(current_time, trade) trade.recalc_trade_from_orders() else: trade.close_date = current_time @@ -1191,7 +1197,8 @@ class Backtesting: row_index += 1 indexes[pair] = row_index - self.dataprovider._set_dataframe_max_index(row_index) + self.dataprovider._set_dataframe_max_index(self.required_startup + row_index) + self.dataprovider._set_dataframe_max_date(current_time) current_detail_time: datetime = row[DATE_IDX].to_pydatetime() trade_dir: Optional[LongShort] = self.check_for_trade_entry(row) @@ -1224,12 +1231,14 @@ class Backtesting: is_first = True current_time_det = current_time for det_row in detail_data[HEADERS].values.tolist(): + self.dataprovider._set_dataframe_max_date(current_time_det) open_trade_count_start = self.backtest_loop( det_row, pair, current_time_det, end_date, open_trade_count_start, trade_dir, is_first) current_time_det += timedelta(minutes=self.timeframe_detail_min) is_first = False else: + self.dataprovider._set_dataframe_max_date(current_time) open_trade_count_start = self.backtest_loop( row, pair, current_time, end_date, open_trade_count_start, trade_dir) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index dcc1088b3..80418da95 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -48,6 +48,7 @@ class LookaheadAnalysis: self.entry_varHolders: List[VarHolder] = [] self.exit_varHolders: List[VarHolder] = [] self.exchange: Optional[Any] = None + self._fee = None # pull variables the scope of the lookahead_analysis-instance self.local_config = deepcopy(config) @@ -145,8 +146,13 @@ class LookaheadAnalysis: str(self.dt_to_timestamp(varholder.to_dt))) prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load + if self._fee is not None: + # Don't re-calculate fee per pair, as fee might differ per pair. + prepare_data_config['fee'] = self._fee + backtesting = Backtesting(prepare_data_config, self.exchange) self.exchange = backtesting.exchange + self._fee = backtesting.fee backtesting._set_strategy(backtesting.strategylist[0]) varholder.data, varholder.timerange = backtesting.load_bt_data() @@ -198,7 +204,7 @@ class LookaheadAnalysis: self.prepare_data(exit_varHolder, [result_row['pair']]) # now we analyze a full trade of full_varholder and look for analyze its bias - def analyze_row(self, idx, result_row): + def analyze_row(self, idx: int, result_row): # if force-sold, ignore this signal since here it will unconditionally exit. if result_row.close_date == self.dt_to_timestamp(self.full_varHolder.to_dt): return @@ -209,12 +215,16 @@ class LookaheadAnalysis: # fill entry_varHolder and exit_varHolder self.fill_entry_and_exit_varHolders(result_row) + # this will trigger a logger-message + buy_or_sell_biased: bool = False + # register if buy signal is broken if not self.report_signal( self.entry_varHolders[idx].result, "open_date", self.entry_varHolders[idx].compared_dt): self.current_analysis.false_entry_signals += 1 + buy_or_sell_biased = True # register if buy or sell signal is broken if not self.report_signal( @@ -222,6 +232,13 @@ class LookaheadAnalysis: "close_date", self.exit_varHolders[idx].compared_dt): self.current_analysis.false_exit_signals += 1 + buy_or_sell_biased = True + + if buy_or_sell_biased: + logger.info(f"found lookahead-bias in trade " + f"pair: {result_row['pair']}, " + f"timerange:{result_row['open_date']} - {result_row['close_date']}, " + f"idx: {idx}") # check if the indicators themselves contain biased data self.analyze_indicators(self.full_varHolder, self.entry_varHolders[idx], result_row['pair']) @@ -251,9 +268,33 @@ class LookaheadAnalysis: # starting from the same datetime to avoid miss-reports of bias for idx, result_row in self.full_varHolder.result['results'].iterrows(): if self.current_analysis.total_signals == self.targeted_trade_amount: + logger.info(f"Found targeted trade amount = {self.targeted_trade_amount} signals.") break + if found_signals < self.minimum_trade_amount: + logger.info(f"only found {found_signals} " + f"which is smaller than " + f"minimum trade amount = {self.minimum_trade_amount}. " + f"Exiting this lookahead-analysis") + return None + if "force_exit" in result_row['exit_reason']: + logger.info("found force-exit in pair: {result_row['pair']}, " + f"timerange:{result_row['open_date']}-{result_row['close_date']}, " + f"idx: {idx}, skipping this one to avoid a false-positive.") + + # just to keep the IDs of both full, entry and exit varholders the same + # to achieve a better debugging experience + self.entry_varHolders.append(VarHolder()) + self.exit_varHolders.append(VarHolder()) + continue + self.analyze_row(idx, result_row) + if len(self.entry_varHolders) < self.minimum_trade_amount: + logger.info(f"only found {found_signals} after skipping forced exits " + f"which is smaller than " + f"minimum trade amount = {self.minimum_trade_amount}. " + f"Exiting this lookahead-analysis") + # Restore verbosity, so it's not too quiet for the next strategy restore_verbosity_for_bias_tester() # check and report signals diff --git a/freqtrade/optimize/lookahead_analysis_helpers.py b/freqtrade/optimize/lookahead_analysis_helpers.py index 702eee774..422026780 100644 --- a/freqtrade/optimize/lookahead_analysis_helpers.py +++ b/freqtrade/optimize/lookahead_analysis_helpers.py @@ -137,6 +137,19 @@ class LookaheadAnalysisSubFunctions: 'just to avoid false positives') config['dry_run_wallet'] = min_dry_run_wallet + if 'timerange' not in config: + # setting a timerange is enforced here + raise OperationalException( + "Please set a timerange. " + "Usually a few months are enough depending on your needs and strategy." + ) + # fix stake_amount to 10k. + # in a combination with a wallet size of 1 billion it should always be able to trade + # no matter if they use custom_stake_amount as a small percentage of wallet size + # or fixate custom_stake_amount to a certain value. + logger.info('fixing stake_amount to 10k') + config['stake_amount'] = 10000 + # enforce cache to be 'none', shift it to 'none' if not already # (since the default value is 'day') if config.get('backtest_cache') is None: diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 5d772febe..0c83b3d80 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -648,11 +648,9 @@ class LocalTrade: """ Method used internally to set self.stop_loss. """ - stop_loss_norm = price_to_precision(stop_loss, self.price_precision, self.precision_mode, - rounding_mode=ROUND_DOWN if self.is_short else ROUND_UP) if not self.stop_loss: - self.initial_stop_loss = stop_loss_norm - self.stop_loss = stop_loss_norm + self.initial_stop_loss = stop_loss + self.stop_loss = stop_loss self.stop_loss_pct = -1 * abs(percent) @@ -676,26 +674,27 @@ class LocalTrade: else: new_loss = float(current_price * (1 - abs(stoploss / leverage))) + stop_loss_norm = price_to_precision(new_loss, self.price_precision, self.precision_mode, + rounding_mode=ROUND_DOWN if self.is_short else ROUND_UP) # no stop loss assigned yet if self.initial_stop_loss_pct is None or refresh: - self.__set_stop_loss(new_loss, stoploss) + self.__set_stop_loss(stop_loss_norm, stoploss) self.initial_stop_loss = price_to_precision( - new_loss, self.price_precision, self.precision_mode, + stop_loss_norm, self.price_precision, self.precision_mode, rounding_mode=ROUND_DOWN if self.is_short else ROUND_UP) self.initial_stop_loss_pct = -1 * abs(stoploss) # evaluate if the stop loss needs to be updated else: - - higher_stop = new_loss > self.stop_loss - lower_stop = new_loss < self.stop_loss + higher_stop = stop_loss_norm > self.stop_loss + lower_stop = stop_loss_norm < self.stop_loss # stop losses only walk up, never down!, # ? But adding more to a leveraged trade would create a lower liquidation price, # ? decreasing the minimum stoploss if (higher_stop and not self.is_short) or (lower_stop and self.is_short): logger.debug(f"{self.pair} - Adjusting stoploss...") - self.__set_stop_loss(new_loss, stoploss) + self.__set_stop_loss(stop_loss_norm, stoploss) else: logger.debug(f"{self.pair} - Keeping current stoploss...") @@ -769,10 +768,8 @@ class LocalTrade: self.open_order_id = None self.recalc_trade_from_orders(is_closing=True) if show_msg: - logger.info( - 'Marking %s as closed as the trade is fulfilled and found no open orders for it.', - self - ) + logger.info(f"Marking {self} as closed as the trade is fulfilled " + "and found no open orders for it.") def update_fee(self, fee_cost: float, fee_currency: Optional[str], fee_rate: Optional[float], side: str) -> None: @@ -1220,12 +1217,13 @@ class LocalTrade: return LocalTrade.bt_open_open_trade_count @staticmethod - def stoploss_reinitialization(desired_stoploss): + def stoploss_reinitialization(desired_stoploss: float): """ Adjust initial Stoploss to desired stoploss for all open trades. """ + trade: Trade for trade in Trade.get_open_trades(): - logger.info("Found open trade: %s", trade) + logger.info(f"Found open trade: {trade}") # skip case if trailing-stop changed the stoploss already. if (trade.stop_loss == trade.initial_stop_loss @@ -1234,7 +1232,7 @@ class LocalTrade: logger.info(f"Stoploss for {trade} needs adjustment...") # Force reset of stoploss - trade.stop_loss = None + trade.stop_loss = 0.0 trade.initial_stop_loss_pct = None trade.adjust_stop_loss(trade.open_rate, desired_stoploss) logger.info(f"New stoploss: {trade.stop_loss}.") diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index f1ac9db54..892865d43 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -1,7 +1,7 @@ from datetime import date, datetime from typing import Any, Dict, List, Optional, Union -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, RootModel, SerializeAsAny from freqtrade.constants import DATETIME_PRINT_FORMAT, IntOrInf from freqtrade.enums import MarginMode, OrderTypeValues, SignalDirection, TradingMode @@ -9,9 +9,9 @@ from freqtrade.types import ValidExchangesType class ExchangeModePayloadMixin(BaseModel): - trading_mode: Optional[TradingMode] - margin_mode: Optional[MarginMode] - exchange: Optional[str] + trading_mode: Optional[TradingMode] = None + margin_mode: Optional[MarginMode] = None + exchange: Optional[str] = None class Ping(BaseModel): @@ -43,11 +43,11 @@ class BackgroundTaskStatus(BaseModel): job_category: str status: str running: bool - progress: Optional[float] + progress: Optional[float] = None class BackgroundTaskResult(BaseModel): - error: Optional[str] + error: Optional[str] = None status: str @@ -60,9 +60,9 @@ class Balance(BaseModel): free: float balance: float used: float - bot_owned: Optional[float] + bot_owned: Optional[float] = None est_stake: float - est_stake_bot: Optional[float] + est_stake_bot: Optional[float] = None stake: str # Starting with 2.x side: str @@ -141,7 +141,7 @@ class Profit(BaseModel): expectancy_ratio: float max_drawdown: float max_drawdown_abs: float - trading_volume: Optional[float] + trading_volume: Optional[float] = None bot_start_timestamp: int bot_start_date: str @@ -173,50 +173,50 @@ class Daily(BaseModel): class UnfilledTimeout(BaseModel): - entry: Optional[int] - exit: Optional[int] - unit: Optional[str] - exit_timeout_count: Optional[int] + entry: Optional[int] = None + exit: Optional[int] = None + unit: Optional[str] = None + exit_timeout_count: Optional[int] = None class OrderTypes(BaseModel): entry: OrderTypeValues exit: OrderTypeValues - emergency_exit: Optional[OrderTypeValues] - force_exit: Optional[OrderTypeValues] - force_entry: Optional[OrderTypeValues] + emergency_exit: Optional[OrderTypeValues] = None + force_exit: Optional[OrderTypeValues] = None + force_entry: Optional[OrderTypeValues] = None stoploss: OrderTypeValues stoploss_on_exchange: bool - stoploss_on_exchange_interval: Optional[int] + stoploss_on_exchange_interval: Optional[int] = None class ShowConfig(BaseModel): version: str - strategy_version: Optional[str] + strategy_version: Optional[str] = None api_version: float dry_run: bool trading_mode: str short_allowed: bool stake_currency: str stake_amount: str - available_capital: Optional[float] + available_capital: Optional[float] = None stake_currency_decimals: int max_open_trades: IntOrInf minimal_roi: Dict[str, Any] - stoploss: Optional[float] + stoploss: Optional[float] = None stoploss_on_exchange: bool - trailing_stop: Optional[bool] - trailing_stop_positive: Optional[float] - trailing_stop_positive_offset: Optional[float] - trailing_only_offset_is_reached: Optional[bool] - unfilledtimeout: Optional[UnfilledTimeout] # Empty in webserver mode - order_types: Optional[OrderTypes] - use_custom_stoploss: Optional[bool] - timeframe: Optional[str] + trailing_stop: Optional[bool] = None + trailing_stop_positive: Optional[float] = None + trailing_stop_positive_offset: Optional[float] = None + trailing_only_offset_is_reached: Optional[bool] = None + unfilledtimeout: Optional[UnfilledTimeout] = None # Empty in webserver mode + order_types: Optional[OrderTypes] = None + use_custom_stoploss: Optional[bool] = None + timeframe: Optional[str] = None timeframe_ms: int timeframe_min: int exchange: str - strategy: Optional[str] + strategy: Optional[str] = None force_entry_enable: bool exit_pricing: Dict[str, Any] entry_pricing: Dict[str, Any] @@ -231,17 +231,17 @@ class OrderSchema(BaseModel): pair: str order_id: str status: str - remaining: Optional[float] + remaining: Optional[float] = None amount: float safe_price: float cost: float - filled: Optional[float] + filled: Optional[float] = None ft_order_side: str order_type: str is_open: bool - order_timestamp: Optional[int] - order_filled_timestamp: Optional[int] - ft_fee_base: Optional[float] + order_timestamp: Optional[int] = None + order_filled_timestamp: Optional[int] = None + ft_fee_base: Optional[float] = None class TradeSchema(BaseModel): @@ -255,81 +255,81 @@ class TradeSchema(BaseModel): amount: float amount_requested: float stake_amount: float - max_stake_amount: Optional[float] + max_stake_amount: Optional[float] = None strategy: str - enter_tag: Optional[str] + enter_tag: Optional[str] = None timeframe: int - fee_open: Optional[float] - fee_open_cost: Optional[float] - fee_open_currency: Optional[str] - fee_close: Optional[float] - fee_close_cost: Optional[float] - fee_close_currency: Optional[str] + fee_open: Optional[float] = None + fee_open_cost: Optional[float] = None + fee_open_currency: Optional[str] = None + fee_close: Optional[float] = None + fee_close_cost: Optional[float] = None + fee_close_currency: Optional[str] = None open_date: str open_timestamp: int open_rate: float - open_rate_requested: Optional[float] + open_rate_requested: Optional[float] = None open_trade_value: float - close_date: Optional[str] - close_timestamp: Optional[int] - close_rate: Optional[float] - close_rate_requested: Optional[float] + close_date: Optional[str] = None + close_timestamp: Optional[int] = None + close_rate: Optional[float] = None + close_rate_requested: Optional[float] = None - close_profit: Optional[float] - close_profit_pct: Optional[float] - close_profit_abs: Optional[float] + close_profit: Optional[float] = None + close_profit_pct: Optional[float] = None + close_profit_abs: Optional[float] = None - profit_ratio: Optional[float] - profit_pct: Optional[float] - profit_abs: Optional[float] - profit_fiat: Optional[float] + profit_ratio: Optional[float] = None + profit_pct: Optional[float] = None + profit_abs: Optional[float] = None + profit_fiat: Optional[float] = None realized_profit: float - realized_profit_ratio: Optional[float] + realized_profit_ratio: Optional[float] = None - exit_reason: Optional[str] - exit_order_status: Optional[str] + exit_reason: Optional[str] = None + exit_order_status: Optional[str] = None - stop_loss_abs: Optional[float] - stop_loss_ratio: Optional[float] - stop_loss_pct: Optional[float] - stoploss_order_id: Optional[str] - stoploss_last_update: Optional[str] - stoploss_last_update_timestamp: Optional[int] - initial_stop_loss_abs: Optional[float] - initial_stop_loss_ratio: Optional[float] - initial_stop_loss_pct: Optional[float] + stop_loss_abs: Optional[float] = None + stop_loss_ratio: Optional[float] = None + stop_loss_pct: Optional[float] = None + stoploss_order_id: Optional[str] = None + stoploss_last_update: Optional[str] = None + stoploss_last_update_timestamp: Optional[int] = None + initial_stop_loss_abs: Optional[float] = None + initial_stop_loss_ratio: Optional[float] = None + initial_stop_loss_pct: Optional[float] = None - min_rate: Optional[float] - max_rate: Optional[float] - open_order_id: Optional[str] + min_rate: Optional[float] = None + max_rate: Optional[float] = None + open_order_id: Optional[str] = None orders: List[OrderSchema] - leverage: Optional[float] - interest_rate: Optional[float] - liquidation_price: Optional[float] - funding_fees: Optional[float] - trading_mode: Optional[TradingMode] + leverage: Optional[float] = None + interest_rate: Optional[float] = None + liquidation_price: Optional[float] = None + funding_fees: Optional[float] = None + trading_mode: Optional[TradingMode] = None - amount_precision: Optional[float] - price_precision: Optional[float] - precision_mode: Optional[int] + amount_precision: Optional[float] = None + price_precision: Optional[float] = None + precision_mode: Optional[int] = None class OpenTradeSchema(TradeSchema): - stoploss_current_dist: Optional[float] - stoploss_current_dist_pct: Optional[float] - stoploss_current_dist_ratio: Optional[float] - stoploss_entry_dist: Optional[float] - stoploss_entry_dist_ratio: Optional[float] + stoploss_current_dist: Optional[float] = None + stoploss_current_dist_pct: Optional[float] = None + stoploss_current_dist_ratio: Optional[float] = None + stoploss_entry_dist: Optional[float] = None + stoploss_entry_dist_ratio: Optional[float] = None current_rate: float total_profit_abs: float - total_profit_fiat: Optional[float] - total_profit_ratio: Optional[float] + total_profit_fiat: Optional[float] = None + total_profit_ratio: Optional[float] = None - open_order: Optional[str] + open_order: Optional[str] = None class TradeResponse(BaseModel): @@ -339,8 +339,7 @@ class TradeResponse(BaseModel): total_trades: int -class ForceEnterResponse(BaseModel): - __root__: Union[TradeSchema, StatusMsg] +ForceEnterResponse = RootModel[Union[TradeSchema, StatusMsg]] class LockModel(BaseModel): @@ -352,7 +351,7 @@ class LockModel(BaseModel): lock_timestamp: int pair: str side: str - reason: Optional[str] + reason: Optional[str] = None class Locks(BaseModel): @@ -361,8 +360,8 @@ class Locks(BaseModel): class DeleteLockRequest(BaseModel): - pair: Optional[str] - lockid: Optional[int] + pair: Optional[str] = None + lockid: Optional[int] = None class Logs(BaseModel): @@ -373,17 +372,17 @@ class Logs(BaseModel): class ForceEnterPayload(BaseModel): pair: str side: SignalDirection = SignalDirection.LONG - price: Optional[float] - ordertype: Optional[OrderTypeValues] - stakeamount: Optional[float] - entry_tag: Optional[str] - leverage: Optional[float] + price: Optional[float] = None + ordertype: Optional[OrderTypeValues] = None + stakeamount: Optional[float] = None + entry_tag: Optional[str] = None + leverage: Optional[float] = None class ForceExitPayload(BaseModel): tradeid: str - ordertype: Optional[OrderTypeValues] - amount: Optional[float] + ordertype: Optional[OrderTypeValues] = None + amount: Optional[float] = None class BlacklistPayload(BaseModel): @@ -405,7 +404,7 @@ class WhitelistResponse(BaseModel): class WhitelistEvaluateResponse(BackgroundTaskResult): - result: Optional[WhitelistResponse] + result: Optional[WhitelistResponse] = None class DeleteTrade(BaseModel): @@ -420,8 +419,7 @@ class PlotConfig_(BaseModel): subplots: Dict[str, Any] -class PlotConfig(BaseModel): - __root__: Union[PlotConfig_, Dict] +PlotConfig = RootModel[Union[PlotConfig_, Dict]] class StrategyListResponse(BaseModel): @@ -470,7 +468,7 @@ class PairHistory(BaseModel): timeframe: str timeframe_ms: int columns: List[str] - data: List[Any] + data: SerializeAsAny[List[Any]] length: int buy_signals: int sell_signals: int @@ -484,11 +482,11 @@ class PairHistory(BaseModel): data_start: str data_stop: str data_stop_ts: int - - class Config: - json_encoders = { - datetime: lambda v: v.strftime(DATETIME_PRINT_FORMAT), - } + # TODO[pydantic]: The following keys were removed: `json_encoders`. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. + model_config = ConfigDict(json_encoders={ + datetime: lambda v: v.strftime(DATETIME_PRINT_FORMAT), + }) class BacktestFreqAIInputs(BaseModel): @@ -497,16 +495,16 @@ class BacktestFreqAIInputs(BaseModel): class BacktestRequest(BaseModel): strategy: str - timeframe: Optional[str] - timeframe_detail: Optional[str] - timerange: Optional[str] - max_open_trades: Optional[IntOrInf] - stake_amount: Optional[str] + timeframe: Optional[str] = None + timeframe_detail: Optional[str] = None + timerange: Optional[str] = None + max_open_trades: Optional[IntOrInf] = None + stake_amount: Optional[Union[str, float]] = None enable_protections: bool - dry_run_wallet: Optional[float] - backtest_cache: Optional[str] - freqaimodel: Optional[str] - freqai: Optional[BacktestFreqAIInputs] + dry_run_wallet: Optional[float] = None + backtest_cache: Optional[str] = None + freqaimodel: Optional[str] = None + freqai: Optional[BacktestFreqAIInputs] = None class BacktestResponse(BaseModel): @@ -515,9 +513,9 @@ class BacktestResponse(BaseModel): status_msg: str step: str progress: float - trade_count: Optional[float] + trade_count: Optional[float] = None # TODO: Properly type backtestresult... - backtest_result: Optional[Dict[str, Any]] + backtest_result: Optional[Dict[str, Any]] = None # TODO: This is a copy of BacktestHistoryEntryType @@ -540,5 +538,5 @@ class SysInfo(BaseModel): class Health(BaseModel): - last_process: Optional[datetime] - last_process_ts: Optional[int] + last_process: Optional[datetime] = None + last_process_ts: Optional[int] = None diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index bc0c88fe4..7299364a8 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -175,9 +175,9 @@ def force_entry(payload: ForceEnterPayload, rpc: RPC = Depends(get_rpc)): leverage=payload.leverage) if trade: - return ForceEnterResponse.parse_obj(trade.to_json()) + return ForceEnterResponse.model_validate(trade.to_json()) else: - return ForceEnterResponse.parse_obj( + return ForceEnterResponse.model_validate( {"status": f"Error entering {payload.side} trade for pair {payload.pair}."}) @@ -282,14 +282,14 @@ def plot_config(strategy: Optional[str] = None, config=Depends(get_config), if not strategy: if not rpc: raise RPCException("Strategy is mandatory in webserver mode.") - return PlotConfig.parse_obj(rpc._rpc_plot_config()) + return PlotConfig.model_validate(rpc._rpc_plot_config()) else: config1 = deepcopy(config) config1.update({ 'strategy': strategy }) try: - return PlotConfig.parse_obj(RPC._rpc_plot_config_with_strategy(config1)) + return PlotConfig.model_validate(RPC._rpc_plot_config_with_strategy(config1)) except Exception as e: raise HTTPException(status_code=502, detail=str(e)) diff --git a/freqtrade/rpc/api_server/api_ws.py b/freqtrade/rpc/api_server/api_ws.py index 40a5a75fd..16aeb56f3 100644 --- a/freqtrade/rpc/api_server/api_ws.py +++ b/freqtrade/rpc/api_server/api_ws.py @@ -65,7 +65,7 @@ async def _process_consumer_request( """ # Validate the request, makes sure it matches the schema try: - websocket_request = WSRequestSchema.parse_obj(request) + websocket_request = WSRequestSchema.model_validate(request) except ValidationError as e: logger.error(f"Invalid request from {channel}: {e}") return @@ -94,7 +94,7 @@ async def _process_consumer_request( # Format response response = WSWhitelistMessage(data=whitelist) - await channel.send(response.dict(exclude_none=True)) + await channel.send(response.model_dump(exclude_none=True)) elif type_ == RPCRequestType.ANALYZED_DF: # Limit the amount of candles per dataframe to 'limit' or 1500 @@ -105,7 +105,7 @@ async def _process_consumer_request( for message in rpc._ws_request_analyzed_df(limit, pair): # Format response response = WSAnalyzedDFMessage(data=message) - await channel.send(response.dict(exclude_none=True)) + await channel.send(response.model_dump(exclude_none=True)) @router.websocket("/message/ws") diff --git a/freqtrade/rpc/api_server/ws_schemas.py b/freqtrade/rpc/api_server/ws_schemas.py index af98bd532..34eaf0245 100644 --- a/freqtrade/rpc/api_server/ws_schemas.py +++ b/freqtrade/rpc/api_server/ws_schemas.py @@ -2,15 +2,14 @@ from datetime import datetime from typing import Any, Dict, List, Optional, TypedDict from pandas import DataFrame -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from freqtrade.constants import PairWithTimeframe from freqtrade.enums.rpcmessagetype import RPCMessageType, RPCRequestType class BaseArbitraryModel(BaseModel): - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict(arbitrary_types_allowed=True) class WSRequestSchema(BaseArbitraryModel): @@ -27,9 +26,7 @@ class WSMessageSchemaType(TypedDict): class WSMessageSchema(BaseArbitraryModel): type: RPCMessageType data: Optional[Any] = None - - class Config: - extra = 'allow' + model_config = ConfigDict(extra='allow') # ------------------------------ REQUEST SCHEMAS ---------------------------- diff --git a/freqtrade/rpc/external_message_consumer.py b/freqtrade/rpc/external_message_consumer.py index e888191ea..200d408e6 100644 --- a/freqtrade/rpc/external_message_consumer.py +++ b/freqtrade/rpc/external_message_consumer.py @@ -41,7 +41,7 @@ logger = logging.getLogger(__name__) def schema_to_dict(schema: Union[WSMessageSchema, WSRequestSchema]): - return schema.dict(exclude_none=True) + return schema.model_dump(exclude_none=True) class ExternalMessageConsumer: @@ -322,7 +322,7 @@ class ExternalMessageConsumer: producer_name = producer.get('name', 'default') try: - producer_message = WSMessageSchema.parse_obj(message) + producer_message = WSMessageSchema.model_validate(message) except ValidationError as e: logger.error(f"Invalid message from `{producer_name}`: {e}") return @@ -344,7 +344,7 @@ class ExternalMessageConsumer: def _consume_whitelist_message(self, producer_name: str, message: WSMessageSchema): try: # Validate the message - whitelist_message = WSWhitelistMessage.parse_obj(message) + whitelist_message = WSWhitelistMessage.model_validate(message.model_dump()) except ValidationError as e: logger.error(f"Invalid message from `{producer_name}`: {e}") return @@ -356,7 +356,7 @@ class ExternalMessageConsumer: def _consume_analyzed_df_message(self, producer_name: str, message: WSMessageSchema): try: - df_message = WSAnalyzedDFMessage.parse_obj(message) + df_message = WSAnalyzedDFMessage.model_validate(message.model_dump()) except ValidationError as e: logger.error(f"Invalid message from `{producer_name}`: {e}") return diff --git a/freqtrade/rpc/fiat_convert.py b/freqtrade/rpc/fiat_convert.py index ea13209c4..d790c4b48 100644 --- a/freqtrade/rpc/fiat_convert.py +++ b/freqtrade/rpc/fiat_convert.py @@ -26,6 +26,7 @@ coingecko_mapping = { 'sol': 'solana', 'usdt': 'tether', 'busd': 'binance-usd', + 'tusd': 'true-usd', } diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 78616a3fc..5697ce15b 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -612,17 +612,13 @@ class RPC: est_stake = balance.free est_bot_stake = amount else: - try: - pair = self._freqtrade.exchange.get_valid_pair_combination(coin, stake_currency) - rate: Optional[float] = tickers.get(pair, {}).get('last', None) - if rate: - if pair.startswith(stake_currency) and not pair.endswith(stake_currency): - rate = 1.0 / rate - est_stake = rate * balance.total - est_bot_stake = rate * amount - except (ExchangeError): - logger.warning(f"Could not get rate for pair {coin}.") - raise ValueError() + pair = self._freqtrade.exchange.get_valid_pair_combination(coin, stake_currency) + rate: Optional[float] = tickers.get(pair, {}).get('last', None) + if rate: + if pair.startswith(stake_currency) and not pair.endswith(stake_currency): + rate = 1.0 / rate + est_stake = rate * balance.total + est_bot_stake = rate * amount return est_stake, est_bot_stake diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 0f848130f..eca3e3ede 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -381,7 +381,7 @@ class IStrategy(ABC, HyperStrategyMixin): For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ - When not implemented by a strategy, returns the initial stoploss value + When not implemented by a strategy, returns the initial stoploss value. Only called when use_custom_stoploss is set to True. :param pair: Pair that's currently analyzed @@ -1181,7 +1181,8 @@ class IStrategy(ABC, HyperStrategyMixin): bound = (low if trade.is_short else high) bound_profit = current_profit if not bound else trade.calc_profit_ratio(bound) if self.use_custom_stoploss and dir_correct: - stop_loss_value = strategy_safe_wrapper(self.custom_stoploss, default_retval=None + stop_loss_value = strategy_safe_wrapper(self.custom_stoploss, default_retval=None, + supress_error=True )(pair=trade.pair, trade=trade, current_time=current_time, current_rate=(bound or current_rate), diff --git a/freqtrade/templates/base_strategy.py.j2 b/freqtrade/templates/base_strategy.py.j2 index 53426b211..a4e0a2b24 100644 --- a/freqtrade/templates/base_strategy.py.j2 +++ b/freqtrade/templates/base_strategy.py.j2 @@ -78,19 +78,7 @@ class {{ strategy }}(IStrategy): buy_rsi = IntParameter(10, 40, default=30, space="buy") sell_rsi = IntParameter(60, 90, default=70, space="sell") - # Optional order type mapping. - order_types = { - 'entry': 'limit', - 'exit': 'limit', - 'stoploss': 'market', - 'stoploss_on_exchange': False - } - - # Optional order time in force. - order_time_in_force = { - 'entry': 'GTC', - 'exit': 'GTC' - } + {{ attributes | indent(4) }} {{ plot_config | indent(4) }} def informative_pairs(self): diff --git a/freqtrade/templates/strategy_subtemplates/strategy_attributes_full.j2 b/freqtrade/templates/strategy_subtemplates/strategy_attributes_full.j2 new file mode 100644 index 000000000..86445510d --- /dev/null +++ b/freqtrade/templates/strategy_subtemplates/strategy_attributes_full.j2 @@ -0,0 +1,13 @@ +# Optional order type mapping. +order_types = { + 'entry': 'limit', + 'exit': 'limit', + 'stoploss': 'market', + 'stoploss_on_exchange': False +} + +# Optional order time in force. +order_time_in_force = { + 'entry': 'GTC', + 'exit': 'GTC' +} diff --git a/freqtrade/templates/strategy_subtemplates/strategy_attributes_minimal.j2 b/freqtrade/templates/strategy_subtemplates/strategy_attributes_minimal.j2 new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index 92c79b899..af09624ac 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -2,6 +2,7 @@ from freqtrade.util.datetime_helpers import (dt_floor_day, dt_from_ts, dt_humani dt_utc, format_ms_time, shorten_date) from freqtrade.util.ft_precise import FtPrecise from freqtrade.util.periodic_cache import PeriodicCache +from freqtrade.util.template_renderer import render_template, render_template_with_fallback # noqa __all__ = [ diff --git a/freqtrade/util/template_renderer.py b/freqtrade/util/template_renderer.py new file mode 100644 index 000000000..362d0f875 --- /dev/null +++ b/freqtrade/util/template_renderer.py @@ -0,0 +1,27 @@ +""" +Jinja2 rendering utils, used to generate new strategy and configurations. +""" + + +def render_template(templatefile: str, arguments: dict = {}) -> str: + + from jinja2 import Environment, PackageLoader, select_autoescape + + env = Environment( + loader=PackageLoader('freqtrade', 'templates'), + autoescape=select_autoescape(['html', 'xml']) + ) + template = env.get_template(templatefile) + return template.render(**arguments) + + +def render_template_with_fallback(templatefile: str, templatefallbackfile: str, + arguments: dict = {}) -> str: + """ + Use templatefile if possible, otherwise fall back to templatefallbackfile + """ + from jinja2.exceptions import TemplateNotFound + try: + return render_template(templatefile, arguments) + except TemplateNotFound: + return render_template(templatefallbackfile, arguments) diff --git a/mkdocs.yml b/mkdocs.yml index 815a10419..bb5ae0010 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -47,7 +47,6 @@ nav: - Advanced Hyperopt: advanced-hyperopt.md - Producer/Consumer mode: producer-consumer.md - Edge Positioning: edge.md - - Sandbox Testing: sandbox-testing.md - FAQ: faq.md - SQL Cheat-sheet: sql_cheatsheet.md - Strategy migration: strategy_migration.md diff --git a/pyproject.toml b/pyproject.toml index 17f91c7b2..40c0e2005 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,7 @@ ignore = ["freqtrade/vendor/**"] [tool.ruff] line-length = 100 -extend-exclude = [".env"] +extend-exclude = [".env", ".venv"] target-version = "py38" extend-select = [ "C90", # mccabe diff --git a/requirements-dev.txt b/requirements-dev.txt index 9d1873f2e..cac9d5cd7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,8 +7,8 @@ -r docs/requirements-docs.txt coveralls==3.3.1 -ruff==0.0.282 -mypy==1.4.1 +ruff==0.0.285 +mypy==1.5.1 pre-commit==3.3.3 pytest==7.4.0 pytest-asyncio==0.21.1 @@ -17,10 +17,10 @@ pytest-mock==3.11.1 pytest-random-order==1.1.0 isort==5.12.0 # For datetime mocking -time-machine==2.11.0 +time-machine==2.12.0 # Convert jupyter notebooks to markdown documents -nbconvert==7.7.3 +nbconvert==7.7.4 # mypy types types-cachetools==5.3.0.6 diff --git a/requirements-freqai-rl.txt b/requirements-freqai-rl.txt index 74c6d4ebe..c2cca5427 100644 --- a/requirements-freqai-rl.txt +++ b/requirements-freqai-rl.txt @@ -4,8 +4,8 @@ # Required for freqai-rl torch==2.0.1 #until these branches will be released we can use this -gymnasium==0.28.1 -stable_baselines3==2.0.0 +gymnasium==0.29.1 +stable_baselines3==2.1.0 sb3_contrib>=2.0.0a9 # Progress bar for stable-baselines3 and sb3-contrib -tqdm==4.65.0 +tqdm==4.66.1 diff --git a/requirements-freqai.txt b/requirements-freqai.txt index 325b92544..d8421e968 100644 --- a/requirements-freqai.txt +++ b/requirements-freqai.txt @@ -4,9 +4,9 @@ # Required for freqai scikit-learn==1.1.3 -joblib==1.3.1 +joblib==1.3.2 catboost==1.2; 'arm' not in platform_machine lightgbm==4.0.0 xgboost==1.7.6 -tensorboard==2.13.0 +tensorboard==2.14.0 datasieve==0.1.7 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index aafbad608..80b62e2f3 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -2,7 +2,7 @@ -r requirements.txt # Required for hyperopt -scipy==1.11.1; python_version >= '3.9' +scipy==1.11.2; python_version >= '3.9' scipy==1.10.1; python_version < '3.9' scikit-learn==1.1.3 scikit-optimize==0.9.0 diff --git a/requirements-plot.txt b/requirements-plot.txt index 72303efcb..9a8c596ad 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,4 +1,4 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.15.0 +plotly==5.16.1 diff --git a/requirements.txt b/requirements.txt index d0f61d2ab..bf346382f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,11 +3,11 @@ numpy==1.24.3; python_version <= '3.8' pandas==2.0.3 pandas-ta==0.3.14b -ccxt==4.0.50 +ccxt==4.0.71 cryptography==41.0.3; platform_machine != 'armv7l' cryptography==40.0.1; platform_machine == 'armv7l' aiohttp==3.8.5 -SQLAlchemy==2.0.19 +SQLAlchemy==2.0.20 python-telegram-bot==20.4 # can't be hard-pinned due to telegram-bot pinning httpx with ~ httpx>=0.24.1 @@ -15,15 +15,15 @@ arrow==1.2.3 cachetools==5.3.1 requests==2.31.0 urllib3==2.0.4 -jsonschema==4.18.6 -TA-Lib==0.4.27 +jsonschema==4.19.0 +TA-Lib==0.4.28 technical==1.4.0 tabulate==0.9.0 pycoingecko==3.1.0 jinja2==3.1.2 tables==3.8.0 blosc==1.11.1 -joblib==1.3.1 +joblib==1.3.2 rich==13.5.2 pyarrow==12.0.1; platform_machine != 'armv7l' @@ -33,24 +33,24 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.10 # Properly format api responses -orjson==3.9.3 +orjson==3.9.5 # Notify systemd sdnotify==0.3.2 # API Server -fastapi==0.101.0 -pydantic==1.10.11 +fastapi==0.101.1 +pydantic==2.2.1 uvicorn==0.23.2 pyjwt==2.8.0 -aiofiles==23.1.0 +aiofiles==23.2.1 psutil==5.9.5 # Support for colorized terminal output colorama==0.4.6 # Building config files interactively -questionary==1.10.0 -prompt-toolkit==3.0.39 +questionary==2.0.0 +prompt-toolkit==3.0.36 # Extensions to datetime library python-dateutil==2.8.2 diff --git a/setup.py b/setup.py index c2b725f87..2e92d0839 100644 --- a/setup.py +++ b/setup.py @@ -97,7 +97,7 @@ setup( 'rich', 'pyarrow; platform_machine != "armv7l"', 'fastapi', - 'pydantic>=1.8.0,<2.0', + 'pydantic>=2.2.0', 'uvicorn', 'psutil', 'pyjwt', diff --git a/setup.sh b/setup.sh index 84f804021..cd9b85d2a 100755 --- a/setup.sh +++ b/setup.sh @@ -11,7 +11,7 @@ function check_installed_pip() { ${PYTHON} -m pip > /dev/null if [ $? -ne 0 ]; then echo_block "Installing Pip for ${PYTHON}" - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py + curl https://bootstrap.pypa.io/get-pip.py -s -o get-pip.py ${PYTHON} get-pip.py rm get-pip.py fi @@ -41,12 +41,12 @@ function check_installed_python() { } function updateenv() { - echo_block "Updating your virtual env" - if [ ! -f .env/bin/activate ]; then + echo_block "Updating your virtual environment" + if [ ! -f .venv/bin/activate ]; then echo "Something went wrong, no virtual environment found." exit 1 fi - source .env/bin/activate + source .venv/bin/activate SYS_ARCH=$(uname -m) echo "pip install in-progress. Please wait..." ${PYTHON} -m pip install --upgrade pip wheel setuptools @@ -120,7 +120,7 @@ function updateenv() { # Install tab lib function install_talib() { - if [ -f /usr/local/lib/libta_lib.a ]; then + if [ -f /usr/local/lib/libta_lib.a ] || [ -f /usr/local/lib/libta_lib.so ] || [ -f /usr/lib/libta_lib.so ]; then echo "ta-lib already installed, skipping" return fi @@ -186,7 +186,14 @@ function install_redhat() { # Upgrade the bot function update() { git pull + if [ -f .env/bin/activate ]; then + # Old environment found - updating to new environment. + recreate_environments + fi updateenv + echo "Update completed." + echo_block "Don't forget to activate your virtual enviorment with 'source .venv/bin/activate'!" + } function check_git_changes() { @@ -199,6 +206,27 @@ function check_git_changes() { fi } +function recreate_environments() { + if [ -d ".env" ]; then + # Remove old virtual env + echo "- Deleting your previous virtual env" + echo "Warning: Your new environment will be at .venv!" + rm -rf .env + fi + if [ -d ".venv" ]; then + echo "- Deleting your previous virtual env" + rm -rf .venv + fi + + echo + ${PYTHON} -m venv .venv + if [ $? -ne 0 ]; then + echo "Could not create virtual environment. Leaving now" + exit 1 + fi + +} + # Reset Develop or Stable branch function reset() { echo_block "Resetting branch and virtual env" @@ -225,22 +253,13 @@ function reset() { else echo "Reset ignored because you are not on 'stable' or 'develop'." fi + recreate_environments - if [ -d ".env" ]; then - echo "- Deleting your previous virtual env" - rm -rf .env - fi - echo - ${PYTHON} -m venv .env - if [ $? -ne 0 ]; then - echo "Could not create virtual environment. Leaving now" - exit 1 - fi updateenv } function config() { - echo_block "Please use 'freqtrade new-config -c config.json' to generate a new configuration file." + echo_block "Please use 'freqtrade new-config -c user_data/config.json' to generate a new configuration file." } function install() { @@ -266,9 +285,9 @@ function install() { reset config echo_block "Run the bot !" - echo "You can now use the bot by executing 'source .env/bin/activate; freqtrade '." - echo "You can see the list of available bot sub-commands by executing 'source .env/bin/activate; freqtrade --help'." - echo "You verify that freqtrade is installed successfully by running 'source .env/bin/activate; freqtrade --version'." + echo "You can now use the bot by executing 'source .venv/bin/activate; freqtrade '." + echo "You can see the list of available bot sub-commands by executing 'source .venv/bin/activate; freqtrade --help'." + echo "You verify that freqtrade is installed successfully by running 'source .venv/bin/activate; freqtrade --version'." } function plot() { diff --git a/tests/conftest.py b/tests/conftest.py index 3bfc88d70..4372534ce 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,7 +14,7 @@ import pytest from freqtrade import constants from freqtrade.commands import Arguments -from freqtrade.data.converter import ohlcv_to_dataframe +from freqtrade.data.converter import ohlcv_to_dataframe, trades_list_to_df from freqtrade.edge import PairInfo from freqtrade.enums import CandleType, MarginMode, RunMode, SignalDirection, TradingMode from freqtrade.exchange import Exchange @@ -2346,7 +2346,15 @@ def trades_history(): [1565798399629, '1261813bb30', None, 'buy', 0.019627, 0.244, 0.004788987999999999], [1565798399752, '1261813cc31', None, 'sell', 0.019626, 0.011, 0.00021588599999999999], [1565798399862, '126181cc332', None, 'sell', 0.019626, 0.011, 0.00021588599999999999], - [1565798399872, '1261aa81333', None, 'sell', 0.019626, 0.011, 0.00021588599999999999]] + [1565798399862, '126181cc333', None, 'sell', 0.019626, 0.012, 0.00021588599999999999], + [1565798399872, '1261aa81334', None, 'sell', 0.019626, 0.011, 0.00021588599999999999]] + + +@pytest.fixture(scope="function") +def trades_history_df(trades_history): + trades = trades_list_to_df(trades_history) + trades['date'] = pd.to_datetime(trades['timestamp'], unit='ms', utc=True) + return trades @pytest.fixture(scope="function") diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 6a2cb5638..7ca236442 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -4,13 +4,14 @@ from pathlib import Path from shutil import copyfile import numpy as np +import pandas as pd import pytest from freqtrade.configuration.timerange import TimeRange from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_format, ohlcv_fill_up_missing_data, ohlcv_to_dataframe, - reduce_dataframe_footprint, trades_dict_to_list, - trades_remove_duplicates, trades_to_ohlcv, trim_dataframe) + reduce_dataframe_footprint, trades_df_remove_duplicates, + trades_dict_to_list, trades_to_ohlcv, trim_dataframe) from freqtrade.data.history import (get_timerange, load_data, load_pair_history, validate_backtest_data) from freqtrade.data.history.idatahandler import IDataHandler @@ -34,26 +35,21 @@ def test_ohlcv_to_dataframe(ohlcv_history_list, caplog): assert log_has('Converting candle (OHLCV) data to dataframe for pair UNITTEST/BTC.', caplog) -def test_trades_to_ohlcv(ohlcv_history_list, caplog): +def test_trades_to_ohlcv(trades_history_df, caplog): caplog.set_level(logging.DEBUG) with pytest.raises(ValueError, match="Trade-list empty."): - trades_to_ohlcv([], '1m') + trades_to_ohlcv(pd.DataFrame(columns=trades_history_df.columns), '1m') - trades = [ - [1570752011620, "13519807", None, "sell", 0.00141342, 23.0, 0.03250866], - [1570752011620, "13519808", None, "sell", 0.00141266, 54.0, 0.07628364], - [1570752017964, "13519809", None, "sell", 0.00141266, 8.0, 0.01130128]] - - df = trades_to_ohlcv(trades, '1m') + df = trades_to_ohlcv(trades_history_df, '1m') assert not df.empty assert len(df) == 1 assert 'open' in df.columns assert 'high' in df.columns assert 'low' in df.columns assert 'close' in df.columns - assert df.loc[:, 'high'][0] == 0.00141342 - assert df.loc[:, 'low'][0] == 0.00141266 + assert df.loc[:, 'high'][0] == 0.019627 + assert df.loc[:, 'low'][0] == 0.019626 def test_ohlcv_fill_up_missing_data(testdatadir, caplog): @@ -302,13 +298,13 @@ def test_trim_dataframe(testdatadir) -> None: assert all(data_modify.iloc[0] == data.iloc[25]) -def test_trades_remove_duplicates(trades_history): - trades_history1 = trades_history * 3 - assert len(trades_history1) == len(trades_history) * 3 - res = trades_remove_duplicates(trades_history1) - assert len(res) == len(trades_history) - for i, t in enumerate(res): - assert t == trades_history[i] +def test_trades_df_remove_duplicates(trades_history_df): + trades_history1 = pd.concat([trades_history_df, trades_history_df, trades_history_df] + ).reset_index(drop=True) + assert len(trades_history1) == len(trades_history_df) * 3 + res = trades_df_remove_duplicates(trades_history1) + assert len(res) == len(trades_history_df) + assert res.equals(trades_history_df) def test_trades_dict_to_list(fetch_trades_result): diff --git a/tests/data/test_datahandler.py b/tests/data/test_datahandler.py index 5c7cd95b8..8415bea7c 100644 --- a/tests/data/test_datahandler.py +++ b/tests/data/test_datahandler.py @@ -6,7 +6,8 @@ from pathlib import Path from unittest.mock import MagicMock import pytest -from pandas import DataFrame +from pandas import DataFrame, Timestamp +from pandas.testing import assert_frame_equal from freqtrade.configuration import TimeRange from freqtrade.constants import AVAILABLE_DATAHANDLERS @@ -117,12 +118,6 @@ def test_datahandler_ohlcv_get_available_data(testdatadir): assert set(paircombs) == {('UNITTEST/BTC', '5m', CandleType.SPOT)} -def test_jsondatahandler_trades_get_pairs(testdatadir): - pairs = JsonGzDataHandler.trades_get_pairs(testdatadir) - # Convert to set to avoid failures due to sorting - assert set(pairs) == {'XRP/ETH', 'XRP/OLD'} - - def test_jsondatahandler_ohlcv_purge(mocker, testdatadir): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) unlinkmock = mocker.patch.object(Path, "unlink", MagicMock()) @@ -246,8 +241,10 @@ def test_datahandler__check_empty_df(testdatadir, caplog): assert log_has_re(expected_text, caplog) -@pytest.mark.parametrize('datahandler', ['parquet']) +# @pytest.mark.parametrize('datahandler', []) +@pytest.mark.skip("All datahandlers currently support trades data.") def test_datahandler_trades_not_supported(datahandler, testdatadir, ): + # Currently disabled. Reenable should a new provider not support trades data. dh = get_datahandler(testdatadir, datahandler) with pytest.raises(NotImplementedError): dh.trades_load('UNITTEST/ETH') @@ -266,18 +263,6 @@ def test_jsondatahandler_trades_load(testdatadir, caplog): assert log_has(logmsg, caplog) -def test_jsondatahandler_trades_purge(mocker, testdatadir): - mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - unlinkmock = mocker.patch.object(Path, "unlink", MagicMock()) - dh = JsonGzDataHandler(testdatadir) - assert not dh.trades_purge('UNITTEST/NONEXIST') - assert unlinkmock.call_count == 0 - - mocker.patch.object(Path, "exists", MagicMock(return_value=True)) - assert dh.trades_purge('UNITTEST/NONEXIST') - assert unlinkmock.call_count == 1 - - @pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS) def test_datahandler_ohlcv_append(datahandler, testdatadir, ): dh = get_datahandler(testdatadir, datahandler) @@ -291,79 +276,48 @@ def test_datahandler_ohlcv_append(datahandler, testdatadir, ): def test_datahandler_trades_append(datahandler, testdatadir): dh = get_datahandler(testdatadir, datahandler) with pytest.raises(NotImplementedError): - dh.trades_append('UNITTEST/ETH', []) + dh.trades_append('UNITTEST/ETH', DataFrame()) -def test_hdf5datahandler_trades_get_pairs(testdatadir): - pairs = HDF5DataHandler.trades_get_pairs(testdatadir) +@pytest.mark.parametrize('datahandler,expected', [ + ('jsongz', {'XRP/ETH', 'XRP/OLD'}), + ('hdf5', {'XRP/ETH'}), + ('feather', {'XRP/ETH'}), + ('parquet', {'XRP/ETH'}), +]) +def test_datahandler_trades_get_pairs(testdatadir, datahandler, expected): + + pairs = get_datahandlerclass(datahandler).trades_get_pairs(testdatadir) # Convert to set to avoid failures due to sorting - assert set(pairs) == {'XRP/ETH'} + assert set(pairs) == expected def test_hdf5datahandler_trades_load(testdatadir): dh = get_datahandler(testdatadir, 'hdf5') trades = dh.trades_load('XRP/ETH') - assert isinstance(trades, list) + assert isinstance(trades, DataFrame) trades1 = dh.trades_load('UNITTEST/NONEXIST') - assert trades1 == [] + assert isinstance(trades1, DataFrame) + assert trades1.empty # data goes from 2019-10-11 - 2019-10-13 timerange = TimeRange.parse_timerange('20191011-20191012') trades2 = dh._trades_load('XRP/ETH', timerange) assert len(trades) > len(trades2) # Check that ID is None (If it's nan, it's wrong) - assert trades2[0][2] is None + assert trades2.iloc[0]['type'] is None # unfiltered load has trades before starttime - assert len([t for t in trades if t[0] < timerange.startts * 1000]) >= 0 + + assert len(trades.loc[trades['timestamp'] < timerange.startts * 1000]) >= 0 # filtered list does not have trades before starttime - assert len([t for t in trades2 if t[0] < timerange.startts * 1000]) == 0 + assert len(trades2.loc[trades2['timestamp'] < timerange.startts * 1000]) == 0 # unfiltered load has trades after endtime - assert len([t for t in trades if t[0] > timerange.stopts * 1000]) > 0 + assert len(trades.loc[trades['timestamp'] > timerange.stopts * 1000]) >= 0 # filtered list does not have trades after endtime - assert len([t for t in trades2 if t[0] > timerange.stopts * 1000]) == 0 - - -def test_hdf5datahandler_trades_store(testdatadir, tmpdir): - tmpdir1 = Path(tmpdir) - dh = get_datahandler(testdatadir, 'hdf5') - trades = dh.trades_load('XRP/ETH') - - dh1 = get_datahandler(tmpdir1, 'hdf5') - dh1.trades_store('XRP/NEW', trades) - file = tmpdir1 / 'XRP_NEW-trades.h5' - assert file.is_file() - # Load trades back - trades_new = dh1.trades_load('XRP/NEW') - - assert len(trades_new) == len(trades) - assert trades[0][0] == trades_new[0][0] - assert trades[0][1] == trades_new[0][1] - # assert trades[0][2] == trades_new[0][2] # This is nan - so comparison does not make sense - assert trades[0][3] == trades_new[0][3] - assert trades[0][4] == trades_new[0][4] - assert trades[0][5] == trades_new[0][5] - assert trades[0][6] == trades_new[0][6] - assert trades[-1][0] == trades_new[-1][0] - assert trades[-1][1] == trades_new[-1][1] - # assert trades[-1][2] == trades_new[-1][2] # This is nan - so comparison does not make sense - assert trades[-1][3] == trades_new[-1][3] - assert trades[-1][4] == trades_new[-1][4] - assert trades[-1][5] == trades_new[-1][5] - assert trades[-1][6] == trades_new[-1][6] - - -def test_hdf5datahandler_trades_purge(mocker, testdatadir): - mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - unlinkmock = mocker.patch.object(Path, "unlink", MagicMock()) - dh = get_datahandler(testdatadir, 'hdf5') - assert not dh.trades_purge('UNITTEST/NONEXIST') - assert unlinkmock.call_count == 0 - - mocker.patch.object(Path, "exists", MagicMock(return_value=True)) - assert dh.trades_purge('UNITTEST/NONEXIST') - assert unlinkmock.call_count == 1 + assert len(trades2.loc[trades2['timestamp'] > timerange.stopts * 1000]) == 0 + # assert len([t for t in trades2 if t[0] > timerange.stopts * 1000]) == 0 @pytest.mark.parametrize('pair,timeframe,candle_type,candle_append,startdt,enddt', [ @@ -490,50 +444,42 @@ def test_hdf5datahandler_ohlcv_purge(mocker, testdatadir): assert unlinkmock.call_count == 2 -def test_featherdatahandler_trades_load(testdatadir): - dh = get_datahandler(testdatadir, 'feather') +@pytest.mark.parametrize('datahandler', ['jsongz', 'hdf5', 'feather', 'parquet']) +def test_datahandler_trades_load(testdatadir, datahandler): + dh = get_datahandler(testdatadir, datahandler) trades = dh.trades_load('XRP/ETH') - assert isinstance(trades, list) - assert trades[0][0] == 1570752011620 - assert trades[-1][-1] == 0.1986231 + assert isinstance(trades, DataFrame) + assert trades.iloc[0]['timestamp'] == 1570752011620 + assert trades.iloc[0]['date'] == Timestamp('2019-10-11 00:00:11.620000+0000') + assert trades.iloc[-1]['cost'] == 0.1986231 trades1 = dh.trades_load('UNITTEST/NONEXIST') - assert trades1 == [] + assert isinstance(trades, DataFrame) + assert trades1.empty -def test_featherdatahandler_trades_store(testdatadir, tmpdir): +@pytest.mark.parametrize('datahandler', ['jsongz', 'hdf5', 'feather', 'parquet']) +def test_datahandler_trades_store(testdatadir, tmpdir, datahandler): tmpdir1 = Path(tmpdir) - dh = get_datahandler(testdatadir, 'feather') + dh = get_datahandler(testdatadir, datahandler) trades = dh.trades_load('XRP/ETH') - dh1 = get_datahandler(tmpdir1, 'feather') + dh1 = get_datahandler(tmpdir1, datahandler) dh1.trades_store('XRP/NEW', trades) - file = tmpdir1 / 'XRP_NEW-trades.feather' + + file = tmpdir1 / f'XRP_NEW-trades.{dh1._get_file_extension()}' assert file.is_file() # Load trades back trades_new = dh1.trades_load('XRP/NEW') - + assert_frame_equal(trades, trades_new, check_exact=True) assert len(trades_new) == len(trades) - assert trades[0][0] == trades_new[0][0] - assert trades[0][1] == trades_new[0][1] - # assert trades[0][2] == trades_new[0][2] # This is nan - so comparison does not make sense - assert trades[0][3] == trades_new[0][3] - assert trades[0][4] == trades_new[0][4] - assert trades[0][5] == trades_new[0][5] - assert trades[0][6] == trades_new[0][6] - assert trades[-1][0] == trades_new[-1][0] - assert trades[-1][1] == trades_new[-1][1] - # assert trades[-1][2] == trades_new[-1][2] # This is nan - so comparison does not make sense - assert trades[-1][3] == trades_new[-1][3] - assert trades[-1][4] == trades_new[-1][4] - assert trades[-1][5] == trades_new[-1][5] - assert trades[-1][6] == trades_new[-1][6] -def test_featherdatahandler_trades_purge(mocker, testdatadir): +@pytest.mark.parametrize('datahandler', ['jsongz', 'hdf5', 'feather', 'parquet']) +def test_datahandler_trades_purge(mocker, testdatadir, datahandler): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) unlinkmock = mocker.patch.object(Path, "unlink", MagicMock()) - dh = get_datahandler(testdatadir, 'feather') + dh = get_datahandler(testdatadir, datahandler) assert not dh.trades_purge('UNITTEST/NONEXIST') assert unlinkmock.call_count == 0 diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index 4ff4f214b..1eeaffb2c 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -129,9 +129,14 @@ def test_get_pair_dataframe(mocker, default_conf, ohlcv_history, candle_type): default_conf["runmode"] = RunMode.BACKTEST dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.BACKTEST - assert isinstance(dp.get_pair_dataframe( - "UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame) - # assert dp.get_pair_dataframe("NONESENSE/AAA", timeframe).empty + df = dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candle_type) + assert isinstance(df, DataFrame) + assert len(df) == 3 # ohlcv_history mock has just 3 rows + + dp._set_dataframe_max_date(ohlcv_history.iloc[-1]['date']) + df = dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candle_type) + assert isinstance(df, DataFrame) + assert len(df) == 2 # ohlcv_history is limited to 2 rows now def test_available_pairs(mocker, default_conf, ohlcv_history): @@ -259,7 +264,7 @@ def test_orderbook(mocker, default_conf, order_book_l2): assert order_book_l2.call_args_list[0][0][0] == 'ETH/BTC' assert order_book_l2.call_args_list[0][0][1] >= 5 - assert type(res) is dict + assert isinstance(res, dict) assert 'bids' in res assert 'asks' in res @@ -272,7 +277,7 @@ def test_market(mocker, default_conf, markets): dp = DataProvider(default_conf, exchange) res = dp.market('ETH/BTC') - assert type(res) is dict + assert isinstance(res, dict) assert 'symbol' in res assert res['symbol'] == 'ETH/BTC' @@ -286,7 +291,7 @@ def test_ticker(mocker, default_conf, tickers): exchange = get_patched_exchange(mocker, default_conf) dp = DataProvider(default_conf, exchange) res = dp.ticker('ETH/BTC') - assert type(res) is dict + assert isinstance(res, dict) assert 'symbol' in res assert res['symbol'] == 'ETH/BTC' diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 417f984ca..84f00b8f4 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -3,6 +3,7 @@ import json import logging import uuid +from datetime import timedelta from pathlib import Path from shutil import copyfile from unittest.mock import MagicMock, PropertyMock @@ -26,7 +27,7 @@ from freqtrade.enums import CandleType from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json from freqtrade.resolvers import StrategyResolver -from freqtrade.util import dt_utc +from freqtrade.util import dt_ts, dt_utc from tests.conftest import (CURRENT_TEST_STRATEGY, EXMS, get_patched_exchange, log_has, log_has_re, patch_exchange) @@ -569,7 +570,10 @@ def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, tes def test_download_trades_history(trades_history, mocker, default_conf, testdatadir, caplog, - tmpdir) -> None: + tmpdir, time_machine) -> None: + start_dt = dt_utc(2023, 1, 1) + time_machine.move_to(start_dt, tick=False) + tmpdir1 = Path(tmpdir) ght_mock = MagicMock(side_effect=lambda pair, *args, **kwargs: (pair, trades_history)) mocker.patch(f'{EXMS}.get_historic_trades', ght_mock) @@ -581,8 +585,13 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad assert _download_trades_history(data_handler=data_handler, exchange=exchange, pair='ETH/BTC') - assert log_has("New Amount of trades: 5", caplog) + assert log_has("Current Amount of trades: 0", caplog) + assert log_has("New Amount of trades: 6", caplog) + assert ght_mock.call_count == 1 + # Default "since" - 30 days before current day. + assert ght_mock.call_args_list[0][1]['since'] == dt_ts(start_dt - timedelta(days=30)) assert file1.is_file() + caplog.clear() ght_mock.reset_mock() since_time = int(trades_history[-3][0] // 1000) @@ -599,6 +608,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad file1.unlink() mocker.patch(f'{EXMS}.get_historic_trades', MagicMock(side_effect=ValueError)) + caplog.clear() assert not _download_trades_history(data_handler=data_handler, exchange=exchange, pair='ETH/BTC') @@ -620,7 +630,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad assert int(ght_mock.call_args_list[0][1]['since'] // 1000) == since_time assert ght_mock.call_args_list[0][1]['from_id'] is None - assert log_has_re(r'Start earlier than available data. Redownloading trades for.*', caplog) + assert log_has_re(r'Start .* earlier than available data. Redownloading trades for.*', caplog) _clean_test_file(file2) @@ -651,10 +661,10 @@ def test_convert_trades_to_ohlcv(testdatadir, tmpdir, caplog): assert_frame_equal(dfbak_1m, df_1m, check_exact=True) assert_frame_equal(dfbak_5m, df_5m, check_exact=True) - - assert not log_has('Could not convert NoDatapair to OHLCV.', caplog) + msg = 'Could not convert NoDatapair to OHLCV.' + assert not log_has(msg, caplog) convert_trades_to_ohlcv(['NoDatapair'], timeframes=['1m', '5m'], data_format_trades='jsongz', datadir=tmpdir1, timerange=tr, erase=True) - assert log_has('Could not convert NoDatapair to OHLCV.', caplog) + assert log_has(msg, caplog) diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index 9018d2db9..c4e657ad9 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -35,7 +35,7 @@ def test__get_params_binance(default_conf, mocker, side, type, time_in_force, ex ]) def test_create_stoploss_order_binance(default_conf, mocker, limitratio, expected, side, trademode): api_mock = MagicMock() - order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_id = f'test_prod_buy_{randint(0, 10 ** 6)}' order_type = 'stop_loss_limit' if trademode == TradingMode.SPOT else 'stop' api_mock.create_order = MagicMock(return_value={ diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index a8c765553..adb8b7497 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -556,41 +556,6 @@ def test_get_min_pair_stake_amount_real_data(mocker, default_conf) -> None: assert result == 4000 -def test_set_sandbox(default_conf, mocker): - """ - Test working scenario - """ - api_mock = MagicMock() - api_mock.load_markets = MagicMock(return_value={ - 'ETH/BTC': '', 'LTC/BTC': '', 'XRP/BTC': '', 'NEO/BTC': '' - }) - url_mock = PropertyMock(return_value={'test': "api-public.sandbox.gdax.com", - 'api': 'https://api.gdax.com'}) - type(api_mock).urls = url_mock - exchange = get_patched_exchange(mocker, default_conf, api_mock) - liveurl = exchange._api.urls['api'] - default_conf['exchange']['sandbox'] = True - exchange.set_sandbox(exchange._api, default_conf['exchange'], 'Logname') - assert exchange._api.urls['api'] != liveurl - - -def test_set_sandbox_exception(default_conf, mocker): - """ - Test Fail scenario - """ - api_mock = MagicMock() - api_mock.load_markets = MagicMock(return_value={ - 'ETH/BTC': '', 'LTC/BTC': '', 'XRP/BTC': '', 'NEO/BTC': '' - }) - url_mock = PropertyMock(return_value={'api': 'https://api.gdax.com'}) - type(api_mock).urls = url_mock - - with pytest.raises(OperationalException, match=r'does not provide a sandbox api'): - exchange = get_patched_exchange(mocker, default_conf, api_mock) - default_conf['exchange']['sandbox'] = True - exchange.set_sandbox(exchange._api, default_conf['exchange'], 'Logname') - - def test__load_async_markets(default_conf, mocker, caplog): mocker.patch(f'{EXMS}._init_ccxt') mocker.patch(f'{EXMS}.validate_pairs') @@ -1372,7 +1337,7 @@ def test_create_dry_run_order_market_fill(default_conf, mocker, side, rate, amou @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_create_order(default_conf, mocker, side, ordertype, rate, marketprice, exchange_name): api_mock = MagicMock() - order_id = 'test_prod_{}_{}'.format(side, randint(0, 10 ** 6)) + order_id = f'test_prod_{side}_{randint(0, 10 ** 6)}' api_mock.options = {} if not marketprice else {"createMarketBuyOrderRequiresPrice": True} api_mock.create_order = MagicMock(return_value={ 'id': order_id, @@ -1452,7 +1417,7 @@ def test_buy_dry_run(default_conf, mocker, exchange_name): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_buy_prod(default_conf, mocker, exchange_name): api_mock = MagicMock() - order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_id = f'test_prod_buy_{randint(0, 10 ** 6)}' order_type = 'market' time_in_force = 'gtc' api_mock.options = {} @@ -1541,7 +1506,7 @@ def test_buy_prod(default_conf, mocker, exchange_name): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_buy_considers_time_in_force(default_conf, mocker, exchange_name): api_mock = MagicMock() - order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_id = f'test_prod_buy_{randint(0, 10 ** 6)}' api_mock.options = {} api_mock.create_order = MagicMock(return_value={ 'id': order_id, @@ -1608,7 +1573,7 @@ def test_sell_dry_run(default_conf, mocker): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_sell_prod(default_conf, mocker, exchange_name): api_mock = MagicMock() - order_id = 'test_prod_sell_{}'.format(randint(0, 10 ** 6)) + order_id = f'test_prod_sell_{randint(0, 10 ** 6)}' order_type = 'market' api_mock.options = {} api_mock.create_order = MagicMock(return_value={ @@ -1686,7 +1651,7 @@ def test_sell_prod(default_conf, mocker, exchange_name): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_sell_considers_time_in_force(default_conf, mocker, exchange_name): api_mock = MagicMock() - order_id = 'test_prod_sell_{}'.format(randint(0, 10 ** 6)) + order_id = f'test_prod_sell_{randint(0, 10 ** 6)}' api_mock.create_order = MagicMock(return_value={ 'id': order_id, 'symbol': 'ETH/BTC', @@ -2505,7 +2470,7 @@ def test_refresh_latest_ohlcv_inv_result(default_conf, mocker, caplog): assert exchange._klines assert exchange._api_async.fetch_ohlcv.call_count == 2 - assert type(res) is dict + assert isinstance(res, dict) assert len(res) == 1 # Test that each is in list at least once as order is not guaranteed assert log_has("Error loading ETH/BTC. Result was [[]].", caplog) @@ -2889,7 +2854,7 @@ async def test__async_fetch_trades(default_conf, mocker, caplog, exchange_name, pair = 'ETH/BTC' res = await exchange._async_fetch_trades(pair, since=None, params=None) - assert type(res) is list + assert isinstance(res, list) assert isinstance(res[0], list) assert isinstance(res[1], list) @@ -2989,9 +2954,9 @@ async def test__async_get_trade_history_id(default_conf, mocker, exchange_name, ret = await exchange._async_get_trade_history_id(pair, since=fetch_trades_result[0]['timestamp'], until=fetch_trades_result[-1]['timestamp'] - 1) - assert type(ret) is tuple + assert isinstance(ret, tuple) assert ret[0] == pair - assert type(ret[1]) is list + assert isinstance(ret[1], list) assert len(ret[1]) == len(fetch_trades_result) assert exchange._api_async.fetch_trades.call_count == 3 fetch_trades_cal = exchange._api_async.fetch_trades.call_args_list @@ -3027,9 +2992,9 @@ async def test__async_get_trade_history_time(default_conf, mocker, caplog, excha pair, since=fetch_trades_result[0]['timestamp'], until=fetch_trades_result[-1]['timestamp'] - 1) - assert type(ret) is tuple + assert isinstance(ret, tuple) assert ret[0] == pair - assert type(ret[1]) is list + assert isinstance(ret[1], list) assert len(ret[1]) == len(fetch_trades_result) assert exchange._api_async.fetch_trades.call_count == 2 fetch_trades_cal = exchange._api_async.fetch_trades.call_args_list @@ -3063,9 +3028,9 @@ async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog, pair = 'ETH/BTC' ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0][0], until=trades_history[-1][0] - 1) - assert type(ret) is tuple + assert isinstance(ret, tuple) assert ret[0] == pair - assert type(ret[1]) is list + assert isinstance(ret[1], list) assert len(ret[1]) == len(trades_history) - 1 assert exchange._async_fetch_trades.call_count == 2 fetch_trades_cal = exchange._async_fetch_trades.call_args_list @@ -3557,7 +3522,7 @@ def test_get_valid_pair_combination(default_conf, mocker, markets): assert ex.get_valid_pair_combination("ETH", "BTC") == "ETH/BTC" assert ex.get_valid_pair_combination("BTC", "ETH") == "ETH/BTC" - with pytest.raises(DependencyException, match=r"Could not combine.* to get a valid pair."): + with pytest.raises(ValueError, match=r"Could not combine.* to get a valid pair."): ex.get_valid_pair_combination("NOPAIR", "ETH") @@ -5392,7 +5357,7 @@ def test_get_liquidation_price( ]) def test_stoploss_contract_size(mocker, default_conf, contract_size, order_amount): api_mock = MagicMock() - order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_id = f'test_prod_buy_{randint(0, 10 ** 6)}' api_mock.create_order = MagicMock(return_value={ 'id': order_id, diff --git a/tests/exchange/test_huobi.py b/tests/exchange/test_huobi.py index 8be8ef8b3..b3f3c0900 100644 --- a/tests/exchange/test_huobi.py +++ b/tests/exchange/test_huobi.py @@ -16,7 +16,7 @@ from tests.exchange.test_exchange import ccxt_exceptionhandlers ]) def test_create_stoploss_order_huobi(default_conf, mocker, limitratio, expected, side): api_mock = MagicMock() - order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_id = f'test_prod_buy_{randint(0, 10 ** 6)}' order_type = 'stop-limit' api_mock.create_order = MagicMock(return_value={ diff --git a/tests/exchange/test_kraken.py b/tests/exchange/test_kraken.py index 8fc23b94e..7db3eeeeb 100644 --- a/tests/exchange/test_kraken.py +++ b/tests/exchange/test_kraken.py @@ -15,7 +15,7 @@ STOPLOSS_LIMIT_ORDERTYPE = 'stop-loss-limit' def test_buy_kraken_trading_agreement(default_conf, mocker): api_mock = MagicMock() - order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_id = f'test_prod_buy_{randint(0, 10 ** 6)}' order_type = 'limit' time_in_force = 'ioc' api_mock.options = {} @@ -56,7 +56,7 @@ def test_buy_kraken_trading_agreement(default_conf, mocker): def test_sell_kraken_trading_agreement(default_conf, mocker): api_mock = MagicMock() - order_id = 'test_prod_sell_{}'.format(randint(0, 10 ** 6)) + order_id = f'test_prod_sell_{randint(0, 10 ** 6)}' order_type = 'market' api_mock.options = {} api_mock.create_order = MagicMock(return_value={ @@ -181,7 +181,7 @@ def test_get_balances_prod(default_conf, mocker): ]) def test_create_stoploss_order_kraken(default_conf, mocker, ordertype, side, adjustedprice): api_mock = MagicMock() - order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_id = f'test_prod_buy_{randint(0, 10 ** 6)}' api_mock.create_order = MagicMock(return_value={ 'id': order_id, diff --git a/tests/exchange/test_kucoin.py b/tests/exchange/test_kucoin.py index 741ee27be..a74b77859 100644 --- a/tests/exchange/test_kucoin.py +++ b/tests/exchange/test_kucoin.py @@ -17,7 +17,7 @@ from tests.exchange.test_exchange import ccxt_exceptionhandlers ]) def test_create_stoploss_order_kucoin(default_conf, mocker, limitratio, expected, side, order_type): api_mock = MagicMock() - order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_id = f'test_prod_buy_{randint(0, 10 ** 6)}' api_mock.create_order = MagicMock(return_value={ 'id': order_id, @@ -136,7 +136,7 @@ def test_stoploss_adjust_kucoin(mocker, default_conf): ]) def test_kucoin_create_order(default_conf, mocker, side, ordertype, rate): api_mock = MagicMock() - order_id = 'test_prod_{}_{}'.format(side, randint(0, 10 ** 6)) + order_id = f'test_prod_{side}_{randint(0, 10 ** 6)}' api_mock.create_order = MagicMock(return_value={ 'id': order_id, 'info': { diff --git a/tests/exchange_online/__init__.py b/tests/exchange_online/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/exchange_online/conftest.py b/tests/exchange_online/conftest.py new file mode 100644 index 000000000..45da16dd5 --- /dev/null +++ b/tests/exchange_online/conftest.py @@ -0,0 +1,334 @@ +from copy import deepcopy +from pathlib import Path +from typing import Tuple + +import pytest + +from freqtrade.constants import Config +from freqtrade.exchange.exchange import Exchange +from freqtrade.resolvers.exchange_resolver import ExchangeResolver +from tests.conftest import EXMS, get_default_conf_usdt + + +EXCHANGE_FIXTURE_TYPE = Tuple[Exchange, str] + +# Exchanges that should be tested online +EXCHANGES = { + 'bittrex': { + 'pair': 'BTC/USDT', + 'stake_currency': 'USDT', + 'hasQuoteVolume': False, + 'timeframe': '1h', + 'leverage_tiers_public': False, + 'leverage_in_spot_market': False, + }, + 'binance': { + 'pair': 'BTC/USDT', + 'stake_currency': 'USDT', + 'use_ci_proxy': True, + 'hasQuoteVolume': True, + 'timeframe': '1h', + 'futures': True, + 'futures_pair': 'BTC/USDT:USDT', + 'hasQuoteVolumeFutures': True, + 'leverage_tiers_public': False, + 'leverage_in_spot_market': False, + 'trades_lookback_hours': 4, + 'private_methods': [ + 'fapiPrivateGetPositionSideDual', + 'fapiPrivateGetMultiAssetsMargin' + ], + 'sample_order': [{ + "symbol": "SOLUSDT", + "orderId": 3551312894, + "orderListId": -1, + "clientOrderId": "x-R4DD3S8297c73a11ccb9dc8f2811ba", + "transactTime": 1674493798550, + "price": "15.50000000", + "origQty": "1.10000000", + "executedQty": "0.00000000", + "cummulativeQuoteQty": "0.00000000", + "status": "NEW", + "timeInForce": "GTC", + "type": "LIMIT", + "side": "BUY", + "workingTime": 1674493798550, + "fills": [], + "selfTradePreventionMode": "NONE", + }] + }, + 'binanceus': { + 'pair': 'BTC/USDT', + 'stake_currency': 'USDT', + 'hasQuoteVolume': True, + 'timeframe': '1h', + 'futures': False, + 'sample_order': [{ + "symbol": "SOLUSDT", + "orderId": 3551312894, + "orderListId": -1, + "clientOrderId": "x-R4DD3S8297c73a11ccb9dc8f2811ba", + "transactTime": 1674493798550, + "price": "15.50000000", + "origQty": "1.10000000", + "executedQty": "0.00000000", + "cummulativeQuoteQty": "0.00000000", + "status": "NEW", + "timeInForce": "GTC", + "type": "LIMIT", + "side": "BUY", + "workingTime": 1674493798550, + "fills": [], + "selfTradePreventionMode": "NONE", + }] + }, + 'kraken': { + 'pair': 'BTC/USD', + 'stake_currency': 'USD', + 'hasQuoteVolume': True, + 'timeframe': '1h', + 'leverage_tiers_public': False, + 'leverage_in_spot_market': True, + 'trades_lookback_hours': 12, + }, + 'kucoin': { + 'pair': 'XRP/USDT', + 'stake_currency': 'USDT', + 'hasQuoteVolume': True, + 'timeframe': '1h', + 'leverage_tiers_public': False, + 'leverage_in_spot_market': True, + 'sample_order': [ + {'id': '63d6742d0adc5570001d2bbf7'}, # create order + { + 'id': '63d6742d0adc5570001d2bbf7', + 'symbol': 'SOL-USDT', + 'opType': 'DEAL', + 'type': 'limit', + 'side': 'buy', + 'price': '15.5', + 'size': '1.1', + 'funds': '0', + 'dealFunds': '17.05', + 'dealSize': '1.1', + 'fee': '0.000065252', + 'feeCurrency': 'USDT', + 'stp': '', + 'stop': '', + 'stopTriggered': False, + 'stopPrice': '0', + 'timeInForce': 'GTC', + 'postOnly': False, + 'hidden': False, + 'iceberg': False, + 'visibleSize': '0', + 'cancelAfter': 0, + 'channel': 'API', + 'clientOid': '0a053870-11bf-41e5-be61-b272a4cb62e1', + 'remark': None, + 'tags': 'partner:ccxt', + 'isActive': False, + 'cancelExist': False, + 'createdAt': 1674493798550, + 'tradeType': 'TRADE' + }], + }, + 'gate': { + 'pair': 'BTC/USDT', + 'stake_currency': 'USDT', + 'hasQuoteVolume': True, + 'timeframe': '1h', + 'futures': True, + 'futures_pair': 'BTC/USDT:USDT', + 'hasQuoteVolumeFutures': True, + 'leverage_tiers_public': True, + 'leverage_in_spot_market': True, + 'sample_order': [ + { + "id": "276266139423", + "text": "apiv4", + "create_time": "1674493798", + "update_time": "1674493798", + "create_time_ms": "1674493798550", + "update_time_ms": "1674493798550", + "status": "closed", + "currency_pair": "SOL_USDT", + "type": "limit", + "account": "spot", + "side": "buy", + "amount": "1.1", + "price": "15.5", + "time_in_force": "gtc", + "iceberg": "0", + "left": "0", + "fill_price": "17.05", + "filled_total": "17.05", + "avg_deal_price": "15.5", + "fee": "0.0000018", + "fee_currency": "SOL", + "point_fee": "0", + "gt_fee": "0", + "gt_maker_fee": "0", + "gt_taker_fee": "0.0015", + "gt_discount": True, + "rebated_fee": "0", + "rebated_fee_currency": "USDT" + }, + { + # market order + 'id': '276401180529', + 'text': 'apiv4', + 'create_time': '1674493798', + 'update_time': '1674493798', + 'create_time_ms': '1674493798550', + 'update_time_ms': '1674493798550', + 'status': 'cancelled', + 'currency_pair': 'SOL_USDT', + 'type': 'market', + 'account': 'spot', + 'side': 'buy', + 'amount': '17.05', + 'price': '0', + 'time_in_force': 'ioc', + 'iceberg': '0', + 'left': '0.0000000016228', + 'fill_price': '17.05', + 'filled_total': '17.05', + 'avg_deal_price': '15.5', + 'fee': '0', + 'fee_currency': 'SOL', + 'point_fee': '0.0199999999967544', + 'gt_fee': '0', + 'gt_maker_fee': '0', + 'gt_taker_fee': '0', + 'gt_discount': False, + 'rebated_fee': '0', + 'rebated_fee_currency': 'USDT' + } + ], + }, + 'okx': { + 'pair': 'BTC/USDT', + 'stake_currency': 'USDT', + 'hasQuoteVolume': True, + 'timeframe': '1h', + 'futures': True, + 'futures_pair': 'BTC/USDT:USDT', + 'hasQuoteVolumeFutures': False, + 'leverage_tiers_public': True, + 'leverage_in_spot_market': True, + 'private_methods': ['fetch_accounts'], + }, + 'bybit': { + 'pair': 'BTC/USDT', + 'stake_currency': 'USDT', + 'hasQuoteVolume': True, + 'use_ci_proxy': True, + 'timeframe': '1h', + 'futures_pair': 'BTC/USDT:USDT', + 'futures': True, + 'leverage_tiers_public': True, + 'leverage_in_spot_market': True, + 'sample_order': [ + { + "orderId": "1274754916287346280", + "orderLinkId": "1666798627015730", + "symbol": "SOLUSDT", + "createTime": "1674493798550", + "orderPrice": "15.5", + "orderQty": "1.1", + "orderType": "LIMIT", + "side": "BUY", + "status": "NEW", + "timeInForce": "GTC", + "accountId": "5555555", + "execQty": "0", + "orderCategory": "0" + } + ] + }, + 'huobi': { + 'pair': 'ETH/BTC', + 'stake_currency': 'BTC', + 'hasQuoteVolume': True, + 'timeframe': '1h', + 'futures': False, + }, + 'bitvavo': { + 'pair': 'BTC/EUR', + 'stake_currency': 'EUR', + 'hasQuoteVolume': True, + 'timeframe': '1h', + 'leverage_tiers_public': False, + 'leverage_in_spot_market': False, + }, +} + + +@pytest.fixture(scope="class") +def exchange_conf(): + config = get_default_conf_usdt((Path(__file__).parent / "testdata").resolve()) + config['exchange']['pair_whitelist'] = [] + config['exchange']['key'] = '' + config['exchange']['secret'] = '' + config['dry_run'] = False + config['entry_pricing']['use_order_book'] = True + config['exit_pricing']['use_order_book'] = True + return config + + +def set_test_proxy(config: Config, use_proxy: bool) -> Config: + # Set proxy to test in CI. + import os + if use_proxy and (proxy := os.environ.get('CI_WEB_PROXY')): + config1 = deepcopy(config) + config1['exchange']['ccxt_config'] = { + "httpsProxy": proxy, + } + return config1 + + return config + + +def get_exchange(exchange_name, exchange_conf): + exchange_conf = set_test_proxy( + exchange_conf, EXCHANGES[exchange_name].get('use_ci_proxy', False)) + exchange_conf['exchange']['name'] = exchange_name + exchange_conf['stake_currency'] = EXCHANGES[exchange_name]['stake_currency'] + exchange = ExchangeResolver.load_exchange(exchange_conf, validate=True, + load_leverage_tiers=True) + + yield exchange, exchange_name + + +def get_futures_exchange(exchange_name, exchange_conf, class_mocker): + if EXCHANGES[exchange_name].get('futures') is not True: + pytest.skip(f"Exchange {exchange_name} does not support futures.") + else: + exchange_conf = deepcopy(exchange_conf) + exchange_conf = set_test_proxy( + exchange_conf, EXCHANGES[exchange_name].get('use_ci_proxy', False)) + exchange_conf['trading_mode'] = 'futures' + exchange_conf['margin_mode'] = 'isolated' + + class_mocker.patch( + 'freqtrade.exchange.binance.Binance.fill_leverage_tiers') + class_mocker.patch(f'{EXMS}.fetch_trading_fees') + class_mocker.patch('freqtrade.exchange.okx.Okx.additional_exchange_init') + class_mocker.patch('freqtrade.exchange.binance.Binance.additional_exchange_init') + class_mocker.patch('freqtrade.exchange.bybit.Bybit.additional_exchange_init') + class_mocker.patch(f'{EXMS}.load_cached_leverage_tiers', return_value=None) + class_mocker.patch(f'{EXMS}.cache_leverage_tiers') + + yield from get_exchange(exchange_name, exchange_conf) + + +@pytest.fixture(params=EXCHANGES, scope="class") +def exchange(request, exchange_conf): + yield from get_exchange(request.param, exchange_conf) + + +@pytest.fixture(params=EXCHANGES, scope="class") +def exchange_futures(request, exchange_conf, class_mocker): + + yield from get_futures_exchange(request.param, exchange_conf, class_mocker) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange_online/test_ccxt_compat.py similarity index 59% rename from tests/exchange/test_ccxt_compat.py rename to tests/exchange_online/test_ccxt_compat.py index 2c370145d..e33875416 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange_online/test_ccxt_compat.py @@ -5,338 +5,14 @@ However, these tests should give a good idea to determine if a new exchange is suitable to run with freqtrade. """ -from copy import deepcopy from datetime import datetime, timedelta, timezone -from pathlib import Path -from typing import Tuple import pytest -from freqtrade.constants import Config from freqtrade.enums import CandleType from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date -from freqtrade.exchange.exchange import Exchange, timeframe_to_msecs -from freqtrade.resolvers.exchange_resolver import ExchangeResolver -from tests.conftest import EXMS, get_default_conf_usdt - - -EXCHANGE_FIXTURE_TYPE = Tuple[Exchange, str] - -# Exchanges that should be tested -EXCHANGES = { - 'bittrex': { - 'pair': 'BTC/USDT', - 'stake_currency': 'USDT', - 'hasQuoteVolume': False, - 'timeframe': '1h', - 'leverage_tiers_public': False, - 'leverage_in_spot_market': False, - }, - 'binance': { - 'pair': 'BTC/USDT', - 'stake_currency': 'USDT', - 'use_ci_proxy': True, - 'hasQuoteVolume': True, - 'timeframe': '1h', - 'futures': True, - 'futures_pair': 'BTC/USDT:USDT', - 'hasQuoteVolumeFutures': True, - 'leverage_tiers_public': False, - 'leverage_in_spot_market': False, - 'trades_lookback_hours': 4, - 'private_methods': [ - 'fapiPrivateGetPositionSideDual', - 'fapiPrivateGetMultiAssetsMargin' - ], - 'sample_order': [{ - "symbol": "SOLUSDT", - "orderId": 3551312894, - "orderListId": -1, - "clientOrderId": "x-R4DD3S8297c73a11ccb9dc8f2811ba", - "transactTime": 1674493798550, - "price": "15.50000000", - "origQty": "1.10000000", - "executedQty": "0.00000000", - "cummulativeQuoteQty": "0.00000000", - "status": "NEW", - "timeInForce": "GTC", - "type": "LIMIT", - "side": "BUY", - "workingTime": 1674493798550, - "fills": [], - "selfTradePreventionMode": "NONE", - }] - }, - 'binanceus': { - 'pair': 'BTC/USDT', - 'stake_currency': 'USDT', - 'hasQuoteVolume': True, - 'timeframe': '1h', - 'futures': False, - 'sample_order': [{ - "symbol": "SOLUSDT", - "orderId": 3551312894, - "orderListId": -1, - "clientOrderId": "x-R4DD3S8297c73a11ccb9dc8f2811ba", - "transactTime": 1674493798550, - "price": "15.50000000", - "origQty": "1.10000000", - "executedQty": "0.00000000", - "cummulativeQuoteQty": "0.00000000", - "status": "NEW", - "timeInForce": "GTC", - "type": "LIMIT", - "side": "BUY", - "workingTime": 1674493798550, - "fills": [], - "selfTradePreventionMode": "NONE", - }] - }, - 'kraken': { - 'pair': 'BTC/USD', - 'stake_currency': 'USD', - 'hasQuoteVolume': True, - 'timeframe': '1h', - 'leverage_tiers_public': False, - 'leverage_in_spot_market': True, - 'trades_lookback_hours': 12, - }, - 'kucoin': { - 'pair': 'XRP/USDT', - 'stake_currency': 'USDT', - 'hasQuoteVolume': True, - 'timeframe': '1h', - 'leverage_tiers_public': False, - 'leverage_in_spot_market': True, - 'sample_order': [ - {'id': '63d6742d0adc5570001d2bbf7'}, # create order - { - 'id': '63d6742d0adc5570001d2bbf7', - 'symbol': 'SOL-USDT', - 'opType': 'DEAL', - 'type': 'limit', - 'side': 'buy', - 'price': '15.5', - 'size': '1.1', - 'funds': '0', - 'dealFunds': '17.05', - 'dealSize': '1.1', - 'fee': '0.000065252', - 'feeCurrency': 'USDT', - 'stp': '', - 'stop': '', - 'stopTriggered': False, - 'stopPrice': '0', - 'timeInForce': 'GTC', - 'postOnly': False, - 'hidden': False, - 'iceberg': False, - 'visibleSize': '0', - 'cancelAfter': 0, - 'channel': 'API', - 'clientOid': '0a053870-11bf-41e5-be61-b272a4cb62e1', - 'remark': None, - 'tags': 'partner:ccxt', - 'isActive': False, - 'cancelExist': False, - 'createdAt': 1674493798550, - 'tradeType': 'TRADE' - }], - }, - 'gate': { - 'pair': 'BTC/USDT', - 'stake_currency': 'USDT', - 'hasQuoteVolume': True, - 'timeframe': '1h', - 'futures': True, - 'futures_pair': 'BTC/USDT:USDT', - 'hasQuoteVolumeFutures': True, - 'leverage_tiers_public': True, - 'leverage_in_spot_market': True, - 'sample_order': [ - { - "id": "276266139423", - "text": "apiv4", - "create_time": "1674493798", - "update_time": "1674493798", - "create_time_ms": "1674493798550", - "update_time_ms": "1674493798550", - "status": "closed", - "currency_pair": "SOL_USDT", - "type": "limit", - "account": "spot", - "side": "buy", - "amount": "1.1", - "price": "15.5", - "time_in_force": "gtc", - "iceberg": "0", - "left": "0", - "fill_price": "17.05", - "filled_total": "17.05", - "avg_deal_price": "15.5", - "fee": "0.0000018", - "fee_currency": "SOL", - "point_fee": "0", - "gt_fee": "0", - "gt_maker_fee": "0", - "gt_taker_fee": "0.0015", - "gt_discount": True, - "rebated_fee": "0", - "rebated_fee_currency": "USDT" - }, - { - # market order - 'id': '276401180529', - 'text': 'apiv4', - 'create_time': '1674493798', - 'update_time': '1674493798', - 'create_time_ms': '1674493798550', - 'update_time_ms': '1674493798550', - 'status': 'cancelled', - 'currency_pair': 'SOL_USDT', - 'type': 'market', - 'account': 'spot', - 'side': 'buy', - 'amount': '17.05', - 'price': '0', - 'time_in_force': 'ioc', - 'iceberg': '0', - 'left': '0.0000000016228', - 'fill_price': '17.05', - 'filled_total': '17.05', - 'avg_deal_price': '15.5', - 'fee': '0', - 'fee_currency': 'SOL', - 'point_fee': '0.0199999999967544', - 'gt_fee': '0', - 'gt_maker_fee': '0', - 'gt_taker_fee': '0', - 'gt_discount': False, - 'rebated_fee': '0', - 'rebated_fee_currency': 'USDT' - } - ], - }, - 'okx': { - 'pair': 'BTC/USDT', - 'stake_currency': 'USDT', - 'hasQuoteVolume': True, - 'timeframe': '1h', - 'futures': True, - 'futures_pair': 'BTC/USDT:USDT', - 'hasQuoteVolumeFutures': False, - 'leverage_tiers_public': True, - 'leverage_in_spot_market': True, - 'private_methods': ['fetch_accounts'], - }, - 'bybit': { - 'pair': 'BTC/USDT', - 'stake_currency': 'USDT', - 'hasQuoteVolume': True, - 'use_ci_proxy': True, - 'timeframe': '1h', - 'futures_pair': 'BTC/USDT:USDT', - 'futures': True, - 'leverage_tiers_public': True, - 'leverage_in_spot_market': True, - 'sample_order': [ - { - "orderId": "1274754916287346280", - "orderLinkId": "1666798627015730", - "symbol": "SOLUSDT", - "createTime": "1674493798550", - "orderPrice": "15.5", - "orderQty": "1.1", - "orderType": "LIMIT", - "side": "BUY", - "status": "NEW", - "timeInForce": "GTC", - "accountId": "5555555", - "execQty": "0", - "orderCategory": "0" - } - ] - }, - 'huobi': { - 'pair': 'ETH/BTC', - 'stake_currency': 'BTC', - 'hasQuoteVolume': True, - 'timeframe': '1h', - 'futures': False, - }, - 'bitvavo': { - 'pair': 'BTC/EUR', - 'stake_currency': 'EUR', - 'hasQuoteVolume': True, - 'timeframe': '1h', - 'leverage_tiers_public': False, - 'leverage_in_spot_market': False, - }, -} - - -@pytest.fixture(scope="class") -def exchange_conf(): - config = get_default_conf_usdt((Path(__file__).parent / "testdata").resolve()) - config['exchange']['pair_whitelist'] = [] - config['exchange']['key'] = '' - config['exchange']['secret'] = '' - config['dry_run'] = False - config['entry_pricing']['use_order_book'] = True - config['exit_pricing']['use_order_book'] = True - return config - - -def set_test_proxy(config: Config, use_proxy: bool) -> Config: - # Set proxy to test in CI. - import os - if use_proxy and (proxy := os.environ.get('CI_WEB_PROXY')): - config1 = deepcopy(config) - config1['exchange']['ccxt_config'] = { - "httpsProxy": proxy, - } - return config1 - - return config - - -@pytest.fixture(params=EXCHANGES, scope="class") -def exchange(request, exchange_conf): - exchange_conf = set_test_proxy( - exchange_conf, EXCHANGES[request.param].get('use_ci_proxy', False)) - exchange_conf['exchange']['name'] = request.param - exchange_conf['stake_currency'] = EXCHANGES[request.param]['stake_currency'] - exchange = ExchangeResolver.load_exchange(exchange_conf, validate=True) - - yield exchange, request.param - - -@pytest.fixture(params=EXCHANGES, scope="class") -def exchange_futures(request, exchange_conf, class_mocker): - if EXCHANGES[request.param].get('futures') is not True: - yield None, request.param - else: - exchange_conf = set_test_proxy( - exchange_conf, EXCHANGES[request.param].get('use_ci_proxy', False)) - exchange_conf = deepcopy(exchange_conf) - exchange_conf['exchange']['name'] = request.param - exchange_conf['trading_mode'] = 'futures' - exchange_conf['margin_mode'] = 'isolated' - exchange_conf['stake_currency'] = EXCHANGES[request.param]['stake_currency'] - - class_mocker.patch( - 'freqtrade.exchange.binance.Binance.fill_leverage_tiers') - class_mocker.patch(f'{EXMS}.fetch_trading_fees') - class_mocker.patch('freqtrade.exchange.okx.Okx.additional_exchange_init') - class_mocker.patch('freqtrade.exchange.binance.Binance.additional_exchange_init') - class_mocker.patch('freqtrade.exchange.bybit.Bybit.additional_exchange_init') - class_mocker.patch(f'{EXMS}.load_cached_leverage_tiers', return_value=None) - class_mocker.patch(f'{EXMS}.cache_leverage_tiers') - - exchange = ExchangeResolver.load_exchange( - exchange_conf, validate=True, load_leverage_tiers=True) - - yield exchange, request.param +from freqtrade.exchange.exchange import timeframe_to_msecs +from tests.exchange_online.conftest import EXCHANGE_FIXTURE_TYPE, EXCHANGES @pytest.mark.longrun @@ -371,9 +47,6 @@ class TestCCXTExchange: def test_load_markets_futures(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): exchange, exchangename = exchange_futures - if not exchange: - # exchange_futures only returns values for supported exchanges - return pair = EXCHANGES[exchangename]['pair'] pair = EXCHANGES[exchangename].get('futures_pair', pair) markets = exchange.markets @@ -561,9 +234,6 @@ class TestCCXTExchange: def test_ccxt__async_get_candle_history_futures( self, exchange_futures: EXCHANGE_FIXTURE_TYPE, candle_type): exchange, exchangename = exchange_futures - if not exchange: - # exchange_futures only returns values for supported exchanges - return pair = EXCHANGES[exchangename].get('futures_pair', EXCHANGES[exchangename]['pair']) timeframe = EXCHANGES[exchangename]['timeframe'] if candle_type == CandleType.FUNDING_RATE: @@ -579,9 +249,6 @@ class TestCCXTExchange: def test_ccxt_fetch_funding_rate_history(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): exchange, exchangename = exchange_futures - if not exchange: - # exchange_futures only returns values for supported exchanges - return pair = EXCHANGES[exchangename].get('futures_pair', EXCHANGES[exchangename]['pair']) since = int((datetime.now(timezone.utc) - timedelta(days=5)).timestamp() * 1000) @@ -617,9 +284,6 @@ class TestCCXTExchange: def test_ccxt_fetch_mark_price_history(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): exchange, exchangename = exchange_futures - if not exchange: - # exchange_futures only returns values for supported exchanges - return pair = EXCHANGES[exchangename].get('futures_pair', EXCHANGES[exchangename]['pair']) since = int((datetime.now(timezone.utc) - timedelta(days=5)).timestamp() * 1000) pair_tf = (pair, '1h', CandleType.MARK) @@ -641,9 +305,6 @@ class TestCCXTExchange: def test_ccxt__calculate_funding_fees(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): exchange, exchangename = exchange_futures - if not exchange: - # exchange_futures only returns values for supported exchanges - return pair = EXCHANGES[exchangename].get('futures_pair', EXCHANGES[exchangename]['pair']) since = datetime.now(timezone.utc) - timedelta(days=5) @@ -690,31 +351,29 @@ class TestCCXTExchange: def test_ccxt_get_max_leverage_futures(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): futures, futures_name = exchange_futures - if futures: - leverage_tiers_public = EXCHANGES[futures_name].get('leverage_tiers_public') - if leverage_tiers_public: - futures_pair = EXCHANGES[futures_name].get( - 'futures_pair', - EXCHANGES[futures_name]['pair'] - ) - futures_leverage = futures.get_max_leverage(futures_pair, 20) - assert (isinstance(futures_leverage, float) or isinstance(futures_leverage, int)) - assert futures_leverage >= 1.0 - - def test_ccxt_get_contract_size(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): - futures, futures_name = exchange_futures - if futures: + leverage_tiers_public = EXCHANGES[futures_name].get('leverage_tiers_public') + if leverage_tiers_public: futures_pair = EXCHANGES[futures_name].get( 'futures_pair', EXCHANGES[futures_name]['pair'] ) - contract_size = futures.get_contract_size(futures_pair) - assert (isinstance(contract_size, float) or isinstance(contract_size, int)) - assert contract_size >= 0.0 + futures_leverage = futures.get_max_leverage(futures_pair, 20) + assert (isinstance(futures_leverage, float) or isinstance(futures_leverage, int)) + assert futures_leverage >= 1.0 + + def test_ccxt_get_contract_size(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): + futures, futures_name = exchange_futures + futures_pair = EXCHANGES[futures_name].get( + 'futures_pair', + EXCHANGES[futures_name]['pair'] + ) + contract_size = futures.get_contract_size(futures_pair) + assert (isinstance(contract_size, float) or isinstance(contract_size, int)) + assert contract_size >= 0.0 def test_ccxt_load_leverage_tiers(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): futures, futures_name = exchange_futures - if futures and EXCHANGES[futures_name].get('leverage_tiers_public'): + if EXCHANGES[futures_name].get('leverage_tiers_public'): leverage_tiers = futures.load_leverage_tiers() futures_pair = EXCHANGES[futures_name].get( 'futures_pair', @@ -747,7 +406,7 @@ class TestCCXTExchange: def test_ccxt_dry_run_liquidation_price(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): futures, futures_name = exchange_futures - if futures and EXCHANGES[futures_name].get('leverage_tiers_public'): + if EXCHANGES[futures_name].get('leverage_tiers_public'): futures_pair = EXCHANGES[futures_name].get( 'futures_pair', @@ -780,14 +439,13 @@ class TestCCXTExchange: def test_ccxt_get_max_pair_stake_amount(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): futures, futures_name = exchange_futures - if futures: - futures_pair = EXCHANGES[futures_name].get( - 'futures_pair', - EXCHANGES[futures_name]['pair'] - ) - max_stake_amount = futures.get_max_pair_stake_amount(futures_pair, 40000) - assert (isinstance(max_stake_amount, float)) - assert max_stake_amount >= 0.0 + futures_pair = EXCHANGES[futures_name].get( + 'futures_pair', + EXCHANGES[futures_name]['pair'] + ) + max_stake_amount = futures.get_max_pair_stake_amount(futures_pair, 40000) + assert (isinstance(max_stake_amount, float)) + assert max_stake_amount >= 0.0 def test_private_method_presence(self, exchange: EXCHANGE_FIXTURE_TYPE): exch, exchangename = exchange diff --git a/tests/freqai/conftest.py b/tests/freqai/conftest.py index 4c4891ceb..9c7a950e7 100644 --- a/tests/freqai/conftest.py +++ b/tests/freqai/conftest.py @@ -97,9 +97,9 @@ def mock_pytorch_mlp_model_training_parameters() -> Dict[str, Any]: return { "learning_rate": 3e-4, "trainer_kwargs": { - "max_iters": 1, + "n_steps": None, "batch_size": 64, - "max_n_eval_batches": 1, + "n_epochs": 1, }, "model_kwargs": { "hidden_dim": 32, diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index f0bd18768..ac409bf71 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -20,7 +20,7 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import get_timerange from freqtrade.enums import CandleType, ExitType, RunMode from freqtrade.exceptions import DependencyException, OperationalException -from freqtrade.exchange.exchange import timeframe_to_next_date +from freqtrade.exchange import timeframe_to_next_date, timeframe_to_prev_date from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename, get_strategy_run_id from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import LocalTrade, Trade @@ -1122,10 +1122,10 @@ def test_backtest_dataprovider_analyzed_df(default_conf, fee, mocker, testdatadi processed = backtesting.strategy.advise_all_indicators(data) min_date, max_date = get_timerange(processed) - global count count = 0 def tmp_confirm_entry(pair, current_time, **kwargs): + nonlocal count dp = backtesting.strategy.dp df, _ = dp.get_analyzed_dataframe(pair, backtesting.strategy.timeframe) current_candle = df.iloc[-1].squeeze() @@ -1135,8 +1135,13 @@ def test_backtest_dataprovider_analyzed_df(default_conf, fee, mocker, testdatadi assert candle_date == current_time # These asserts don't properly raise as they are nested, # therefore we increment count and assert for that. - global count - count = count + 1 + df = dp.get_pair_dataframe(pair, backtesting.strategy.timeframe) + prior_time = timeframe_to_prev_date(backtesting.strategy.timeframe, + candle_date - timedelta(seconds=1)) + assert prior_time == df.iloc[-1].squeeze()['date'] + assert df.iloc[-1].squeeze()['date'] < current_time + + count += 1 backtesting.strategy.confirm_trade_entry = tmp_confirm_entry backtesting.backtest( @@ -1354,11 +1359,11 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) # Cached data correctly removed amounts offset = 1 if tres == 0 else 0 - removed_candles = len(data[pair]) - offset - backtesting.strategy.startup_candle_count + removed_candles = len(data[pair]) - offset assert len(backtesting.dataprovider.get_analyzed_dataframe(pair, '5m')[0]) == removed_candles assert len( backtesting.dataprovider.get_analyzed_dataframe('NXT/BTC', '5m')[0] - ) == len(data['NXT/BTC']) - 1 - backtesting.strategy.startup_candle_count + ) == len(data['NXT/BTC']) - 1 backtesting.strategy.max_open_trades = 1 backtesting.config.update({'max_open_trades': 1}) diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 3c6a5ad6d..decc4706d 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -17,6 +17,8 @@ from tests.conftest import EXMS, get_args, log_has_re, patch_exchange def lookahead_conf(default_conf_usdt): default_conf_usdt['minimum_trade_amount'] = 10 default_conf_usdt['targeted_trade_amount'] = 20 + default_conf_usdt['timerange'] = '20220101-20220501' + default_conf_usdt['strategy_path'] = str( Path(__file__).parent.parent / "strategy/strats/lookahead_bias") default_conf_usdt['strategy'] = 'strategy_test_v3_with_lookahead_bias' @@ -43,7 +45,9 @@ def test_start_lookahead_analysis(mocker): "--pairs", "UNITTEST/BTC", "--max-open-trades", - "1" + "1", + "--timerange", + "20220101-20220201" ] pargs = get_args(args) pargs['config'] = None @@ -72,6 +76,24 @@ def test_start_lookahead_analysis(mocker): match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): start_lookahead_analysis(pargs) + # Missing timerange + args = [ + "lookahead-analysis", + "--strategy", + "strategy_test_v3_with_lookahead_bias", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), + "--pairs", + "UNITTEST/BTC", + "--max-open-trades", + "1", + ] + pargs = get_args(args) + pargs['config'] = None + with pytest.raises(OperationalException, + match=r"Please set a timerange\..*"): + start_lookahead_analysis(pargs) + def test_lookahead_helper_invalid_config(lookahead_conf) -> None: conf = deepcopy(lookahead_conf) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index cbbf1e6da..369a7f223 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -553,7 +553,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t assert isinstance(whitelist, list) # Verify length of pairlist matches (used for ShuffleFilter without seed) - if type(whitelist_result) is list: + if isinstance(whitelist_result, list): assert whitelist == whitelist_result else: len(whitelist) == whitelist_result diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index d89bbbff9..f3762fbc9 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1424,12 +1424,12 @@ def test_api_pair_candles(botclient, ohlcv_history): assert len(rc.json()['data']) == amount assert (rc.json()['data'] == - [['2017-11-26 08:50:00', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, 0.0877869, + [['2017-11-26T08:50:00Z', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, 0.0877869, None, 0, 0, 0, 0, 1511686200000, None, None, None, None], - ['2017-11-26 08:55:00', 8.88e-05, 8.942e-05, 8.88e-05, + ['2017-11-26T08:55:00Z', 8.88e-05, 8.942e-05, 8.88e-05, 8.893e-05, 0.05874751, 8.886500000000001e-05, 1, 0, 0, 0, 1511686500000, 8.893e-05, None, None, None], - ['2017-11-26 09:00:00', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05, + ['2017-11-26T09:00:00Z', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05, 0.7039405, 8.885e-05, 0, 0, 0, 0, 1511686800000, None, None, None, None] ]) @@ -1443,13 +1443,13 @@ def test_api_pair_candles(botclient, ohlcv_history): f"{BASE_URI}/pair_candles?limit={amount}&pair=XRP%2FBTC&timeframe={timeframe}") assert_response(rc) assert (rc.json()['data'] == - [['2017-11-26 08:50:00', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, 0.0877869, + [['2017-11-26T08:50:00Z', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, 0.0877869, None, 0, None, 0, 0, None, 1511686200000, None, None, None, None], - ['2017-11-26 08:55:00', 8.88e-05, 8.942e-05, 8.88e-05, - 8.893e-05, 0.05874751, 8.886500000000001e-05, 1, 0.0, 0, 0, '2017-11-26 08:55:00', + ['2017-11-26T08:55:00Z', 8.88e-05, 8.942e-05, 8.88e-05, + 8.893e-05, 0.05874751, 8.886500000000001e-05, 1, 0.0, 0, 0, '2017-11-26T08:55:00Z', 1511686500000, 8.893e-05, None, None, None], - ['2017-11-26 09:00:00', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05, - 0.7039405, 8.885e-05, 0, 0.0, 0, 0, '2017-11-26 09:00:00', 1511686800000, + ['2017-11-26T09:00:00Z', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05, + 0.7039405, 8.885e-05, 0, 0.0, 0, 0, '2017-11-26T09:00:00Z', 1511686800000, None, None, None, None] ]) @@ -1506,7 +1506,7 @@ def test_api_pair_history(botclient, mocker): date_col_idx = [idx for idx, c in enumerate(result['columns']) if c == 'date'][0] rsi_col_idx = [idx for idx, c in enumerate(result['columns']) if c == 'rsi'][0] - assert data[0][date_col_idx] == '2018-01-11 00:00:00' + assert data[0][date_col_idx] == '2018-01-11T00:00:00Z' assert data[0][rsi_col_idx] is not None assert data[0][rsi_col_idx] > 0 assert lfm.call_count == 1 diff --git a/tests/strategy/test_default_strategy.py b/tests/strategy/test_default_strategy.py index 5f41177eb..b5b07e0cd 100644 --- a/tests/strategy/test_default_strategy.py +++ b/tests/strategy/test_default_strategy.py @@ -25,13 +25,13 @@ def test_strategy_test_v3(dataframe_1m, fee, is_short, side): strategy = StrategyTestV3({}) metadata = {'pair': 'ETH/BTC'} - assert type(strategy.minimal_roi) is dict - assert type(strategy.stoploss) is float - assert type(strategy.timeframe) is str + assert isinstance(strategy.minimal_roi, dict) + assert isinstance(strategy.stoploss, float) + assert isinstance(strategy.timeframe, str) indicators = strategy.populate_indicators(dataframe_1m, metadata) - assert type(indicators) is DataFrame - assert type(strategy.populate_buy_trend(indicators, metadata)) is DataFrame - assert type(strategy.populate_sell_trend(indicators, metadata)) is DataFrame + assert isinstance(indicators, DataFrame) + assert isinstance(strategy.populate_buy_trend(indicators, metadata), DataFrame) + assert isinstance(strategy.populate_sell_trend(indicators, metadata), DataFrame) trade = Trade( open_rate=19_000, diff --git a/tests/test_arguments.py b/tests/test_arguments.py index 0c63f636d..b4f6e7279 100644 --- a/tests/test_arguments.py +++ b/tests/test_arguments.py @@ -133,7 +133,7 @@ def test_parse_args_backtesting_custom() -> None: assert call_args['command'] == 'backtesting' assert call_args['func'] is not None assert call_args['timeframe'] == '1m' - assert type(call_args['strategy_list']) is list + assert isinstance(call_args['strategy_list'], list) assert len(call_args['strategy_list']) == 2 diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 7808fb5c8..3f2fb0669 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -1038,8 +1038,7 @@ def test_load_config_stoploss_exchange_limit_ratio(all_conf) -> None: validate_config_schema(all_conf) -@pytest.mark.parametrize("keys", [("exchange", "sandbox", False), - ("exchange", "key", ""), +@pytest.mark.parametrize("keys", [("exchange", "key", ""), ("exchange", "secret", ""), ("exchange", "password", ""), ]) diff --git a/tests/test_misc.py b/tests/test_misc.py index 3943e7f15..e94e299fd 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -9,8 +9,7 @@ import pytest from freqtrade.misc import (dataframe_to_json, decimals_per_coin, deep_merge_dicts, file_dump_json, file_load_json, is_file_in_dir, json_to_dataframe, pair_to_filename, - parse_db_uri_for_logging, plural, render_template, - render_template_with_fallback, round_coin_value, safe_value_fallback, + parse_db_uri_for_logging, plural, round_coin_value, safe_value_fallback, safe_value_fallback2) @@ -177,20 +176,6 @@ def test_plural() -> None: assert plural(-1.5, "ox", "oxen") == "oxen" -def test_render_template_fallback(mocker): - from jinja2.exceptions import TemplateNotFound - with pytest.raises(TemplateNotFound): - val = render_template( - templatefile='subtemplates/indicators_does-not-exist.j2',) - - val = render_template_with_fallback( - templatefile='strategy_subtemplates/indicators_does-not-exist.j2', - templatefallbackfile='strategy_subtemplates/indicators_minimal.j2', - ) - assert isinstance(val, str) - assert 'if self.dp' in val - - @pytest.mark.parametrize('conn_url,expected', [ ("postgresql+psycopg2://scott123:scott123@host:1245/dbname", "postgresql+psycopg2://scott123:*****@host:1245/dbname"), diff --git a/tests/testdata/XRP_ETH-trades.parquet b/tests/testdata/XRP_ETH-trades.parquet new file mode 100644 index 000000000..c2913afdd Binary files /dev/null and b/tests/testdata/XRP_ETH-trades.parquet differ diff --git a/tests/utils/test_datetime_helpers.py b/tests/utils/test_datetime_helpers.py index 222410027..6ce975732 100644 --- a/tests/utils/test_datetime_helpers.py +++ b/tests/utils/test_datetime_helpers.py @@ -63,7 +63,7 @@ def test_format_ms_time() -> None: # Date 2018-04-10 18:02:01 date_in_epoch_ms = 1523383321000 date = format_ms_time(date_in_epoch_ms) - assert type(date) is str + assert isinstance(date, str) res = datetime(2018, 4, 10, 18, 2, 1, tzinfo=timezone.utc) assert date == res.astimezone(None).strftime('%Y-%m-%dT%H:%M:%S') res = datetime(2017, 12, 13, 8, 2, 1, tzinfo=timezone.utc) diff --git a/tests/utils/test_rendering_utils.py b/tests/utils/test_rendering_utils.py new file mode 100644 index 000000000..7d52b4f26 --- /dev/null +++ b/tests/utils/test_rendering_utils.py @@ -0,0 +1,17 @@ +import pytest + +from freqtrade.util import render_template, render_template_with_fallback + + +def test_render_template_fallback(): + from jinja2.exceptions import TemplateNotFound + with pytest.raises(TemplateNotFound): + val = render_template( + templatefile='subtemplates/indicators_does-not-exist.j2',) + + val = render_template_with_fallback( + templatefile='strategy_subtemplates/indicators_does-not-exist.j2', + templatefallbackfile='strategy_subtemplates/indicators_minimal.j2', + ) + assert isinstance(val, str) + assert 'if self.dp' in val