diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f06d7c8ef..a8a261d0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: strategy: matrix: os: [ ubuntu-20.04, ubuntu-22.04 ] - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 @@ -36,15 +36,14 @@ jobs: python-version: ${{ matrix.python-version }} - name: Cache_dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache with: path: ~/dependencies/ key: ${{ runner.os }}-dependencies - name: pip cache (linux) - uses: actions/cache@v3 - if: runner.os == 'Linux' + uses: actions/cache@v4 with: path: ~/.cache/pip key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip @@ -55,7 +54,6 @@ jobs: cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd .. - name: Installation - *nix - if: runner.os == 'Linux' run: | python -m pip install --upgrade pip wheel export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH @@ -126,8 +124,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ "macos-latest" ] - python-version: ["3.9", "3.10", "3.11"] + os: [ "macos-latest", "macos-13" ] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 @@ -139,14 +137,14 @@ jobs: check-latest: true - name: Cache_dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache with: path: ~/dependencies/ key: ${{ matrix.os }}-dependencies - name: pip cache (macOS) - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/Library/Caches/pip key: ${{ matrix.os }}-${{ matrix.python-version }}-pip @@ -164,14 +162,19 @@ jobs: # https://github.com/actions/runner-images/issues/6817 rm /usr/local/bin/2to3 || true rm /usr/local/bin/2to3-3.11 || true + rm /usr/local/bin/2to3-3.12 || true rm /usr/local/bin/idle3 || true rm /usr/local/bin/idle3.11 || true + rm /usr/local/bin/idle3.12 || true rm /usr/local/bin/pydoc3 || true rm /usr/local/bin/pydoc3.11 || true + rm /usr/local/bin/pydoc3.12 || true rm /usr/local/bin/python3 || true rm /usr/local/bin/python3.11 || true + rm /usr/local/bin/python3.12 || true rm /usr/local/bin/python3-config || true rm /usr/local/bin/python3.11-config || true + rm /usr/local/bin/python3.12-config || true brew install hdf5 c-blosc libomp python -m pip install --upgrade pip wheel @@ -235,7 +238,7 @@ jobs: strategy: matrix: os: [ windows-latest ] - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 @@ -246,7 +249,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Pip cache (Windows) - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~\AppData\Local\pip\Cache key: ${{ matrix.os }}-${{ matrix.python-version }}-pip @@ -362,18 +365,17 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.11" - name: Cache_dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache with: path: ~/dependencies/ key: ${{ runner.os }}-dependencies - name: pip cache (linux) - uses: actions/cache@v3 - if: runner.os == 'Linux' + uses: actions/cache@v4 with: path: ~/.cache/pip key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip @@ -384,7 +386,6 @@ jobs: cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd .. - name: Installation - *nix - if: runner.os == 'Linux' run: | python -m pip install --upgrade pip wheel export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH @@ -397,7 +398,7 @@ jobs: env: CI_WEB_PROXY: http://152.67.78.211:13128 run: | - pytest --random-order --cov=freqtrade --cov-config=.coveragerc --longrun + pytest --random-order --longrun --durations 20 -n auto --dist loadscope # Notify only once - when CI completes (and after deploy) in case it's successfull @@ -504,9 +505,10 @@ jobs: python-version: "3.11" - name: Extract branch name - shell: bash - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF##*/})" - id: extract_branch + id: extract-branch + run: | + echo "GITHUB_REF='${GITHUB_REF}'" + echo "branch=${GITHUB_REF##*/}" >> "$GITHUB_OUTPUT" - name: Dockerhub login env: @@ -535,7 +537,7 @@ jobs: - name: Build and test and push docker images env: - BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }} + BRANCH_NAME: ${{ steps.extract-branch.outputs.branch }} run: | build_helpers/publish_docker_multi.sh @@ -552,9 +554,10 @@ jobs: - uses: actions/checkout@v4 - name: Extract branch name - shell: bash - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF##*/})" - id: extract_branch + id: extract-branch + run: | + echo "GITHUB_REF='${GITHUB_REF}'" + echo "branch=${GITHUB_REF##*/}" >> "$GITHUB_OUTPUT" - name: Dockerhub login env: @@ -565,7 +568,7 @@ jobs: - name: Build and test and push docker images env: - BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }} + BRANCH_NAME: ${{ steps.extract-branch.outputs.branch }} GHCR_USERNAME: ${{ github.actor }} GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | diff --git a/.github/workflows/docker_update_readme.yml b/.github/workflows/docker_update_readme.yml index 9d2f8b768..d91e79099 100644 --- a/.github/workflows/docker_update_readme.yml +++ b/.github/workflows/docker_update_readme.yml @@ -10,7 +10,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Docker Hub Description - uses: peter-evans/dockerhub-description@v3 + uses: peter-evans/dockerhub-description@v4 env: DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKERHUB_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} diff --git a/.github/workflows/pre-commit-update.yml b/.github/workflows/pre-commit-update.yml new file mode 100644 index 000000000..bd50a9c3c --- /dev/null +++ b/.github/workflows/pre-commit-update.yml @@ -0,0 +1,45 @@ +name: Pre-commit auto-update + +on: + # every day at midnight + schedule: + - cron: "0 3 * * 2" + # on demand + workflow_dispatch: + +permissions: + contents: read + +jobs: + auto-update: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + + - name: Install pre-commit + run: pip install pre-commit + + - name: Run auto-update + run: pre-commit autoupdate + + - name: Run pre-commit + run: pre-commit run --all-files + + - uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.REPO_SCOPED_TOKEN }} + add-paths: .pre-commit-config.yaml + labels: | + Tech maintenance + Dependencies + branch: update/pre-commit-hooks + title: Update pre-commit hooks + commit-message: "chore: update pre-commit hooks" + committer: Freqtrade Bot + body: Update versions of pre-commit hooks to latest version. + delete-branch: true diff --git a/.gitignore b/.gitignore index a0ee2a228..8a9fc9cfa 100644 --- a/.gitignore +++ b/.gitignore @@ -112,7 +112,6 @@ target/ #exceptions !*.gitkeep !config_examples/config_binance.example.json -!config_examples/config_bittrex.example.json !config_examples/config_full.example.json !config_examples/config_kraken.example.json !config_examples/config_freqai.example.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 298b66956..979fbe083 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,28 +2,28 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pycqa/flake8 - rev: "6.0.0" + rev: "7.0.0" hooks: - id: flake8 additional_dependencies: [Flake8-pyproject] # stages: [push] - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.7.0" + rev: "v1.8.0" hooks: - id: mypy exclude: build_helpers additional_dependencies: - types-cachetools==5.3.0.7 - types-filelock==3.2.7 - - types-requests==2.31.0.10 - - types-tabulate==0.9.0.3 - - types-python-dateutil==2.8.19.14 - - SQLAlchemy==2.0.23 + - types-requests==2.31.0.20240125 + - types-tabulate==0.9.0.20240106 + - types-python-dateutil==2.8.19.20240106 + - SQLAlchemy==2.0.25 # stages: [push] - repo: https://github.com/pycqa/isort - rev: "5.12.0" + rev: "5.13.2" hooks: - id: isort name: isort (python) @@ -31,12 +31,12 @@ repos: - repo: https://github.com/charliermarsh/ruff-pre-commit # Ruff version. - rev: 'v0.1.1' + rev: 'v0.1.15' hooks: - id: ruff - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: end-of-file-fixer exclude: | diff --git a/Dockerfile b/Dockerfile index 38f9ca788..e5a33df87 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.6-slim-bookworm as base +FROM python:3.11.7-slim-bookworm as base # Setup env ENV LANG C.UTF-8 diff --git a/README.md b/README.md index c00d2c999..c6e54b112 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Please read the [exchange specific notes](docs/exchanges.md) to learn about even - [X] [Binance](https://www.binance.com/) - [X] [Bitmart](https://bitmart.com/) - [X] [Gate.io](https://www.gate.io/ref/6266643) -- [X] [Huobi](http://huobi.com/) +- [X] [HTX](https://www.htx.com/) (Former Huobi) - [X] [Kraken](https://kraken.com/) - [X] [OKX](https://okx.com/) (Former OKEX) - [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ diff --git a/build_helpers/pyarrow-14.0.1-cp39-cp39-linux_armv7l.whl b/build_helpers/pyarrow-15.0.0-cp311-cp311-linux_armv7l.whl similarity index 64% rename from build_helpers/pyarrow-14.0.1-cp39-cp39-linux_armv7l.whl rename to build_helpers/pyarrow-15.0.0-cp311-cp311-linux_armv7l.whl index 08226b794..11cfc1895 100644 Binary files a/build_helpers/pyarrow-14.0.1-cp39-cp39-linux_armv7l.whl and b/build_helpers/pyarrow-15.0.0-cp311-cp311-linux_armv7l.whl differ diff --git a/build_helpers/pyarrow-14.0.1-cp311-cp311-linux_armv7l.whl b/build_helpers/pyarrow-15.0.0-cp39-cp39-linux_armv7l.whl similarity index 64% rename from build_helpers/pyarrow-14.0.1-cp311-cp311-linux_armv7l.whl rename to build_helpers/pyarrow-15.0.0-cp39-cp39-linux_armv7l.whl index 46529fbe4..fe2a1e85a 100644 Binary files a/build_helpers/pyarrow-14.0.1-cp311-cp311-linux_armv7l.whl and b/build_helpers/pyarrow-15.0.0-cp39-cp39-linux_armv7l.whl differ diff --git a/config_examples/config_freqai.example.json b/config_examples/config_freqai.example.json index 65a93379e..27bc4532c 100644 --- a/config_examples/config_freqai.example.json +++ b/config_examples/config_freqai.example.json @@ -52,7 +52,7 @@ "train_period_days": 15, "backtest_period_days": 7, "live_retrain_hours": 0, - "identifier": "uniqe-id", + "identifier": "unique-id", "feature_parameters": { "include_timeframes": [ "3m", diff --git a/docker/Dockerfile.armhf b/docker/Dockerfile.armhf index c8efa4232..4cb8f5fea 100644 --- a/docker/Dockerfile.armhf +++ b/docker/Dockerfile.armhf @@ -1,4 +1,4 @@ -FROM python:3.11.6-slim-bookworm as base +FROM python:3.11.7-slim-bookworm as base # Setup env ENV LANG C.UTF-8 diff --git a/docker/Dockerfile.jupyter b/docker/Dockerfile.jupyter index d86980bdf..79fefb3a7 100644 --- a/docker/Dockerfile.jupyter +++ b/docker/Dockerfile.jupyter @@ -1,8 +1,8 @@ FROM freqtradeorg/freqtrade:develop_plot -# Pin jupyter-client to avoid tornado version conflict -RUN pip install jupyterlab jupyter-client==7.3.4 --user --no-cache-dir +# Pin prompt-toolkit to avoid questionary version conflict +RUN pip install jupyterlab "prompt-toolkit<=3.0.36" jupyter-client --user --no-cache-dir # Empty the ENTRYPOINT to allow all commands ENTRYPOINT [] diff --git a/docker/docker-compose-jupyter.yml b/docker/docker-compose-jupyter.yml index 3df82365f..b72c8df84 100644 --- a/docker/docker-compose-jupyter.yml +++ b/docker/docker-compose-jupyter.yml @@ -6,7 +6,7 @@ services: context: .. dockerfile: docker/Dockerfile.jupyter restart: unless-stopped - container_name: freqtrade + # container_name: freqtrade ports: - "127.0.0.1:8888:8888" volumes: diff --git a/docs/configuration.md b/docs/configuration.md index 313e3e456..202fa49bf 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -321,7 +321,7 @@ For example, if you have 10 ETH available in your wallet on the exchange and `tr To fully utilize compounding profits when using multiple bots on the same exchange account, you'll want to limit each bot to a certain starting balance. This can be accomplished by setting `available_capital` to the desired starting balance. -Assuming your account has 10.000 USDT and you want to run 2 different strategies on this exchange. +Assuming your account has 10000 USDT and you want to run 2 different strategies on this exchange. You'd set `available_capital=5000` - granting each bot an initial capital of 5000 USDT. The bot will then split this starting balance equally into `max_open_trades` buckets. Profitable trades will result in increased stake-sizes for this bot - without affecting the stake-sizes of the other bot. @@ -572,9 +572,11 @@ In addition to fiat currencies, a range of crypto currencies is supported. The valid values are: ```json -"BTC", "ETH", "XRP", "LTC", "BCH", "USDT" +"BTC", "ETH", "XRP", "LTC", "BCH", "BNB" ``` +Removing `fiat_display_currency` completely from the configuration will skip initializing coingecko, and will not show any FIAT currency conversion. This has no importance for the correct functioning of the bot. + ## Using Dry-run mode We recommend starting the bot in the Dry-run mode to see how your bot will diff --git a/docs/exchanges.md b/docs/exchanges.md index ac3957b07..c106ea4cf 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -127,6 +127,8 @@ Freqtrade will not attempt to change these settings. ## Kraken +Kraken supports [time_in_force](configuration.md#understand-order_time_in_force) with settings "GTC" (good till cancelled), "IOC" (immediate-or-cancel) and "PO" (Post only) settings. + !!! Tip "Stoploss on Exchange" Kraken supports `stoploss_on_exchange` and can use both stop-loss-market and stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it. You can use either `"limit"` or `"market"` in the `order_types.stoploss` configuration setting to decide which type to use. @@ -181,48 +183,6 @@ freqtrade download-data --exchange kraken --dl-trades -p BTC/EUR BCH/EUR Please pay attention that rateLimit configuration entry holds delay in milliseconds between requests, NOT requests\sec rate. So, in order to mitigate Kraken API "Rate limit exceeded" exception, this configuration should be increased, NOT decreased. -## Bittrex - -### Order types - -Bittrex does not support market orders. If you have a message at the bot startup about this, you should change order type values set in your configuration and/or in the strategy from `"market"` to `"limit"`. See some more details on this [here in the FAQ](faq.md#im-getting-the-exchange-bittrex-does-not-support-market-orders-message-and-cannot-run-my-strategy). - -Bittrex also does not support `VolumePairlist` due to limited / split API constellation at the moment. -Please use `StaticPairlist`. Other pairlists (other than `VolumePairlist`) should not be affected. - -### Volume pairlist - -Bittrex does not support the direct usage of VolumePairList. This can however be worked around by using the advanced mode with `lookback_days: 1` (or more), which will emulate 24h volume. - -Read more in the [pairlist documentation](plugins.md#volumepairlist-advanced-mode). - -### Restricted markets - -Bittrex split its exchange into US and International versions. -The International version has more pairs available, however the API always returns all pairs, so there is currently no automated way to detect if you're affected by the restriction. - -If you have restricted pairs in your whitelist, you'll get a warning message in the log on Freqtrade startup for each restricted pair. - -The warning message will look similar to the following: - -``` output -[...] Message: bittrex {"success":false,"message":"RESTRICTED_MARKET","result":null,"explanation":null}" -``` - -If you're an "International" customer on the Bittrex exchange, then this warning will probably not impact you. -If you're a US customer, the bot will fail to create orders for these pairs, and you should remove them from your whitelist. - -You can get a list of restricted markets by using the following snippet: - -``` python -import ccxt -ct = ccxt.bittrex() -lm = ct.load_markets() - -res = [p for p, x in lm.items() if 'US' in x['info']['prohibitedIn']] -print(res) -``` - ## Kucoin Kucoin requires a passphrase for each api key, you will therefore need to add this key into the configuration so your exchange section looks as follows: @@ -248,10 +208,10 @@ Kucoin supports [time_in_force](configuration.md#understand-order_time_in_force) For Kucoin, it is suggested to add `"KCS/"` to your blacklist to avoid issues, unless you are willing to maintain enough extra `KCS` on the account or unless you're willing to disable using `KCS` for fees. Kucoin accounts may use `KCS` for fees, and if a trade happens to be on `KCS`, further trades may consume this position and make the initial `KCS` trade unsellable as the expected amount is not there anymore. -## Huobi +## HTX (formerly Huobi) !!! Tip "Stoploss on Exchange" - Huobi supports `stoploss_on_exchange` and uses `stop-limit` orders. It provides great advantages, so we recommend to benefit from it by enabling stoploss on exchange. + HTX supports `stoploss_on_exchange` and uses `stop-limit` orders. It provides great advantages, so we recommend to benefit from it by enabling stoploss on exchange. ## OKX (former OKEX) diff --git a/docs/faq.md b/docs/faq.md index 196bd4308..95a9924f9 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -130,7 +130,7 @@ This warning can point to one of the below problems: ### I'm getting the "Exchange XXX does not support market orders." message and cannot run my strategy -As the message says, your exchange does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Your strategy was probably written with other exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of the exchanges supporting market orders (but not for Bittrex and Gate.io). +As the message says, your exchange does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Your strategy was probably written with other exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of the exchanges supporting market orders (but not for Gate.io). To fix this, redefine order types in the strategy to use "limit" instead of "market": diff --git a/docs/freqai-configuration.md b/docs/freqai-configuration.md index 090fa8415..dd0dde5d4 100644 --- a/docs/freqai-configuration.md +++ b/docs/freqai-configuration.md @@ -162,7 +162,8 @@ Below are the values you can expect to include/use inside a typical strategy dat | `df['&*_std/mean']` | Standard deviation and mean values of the defined labels during training (or live tracking with `fit_live_predictions_candles`). Commonly used to understand the rarity of a prediction (use the z-score as shown in `templates/FreqaiExampleStrategy.py` and explained [here](#creating-a-dynamic-target-threshold) to evaluate how often a particular prediction was observed during training or historically with `fit_live_predictions_candles`).
**Datatype:** Float. | `df['do_predict']` | Indication of an outlier data point. The return value is integer between -2 and 2, which lets you know if the prediction is trustworthy or not. `do_predict==1` means that the prediction is trustworthy. If the Dissimilarity Index (DI, see details [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di)) of the input data point is above the threshold defined in the config, FreqAI will subtract 1 from `do_predict`, resulting in `do_predict==0`. If `use_SVM_to_remove_outliers` is active, the Support Vector Machine (SVM, see details [here](freqai-feature-engineering.md#identifying-outliers-using-a-support-vector-machine-svm)) may also detect outliers in training and prediction data. In this case, the SVM will also subtract 1 from `do_predict`. If the input data point was considered an outlier by the SVM but not by the DI, or vice versa, the result will be `do_predict==0`. If both the DI and the SVM considers the input data point to be an outlier, the result will be `do_predict==-1`. As with the SVM, if `use_DBSCAN_to_remove_outliers` is active, DBSCAN (see details [here](freqai-feature-engineering.md#identifying-outliers-with-dbscan)) may also detect outliers and subtract 1 from `do_predict`. Hence, if both the SVM and DBSCAN are active and identify a datapoint that was above the DI threshold as an outlier, the result will be `do_predict==-2`. A particular case is when `do_predict == 2`, which means that the model has expired due to exceeding `expired_hours`.
**Datatype:** Integer between -2 and 2. | `df['DI_values']` | Dissimilarity Index (DI) values are proxies for the level of confidence FreqAI has in the prediction. A lower DI means the prediction is close to the training data, i.e., higher prediction confidence. See details about the DI [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di).
**Datatype:** Float. -| `df['%*']` | Any dataframe column prepended with `%` in `feature_engineering_*()` is treated as a training feature. For example, you can include the RSI in the training feature set (similar to in `templates/FreqaiExampleStrategy.py`) by setting `df['%-rsi']`. See more details on how this is done [here](freqai-feature-engineering.md).
**Note:** Since the number of features prepended with `%` can multiply very quickly (10s of thousands of features are easily engineered using the multiplictative functionality of, e.g., `include_shifted_candles` and `include_timeframes` as described in the [parameter table](freqai-parameter-table.md)), these features are removed from the dataframe that is returned from FreqAI to the strategy. To keep a particular type of feature for plotting purposes, you would prepend it with `%%`.
**Datatype:** Depends on the output of the model. +| `df['%*']` | Any dataframe column prepended with `%` in `feature_engineering_*()` is treated as a training feature. For example, you can include the RSI in the training feature set (similar to in `templates/FreqaiExampleStrategy.py`) by setting `df['%-rsi']`. See more details on how this is done [here](freqai-feature-engineering.md).
**Note:** Since the number of features prepended with `%` can multiply very quickly (10s of thousands of features are easily engineered using the multiplictative functionality of, e.g., `include_shifted_candles` and `include_timeframes` as described in the [parameter table](freqai-parameter-table.md)), these features are removed from the dataframe that is returned from FreqAI to the strategy. To keep a particular type of feature for plotting purposes, you would prepend it with `%%` (see details below).
**Datatype:** Depends on the feature created by the user. +| `df['%%*']` | Any dataframe column prepended with `%%` in `feature_engineering_*()` is treated as a training feature, just the same as the above `%` prepend. However, in this case, the features are returned back to the strategy for FreqUI/plot-dataframe plotting and monitoring in Dry/Live/Backtesting
**Datatype:** Depends on the feature created by the user. Please note that features created in `feature_engineering_expand()` will have automatic FreqAI naming schemas depending on the expansions that you configured (i.e. `include_timeframes`, `include_corr_pairlist`, `indicators_periods_candles`, `include_shifted_candles`). So if you want to plot `%%-rsi` from `feature_engineering_expand_all()`, the final naming scheme for your plotting config would be: `%%-rsi-period_10_ETH/USDT:USDT_1h` for the `rsi` feature with `period=10`, `timeframe=1h`, and `pair=ETH/USDT:USDT` (the `:USDT` is added if you are using futures pairs). It is useful to simply add `print(dataframe.columns)` in your `populate_indicators()` after `self.freqai.start()` to see the full list of available features that are returned to the strategy for plotting purposes. ## Setting the `startup_candle_count` diff --git a/docs/freqai-running.md b/docs/freqai-running.md index 55f302d40..553a8b698 100644 --- a/docs/freqai-running.md +++ b/docs/freqai-running.md @@ -41,11 +41,11 @@ FreqAI stores new model files after each successful training. These files become ```json "freqai": { - "purge_old_models": true, + "purge_old_models": 4, } ``` -This will automatically purge all models older than the two most recently trained ones to save disk space. +This will automatically purge all models older than the four most recently trained ones to save disk space. Inputing "0" will never purge any models. ## Backtesting @@ -68,7 +68,7 @@ Backtesting mode requires [downloading the necessary data](#downloading-data-to- This way, you can return to using any model you wish by simply specifying the `identifier`. !!! Note - Backtesting calls `set_freqai_targets()` one time for each backtest window (where the number of windows is the full backtest timerange divided by the `backtest_period_days` parameter). Doing this means that the targets simulate dry/live behavior without look ahead bias. However, the definition of the features in `feature_engineering_*()` is performed once on the entire backtest timerange. This means that you should be sure that features do look-ahead into the future. + Backtesting calls `set_freqai_targets()` one time for each backtest window (where the number of windows is the full backtest timerange divided by the `backtest_period_days` parameter). Doing this means that the targets simulate dry/live behavior without look ahead bias. However, the definition of the features in `feature_engineering_*()` is performed once on the entire training timerange. This means that you should be sure that features do not look-ahead into the future. More details about look-ahead bias can be found in [Common Mistakes](strategy-customization.md#common-mistakes-when-developing-strategies). --- diff --git a/docs/freqai.md b/docs/freqai.md index 820fb81f6..a81c9a81b 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -114,6 +114,11 @@ Here we compile some external materials that provide deeper looks into various c - [Real-time head-to-head: Adaptive modeling of financial market data using XGBoost and CatBoost](https://emergentmethods.medium.com/real-time-head-to-head-adaptive-modeling-of-financial-market-data-using-xgboost-and-catboost-995a115a7495) - [FreqAI - from price to prediction](https://emergentmethods.medium.com/freqai-from-price-to-prediction-6fadac18b665) + +## Support + +You can find support for FreqAI in a variety of places, including the [Freqtrade discord](https://discord.gg/Jd8JYeWHc4), the dedicated [FreqAI discord](https://discord.gg/7AMWACmbjT), and in [github issues](https://github.com/freqtrade/freqtrade/issues). + ## Credits FreqAI is developed by a group of individuals who all contribute specific skillsets to the project. diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 444279195..d3371d771 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -439,7 +439,7 @@ While this strategy is most likely too simple to provide consistent profit, it s ??? Hint "Performance tip" During normal hyperopting, indicators are calculated once and supplied to each epoch, linearly increasing RAM usage as a factor of increasing cores. As this also has performance implications, there are two alternatives to reduce RAM usage - * Move `ema_short` and `ema_long` calculations from `populate_indicators()` to `populate_entry_trend()`. Since `populate_entry_trend()` gonna be calculated every epochs, you don't need to use `.range` functionality. + * Move `ema_short` and `ema_long` calculations from `populate_indicators()` to `populate_entry_trend()`. Since `populate_entry_trend()` will be calculated every epoch, you don't need to use `.range` functionality. * hyperopt provides `--analyze-per-epoch` which will move the execution of `populate_indicators()` to the epoch process, calculating a single value per parameter per epoch instead of using the `.range` functionality. In this case, `.range` functionality will only return the actually used value. These alternatives will reduce RAM usage, but increase CPU usage. However, your hyperopting run will be less likely to fail due to Out Of Memory (OOM) issues. @@ -926,6 +926,12 @@ Once the optimized strategy has been implemented into your strategy, you should To achieve same the results (number of trades, their durations, profit, etc.) as during Hyperopt, please use the same configuration and parameters (timerange, timeframe, ...) used for hyperopt `--dmmp`/`--disable-max-market-positions` and `--eps`/`--enable-position-stacking` for Backtesting. -Should results not match, please double-check to make sure you transferred all conditions correctly. -Pay special care to the stoploss, max_open_trades and trailing stoploss parameters, as these are often set in configuration files, which override changes to the strategy. -You should also carefully review the log of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss`, `max_open_trades` or `trailing_stop`). +### Why do my backtest results not match my hyperopt results? +Should results not match, check the following factors: + +* You may have added parameters to hyperopt in `populate_indicators()` where they will be calculated only once **for all epochs**. If you are, for example, trying to optimise multiple SMA timeperiod values, the hyperoptable timeperiod parameter should be placed in `populate_entry_trend()` which is calculated every epoch. See [Optimizing an indicator parameter](https://www.freqtrade.io/en/stable/hyperopt/#optimizing-an-indicator-parameter). +* If you have disabled the auto-export of hyperopt parameters into the JSON parameters file, double-check to make sure you transferred all hyperopted values into your strategy correctly. +* Check the logs to verify what parameters are being set and what values are being used. +* Pay special care to the stoploss, max_open_trades and trailing stoploss parameters, as these are often set in configuration files, which override changes to the strategy. Check the logs of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss`, `max_open_trades` or `trailing_stop`). +* Verify that you do not have an unexpected parameters JSON file overriding the parameters or the default hyperopt settings in your strategy. +* Verify that any protections that are enabled in backtesting are also enabled when hyperopting, and vice versa. When using `--space protection`, protections are auto-enabled for hyperopting. diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index b3b23d6ff..9781edf10 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -6,7 +6,7 @@ In your configuration, you can use Static Pairlist (defined by the [`StaticPairL Additionally, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter), [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist. -If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You should always configure either `StaticPairList` or `VolumePairList` as the starting Pairlist Handler. +If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You can define either `StaticPairList`, `VolumePairList`, `ProducerPairList`, `RemotePairList` or `MarketCapPairList` as the starting Pairlist Handler. Inactive markets are always removed from the resulting pairlist. Explicitly blacklisted pairs (those in the `pair_blacklist` configuration setting) are also always removed from the resulting pairlist. @@ -24,6 +24,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged * [`VolumePairList`](#volume-pair-list) * [`ProducerPairList`](#producerpairlist) * [`RemotePairList`](#remotepairlist) +* [`MarketCapPairList`](#marketcappairlist) * [`AgeFilter`](#agefilter) * [`FullTradesFilter`](#fulltradesfilter) * [`OffsetFilter`](#offsetfilter) @@ -112,8 +113,8 @@ For convenience `lookback_days` can be specified, which will imply that 1d candl !!! Warning "Performance implications when using lookback range" If used in first position in combination with lookback, the computation of the range based volume can be time and resource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `VolumeFilter` to narrow the pairlist down for further range volume calculation. -??? Tip "Unsupported exchanges (Bittrex, Gemini)" - On some exchanges (like Bittrex and Gemini), regular VolumePairList does not work as the api does not natively provide 24h volume. This can be worked around by using candle data to build the volume. +??? Tip "Unsupported exchanges" + On some exchanges (like Gemini), regular VolumePairList does not work as the api does not natively provide 24h volume. This can be worked around by using candle data to build the volume. To roughly simulate 24h volume, you can use the following configuration. Please note that These pairlists will only refresh once per day. @@ -192,7 +193,8 @@ The RemotePairList is defined in the pairlists section of the configuration sett "refresh_period": 1800, "keep_pairlist_on_failure": true, "read_timeout": 60, - "bearer_token": "my-bearer-token" + "bearer_token": "my-bearer-token", + "save_to_file": "user_data/filename.json" } ] ``` @@ -207,6 +209,42 @@ In "append" mode, the retrieved pairlist is added to the original pairlist. All The `pairlist_url` option specifies the URL of the remote server where the pairlist is located, or the path to a local file (if file:/// is prepended). This allows the user to use either a remote server or a local file as the source for the pairlist. +The `save_to_file` option, when provided with a valid filename, saves the processed pairlist to that file in JSON format. This option is optional, and by default, the pairlist is not saved to a file. + +??? Example "Multi bot with shared pairlist example" + + `save_to_file` can be used to save the pairlist to a file with Bot1: + + ```json + "pairlists": [ + { + "method": "RemotePairList", + "mode": "whitelist", + "pairlist_url": "https://example.com/pairlist", + "number_assets": 10, + "refresh_period": 1800, + "keep_pairlist_on_failure": true, + "read_timeout": 60, + "save_to_file": "user_data/filename.json" + } + ] + ``` + + This saved pairlist file can be loaded by Bot2, or any additional bot with this configuration: + + ```json + "pairlists": [ + { + "method": "RemotePairList", + "mode": "whitelist", + "pairlist_url": "file:///user_data/filename.json", + "number_assets": 10, + "refresh_period": 10, + "keep_pairlist_on_failure": true, + } + ] + ``` + The user is responsible for providing a server or local file that returns a JSON object with the following structure: ```json @@ -227,6 +265,25 @@ The optional `bearer_token` will be included in the requests Authorization Heade !!! Note In case of a server error the last received pairlist will be kept if `keep_pairlist_on_failure` is set to true, when set to false a empty pairlist is returned. +#### MarketCapPairList + +`MarketCapPairList` employs sorting/filtering of pairs by their marketcap rank based of CoinGecko. It will only recognize coins up to the coin placed at rank 250. The returned pairlist will be sorted based of their marketcap ranks. + +```json +"pairlists": [ + { + "method": "MarketCapPairList", + "number_assets": 20, + "max_rank": 50, + "refresh_period": 86400 + } +] +``` + +`number_assets` defines the maximum number of pairs returned by the pairlist. `max_rank` will determine the maximum rank used in creating/filtering the pairlist. It's expected that some coins within the top `max_rank` marketcap will not be included in the resulting pairlist since not all pairs will have active trading pairs in your preferred market/stake/exchange combination. + +`refresh_period` setting defines the period (in seconds) at which the marketcap rank data will be refreshed. Defaults to 86,400s (1 day). The pairlist cache (`refresh_period`) is applicable on both generating pairlists (first position in the list) and filtering instances (not the first position in the list). + #### AgeFilter Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`) or more than `max_days_listed` days (defaults `None` mean infinity). diff --git a/docs/includes/showcase.md b/docs/includes/showcase.md index 297685ad4..766703b9f 100644 --- a/docs/includes/showcase.md +++ b/docs/includes/showcase.md @@ -5,7 +5,7 @@ This section will highlight a few projects from members of the community. - [Example freqtrade strategies](https://github.com/freqtrade/freqtrade-strategies/) - [FrequentHippo - Grafana dashboard with dry/live runs and backtests](http://frequenthippo.ddns.net:3000/) (by hippocritical). - [Online pairlist generator](https://remotepairlist.com/) (by Blood4rc). -- [Freqtrade Backtesting Project](https://bt.robot.co.network/) (by Blood4rc). +- [Freqtrade Backtesting Project](https://strat.ninja/) (by Blood4rc). - [Freqtrade analysis notebook](https://github.com/froggleston/freqtrade_analysis_notebook) (by Froggleston). - [TUI for freqtrade](https://github.com/froggleston/freqtrade-frogtrade9000) (by Froggleston). - [Bot Academy](https://botacademy.ddns.net/) (by stash86) - Blog about crypto bot projects. diff --git a/docs/index.md b/docs/index.md index 1df5424de..26debe993 100644 --- a/docs/index.md +++ b/docs/index.md @@ -42,7 +42,7 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual, - [X] [Binance](https://www.binance.com/) - [X] [Bitmart](https://bitmart.com/) - [X] [Gate.io](https://www.gate.io/ref/6266643) -- [X] [Huobi](http://huobi.com/) +- [X] [HTX](https://www.htx.com/) (Former Huobi) - [X] [Kraken](https://kraken.com/) - [X] [OKX](https://okx.com/) (Former OKEX) - [ ] [potentially many others through ccxt](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 6df4bef97..d6d2c29cc 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ -markdown==3.5.1 +markdown==3.5.2 mkdocs==1.5.3 -mkdocs-material==9.5.2 +mkdocs-material==9.5.6 mdx_truly_sane_lists==1.3 -pymdown-extensions==10.5 -jinja2==3.1.2 +pymdown-extensions==10.7 +jinja2==3.1.3 diff --git a/docs/stoploss.md b/docs/stoploss.md index 8fc73be21..935950d06 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -30,7 +30,7 @@ The Order-type will be ignored if only one mode is available. |----------|-------------| | Binance | limit | | Binance Futures | market, limit | -| Huobi | limit | +| HTX (former Huobi) | limit | | kraken | market, limit | | Gate | limit | | Okx | limit | diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 34d72a6ef..2292b7ed0 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -489,7 +489,7 @@ The helper function `stoploss_from_absolute()` can be used to convert from an ab dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe) trade_date = timeframe_to_prev_date(self.timeframe, trade.open_date_utc) candle = dataframe.iloc[-1].squeeze() - sign = 1 if trade.is_short else -1 + side = 1 if trade.is_short else -1 return stoploss_from_absolute(current_rate + (side * candle['atr'] * 2), current_rate, is_short=trade.is_short, leverage=trade.leverage) @@ -760,22 +760,32 @@ The `position_adjustment_enable` strategy property enables the usage of `adjust_ For performance reasons, it's disabled by default and freqtrade will show a warning message on startup if enabled. `adjust_trade_position()` can be used to perform additional orders, for example to manage risk with DCA (Dollar Cost Averaging) or to increase or decrease positions. -`max_entry_position_adjustment` property is used to limit the number of additional entries per trade (on top of the first entry order) that the bot can execute. By default, the value is -1 which means the bot have no limit on number of adjustment entries. - -The strategy is expected to return a stake_amount (in stake currency) between `min_stake` and `max_stake` if and when an additional entry order should be made (position is increased -> buy order for long trades, sell order for short trades). -If there are not enough funds in the wallet (the return value is above `max_stake`) then the signal will be ignored. Additional orders also result in additional fees and those orders don't count towards `max_open_trades`. This callback is **not** called when there is an open order (either buy or sell) waiting for execution. `adjust_trade_position()` is called very frequently for the duration of a trade, so you must keep your implementation as performant as possible. -Additional entries are ignored once you have reached the maximum amount of extra entries that you have set on `max_entry_position_adjustment`, but the callback is called anyway looking for partial exits. - Position adjustments will always be applied in the direction of the trade, so a positive value will always increase your position (negative values will decrease your position), no matter if it's a long or short trade. +Adjustment orders can be assigned with a tag by returning a 2 element Tuple, with the first element being the adjustment amount, and the 2nd element the tag (e.g. `return 250, 'increase_favorable_conditions'`). Modifications to leverage are not possible, and the stake-amount returned is assumed to be before applying leverage. +### Increase position + +The strategy is expected to return a positive **stake_amount** (in stake currency) between `min_stake` and `max_stake` if and when an additional entry order should be made (position is increased -> buy order for long trades, sell order for short trades). + +If there are not enough funds in the wallet (the return value is above `max_stake`) then the signal will be ignored. +`max_entry_position_adjustment` property is used to limit the number of additional entries per trade (on top of the first entry order) that the bot can execute. By default, the value is -1 which means the bot have no limit on number of adjustment entries. + +Additional entries are ignored once you have reached the maximum amount of extra entries that you have set on `max_entry_position_adjustment`, but the callback is called anyway looking for partial exits. + +### Decrease position + +The strategy is expected to return a negative stake_amount (in stake currency) for a partial exit. +Returning the full owned stake at that point (based on the current price) (`-(trade.amount / trade.leverage) * current_exit_rate`) results in a full exit. +Returning a value more than the above (so remaining stake_amount would become negative) will result in the bot ignoring the signal. + !!! Note "About stake size" Using fixed stake size means it will be the amount used for the first order, just like without position adjustment. If you wish to buy additional orders with DCA, then make sure to leave enough funds in the wallet for that. @@ -824,7 +834,8 @@ class DigDeeperStrategy(IStrategy): min_stake: Optional[float], max_stake: float, current_entry_rate: float, current_exit_rate: float, current_entry_profit: float, current_exit_profit: float, - **kwargs) -> Optional[float]: + **kwargs + ) -> Union[Optional[float], Tuple[Optional[float], Optional[str]]]: """ Custom trade adjustment logic, returning the stake amount that a trade should be increased or decreased. @@ -850,11 +861,12 @@ class DigDeeperStrategy(IStrategy): :return float: Stake amount to adjust your trade, Positive values to increase position, Negative values to decrease position. Return None for no action. + Optionally, return a tuple with a 2nd element with an order reason """ if current_profit > 0.05 and trade.nr_of_successful_exits == 0: # Take half of the profit at +5% - return -(trade.stake_amount / 2) + return -(trade.stake_amount / 2), 'half_profit_5%' if current_profit > -0.05: return None @@ -882,7 +894,7 @@ class DigDeeperStrategy(IStrategy): stake_amount = filled_entries[0].stake_amount # This then calculates current safety order size stake_amount = stake_amount * (1 + (count_of_entries * 0.25)) - return stake_amount + return stake_amount, '1/3rd_increase' except Exception as exception: return None diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index e80a30c64..900dafb33 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -156,9 +156,9 @@ def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame Out of the box, freqtrade installs the following technical libraries: -* [ta-lib](http://mrjbq7.github.io/ta-lib/) -* [pandas-ta](https://twopirllc.github.io/pandas-ta/) -* [technical](https://github.com/freqtrade/technical/) +- [ta-lib](https://ta-lib.github.io/ta-lib-python/) +- [pandas-ta](https://twopirllc.github.io/pandas-ta/) +- [technical](https://github.com/freqtrade/technical/) Additional technical libraries can be installed as necessary, or custom indicators may be written / invented by the strategy author. @@ -367,6 +367,11 @@ class AwesomeStrategy(IStrategy): } ``` +??? info "Orders that don't fill immediately" + `minimal_roi` will take the `trade.open_date` as reference, which is the time the trade was initialized / the first order for this trade was placed. + This will also hold true for limit orders that don't fill immediately (usually in combination with "off-spot" prices through `custom_entry_price()`), as well as for cases where the initial order is replaced through `adjust_entry_price()`. + The time used will still be from the initial `trade.open_date` (when the initial order was first placed), not from the newly placed order date. + ### Stoploss Setting a stoploss is highly recommended to protect your capital from strong moves against you. @@ -1004,8 +1009,8 @@ This is a common pain-point, which can cause huge differences between backtestin The following lists some common patterns which should be avoided to prevent frustration: -- don't use `shift(-1)`. This uses data from the future, which is not available. -- don't use `.iloc[-1]` or any other absolute position in the dataframe, this will be different between dry-run and backtesting. +- don't use `shift(-1)` or other negative values. This uses data from the future in backtesting, which is not available in dry or live modes. +- don't use `.iloc[-1]` or any other absolute position in the dataframe within `populate_` functions, as this will be different between dry-run and backtesting. Absolute `iloc` indexing is safe to use in callbacks however - see [Strategy Callbacks](strategy-callbacks.md). - don't use `dataframe['volume'].mean()`. This uses the full DataFrame for backtesting, including data from the future. Use `dataframe['volume'].rolling().mean()` instead - don't use `.resample('1h')`. This uses the left border of the interval, so moves data from an hour to the start of the hour. Use `.resample('1h', label='right')` instead. diff --git a/docs/utils.md b/docs/utils.md index ac0d341bd..b4432833d 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -242,7 +242,6 @@ bitkk True missing opt: fetchMyTrades bitmart True bitmax True missing opt: fetchMyTrades bitpanda True -bittrex True bitvavo True bitz True missing opt: fetchMyTrades btcalpha True missing opt: fetchTicker, fetchTickers @@ -324,7 +323,6 @@ bitpanda True bitso False missing: fetchOHLCV bitstamp True missing opt: fetchTickers bitstamp1 False missing: fetchOrder, fetchOHLCV -bittrex True bitvavo True bitz True missing opt: fetchMyTrades bl3p False missing: fetchOrder, fetchOHLCV diff --git a/docs/webhook-config.md b/docs/webhook-config.md index 811fa1b8e..b4044655c 100644 --- a/docs/webhook-config.md +++ b/docs/webhook-config.md @@ -134,6 +134,7 @@ Possible parameters are: * `stake_amount` * `stake_currency` * `base_currency` +* `quote_currency` * `fiat_currency` * `order_type` * `current_rate` @@ -155,6 +156,7 @@ Possible parameters are: * `stake_amount` * `stake_currency` * `base_currency` +* `quote_currency` * `fiat_currency` * `order_type` * `current_rate` @@ -176,6 +178,7 @@ Possible parameters are: * `stake_amount` * `stake_currency` * `base_currency` +* `quote_currency` * `fiat_currency` * `order_type` * `current_rate` @@ -199,6 +202,7 @@ Possible parameters are: * `profit_ratio` * `stake_currency` * `base_currency` +* `quote_currency` * `fiat_currency` * `exit_reason` * `order_type` @@ -224,6 +228,7 @@ Possible parameters are: * `profit_ratio` * `stake_currency` * `base_currency` +* `quote_currency` * `fiat_currency` * `exit_reason` * `order_type` @@ -249,6 +254,7 @@ Possible parameters are: * `profit_ratio` * `stake_currency` * `base_currency` +* `quote_currency` * `fiat_currency` * `exit_reason` * `order_type` diff --git a/docs/windows_installation.md b/docs/windows_installation.md index 5ac3d5e3d..cd9007d98 100644 --- a/docs/windows_installation.md +++ b/docs/windows_installation.md @@ -22,7 +22,7 @@ git clone https://github.com/freqtrade/freqtrade.git ### 2. Install ta-lib -Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows). +Install ta-lib according to the [ta-lib documentation](https://github.com/TA-Lib/ta-lib-python#windows). As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), Freqtrade provides these dependencies (in the binary wheel format) for the latest 3 Python versions (3.9, 3.10 and 3.11) and for 64bit Windows. These Wheels are also used by CI running on windows, and are therefore tested together with freqtrade. diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 464a9df97..7c699d643 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2023.12-dev' +__version__ = '2024.2-dev' if 'dev' in __version__: from pathlib import Path diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 26d582465..f72164675 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -219,27 +219,35 @@ class Arguments: ) # Add trade subcommand - trade_cmd = subparsers.add_parser('trade', help='Trade module.', - parents=[_common_parser, _strategy_parser]) + trade_cmd = subparsers.add_parser( + 'trade', + help='Trade module.', + parents=[_common_parser, _strategy_parser] + ) trade_cmd.set_defaults(func=start_trading) self._build_args(optionlist=ARGS_TRADE, parser=trade_cmd) # add create-userdir subcommand - create_userdir_cmd = subparsers.add_parser('create-userdir', - help="Create user-data directory.", - ) + create_userdir_cmd = subparsers.add_parser( + 'create-userdir', + help="Create user-data directory.", + ) create_userdir_cmd.set_defaults(func=start_create_userdir) self._build_args(optionlist=ARGS_CREATE_USERDIR, parser=create_userdir_cmd) # add new-config subcommand - build_config_cmd = subparsers.add_parser('new-config', - help="Create new config") + build_config_cmd = subparsers.add_parser( + 'new-config', + help="Create new config", + ) build_config_cmd.set_defaults(func=start_new_config) self._build_args(optionlist=ARGS_BUILD_CONFIG, parser=build_config_cmd) # add new-strategy subcommand - build_strategy_cmd = subparsers.add_parser('new-strategy', - help="Create new strategy") + build_strategy_cmd = subparsers.add_parser( + 'new-strategy', + help="Create new strategy", + ) build_strategy_cmd.set_defaults(func=start_new_strategy) self._build_args(optionlist=ARGS_BUILD_STRATEGY, parser=build_strategy_cmd) @@ -289,8 +297,11 @@ class Arguments: self._build_args(optionlist=ARGS_LIST_DATA, parser=list_data_cmd) # Add backtesting subcommand - backtesting_cmd = subparsers.add_parser('backtesting', help='Backtesting module.', - parents=[_common_parser, _strategy_parser]) + backtesting_cmd = subparsers.add_parser( + 'backtesting', + help='Backtesting module.', + parents=[_common_parser, _strategy_parser] + ) backtesting_cmd.set_defaults(func=start_backtesting) self._build_args(optionlist=ARGS_BACKTEST, parser=backtesting_cmd) @@ -304,22 +315,29 @@ class Arguments: self._build_args(optionlist=ARGS_BACKTEST_SHOW, parser=backtesting_show_cmd) # Add backtesting analysis subcommand - analysis_cmd = subparsers.add_parser('backtesting-analysis', - help='Backtest Analysis module.', - parents=[_common_parser]) + analysis_cmd = subparsers.add_parser( + 'backtesting-analysis', + help='Backtest Analysis module.', + parents=[_common_parser] + ) analysis_cmd.set_defaults(func=start_analysis_entries_exits) self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd) # Add edge subcommand - edge_cmd = subparsers.add_parser('edge', help='Edge module.', - parents=[_common_parser, _strategy_parser]) + edge_cmd = subparsers.add_parser( + 'edge', + help='Edge module.', + parents=[_common_parser, _strategy_parser] + ) edge_cmd.set_defaults(func=start_edge) self._build_args(optionlist=ARGS_EDGE, parser=edge_cmd) # Add hyperopt subcommand - hyperopt_cmd = subparsers.add_parser('hyperopt', help='Hyperopt module.', - parents=[_common_parser, _strategy_parser], - ) + hyperopt_cmd = subparsers.add_parser( + 'hyperopt', + help='Hyperopt module.', + parents=[_common_parser, _strategy_parser], + ) hyperopt_cmd.set_defaults(func=start_hyperopt) self._build_args(optionlist=ARGS_HYPEROPT, parser=hyperopt_cmd) @@ -447,16 +465,20 @@ class Arguments: self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd) # Add webserver subcommand - webserver_cmd = subparsers.add_parser('webserver', help='Webserver module.', - parents=[_common_parser]) + webserver_cmd = subparsers.add_parser( + 'webserver', + help='Webserver module.', + parents=[_common_parser] + ) webserver_cmd.set_defaults(func=start_webserver) self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd) # Add strategy_updater subcommand - strategy_updater_cmd = subparsers.add_parser('strategy-updater', - help='updates outdated strategy' - 'files to the current version', - parents=[_common_parser]) + strategy_updater_cmd = subparsers.add_parser( + 'strategy-updater', + help='updates outdated strategy files to the current version', + parents=[_common_parser] + ) strategy_updater_cmd.set_defaults(func=start_strategy_update) self._build_args(optionlist=ARGS_STRATEGY_UPDATER, parser=strategy_updater_cmd) @@ -464,8 +486,8 @@ class Arguments: lookahead_analayis_cmd = subparsers.add_parser( 'lookahead-analysis', help="Check for potential look ahead bias.", - parents=[_common_parser, _strategy_parser]) - + parents=[_common_parser, _strategy_parser] + ) lookahead_analayis_cmd.set_defaults(func=start_lookahead_analysis) self._build_args(optionlist=ARGS_LOOKAHEAD_ANALYSIS, @@ -475,8 +497,8 @@ class Arguments: recursive_analayis_cmd = subparsers.add_parser( 'recursive-analysis', help="Check for potential recursive formula issue.", - parents=[_common_parser, _strategy_parser]) - + parents=[_common_parser, _strategy_parser] + ) recursive_analayis_cmd.set_defaults(func=start_recursive_analysis) self._build_args(optionlist=ARGS_RECURSIVE_ANALYSIS, diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 2dff87a7d..5b3447d70 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -109,7 +109,7 @@ def ask_user_config() -> Dict[str, Any]: "binance", "binanceus", "gate", - "huobi", + "htx", "kraken", "kucoin", "okx", diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 229373400..33069885a 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -12,7 +12,7 @@ from freqtrade.enums import RunMode, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes from freqtrade.resolvers import ExchangeResolver -from freqtrade.util.binance_mig import migrate_binance_futures_data +from freqtrade.util.migrations import migrate_data logger = logging.getLogger(__name__) @@ -78,7 +78,7 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: """ config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) if ohlcv: - migrate_binance_futures_data(config) + migrate_data(config) convert_ohlcv_format(config, convert_from=args['format_from'], convert_to=args['format_to'], @@ -134,10 +134,10 @@ def start_list_data(args: Dict[str, Any]) -> None: print(tabulate([ (pair, timeframe, candle_type, start.strftime(DATETIME_PRINT_FORMAT), - end.strftime(DATETIME_PRINT_FORMAT)) - for pair, timeframe, candle_type, start, end in sorted( + end.strftime(DATETIME_PRINT_FORMAT), length) + for pair, timeframe, candle_type, start, end, length in sorted( paircombs1, key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2])) ], - headers=("Pair", "Timeframe", "Type", 'From', 'To'), + headers=("Pair", "Timeframe", "Type", 'From', 'To', 'Candles'), tablefmt='psql', stralign='right')) diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index 2238c5e0a..837a5e4f3 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -5,7 +5,7 @@ from freqtrade import constants from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.misc import round_coin_value +from freqtrade.util import fmt_coin logger = logging.getLogger(__name__) @@ -29,8 +29,8 @@ def setup_optimize_configuration(args: Dict[str, Any], method: RunMode) -> Dict[ # tradable_balance_ratio if (config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT and config['stake_amount'] > wallet_size): - wallet = round_coin_value(wallet_size, config['stake_currency']) - stake = round_coin_value(config['stake_amount'], config['stake_currency']) + wallet = fmt_coin(wallet_size, config['stake_currency']) + stake = fmt_coin(config['stake_amount'], config['stake_currency']) raise OperationalException( f"Starting balance ({wallet}) is smaller than stake_amount {stake}. " f"Wallet is calculated as `dry_run_wallet * tradable_balance_ratio`." diff --git a/freqtrade/commands/pairlist_commands.py b/freqtrade/commands/pairlist_commands.py index a815cd5f3..b1df7e98b 100644 --- a/freqtrade/commands/pairlist_commands.py +++ b/freqtrade/commands/pairlist_commands.py @@ -15,6 +15,7 @@ def start_test_pairlist(args: Dict[str, Any]) -> None: """ Test Pairlist configuration """ + from freqtrade.persistence import FtNoDBContext from freqtrade.plugins.pairlistmanager import PairListManager config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) @@ -24,11 +25,12 @@ def start_test_pairlist(args: Dict[str, Any]) -> None: if not quote_currencies: quote_currencies = [config.get('stake_currency')] results = {} - for curr in quote_currencies: - config['stake_currency'] = curr - pairlists = PairListManager(exchange, config) - pairlists.refresh_pairlist() - results[curr] = pairlists.whitelist + with FtNoDBContext(): + for curr in quote_currencies: + config['stake_currency'] = curr + pairlists = PairListManager(exchange, config) + pairlists.refresh_pairlist() + results[curr] = pairlists.whitelist for curr, pairlist in results.items(): if not args.get('print_one_column', False) and not args.get('list_pairs_print_json', False): diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index e5e4d28a0..58b1fdf84 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -5,7 +5,7 @@ import logging import warnings from copy import deepcopy from pathlib import Path -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Tuple from freqtrade import constants from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings @@ -68,8 +68,10 @@ class Configuration: config: Config = load_from_files(self.args.get("config", [])) # Load environment variables - env_data = enironment_vars_to_dict() - config = deep_merge_dicts(env_data, config) + from freqtrade.commands.arguments import NO_CONF_ALLOWED + if self.args.get('command') not in NO_CONF_ALLOWED: + env_data = enironment_vars_to_dict() + config = deep_merge_dicts(env_data, config) # Normalize config if 'internals' not in config: @@ -233,54 +235,37 @@ class Configuration: except ValueError: pass - self._args_to_config(config, argname='timeframe_detail', - logstring='Parameter --timeframe-detail detected, ' - 'using {} for intra-candle backtesting ...') + configurations = [ + ('timeframe_detail', + 'Parameter --timeframe-detail detected, using {} for intra-candle backtesting ...'), + ('backtest_show_pair_list', 'Parameter --show-pair-list detected.'), + ('stake_amount', + 'Parameter --stake-amount detected, overriding stake_amount to: {} ...'), + ('dry_run_wallet', + 'Parameter --dry-run-wallet detected, overriding dry_run_wallet to: {} ...'), + ('fee', 'Parameter --fee detected, setting fee to: {} ...'), + ('timerange', 'Parameter --timerange detected: {} ...'), + ] - self._args_to_config(config, argname='backtest_show_pair_list', - logstring='Parameter --show-pair-list detected.') - - self._args_to_config(config, argname='stake_amount', - logstring='Parameter --stake-amount detected, ' - 'overriding stake_amount to: {} ...') - self._args_to_config(config, argname='dry_run_wallet', - logstring='Parameter --dry-run-wallet detected, ' - 'overriding dry_run_wallet to: {} ...') - self._args_to_config(config, argname='fee', - logstring='Parameter --fee detected, ' - 'setting fee to: {} ...') - - self._args_to_config(config, argname='timerange', - logstring='Parameter --timerange detected: {} ...') + self._args_to_config_loop(config, configurations) self._process_datadir_options(config) self._args_to_config(config, argname='strategy_list', logstring='Using strategy list of {} strategies', logfun=len) - self._args_to_config( - config, - argname='recursive_strategy_search', - logstring='Recursively searching for a strategy in the strategies folder.', - ) - - self._args_to_config(config, argname='timeframe', - logstring='Overriding timeframe with Command line argument') - - self._args_to_config(config, argname='export', - logstring='Parameter --export detected: {} ...') - - self._args_to_config(config, argname='backtest_breakdown', - logstring='Parameter --breakdown detected ...') - - self._args_to_config(config, argname='backtest_cache', - logstring='Parameter --cache={} detected ...') - - self._args_to_config(config, argname='disableparamexport', - logstring='Parameter --disableparamexport detected: {} ...') - - self._args_to_config(config, argname='freqai_backtest_live_models', - logstring='Parameter --freqai-backtest-live-models detected ...') + configurations = [ + ('recursive_strategy_search', + 'Recursively searching for a strategy in the strategies folder.'), + ('timeframe', 'Overriding timeframe with Command line argument'), + ('export', 'Parameter --export detected: {} ...'), + ('backtest_breakdown', 'Parameter --breakdown detected ...'), + ('backtest_cache', 'Parameter --cache={} detected ...'), + ('disableparamexport', 'Parameter --disableparamexport detected: {} ...'), + ('freqai_backtest_live_models', + 'Parameter --freqai-backtest-live-models detected ...'), + ] + self._args_to_config_loop(config, configurations) # Edge section: if 'stoploss_range' in self.args and self.args["stoploss_range"]: @@ -291,31 +276,18 @@ class Configuration: logger.info('Parameter --stoplosses detected: %s ...', self.args["stoploss_range"]) # Hyperopt section - self._args_to_config(config, argname='hyperopt', - logstring='Using Hyperopt class name: {}') - self._args_to_config(config, argname='hyperopt_path', - logstring='Using additional Hyperopt lookup path: {}') - - self._args_to_config(config, argname='hyperoptexportfilename', - logstring='Using hyperopt file: {}') - - self._args_to_config(config, argname='lookahead_analysis_exportfilename', - logstring='Saving lookahead analysis results into {} ...') - - self._args_to_config(config, argname='epochs', - logstring='Parameter --epochs detected ... ' - 'Will run Hyperopt with for {} epochs ...' - ) - - self._args_to_config(config, argname='spaces', - logstring='Parameter -s/--spaces detected: {}') - - self._args_to_config(config, argname='analyze_per_epoch', - logstring='Parameter --analyze-per-epoch detected.') - - self._args_to_config(config, argname='print_all', - logstring='Parameter --print-all detected ...') + configurations = [ + ('hyperopt', 'Using Hyperopt class name: {}'), + ('hyperopt_path', 'Using additional Hyperopt lookup path: {}'), + ('hyperoptexportfilename', 'Using hyperopt file: {}'), + ('lookahead_analysis_exportfilename', 'Saving lookahead analysis results into {} ...'), + ('epochs', 'Parameter --epochs detected ... Will run Hyperopt with for {} epochs ...'), + ('spaces', 'Parameter -s/--spaces detected: {}'), + ('analyze_per_epoch', 'Parameter --analyze-per-epoch detected.'), + ('print_all', 'Parameter --print-all detected ...'), + ] + self._args_to_config_loop(config, configurations) if 'print_colorized' in self.args and not self.args["print_colorized"]: logger.info('Parameter --no-color detected ...') @@ -323,123 +295,55 @@ class Configuration: else: config.update({'print_colorized': True}) - self._args_to_config(config, argname='print_json', - logstring='Parameter --print-json detected ...') + configurations = [ + ('print_json', 'Parameter --print-json detected ...'), + ('export_csv', 'Parameter --export-csv detected: {}'), + ('hyperopt_jobs', 'Parameter -j/--job-workers detected: {}'), + ('hyperopt_random_state', 'Parameter --random-state detected: {}'), + ('hyperopt_min_trades', 'Parameter --min-trades detected: {}'), + ('hyperopt_loss', 'Using Hyperopt loss class name: {}'), + ('hyperopt_show_index', 'Parameter -n/--index detected: {}'), + ('hyperopt_list_best', 'Parameter --best detected: {}'), + ('hyperopt_list_profitable', 'Parameter --profitable detected: {}'), + ('hyperopt_list_min_trades', 'Parameter --min-trades detected: {}'), + ('hyperopt_list_max_trades', 'Parameter --max-trades detected: {}'), + ('hyperopt_list_min_avg_time', 'Parameter --min-avg-time detected: {}'), + ('hyperopt_list_max_avg_time', 'Parameter --max-avg-time detected: {}'), + ('hyperopt_list_min_avg_profit', 'Parameter --min-avg-profit detected: {}'), + ('hyperopt_list_max_avg_profit', 'Parameter --max-avg-profit detected: {}'), + ('hyperopt_list_min_total_profit', 'Parameter --min-total-profit detected: {}'), + ('hyperopt_list_max_total_profit', 'Parameter --max-total-profit detected: {}'), + ('hyperopt_list_min_objective', 'Parameter --min-objective detected: {}'), + ('hyperopt_list_max_objective', 'Parameter --max-objective detected: {}'), + ('hyperopt_list_no_details', 'Parameter --no-details detected: {}'), + ('hyperopt_show_no_header', 'Parameter --no-header detected: {}'), + ('hyperopt_ignore_missing_space', 'Paramter --ignore-missing-space detected: {}'), + ] - self._args_to_config(config, argname='export_csv', - logstring='Parameter --export-csv detected: {}') - - self._args_to_config(config, argname='hyperopt_jobs', - logstring='Parameter -j/--job-workers detected: {}') - - self._args_to_config(config, argname='hyperopt_random_state', - logstring='Parameter --random-state detected: {}') - - self._args_to_config(config, argname='hyperopt_min_trades', - logstring='Parameter --min-trades detected: {}') - - self._args_to_config(config, argname='hyperopt_loss', - logstring='Using Hyperopt loss class name: {}') - - self._args_to_config(config, argname='hyperopt_show_index', - logstring='Parameter -n/--index detected: {}') - - self._args_to_config(config, argname='hyperopt_list_best', - logstring='Parameter --best detected: {}') - - self._args_to_config(config, argname='hyperopt_list_profitable', - logstring='Parameter --profitable detected: {}') - - self._args_to_config(config, argname='hyperopt_list_min_trades', - logstring='Parameter --min-trades detected: {}') - - self._args_to_config(config, argname='hyperopt_list_max_trades', - logstring='Parameter --max-trades detected: {}') - - self._args_to_config(config, argname='hyperopt_list_min_avg_time', - logstring='Parameter --min-avg-time detected: {}') - - self._args_to_config(config, argname='hyperopt_list_max_avg_time', - logstring='Parameter --max-avg-time detected: {}') - - self._args_to_config(config, argname='hyperopt_list_min_avg_profit', - logstring='Parameter --min-avg-profit detected: {}') - - self._args_to_config(config, argname='hyperopt_list_max_avg_profit', - logstring='Parameter --max-avg-profit detected: {}') - - self._args_to_config(config, argname='hyperopt_list_min_total_profit', - logstring='Parameter --min-total-profit detected: {}') - - self._args_to_config(config, argname='hyperopt_list_max_total_profit', - logstring='Parameter --max-total-profit detected: {}') - - self._args_to_config(config, argname='hyperopt_list_min_objective', - logstring='Parameter --min-objective detected: {}') - - self._args_to_config(config, argname='hyperopt_list_max_objective', - logstring='Parameter --max-objective detected: {}') - - self._args_to_config(config, argname='hyperopt_list_no_details', - logstring='Parameter --no-details detected: {}') - - self._args_to_config(config, argname='hyperopt_show_no_header', - logstring='Parameter --no-header detected: {}') - - self._args_to_config(config, argname="hyperopt_ignore_missing_space", - logstring="Paramter --ignore-missing-space detected: {}") + self._args_to_config_loop(config, configurations) def _process_plot_options(self, config: Config) -> None: - self._args_to_config(config, argname='pairs', - logstring='Using pairs {}') - - self._args_to_config(config, argname='indicators1', - logstring='Using indicators1: {}') - - self._args_to_config(config, argname='indicators2', - logstring='Using indicators2: {}') - - self._args_to_config(config, argname='trade_ids', - logstring='Filtering on trade_ids: {}') - - self._args_to_config(config, argname='plot_limit', - logstring='Limiting plot to: {}') - - self._args_to_config(config, argname='plot_auto_open', - logstring='Parameter --auto-open detected.') - - self._args_to_config(config, argname='trade_source', - logstring='Using trades from: {}') - - self._args_to_config(config, argname='prepend_data', - logstring='Prepend detected. Allowing data prepending.') - self._args_to_config(config, argname='erase', - logstring='Erase detected. Deleting existing data.') - - self._args_to_config(config, argname='no_trades', - logstring='Parameter --no-trades detected.') - - self._args_to_config(config, argname='timeframes', - logstring='timeframes --timeframes: {}') - - self._args_to_config(config, argname='days', - logstring='Detected --days: {}') - - self._args_to_config(config, argname='include_inactive', - logstring='Detected --include-inactive-pairs: {}') - - self._args_to_config(config, argname='download_trades', - logstring='Detected --dl-trades: {}') - - self._args_to_config(config, argname='dataformat_ohlcv', - logstring='Using "{}" to store OHLCV data.') - - self._args_to_config(config, argname='dataformat_trades', - logstring='Using "{}" to store trades data.') - - self._args_to_config(config, argname='show_timerange', - logstring='Detected --show-timerange') + configurations = [ + ('pairs', 'Using pairs {}'), + ('indicators1', 'Using indicators1: {}'), + ('indicators2', 'Using indicators2: {}'), + ('trade_ids', 'Filtering on trade_ids: {}'), + ('plot_limit', 'Limiting plot to: {}'), + ('plot_auto_open', 'Parameter --auto-open detected.'), + ('trade_source', 'Using trades from: {}'), + ('prepend_data', 'Prepend detected. Allowing data prepending.'), + ('erase', 'Erase detected. Deleting existing data.'), + ('no_trades', 'Parameter --no-trades detected.'), + ('timeframes', 'timeframes --timeframes: {}'), + ('days', 'Detected --days: {}'), + ('include_inactive', 'Detected --include-inactive-pairs: {}'), + ('download_trades', 'Detected --dl-trades: {}'), + ('dataformat_ohlcv', 'Using "{}" to store OHLCV data.'), + ('dataformat_trades', 'Using "{}" to store trades data.'), + ('show_timerange', 'Detected --show-timerange'), + ] + self._args_to_config_loop(config, configurations) def _process_data_options(self, config: Config) -> None: self._args_to_config(config, argname='new_pairs_days', @@ -453,45 +357,27 @@ class Configuration: logstring='Detected --candle-types: {}') def _process_analyze_options(self, config: Config) -> None: - self._args_to_config(config, argname='analysis_groups', - logstring='Analysis reason groups: {}') + configurations = [ + ('analysis_groups', 'Analysis reason groups: {}'), + ('enter_reason_list', 'Analysis enter tag list: {}'), + ('exit_reason_list', 'Analysis exit tag list: {}'), + ('indicator_list', 'Analysis indicator list: {}'), + ('timerange', 'Filter trades by timerange: {}'), + ('analysis_rejected', 'Analyse rejected signals: {}'), + ('analysis_to_csv', 'Store analysis tables to CSV: {}'), + ('analysis_csv_path', 'Path to store analysis CSVs: {}'), + # Lookahead analysis results + ('targeted_trade_amount', 'Targeted Trade amount: {}'), + ('minimum_trade_amount', 'Minimum Trade amount: {}'), + ('lookahead_analysis_exportfilename', 'Path to store lookahead-analysis-results: {}'), + ('startup_candle', 'Startup candle to be used on recursive analysis: {}'), + ] + self._args_to_config_loop(config, configurations) - self._args_to_config(config, argname='enter_reason_list', - logstring='Analysis enter tag list: {}') + def _args_to_config_loop(self, config, configurations: List[Tuple[str, str]]) -> None: - self._args_to_config(config, argname='exit_reason_list', - logstring='Analysis exit tag list: {}') - - self._args_to_config(config, argname='indicator_list', - logstring='Analysis indicator list: {}') - - self._args_to_config(config, argname='timerange', - logstring='Filter trades by timerange: {}') - - self._args_to_config(config, argname='analysis_rejected', - logstring='Analyse rejected signals: {}') - - self._args_to_config(config, argname='analysis_to_csv', - logstring='Store analysis tables to CSV: {}') - - self._args_to_config(config, argname='analysis_csv_path', - logstring='Path to store analysis CSVs: {}') - - self._args_to_config(config, argname='analysis_csv_path', - logstring='Path to store analysis CSVs: {}') - - # Lookahead analysis results - self._args_to_config(config, argname='targeted_trade_amount', - logstring='Targeted Trade amount: {}') - - self._args_to_config(config, argname='minimum_trade_amount', - logstring='Minimum Trade amount: {}') - - self._args_to_config(config, argname='lookahead_analysis_exportfilename', - logstring='Path to store lookahead-analysis-results: {}') - - self._args_to_config(config, argname='startup_candle', - logstring='Startup candle to be used on recursive analysis: {}') + for argname, logstring in configurations: + self._args_to_config(config, argname=argname, logstring=logstring) def _process_runmode(self, config: Config) -> None: diff --git a/freqtrade/configuration/environment_vars.py b/freqtrade/configuration/environment_vars.py index c5efd45b7..b59b10fa2 100644 --- a/freqtrade/configuration/environment_vars.py +++ b/freqtrade/configuration/environment_vars.py @@ -9,7 +9,7 @@ from freqtrade.misc import deep_merge_dicts logger = logging.getLogger(__name__) -def get_var_typed(val): +def _get_var_typed(val): try: return int(val) except ValueError: @@ -24,7 +24,7 @@ def get_var_typed(val): return val -def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, Any]: +def _flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, Any]: """ Environment variables must be prefixed with FREQTRADE. FREQTRADE__{section}__{key} @@ -40,7 +40,7 @@ def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, logger.info(f"Loading variable '{env_var}'") key = env_var.replace(prefix, '') for k in reversed(key.split('__')): - val = {k.lower(): get_var_typed(val) + val = {k.lower(): _get_var_typed(val) if not isinstance(val, dict) and k not in no_convert else val} relevant_vars = deep_merge_dicts(val, relevant_vars) return relevant_vars @@ -52,4 +52,4 @@ def enironment_vars_to_dict() -> Dict[str, Any]: Relevant variables must follow the FREQTRADE__{section}__{key} pattern :return: Nested dict based on available and relevant variables. """ - return flat_vars_to_nested_dict(os.environ.copy(), ENV_VAR_PREFIX) + return _flat_vars_to_nested_dict(os.environ.copy(), ENV_VAR_PREFIX) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 060f2e53c..ce9080f6e 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -33,9 +33,10 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss', 'ProfitDrawDownHyperOptLoss'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', 'RemotePairList', - 'AgeFilter', "FullTradesFilter", 'OffsetFilter', 'PerformanceFilter', - 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', - 'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter'] + 'MarketCapPairList', 'AgeFilter', "FullTradesFilter", 'OffsetFilter', + 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', + 'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter', + 'VolatilityFilter'] AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5', 'feather', 'parquet'] @@ -107,7 +108,7 @@ SUPPORTED_FIAT = [ "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "UAH", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", - "USD", "BTC", "ETH", "XRP", "LTC", "BCH" + "USD", "BTC", "ETH", "XRP", "LTC", "BCH", "BNB" ] MINIMAL_CONFIG = { diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 96ab4927e..9d9b17552 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -175,36 +175,40 @@ def _get_backtest_files(dirname: Path) -> List[Path]: return list(reversed(sorted(dirname.glob('backtest-result-*-[0-9][0-9].json')))) -def get_backtest_result(filename: Path) -> List[BacktestHistoryEntryType]: - """ - Get backtest result read from metadata file - """ +def _extract_backtest_result(filename: Path) -> List[BacktestHistoryEntryType]: + metadata = load_backtest_metadata(filename) return [ { 'filename': filename.stem, 'strategy': s, - 'notes': v.get('notes', ''), 'run_id': v['run_id'], + 'notes': v.get('notes', ''), + # Backtest "run" time 'backtest_start_time': v['backtest_start_time'], - } for s, v in load_backtest_metadata(filename).items() + # Backtest timerange + 'backtest_start_ts': v.get('backtest_start_ts', None), + 'backtest_end_ts': v.get('backtest_end_ts', None), + 'timeframe': v.get('timeframe', None), + 'timeframe_detail': v.get('timeframe_detail', None), + } for s, v in metadata.items() ] +def get_backtest_result(filename: Path) -> List[BacktestHistoryEntryType]: + """ + Get backtest result read from metadata file + """ + return _extract_backtest_result(filename) + + def get_backtest_resultlist(dirname: Path) -> List[BacktestHistoryEntryType]: """ Get list of backtest results read from metadata files """ return [ - { - 'filename': filename.stem, - 'strategy': s, - 'run_id': v['run_id'], - 'notes': v.get('notes', ''), - 'backtest_start_time': v['backtest_start_time'], - } + result for filename in _get_backtest_files(dirname) - for s, v in load_backtest_metadata(filename).items() - if v + for result in _extract_backtest_result(filename) ] @@ -326,7 +330,10 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non "Please specify a strategy.") if strategy not in data['strategy']: - raise ValueError(f"Strategy {strategy} not available in the backtest result.") + raise ValueError( + f"Strategy {strategy} not available in the backtest result. " + f"Available strategies are '{','.join(data['strategy'].keys())}'" + ) data = data['strategy'][strategy]['trades'] df = pd.DataFrame(data) @@ -350,10 +357,10 @@ def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataF :param timeframe: Timeframe used for backtest :return: dataframe with open-counts per time-period in timeframe """ - from freqtrade.exchange import timeframe_to_minutes - timeframe_min = timeframe_to_minutes(timeframe) + from freqtrade.exchange import timeframe_to_resample_freq + timeframe_freq = timeframe_to_resample_freq(timeframe) dates = [pd.Series(pd.date_range(row[1]['open_date'], row[1]['close_date'], - freq=f"{timeframe_min}min")) + freq=timeframe_freq)) for row in results[['open_date', 'close_date']].iterrows()] deltas = [len(x) for x in dates] dates = pd.Series(pd.concat(dates).values, name='date') @@ -361,7 +368,7 @@ def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataF df2 = pd.concat([dates, df2], axis=1) df2 = df2.set_index('date') - df_final = df2.resample(f"{timeframe_min}min")[['pair']].count() + df_final = df2.resample(timeframe_freq)[['pair']].count() df_final = df_final.rename({'pair': 'open_trades'}, axis=1) return df_final diff --git a/freqtrade/data/converter/converter.py b/freqtrade/data/converter/converter.py index 2184bb0a0..96f73e8b1 100644 --- a/freqtrade/data/converter/converter.py +++ b/freqtrade/data/converter/converter.py @@ -431,7 +431,7 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str) using the previous close as price for "open", "high" "low" and "close", volume is set to 0 """ - from freqtrade.exchange import timeframe_to_minutes + from freqtrade.exchange import timeframe_to_resample_freq ohlcv_dict = { 'open': 'first', @@ -440,13 +440,7 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str) 'close': 'last', 'volume': 'sum' } - timeframe_minutes = timeframe_to_minutes(timeframe) - resample_interval = f'{timeframe_minutes}min' - if timeframe_minutes >= 43200 and timeframe_minutes < 525600: - # Monthly candles need special treatment to stick to the 1st of the month - resample_interval = f'{timeframe}S' - elif timeframe_minutes > 43200: - resample_interval = timeframe + resample_interval = timeframe_to_resample_freq(timeframe) # Resample to create "NAN" values df = dataframe.resample(resample_interval, on='date').agg(ohlcv_dict) diff --git a/freqtrade/data/converter/trade_converter.py b/freqtrade/data/converter/trade_converter.py index 0485930a6..bd4efb77e 100644 --- a/freqtrade/data/converter/trade_converter.py +++ b/freqtrade/data/converter/trade_converter.py @@ -70,14 +70,13 @@ def trades_to_ohlcv(trades: DataFrame, timeframe: str) -> DataFrame: :return: OHLCV Dataframe. :raises: ValueError if no trades are provided """ - from freqtrade.exchange import timeframe_to_minutes - timeframe_minutes = timeframe_to_minutes(timeframe) + from freqtrade.exchange import timeframe_to_resample_freq if trades.empty: raise ValueError('Trade-list empty.') df = trades.set_index('date', drop=True) - - df_new = df['price'].resample(f'{timeframe_minutes}min').ohlc() - df_new['volume'] = df['amount'].resample(f'{timeframe_minutes}min').sum() + resample_interval = timeframe_to_resample_freq(timeframe) + df_new = df['price'].resample(resample_interval).ohlc() + df_new['volume'] = df['amount'].resample(resample_interval).sum() df_new['date'] = df_new.index # Drop 0 volume rows df_new = df_new.dropna() diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index bf769df29..856f5d81b 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -313,11 +313,13 @@ class DataProvider: timerange = TimeRange.parse_timerange(None if self._config.get( 'timerange') is None else str(self._config.get('timerange'))) - # It is not necessary to add the training candles, as they - # were already added at the beginning of the backtest. - startup_candles = self.get_required_startup(str(timeframe), False) + startup_candles = self.get_required_startup(str(timeframe)) tf_seconds = timeframe_to_seconds(str(timeframe)) timerange.subtract_start(tf_seconds * startup_candles) + + logger.info(f"Loading data for {pair} {timeframe} " + f"from {timerange.start_fmt} to {timerange.stop_fmt}") + self.__cached_pairs_backtesting[saved_pair] = load_pair_history( pair=pair, timeframe=timeframe, @@ -329,7 +331,7 @@ class DataProvider: ) return self.__cached_pairs_backtesting[saved_pair].copy() - def get_required_startup(self, timeframe: str, add_train_candles: bool = True) -> int: + def get_required_startup(self, timeframe: str) -> int: freqai_config = self._config.get('freqai', {}) if not freqai_config.get('enabled', False): return self._config.get('startup_candle_count', 0) @@ -339,12 +341,11 @@ class DataProvider: # make sure the startupcandles is at least the set maximum indicator periods self._config['startup_candle_count'] = max(startup_candles, max(indicator_periods)) tf_seconds = timeframe_to_seconds(timeframe) - train_candles = 0 - if add_train_candles: - train_candles = freqai_config['train_period_days'] * 86400 / tf_seconds + train_candles = freqai_config['train_period_days'] * 86400 / tf_seconds total_candles = int(self._config['startup_candle_count'] + train_candles) - logger.info(f'Increasing startup_candle_count for freqai to {total_candles}') - return total_candles + logger.info( + f'Increasing startup_candle_count for freqai on {timeframe} to {total_candles}') + return total_candles def get_pair_dataframe( self, diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 4222b4038..cffcb1826 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -8,7 +8,7 @@ from pandas import DataFrame, concat from freqtrade.configuration import TimeRange from freqtrade.constants import (DATETIME_PRINT_FORMAT, DEFAULT_DATAFRAME_COLUMNS, - DL_DATA_TIMEFRAMES, Config) + DL_DATA_TIMEFRAMES, DOCS_LINK, Config) from freqtrade.data.converter import (clean_ohlcv_dataframe, convert_trades_to_ohlcv, ohlcv_to_dataframe, trades_df_remove_duplicates, trades_list_to_df) @@ -18,8 +18,8 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist from freqtrade.util import dt_ts, format_ms_time -from freqtrade.util.binance_mig import migrate_binance_futures_data from freqtrade.util.datetime_helpers import dt_now +from freqtrade.util.migrations import migrate_data logger = logging.getLogger(__name__) @@ -311,15 +311,19 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes # Predefined candletype (and timeframe) depending on exchange # Downloads what is necessary to backtest based on futures data. tf_mark = exchange.get_option('mark_ohlcv_timeframe') + tf_funding_rate = exchange.get_option('funding_fee_timeframe') + fr_candle_type = CandleType.from_string(exchange.get_option('mark_ohlcv_price')) # All exchanges need FundingRate for futures trading. # The timeframe is aligned to the mark-price timeframe. - for funding_candle_type in (CandleType.FUNDING_RATE, fr_candle_type): + combs = ((CandleType.FUNDING_RATE, tf_funding_rate), (fr_candle_type, tf_mark)) + for candle_type_f, tf in combs: + logger.debug(f'Downloading pair {pair}, {candle_type_f}, interval {tf}.') _download_pair_history(pair=pair, process=process, datadir=datadir, exchange=exchange, timerange=timerange, data_handler=data_handler, - timeframe=str(tf_mark), new_pairs_days=new_pairs_days, - candle_type=funding_candle_type, + timeframe=str(tf), new_pairs_days=new_pairs_days, + candle_type=candle_type_f, erase=erase, prepend=prepend) return pairs_not_available @@ -502,6 +506,12 @@ def download_data_main(config: Config) -> None: logger.info(f"About to download pairs: {expanded_pairs}, " f"intervals: {config['timeframes']} to {config['datadir']}") + if len(expanded_pairs) == 0: + logger.warning( + "No pairs available for download. " + "Please make sure you're using the correct Pair naming for your selected trade mode. \n" + f"More info: {DOCS_LINK}/bot-basics/#pair-naming") + for timeframe in config['timeframes']: exchange.validate_timeframes(timeframe) @@ -527,7 +537,7 @@ def download_data_main(config: Config) -> None: "Please use `--dl-trades` instead for this exchange " "(will unfortunately take a long time)." ) - migrate_binance_futures_data(config) + migrate_data(config, exchange) pairs_not_available = refresh_backtest_ohlcv_data( exchange, pairs=expanded_pairs, timeframes=config['timeframes'], datadir=config['datadir'], timerange=timerange, diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index d8c063f2a..47c2dd838 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -94,21 +94,22 @@ class IDataHandler(ABC): """ def ohlcv_data_min_max(self, pair: str, timeframe: str, - candle_type: CandleType) -> Tuple[datetime, datetime]: + candle_type: CandleType) -> Tuple[datetime, datetime, int]: """ Returns the min and max timestamp for the given pair and timeframe. :param pair: Pair to get min/max for :param timeframe: Timeframe to get min/max for :param candle_type: Any of the enum CandleType (must match trading mode!) - :return: (min, max) + :return: (min, max, len) """ - data = self._ohlcv_load(pair, timeframe, None, candle_type) - if data.empty: + df = self._ohlcv_load(pair, timeframe, None, candle_type) + if df.empty: return ( datetime.fromtimestamp(0, tz=timezone.utc), - datetime.fromtimestamp(0, tz=timezone.utc) + datetime.fromtimestamp(0, tz=timezone.utc), + 0, ) - return data.iloc[0]['date'].to_pydatetime(), data.iloc[-1]['date'].to_pydatetime() + return df.iloc[0]['date'].to_pydatetime(), df.iloc[-1]['date'].to_pydatetime(), len(df) @abstractmethod def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange], @@ -403,6 +404,34 @@ class IDataHandler(ABC): return file_old.rename(file_new) + def fix_funding_fee_timeframe(self, ff_timeframe: str): + """ + Temporary method to migrate data from old funding fee timeframe to the correct timeframe + Applies to bybit and okx, where funding-fee and mark candles have different timeframes. + """ + paircombs = self.ohlcv_get_available_data(self._datadir, TradingMode.FUTURES) + funding_rate_combs = [ + f for f in paircombs if f[2] == CandleType.FUNDING_RATE and f[1] != ff_timeframe + ] + + if funding_rate_combs: + logger.warning( + f'Migrating {len(funding_rate_combs)} funding fees to correct timeframe.') + + for pair, timeframe, candletype in funding_rate_combs: + old_name = self._pair_data_filename(self._datadir, pair, timeframe, candletype) + new_name = self._pair_data_filename(self._datadir, pair, ff_timeframe, candletype) + + if not Path(old_name).exists(): + logger.warning(f'{old_name} does not exist, skipping.') + continue + + if Path(new_name).exists(): + logger.warning(f'{new_name} already exists, Removing.') + Path(new_name).unlink() + + Path(old_name).rename(new_name) + def get_datahandlerclass(datatype: str) -> Type[IDataHandler]: """ diff --git a/freqtrade/data/metrics.py b/freqtrade/data/metrics.py index c22dcccef..7b45342bb 100644 --- a/freqtrade/data/metrics.py +++ b/freqtrade/data/metrics.py @@ -61,10 +61,10 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, """ if len(trades) == 0: raise ValueError("Trade dataframe empty.") - from freqtrade.exchange import timeframe_to_minutes - timeframe_minutes = timeframe_to_minutes(timeframe) + from freqtrade.exchange import timeframe_to_resample_freq + timeframe_freq = timeframe_to_resample_freq(timeframe) # Resample to timeframe to make sure trades match candles - _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date' + _trades_sum = trades.resample(timeframe_freq, on='close_date' )[['profit_abs']].sum() df.loc[:, col_name] = _trades_sum['profit_abs'].cumsum() # Set first value to 0 diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 8de9120dc..145332a33 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -6,7 +6,6 @@ from freqtrade.exchange.exchange import Exchange from freqtrade.exchange.binance import Binance from freqtrade.exchange.bitmart import Bitmart from freqtrade.exchange.bitpanda import Bitpanda -from freqtrade.exchange.bittrex import Bittrex from freqtrade.exchange.bitvavo import Bitvavo from freqtrade.exchange.bybit import Bybit from freqtrade.exchange.coinbasepro import Coinbasepro @@ -18,10 +17,11 @@ from freqtrade.exchange.exchange_utils import (ROUND_DOWN, ROUND_UP, amount_to_c market_is_active, price_to_precision, timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date, timeframe_to_prev_date, - timeframe_to_seconds, validate_exchange) + timeframe_to_resample_freq, timeframe_to_seconds, + validate_exchange) from freqtrade.exchange.gate import Gate from freqtrade.exchange.hitbtc import Hitbtc -from freqtrade.exchange.huobi import Huobi +from freqtrade.exchange.htx import Htx from freqtrade.exchange.kraken import Kraken from freqtrade.exchange.kucoin import Kucoin from freqtrade.exchange.okx import Okx diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py deleted file mode 100644 index 69e2f2b8d..000000000 --- a/freqtrade/exchange/bittrex.py +++ /dev/null @@ -1,25 +0,0 @@ -""" Bittrex exchange subclass """ -import logging -from typing import Dict - -from freqtrade.exchange import Exchange - - -logger = logging.getLogger(__name__) - - -class Bittrex(Exchange): - """ - Bittrex exchange class. Contains adjustments needed for Freqtrade to work - with this exchange. - """ - - _ft_has: Dict = { - "ohlcv_candle_limit_per_timeframe": { - '1m': 1440, - '5m': 288, - '1h': 744, - '1d': 365, - }, - "l2_limit_range": [1, 25, 500], - } diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index ca986d2be..72ad774b6 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -48,13 +48,14 @@ MAP_EXCHANGE_CHILDCLASS = { 'binanceusdm': 'binance', 'okex': 'okx', 'gateio': 'gate', + 'huboi': 'htx', } SUPPORTED_EXCHANGES = [ 'binance', 'bitmart', 'gate', - 'huobi', + 'htx', 'kraken', 'okx', ] diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 4eb57396c..d8b57fcca 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -81,6 +81,7 @@ class Exchange: "l2_limit_range_required": True, # Allow Empty L2 limit (kucoin) "mark_ohlcv_price": "mark", "mark_ohlcv_timeframe": "8h", + "funding_fee_timeframe": "8h", "ccxt_futures_name": "swap", "needs_trading_fees": False, # use fetch_trading_fees to cache fees "order_props_in_contracts": ['amount', 'filled', 'remaining'], @@ -123,11 +124,12 @@ class Exchange: # Cache for 10 minutes ... self._cache_lock = Lock() self._fetch_tickers_cache: TTLCache = TTLCache(maxsize=2, ttl=60 * 10) - # Cache values for 1800 to avoid frequent polling of the exchange for prices + # Cache values for 300 to avoid frequent polling of the exchange for prices # Caching only applies to RPC methods, so prices for open trades are still # refreshed once every iteration. - self._exit_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) - self._entry_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) + # Shouldn't be too high either, as it'll freeze UI updates in case of open orders. + self._exit_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=300) + self._entry_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=300) # Holds candles self._klines: Dict[PairWithTimeframe, DataFrame] = {} @@ -328,10 +330,11 @@ class Exchange: """ pass - def _log_exchange_response(self, endpoint, response) -> None: + def _log_exchange_response(self, endpoint: str, response, *, add_info=None) -> None: """ Log exchange responses """ if self.log_responses: - logger.info(f"API {endpoint}: {response}") + add_info_str = "" if add_info is None else f" {add_info}: " + logger.info(f"API {endpoint}: {add_info_str}{response}") def ohlcv_candle_limit( self, timeframe: str, candle_type: CandleType, since_ms: Optional[int] = None) -> int: @@ -339,6 +342,7 @@ class Exchange: Exchange ohlcv candle limit Uses ohlcv_candle_limit_per_timeframe if the exchange has different limits per timeframe (e.g. bittrex), otherwise falls back to ohlcv_candle_limit + TODO: this is most likely no longer needed since only bittrex needed this. :param timeframe: Timeframe to check :param candle_type: Candle-type :param since_ms: Starting timestamp @@ -1418,7 +1422,7 @@ class Exchange: order = self.fetch_stoploss_order(order_id, pair) except InvalidOrderException: logger.warning(f"Could not fetch cancelled stoploss order {order_id}.") - order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}} + order = {'id': order_id, 'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}} return order @@ -2564,13 +2568,13 @@ class Exchange: @retrier_async async def _async_fetch_trades(self, pair: str, since: Optional[int] = None, - params: Optional[dict] = None) -> List[List]: + params: Optional[dict] = None) -> Tuple[List[List], Any]: """ Asyncronously gets trade history using fetch_trades. Handles exchange errors, does one call to the exchange. :param pair: Pair to fetch trade data for :param since: Since as integer timestamp in milliseconds - returns: List of dicts containing trades + returns: List of dicts containing trades, the next iteration value (new "since" or trade_id) """ try: candle_limit = self.trades_candle_limit("1m", candle_type=CandleType.FUTURES, since_ms=since) @@ -2586,10 +2590,8 @@ class Exchange: ) trades = await self._api_async.fetch_trades(pair, since=since, limit=candle_limit) trades = self._trades_contracts_to_amount(trades) - - if trades: - logger.debug("Fetched trades for pair %s, datetime: %s (%d).", pair, trades[0]['datetime'], trades[0]['timestamp'] ) - return trades_dict_to_list(trades) + pagination_value = self._get_trade_pagination_next_value(trades) + return trades_dict_to_list(trades), pagination_value except ccxt.NotSupported as e: raise OperationalException( f'Exchange {self._api.name} does not support fetching historical trade data.' @@ -2602,6 +2604,25 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(f'Could not fetch trade data. Msg: {e}') from e + def _valid_trade_pagination_id(self, pair: str, from_id: str) -> bool: + """ + Verify trade-pagination id is valid. + Workaround for odd Kraken issue where ID is sometimes wrong. + """ + return True + + def _get_trade_pagination_next_value(self, trades: List[Dict]): + """ + Extract pagination id for the next "from_id" value + Applies only to fetch_trade_history by id. + """ + if not trades: + return None + if self._trades_pagination == 'id': + return trades[-1].get('id') + else: + return trades[-1].get('timestamp') + async def _async_get_trade_history_id(self, pair: str, until: int, since: Optional[int] = None, @@ -2618,39 +2639,37 @@ class Exchange: """ trades: List[List] = [] + # DEFAULT_TRADES_COLUMNS: 0 -> timestamp + # DEFAULT_TRADES_COLUMNS: 1 -> id + has_overlap = self._ft_has.get('trades_pagination_overlap', True) + # Skip last trade by default since its the key for the next call + x = slice(None, -1) if has_overlap else slice(None) if not until and not stop_on_from_id: raise "stop_on_from_id must be set if until is not set" - - if not from_id: + if not from_id or not self._valid_trade_pagination_id(pair, from_id): # Fetch first elements using timebased method to get an ID to paginate on # Depending on the Exchange, this can introduce a drift at the start of the interval # of up to an hour. # e.g. Binance returns the "last 1000" candles within a 1h time interval # - so we will miss the first trades. - trade = await self._async_fetch_trades(pair, since=since) - if trade: - # DEFAULT_TRADES_COLUMNS: 0 -> timestamp - # DEFAULT_TRADES_COLUMNS: 1 -> id - from_id = trade[-1][1] - trades.extend(trade[:-1]) - else: - return (pair, trades) + t, from_id = await self._async_fetch_trades(pair, since=since) + trades.extend(t[x]) while True: try: - t = await self._async_fetch_trades(pair, - params={self._trades_pagination_arg: from_id}) + t, from_id_next = await self._async_fetch_trades( + pair, params={self._trades_pagination_arg: from_id}) if t: - # Skip last id since its the key for the next call - trades.extend(t[:-1]) - if from_id == t[-1][1] or t[-1][0] > until: + trades.extend(t[x]) + if from_id == from_id_next or t[-1][0] > until: logger.debug(f"Stopping because from_id did not change. " f"Reached {t[-1][0]} > {until}") # Reached the end of the defined-download period - add last trade as well. - trades.extend(t[-1:]) + if has_overlap: + trades.extend(t[-1:]) break - from_id = t[-1][1] + from_id = from_id_next else: logger.debug("Stopping as no more trades were returned.") break @@ -2676,19 +2695,19 @@ class Exchange: # DEFAULT_TRADES_COLUMNS: 1 -> id while True: try: - t = await self._async_fetch_trades(pair, since=since) + t, since_next = await self._async_fetch_trades(pair, since=since) if t: # No more trades to download available at the exchange, # So we repeatedly get the same trade over and over again. - if since == t[-1][0] and len(t) == 1: + if since == since_next and len(t) == 1: logger.debug("Stopping because no more trades are available.") break - since = t[-1][0] + since = since_next trades.extend(t) # Reached the end of the defined-download period - if until and t[-1][0] > until: + if until and since_next > until: logger.debug( - f"Stopping because until was reached. {t[-1][0]} > {until}") + f"Stopping because until was reached. {since_next} > {until}") break else: logger.debug("Stopping as no more trades were returned.") @@ -2806,6 +2825,8 @@ class Exchange: symbol=pair, since=since ) + self._log_exchange_response('funding_history', funding_history, + add_info=f"pair: {pair}, since: {since}") return sum(fee['amount'] for fee in funding_history) except ccxt.DDoSProtection as e: raise DDosProtection(e) from e @@ -3122,17 +3143,16 @@ class Exchange: # Only really relevant for trades very close to the full hour open_date = timeframe_to_prev_date('1h', open_date) timeframe = self._ft_has['mark_ohlcv_timeframe'] - timeframe_ff = self._ft_has.get('funding_fee_timeframe', - self._ft_has['mark_ohlcv_timeframe']) + timeframe_ff = self._ft_has['funding_fee_timeframe'] + mark_price_type = CandleType.from_string(self._ft_has["mark_ohlcv_price"]) if not close_date: close_date = datetime.now(timezone.utc) since_ms = int(timeframe_to_prev_date(timeframe, open_date).timestamp()) * 1000 - mark_comb: PairWithTimeframe = ( - pair, timeframe, CandleType.from_string(self._ft_has["mark_ohlcv_price"])) - + mark_comb: PairWithTimeframe = (pair, timeframe, mark_price_type) funding_comb: PairWithTimeframe = (pair, timeframe_ff, CandleType.FUNDING_RATE) + candle_histories = self.refresh_latest_ohlcv( [mark_comb, funding_comb], since_ms=since_ms, diff --git a/freqtrade/exchange/exchange_utils.py b/freqtrade/exchange/exchange_utils.py index 4dfa97b28..98e05bf7a 100644 --- a/freqtrade/exchange/exchange_utils.py +++ b/freqtrade/exchange/exchange_utils.py @@ -118,6 +118,27 @@ def timeframe_to_msecs(timeframe: str) -> int: return ccxt.Exchange.parse_timeframe(timeframe) * 1000 +def timeframe_to_resample_freq(timeframe: str) -> str: + """ + Translates the timeframe interval value written in the human readable + form ('1m', '5m', '1h', '1d', '1w', etc.) to the resample frequency + used by pandas ('1T', '5T', '1H', '1D', '1W', etc.) + """ + if timeframe == '1y': + return '1YS' + timeframe_seconds = timeframe_to_seconds(timeframe) + timeframe_minutes = timeframe_seconds // 60 + resample_interval = f'{timeframe_seconds}s' + if 10000 < timeframe_minutes < 43200: + resample_interval = '1W-MON' + elif timeframe_minutes >= 43200 and timeframe_minutes < 525600: + # Monthly candles need special treatment to stick to the 1st of the month + resample_interval = f'{timeframe}S' + elif timeframe_minutes > 43200: + resample_interval = timeframe + return resample_interval + + def timeframe_to_prev_date(timeframe: str, date: Optional[datetime] = None) -> datetime: """ Use Timeframe and determine the candle start date for this date. diff --git a/freqtrade/exchange/huobi.py b/freqtrade/exchange/htx.py similarity index 85% rename from freqtrade/exchange/huobi.py rename to freqtrade/exchange/htx.py index 207520862..2e9aff77b 100644 --- a/freqtrade/exchange/huobi.py +++ b/freqtrade/exchange/htx.py @@ -1,4 +1,4 @@ -""" Huobi exchange subclass """ +""" HTX exchange subclass """ import logging from typing import Dict @@ -9,9 +9,9 @@ from freqtrade.exchange import Exchange logger = logging.getLogger(__name__) -class Huobi(Exchange): +class Htx(Exchange): """ - Huobi exchange class. Contains adjustments needed for Freqtrade to work + HTX exchange class. Contains adjustments needed for Freqtrade to work with this exchange. """ diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index 46e34cec8..c8fecdd10 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -8,11 +8,9 @@ from pandas import DataFrame from freqtrade.constants import BuySell from freqtrade.enums import MarginMode, TradingMode -from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException, - OperationalException, TemporaryError) +from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError from freqtrade.exchange import Exchange from freqtrade.exchange.common import retrier -from freqtrade.exchange.exchange_utils import ROUND_DOWN, ROUND_UP from freqtrade.exchange.types import Tickers @@ -24,12 +22,15 @@ class Kraken(Exchange): _params: Dict = {"trading_agreement": "agree"} _ft_has: Dict = { "stoploss_on_exchange": True, - "stop_price_param": "stopPrice", - "stop_price_prop": "stopPrice", + "stop_price_param": "stopLossPrice", + "stop_price_prop": "stopLossPrice", + "stoploss_order_types": {"limit": "limit", "market": "market"}, + "order_time_in_force": ["GTC", "IOC", "PO"], "ohlcv_candle_limit": 720, "ohlcv_has_history": False, "trades_pagination": "id", "trades_pagination_arg": "since", + "trades_pagination_overlap": False, "mark_ohlcv_timeframe": "4h", } @@ -89,75 +90,6 @@ class Kraken(Exchange): except ccxt.BaseError as e: raise OperationalException(e) from e - def stoploss_adjust(self, stop_loss: float, order: Dict, side: str) -> bool: - """ - Verify stop_loss against stoploss-order value (limit or price) - Returns True if adjustment is necessary. - """ - return (order['type'] in ('stop-loss', 'stop-loss-limit') and ( - (side == "sell" and stop_loss > float(order['price'])) or - (side == "buy" and stop_loss < float(order['price'])) - )) - - @retrier(retries=0) - def create_stoploss(self, pair: str, amount: float, stop_price: float, - order_types: Dict, side: BuySell, leverage: float) -> Dict: - """ - Creates a stoploss market order. - Stoploss market orders is the only stoploss type supported by kraken. - TODO: investigate if this can be combined with generic implementation - (careful, prices are reversed) - """ - params = self._params.copy() - if self.trading_mode == TradingMode.FUTURES: - params.update({'reduceOnly': True}) - - round_mode = ROUND_DOWN if side == 'buy' else ROUND_UP - if order_types.get('stoploss', 'market') == 'limit': - ordertype = "stop-loss-limit" - limit_price_pct = order_types.get('stoploss_on_exchange_limit_ratio', 0.99) - if side == "sell": - limit_rate = stop_price * limit_price_pct - else: - limit_rate = stop_price * (2 - limit_price_pct) - params['price2'] = self.price_to_precision(pair, limit_rate, rounding_mode=round_mode) - else: - ordertype = "stop-loss" - - stop_price = self.price_to_precision(pair, stop_price, rounding_mode=round_mode) - - if self._config['dry_run']: - dry_order = self.create_dry_run_order( - pair, ordertype, side, amount, stop_price, leverage, stop_loss=True) - return dry_order - - try: - amount = self.amount_to_precision(pair, amount) - - order = self._api.create_order(symbol=pair, type=ordertype, side=side, - amount=amount, price=stop_price, params=params) - self._log_exchange_response('create_stoploss_order', order) - logger.info('stoploss order added for %s. ' - 'stop price: %s.', pair, stop_price) - return order - except ccxt.InsufficientFunds as e: - raise InsufficientFundsError( - f'Insufficient funds to create {ordertype} {side} order on market {pair}. ' - f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' - f'Message: {e}') from e - except ccxt.InvalidOrder as e: - raise InvalidOrderException( - f'Could not create {ordertype} {side} order on market {pair}. ' - f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' - f'Message: {e}') from e - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not place {side} order due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - def _set_leverage( self, leverage: float, @@ -187,6 +119,9 @@ class Kraken(Exchange): ) if leverage > 1.0: params['leverage'] = round(leverage) + if time_in_force == 'PO': + params.pop('timeInForce', None) + params['postOnly'] = True return params def calculate_funding_fees( @@ -223,18 +158,30 @@ class Kraken(Exchange): return fees if is_short else -fees - def _trades_contracts_to_amount(self, trades: List) -> List: + def _get_trade_pagination_next_value(self, trades: List[Dict]): """ - Fix "last" id issue for kraken data downloads - This whole override can probably be removed once the following - issue is closed in ccxt: https://github.com/ccxt/ccxt/issues/15827 + Extract pagination id for the next "from_id" value + Applies only to fetch_trade_history by id. """ - super()._trades_contracts_to_amount(trades) - if ( - len(trades) > 0 - and isinstance(trades[-1].get('info'), list) - and len(trades[-1].get('info', [])) > 7 - ): + if len(trades) > 0: + if ( + isinstance(trades[-1].get('info'), list) + and len(trades[-1].get('info', [])) > 7 + ): + # Trade response's "last" value. + return trades[-1].get('info', [])[-1] + # Fall back to timestamp if info is somehow empty. + return trades[-1].get('timestamp') + return None - trades[-1]['id'] = trades[-1].get('info', [])[-1] - return trades + def _valid_trade_pagination_id(self, pair: str, from_id: str) -> bool: + """ + Verify trade-pagination id is valid. + Workaround for odd Kraken issue where ID is sometimes wrong. + """ + # Regular id's are in timestamp format 1705443695120072285 + # If the id is smaller than 19 characters, it's not a valid timestamp. + if len(from_id) >= 19: + return True + logger.debug(f"{pair} - trade-pagination id is not valid. Fallback to timestamp.") + return False diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index 7d7c15f49..783a197d2 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -228,7 +228,7 @@ class Okx(Exchange): f'StoplossOrder not found (pair: {pair} id: {order_id}).') def get_order_id_conditional(self, order: Dict[str, Any]) -> str: - if order['type'] == 'stop': + if order.get('type', '') == 'stop': return safe_value_fallback2(order, order, 'id_stop', 'id') return order['id'] diff --git a/freqtrade/freqai/base_models/FreqaiMultiOutputClassifier.py b/freqtrade/freqai/base_models/FreqaiMultiOutputClassifier.py index 435c0e646..4646bb9a8 100644 --- a/freqtrade/freqai/base_models/FreqaiMultiOutputClassifier.py +++ b/freqtrade/freqai/base_models/FreqaiMultiOutputClassifier.py @@ -1,9 +1,8 @@ import numpy as np -from joblib import Parallel from sklearn.base import is_classifier from sklearn.multioutput import MultiOutputClassifier, _fit_estimator -from sklearn.utils.fixes import delayed from sklearn.utils.multiclass import check_classification_targets +from sklearn.utils.parallel import Parallel, delayed from sklearn.utils.validation import has_fit_parameter from freqtrade.exceptions import OperationalException diff --git a/freqtrade/freqai/base_models/FreqaiMultiOutputRegressor.py b/freqtrade/freqai/base_models/FreqaiMultiOutputRegressor.py index 54136d5e0..a6cc4f39b 100644 --- a/freqtrade/freqai/base_models/FreqaiMultiOutputRegressor.py +++ b/freqtrade/freqai/base_models/FreqaiMultiOutputRegressor.py @@ -1,6 +1,5 @@ -from joblib import Parallel from sklearn.multioutput import MultiOutputRegressor, _fit_estimator -from sklearn.utils.fixes import delayed +from sklearn.utils.parallel import Parallel, delayed from sklearn.utils.validation import has_fit_parameter diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 1bdd8b0d5..4c30ce690 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -255,7 +255,7 @@ class FreqaiDataKitchen: if (1 - len(filtered_df) / len(unfiltered_df)) > 0.1 and self.live: worst_indicator = str(unfiltered_df.count().idxmin()) logger.warning( - f" {(1 - len(filtered_df)/len(unfiltered_df)) * 100:.0f} percent " + f" {(1 - len(filtered_df) / len(unfiltered_df)) * 100:.0f} percent " " of training data dropped due to NaNs, model may perform inconsistent " f"with expectations. Verify {worst_indicator}" ) @@ -432,8 +432,12 @@ class FreqaiDataKitchen: if self.freqai_config["feature_parameters"].get("DI_threshold", 0) > 0: append_df["DI_values"] = self.DI_values + user_cols = [col for col in dataframe_backtest.columns if col.startswith("%%")] + cols = ["date"] + cols.extend(user_cols) + dataframe_backtest.reset_index(drop=True, inplace=True) - merged_df = pd.concat([dataframe_backtest["date"], append_df], axis=1) + merged_df = pd.concat([dataframe_backtest[cols], append_df], axis=1) return merged_df def append_predictions(self, append_df: DataFrame) -> None: @@ -451,7 +455,8 @@ class FreqaiDataKitchen: Back fill values to before the backtesting range so that the dataframe matches size when it goes back to the strategy. These rows are not included in the backtest. """ - to_keep = [col for col in dataframe.columns if not col.startswith("&")] + to_keep = [col for col in dataframe.columns if + not col.startswith("&") and not col.startswith("%%")] self.return_dataframe = pd.merge(dataframe[to_keep], self.full_df, how='left', on='date') self.return_dataframe[self.full_df.columns] = ( @@ -709,6 +714,8 @@ class FreqaiDataKitchen: pair, tf, strategy, corr_dataframes, base_dataframes, is_corr_pairs) informative_copy = informative_df.copy() + logger.debug(f"Populating features for {pair} {tf}") + for t in self.freqai_config["feature_parameters"]["indicator_periods_candles"]: df_features = strategy.feature_engineering_expand_all( informative_copy.copy(), t, metadata=metadata) @@ -788,6 +795,7 @@ class FreqaiDataKitchen: if not prediction_dataframe.empty: dataframe = prediction_dataframe.copy() + base_dataframes[self.config["timeframe"]] = dataframe.copy() else: dataframe = base_dataframes[self.config["timeframe"]].copy() diff --git a/freqtrade/freqai/prediction_models/SKLearnRandomForestClassifier.py b/freqtrade/freqai/prediction_models/SKLearnRandomForestClassifier.py new file mode 100644 index 000000000..4462efc49 --- /dev/null +++ b/freqtrade/freqai/prediction_models/SKLearnRandomForestClassifier.py @@ -0,0 +1,82 @@ +import logging +from typing import Any, Dict, Tuple + +import numpy as np +import numpy.typing as npt +from pandas import DataFrame +from sklearn.ensemble import RandomForestClassifier +from sklearn.preprocessing import LabelEncoder + +from freqtrade.freqai.base_models.BaseClassifierModel import BaseClassifierModel +from freqtrade.freqai.data_kitchen import FreqaiDataKitchen + + +logger = logging.getLogger(__name__) + + +class SKLearnRandomForestClassifier(BaseClassifierModel): + """ + User created prediction model. The class inherits IFreqaiModel, which + means it has full access to all Frequency AI functionality. Typically, + users would use this to override the common `fit()`, `train()`, or + `predict()` methods to add their custom data handling tools or change + various aspects of the training that cannot be configured via the + top level config.json file. + """ + + def fit(self, data_dictionary: Dict, dk: FreqaiDataKitchen, **kwargs) -> Any: + """ + User sets up the training and test data to fit their desired model here + :param data_dictionary: the dictionary holding all data for train, test, + labels, weights + :param dk: The datakitchen object for the current coin/model + """ + + X = data_dictionary["train_features"].to_numpy() + y = data_dictionary["train_labels"].to_numpy()[:, 0] + + if self.freqai_info.get('data_split_parameters', {}).get('test_size', 0.1) == 0: + eval_set = None + else: + test_features = data_dictionary["test_features"].to_numpy() + test_labels = data_dictionary["test_labels"].to_numpy()[:, 0] + + eval_set = (test_features, test_labels) + + if self.freqai_info.get("continual_learning", False): + logger.warning("Continual learning is not supported for " + "SKLearnRandomForestClassifier, ignoring.") + + train_weights = data_dictionary["train_weights"] + + model = RandomForestClassifier(**self.model_training_parameters) + + model.fit(X=X, y=y, sample_weight=train_weights) + if eval_set: + logger.info("Score: %s", model.score(eval_set[0], eval_set[1])) + + return model + + def predict( + self, unfiltered_df: DataFrame, dk: FreqaiDataKitchen, **kwargs + ) -> Tuple[DataFrame, npt.NDArray[np.int_]]: + """ + Filter the prediction features data and predict with it. + :param unfiltered_df: Full dataframe for the current backtest period. + :return: + :pred_df: dataframe containing the predictions + :do_predict: np.array of 1s and 0s to indicate places where freqai needed to remove + data (NaNs) or felt uncertain about data (PCA and DI index) + """ + + (pred_df, dk.do_predict) = super().predict(unfiltered_df, dk, **kwargs) + + le = LabelEncoder() + label = dk.label_list[0] + labels_before = list(dk.data['labels_std'].keys()) + labels_after = le.fit_transform(labels_before).tolist() + pred_df[label] = le.inverse_transform(pred_df[label]) + pred_df = pred_df.rename( + columns={labels_after[i]: labels_before[i] for i in range(len(labels_before))}) + + return (pred_df, dk.do_predict) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index bb2c3dd8f..4428d9578 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -13,7 +13,6 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds -from freqtrade.exchange.exchange import market_is_active from freqtrade.freqai.data_drawer import FreqaiDataDrawer from freqtrade.freqai.data_kitchen import FreqaiDataKitchen from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist @@ -33,8 +32,11 @@ def download_all_data_for_training(dp: DataProvider, config: Config) -> None: if dp._exchange is None: raise OperationalException('No exchange object found.') - markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m) - or config.get('include_inactive')] + markets = [ + p for p in dp._exchange.get_markets( + tradable_only=True, active_only=not config.get('include_inactive') + ).keys() + ] all_pairs = dynamic_expand_pairlist(config, markets) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7b820bfc1..0eb1c608a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -18,8 +18,8 @@ from freqtrade.constants import BuySell, Config, EntryExecuteMode, ExchangeConfi from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge -from freqtrade.enums import (ExitCheckTuple, ExitType, RPCMessageType, RunMode, SignalDirection, - State, TradingMode) +from freqtrade.enums import (ExitCheckTuple, ExitType, RPCMessageType, SignalDirection, State, + TradingMode) from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, PricingError) from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, remove_exchange_credentials, @@ -33,12 +33,12 @@ from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.rpc import RPCManager from freqtrade.rpc.external_message_consumer import ExternalMessageConsumer -from freqtrade.rpc.rpc_types import (RPCBuyMsg, RPCCancelMsg, RPCProtectionMsg, RPCSellCancelMsg, - RPCSellMsg) +from freqtrade.rpc.rpc_types import (ProfitLossStr, RPCCancelMsg, RPCEntryMsg, RPCExitCancelMsg, + RPCExitMsg, RPCProtectionMsg) from freqtrade.strategy.interface import IStrategy from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.util import FtPrecise -from freqtrade.util.binance_mig import migrate_binance_futures_names +from freqtrade.util.migrations import migrate_binance_futures_names from freqtrade.wallets import Wallets @@ -83,6 +83,8 @@ class FreqtradeBot(LoggingMixin): PairLocks.timeframe = self.config['timeframe'] self.pairlists = PairListManager(self.exchange, self.config) + self.trading_mode: TradingMode = self.config.get('trading_mode', TradingMode.SPOT) + self.last_process: Optional[datetime] = None # RPC runs in separate threads, can start handling external commands just after # initialization, even before Freqtradebot has a chance to start its throttling, @@ -119,8 +121,6 @@ class FreqtradeBot(LoggingMixin): self._exit_lock = Lock() LoggingMixin.__init__(self, logger, timeframe_to_seconds(self.strategy.timeframe)) - self.trading_mode: TradingMode = self.config.get('trading_mode', TradingMode.SPOT) - self._schedule = Scheduler() if self.trading_mode == TradingMode.FUTURES: @@ -135,7 +135,6 @@ class FreqtradeBot(LoggingMixin): for minutes in [1, 31]: t = str(time(time_slot, minutes, 2)) self._schedule.every().day.at(t).do(update) - self.last_process: Optional[datetime] = None self.strategy.ft_bot_start() # Initialize protections AFTER bot start - otherwise parameters are not loaded. @@ -646,8 +645,7 @@ class FreqtradeBot(LoggingMixin): max_entry_stake = self.exchange.get_max_pair_stake_amount(trade.pair, current_entry_rate) stake_available = self.wallets.get_available_stake_amount() logger.debug(f"Calling adjust_trade_position for pair {trade.pair}") - stake_amount = strategy_safe_wrapper(self.strategy.adjust_trade_position, - default_retval=None, supress_error=True)( + stake_amount, order_tag = self.strategy._adjust_trade_position_internal( trade=trade, current_time=datetime.now(timezone.utc), current_rate=current_entry_rate, current_profit=current_entry_profit, min_stake=min_entry_stake, @@ -666,33 +664,27 @@ class FreqtradeBot(LoggingMixin): else: logger.debug("Max adjustment entries is set to unlimited.") self.execute_entry(trade.pair, stake_amount, price=current_entry_rate, - trade=trade, is_short=trade.is_short, mode='pos_adjust') + trade=trade, is_short=trade.is_short, mode='pos_adjust', + enter_tag=order_tag) if stake_amount is not None and stake_amount < 0.0: # We should decrease our position amount = self.exchange.amount_to_contract_precision( trade.pair, abs(float(FtPrecise(stake_amount * trade.leverage) / FtPrecise(current_exit_rate)))) - if amount > trade.amount: - # This is currently ineffective as remaining would become < min tradable - # Fixing this would require checking for 0.0 there - - # if we decide that this callback is allowed to "fully exit" - logger.info( - f"Adjusting amount to trade.amount as it is higher. {amount} > {trade.amount}") - amount = trade.amount if amount == 0.0: logger.info("Amount to exit is 0.0 due to exchange limits - not exiting.") return remaining = (trade.amount - amount) * current_exit_rate - if min_exit_stake and remaining < min_exit_stake: + if min_exit_stake and remaining != 0 and remaining < min_exit_stake: logger.info(f"Remaining amount of {remaining} would be smaller " f"than the minimum of {min_exit_stake}.") return self.execute_trade_exit(trade, current_exit_rate, exit_check=ExitCheckTuple( - exit_type=ExitType.PARTIAL_EXIT), sub_trade_amt=amount) + exit_type=ExitType.PARTIAL_EXIT), sub_trade_amt=amount, exit_tag=order_tag) def _check_depth_of_market(self, pair: str, conf: Dict, side: SignalDirection) -> bool: """ @@ -790,6 +782,7 @@ class FreqtradeBot(LoggingMixin): leverage=leverage ) order_obj = Order.parse_from_ccxt_object(order, pair, side, amount, enter_limit_requested) + order_obj.ft_order_tag = enter_tag order_id = order['id'] order_status = order.get('status') logger.info(f"Order {order_id} was created for {pair} and status is {order_status}.") @@ -904,7 +897,7 @@ class FreqtradeBot(LoggingMixin): # First cancelling stoploss on exchange ... if trade.stoploss_order_id: try: - logger.info(f"Canceling stoploss on exchange for {trade}") + logger.info(f"Cancelling stoploss on exchange for {trade}") co = self.exchange.cancel_stoploss_order_with_result( trade.stoploss_order_id, trade.pair, trade.amount) self.update_trade_state(trade, trade.stoploss_order_id, co, stoploss_order=True) @@ -1010,12 +1003,10 @@ class FreqtradeBot(LoggingMixin): if open_rate is None: open_rate = trade.open_rate - current_rate = trade.open_rate_requested - if self.dataprovider.runmode in (RunMode.DRY_RUN, RunMode.LIVE): - current_rate = self.exchange.get_rate( - trade.pair, side='entry', is_short=trade.is_short, refresh=False) + current_rate = self.exchange.get_rate( + trade.pair, side='entry', is_short=trade.is_short, refresh=False) - msg: RPCBuyMsg = { + msg: RPCEntryMsg = { 'trade_id': trade.id, 'type': RPCMessageType.ENTRY_FILL if fill else RPCMessageType.ENTRY, 'buy_tag': trade.enter_tag, @@ -1030,6 +1021,7 @@ class FreqtradeBot(LoggingMixin): 'stake_amount': trade.stake_amount, 'stake_currency': self.config['stake_currency'], 'base_currency': self.exchange.get_pair_base_currency(trade.pair), + 'quote_currency': self.exchange.get_pair_quote_currency(trade.pair), 'fiat_currency': self.config.get('fiat_display_currency', None), 'amount': order.safe_amount_after_fee if fill else (order.amount or trade.amount), 'open_date': trade.open_date_utc or datetime.now(timezone.utc), @@ -1063,6 +1055,7 @@ class FreqtradeBot(LoggingMixin): 'open_rate': trade.open_rate, 'stake_currency': self.config['stake_currency'], 'base_currency': self.exchange.get_pair_base_currency(trade.pair), + 'quote_currency': self.exchange.get_pair_quote_currency(trade.pair), 'fiat_currency': self.config.get('fiat_display_currency', None), 'amount': trade.amount, 'open_date': trade.open_date, @@ -1348,9 +1341,11 @@ class FreqtradeBot(LoggingMixin): not_closed = order['status'] == 'open' or fully_cancelled if not_closed: - if fully_cancelled or ( - open_order and self.strategy.ft_check_timed_out( - trade, open_order, datetime.now(timezone.utc) + if ( + fully_cancelled or ( + open_order and self.strategy.ft_check_timed_out( + trade, open_order, datetime.now(timezone.utc) + ) ) ): self.handle_cancel_order( @@ -1430,11 +1425,11 @@ class FreqtradeBot(LoggingMixin): # New candle proposed_rate = self.exchange.get_rate( trade.pair, side='entry', is_short=trade.is_short, refresh=True) - adjusted_entry_price = strategy_safe_wrapper(self.strategy.adjust_entry_price, - default_retval=order_obj.price)( + adjusted_entry_price = strategy_safe_wrapper( + self.strategy.adjust_entry_price, default_retval=order_obj.safe_placement_price)( trade=trade, order=order_obj, pair=trade.pair, current_time=datetime.now(timezone.utc), proposed_rate=proposed_rate, - current_order_rate=order_obj.safe_price, entry_tag=trade.enter_tag, + current_order_rate=order_obj.safe_placement_price, entry_tag=trade.enter_tag, side=trade.trade_direction) replacing = True @@ -1442,7 +1437,7 @@ class FreqtradeBot(LoggingMixin): if not adjusted_entry_price: replacing = False cancel_reason = constants.CANCEL_REASON['USER_CANCEL'] - if order_obj.price != adjusted_entry_price: + if order_obj.safe_placement_price != adjusted_entry_price: # cancel existing order if new price is supplied or None res = self.handle_cancel_enter(trade, order, order_obj, cancel_reason, replacing=replacing) @@ -1759,6 +1754,7 @@ class FreqtradeBot(LoggingMixin): return False order_obj = Order.parse_from_ccxt_object(order, trade.pair, trade.exit_side, amount, limit) + order_obj.ft_order_tag = exit_reason trade.orders.append(order_obj) trade.exit_order_status = '' @@ -1792,9 +1788,9 @@ class FreqtradeBot(LoggingMixin): order_rate = trade.safe_close_rate profit = trade.calculate_profit(rate=order_rate) amount = trade.amount - gain = "profit" if profit.profit_ratio > 0 else "loss" + gain: ProfitLossStr = "profit" if profit.profit_ratio > 0 else "loss" - msg: RPCSellMsg = { + msg: RPCExitMsg = { 'type': (RPCMessageType.EXIT_FILL if fill else RPCMessageType.EXIT), 'trade_id': trade.id, @@ -1810,20 +1806,22 @@ class FreqtradeBot(LoggingMixin): 'open_rate': trade.open_rate, 'close_rate': order_rate, 'current_rate': current_rate, - 'profit_amount': profit.profit_abs if fill else profit.total_profit, + 'profit_amount': profit.profit_abs, 'profit_ratio': profit.profit_ratio, 'buy_tag': trade.enter_tag, 'enter_tag': trade.enter_tag, - 'sell_reason': trade.exit_reason, # Deprecated 'exit_reason': trade.exit_reason, 'open_date': trade.open_date_utc, 'close_date': trade.close_date_utc or datetime.now(timezone.utc), 'stake_amount': trade.stake_amount, 'stake_currency': self.config['stake_currency'], 'base_currency': self.exchange.get_pair_base_currency(trade.pair), + 'quote_currency': self.exchange.get_pair_quote_currency(trade.pair), 'fiat_currency': self.config.get('fiat_display_currency'), 'sub_trade': sub_trade, 'cumulative_profit': trade.realized_profit, + 'final_profit_ratio': trade.close_profit if not trade.is_open else None, + 'is_final_exit': trade.is_open is False, } # Send the message @@ -1846,9 +1844,9 @@ class FreqtradeBot(LoggingMixin): profit = trade.calculate_profit(rate=profit_rate) current_rate = self.exchange.get_rate( trade.pair, side='exit', is_short=trade.is_short, refresh=False) - gain = "profit" if profit.profit_ratio > 0 else "loss" + gain: ProfitLossStr = "profit" if profit.profit_ratio > 0 else "loss" - msg: RPCSellCancelMsg = { + msg: RPCExitCancelMsg = { 'type': RPCMessageType.EXIT_CANCEL, 'trade_id': trade.id, 'exchange': trade.exchange.capitalize(), @@ -1865,12 +1863,12 @@ class FreqtradeBot(LoggingMixin): 'profit_ratio': profit.profit_ratio, 'buy_tag': trade.enter_tag, 'enter_tag': trade.enter_tag, - 'sell_reason': trade.exit_reason, # Deprecated 'exit_reason': trade.exit_reason, 'open_date': trade.open_date, 'close_date': trade.close_date or datetime.now(timezone.utc), 'stake_currency': self.config['stake_currency'], 'base_currency': self.exchange.get_pair_base_currency(trade.pair), + 'quote_currency': self.exchange.get_pair_quote_currency(trade.pair), 'fiat_currency': self.config.get('fiat_display_currency', None), 'reason': reason, 'sub_trade': sub_trade, @@ -1978,15 +1976,16 @@ class FreqtradeBot(LoggingMixin): self, trade: Trade, order: Order, stoploss_order: bool, send_msg: bool): """send "fill" notifications""" - sub_trade = not isclose(order.safe_amount_after_fee, - trade.amount, abs_tol=constants.MATH_CLOSE_PREC) if order.ft_order_side == trade.exit_side: # Exit notification if send_msg and not stoploss_order and order.order_id not in trade.open_orders_ids: - self._notify_exit(trade, '', fill=True, sub_trade=sub_trade, order=order) + self._notify_exit(trade, order.order_type, fill=True, + sub_trade=trade.is_open, order=order) if not trade.is_open: self.handle_protections(trade.pair, trade.trade_direction) elif send_msg and order.order_id not in trade.open_orders_ids and not stoploss_order: + sub_trade = not isclose(order.safe_amount_after_fee, + trade.amount, abs_tol=constants.MATH_CLOSE_PREC) # Enter fill self._notify_enter(trade, order, order.order_type, fill=True, sub_trade=sub_trade) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index bd4f17d05..a6c6f15fd 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -11,41 +11,12 @@ from urllib.parse import urlparse import pandas as pd import rapidjson -from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN from freqtrade.enums import SignalTagType, SignalType logger = logging.getLogger(__name__) -def decimals_per_coin(coin: str): - """ - Helper method getting decimal amount for this coin - example usage: f".{decimals_per_coin('USD')}f" - :param coin: Which coin are we printing the price / value for - """ - return DECIMALS_PER_COIN.get(coin, DECIMAL_PER_COIN_FALLBACK) - - -def round_coin_value( - value: float, coin: str, show_coin_name=True, keep_trailing_zeros=False) -> str: - """ - Get price value for this coin - :param value: Value to be printed - :param coin: Which coin are we printing the price / value for - :param show_coin_name: Return string in format: "222.22 USDT" or "222.22" - :param keep_trailing_zeros: Keep trailing zeros "222.200" vs. "222.2" - :return: Formatted / rounded value (with or without coin name) - """ - val = f"{value:.{decimals_per_coin(coin)}f}" - if not keep_trailing_zeros: - val = val.rstrip('0').rstrip('.') - if show_coin_name: - val = f"{val} {coin}" - - return val - - def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool = True) -> None: """ Dump JSON data into a file diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index b8145b6c8..21e9c75cc 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -33,14 +33,15 @@ from freqtrade.optimize.optimize_reports import (generate_backtest_stats, genera show_backtest_results, store_backtest_analysis_results, store_backtest_stats) -from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade +from freqtrade.persistence import (LocalTrade, Order, PairLocks, Trade, disable_database_use, + enable_database_use) from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.strategy.interface import IStrategy from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.types import BacktestResultType, get_BacktestResultType_default -from freqtrade.util.binance_mig import migrate_binance_futures_data +from freqtrade.util.migrations import migrate_data from freqtrade.wallets import Wallets @@ -116,8 +117,9 @@ class Backtesting: raise OperationalException("Timeframe needs to be set in either " "configuration or as cli argument `--timeframe 5m`") self.timeframe = str(self.config.get('timeframe')) - self.disable_database_use() self.timeframe_min = timeframe_to_minutes(self.timeframe) + self.timeframe_td = timedelta(minutes=self.timeframe_min) + self.disable_database_use() self.init_backtest_detail() self.pairlists = PairListManager(self.exchange, self.config, self.dataprovider) self._validate_pairlists_for_backtesting() @@ -145,19 +147,20 @@ class Backtesting: self.required_startup = max([strat.startup_candle_count for strat in self.strategylist]) self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe) - if self.config.get('freqai', {}).get('enabled', False): - # For FreqAI, increase the required_startup to includes the training data - self.required_startup = self.dataprovider.get_required_startup(self.timeframe) - # Add maximum startup candle count to configuration for informative pairs support self.config['startup_candle_count'] = self.required_startup + if self.config.get('freqai', {}).get('enabled', False): + # For FreqAI, increase the required_startup to includes the training data + # This value should NOT be written to startup_candle_count + self.required_startup = self.dataprovider.get_required_startup(self.timeframe) + self.trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT) # strategies which define "can_short=True" will fail to load in Spot mode. self._can_short = self.trading_mode != TradingMode.SPOT self._position_stacking: bool = self.config.get('position_stacking', False) self.enable_protections: bool = self.config.get('enable_protections', False) - migrate_binance_futures_data(config) + migrate_data(config, self.exchange) self.init_backtest() @@ -176,8 +179,7 @@ class Backtesting: @staticmethod def cleanup(): LoggingMixin.show_output = True - PairLocks.use_db = True - Trade.use_db = True + enable_database_use() def init_backtest_detail(self) -> None: # Load detail timeframe if specified @@ -239,7 +241,7 @@ class Backtesting: pairs=self.pairlists.whitelist, timeframe=self.timeframe, timerange=self.timerange, - startup_candles=self.config['startup_candle_count'], + startup_candles=self.required_startup, fail_without_data=True, data_format=self.config['dataformat_ohlcv'], candle_type=self.config.get('candle_type_def', CandleType.SPOT) @@ -276,8 +278,10 @@ class Backtesting: else: self.detail_data = {} if self.trading_mode == TradingMode.FUTURES: - self.funding_fee_timeframe: str = self.exchange.get_option('mark_ohlcv_timeframe') + self.funding_fee_timeframe: str = self.exchange.get_option('funding_fee_timeframe') self.funding_fee_timeframe_secs: int = timeframe_to_seconds(self.funding_fee_timeframe) + mark_timeframe: str = self.exchange.get_option('mark_ohlcv_timeframe') + # Load additional futures data. funding_rates_dict = history.load_data( datadir=self.config['datadir'], @@ -294,7 +298,7 @@ class Backtesting: mark_rates_dict = history.load_data( datadir=self.config['datadir'], pairs=self.pairlists.whitelist, - timeframe=self.funding_fee_timeframe, + timeframe=mark_timeframe, timerange=self.timerange, startup_candles=0, fail_without_data=True, @@ -322,9 +326,7 @@ class Backtesting: self.futures_data = {} def disable_database_use(self): - PairLocks.use_db = False - PairLocks.timeframe = self.timeframe - Trade.use_db = False + disable_database_use(self.timeframe) def prepare_backtest(self, enable_protections): """ @@ -530,19 +532,19 @@ class Backtesting: def _get_adjust_trade_entry_for_candle( self, trade: LocalTrade, row: Tuple, current_time: datetime ) -> LocalTrade: - current_rate = row[OPEN_IDX] + current_rate: float = row[OPEN_IDX] current_profit = trade.calc_profit_ratio(current_rate) min_stake = self.exchange.get_min_pair_stake_amount(trade.pair, current_rate, -0.1) max_stake = self.exchange.get_max_pair_stake_amount(trade.pair, current_rate) stake_available = self.wallets.get_available_stake_amount() - stake_amount = strategy_safe_wrapper(self.strategy.adjust_trade_position, - default_retval=None, supress_error=True)( + stake_amount, order_tag = self.strategy._adjust_trade_position_internal( trade=trade, # type: ignore[arg-type] current_time=current_time, current_rate=current_rate, current_profit=current_profit, min_stake=min_stake, max_stake=min(max_stake, stake_available), current_entry_rate=current_rate, current_exit_rate=current_rate, - current_entry_profit=current_profit, current_exit_profit=current_profit) + current_entry_profit=current_profit, current_exit_profit=current_profit + ) # Check if we should increase our position if stake_amount is not None and stake_amount > 0.0: @@ -552,7 +554,8 @@ class Backtesting: check_adjust_entry = (entry_count <= self.strategy.max_entry_position_adjustment) if check_adjust_entry: pos_trade = self._enter_trade( - trade.pair, row, 'short' if trade.is_short else 'long', stake_amount, trade) + trade.pair, row, 'short' if trade.is_short else 'long', stake_amount, trade, + entry_tag1=order_tag) if pos_trade is not None: self.wallets.update() return pos_trade @@ -563,14 +566,11 @@ class Backtesting: self.precision_mode, trade.contract_size) if amount == 0.0: return trade - if amount > trade.amount: - # This is currently ineffective as remaining would become < min tradable - amount = trade.amount remaining = (trade.amount - amount) * current_rate - if remaining < min_stake: + if min_stake and remaining != 0 and remaining < min_stake: # Remaining stake is too low to be sold. return trade - exit_ = ExitCheckTuple(ExitType.PARTIAL_EXIT) + exit_ = ExitCheckTuple(ExitType.PARTIAL_EXIT, order_tag) pos_trade = self._get_exit_for_signal(trade, row, exit_, current_time, amount) if pos_trade is not None: order = pos_trade.orders[-1] @@ -682,11 +682,11 @@ class Backtesting: trade.exit_reason = exit_reason - return self._exit_trade(trade, row, close_rate, amount_) + return self._exit_trade(trade, row, close_rate, amount_, exit_reason) return None - def _exit_trade(self, trade: LocalTrade, sell_row: Tuple, - close_rate: float, amount: Optional[float] = None) -> Optional[LocalTrade]: + def _exit_trade(self, trade: LocalTrade, sell_row: Tuple, close_rate: float, + amount: float, exit_reason: Optional[str]) -> Optional[LocalTrade]: self.order_id_counter += 1 exit_candle_time = sell_row[DATE_IDX].to_pydatetime() order_type = self.strategy.order_types['exit'] @@ -713,6 +713,7 @@ class Backtesting: filled=0, remaining=amount, cost=amount * close_rate, + ft_order_tag=exit_reason, ) order._trade_bt = trade trade.orders.append(order) @@ -836,7 +837,9 @@ class Backtesting: stake_amount: Optional[float] = None, trade: Optional[LocalTrade] = None, requested_rate: Optional[float] = None, - requested_stake: Optional[float] = None) -> Optional[LocalTrade]: + requested_stake: Optional[float] = None, + entry_tag1: Optional[str] = None + ) -> Optional[LocalTrade]: """ :param trade: Trade to adjust - initial entry if None :param requested_rate: Adjusted entry rate @@ -844,7 +847,7 @@ class Backtesting: """ current_time = row[DATE_IDX].to_pydatetime() - entry_tag = row[ENTER_TAG_IDX] if len(row) >= ENTER_TAG_IDX + 1 else None + entry_tag = entry_tag1 or (row[ENTER_TAG_IDX] if len(row) >= ENTER_TAG_IDX + 1 else None) # let's call the custom entry price, using the open price as default price order_type = self.strategy.order_types['entry'] pos_adjust = trade is not None and requested_rate is None @@ -945,6 +948,7 @@ class Backtesting: filled=0, remaining=amount, cost=amount * propose_rate + trade.fee_open, + ft_order_tag=entry_tag, ) order._trade_bt = trade trade.orders.append(order) @@ -964,7 +968,8 @@ class Backtesting: # Ignore trade if entry-order did not fill yet continue exit_row = data[pair][-1] - self._exit_trade(trade, exit_row, exit_row[OPEN_IDX], trade.amount) + self._exit_trade(trade, exit_row, exit_row[OPEN_IDX], trade.amount, + ExitType.FORCE_EXIT.value) trade.orders[-1].close_bt_order(exit_row[DATE_IDX].to_pydatetime(), trade) trade.close_date = exit_row[DATE_IDX].to_pydatetime() @@ -1207,10 +1212,10 @@ class Backtesting: # Indexes per pair, so some pairs are allowed to have a missing start. indexes: Dict = defaultdict(int) - current_time = start_date + timedelta(minutes=self.timeframe_min) + current_time = start_date + self.timeframe_td self.progress.init_step(BacktestState.BACKTEST, int( - (end_date - start_date) / timedelta(minutes=self.timeframe_min))) + (end_date - start_date) / self.timeframe_td)) # Loop timerange and get candle for each pair at that point in time while current_time <= end_date: open_trade_count_start = LocalTrade.bt_open_open_trade_count @@ -1237,7 +1242,7 @@ class Backtesting: # Spread out into detail timeframe. # Should only happen when we are either in a trade for this pair # or when we got the signal for a new trade. - exit_candle_end = current_detail_time + timedelta(minutes=self.timeframe_min) + exit_candle_end = current_detail_time + self.timeframe_td detail_data = self.detail_data[pair] detail_data = detail_data.loc[ @@ -1273,7 +1278,7 @@ class Backtesting: # Move time one configured time_interval ahead. self.progress.increment() - current_time += timedelta(minutes=self.timeframe_min) + current_time += self.timeframe_td self.handle_left_open(LocalTrade.bt_trades_open_pp, data=data) self.wallets.update() diff --git a/freqtrade/optimize/base_analysis.py b/freqtrade/optimize/base_analysis.py index 190ac882f..eb0a5e002 100644 --- a/freqtrade/optimize/base_analysis.py +++ b/freqtrade/optimize/base_analysis.py @@ -54,7 +54,7 @@ class BaseAnalysis: self.full_varHolder.from_dt = parsed_timerange.startdt if parsed_timerange.stopdt is None: - self.full_varHolder.to_dt = datetime.utcnow() + self.full_varHolder.to_dt = datetime.now(timezone.utc) else: self.full_varHolder.to_dt = parsed_timerange.stopdt diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index ee03aae1a..763fed747 100644 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -14,9 +14,10 @@ from pandas import isna, json_normalize from freqtrade.constants import FTHYPT_FILEVERSION, Config from freqtrade.enums import HyperoptState from freqtrade.exceptions import OperationalException -from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict, safe_value_fallback2 +from freqtrade.misc import deep_merge_dicts, round_dict, safe_value_fallback2 from freqtrade.optimize.hyperopt_epoch_filters import hyperopt_filter_epochs from freqtrade.optimize.optimize_reports import generate_wins_draws_losses +from freqtrade.util import fmt_coin logger = logging.getLogger(__name__) @@ -405,7 +406,7 @@ class HyperoptTools: trials[f"Max Drawdown{' (Acct)' if has_account_drawdown else ''}"] = trials.apply( lambda x: "{} {}".format( - round_coin_value(x['max_drawdown_abs'], stake_currency, keep_trailing_zeros=True), + fmt_coin(x['max_drawdown_abs'], stake_currency, keep_trailing_zeros=True), (f"({x['max_drawdown_account']:,.2%})" if has_account_drawdown else f"({x['max_drawdown']:,.2%})" @@ -420,7 +421,7 @@ class HyperoptTools: trials['Profit'] = trials.apply( lambda x: '{} {}'.format( - round_coin_value(x['Total profit'], stake_currency, keep_trailing_zeros=True), + fmt_coin(x['Total profit'], stake_currency, keep_trailing_zeros=True), f"({x['Profit']:,.2%})".rjust(10, ' ') ).rjust(25 + len(stake_currency)) if x['Total profit'] != 0.0 else '--'.rjust(25 + len(stake_currency)), diff --git a/freqtrade/optimize/optimize_reports/bt_output.py b/freqtrade/optimize/optimize_reports/bt_output.py index 532796f4a..059207a88 100644 --- a/freqtrade/optimize/optimize_reports/bt_output.py +++ b/freqtrade/optimize/optimize_reports/bt_output.py @@ -4,9 +4,9 @@ from typing import Any, Dict, List from tabulate import tabulate from freqtrade.constants import UNLIMITED_STAKE_AMOUNT, Config -from freqtrade.misc import decimals_per_coin, round_coin_value from freqtrade.optimize.optimize_reports.optimize_reports import generate_periodic_breakdown_stats from freqtrade.types import BacktestResultType +from freqtrade.util import decimals_per_coin, fmt_coin logger = logging.getLogger(__name__) @@ -63,7 +63,7 @@ def text_table_bt_results(pair_results: List[Dict[str, Any]], stake_currency: st def text_table_exit_reason(exit_reason_stats: List[Dict[str, Any]], stake_currency: str) -> str: """ Generate small table outlining Backtest results - :param sell_reason_stats: Exit reason metrics + :param exit_reason_stats: Exit reason metrics :param stake_currency: Stakecurrency used :return: pretty printed table with tabulate as string """ @@ -81,7 +81,7 @@ def text_table_exit_reason(exit_reason_stats: List[Dict[str, Any]], stake_curren t.get('exit_reason', t.get('sell_reason')), t['trades'], generate_wins_draws_losses(t['wins'], t['draws'], t['losses']), t['profit_mean_pct'], t['profit_sum_pct'], - round_coin_value(t['profit_total_abs'], stake_currency, False), + fmt_coin(t['profit_total_abs'], stake_currency, False), t['profit_total_pct'], ] for t in exit_reason_stats] return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right") @@ -134,7 +134,7 @@ def text_table_periodic_breakdown(days_breakdown_stats: List[Dict[str, Any]], 'Losses', ] output = [[ - d['date'], round_coin_value(d['profit_abs'], stake_currency, False), + d['date'], fmt_coin(d['profit_abs'], stake_currency, False), d['wins'], d['draws'], d['loses'], ] for d in days_breakdown_stats] return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right") @@ -187,10 +187,10 @@ def text_table_add_metrics(strat_results: Dict) -> str: f"{strat_results.get('trade_count_short', 0)}"), ('Total profit Long %', f"{strat_results['profit_total_long']:.2%}"), ('Total profit Short %', f"{strat_results['profit_total_short']:.2%}"), - ('Absolute profit Long', round_coin_value(strat_results['profit_total_long_abs'], - strat_results['stake_currency'])), - ('Absolute profit Short', round_coin_value(strat_results['profit_total_short_abs'], - strat_results['stake_currency'])), + ('Absolute profit Long', fmt_coin(strat_results['profit_total_long_abs'], + strat_results['stake_currency'])), + ('Absolute profit Short', fmt_coin(strat_results['profit_total_short_abs'], + strat_results['stake_currency'])), ] if strat_results.get('trade_count_short', 0) > 0 else [] drawdown_metrics = [] @@ -203,12 +203,12 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Absolute Drawdown (Account)', f"{strat_results['max_drawdown_account']:.2%}") if 'max_drawdown_account' in strat_results else ( 'Drawdown', f"{strat_results['max_drawdown']:.2%}"), - ('Absolute Drawdown', round_coin_value(strat_results['max_drawdown_abs'], - strat_results['stake_currency'])), - ('Drawdown high', round_coin_value(strat_results['max_drawdown_high'], - strat_results['stake_currency'])), - ('Drawdown low', round_coin_value(strat_results['max_drawdown_low'], - strat_results['stake_currency'])), + ('Absolute Drawdown', fmt_coin(strat_results['max_drawdown_abs'], + strat_results['stake_currency'])), + ('Drawdown high', fmt_coin(strat_results['max_drawdown_high'], + strat_results['stake_currency'])), + ('Drawdown low', fmt_coin(strat_results['max_drawdown_low'], + strat_results['stake_currency'])), ('Drawdown Start', strat_results['drawdown_start']), ('Drawdown End', strat_results['drawdown_end']), ]) @@ -230,12 +230,12 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Total/Daily Avg Trades', f"{strat_results['total_trades']} / {strat_results['trades_per_day']}"), - ('Starting balance', round_coin_value(strat_results['starting_balance'], - strat_results['stake_currency'])), - ('Final balance', round_coin_value(strat_results['final_balance'], - strat_results['stake_currency'])), - ('Absolute profit ', round_coin_value(strat_results['profit_total_abs'], - strat_results['stake_currency'])), + ('Starting balance', fmt_coin(strat_results['starting_balance'], + strat_results['stake_currency'])), + ('Final balance', fmt_coin(strat_results['final_balance'], + strat_results['stake_currency'])), + ('Absolute profit ', fmt_coin(strat_results['profit_total_abs'], + strat_results['stake_currency'])), ('Total profit %', f"{strat_results['profit_total']:.2%}"), ('CAGR %', f"{strat_results['cagr']:.2%}" if 'cagr' in strat_results else 'N/A'), ('Sortino', f"{strat_results['sortino']:.2f}" if 'sortino' in strat_results else 'N/A'), @@ -249,10 +249,10 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Trades per day', strat_results['trades_per_day']), ('Avg. daily profit %', f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"), - ('Avg. stake amount', round_coin_value(strat_results['avg_stake_amount'], - strat_results['stake_currency'])), - ('Total trade volume', round_coin_value(strat_results['total_volume'], - strat_results['stake_currency'])), + ('Avg. stake amount', fmt_coin(strat_results['avg_stake_amount'], + strat_results['stake_currency'])), + ('Total trade volume', fmt_coin(strat_results['total_volume'], + strat_results['stake_currency'])), *short_metrics, ('', ''), # Empty line to improve readability ('Best Pair', f"{strat_results['best_pair']['key']} " @@ -263,10 +263,10 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Worst trade', f"{worst_trade['pair']} " f"{worst_trade['profit_ratio']:.2%}"), - ('Best day', round_coin_value(strat_results['backtest_best_day_abs'], - strat_results['stake_currency'])), - ('Worst day', round_coin_value(strat_results['backtest_worst_day_abs'], - strat_results['stake_currency'])), + ('Best day', fmt_coin(strat_results['backtest_best_day_abs'], + strat_results['stake_currency'])), + ('Worst day', fmt_coin(strat_results['backtest_worst_day_abs'], + strat_results['stake_currency'])), ('Days win/draw/lose', f"{strat_results['winning_days']} / " f"{strat_results['draw_days']} / {strat_results['losing_days']}"), ('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"), @@ -281,10 +281,8 @@ def text_table_add_metrics(strat_results: Dict) -> str: *entry_adjustment_metrics, ('', ''), # Empty line to improve readability - ('Min balance', round_coin_value(strat_results['csum_min'], - strat_results['stake_currency'])), - ('Max balance', round_coin_value(strat_results['csum_max'], - strat_results['stake_currency'])), + ('Min balance', fmt_coin(strat_results['csum_min'], strat_results['stake_currency'])), + ('Max balance', fmt_coin(strat_results['csum_max'], strat_results['stake_currency'])), *drawdown_metrics, ('Market change', f"{strat_results['market_change']:.2%}"), @@ -292,9 +290,8 @@ def text_table_add_metrics(strat_results: Dict) -> str: return tabulate(metrics, headers=["Metric", "Value"], tablefmt="orgtbl") else: - start_balance = round_coin_value(strat_results['starting_balance'], - strat_results['stake_currency']) - stake_amount = round_coin_value( + start_balance = fmt_coin(strat_results['starting_balance'], strat_results['stake_currency']) + stake_amount = fmt_coin( strat_results['stake_amount'], strat_results['stake_currency'] ) if strat_results['stake_amount'] != UNLIMITED_STAKE_AMOUNT else 'unlimited' @@ -322,24 +319,20 @@ def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency: print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '=')) print(table) - if (results.get('results_per_enter_tag') is not None - or results.get('results_per_buy_tag') is not None): - # results_per_buy_tag is deprecated and should be removed 2 versions after short golive. - table = text_table_tags( - "enter_tag", - results.get('results_per_enter_tag', results.get('results_per_buy_tag')), - stake_currency=stake_currency) + if (results.get('results_per_enter_tag') is not None): + table = text_table_tags("enter_tag", results['results_per_enter_tag'], stake_currency) if isinstance(table, str) and len(table) > 0: print(' ENTER TAG STATS '.center(len(table.splitlines()[0]), '=')) print(table) - exit_reasons = results.get('exit_reason_summary', results.get('sell_reason_summary')) - table = text_table_exit_reason(exit_reason_stats=exit_reasons, - stake_currency=stake_currency) - if isinstance(table, str) and len(table) > 0: - print(' EXIT REASON STATS '.center(len(table.splitlines()[0]), '=')) - print(table) + exit_reasons = results.get('exit_reason_summary') + if exit_reasons: + table = text_table_exit_reason(exit_reason_stats=exit_reasons, + stake_currency=stake_currency) + if isinstance(table, str) and len(table) > 0: + print(' EXIT REASON STATS '.center(len(table.splitlines()[0]), '=')) + print(table) for period in backtest_breakdown: if period in results.get('periodic_breakdown', {}): diff --git a/freqtrade/optimize/optimize_reports/optimize_reports.py b/freqtrade/optimize/optimize_reports/optimize_reports.py index 2ca6ee947..47a13dcd8 100644 --- a/freqtrade/optimize/optimize_reports/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports/optimize_reports.py @@ -10,8 +10,8 @@ from freqtrade.constants import BACKTEST_BREAKDOWNS, DATETIME_PRINT_FORMAT, IntO from freqtrade.data.metrics import (calculate_cagr, calculate_calmar, calculate_csum, calculate_expectancy, calculate_market_change, calculate_max_drawdown, calculate_sharpe, calculate_sortino) -from freqtrade.misc import decimals_per_coin, round_coin_value from freqtrade.types import BacktestResultType +from freqtrade.util import decimals_per_coin, fmt_coin logger = logging.getLogger(__name__) @@ -203,7 +203,7 @@ def generate_strategy_comparison(bt_stats: Dict) -> List[Dict]: # Update "key" to strategy (results_per_pair has it as "Total"). tabular_data[-1]['key'] = strategy tabular_data[-1]['max_drawdown_account'] = result['max_drawdown_account'] - tabular_data[-1]['max_drawdown_abs'] = round_coin_value( + tabular_data[-1]['max_drawdown_abs'] = fmt_coin( result['max_drawdown_abs'], result['stake_currency'], False) return tabular_data @@ -561,6 +561,10 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame], metadata[strategy] = { 'run_id': content['run_id'], 'backtest_start_time': content['backtest_start_time'], + 'timeframe': content['config']['timeframe'], + 'timeframe_detail': content['config'].get('timeframe_detail', None), + 'backtest_start_ts': int(min_date.timestamp()), + 'backtest_end_ts': int(max_date.timestamp()), } result['strategy'][strategy] = strat_stats diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py index 4cf7aa455..6205174a7 100644 --- a/freqtrade/persistence/__init__.py +++ b/freqtrade/persistence/__init__.py @@ -4,3 +4,5 @@ from freqtrade.persistence.key_value_store import KeyStoreKeys, KeyValueStore from freqtrade.persistence.models import init_db from freqtrade.persistence.pairlock_middleware import PairLocks from freqtrade.persistence.trade_model import LocalTrade, Order, Trade +from freqtrade.persistence.usedb_context import (FtNoDBContext, disable_database_use, + enable_database_use) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index bb6c04922..f4d5a7174 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -223,6 +223,7 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List): ft_amount = get_column_def(cols_order, 'ft_amount', 'coalesce(amount, 0.0)') ft_price = get_column_def(cols_order, 'ft_price', 'coalesce(price, 0.0)') ft_cancel_reason = get_column_def(cols_order, 'ft_cancel_reason', 'null') + ft_order_tag = get_column_def(cols_order, 'ft_order_tag', 'null') # sqlite does not support literals for booleans with engine.begin() as connection: @@ -230,13 +231,14 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List): insert into orders (id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, symbol, order_type, side, price, amount, filled, average, remaining, cost, stop_price, order_date, order_filled_date, order_update_date, ft_fee_base, funding_fee, - ft_amount, ft_price, ft_cancel_reason + ft_amount, ft_price, ft_cancel_reason, ft_order_tag ) select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, symbol, order_type, side, price, amount, filled, {average} average, remaining, cost, {stop_price} stop_price, order_date, order_filled_date, order_update_date, {ft_fee_base} ft_fee_base, {funding_fee} funding_fee, - {ft_amount} ft_amount, {ft_price} ft_price, {ft_cancel_reason} ft_cancel_reason + {ft_amount} ft_amount, {ft_price} ft_price, {ft_cancel_reason} ft_cancel_reason, + {ft_order_tag} ft_order_tag from {table_back_name} """)) @@ -331,8 +333,8 @@ def check_migrate(engine, decl_base, previous_tables) -> None: # if ('orders' not in previous_tables # or not has_column(cols_orders, 'funding_fee')): migrating = False - # if not has_column(cols_orders, 'ft_cancel_reason'): - if not has_column(cols_trades, 'funding_fee_running'): + # if not has_column(cols_trades, 'funding_fee_running'): + if not has_column(cols_orders, 'ft_order_tag'): migrating = True logger.info(f"Running database migration for trades - " f"backup: {table_back_name}, {order_table_bak_name}") diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 3f721f1e8..7e3cf970f 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -89,6 +89,8 @@ class Order(ModelBase): funding_fee: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) ft_fee_base: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) + ft_order_tag: Mapped[Optional[str]] = mapped_column(String(CUSTOM_TAG_MAX_LENGTH), + nullable=True) @property def order_date_utc(self) -> datetime: @@ -106,6 +108,11 @@ class Order(ModelBase): def safe_amount(self) -> float: return self.amount or self.ft_amount + @property + def safe_placement_price(self) -> float: + """Price at which the order was placed""" + return self.price or self.stop_price or self.ft_price + @property def safe_price(self) -> float: return self.average or self.price or self.stop_price or self.ft_price @@ -146,7 +153,7 @@ class Order(ModelBase): return (f"Order(id={self.id}, trade={self.ft_trade_id}, order_id={self.order_id}, " f"side={self.side}, filled={self.safe_filled}, price={self.safe_price}, " - f"status={self.status}, date={self.order_date:{DATETIME_PRINT_FORMAT}})") + f"status={self.status}, date={self.order_date_utc:{DATETIME_PRINT_FORMAT}})") def update_from_ccxt_object(self, order): """ @@ -156,20 +163,20 @@ class Order(ModelBase): if self.order_id != str(order['id']): raise DependencyException("Order-id's don't match") - self.status = order.get('status', self.status) - self.symbol = order.get('symbol', self.symbol) - self.order_type = order.get('type', self.order_type) - self.side = order.get('side', self.side) - self.price = order.get('price', self.price) - self.amount = order.get('amount', self.amount) - self.filled = order.get('filled', self.filled) - self.average = order.get('average', self.average) - self.remaining = order.get('remaining', self.remaining) - self.cost = order.get('cost', self.cost) - self.stop_price = order.get('stopPrice', self.stop_price) - - if 'timestamp' in order and order['timestamp'] is not None: - self.order_date = datetime.fromtimestamp(order['timestamp'] / 1000, tz=timezone.utc) + self.status = safe_value_fallback(order, 'status', default_value=self.status) + self.symbol = safe_value_fallback(order, 'symbol', default_value=self.symbol) + self.order_type = safe_value_fallback(order, 'type', default_value=self.order_type) + self.side = safe_value_fallback(order, 'side', default_value=self.side) + self.price = safe_value_fallback(order, 'price', default_value=self.price) + self.amount = safe_value_fallback(order, 'amount', default_value=self.amount) + self.filled = safe_value_fallback(order, 'filled', default_value=self.filled) + self.average = safe_value_fallback(order, 'average', default_value=self.average) + self.remaining = safe_value_fallback(order, 'remaining', default_value=self.remaining) + self.cost = safe_value_fallback(order, 'cost', default_value=self.cost) + self.stop_price = safe_value_fallback(order, 'stopPrice', default_value=self.stop_price) + order_date = safe_value_fallback(order, 'timestamp') + if order_date: + self.order_date = datetime.fromtimestamp(order_date / 1000, tz=timezone.utc) self.ft_is_open = True if self.status in NON_OPEN_EXCHANGE_STATES: @@ -207,6 +214,10 @@ class Order(ModelBase): return order def to_json(self, entry_side: str, minified: bool = False) -> Dict[str, Any]: + """ + :param minified: If True, only return a subset of the data is returned. + Only used for backtesting. + """ resp = { 'amount': self.safe_amount, 'safe_price': self.safe_price, @@ -214,6 +225,7 @@ class Order(ModelBase): 'order_filled_timestamp': int(self.order_filled_date.replace( tzinfo=timezone.utc).timestamp() * 1000) if self.order_filled_date else None, 'ft_is_entry': self.ft_order_side == entry_side, + 'ft_order_tag': self.ft_order_tag, } if not minified: resp.update({ @@ -542,7 +554,9 @@ class LocalTrade: f"{self.trading_mode.value} trading requires param interest_rate on trades") def __repr__(self): - open_since = self.open_date.strftime(DATETIME_PRINT_FORMAT) if self.is_open else 'closed' + open_since = ( + self.open_date_utc.strftime(DATETIME_PRINT_FORMAT) if self.is_open else 'closed' + ) return ( f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, ' @@ -551,6 +565,11 @@ class LocalTrade: ) def to_json(self, minified: bool = False) -> Dict[str, Any]: + """ + :param minified: If True, only return a subset of the data is returned. + Only used for backtesting. + :return: Dictionary with trade data + """ filled_or_open_orders = self.select_filled_or_open_orders() orders_json = [order.to_json(self.entry_side, minified) for order in filled_or_open_orders] @@ -1393,6 +1412,7 @@ class LocalTrade: ft_price=order["price"], remaining=order["remaining"], funding_fee=order.get("funding_fee", None), + ft_order_tag=order.get("ft_order_tag", None), ) trade.orders.append(order_obj) @@ -1603,7 +1623,7 @@ class Trade(ModelBase, LocalTrade): :return: unsorted query object """ query = Trade.get_trades_query(trade_filter, include_orders) - # this sholud remain split. if use_db is False, session is not available and the above will + # this should remain split. if use_db is False, session is not available and the above will # raise an exception. return Trade.session.scalars(query) @@ -1635,7 +1655,7 @@ class Trade(ModelBase, LocalTrade): Retrieves total realized profit """ if Trade.use_db: - total_profit: float = Trade.session.execute( + total_profit = Trade.session.execute( select(func.sum(Trade.close_profit_abs)).filter(Trade.is_open.is_(False)) ).scalar_one() else: @@ -1843,4 +1863,4 @@ class Trade(ModelBase, LocalTrade): Order.order_filled_date >= start_date, Order.status == 'closed' )).scalar_one() - return trading_volume + return trading_volume or 0.0 diff --git a/freqtrade/persistence/usedb_context.py b/freqtrade/persistence/usedb_context.py new file mode 100644 index 000000000..6fffd2fb5 --- /dev/null +++ b/freqtrade/persistence/usedb_context.py @@ -0,0 +1,33 @@ + +from freqtrade.persistence.pairlock_middleware import PairLocks +from freqtrade.persistence.trade_model import Trade + + +def disable_database_use(timeframe: str) -> None: + """ + Disable database usage for PairLocks and Trade models. + Used for backtesting, and some other utility commands. + """ + PairLocks.use_db = False + PairLocks.timeframe = timeframe + Trade.use_db = False + + +def enable_database_use() -> None: + """ + Cleanup function to restore database usage. + """ + PairLocks.use_db = True + PairLocks.timeframe = '' + Trade.use_db = True + + +class FtNoDBContext: + def __init__(self, timeframe: str = ''): + self.timeframe = timeframe + + def __enter__(self): + disable_database_use(self.timeframe) + + def __exit__(self, exc_type, exc_val, exc_tb): + enable_database_use() diff --git a/freqtrade/plugins/pairlist/MarketCapPairList.py b/freqtrade/plugins/pairlist/MarketCapPairList.py new file mode 100644 index 000000000..a618f72d2 --- /dev/null +++ b/freqtrade/plugins/pairlist/MarketCapPairList.py @@ -0,0 +1,157 @@ +""" +Market Cap PairList provider + +Provides dynamic pair list based on Market Cap +""" +import logging +from typing import Any, Dict, List + +from cachetools import TTLCache +from pycoingecko import CoinGeckoAPI + +from freqtrade.constants import Config +from freqtrade.exceptions import OperationalException +from freqtrade.exchange.types import Tickers +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter + + +logger = logging.getLogger(__name__) + + +class MarketCapPairList(IPairList): + + is_pairlist_generator = True + + def __init__(self, exchange, pairlistmanager, + config: Config, pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + if 'number_assets' not in self._pairlistconfig: + raise OperationalException( + '`number_assets` not specified. Please check your configuration ' + 'for "pairlist.config.number_assets"') + + self._stake_currency = config['stake_currency'] + self._number_assets = self._pairlistconfig['number_assets'] + self._max_rank = self._pairlistconfig.get('max_rank', 30) + self._refresh_period = self._pairlistconfig.get('refresh_period', 86400) + self._marketcap_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) + self._def_candletype = self._config['candle_type_def'] + self._coingekko: CoinGeckoAPI = CoinGeckoAPI() + + if self._max_rank > 250: + raise OperationalException( + "This filter only support marketcap rank up to 250." + ) + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty Dict is passed + as tickers argument to filter_pairlist + """ + return False + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + num = self._number_assets + rank = self._max_rank + msg = f"{self.name} - {num} pairs placed within top {rank} market cap." + return msg + + @staticmethod + def description() -> str: + return "Provides pair list based on CoinGecko's market cap rank." + + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "number_assets": { + "type": "number", + "default": 30, + "description": "Number of assets", + "help": "Number of assets to use from the pairlist", + }, + "max_rank": { + "type": "number", + "default": 30, + "description": "Max rank of assets", + "help": "Maximum rank of assets to use from the pairlist", + }, + "refresh_period": { + "type": "number", + "default": 86400, + "description": "Refresh period", + "help": "Refresh period in seconds", + } + } + + def gen_pairlist(self, tickers: Tickers) -> List[str]: + """ + Generate the pairlist + :param tickers: Tickers (from exchange.get_tickers). May be cached. + :return: List of pairs + """ + # Generate dynamic whitelist + # Must always run if this pairlist is the first in the list. + pairlist = self._marketcap_cache.get('pairlist_mc') + if pairlist: + # Item found - no refresh necessary + return pairlist.copy() + else: + # Use fresh pairlist + # Check if pair quote currency equals to the stake currency. + _pairlist = [k for k in self._exchange.get_markets( + quote_currencies=[self._stake_currency], + tradable_only=True, active_only=True).keys()] + # No point in testing for blacklisted pairs... + _pairlist = self.verify_blacklist(_pairlist, logger.info) + + pairlist = self.filter_pairlist(_pairlist, tickers) + self._marketcap_cache['pairlist_mc'] = pairlist.copy() + + return pairlist + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers). May be cached. + :return: new whitelist + """ + marketcap_list = self._marketcap_cache.get('marketcap') + + if marketcap_list is None: + data = self._coingekko.get_coins_markets(vs_currency='usd', order='market_cap_desc', + per_page='250', page='1', sparkline='false', + locale='en') + if data: + marketcap_list = [row['symbol'] for row in data] + self._marketcap_cache['marketcap'] = marketcap_list + + if marketcap_list: + filtered_pairlist = [] + + market = self._config['trading_mode'] + pair_format = f"{self._stake_currency.upper()}" + if (market == 'futures'): + pair_format += f":{self._stake_currency.upper()}" + + top_marketcap = marketcap_list[:self._max_rank:] + + for mc_pair in top_marketcap: + test_pair = f"{mc_pair.upper()}/{pair_format}" + if test_pair in pairlist: + filtered_pairlist.append(test_pair) + if len(filtered_pairlist) == self._number_assets: + break + + if len(filtered_pairlist) > 0: + return filtered_pairlist + + return pairlist diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index 2f03678e2..78e3c8351 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -52,6 +52,7 @@ class RemotePairList(IPairList): self._read_timeout = self._pairlistconfig.get('read_timeout', 60) self._bearer_token = self._pairlistconfig.get('bearer_token', '') self._init_done = False + self._save_to_file = self._pairlistconfig.get('save_to_file', None) self._last_pairlist: List[Any] = list() if self._mode not in ['whitelist', 'blacklist']: @@ -136,6 +137,12 @@ class RemotePairList(IPairList): "description": "Bearer token", "help": "Bearer token - used for auth against the upstream service.", }, + "save_to_file": { + "type": "string", + "default": "", + "description": "Filename to save processed pairlist to.", + "help": "Specify a filename to save the processed pairlist in JSON format.", + }, } def process_json(self, jsonparse) -> List[str]: @@ -184,31 +191,26 @@ class RemotePairList(IPairList): try: pairlist = self.process_json(jsonparse) except Exception as e: - - if self._init_done: - pairlist = self.return_last_pairlist() - logger.warning(f'Error while processing JSON data: {type(e)}') - else: - raise OperationalException(f'Error while processing JSON data: {type(e)}') - + pairlist = self._handle_error(f'Failed processing JSON data: {type(e)}') else: - if self._init_done: - self.log_once(f'Error: RemotePairList is not of type JSON: ' - f' {self._pairlist_url}', logger.info) - pairlist = self.return_last_pairlist() - else: - raise OperationalException('RemotePairList is not of type JSON, abort.') + pairlist = self._handle_error(f'RemotePairList is not of type JSON.' + f' {self._pairlist_url}') except requests.exceptions.RequestException: - self.log_once(f'Was not able to fetch pairlist from:' - f' {self._pairlist_url}', logger.info) - - pairlist = self.return_last_pairlist() + pairlist = self._handle_error(f'Was not able to fetch pairlist from:' + f' {self._pairlist_url}') time_elapsed = 0 return pairlist, time_elapsed + def _handle_error(self, error: str) -> List[str]: + if self._init_done: + self.log_once("Error: " + error, logger.info) + return self.return_last_pairlist() + else: + raise OperationalException(error) + def gen_pairlist(self, tickers: Tickers) -> List[str]: """ Generate the pairlist @@ -236,20 +238,15 @@ class RemotePairList(IPairList): if file_path.exists(): with file_path.open() as json_file: - # Load the JSON data into a dictionary - jsonparse = rapidjson.load(json_file, parse_mode=CONFIG_PARSE_MODE) - try: + # Load the JSON data into a dictionary + jsonparse = rapidjson.load(json_file, parse_mode=CONFIG_PARSE_MODE) pairlist = self.process_json(jsonparse) except Exception as e: - if self._init_done: - pairlist = self.return_last_pairlist() - logger.warning(f'Error while processing JSON data: {type(e)}') - else: - raise OperationalException('Error while processing' - f'JSON data: {type(e)}') + pairlist = self._handle_error(f'processing JSON data: {type(e)}') else: - raise ValueError(f"{self._pairlist_url} does not exist.") + pairlist = self._handle_error(f"{self._pairlist_url} does not exist.") + else: # Fetch Pairlist from Remote URL pairlist, time_elapsed = self.fetch_pairlist() @@ -273,8 +270,23 @@ class RemotePairList(IPairList): self._last_pairlist = list(pairlist) + if self._save_to_file: + self.save_pairlist(pairlist, self._save_to_file) + return pairlist + def save_pairlist(self, pairlist: List[str], filename: str) -> None: + pairlist_data = { + "pairs": pairlist + } + try: + file_path = Path(filename) + with file_path.open('w') as json_file: + rapidjson.dump(pairlist_data, json_file) + logger.info(f"Processed pairlist saved to {filename}") + except Exception as e: + logger.error(f"Error saving processed pairlist to {filename}: {e}") + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index c2f102777..b5525e950 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -62,16 +62,16 @@ class VolumePairList(IPairList): # get timeframe in minutes and seconds self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe) - self._tf_in_sec = self._tf_in_min * 60 + _tf_in_sec = self._tf_in_min * 60 # wether to use range lookback or not self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0) - if self._use_range & (self._refresh_period < self._tf_in_sec): + if self._use_range & (self._refresh_period < _tf_in_sec): raise OperationalException( f'Refresh period of {self._refresh_period} seconds is smaller than one ' f'timeframe of {self._lookback_timeframe}. Please adjust refresh_period ' - f'to at least {self._tf_in_sec} and restart the bot.' + f'to at least {_tf_in_sec} and restart the bot.' ) if (not self._use_range and not ( diff --git a/freqtrade/rpc/api_server/api_auth.py b/freqtrade/rpc/api_server/api_auth.py index 71f1145a9..dd4749100 100644 --- a/freqtrade/rpc/api_server/api_auth.py +++ b/freqtrade/rpc/api_server/api_auth.py @@ -1,6 +1,6 @@ import logging import secrets -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Any, Dict, Union import jwt @@ -88,14 +88,14 @@ async def validate_ws_token( def create_token(data: dict, secret_key: str, token_type: str = "access") -> str: to_encode = data.copy() if token_type == "access": - expire = datetime.utcnow() + timedelta(minutes=15) + expire = datetime.now(timezone.utc) + timedelta(minutes=15) elif token_type == "refresh": - expire = datetime.utcnow() + timedelta(days=30) + expire = datetime.now(timezone.utc) + timedelta(days=30) else: raise ValueError() to_encode.update({ "exp": expire, - "iat": datetime.utcnow(), + "iat": datetime.now(timezone.utc), "type": token_type, }) encoded_jwt = jwt.encode(to_encode, secret_key, algorithm=ALGORITHM) diff --git a/freqtrade/rpc/api_server/api_background_tasks.py b/freqtrade/rpc/api_server/api_background_tasks.py index c13fa31e4..04e98c609 100644 --- a/freqtrade/rpc/api_server/api_background_tasks.py +++ b/freqtrade/rpc/api_server/api_background_tasks.py @@ -7,6 +7,7 @@ from fastapi.exceptions import HTTPException from freqtrade.constants import Config from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException +from freqtrade.persistence import FtNoDBContext from freqtrade.rpc.api_server.api_schemas import (BackgroundTaskStatus, BgJobStarted, ExchangeModePayloadMixin, PairListsPayload, PairListsResponse, WhitelistEvaluateResponse) @@ -57,16 +58,16 @@ def __run_pairlist(job_id: str, config_loc: Config): ApiBG.jobs[job_id]['is_running'] = True from freqtrade.plugins.pairlistmanager import PairListManager - - exchange = get_exchange(config_loc) - pairlists = PairListManager(exchange, config_loc) - pairlists.refresh_pairlist() - ApiBG.jobs[job_id]['result'] = { - 'method': pairlists.name_list, - 'length': len(pairlists.whitelist), - 'whitelist': pairlists.whitelist - } - ApiBG.jobs[job_id]['status'] = 'success' + with FtNoDBContext(): + exchange = get_exchange(config_loc) + pairlists = PairListManager(exchange, config_loc) + pairlists.refresh_pairlist() + ApiBG.jobs[job_id]['result'] = { + 'method': pairlists.name_list, + 'length': len(pairlists.whitelist), + 'whitelist': pairlists.whitelist + } + ApiBG.jobs[job_id]['status'] = 'success' except (OperationalException, Exception) as e: logger.exception(e) ApiBG.jobs[job_id]['error'] = str(e) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 770f68a5d..791f70fa0 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -261,6 +261,7 @@ class OrderSchema(BaseModel): order_timestamp: Optional[int] = None order_filled_timestamp: Optional[int] = None ft_fee_base: Optional[float] = None + ft_order_tag: Optional[str] = None class TradeSchema(BaseModel): @@ -538,6 +539,10 @@ class BacktestHistoryEntry(BaseModel): run_id: str backtest_start_time: int notes: Optional[str] = '' + backtest_start_ts: Optional[int] = None + backtest_end_ts: Optional[int] = None + timeframe: Optional[str] = None + timeframe_detail: Optional[str] = None class BacktestMetadataUpdate(BaseModel): diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index 44c1aa3d9..c28d6da6c 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -107,7 +107,7 @@ class ApiServer(RPCHandler): ApiServer._message_stream.publish(msg) def handle_rpc_exception(self, request, exc): - logger.exception(f"API Error calling: {exc}") + logger.error(f"API Error calling: {exc}") return JSONResponse( status_code=502, content={'error': f"Error querying {request.url.path}: {exc.message}"} diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 809c63e78..6decd7f7b 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -25,13 +25,13 @@ from freqtrade.exceptions import ExchangeError, PricingError from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs from freqtrade.exchange.types import Tickers from freqtrade.loggers import bufferHandler -from freqtrade.misc import decimals_per_coin from freqtrade.persistence import KeyStoreKeys, KeyValueStore, PairLocks, Trade from freqtrade.persistence.models import PairLock from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.rpc.rpc_types import RPCSendMsg -from freqtrade.util import dt_humanize, dt_now, dt_ts_def, format_date, shorten_date +from freqtrade.util import (decimals_per_coin, dt_humanize, dt_now, dt_ts_def, format_date, + shorten_date) from freqtrade.wallets import PositionWallet, Wallet diff --git a/freqtrade/rpc/rpc_types.py b/freqtrade/rpc/rpc_types.py index 23f3ed5a9..72a382f48 100644 --- a/freqtrade/rpc/rpc_types.py +++ b/freqtrade/rpc/rpc_types.py @@ -5,6 +5,9 @@ from freqtrade.constants import PairWithTimeframe from freqtrade.enums import RPCMessageType +ProfitLossStr = Literal["profit", "loss"] + + class RPCSendMsgBase(TypedDict): pass # ty1pe: Literal[RPCMessageType] @@ -41,13 +44,14 @@ class RPCWhitelistMsg(RPCSendMsgBase): data: List[str] -class __RPCBuyMsgBase(RPCSendMsgBase): +class __RPCEntryExitMsgBase(RPCSendMsgBase): trade_id: int buy_tag: Optional[str] enter_tag: Optional[str] exchange: str pair: str base_currency: str + quote_currency: str leverage: Optional[float] direction: str limit: float @@ -62,36 +66,36 @@ class __RPCBuyMsgBase(RPCSendMsgBase): sub_trade: bool -class RPCBuyMsg(__RPCBuyMsgBase): +class RPCEntryMsg(__RPCEntryExitMsgBase): type: Literal[RPCMessageType.ENTRY, RPCMessageType.ENTRY_FILL] -class RPCCancelMsg(__RPCBuyMsgBase): +class RPCCancelMsg(__RPCEntryExitMsgBase): type: Literal[RPCMessageType.ENTRY_CANCEL] reason: str -class RPCSellMsg(__RPCBuyMsgBase): +class RPCExitMsg(__RPCEntryExitMsgBase): type: Literal[RPCMessageType.EXIT, RPCMessageType.EXIT_FILL] cumulative_profit: float - gain: str # Literal["profit", "loss"] + gain: ProfitLossStr close_rate: float profit_amount: float profit_ratio: float - sell_reason: Optional[str] exit_reason: Optional[str] close_date: datetime # current_rate: Optional[float] order_rate: Optional[float] + final_profit_ratio: Optional[float] + is_final_exit: bool -class RPCSellCancelMsg(__RPCBuyMsgBase): +class RPCExitCancelMsg(__RPCEntryExitMsgBase): type: Literal[RPCMessageType.EXIT_CANCEL] reason: str - gain: str # Literal["profit", "loss"] + gain: ProfitLossStr profit_amount: float profit_ratio: float - sell_reason: Optional[str] exit_reason: Optional[str] close_date: datetime @@ -114,15 +118,18 @@ class RPCNewCandleMsg(RPCSendMsgBase): data: PairWithTimeframe +RPCOrderMsg = Union[RPCEntryMsg, RPCExitMsg, RPCExitCancelMsg, RPCCancelMsg] + + RPCSendMsg = Union[ RPCStatusMsg, RPCStrategyMsg, RPCProtectionMsg, RPCWhitelistMsg, - RPCBuyMsg, + RPCEntryMsg, RPCCancelMsg, - RPCSellMsg, - RPCSellCancelMsg, + RPCExitMsg, + RPCExitCancelMsg, RPCAnalyzedDFMsg, RPCNewCandleMsg ] diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c9e9a4733..e2fbe1529 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -10,12 +10,12 @@ import re from copy import deepcopy from dataclasses import dataclass from datetime import date, datetime, timedelta -from functools import partial +from functools import partial, wraps from html import escape from itertools import chain from math import isnan from threading import Thread -from typing import Any, Callable, Coroutine, Dict, List, Optional, Union +from typing import Any, Callable, Coroutine, Dict, List, Literal, Optional, Union from tabulate import tabulate from telegram import (CallbackQuery, InlineKeyboardButton, InlineKeyboardMarkup, KeyboardButton, @@ -29,11 +29,11 @@ from freqtrade.__init__ import __version__ from freqtrade.constants import DUST_PER_COIN, Config from freqtrade.enums import MarketDirection, RPCMessageType, SignalDirection, TradingMode from freqtrade.exceptions import OperationalException -from freqtrade.misc import chunks, plural, round_coin_value +from freqtrade.misc import chunks, plural from freqtrade.persistence import Trade from freqtrade.rpc import RPC, RPCException, RPCHandler -from freqtrade.rpc.rpc_types import RPCSendMsg -from freqtrade.util import dt_humanize +from freqtrade.rpc.rpc_types import RPCEntryMsg, RPCExitMsg, RPCOrderMsg, RPCSendMsg +from freqtrade.util import dt_humanize, fmt_coin, round_value MAX_MESSAGE_LENGTH = MessageLimit.MAX_TEXT_LENGTH @@ -44,6 +44,23 @@ logger = logging.getLogger(__name__) logger.debug('Included module rpc.telegram ...') +def safe_async_db(func: Callable[..., Any]): + """ + Decorator to safely handle sessions when switching async context + :param func: function to decorate + :return: decorated function + """ + @wraps(func) + def wrapper(*args, **kwargs): + """ Decorator logic """ + try: + return func(*args, **kwargs) + finally: + Trade.session.remove() + + return wrapper + + @dataclass class TimeunitMappings: header: str @@ -61,6 +78,7 @@ def authorized_only(command_handler: Callable[..., Coroutine[Any, Any, None]]): :return: decorated function """ + @wraps(command_handler) async def wrapper(self, *args, **kwargs): """ Decorator logic """ update = kwargs.get('update') or args[0] @@ -286,7 +304,7 @@ class Telegram(RPCHandler): asyncio.run_coroutine_threadsafe(self._cleanup_telegram(), self._loop) self._thread.join() - def _exchange_from_msg(self, msg: Dict[str, Any]) -> str: + def _exchange_from_msg(self, msg: RPCOrderMsg) -> str: """ Extracts the exchange name from the given message. :param msg: The message to extract the exchange name from. @@ -310,164 +328,172 @@ class Telegram(RPCHandler): return '' - def _format_entry_msg(self, msg: Dict[str, Any]) -> str: - if self._rpc._fiat_converter: - msg['stake_amount_fiat'] = self._rpc._fiat_converter.convert_amount( - msg['stake_amount'], msg['stake_currency'], msg['fiat_currency']) - else: - msg['stake_amount_fiat'] = 0 + def _format_entry_msg(self, msg: RPCEntryMsg) -> str: + is_fill = msg['type'] in [RPCMessageType.ENTRY_FILL] emoji = '\N{CHECK MARK}' if is_fill else '\N{LARGE BLUE CIRCLE}' - entry_side = ({'enter': 'Long', 'entered': 'Longed'} if msg['direction'] == 'Long' - else {'enter': 'Short', 'entered': 'Shorted'}) + terminology = { + '1_enter': 'New Trade', + '1_entered': 'New Trade filled', + 'x_enter': 'Increasing position', + 'x_entered': 'Position increase filled', + } + + key = f"{'x' if msg['sub_trade'] else '1'}_{'entered' if is_fill else 'enter'}" + wording = terminology[key] + message = ( f"{emoji} *{self._exchange_from_msg(msg)}:*" - f" {entry_side['entered'] if is_fill else entry_side['enter']} {msg['pair']}" - f" (#{msg['trade_id']})\n" + f" {wording} (#{msg['trade_id']})\n" + f"*Pair:* `{msg['pair']}`\n" ) message += self._add_analyzed_candle(msg['pair']) message += f"*Enter Tag:* `{msg['enter_tag']}`\n" if msg.get('enter_tag') else "" - message += f"*Amount:* `{msg['amount']:.8f}`\n" + message += f"*Amount:* `{round_value(msg['amount'], 8)}`\n" + message += f"*Direction:* `{msg['direction']}" if msg.get('leverage') and msg.get('leverage', 1.0) != 1.0: - message += f"*Leverage:* `{msg['leverage']}`\n" + message += f" ({msg['leverage']:.1g}x)" + message += "`\n" + message += f"*Open Rate:* `{fmt_coin(msg['open_rate'], msg['quote_currency'])}`\n" + if msg['type'] == RPCMessageType.ENTRY and msg['current_rate']: + message += f"*Current Rate:* `{fmt_coin(msg['current_rate'], msg['quote_currency'])}`\n" - if msg['type'] in [RPCMessageType.ENTRY_FILL]: - message += f"*Open Rate:* `{msg['open_rate']:.8f}`\n" - elif msg['type'] in [RPCMessageType.ENTRY]: - message += f"*Open Rate:* `{msg['open_rate']:.8f}`\n"\ - f"*Current Rate:* `{msg['current_rate']:.8f}`\n" + profit_fiat_extra = self.__format_profit_fiat(msg, 'stake_amount') # type: ignore + total = fmt_coin(msg['stake_amount'], msg['quote_currency']) - message += f"*Total:* `({round_coin_value(msg['stake_amount'], msg['stake_currency'])}" + message += f"*{'New ' if msg['sub_trade'] else ''}Total:* `{total}{profit_fiat_extra}`" - if msg.get('fiat_currency'): - message += f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}" - - message += ")`" return message - def _format_exit_msg(self, msg: Dict[str, Any]) -> str: - msg['amount'] = round(msg['amount'], 8) - msg['profit_percent'] = round(msg['profit_ratio'] * 100, 2) - msg['duration'] = msg['close_date'].replace( + def _format_exit_msg(self, msg: RPCExitMsg) -> str: + duration = msg['close_date'].replace( microsecond=0) - msg['open_date'].replace(microsecond=0) - msg['duration_min'] = msg['duration'].total_seconds() / 60 + duration_min = duration.total_seconds() / 60 - msg['enter_tag'] = msg['enter_tag'] if "enter_tag" in msg.keys() else None - msg['emoji'] = self._get_sell_emoji(msg) - msg['leverage_text'] = (f"*Leverage:* `{msg['leverage']:.1f}`\n" - if msg.get('leverage') and msg.get('leverage', 1.0) != 1.0 - else "") + leverage_text = (f" ({msg['leverage']:.1g}x)" + if msg.get('leverage') and msg.get('leverage', 1.0) != 1.0 + else "") - # Check if all sell properties are available. - # This might not be the case if the message origin is triggered by /forceexit - if (all(prop in msg for prop in ['gain', 'fiat_currency', 'stake_currency']) - and self._rpc._fiat_converter): - msg['profit_fiat'] = self._rpc._fiat_converter.convert_amount( - msg['profit_amount'], msg['stake_currency'], msg['fiat_currency']) - msg['profit_extra'] = f" / {msg['profit_fiat']:.3f} {msg['fiat_currency']}" - else: - msg['profit_extra'] = '' - msg['profit_extra'] = ( - f" ({msg['gain']}: {msg['profit_amount']:.8f} {msg['stake_currency']}" - f"{msg['profit_extra']})") + profit_fiat_extra = self.__format_profit_fiat(msg, 'profit_amount') + + profit_extra = ( + f" ({msg['gain']}: {fmt_coin(msg['profit_amount'], msg['quote_currency'])}" + f"{profit_fiat_extra})") is_fill = msg['type'] == RPCMessageType.EXIT_FILL is_sub_trade = msg.get('sub_trade') is_sub_profit = msg['profit_amount'] != msg.get('cumulative_profit') - profit_prefix = ('Sub ' if is_sub_profit else 'Cumulative ') if is_sub_trade else '' + is_final_exit = msg.get('is_final_exit', False) and is_sub_profit + profit_prefix = 'Sub ' if is_sub_trade else '' cp_extra = '' exit_wording = 'Exited' if is_fill else 'Exiting' - if is_sub_profit and is_sub_trade: - if self._rpc._fiat_converter: - cp_fiat = self._rpc._fiat_converter.convert_amount( - msg['cumulative_profit'], msg['stake_currency'], msg['fiat_currency']) - cp_extra = f" / {cp_fiat:.3f} {msg['fiat_currency']}" - exit_wording = f"Partially {exit_wording.lower()}" - cp_extra = ( - f"*Cumulative Profit:* (`{msg['cumulative_profit']:.8f} " - f"{msg['stake_currency']}{cp_extra}`)\n" - ) + if is_sub_trade or is_final_exit: + cp_fiat = self.__format_profit_fiat(msg, 'cumulative_profit') + if is_final_exit: + profit_prefix = 'Sub ' + cp_extra = ( + f"*Final Profit:* `{msg['final_profit_ratio']:.2%} " + f"({msg['cumulative_profit']:.8f} {msg['quote_currency']}{cp_fiat})`\n" + ) + else: + exit_wording = f"Partially {exit_wording.lower()}" + if msg['cumulative_profit']: + cp_extra = ( + f"*Cumulative Profit:* `" + f"{fmt_coin(msg['cumulative_profit'], msg['stake_currency'])}{cp_fiat}`\n" + ) + enter_tag = f"*Enter Tag:* `{msg['enter_tag']}`\n" if msg.get('enter_tag') else "" message = ( - f"{msg['emoji']} *{self._exchange_from_msg(msg)}:* " + f"{self._get_exit_emoji(msg)} *{self._exchange_from_msg(msg)}:* " f"{exit_wording} {msg['pair']} (#{msg['trade_id']})\n" f"{self._add_analyzed_candle(msg['pair'])}" f"*{f'{profit_prefix}Profit' if is_fill else f'Unrealized {profit_prefix}Profit'}:* " - f"`{msg['profit_ratio']:.2%}{msg['profit_extra']}`\n" + f"`{msg['profit_ratio']:.2%}{profit_extra}`\n" f"{cp_extra}" - f"*Enter Tag:* `{msg['enter_tag']}`\n" + f"{enter_tag}" f"*Exit Reason:* `{msg['exit_reason']}`\n" - f"*Direction:* `{msg['direction']}`\n" - f"{msg['leverage_text']}" - f"*Amount:* `{msg['amount']:.8f}`\n" - f"*Open Rate:* `{msg['open_rate']:.8f}`\n" + f"*Direction:* `{msg['direction']}" + f"{leverage_text}`\n" + f"*Amount:* `{round_value(msg['amount'], 8)}`\n" + f"*Open Rate:* `{fmt_coin(msg['open_rate'], msg['quote_currency'])}`\n" ) - if msg['type'] == RPCMessageType.EXIT: - message += f"*Current Rate:* `{msg['current_rate']:.8f}`\n" + if msg['type'] == RPCMessageType.EXIT and msg['current_rate']: + message += f"*Current Rate:* `{fmt_coin(msg['current_rate'], msg['quote_currency'])}`\n" if msg['order_rate']: - message += f"*Exit Rate:* `{msg['order_rate']:.8f}`" - + message += f"*Exit Rate:* `{fmt_coin(msg['order_rate'], msg['quote_currency'])}`" elif msg['type'] == RPCMessageType.EXIT_FILL: - message += f"*Exit Rate:* `{msg['close_rate']:.8f}`" + message += f"*Exit Rate:* `{fmt_coin(msg['close_rate'], msg['quote_currency'])}`" + if is_sub_trade: - if self._rpc._fiat_converter: - msg['stake_amount_fiat'] = self._rpc._fiat_converter.convert_amount( - msg['stake_amount'], msg['stake_currency'], msg['fiat_currency']) - else: - msg['stake_amount_fiat'] = 0 - rem = round_coin_value(msg['stake_amount'], msg['stake_currency']) - message += f"\n*Remaining:* `({rem}" + stake_amount_fiat = self.__format_profit_fiat(msg, 'stake_amount') - if msg.get('fiat_currency', None): - message += f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}" - - message += ")`" + rem = fmt_coin(msg['stake_amount'], msg['quote_currency']) + message += f"\n*Remaining:* `{rem}{stake_amount_fiat}`" else: - message += f"\n*Duration:* `{msg['duration']} ({msg['duration_min']:.1f} min)`" + message += f"\n*Duration:* `{duration} ({duration_min:.1f} min)`" return message - def compose_message(self, msg: Dict[str, Any], msg_type: RPCMessageType) -> Optional[str]: - if msg_type in [RPCMessageType.ENTRY, RPCMessageType.ENTRY_FILL]: + def __format_profit_fiat( + self, + msg: RPCExitMsg, + key: Literal['stake_amount', 'profit_amount', 'cumulative_profit'] + ) -> str: + """ + Format Fiat currency to append to regular profit output + """ + profit_fiat_extra = '' + if self._rpc._fiat_converter and (fiat_currency := msg.get('fiat_currency')): + profit_fiat = self._rpc._fiat_converter.convert_amount( + msg[key], msg['stake_currency'], fiat_currency) + profit_fiat_extra = f" / {profit_fiat:.3f} {fiat_currency}" + return profit_fiat_extra + + def compose_message(self, msg: RPCSendMsg) -> Optional[str]: + if msg['type'] == RPCMessageType.ENTRY or msg['type'] == RPCMessageType.ENTRY_FILL: message = self._format_entry_msg(msg) - elif msg_type in [RPCMessageType.EXIT, RPCMessageType.EXIT_FILL]: + elif msg['type'] == RPCMessageType.EXIT or msg['type'] == RPCMessageType.EXIT_FILL: message = self._format_exit_msg(msg) - elif msg_type in (RPCMessageType.ENTRY_CANCEL, RPCMessageType.EXIT_CANCEL): - msg['message_side'] = 'enter' if msg_type in [RPCMessageType.ENTRY_CANCEL] else 'exit' + elif ( + msg['type'] == RPCMessageType.ENTRY_CANCEL + or msg['type'] == RPCMessageType.EXIT_CANCEL + ): + message_side = 'enter' if msg['type'] == RPCMessageType.ENTRY_CANCEL else 'exit' message = (f"\N{WARNING SIGN} *{self._exchange_from_msg(msg)}:* " f"Cancelling {'partial ' if msg.get('sub_trade') else ''}" - f"{msg['message_side']} Order for {msg['pair']} " + f"{message_side} Order for {msg['pair']} " f"(#{msg['trade_id']}). Reason: {msg['reason']}.") - elif msg_type == RPCMessageType.PROTECTION_TRIGGER: + elif msg['type'] == RPCMessageType.PROTECTION_TRIGGER: message = ( f"*Protection* triggered due to {msg['reason']}. " f"`{msg['pair']}` will be locked until `{msg['lock_end_time']}`." ) - elif msg_type == RPCMessageType.PROTECTION_TRIGGER_GLOBAL: + elif msg['type'] == RPCMessageType.PROTECTION_TRIGGER_GLOBAL: message = ( f"*Protection* triggered due to {msg['reason']}. " f"*All pairs* will be locked until `{msg['lock_end_time']}`." ) - elif msg_type == RPCMessageType.STATUS: + elif msg['type'] == RPCMessageType.STATUS: message = f"*Status:* `{msg['status']}`" - elif msg_type == RPCMessageType.WARNING: + elif msg['type'] == RPCMessageType.WARNING: message = f"\N{WARNING SIGN} *Warning:* `{msg['status']}`" - elif msg_type == RPCMessageType.EXCEPTION: + elif msg['type'] == RPCMessageType.EXCEPTION: # Errors will contain exceptions, which are wrapped in tripple ticks. message = f"\N{WARNING SIGN} *ERROR:* \n {msg['status']}" - elif msg_type == RPCMessageType.STARTUP: + elif msg['type'] == RPCMessageType.STARTUP: message = f"{msg['status']}" - elif msg_type == RPCMessageType.STRATEGY_MSG: + elif msg['type'] == RPCMessageType.STRATEGY_MSG: message = f"{msg['msg']}" else: - logger.debug("Unknown message type: %s", msg_type) + logger.debug("Unknown message type: %s", msg['type']) return None return message @@ -495,20 +521,20 @@ class Telegram(RPCHandler): # Notification disabled return - message = self.compose_message(deepcopy(msg), msg_type) # type: ignore + message = self.compose_message(deepcopy(msg)) if message: asyncio.run_coroutine_threadsafe( self._send_msg(message, disable_notification=(noti == 'silent')), self._loop) - def _get_sell_emoji(self, msg): + def _get_exit_emoji(self, msg): """ - Get emoji for sell-side + Get emoji for exit-messages """ - if float(msg['profit_percent']) >= 5.0: + if float(msg['profit_ratio']) >= 0.05: return "\N{ROCKET}" - elif float(msg['profit_percent']) >= 0.0: + elif float(msg['profit_ratio']) >= 0.0: return "\N{EIGHT SPOKED ASTERISK}" elif msg['exit_reason'] == "stop_loss": return "\N{WARNING SIGN}" @@ -537,7 +563,7 @@ class Telegram(RPCHandler): if order_nr == 1: lines.append( f"*Amount:* {cur_entry_amount:.8g} " - f"({round_coin_value(order['cost'], quote_currency)})" + f"({fmt_coin(order['cost'], quote_currency)})" ) lines.append(f"*Average Price:* {cur_entry_average:.8g}") else: @@ -547,7 +573,7 @@ class Telegram(RPCHandler): lines.append("({})".format(dt_humanize(order["order_filled_date"], granularity=["day", "hour", "minute"]))) lines.append(f"*Amount:* {cur_entry_amount:.8g} " - f"({round_coin_value(order['cost'], quote_currency)})") + f"({fmt_coin(order['cost'], quote_currency)})") lines.append(f"*Average {wording} Price:* {cur_entry_average:.8g} " f"({price_to_1st_entry:.2%} from 1st entry rate)") lines.append(f"*Order Filled:* {order['order_filled_date']}") @@ -633,12 +659,12 @@ class Telegram(RPCHandler): r['num_exits'] = len([o for o in r['orders'] if not o['ft_is_entry'] and not o['ft_order_side'] == 'stoploss']) r['exit_reason'] = r.get('exit_reason', "") - r['stake_amount_r'] = round_coin_value(r['stake_amount'], r['quote_currency']) - r['max_stake_amount_r'] = round_coin_value( + r['stake_amount_r'] = fmt_coin(r['stake_amount'], r['quote_currency']) + r['max_stake_amount_r'] = fmt_coin( r['max_stake_amount'] or r['stake_amount'], r['quote_currency']) - r['profit_abs_r'] = round_coin_value(r['profit_abs'], r['quote_currency']) - r['realized_profit_r'] = round_coin_value(r['realized_profit'], r['quote_currency']) - r['total_profit_abs_r'] = round_coin_value( + r['profit_abs_r'] = fmt_coin(r['profit_abs'], r['quote_currency']) + r['realized_profit_r'] = fmt_coin(r['realized_profit'], r['quote_currency']) + r['total_profit_abs_r'] = fmt_coin( r['total_profit_abs'], r['quote_currency']) lines = [ "*Trade ID:* `{trade_id}`" + @@ -781,7 +807,7 @@ class Telegram(RPCHandler): ) stats_tab = tabulate( [[f"{period['date']:{val.dateformat}} ({period['trade_count']})", - f"{round_coin_value(period['abs_profit'], stats['stake_currency'])}", + f"{fmt_coin(period['abs_profit'], stats['stake_currency'])}", f"{period['fiat_value']:.2f} {stats['fiat_display_currency']}", f"{period['rel_profit']:.2%}", ] for period in stats['data']], @@ -883,19 +909,19 @@ class Telegram(RPCHandler): # Message to display if stats['closed_trade_count'] > 0: markdown_msg = ("*ROI:* Closed trades\n" - f"∙ `{round_coin_value(profit_closed_coin, stake_cur)} " + f"∙ `{fmt_coin(profit_closed_coin, stake_cur)} " f"({profit_closed_ratio_mean:.2%}) " f"({profit_closed_percent} \N{GREEK CAPITAL LETTER SIGMA}%)`\n" - f"∙ `{round_coin_value(profit_closed_fiat, fiat_disp_cur)}`\n") + f"∙ `{fmt_coin(profit_closed_fiat, fiat_disp_cur)}`\n") else: markdown_msg = "`No closed trade` \n" markdown_msg += ( f"*ROI:* All trades\n" - f"∙ `{round_coin_value(profit_all_coin, stake_cur)} " + f"∙ `{fmt_coin(profit_all_coin, stake_cur)} " f"({profit_all_ratio_mean:.2%}) " f"({profit_all_percent} \N{GREEK CAPITAL LETTER SIGMA}%)`\n" - f"∙ `{round_coin_value(profit_all_fiat, fiat_disp_cur)}`\n" + f"∙ `{fmt_coin(profit_all_fiat, fiat_disp_cur)}`\n" f"*Total Trade Count:* `{trade_count}`\n" f"*Bot started:* `{stats['bot_start_date']}`\n" f"*{'First Trade opened' if not timescale else 'Showing Profit since'}:* " @@ -909,14 +935,14 @@ class Telegram(RPCHandler): markdown_msg += ( f"\n*Avg. Duration:* `{avg_duration}`\n" f"*Best Performing:* `{best_pair}: {best_pair_profit_ratio:.2%}`\n" - f"*Trading volume:* `{round_coin_value(stats['trading_volume'], stake_cur)}`\n" + f"*Trading volume:* `{fmt_coin(stats['trading_volume'], stake_cur)}`\n" f"*Profit factor:* `{stats['profit_factor']:.2f}`\n" f"*Max Drawdown:* `{stats['max_drawdown']:.2%} " - f"({round_coin_value(stats['max_drawdown_abs'], stake_cur)})`\n" + f"({fmt_coin(stats['max_drawdown_abs'], stake_cur)})`\n" f" from `{stats['max_drawdown_start']} " - f"({round_coin_value(stats['drawdown_high'], stake_cur)})`\n" + f"({fmt_coin(stats['drawdown_high'], stake_cur)})`\n" f" to `{stats['max_drawdown_end']} " - f"({round_coin_value(stats['drawdown_low'], stake_cur)})`\n" + f"({fmt_coin(stats['drawdown_low'], stake_cur)})`\n" ) await self._send_msg(markdown_msg, reload_able=True, callback_path="update_profit", query=update.callback_query) @@ -984,9 +1010,9 @@ class Telegram(RPCHandler): output = '' if self._config['dry_run']: output += "*Warning:* Simulated balances in Dry Mode.\n" - starting_cap = round_coin_value(result['starting_capital'], self._config['stake_currency']) + starting_cap = fmt_coin(result['starting_capital'], self._config['stake_currency']) output += f"Starting capital: `{starting_cap}`" - starting_cap_fiat = round_coin_value( + starting_cap_fiat = fmt_coin( result['starting_capital_fiat'], self._config['fiat_display_currency'] ) if result['starting_capital_fiat'] > 0 else '' output += (f" `, {starting_cap_fiat}`.\n" @@ -1006,9 +1032,9 @@ class Telegram(RPCHandler): f"\t`{curr['side']}: {curr['position']:.8f}`\n" f"\t`Leverage: {curr['leverage']:.1f}`\n" f"\t`Est. {curr['stake']}: " - f"{round_coin_value(curr['est_stake'], curr['stake'], False)}`\n") + f"{fmt_coin(curr['est_stake'], curr['stake'], False)}`\n") else: - est_stake = round_coin_value( + est_stake = fmt_coin( curr['est_stake' if full_result else 'est_stake_bot'], curr['stake'], False) curr_output = ( @@ -1036,13 +1062,13 @@ class Telegram(RPCHandler): f"{plural(total_dust_currencies, 'Currency', 'Currencies')} " f"(< {balance_dust_level} {result['stake']}):*\n" f"\t`Est. {result['stake']}: " - f"{round_coin_value(total_dust_balance, result['stake'], False)}`\n") + f"{fmt_coin(total_dust_balance, result['stake'], False)}`\n") tc = result['trade_count'] > 0 stake_improve = f" `({result['starting_capital_ratio']:.2%})`" if tc else '' fiat_val = f" `({result['starting_capital_fiat_ratio']:.2%})`" if tc else '' - value = round_coin_value( + value = fmt_coin( result['value' if full_result else 'value_bot'], result['symbol'], False) - total_stake = round_coin_value( + total_stake = fmt_coin( result['total' if full_result else 'total_bot'], result['stake'], False) output += ( f"\n*Estimated Value{' (Bot managed assets only)' if not full_result else ''}*:\n" @@ -1150,7 +1176,7 @@ class Telegram(RPCHandler): try: loop = asyncio.get_running_loop() # Workaround to avoid nested loops - await loop.run_in_executor(None, self._rpc._rpc_force_exit, trade_id) + await loop.run_in_executor(None, safe_async_db(self._rpc._rpc_force_exit), trade_id) except RPCException as e: await self._send_msg(str(e)) @@ -1176,6 +1202,7 @@ class Telegram(RPCHandler): async def _force_enter_action(self, pair, price: Optional[float], order_side: SignalDirection): if pair != 'cancel': try: + @safe_async_db def _force_enter(): self._rpc._rpc_force_entry(pair, price, order_side=order_side) loop = asyncio.get_running_loop() @@ -1319,8 +1346,8 @@ class Telegram(RPCHandler): output = "Performance:\n" for i, trade in enumerate(trades): stat_line = ( - f"{i+1}.\t {trade['pair']}\t" - f"{round_coin_value(trade['profit_abs'], self._config['stake_currency'])} " + f"{i + 1}.\t {trade['pair']}\t" + f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} " f"({trade['profit_ratio']:.2%}) " f"({trade['count']})\n") @@ -1351,8 +1378,8 @@ class Telegram(RPCHandler): output = "Entry Tag Performance:\n" for i, trade in enumerate(trades): stat_line = ( - f"{i+1}.\t {trade['enter_tag']}\t" - f"{round_coin_value(trade['profit_abs'], self._config['stake_currency'])} " + f"{i + 1}.\t {trade['enter_tag']}\t" + f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} " f"({trade['profit_ratio']:.2%}) " f"({trade['count']})\n") @@ -1383,8 +1410,8 @@ class Telegram(RPCHandler): output = "Exit Reason Performance:\n" for i, trade in enumerate(trades): stat_line = ( - f"{i+1}.\t {trade['exit_reason']}\t" - f"{round_coin_value(trade['profit_abs'], self._config['stake_currency'])} " + f"{i + 1}.\t {trade['exit_reason']}\t" + f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} " f"({trade['profit_ratio']:.2%}) " f"({trade['count']})\n") @@ -1415,8 +1442,8 @@ class Telegram(RPCHandler): output = "Mix Tag Performance:\n" for i, trade in enumerate(trades): stat_line = ( - f"{i+1}.\t {trade['mix_tag']}\t" - f"{round_coin_value(trade['profit_abs'], self._config['stake_currency'])} " + f"{i + 1}.\t {trade['mix_tag']}\t" + f"{fmt_coin(trade['profit_abs'], self._config['stake_currency'])} " f"({trade['profit_ratio']:.2%}) " f"({trade['count']})\n") diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index b9bdbd435..9b12b7a21 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -84,7 +84,7 @@ class Webhook(RPCHandler): valuedict = self._get_value_dict(msg) if not valuedict: - logger.info("Message type '%s' not configured for webhooks", msg['type']) + logger.debug("Message type '%s' not configured for webhooks", msg['type']) return payload = {key: value.format(**msg) for (key, value) in valuedict.items()} diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 5f60b53d1..d49b49bdc 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -512,7 +512,8 @@ class IStrategy(ABC, HyperStrategyMixin): min_stake: Optional[float], max_stake: float, current_entry_rate: float, current_exit_rate: float, current_entry_profit: float, current_exit_profit: float, - **kwargs) -> Optional[float]: + **kwargs + ) -> Union[Optional[float], Tuple[Optional[float], Optional[str]]]: """ Custom trade adjustment logic, returning the stake amount that a trade should be increased or decreased. @@ -538,6 +539,7 @@ class IStrategy(ABC, HyperStrategyMixin): :return float: Stake amount to adjust your trade, Positive values to increase position, Negative values to decrease position. Return None for no action. + Optionally, return a tuple with a 2nd element with an order reason """ return None @@ -726,6 +728,36 @@ class IStrategy(ABC, HyperStrategyMixin): _ft_stop_uses_after_fill = False + def _adjust_trade_position_internal( + self, trade: Trade, current_time: datetime, + current_rate: float, current_profit: float, + min_stake: Optional[float], max_stake: float, + current_entry_rate: float, current_exit_rate: float, + current_entry_profit: float, current_exit_profit: float, + **kwargs + ) -> Tuple[Optional[float], str]: + """ + wrapper around adjust_trade_position to handle the return value + """ + resp = strategy_safe_wrapper(self.adjust_trade_position, + default_retval=(None, ''), supress_error=True)( + trade=trade, current_time=current_time, + current_rate=current_rate, current_profit=current_profit, + min_stake=min_stake, max_stake=max_stake, + current_entry_rate=current_entry_rate, current_exit_rate=current_exit_rate, + current_entry_profit=current_entry_profit, current_exit_profit=current_exit_profit, + **kwargs + ) + order_tag = '' + if isinstance(resp, tuple): + if len(resp) >= 1: + stake_amount = resp[0] + if len(resp) > 1: + order_tag = resp[1] or '' + else: + stake_amount = resp + return stake_amount, order_tag + def __informative_pairs_freqai(self) -> ListPairsWithTimeframes: """ Create informative-pairs needed for FreqAI @@ -1006,7 +1038,7 @@ class IStrategy(ABC, HyperStrategyMixin): :param is_short: Indicating existing trade direction. :return: (enter, exit) A bool-tuple with enter / exit values. """ - latest, latest_date = self.get_latest_candle(pair, timeframe, dataframe) + latest, _latest_date = self.get_latest_candle(pair, timeframe, dataframe) if latest is None: return False, False, None @@ -1407,7 +1439,8 @@ class IStrategy(ABC, HyperStrategyMixin): """ logger.debug(f"Populating enter signals for pair {metadata.get('pair')}.") - + # Initialize column to work around Pandas bug #56503. + dataframe.loc[:, 'enter_tag'] = '' df = self.populate_entry_trend(dataframe, metadata) if 'enter_long' not in df.columns: df = df.rename({'buy': 'enter_long', 'buy_tag': 'enter_tag'}, axis='columns') @@ -1423,6 +1456,8 @@ class IStrategy(ABC, HyperStrategyMixin): currently traded pair :return: DataFrame with exit column """ + # Initialize column to work around Pandas bug #56503. + dataframe.loc[:, 'exit_tag'] = '' logger.debug(f"Populating exit signals for pair {metadata.get('pair')}.") df = self.populate_exit_trend(dataframe, metadata) if 'exit_long' not in df.columns: diff --git a/freqtrade/templates/FreqaiExampleHybridStrategy.py b/freqtrade/templates/FreqaiExampleHybridStrategy.py index 03446d76e..5df03bd5d 100644 --- a/freqtrade/templates/FreqaiExampleHybridStrategy.py +++ b/freqtrade/templates/FreqaiExampleHybridStrategy.py @@ -29,7 +29,7 @@ class FreqaiExampleHybridStrategy(IStrategy): "enabled": true, "purge_old_models": 2, "train_period_days": 15, - "identifier": "uniqe-id", + "identifier": "unique-id", "feature_parameters": { "include_timeframes": [ "3m", diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index 8be1f0336..93b916e38 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -6,7 +6,7 @@ import talib.abstract as ta from pandas import DataFrame from technical import qtpylib -from freqtrade.strategy import CategoricalParameter, IStrategy +from freqtrade.strategy import IStrategy logger = logging.getLogger(__name__) @@ -45,11 +45,6 @@ class FreqaiExampleStrategy(IStrategy): startup_candle_count: int = 40 can_short = True - std_dev_multiplier_buy = CategoricalParameter( - [0.75, 1, 1.25, 1.5, 1.75], default=1.25, space="buy", optimize=True) - std_dev_multiplier_sell = CategoricalParameter( - [0.75, 1, 1.25, 1.5, 1.75], space="sell", default=1.25, optimize=True) - def feature_engineering_expand_all(self, dataframe: DataFrame, period: int, metadata: Dict, **kwargs) -> DataFrame: """ @@ -239,21 +234,13 @@ class FreqaiExampleStrategy(IStrategy): dataframe = self.freqai.start(dataframe, metadata, self) - for val in self.std_dev_multiplier_buy.range: - dataframe[f'target_roi_{val}'] = ( - dataframe["&-s_close_mean"] + dataframe["&-s_close_std"] * val - ) - for val in self.std_dev_multiplier_sell.range: - dataframe[f'sell_roi_{val}'] = ( - dataframe["&-s_close_mean"] - dataframe["&-s_close_std"] * val - ) return dataframe def populate_entry_trend(self, df: DataFrame, metadata: dict) -> DataFrame: enter_long_conditions = [ df["do_predict"] == 1, - df["&-s_close"] > df[f"target_roi_{self.std_dev_multiplier_buy.value}"], + df["&-s_close"] > 0.01, ] if enter_long_conditions: @@ -263,7 +250,7 @@ class FreqaiExampleStrategy(IStrategy): enter_short_conditions = [ df["do_predict"] == 1, - df["&-s_close"] < df[f"sell_roi_{self.std_dev_multiplier_sell.value}"], + df["&-s_close"] < -0.01, ] if enter_short_conditions: @@ -276,14 +263,14 @@ class FreqaiExampleStrategy(IStrategy): def populate_exit_trend(self, df: DataFrame, metadata: dict) -> DataFrame: exit_long_conditions = [ df["do_predict"] == 1, - df["&-s_close"] < df[f"sell_roi_{self.std_dev_multiplier_sell.value}"] * 0.25, + df["&-s_close"] < 0 ] if exit_long_conditions: df.loc[reduce(lambda x, y: x & y, exit_long_conditions), "exit_long"] = 1 exit_short_conditions = [ df["do_predict"] == 1, - df["&-s_close"] > df[f"target_roi_{self.std_dev_multiplier_buy.value}"] * 0.25, + df["&-s_close"] > 0 ] if exit_short_conditions: df.loc[reduce(lambda x, y: x & y, exit_short_conditions), "exit_short"] = 1 diff --git a/freqtrade/templates/base_config.json.j2 b/freqtrade/templates/base_config.json.j2 index 1a4552c11..caa27a69e 100644 --- a/freqtrade/templates/base_config.json.j2 +++ b/freqtrade/templates/base_config.json.j2 @@ -39,7 +39,7 @@ }, {{ exchange | indent(4) }}, "pairlists": [ - {{ '{"method": "StaticPairList"}' if exchange_name == 'bittrex' else volume_pairlist }} + {{ volume_pairlist }} ], "telegram": { "enabled": {{ telegram | lower }}, diff --git a/freqtrade/types/backtest_result_type.py b/freqtrade/types/backtest_result_type.py index 1043899f7..7a6fc79fa 100644 --- a/freqtrade/types/backtest_result_type.py +++ b/freqtrade/types/backtest_result_type.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from typing_extensions import TypedDict @@ -26,3 +26,7 @@ class BacktestHistoryEntryType(BacktestMetadataType): filename: str strategy: str notes: str + backtest_start_ts: Optional[int] + backtest_end_ts: Optional[int] + timeframe: Optional[str] + timeframe_detail: Optional[str] diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index fc7cf5f6a..513406fd2 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -1,6 +1,7 @@ from freqtrade.util.datetime_helpers import (dt_floor_day, dt_from_ts, dt_humanize, dt_now, dt_ts, dt_ts_def, dt_utc, format_date, format_ms_time, shorten_date) +from freqtrade.util.formatters import decimals_per_coin, fmt_coin, round_value from freqtrade.util.ft_precise import FtPrecise from freqtrade.util.periodic_cache import PeriodicCache from freqtrade.util.template_renderer import render_template, render_template_with_fallback # noqa @@ -19,4 +20,7 @@ __all__ = [ 'FtPrecise', 'PeriodicCache', 'shorten_date', + 'decimals_per_coin', + 'round_value', + 'fmt_coin', ] diff --git a/freqtrade/util/formatters.py b/freqtrade/util/formatters.py new file mode 100644 index 000000000..1a3d064a9 --- /dev/null +++ b/freqtrade/util/formatters.py @@ -0,0 +1,42 @@ +from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN + + +def decimals_per_coin(coin: str): + """ + Helper method getting decimal amount for this coin + example usage: f".{decimals_per_coin('USD')}f" + :param coin: Which coin are we printing the price / value for + """ + return DECIMALS_PER_COIN.get(coin, DECIMAL_PER_COIN_FALLBACK) + + +def round_value(value: float, decimals: int, keep_trailing_zeros=False) -> str: + """ + Round value to given decimals + :param value: Value to be rounded + :param decimals: Number of decimals to round to + :param keep_trailing_zeros: Keep trailing zeros "222.200" vs. "222.2" + :return: Rounded value as string + """ + val = f"{value:.{decimals}f}" + if not keep_trailing_zeros: + val = val.rstrip('0').rstrip('.') + return val + + +def fmt_coin( + value: float, coin: str, show_coin_name=True, keep_trailing_zeros=False) -> str: + """ + Format price value for this coin + :param value: Value to be printed + :param coin: Which coin are we printing the price / value for + :param show_coin_name: Return string in format: "222.22 USDT" or "222.22" + :param keep_trailing_zeros: Keep trailing zeros "222.200" vs. "222.2" + :return: Formatted / rounded value (with or without coin name) + """ + val = f"{value:.{decimals_per_coin(coin)}f}" + val = round_value(value, decimals_per_coin(coin), keep_trailing_zeros) + if show_coin_name: + val = f"{val} {coin}" + + return val diff --git a/freqtrade/util/migrations/__init__.py b/freqtrade/util/migrations/__init__.py new file mode 100644 index 000000000..9bd6f6288 --- /dev/null +++ b/freqtrade/util/migrations/__init__.py @@ -0,0 +1,12 @@ +from typing import Optional + +from freqtrade.exchange import Exchange +from freqtrade.util.migrations.binance_mig import migrate_binance_futures_names # noqa F401 +from freqtrade.util.migrations.binance_mig import migrate_binance_futures_data +from freqtrade.util.migrations.funding_rate_mig import migrate_funding_fee_timeframe + + +def migrate_data(config, exchange: Optional[Exchange] = None): + migrate_binance_futures_data(config) + + migrate_funding_fee_timeframe(config, exchange) diff --git a/freqtrade/util/binance_mig.py b/freqtrade/util/migrations/binance_mig.py similarity index 100% rename from freqtrade/util/binance_mig.py rename to freqtrade/util/migrations/binance_mig.py diff --git a/freqtrade/util/migrations/funding_rate_mig.py b/freqtrade/util/migrations/funding_rate_mig.py new file mode 100644 index 000000000..9fe433b2d --- /dev/null +++ b/freqtrade/util/migrations/funding_rate_mig.py @@ -0,0 +1,27 @@ +import logging +from typing import Optional + +from freqtrade.constants import Config +from freqtrade.data.history.idatahandler import get_datahandler +from freqtrade.enums import TradingMode +from freqtrade.exchange import Exchange + + +logger = logging.getLogger(__name__) + + +def migrate_funding_fee_timeframe(config: Config, exchange: Optional[Exchange]): + if ( + config.get('trading_mode', TradingMode.SPOT) != TradingMode.FUTURES + ): + # only act on futures + return + + if not exchange: + from freqtrade.resolvers import ExchangeResolver + exchange = ExchangeResolver.load_exchange(config, validate=False) + + ff_timeframe = exchange.get_option('funding_fee_timeframe') + + dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv']) + dhc.fix_funding_fee_timeframe(ff_timeframe) diff --git a/pyproject.toml b/pyproject.toml index c446419e2..1d8d9420d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,7 @@ skip_glob = ["**/.env*", "**/env/*", "**/.venv/*", "**/docs/*", "**/user_data/*" [tool.pytest.ini_options] asyncio_mode = "auto" +addopts = "--dist loadscope" [tool.mypy] ignore_missing_imports = true diff --git a/requirements-dev.txt b/requirements-dev.txt index 32e2cf4f9..04d4a8563 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,24 +7,25 @@ -r docs/requirements-docs.txt coveralls==3.3.1 -ruff==0.1.8 -mypy==1.7.1 +ruff==0.1.15 +mypy==1.8.0 pre-commit==3.6.0 -pytest==7.4.3 -pytest-asyncio==0.21.1 +pytest==7.4.4 +pytest-asyncio==0.23.4 pytest-cov==4.1.0 pytest-mock==3.12.0 -pytest-random-order==1.1.0 +pytest-random-order==1.1.1 +pytest-xdist==3.5.0 isort==5.13.2 # For datetime mocking time-machine==2.13.0 # Convert jupyter notebooks to markdown documents -nbconvert==7.12.0 +nbconvert==7.14.2 # mypy types types-cachetools==5.3.0.7 types-filelock==3.2.7 -types-requests==2.31.0.10 -types-tabulate==0.9.0.3 -types-python-dateutil==2.8.19.14 +types-requests==2.31.0.20240125 +types-tabulate==0.9.0.20240106 +types-python-dateutil==2.8.19.20240106 diff --git a/requirements-freqai-rl.txt b/requirements-freqai-rl.txt index 55a09e6f9..fa5e9f014 100644 --- a/requirements-freqai-rl.txt +++ b/requirements-freqai-rl.txt @@ -2,10 +2,10 @@ -r requirements-freqai.txt # Required for freqai-rl -torch==2.1.2 +torch==2.1.2; python_version < '3.12' #until these branches will be released we can use this -gymnasium==0.29.1 -stable_baselines3==2.2.1 -sb3_contrib>=2.0.0a9 +gymnasium==0.29.1; python_version < '3.12' +stable_baselines3==2.2.1; python_version < '3.12' +sb3_contrib>=2.0.0a9; python_version < '3.12' # Progress bar for stable-baselines3 and sb3-contrib tqdm==4.66.1 diff --git a/requirements-freqai.txt b/requirements-freqai.txt index 7d817c2d9..2d33efc3c 100644 --- a/requirements-freqai.txt +++ b/requirements-freqai.txt @@ -3,10 +3,10 @@ -r requirements-plot.txt # Required for freqai -scikit-learn==1.3.2 +scikit-learn==1.4.0 joblib==1.3.2 -catboost==1.2.2; 'arm' not in platform_machine -lightgbm==4.1.0 -xgboost==2.0.2 +catboost==1.2.2; 'arm' not in platform_machine and python_version < '3.12' +lightgbm==4.3.0 +xgboost==2.0.3 tensorboard==2.15.1 datasieve==0.1.7 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index d7f440c0a..b961b3b04 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -2,7 +2,7 @@ -r requirements.txt # Required for hyperopt -scipy==1.11.4 -scikit-learn==1.3.2 +scipy==1.12.0 +scikit-learn==1.4.0 ft-scikit-optimize==0.9.2 filelock==3.13.1 diff --git a/requirements.txt b/requirements.txt index 6c41cf475..535d94946 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ -numpy==1.26.2 +numpy==1.26.3 pandas==2.1.4 pandas-ta==0.3.14b -ccxt==4.1.84 -cryptography==41.0.7 -aiohttp==3.9.1 -SQLAlchemy==2.0.23 +ccxt==4.2.25 +cryptography==42.0.1 +aiohttp==3.9.2 +SQLAlchemy==2.0.25 python-telegram-bot==20.7 # can't be hard-pinned due to telegram-bot pinning httpx with ~ httpx>=0.24.1 @@ -13,16 +13,16 @@ arrow==1.3.0 cachetools==5.3.2 requests==2.31.0 urllib3==2.1.0 -jsonschema==4.20.0 +jsonschema==4.21.1 TA-Lib==0.4.28 technical==1.4.2 tabulate==0.9.0 pycoingecko==3.1.0 -jinja2==3.1.2 +jinja2==3.1.3 tables==3.9.1 joblib==1.3.2 rich==13.7.0 -pyarrow==14.0.1; platform_machine != 'armv7l' +pyarrow==15.0.0; platform_machine != 'armv7l' # find first, C search in arrays py_find_1st==1.1.6 @@ -30,18 +30,18 @@ py_find_1st==1.1.6 # Load ticker files 30% faster python-rapidjson==1.14 # Properly format api responses -orjson==3.9.10 +orjson==3.9.12 # Notify systemd sdnotify==0.3.2 # API Server -fastapi==0.105.0 -pydantic==2.5.2 -uvicorn==0.24.0.post1 +fastapi==0.109.0 +pydantic==2.5.3 +uvicorn==0.27.0 pyjwt==2.8.0 aiofiles==23.2.1 -psutil==5.9.7 +psutil==5.9.8 # Support for colorized terminal output colorama==0.4.6 @@ -58,5 +58,5 @@ schedule==1.2.1 websockets==12.0 janus==1.0.0 -ast-comments==1.2.0 +ast-comments==1.2.1 packaging==23.2 diff --git a/setup.py b/setup.py index 8100f21ae..64b30ed94 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ setup( ], install_requires=[ # from requirements.txt - 'ccxt>=4.0.0', + 'ccxt>=4.2.15', 'SQLAlchemy>=2.0.6', 'python-telegram-bot>=20.1', 'arrow>=1.0.0', diff --git a/tests/commands/test_build_config.py b/tests/commands/test_build_config.py index 7bf374ae0..f799be3ba 100644 --- a/tests/commands/test_build_config.py +++ b/tests/commands/test_build_config.py @@ -30,7 +30,7 @@ def test_validate_is_int(): assert not validate_is_int('-ee') -@pytest.mark.parametrize('exchange', ['bittrex', 'binance', 'kraken']) +@pytest.mark.parametrize('exchange', ['bybit', 'binance', 'kraken']) def test_start_new_config(mocker, caplog, exchange): wt_mock = mocker.patch.object(Path, "write_text", MagicMock()) mocker.patch.object(Path, "exists", MagicMock(return_value=True)) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 7698740a5..cdad46407 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -32,7 +32,7 @@ from tests.conftest_trades import MOCK_TRADE_COUNT def test_setup_utils_configuration(): args = [ - 'list-exchanges', '--config', 'config_examples/config_bittrex.example.json', + 'list-exchanges', '--config', 'tests/testdata/testconfigs/main_test_config.json', ] config = setup_utils_configuration(get_args(args), RunMode.OTHER) @@ -49,7 +49,7 @@ def test_start_trading_fail(mocker, caplog): exitmock = mocker.patch("freqtrade.worker.Worker.exit", MagicMock()) args = [ 'trade', - '-c', 'config_examples/config_bittrex.example.json' + '-c', 'tests/testdata/testconfigs/main_test_config.json' ] start_trading(get_args(args)) assert exitmock.call_count == 1 @@ -68,7 +68,7 @@ def test_start_webserver(mocker, caplog): args = [ 'webserver', - '-c', 'config_examples/config_bittrex.example.json' + '-c', 'tests/testdata/testconfigs/main_test_config.json' ] start_webserver(get_args(args)) assert api_server_mock.call_count == 1 @@ -84,7 +84,7 @@ def test_list_exchanges(capsys): captured = capsys.readouterr() assert re.match(r"Exchanges available for Freqtrade.*", captured.out) assert re.search(r".*binance.*", captured.out) - assert re.search(r".*bittrex.*", captured.out) + assert re.search(r".*bybit.*", captured.out) # Test with --one-column args = [ @@ -95,7 +95,7 @@ def test_list_exchanges(capsys): start_list_exchanges(get_args(args)) captured = capsys.readouterr() assert re.search(r"^binance$", captured.out, re.MULTILINE) - assert re.search(r"^bittrex$", captured.out, re.MULTILINE) + assert re.search(r"^bybit$", captured.out, re.MULTILINE) # Test with --all args = [ @@ -107,7 +107,7 @@ def test_list_exchanges(capsys): captured = capsys.readouterr() assert re.match(r"All exchanges supported by the ccxt library.*", captured.out) assert re.search(r".*binance.*", captured.out) - assert re.search(r".*bittrex.*", captured.out) + assert re.search(r".*bingx.*", captured.out) assert re.search(r".*bitmex.*", captured.out) # Test with --one-column --all @@ -120,7 +120,7 @@ def test_list_exchanges(capsys): start_list_exchanges(get_args(args)) captured = capsys.readouterr() assert re.search(r"^binance$", captured.out, re.MULTILINE) - assert re.search(r"^bittrex$", captured.out, re.MULTILINE) + assert re.search(r"^bingx$", captured.out, re.MULTILINE) assert re.search(r"^bitmex$", captured.out, re.MULTILINE) @@ -133,7 +133,7 @@ def test_list_timeframes(mocker, capsys): '1h': 'hour', '1d': 'day', } - patch_exchange(mocker, api_mock=api_mock, id='bittrex') + patch_exchange(mocker, api_mock=api_mock, id='bybit') args = [ "list-timeframes", ] @@ -143,25 +143,25 @@ def test_list_timeframes(mocker, capsys): match=r"This command requires a configured exchange.*"): start_list_timeframes(pargs) - # Test with --config config_examples/config_bittrex.example.json + # Test with --config tests/testdata/testconfigs/main_test_config.json args = [ "list-timeframes", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', ] start_list_timeframes(get_args(args)) captured = capsys.readouterr() - assert re.match("Timeframes available for the exchange `Bittrex`: " + assert re.match("Timeframes available for the exchange `Bybit`: " "1m, 5m, 30m, 1h, 1d", captured.out) - # Test with --exchange bittrex + # Test with --exchange bybit args = [ "list-timeframes", - "--exchange", "bittrex", + "--exchange", "bybit", ] start_list_timeframes(get_args(args)) captured = capsys.readouterr() - assert re.match("Timeframes available for the exchange `Bittrex`: " + assert re.match("Timeframes available for the exchange `Bybit`: " "1m, 5m, 30m, 1h, 1d", captured.out) @@ -190,7 +190,7 @@ def test_list_timeframes(mocker, capsys): # Test with --one-column args = [ "list-timeframes", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', "--one-column", ] start_list_timeframes(get_args(args)) @@ -217,7 +217,7 @@ def test_list_timeframes(mocker, capsys): def test_list_markets(mocker, markets_static, capsys): api_mock = MagicMock() - patch_exchange(mocker, api_mock=api_mock, id='bittrex', mock_markets=markets_static) + patch_exchange(mocker, api_mock=api_mock, id='binance', mock_markets=markets_static) # Test with no --config args = [ @@ -229,15 +229,15 @@ def test_list_markets(mocker, markets_static, capsys): match=r"This command requires a configured exchange.*"): start_list_markets(pargs, False) - # Test with --config config_examples/config_bittrex.example.json + # Test with --config tests/testdata/testconfigs/main_test_config.json args = [ "list-markets", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', "--print-list", ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 12 active markets: " + assert ("Exchange Binance has 12 active markets: " "ADA/USDT:USDT, BLK/BTC, ETH/BTC, ETH/USDT, ETH/USDT:USDT, LTC/BTC, " "LTC/ETH, LTC/USD, NEO/BTC, TKN/BTC, XLTCUSDT, XRP/BTC.\n" in captured.out) @@ -255,16 +255,16 @@ def test_list_markets(mocker, markets_static, capsys): assert re.match("\nExchange Binance has 12 active markets:\n", captured.out) - patch_exchange(mocker, api_mock=api_mock, id="bittrex", mock_markets=markets_static) + patch_exchange(mocker, api_mock=api_mock, id="binance", mock_markets=markets_static) # Test with --all: all markets args = [ "list-markets", "--all", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', "--print-list", ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 14 markets: " + assert ("Exchange Binance has 14 markets: " "ADA/USDT:USDT, BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, ETH/USDT:USDT, " "LTC/BTC, LTC/ETH, LTC/USD, LTC/USDT, NEO/BTC, TKN/BTC, XLTCUSDT, XRP/BTC.\n" in captured.out) @@ -272,24 +272,24 @@ def test_list_markets(mocker, markets_static, capsys): # Test list-pairs subcommand: active pairs args = [ "list-pairs", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', "--print-list", ] start_list_markets(get_args(args), True) captured = capsys.readouterr() - assert ("Exchange Bittrex has 9 active pairs: " + assert ("Exchange Binance has 9 active pairs: " "BLK/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/ETH, LTC/USD, NEO/BTC, TKN/BTC, XRP/BTC.\n" in captured.out) # Test list-pairs subcommand with --all: all pairs args = [ "list-pairs", "--all", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', "--print-list", ] start_list_markets(get_args(args), True) captured = capsys.readouterr() - assert ("Exchange Bittrex has 11 pairs: " + assert ("Exchange Binance has 11 pairs: " "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/ETH, LTC/USD, LTC/USDT, NEO/BTC, " "TKN/BTC, XRP/BTC.\n" in captured.out) @@ -297,133 +297,133 @@ def test_list_markets(mocker, markets_static, capsys): # active markets, base=ETH, LTC args = [ "list-markets", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', "--base", "ETH", "LTC", "--print-list", ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 7 active markets with ETH, LTC as base currencies: " + assert ("Exchange Binance has 7 active markets with ETH, LTC as base currencies: " "ETH/BTC, ETH/USDT, ETH/USDT:USDT, LTC/BTC, LTC/ETH, LTC/USD, XLTCUSDT.\n" in captured.out) # active markets, base=LTC args = [ "list-markets", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', "--base", "LTC", "--print-list", ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 4 active markets with LTC as base currency: " + assert ("Exchange Binance has 4 active markets with LTC as base currency: " "LTC/BTC, LTC/ETH, LTC/USD, XLTCUSDT.\n" in captured.out) # active markets, quote=USDT, USD args = [ "list-markets", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', "--quote", "USDT", "USD", "--print-list", ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 5 active markets with USDT, USD as quote currencies: " + assert ("Exchange Binance has 5 active markets with USDT, USD as quote currencies: " "ADA/USDT:USDT, ETH/USDT, ETH/USDT:USDT, LTC/USD, XLTCUSDT.\n" in captured.out) # active markets, quote=USDT args = [ "list-markets", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', "--quote", "USDT", "--print-list", ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 4 active markets with USDT as quote currency: " + assert ("Exchange Binance has 4 active markets with USDT as quote currency: " "ADA/USDT:USDT, ETH/USDT, ETH/USDT:USDT, XLTCUSDT.\n" in captured.out) # active markets, base=LTC, quote=USDT args = [ "list-markets", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', "--base", "LTC", "--quote", "USDT", "--print-list", ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 1 active market with LTC as base currency and " + assert ("Exchange Binance has 1 active market with LTC as base currency and " "with USDT as quote currency: XLTCUSDT.\n" in captured.out) # active pairs, base=LTC, quote=USDT args = [ "list-pairs", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', "--base", "LTC", "--quote", "USD", "--print-list", ] start_list_markets(get_args(args), True) captured = capsys.readouterr() - assert ("Exchange Bittrex has 1 active pair with LTC as base currency and " + assert ("Exchange Binance has 1 active pair with LTC as base currency and " "with USD as quote currency: LTC/USD.\n" in captured.out) # active markets, base=LTC, quote=USDT, NONEXISTENT args = [ "list-markets", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', "--base", "LTC", "--quote", "USDT", "NONEXISTENT", "--print-list", ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 1 active market with LTC as base currency and " + assert ("Exchange Binance has 1 active market with LTC as base currency and " "with USDT, NONEXISTENT as quote currencies: XLTCUSDT.\n" in captured.out) # active markets, base=LTC, quote=NONEXISTENT args = [ "list-markets", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', "--base", "LTC", "--quote", "NONEXISTENT", "--print-list", ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 0 active markets with LTC as base currency and " + assert ("Exchange Binance has 0 active markets with LTC as base currency and " "with NONEXISTENT as quote currency.\n" in captured.out) # Test tabular output args = [ "list-markets", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 12 active markets:\n" + assert ("Exchange Binance has 12 active markets:\n" in captured.out) # Test tabular output, no markets found args = [ "list-markets", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', "--base", "LTC", "--quote", "NONEXISTENT", ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 0 active markets with LTC as base currency and " + assert ("Exchange Binance has 0 active markets with LTC as base currency and " "with NONEXISTENT as quote currency.\n" in captured.out) # Test --print-json args = [ "list-markets", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', "--print-json" ] start_list_markets(get_args(args), False) @@ -435,7 +435,7 @@ def test_list_markets(mocker, markets_static, capsys): # Test --print-csv args = [ "list-markets", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', "--print-csv" ] start_list_markets(get_args(args), False) @@ -447,7 +447,7 @@ def test_list_markets(mocker, markets_static, capsys): # Test --one-column args = [ "list-markets", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', "--one-column" ] start_list_markets(get_args(args), False) @@ -459,7 +459,7 @@ def test_list_markets(mocker, markets_static, capsys): # Test --one-column args = [ "list-markets", - '--config', 'config_examples/config_bittrex.example.json', + '--config', 'tests/testdata/testconfigs/main_test_config.json', "--one-column" ] with pytest.raises(OperationalException, match=r"Cannot get markets.*"): @@ -772,7 +772,7 @@ def test_download_data_all_pairs(mocker, markets): pargs = get_args(args) pargs['config'] = None start_download_data(pargs) - expected = set(['ETH/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT']) + expected = set(['BTC/USDT', 'ETH/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT']) assert set(dl_mock.call_args_list[0][1]['pairs']) == expected assert dl_mock.call_count == 1 @@ -788,7 +788,7 @@ def test_download_data_all_pairs(mocker, markets): pargs = get_args(args) pargs['config'] = None start_download_data(pargs) - expected = set(['ETH/USDT', 'LTC/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT']) + expected = set(['BTC/USDT', 'ETH/USDT', 'LTC/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT']) assert set(dl_mock.call_args_list[0][1]['pairs']) == expected @@ -971,7 +971,7 @@ def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): patched_configuration_load_config_file(mocker, default_conf) args = [ 'test-pairlist', - '-c', 'config_examples/config_bittrex.example.json' + '-c', 'tests/testdata/testconfigs/main_test_config.json' ] start_test_pairlist(get_args(args)) @@ -985,7 +985,7 @@ def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): args = [ 'test-pairlist', - '-c', 'config_examples/config_bittrex.example.json', + '-c', 'tests/testdata/testconfigs/main_test_config.json', '--one-column', ] start_test_pairlist(get_args(args)) @@ -994,7 +994,7 @@ def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): args = [ 'test-pairlist', - '-c', 'config_examples/config_bittrex.example.json', + '-c', 'tests/testdata/testconfigs/main_test_config.json', '--print-json', ] start_test_pairlist(get_args(args)) @@ -1445,12 +1445,13 @@ def test_start_list_data(testdatadir, capsys): start_list_data(pargs) captured = capsys.readouterr() assert "Found 2 pair / timeframe combinations." in captured.out - assert ("\n| Pair | Timeframe | Type | From | To |\n" - in captured.out) + assert ( + "\n| Pair | Timeframe | Type " + "| From | To | Candles |\n") in captured.out assert "UNITTEST/BTC" not in captured.out assert ( - "\n| XRP/ETH | 1m | spot | 2019-10-11 00:00:00 | 2019-10-13 11:19:00 |\n" - in captured.out) + "\n| XRP/ETH | 1m | spot | " + "2019-10-11 00:00:00 | 2019-10-13 11:19:00 | 2469 |\n") in captured.out @pytest.mark.usefixtures("init_persistence") @@ -1508,7 +1509,7 @@ def test_backtesting_show(mocker, testdatadir, capsys): pargs['config'] = None start_backtesting_show(pargs) assert sbr.call_count == 1 - out, err = capsys.readouterr() + out, _err = capsys.readouterr() assert "Pairs for Strategy" in out diff --git a/tests/conftest.py b/tests/conftest.py index 2d7a805b1..0cc3a8ea0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,7 @@ import json import logging import re from copy import deepcopy -from datetime import timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Optional from unittest.mock import MagicMock, Mock, PropertyMock @@ -11,19 +11,18 @@ from unittest.mock import MagicMock, Mock, PropertyMock import numpy as np import pandas as pd import pytest +from xdist.scheduler.loadscope import LoadScopeScheduling from freqtrade import constants from freqtrade.commands import Arguments from freqtrade.data.converter import ohlcv_to_dataframe, trades_list_to_df from freqtrade.edge import PairInfo from freqtrade.enums import CandleType, MarginMode, RunMode, SignalDirection, TradingMode -from freqtrade.exchange import Exchange -from freqtrade.exchange.exchange import timeframe_to_minutes +from freqtrade.exchange import Exchange, timeframe_to_minutes, timeframe_to_seconds from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import LocalTrade, Order, Trade, init_db from freqtrade.resolvers import ExchangeResolver -from freqtrade.util import dt_ts -from freqtrade.util.datetime_helpers import dt_now +from freqtrade.util import dt_now, dt_ts from freqtrade.worker import Worker from tests.conftest_trades import (leverage_trade, mock_trade_1, mock_trade_2, mock_trade_3, mock_trade_4, mock_trade_5, mock_trade_6, short_trade) @@ -56,6 +55,27 @@ def pytest_configure(config): setattr(config.option, 'markexpr', 'not longrun') +class FixtureScheduler(LoadScopeScheduling): + # Based on the suggestion in + # https://github.com/pytest-dev/pytest-xdist/issues/18 + + def _split_scope(self, nodeid): + if 'exchange_online' in nodeid: + try: + # Extract exchange ID from nodeid + exchange_id = nodeid.split('[')[1].split('-')[0].rstrip(']') + return exchange_id + except Exception as e: + print(e) + pass + + return nodeid + + +def pytest_xdist_make_scheduler(config, log): + return FixtureScheduler(config, log) + + def log_has(line, logs): """Check if line is found on some caplog's message.""" return any(line == message for message in logs.messages) @@ -85,17 +105,62 @@ def get_args(args): return Arguments(args).get_parsed_arg() +def generate_trades_history(n_rows, start_date: Optional[datetime] = None, days=5): + np.random.seed(42) + if not start_date: + start_date = datetime(2020, 1, 1, tzinfo=timezone.utc) + + # Generate random data + end_date = start_date + timedelta(days=days) + _start_timestamp = start_date.timestamp() + _end_timestamp = pd.to_datetime(end_date).timestamp() + + random_timestamps_in_seconds = np.random.uniform(_start_timestamp, _end_timestamp, n_rows) + timestamp = pd.to_datetime(random_timestamps_in_seconds, unit='s') + + id = [ + f'a{np.random.randint(1e6, 1e7 - 1)}cd{np.random.randint(100, 999)}' + for _ in range(n_rows) + ] + + side = np.random.choice(['buy', 'sell'], n_rows) + + # Initial price and subsequent changes + initial_price = 0.019626 + price_changes = np.random.normal(0, initial_price * 0.05, n_rows) + price = np.cumsum(np.concatenate(([initial_price], price_changes)))[:n_rows] + + amount = np.random.uniform(0.011, 20, n_rows) + cost = price * amount + + # Create DataFrame + df = pd.DataFrame({'timestamp': timestamp, 'id': id, 'type': None, 'side': side, + 'price': price, 'amount': amount, 'cost': cost}) + df['date'] = pd.to_datetime(df['timestamp'], unit='ms', utc=True) + df = df.sort_values('timestamp').reset_index(drop=True) + assert list(df.columns) == constants.DEFAULT_TRADES_COLUMNS + ['date'] + return df + + def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05'): np.random.seed(42) base = np.random.normal(20, 2, size=size) - if timeframe == '1M': + if timeframe == '1y': + date = pd.date_range(start, periods=size, freq='1YS', tz='UTC') + elif timeframe == '1M': date = pd.date_range(start, periods=size, freq='1MS', tz='UTC') - elif timeframe == '1w': + elif timeframe == '3M': + date = pd.date_range(start, periods=size, freq='3MS', tz='UTC') + elif timeframe == '1w' or timeframe == '7d': date = pd.date_range(start, periods=size, freq='1W-MON', tz='UTC') else: tf_mins = timeframe_to_minutes(timeframe) - date = pd.date_range(start, periods=size, freq=f'{tf_mins}min', tz='UTC') + if tf_mins >= 1: + date = pd.date_range(start, periods=size, freq=f'{tf_mins}min', tz='UTC') + else: + tf_secs = timeframe_to_seconds(timeframe) + date = pd.date_range(start, periods=size, freq=f'{tf_secs}s', tz='UTC') df = pd.DataFrame({ 'date': date, 'open': base, @@ -531,6 +596,7 @@ def get_default_conf(testdatadir): "internals": {}, "export": "none", "dataformat_ohlcv": "feather", + "runmode": "dry_run", "candle_type_def": CandleType.SPOT, } return configuration @@ -939,6 +1005,58 @@ def get_markets(): 'maintenance_rate': '0.005', }, }, + 'BTC/USDT': { + 'id': 'USDT-BTC', + 'symbol': 'BTC/USDT', + 'base': 'BTC', + 'quote': 'USDT', + 'settle': None, + 'baseId': 'BTC', + 'quoteId': 'USDT', + 'settleId': None, + 'type': 'spot', + 'spot': True, + 'margin': True, + 'swap': False, + 'future': False, + 'option': False, + 'active': True, + 'contract': None, + 'linear': None, + 'inverse': None, + 'taker': 0.0006, + 'maker': 0.0002, + 'contractSize': None, + 'expiry': None, + 'expiryDateTime': None, + 'strike': None, + 'optionType': None, + 'precision': { + 'amount': 4, + 'price': 4, + }, + 'limits': { + 'leverage': { + 'min': 1, + 'max': 100, + }, + 'amount': { + 'min': 0.000221, + 'max': None, + }, + 'price': { + 'min': 1e-02, + 'max': None, + }, + 'cost': { + 'min': None, + 'max': None, + }, + }, + 'info': { + 'maintenance_rate': '0.005', + }, + }, 'LTC/USDT': { 'id': 'USDT-LTC', 'symbol': 'LTC/USDT', @@ -2363,14 +2481,7 @@ def trades_history_df(trades_history): @pytest.fixture(scope="function") def fetch_trades_result(): - return [{'info': {'a': 126181329, - 'p': '0.01962700', - 'q': '0.04000000', - 'f': 138604155, - 'l': 138604155, - 'T': 1565798399463, - 'm': False, - 'M': True}, + return [{'info': ['0.01962700', '0.04000000', '1565798399.4631551', 'b', 'm', '', '126181329'], 'timestamp': 1565798399463, 'datetime': '2019-08-14T15:59:59.463Z', 'symbol': 'ETH/BTC', @@ -2383,14 +2494,7 @@ def fetch_trades_result(): 'amount': 0.04, 'cost': 0.00078508, 'fee': None}, - {'info': {'a': 126181330, - 'p': '0.01962700', - 'q': '0.24400000', - 'f': 138604156, - 'l': 138604156, - 'T': 1565798399629, - 'm': False, - 'M': True}, + {'info': ['0.01962700', '0.24400000', '1565798399.6291551', 'b', 'm', '', '126181330'], 'timestamp': 1565798399629, 'datetime': '2019-08-14T15:59:59.629Z', 'symbol': 'ETH/BTC', @@ -2403,14 +2507,7 @@ def fetch_trades_result(): 'amount': 0.244, 'cost': 0.004788987999999999, 'fee': None}, - {'info': {'a': 126181331, - 'p': '0.01962600', - 'q': '0.01100000', - 'f': 138604157, - 'l': 138604157, - 'T': 1565798399752, - 'm': True, - 'M': True}, + {'info': ['0.01962600', '0.01100000', '1565798399.7521551', 's', 'm', '', '126181331'], 'timestamp': 1565798399752, 'datetime': '2019-08-14T15:59:59.752Z', 'symbol': 'ETH/BTC', @@ -2423,14 +2520,7 @@ def fetch_trades_result(): 'amount': 0.011, 'cost': 0.00021588599999999999, 'fee': None}, - {'info': {'a': 126181332, - 'p': '0.01962600', - 'q': '0.01100000', - 'f': 138604158, - 'l': 138604158, - 'T': 1565798399862, - 'm': True, - 'M': True}, + {'info': ['0.01962600', '0.01100000', '1565798399.8621551', 's', 'm', '', '126181332'], 'timestamp': 1565798399862, 'datetime': '2019-08-14T15:59:59.862Z', 'symbol': 'ETH/BTC', @@ -2443,14 +2533,8 @@ def fetch_trades_result(): 'amount': 0.011, 'cost': 0.00021588599999999999, 'fee': None}, - {'info': {'a': 126181333, - 'p': '0.01952600', - 'q': '0.01200000', - 'f': 138604158, - 'l': 138604158, - 'T': 1565798399872, - 'm': True, - 'M': True}, + {'info': ['0.01952600', '0.01200000', '1565798399.8721551', 's', 'm', '', '126181333', + 1565798399872512133], 'timestamp': 1565798399872, 'datetime': '2019-08-14T15:59:59.872Z', 'symbol': 'ETH/BTC', diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 4186708ab..08fc785aa 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -17,7 +17,8 @@ from freqtrade.data.history import (get_timerange, load_data, load_pair_history, validate_backtest_data) from freqtrade.data.history.idatahandler import IDataHandler from freqtrade.enums import CandleType -from tests.conftest import generate_test_data, log_has, log_has_re +from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds +from tests.conftest import generate_test_data, generate_trades_history, log_has, log_has_re from tests.data.test_history import _clean_test_file @@ -51,6 +52,49 @@ def test_trades_to_ohlcv(trades_history_df, caplog): assert 'close' in df.columns assert df.iloc[0, :]['high'] == 0.019627 assert df.iloc[0, :]['low'] == 0.019626 + assert df.iloc[0, :]['date'] == pd.Timestamp('2019-08-14 15:59:00+0000') + + df_1h = trades_to_ohlcv(trades_history_df, '1h') + assert len(df_1h) == 1 + assert df_1h.iloc[0, :]['high'] == 0.019627 + assert df_1h.iloc[0, :]['low'] == 0.019626 + assert df_1h.iloc[0, :]['date'] == pd.Timestamp('2019-08-14 15:00:00+0000') + + df_1s = trades_to_ohlcv(trades_history_df, '1s') + assert len(df_1s) == 2 + assert df_1s.iloc[0, :]['high'] == 0.019627 + assert df_1s.iloc[0, :]['low'] == 0.019627 + assert df_1s.iloc[0, :]['date'] == pd.Timestamp('2019-08-14 15:59:49+0000') + assert df_1s.iloc[-1, :]['date'] == pd.Timestamp('2019-08-14 15:59:59+0000') + + +@pytest.mark.parametrize('timeframe,rows,days,candles,start,end,weekday', [ + ('1s', 20_000, 5, 19522, '2020-01-01 00:00:05', '2020-01-05 23:59:27', None), + ('1m', 20_000, 5, 6745, '2020-01-01 00:00:00', '2020-01-05 23:59:00', None), + ('5m', 20_000, 5, 1440, '2020-01-01 00:00:00', '2020-01-05 23:55:00', None), + ('15m', 20_000, 5, 480, '2020-01-01 00:00:00', '2020-01-05 23:45:00', None), + ('1h', 20_000, 5, 120, '2020-01-01 00:00:00', '2020-01-05 23:00:00', None), + ('2h', 20_000, 5, 60, '2020-01-01 00:00:00', '2020-01-05 22:00:00', None), + ('4h', 20_000, 5, 30, '2020-01-01 00:00:00', '2020-01-05 20:00:00', None), + ('8h', 20_000, 5, 15, '2020-01-01 00:00:00', '2020-01-05 16:00:00', None), + ('12h', 20_000, 5, 10, '2020-01-01 00:00:00', '2020-01-05 12:00:00', None), + ('1d', 20_000, 5, 5, '2020-01-01 00:00:00', '2020-01-05 00:00:00', 'Sunday'), + ('7d', 20_000, 37, 6, '2020-01-06 00:00:00', '2020-02-10 00:00:00', 'Monday'), + ('1w', 20_000, 37, 6, '2020-01-06 00:00:00', '2020-02-10 00:00:00', 'Monday'), + ('1M', 20_000, 74, 3, '2020-01-01 00:00:00', '2020-03-01 00:00:00', None), + ('3M', 20_000, 100, 2, '2020-01-01 00:00:00', '2020-04-01 00:00:00', None), + ('1y', 20_000, 1000, 3, '2020-01-01 00:00:00', '2022-01-01 00:00:00', None), +]) +def test_trades_to_ohlcv_multi(timeframe, rows, days, candles, start, end, weekday): + trades_history = generate_trades_history(n_rows=rows, days=days) + df = trades_to_ohlcv(trades_history, timeframe) + assert not df.empty + assert len(df) == candles + assert df.iloc[0, :]['date'] == pd.Timestamp(f'{start}+0000') + assert df.iloc[-1, :]['date'] == pd.Timestamp(f'{end}+0000') + if weekday: + # Weekday is only relevant for daily and weekly candles. + assert df.iloc[-1, :]['date'].day_name() == weekday def test_ohlcv_fill_up_missing_data(testdatadir, caplog): @@ -132,6 +176,45 @@ def test_ohlcv_fill_up_missing_data2(caplog): f"{len(data)} - after: {len(data2)}.*", caplog) +@pytest.mark.parametrize('timeframe', [ + '1s', '1m', '5m', '15m', '1h', '2h', '4h', '8h', '12h', '1d', '7d', '1w', '1M', '3M', '1y' +]) +def test_ohlcv_to_dataframe_multi(timeframe): + data = generate_test_data(timeframe, 180) + assert len(data) == 180 + df = ohlcv_to_dataframe(data, timeframe, 'UNITTEST/USDT') + assert len(df) == len(data) - 1 + df1 = ohlcv_to_dataframe(data, timeframe, 'UNITTEST/USDT', drop_incomplete=False) + assert len(df1) == len(data) + assert data.equals(df1) + + data1 = data.copy() + if timeframe in ('1M', '3M', '1y'): + data1.loc[:, 'date'] = data1.loc[:, 'date'] + pd.to_timedelta('1w') + else: + # Shift by half a timeframe + data1.loc[:, 'date'] = data1.loc[:, 'date'] + (pd.to_timedelta(timeframe) / 2) + df2 = ohlcv_to_dataframe(data1, timeframe, 'UNITTEST/USDT') + + assert len(df2) == len(data) - 1 + tfs = timeframe_to_seconds(timeframe) + tfm = timeframe_to_minutes(timeframe) + if 1 <= tfm < 10000: + # minute based resampling does not work on timeframes >= 1 week + ohlcv_dict = { + 'open': 'first', + 'high': 'max', + 'low': 'min', + 'close': 'last', + 'volume': 'sum' + } + dfs = data1.resample(f"{tfs}s", on='date').agg(ohlcv_dict).reset_index(drop=False) + dfm = data1.resample(f"{tfm}min", on='date').agg(ohlcv_dict).reset_index(drop=False) + + assert dfs.equals(dfm) + assert dfs.equals(df1) + + def test_ohlcv_to_dataframe_1M(): # Monthly ticks from 2019-09-01 to 2023-07-01 diff --git a/tests/data/test_datahandler.py b/tests/data/test_datahandler.py index 988e7ea55..a0a37c393 100644 --- a/tests/data/test_datahandler.py +++ b/tests/data/test_datahandler.py @@ -148,19 +148,25 @@ def test_jsondatahandler_ohlcv_load(testdatadir, caplog): def test_datahandler_ohlcv_data_min_max(testdatadir): dh = JsonDataHandler(testdatadir) min_max = dh.ohlcv_data_min_max('UNITTEST/BTC', '5m', 'spot') - assert len(min_max) == 2 + assert len(min_max) == 3 # Empty pair min_max = dh.ohlcv_data_min_max('UNITTEST/BTC', '8m', 'spot') - assert len(min_max) == 2 + assert len(min_max) == 3 assert min_max[0] == datetime.fromtimestamp(0, tz=timezone.utc) assert min_max[0] == min_max[1] # Empty pair2 - min_max = dh.ohlcv_data_min_max('NOPAIR/XXX', '4m', 'spot') - assert len(min_max) == 2 + min_max = dh.ohlcv_data_min_max('NOPAIR/XXX', '41m', 'spot') + assert len(min_max) == 3 assert min_max[0] == datetime.fromtimestamp(0, tz=timezone.utc) assert min_max[0] == min_max[1] + # Existing pair ... + min_max = dh.ohlcv_data_min_max('UNITTEST/BTC', '1m', 'spot') + assert len(min_max) == 3 + assert min_max[0] == datetime(2017, 11, 4, 23, 2, tzinfo=timezone.utc) + assert min_max[1] == datetime(2017, 11, 14, 22, 59, tzinfo=timezone.utc) + def test_datahandler__check_empty_df(testdatadir, caplog): dh = JsonDataHandler(testdatadir) @@ -513,11 +519,11 @@ def test_gethandlerclass(): def test_get_datahandler(testdatadir): dh = get_datahandler(testdatadir, 'json') - assert type(dh) == JsonDataHandler + assert isinstance(dh, JsonDataHandler) dh = get_datahandler(testdatadir, 'jsongz') - assert type(dh) == JsonGzDataHandler + assert isinstance(dh, JsonGzDataHandler) dh1 = get_datahandler(testdatadir, 'jsongz', dh) assert id(dh1) == id(dh) dh = get_datahandler(testdatadir, 'hdf5') - assert type(dh) == HDF5DataHandler + assert isinstance(dh, HDF5DataHandler) diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index a61dd9679..cc875b970 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -194,7 +194,7 @@ def test_get_producer_df(default_conf): assert la == empty_la # non existent timeframe, empty dataframe - datframe, la = dataprovider.get_producer_df(pair, timeframe='1h') + _dataframe, la = dataprovider.get_producer_df(pair, timeframe='1h') assert dataframe.empty assert la == empty_la @@ -508,16 +508,13 @@ def test_dp_get_required_startup(default_conf_usdt): dp = DataProvider(default_conf_usdt, None) # No FreqAI config - assert dp.get_required_startup('5m', False) == 0 - assert dp.get_required_startup('1h', False) == 0 - assert dp.get_required_startup('1d', False) == 0 - assert dp.get_required_startup('1d', True) == 0 + assert dp.get_required_startup('5m') == 0 + assert dp.get_required_startup('1h') == 0 assert dp.get_required_startup('1d') == 0 dp._config['startup_candle_count'] = 20 - assert dp.get_required_startup('5m', False) == 20 - assert dp.get_required_startup('5m', True) == 20 - assert dp.get_required_startup('1h', False) == 20 + assert dp.get_required_startup('5m') == 20 + assert dp.get_required_startup('1h') == 20 assert dp.get_required_startup('1h') == 20 # With freqAI config @@ -532,37 +529,19 @@ def test_dp_get_required_startup(default_conf_usdt): ] } } - assert dp.get_required_startup('5m', False) == 20 - assert dp.get_required_startup('5m', True) == 5780 - - assert dp.get_required_startup('1h', False) == 20 - assert dp.get_required_startup('1h', True) == 500 - - assert dp.get_required_startup('1d', False) == 20 - assert dp.get_required_startup('1d', True) == 40 + assert dp.get_required_startup('5m') == 5780 + assert dp.get_required_startup('1h') == 500 assert dp.get_required_startup('1d') == 40 # FreqAI kindof ignores startup_candle_count if it's below indicator_periods_candles dp._config['startup_candle_count'] = 0 - assert dp.get_required_startup('5m', False) == 20 - assert dp.get_required_startup('5m', True) == 5780 - - assert dp.get_required_startup('1h', False) == 20 - assert dp.get_required_startup('1h', True) == 500 - - assert dp.get_required_startup('1d', False) == 20 - assert dp.get_required_startup('1d', True) == 40 + assert dp.get_required_startup('5m') == 5780 + assert dp.get_required_startup('1h') == 500 assert dp.get_required_startup('1d') == 40 dp._config['freqai']['feature_parameters']['indicator_periods_candles'][1] = 50 - assert dp.get_required_startup('5m', False) == 50 - assert dp.get_required_startup('5m', True) == 5810 - - assert dp.get_required_startup('1h', False) == 50 - assert dp.get_required_startup('1h', True) == 530 - - assert dp.get_required_startup('1d', False) == 50 - assert dp.get_required_startup('1d', True) == 70 + assert dp.get_required_startup('5m') == 5810 + assert dp.get_required_startup('1h') == 530 assert dp.get_required_startup('1d') == 70 # scenario from issue https://github.com/freqtrade/freqtrade/issues/9432 @@ -577,12 +556,6 @@ def test_dp_get_required_startup(default_conf_usdt): } } dp._config['startup_candle_count'] = 40 - assert dp.get_required_startup('5m', False) == 40 - assert dp.get_required_startup('5m', True) == 51880 - - assert dp.get_required_startup('1h', False) == 40 - assert dp.get_required_startup('1h', True) == 4360 - - assert dp.get_required_startup('1d', False) == 40 - assert dp.get_required_startup('1d', True) == 220 + assert dp.get_required_startup('5m') == 51880 + assert dp.get_required_startup('1h') == 4360 assert dp.get_required_startup('1d') == 220 diff --git a/tests/data/test_download_data.py b/tests/data/test_download_data.py index 191dbb7d3..97640d01c 100644 --- a/tests/data/test_download_data.py +++ b/tests/data/test_download_data.py @@ -38,7 +38,7 @@ def test_download_data_main_all_pairs(mocker, markets): "timeframes": ["5m", "1h"] }) download_data_main(config) - expected = set(['ETH/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT']) + expected = set(['BTC/USDT', 'ETH/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT']) assert set(dl_mock.call_args_list[0][1]['pairs']) == expected assert dl_mock.call_count == 1 @@ -50,7 +50,7 @@ def test_download_data_main_all_pairs(mocker, markets): "include_inactive": True }) download_data_main(config) - expected = set(['ETH/USDT', 'LTC/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT']) + expected = set(['BTC/USDT', 'ETH/USDT', 'LTC/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT']) assert set(dl_mock.call_args_list[0][1]['pairs']) == expected diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 100916387..a48d34aee 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -508,8 +508,9 @@ def test_refresh_backtest_ohlcv_data( mocker.patch.object(Path, "exists", MagicMock(return_value=True)) mocker.patch.object(Path, "unlink", MagicMock()) + default_conf['trading_mode'] = trademode - ex = get_patched_exchange(mocker, default_conf) + ex = get_patched_exchange(mocker, default_conf, id='bybit') timerange = TimeRange.parse_timerange("20190101-20190102") refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"], timeframes=["1m", "5m"], datadir=testdatadir, @@ -521,6 +522,9 @@ def test_refresh_backtest_ohlcv_data( assert dl_mock.call_args[1]['timerange'].starttype == 'date' assert log_has_re(r"Downloading pair ETH/BTC, .* interval 1m\.", caplog) + if trademode == 'futures': + assert log_has_re(r"Downloading pair ETH/BTC, funding_rate, interval 8h\.", caplog) + assert log_has_re(r"Downloading pair ETH/BTC, mark, interval 4h\.", caplog) def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 1d26b706f..f686959fc 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -13,7 +13,7 @@ from freqtrade.enums import CandleType, MarginMode, TradingMode from freqtrade.exceptions import (DDosProtection, DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, OperationalException, PricingError, TemporaryError) -from freqtrade.exchange import (Binance, Bittrex, Exchange, Kraken, market_is_active, +from freqtrade.exchange import (Binance, Bybit, Exchange, Kraken, market_is_active, timeframe_to_prev_date) from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT, calculate_backoff, remove_exchange_credentials) @@ -55,7 +55,7 @@ get_entry_rate_data = [ ('bid', 6, 5, None, 0, 5), # last not available - uses bid ] -get_sell_rate_data = [ +get_exit_rate_data = [ ('bid', 12.0, 11.0, 11.5, 0.0, 11.0), # full bid side ('bid', 12.0, 11.0, 11.5, 1.0, 11.5), # full last side ('bid', 12.0, 11.0, 11.5, 0.5, 11.25), # between bid and lat @@ -228,10 +228,10 @@ def test_exchange_resolver(default_conf, mocker, caplog): assert log_has_re(r"No .* specific subclass found. Using the generic class instead.", caplog) caplog.clear() - default_conf['exchange']['name'] = 'Bittrex' + default_conf['exchange']['name'] = 'Bybit' exchange = ExchangeResolver.load_exchange(default_conf) assert isinstance(exchange, Exchange) - assert isinstance(exchange, Bittrex) + assert isinstance(exchange, Bybit) assert not log_has_re(r"No .* specific subclass found. Using the generic class instead.", caplog) caplog.clear() @@ -263,8 +263,8 @@ def test_exchange_resolver(default_conf, mocker, caplog): def test_validate_order_time_in_force(default_conf, mocker, caplog): caplog.set_level(logging.INFO) - # explicitly test bittrex, exchanges implementing other policies need separate tests - ex = get_patched_exchange(mocker, default_conf, id="bittrex") + # explicitly test bybit, exchanges implementing other policies need separate tests + ex = get_patched_exchange(mocker, default_conf, id="bybit") tif = { "buy": "gtc", "sell": "gtc", @@ -273,11 +273,14 @@ def test_validate_order_time_in_force(default_conf, mocker, caplog): ex.validate_order_time_in_force(tif) tif2 = { "buy": "fok", - "sell": "ioc", + "sell": "ioc22", } with pytest.raises(OperationalException, match=r"Time in force.*not supported for .*"): ex.validate_order_time_in_force(tif2) - + tif2 = { + "buy": "fok", + "sell": "ioc", + } # Patch to see if this will pass if the values are in the ft dict ex._ft_has.update({"order_time_in_force": ["GTC", "FOK", "IOC"]}) ex.validate_order_time_in_force(tif2) @@ -915,7 +918,6 @@ def test_validate_ordertypes(default_conf, mocker): mocker.patch(f'{EXMS}.validate_timeframes') mocker.patch(f'{EXMS}.validate_stakecurrency') mocker.patch(f'{EXMS}.validate_pricing') - mocker.patch(f'{EXMS}.name', 'Bittrex') default_conf['order_types'] = { 'entry': 'limit', @@ -2510,8 +2512,10 @@ def test_fetch_l2_order_book_exception(default_conf, mocker, exchange_name): @pytest.mark.parametrize("side,ask,bid,last,last_ab,expected", get_entry_rate_data) def test_get_entry_rate(mocker, default_conf, caplog, side, ask, bid, - last, last_ab, expected) -> None: + last, last_ab, expected, time_machine) -> None: caplog.set_level(logging.DEBUG) + start_dt = datetime(2023, 12, 1, 0, 10, 0, tzinfo=timezone.utc) + time_machine.move_to(start_dt, tick=False) if last_ab is None: del default_conf['entry_pricing']['price_last_balance'] else: @@ -2519,39 +2523,65 @@ def test_get_entry_rate(mocker, default_conf, caplog, side, ask, bid, default_conf['entry_pricing']['price_side'] = side exchange = get_patched_exchange(mocker, default_conf) mocker.patch(f'{EXMS}.fetch_ticker', return_value={'ask': ask, 'last': last, 'bid': bid}) + log_msg = "Using cached entry rate for ETH/BTC." assert exchange.get_rate('ETH/BTC', side="entry", is_short=False, refresh=True) == expected - assert not log_has("Using cached entry rate for ETH/BTC.", caplog) + assert not log_has(log_msg, caplog) + time_machine.move_to(start_dt + timedelta(minutes=4), tick=False) + # Running a 2nd time without Refresh! + caplog.clear() assert exchange.get_rate('ETH/BTC', side="entry", is_short=False, refresh=False) == expected - assert log_has("Using cached entry rate for ETH/BTC.", caplog) + assert log_has(log_msg, caplog) + + time_machine.move_to(start_dt + timedelta(minutes=6), tick=False) + # Running a 2nd time - forces refresh due to ttl timeout + caplog.clear() + assert exchange.get_rate('ETH/BTC', side="entry", is_short=False, refresh=False) == expected + assert not log_has(log_msg, caplog) + # Running a 2nd time with Refresh on! caplog.clear() assert exchange.get_rate('ETH/BTC', side="entry", is_short=False, refresh=True) == expected - assert not log_has("Using cached entry rate for ETH/BTC.", caplog) + assert not log_has(log_msg, caplog) -@pytest.mark.parametrize('side,ask,bid,last,last_ab,expected', get_sell_rate_data) +@pytest.mark.parametrize('side,ask,bid,last,last_ab,expected', get_exit_rate_data) def test_get_exit_rate(default_conf, mocker, caplog, side, bid, ask, - last, last_ab, expected) -> None: + last, last_ab, expected, time_machine) -> None: caplog.set_level(logging.DEBUG) + start_dt = datetime(2023, 12, 1, 0, 10, 0, tzinfo=timezone.utc) + time_machine.move_to(start_dt, tick=False) default_conf['exit_pricing']['price_side'] = side if last_ab is not None: default_conf['exit_pricing']['price_last_balance'] = last_ab mocker.patch(f'{EXMS}.fetch_ticker', return_value={'ask': ask, 'bid': bid, 'last': last}) pair = "ETH/BTC" + log_msg = "Using cached exit rate for ETH/BTC." # Test regular mode exchange = get_patched_exchange(mocker, default_conf) rate = exchange.get_rate(pair, side="exit", is_short=False, refresh=True) - assert not log_has("Using cached exit rate for ETH/BTC.", caplog) + assert not log_has(log_msg, caplog) assert isinstance(rate, float) assert rate == expected # Use caching - rate = exchange.get_rate(pair, side="exit", is_short=False, refresh=False) - assert rate == expected - assert log_has("Using cached exit rate for ETH/BTC.", caplog) + caplog.clear() + assert exchange.get_rate(pair, side="exit", is_short=False, refresh=False) == expected + assert log_has(log_msg, caplog) + + time_machine.move_to(start_dt + timedelta(minutes=4), tick=False) + # Caching still active - TTL didn't expire + caplog.clear() + assert exchange.get_rate(pair, side="exit", is_short=False, refresh=False) == expected + assert log_has(log_msg, caplog) + + time_machine.move_to(start_dt + timedelta(minutes=6), tick=False) + # Caching expired - refresh forced + caplog.clear() + assert exchange.get_rate(pair, side="exit", is_short=False, refresh=False) == expected + assert not log_has(log_msg, caplog) @pytest.mark.parametrize("entry,is_short,side,ask,bid,last,last_ab,expected", [ @@ -2647,9 +2677,9 @@ def test_get_exit_rate_exception(default_conf, mocker, is_short): @pytest.mark.parametrize("side,ask,bid,last,last_ab,expected", get_entry_rate_data) @pytest.mark.parametrize("side2", ['bid', 'ask']) @pytest.mark.parametrize("use_order_book", [True, False]) -def test_get_rates_testing_buy(mocker, default_conf, caplog, side, ask, bid, - last, last_ab, expected, - side2, use_order_book, order_book_l2) -> None: +def test_get_rates_testing_entry(mocker, default_conf, caplog, side, ask, bid, + last, last_ab, expected, + side2, use_order_book, order_book_l2) -> None: caplog.set_level(logging.DEBUG) if last_ab is None: del default_conf['entry_pricing']['price_last_balance'] @@ -2683,10 +2713,10 @@ def test_get_rates_testing_buy(mocker, default_conf, caplog, side, ask, bid, assert api_mock.fetch_ticker.call_count == 1 -@pytest.mark.parametrize('side,ask,bid,last,last_ab,expected', get_sell_rate_data) +@pytest.mark.parametrize('side,ask,bid,last,last_ab,expected', get_exit_rate_data) @pytest.mark.parametrize("side2", ['bid', 'ask']) @pytest.mark.parametrize("use_order_book", [True, False]) -def test_get_rates_testing_sell(default_conf, mocker, caplog, side, bid, ask, +def test_get_rates_testing_exit(default_conf, mocker, caplog, side, bid, ask, last, last_ab, expected, side2, use_order_book, order_book_l2) -> None: caplog.set_level(logging.DEBUG) @@ -2766,7 +2796,6 @@ async def test___async_get_candle_history_sort(default_conf, mocker, exchange_na assert res_ohlcv[9][4] == 0.07668 assert res_ohlcv[9][5] == 16.65244264 - # Bittrex use-case (real data from Bittrex) # This OHLCV data is ordered ASC (oldest first, newest last) ohlcv = [ [1527827700000, 0.07659999, 0.0766, 0.07627, 0.07657998, 1.85216924], @@ -2815,10 +2844,17 @@ async def test__async_fetch_trades(default_conf, mocker, caplog, exchange_name, exchange._api_async.fetch_trades = get_mock_coro(fetch_trades_result) pair = 'ETH/BTC' - res = await exchange._async_fetch_trades(pair, since=None, params=None) + res, pagid = await exchange._async_fetch_trades(pair, since=None, params=None) assert isinstance(res, list) assert isinstance(res[0], list) assert isinstance(res[1], list) + if exchange._trades_pagination == 'id': + if exchange_name == 'kraken': + assert pagid == 1565798399872512133 + else: + assert pagid == '126181333' + else: + assert pagid == 1565798399872 assert exchange._api_async.fetch_trades.call_count == 1 assert exchange._api_async.fetch_trades.call_args[0][0] == pair @@ -2827,11 +2863,20 @@ async def test__async_fetch_trades(default_conf, mocker, caplog, exchange_name, assert log_has_re(f"Fetching trades for pair {pair}, since .*", caplog) caplog.clear() exchange._api_async.fetch_trades.reset_mock() - res = await exchange._async_fetch_trades(pair, since=None, params={'from': '123'}) + res, pagid = await exchange._async_fetch_trades(pair, since=None, params={'from': '123'}) assert exchange._api_async.fetch_trades.call_count == 1 assert exchange._api_async.fetch_trades.call_args[0][0] == pair assert exchange._api_async.fetch_trades.call_args[1]['limit'] == 1000 assert exchange._api_async.fetch_trades.call_args[1]['params'] == {'from': '123'} + + if exchange._trades_pagination == 'id': + if exchange_name == 'kraken': + assert pagid == 1565798399872512133 + else: + assert pagid == '126181333' + else: + assert pagid == 1565798399872 + assert log_has_re(f"Fetching trades for pair {pair}, params: .*", caplog) exchange.close() @@ -2886,8 +2931,9 @@ async def test__async_fetch_trades_contract_size(default_conf, mocker, caplog, e ) pair = 'ETH/USDT:USDT' - res = await exchange._async_fetch_trades(pair, since=None, params=None) + res, pagid = await exchange._async_fetch_trades(pair, since=None, params=None) assert res[0][5] == 300 + assert pagid is not None exchange.close() @@ -2897,13 +2943,17 @@ async def test__async_get_trade_history_id(default_conf, mocker, exchange_name, fetch_trades_result): exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + if exchange._trades_pagination != 'id': + exchange.close() + pytest.skip("Exchange does not support pagination by trade id") pagination_arg = exchange._trades_pagination_arg async def mock_get_trade_hist(pair, *args, **kwargs): if 'since' in kwargs: # Return first 3 return fetch_trades_result[:-2] - elif kwargs.get('params', {}).get(pagination_arg) == fetch_trades_result[-3]['id']: + elif kwargs.get('params', {}).get(pagination_arg) in ( + fetch_trades_result[-3]['id'], 1565798399752): # Return 2 return fetch_trades_result[-3:-1] else: @@ -2919,7 +2969,8 @@ async def test__async_get_trade_history_id(default_conf, mocker, exchange_name, assert isinstance(ret, tuple) assert ret[0] == pair assert isinstance(ret[1], list) - assert len(ret[1]) == len(fetch_trades_result) + if exchange_name != 'kraken': + assert len(ret[1]) == len(fetch_trades_result) assert exchange._api_async.fetch_trades.call_count == 3 fetch_trades_cal = exchange._api_async.fetch_trades.call_args_list # first call (using since, not fromId) @@ -2932,6 +2983,21 @@ async def test__async_get_trade_history_id(default_conf, mocker, exchange_name, assert exchange._ft_has['trades_pagination_arg'] in fetch_trades_cal[1][1]['params'] +@pytest.mark.parametrize('trade_id, expected', [ + ('1234', True), + ('170544369512007228', True), + ('1705443695120072285', True), + ('170544369512007228555', True), +]) +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test__valid_trade_pagination_id(mocker, default_conf_usdt, exchange_name, trade_id, expected): + if exchange_name == 'kraken': + pytest.skip("Kraken has a different pagination id format, and an explicit test.") + exchange = get_patched_exchange(mocker, default_conf_usdt, id=exchange_name) + + assert exchange._valid_trade_pagination_id('XRP/USDT', trade_id) == expected + + @pytest.mark.asyncio @pytest.mark.parametrize("exchange_name", EXCHANGES) async def test__async_get_trade_history_time(default_conf, mocker, caplog, exchange_name, @@ -2947,6 +3013,9 @@ async def test__async_get_trade_history_time(default_conf, mocker, caplog, excha caplog.set_level(logging.DEBUG) exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + if exchange._trades_pagination != 'time': + exchange.close() + pytest.skip("Exchange does not support pagination by timestamp") # Monkey-patch async function exchange._api_async.fetch_trades = MagicMock(side_effect=mock_get_trade_hist) pair = 'ETH/BTC' @@ -2979,9 +3048,9 @@ async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog, async def mock_get_trade_hist(pair, *args, **kwargs): if kwargs['since'] == trades_history[0][0]: - return trades_history[:-1] + return trades_history[:-1], trades_history[:-1][-1][0] else: - return [] + return [], None caplog.set_level(logging.DEBUG) exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) @@ -3193,7 +3262,7 @@ def test_cancel_stoploss_order_with_result(default_conf, mocker, exchange_name): mocker.patch(f'{mock_prefix}.fetch_stoploss_order', side_effect=exc) co = exchange.cancel_stoploss_order_with_result(order_id='_', pair='TKN/BTC', amount=555) assert co['amount'] == 555 - assert co == {'fee': {}, 'status': 'canceled', 'amount': 555, 'info': {}} + assert co == {'id': '_', 'fee': {}, 'status': 'canceled', 'amount': 555, 'info': {}} with pytest.raises(InvalidOrderException): exc = InvalidOrderException("Did not find order") @@ -3410,7 +3479,7 @@ def test_get_fee(default_conf, mocker, exchange_name): def test_stoploss_order_unsupported_exchange(default_conf, mocker): - exchange = get_patched_exchange(mocker, default_conf, id='bittrex') + exchange = get_patched_exchange(mocker, default_conf, id='bitpanda') with pytest.raises(OperationalException, match=r"stoploss is not implemented .*"): exchange.create_stoploss( pair='ETH/BTC', @@ -3606,10 +3675,10 @@ def test_ohlcv_candle_limit(default_conf, mocker, exchange_name): timeframes = ('1m', '5m', '1h') expected = exchange._ft_has['ohlcv_candle_limit'] for timeframe in timeframes: - if 'ohlcv_candle_limit_per_timeframe' in exchange._ft_has: - expected = exchange._ft_has['ohlcv_candle_limit_per_timeframe'][timeframe] - # This should only run for bittrex - assert exchange_name == 'bittrex' + # if 'ohlcv_candle_limit_per_timeframe' in exchange._ft_has: + # expected = exchange._ft_has['ohlcv_candle_limit_per_timeframe'][timeframe] + # This should only run for bittrex + # assert exchange_name == 'bittrex' assert exchange.ohlcv_candle_limit(timeframe, CandleType.SPOT) == expected @@ -4522,10 +4591,10 @@ def test_amount_to_contract_precision( @pytest.mark.parametrize('exchange_name,open_rate,is_short,trading_mode,margin_mode', [ - # Bittrex - ('bittrex', 2.0, False, 'spot', None), - ('bittrex', 2.0, False, 'spot', 'cross'), - ('bittrex', 2.0, True, 'spot', 'isolated'), + # Bybit + ('bybit', 2.0, False, 'spot', None), + ('bybit', 2.0, False, 'spot', 'cross'), + ('bybit', 2.0, True, 'spot', 'isolated'), # Binance ('binance', 2.0, False, 'spot', None), ('binance', 2.0, False, 'spot', 'cross'), @@ -4947,7 +5016,7 @@ def test_get_max_leverage_futures(default_conf, mocker, leverage_tiers): exchange.get_max_leverage("BTC/USDT:USDT", 1000000000.01) -@pytest.mark.parametrize("exchange_name", ['bittrex', 'binance', 'kraken', 'gate', 'okx', 'bybit']) +@pytest.mark.parametrize("exchange_name", ['binance', 'kraken', 'gate', 'okx', 'bybit']) def test__get_params(mocker, default_conf, exchange_name): api_mock = MagicMock() mocker.patch(f'{EXMS}.exchange_has', return_value=True) @@ -5283,3 +5352,4 @@ def test_price_to_precision_with_default_conf(default_conf, mocker): patched_ex = get_patched_exchange(mocker, conf) prec_price = patched_ex.price_to_precision("XRP/USDT", 1.0000000101) assert prec_price == 1.00000001 + assert prec_price == 1.00000001 diff --git a/tests/exchange/test_exchange_utils.py b/tests/exchange/test_exchange_utils.py index 6a4137369..dd79bf083 100644 --- a/tests/exchange/test_exchange_utils.py +++ b/tests/exchange/test_exchange_utils.py @@ -10,7 +10,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange import (amount_to_contract_precision, amount_to_precision, date_minus_candles, price_to_precision, timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date, timeframe_to_prev_date, - timeframe_to_seconds) + timeframe_to_resample_freq, timeframe_to_seconds) from freqtrade.exchange.check_exchange import check_exchange from tests.conftest import log_has_re @@ -124,6 +124,21 @@ def test_timeframe_to_msecs(): assert timeframe_to_msecs("1d") == 86400000 +@pytest.mark.parametrize("timeframe,expected", [ + ("1s", '1s'), + ("15s", '15s'), + ("5m", '300s'), + ("10m", '600s'), + ("1h", '3600s'), + ("1d", '86400s'), + ("1w", '1W-MON'), + ("1M", '1MS'), + ("1y", '1YS'), +]) +def test_timeframe_to_resample_freq(timeframe, expected): + assert timeframe_to_resample_freq(timeframe) == expected + + def test_timeframe_to_prev_date(): # 2019-08-12 13:22:08 date = datetime.fromtimestamp(1565616128, tz=timezone.utc) diff --git a/tests/exchange/test_huobi.py b/tests/exchange/test_htx.py similarity index 92% rename from tests/exchange/test_huobi.py rename to tests/exchange/test_htx.py index b3f3c0900..ac136618f 100644 --- a/tests/exchange/test_huobi.py +++ b/tests/exchange/test_htx.py @@ -14,7 +14,7 @@ from tests.exchange.test_exchange import ccxt_exceptionhandlers (0.99, 220 * 0.99, "sell"), (0.98, 220 * 0.98, "sell"), ]) -def test_create_stoploss_order_huobi(default_conf, mocker, limitratio, expected, side): +def test_create_stoploss_order_htx(default_conf, mocker, limitratio, expected, side): api_mock = MagicMock() order_id = f'test_prod_buy_{randint(0, 10 ** 6)}' order_type = 'stop-limit' @@ -29,7 +29,7 @@ def test_create_stoploss_order_huobi(default_conf, mocker, limitratio, expected, mocker.patch(f'{EXMS}.amount_to_precision', lambda s, x, y: y) mocker.patch(f'{EXMS}.price_to_precision', lambda s, x, y, **kwargs: y) - exchange = get_patched_exchange(mocker, default_conf, api_mock, 'huobi') + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'htx') with pytest.raises(InvalidOrderException): order = exchange.create_stoploss(pair='ETH/BTC', amount=1, stop_price=190, @@ -58,7 +58,7 @@ def test_create_stoploss_order_huobi(default_conf, mocker, limitratio, expected, # test exception handling with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds("0 balance")) - exchange = get_patched_exchange(mocker, default_conf, api_mock, 'huobi') + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'htx') exchange.create_stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}, side=side, leverage=1.0) @@ -69,20 +69,20 @@ def test_create_stoploss_order_huobi(default_conf, mocker, limitratio, expected, exchange.create_stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}, side=side, leverage=1.0) - ccxt_exceptionhandlers(mocker, default_conf, api_mock, "huobi", + ccxt_exceptionhandlers(mocker, default_conf, api_mock, "htx", "create_stoploss", "create_order", retries=1, pair='ETH/BTC', amount=1, stop_price=220, order_types={}, side=side, leverage=1.0) -def test_create_stoploss_order_dry_run_huobi(default_conf, mocker): +def test_create_stoploss_order_dry_run_htx(default_conf, mocker): api_mock = MagicMock() order_type = 'stop-limit' default_conf['dry_run'] = True mocker.patch(f'{EXMS}.amount_to_precision', lambda s, x, y: y) mocker.patch(f'{EXMS}.price_to_precision', lambda s, x, y, **kwargs: y) - exchange = get_patched_exchange(mocker, default_conf, api_mock, 'huobi') + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'htx') with pytest.raises(InvalidOrderException): order = exchange.create_stoploss(pair='ETH/BTC', amount=1, stop_price=190, @@ -103,8 +103,8 @@ def test_create_stoploss_order_dry_run_huobi(default_conf, mocker): assert order['amount'] == 1 -def test_stoploss_adjust_huobi(mocker, default_conf): - exchange = get_patched_exchange(mocker, default_conf, id='huobi') +def test_stoploss_adjust_htx(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf, id='htx') order = { 'type': 'stop', 'price': 1500, diff --git a/tests/exchange/test_kraken.py b/tests/exchange/test_kraken.py index 7db3eeeeb..760e18982 100644 --- a/tests/exchange/test_kraken.py +++ b/tests/exchange/test_kraken.py @@ -13,11 +13,14 @@ STOPLOSS_ORDERTYPE = 'stop-loss' STOPLOSS_LIMIT_ORDERTYPE = 'stop-loss-limit' -def test_buy_kraken_trading_agreement(default_conf, mocker): +@pytest.mark.parametrize("order_type,time_in_force,expected_params", [ + ('limit', 'ioc', {'timeInForce': 'IOC', 'trading_agreement': 'agree'}), + ('limit', 'PO', {'postOnly': True, 'trading_agreement': 'agree'}), + ('market', None, {'trading_agreement': 'agree'}) +]) +def test_kraken_trading_agreement(default_conf, mocker, order_type, time_in_force, expected_params): api_mock = MagicMock() - order_id = f'test_prod_buy_{randint(0, 10 ** 6)}' - order_type = 'limit' - time_in_force = 'ioc' + order_id = f'test_prod_{order_type}_{randint(0, 10 ** 6)}' api_mock.options = {} api_mock.create_order = MagicMock(return_value={ 'id': order_id, @@ -49,41 +52,9 @@ def test_buy_kraken_trading_agreement(default_conf, mocker): assert api_mock.create_order.call_args[0][1] == order_type assert api_mock.create_order.call_args[0][2] == 'buy' assert api_mock.create_order.call_args[0][3] == 1 - assert api_mock.create_order.call_args[0][4] == 200 - assert api_mock.create_order.call_args[0][5] == {'timeInForce': 'IOC', - 'trading_agreement': 'agree'} + assert api_mock.create_order.call_args[0][4] == (200 if order_type == 'limit' else None) - -def test_sell_kraken_trading_agreement(default_conf, mocker): - api_mock = MagicMock() - order_id = f'test_prod_sell_{randint(0, 10 ** 6)}' - order_type = 'market' - api_mock.options = {} - api_mock.create_order = MagicMock(return_value={ - 'id': order_id, - 'symbol': 'ETH/BTC', - 'info': { - 'foo': 'bar' - } - }) - default_conf['dry_run'] = False - - mocker.patch(f'{EXMS}.amount_to_precision', lambda s, x, y: y) - mocker.patch(f'{EXMS}.price_to_precision', lambda s, x, y: y) - exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kraken") - - order = exchange.create_order(pair='ETH/BTC', ordertype=order_type, - side="sell", amount=1, rate=200, leverage=1.0) - - assert 'id' in order - assert 'info' in order - assert order['id'] == order_id - assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' - assert api_mock.create_order.call_args[0][1] == order_type - assert api_mock.create_order.call_args[0][2] == 'sell' - assert api_mock.create_order.call_args[0][3] == 1 - assert api_mock.create_order.call_args[0][4] is None - assert api_mock.create_order.call_args[0][5] == {'trading_agreement': 'agree'} + assert api_mock.create_order.call_args[0][5] == expected_params def test_get_balances_prod(default_conf, mocker): @@ -212,19 +183,17 @@ def test_create_stoploss_order_kraken(default_conf, mocker, ordertype, side, adj assert 'info' in order assert order['id'] == order_id assert api_mock.create_order.call_args_list[0][1]['symbol'] == 'ETH/BTC' - if ordertype == 'limit': - assert api_mock.create_order.call_args_list[0][1]['type'] == STOPLOSS_LIMIT_ORDERTYPE - assert api_mock.create_order.call_args_list[0][1]['params'] == { - 'trading_agreement': 'agree', - 'price2': adjustedprice - } - else: - assert api_mock.create_order.call_args_list[0][1]['type'] == STOPLOSS_ORDERTYPE - assert api_mock.create_order.call_args_list[0][1]['params'] == { - 'trading_agreement': 'agree'} + assert api_mock.create_order.call_args_list[0][1]['type'] == ordertype + assert api_mock.create_order.call_args_list[0][1]['params'] == { + 'trading_agreement': 'agree', + 'stopLossPrice': 220 + } assert api_mock.create_order.call_args_list[0][1]['side'] == side assert api_mock.create_order.call_args_list[0][1]['amount'] == 1 - assert api_mock.create_order.call_args_list[0][1]['price'] == 220 + if ordertype == 'limit': + assert api_mock.create_order.call_args_list[0][1]['price'] == adjustedprice + else: + assert api_mock.create_order.call_args_list[0][1]['price'] is None # test exception handling with pytest.raises(DependencyException): @@ -282,7 +251,7 @@ def test_create_stoploss_order_dry_run_kraken(default_conf, mocker, side): assert 'info' in order assert 'type' in order - assert order['type'] == STOPLOSS_ORDERTYPE + assert order['type'] == 'market' assert order['price'] == 220 assert order['amount'] == 1 @@ -294,11 +263,22 @@ def test_create_stoploss_order_dry_run_kraken(default_conf, mocker, side): def test_stoploss_adjust_kraken(mocker, default_conf, sl1, sl2, sl3, side): exchange = get_patched_exchange(mocker, default_conf, id='kraken') order = { - 'type': STOPLOSS_ORDERTYPE, - 'price': 1500, + 'type': 'market', + 'stopLossPrice': 1500, } assert exchange.stoploss_adjust(sl1, order, side=side) assert not exchange.stoploss_adjust(sl2, order, side=side) - # Test with invalid order case ... - order['type'] = 'stop_loss_limit' - assert not exchange.stoploss_adjust(sl3, order, side=side) + # diff. order type ... + order['type'] = 'limit' + assert exchange.stoploss_adjust(sl3, order, side=side) + + +@pytest.mark.parametrize('trade_id, expected', [ + ('1234', False), + ('170544369512007228', False), + ('1705443695120072285', True), + ('170544369512007228555', True), +]) +def test__valid_trade_pagination_id_kraken(mocker, default_conf_usdt, trade_id, expected): + exchange = get_patched_exchange(mocker, default_conf_usdt, id='kraken') + assert exchange._valid_trade_pagination_id('XRP/USDT', trade_id) == expected diff --git a/tests/exchange_online/conftest.py b/tests/exchange_online/conftest.py index 875faeded..a613ae586 100644 --- a/tests/exchange_online/conftest.py +++ b/tests/exchange_online/conftest.py @@ -247,7 +247,7 @@ EXCHANGES = { 'timeframe': '1h', 'orderbook_max_entries': 50, }, - 'huobi': { + 'htx': { 'pair': 'ETH/BTC', 'stake_currency': 'BTC', 'hasQuoteVolume': True, diff --git a/tests/exchange_online/test_ccxt_compat.py b/tests/exchange_online/test_ccxt_compat.py index b48d70de2..f95f4c000 100644 --- a/tests/exchange_online/test_ccxt_compat.py +++ b/tests/exchange_online/test_ccxt_compat.py @@ -218,9 +218,6 @@ class TestCCXTExchange: def test_ccxt__async_get_candle_history(self, exchange: EXCHANGE_FIXTURE_TYPE): exc, exchangename = exchange - if exchangename in ('bittrex'): - # For some weired reason, this test returns random lengths for bittrex. - pytest.skip("Exchange doesn't provide stable ohlcv history") if not exc._ft_has['ohlcv_has_history']: pytest.skip("Exchange does not support candle history") diff --git a/tests/freqai/conftest.py b/tests/freqai/conftest.py index 8a9425a32..81d72d92a 100644 --- a/tests/freqai/conftest.py +++ b/tests/freqai/conftest.py @@ -1,4 +1,5 @@ import platform +import sys from copy import deepcopy from pathlib import Path from typing import Any, Dict @@ -15,6 +16,10 @@ from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver from tests.conftest import get_patched_exchange +def is_py12() -> bool: + return sys.version_info >= (3, 12) + + def is_mac() -> bool: machine = platform.system() return "Darwin" in machine @@ -31,7 +36,7 @@ def patch_torch_initlogs(mocker) -> None: module_name = 'torch' mocked_module = types.ModuleType(module_name) sys.modules[module_name] = mocked_module - else: + elif not is_py12(): mocker.patch("torch._logging._init_logs") @@ -54,7 +59,7 @@ def freqai_conf(default_conf, tmp_path): "backtest_period_days": 10, "live_retrain_hours": 0, "expiration_hours": 1, - "identifier": "uniqe-id100", + "identifier": "unique-id100", "live_trained_timestamp": 0, "data_kitchen_thread_count": 2, "activate_tensorboard": False, diff --git a/tests/freqai/test_freqai_backtesting.py b/tests/freqai/test_freqai_backtesting.py index 0a8059966..c65934c4e 100644 --- a/tests/freqai/test_freqai_backtesting.py +++ b/tests/freqai/test_freqai_backtesting.py @@ -6,11 +6,17 @@ from unittest.mock import PropertyMock import pytest from freqtrade.commands.optimize_commands import setup_optimize_configuration +from freqtrade.configuration.timerange import TimeRange +from freqtrade.data import history +from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import RunMode +from freqtrade.enums.candletype import CandleType from freqtrade.exceptions import OperationalException +from freqtrade.freqai.data_kitchen import FreqaiDataKitchen from freqtrade.optimize.backtesting import Backtesting -from tests.conftest import (CURRENT_TEST_STRATEGY, get_args, log_has_re, patch_exchange, - patched_configuration_load_config_file) +from tests.conftest import (CURRENT_TEST_STRATEGY, get_args, get_patched_exchange, log_has_re, + patch_exchange, patched_configuration_load_config_file) +from tests.freqai.conftest import get_patched_freqai_strategy def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir, caplog): @@ -40,7 +46,16 @@ def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir, c Backtesting.cleanup() -def test_freqai_backtest_load_data(freqai_conf, mocker, caplog): +@pytest.mark.parametrize( + "timeframe, expected_startup_candle_count", + [ + ("5m", 876), + ("15m", 492), + ("1d", 302), + ], +) +def test_freqai_backtest_load_data(freqai_conf, mocker, caplog, + timeframe, expected_startup_candle_count): patch_exchange(mocker) now = datetime.now(timezone.utc) @@ -48,10 +63,14 @@ def test_freqai_backtest_load_data(freqai_conf, mocker, caplog): PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT'])) mocker.patch('freqtrade.optimize.backtesting.history.load_data') mocker.patch('freqtrade.optimize.backtesting.history.get_timerange', return_value=(now, now)) + freqai_conf['timeframe'] = timeframe + freqai_conf.get('freqai', {}).get('feature_parameters', {}).update({'include_timeframes': []}) backtesting = Backtesting(deepcopy(freqai_conf)) backtesting.load_bt_data() - assert log_has_re('Increasing startup_candle_count for freqai to.*', caplog) + assert log_has_re(f'Increasing startup_candle_count for freqai on {timeframe} ' + f'to {expected_startup_candle_count}', caplog) + assert history.load_data.call_args[1]['startup_candles'] == expected_startup_candle_count Backtesting.cleanup() @@ -85,3 +104,35 @@ def test_freqai_backtest_live_models_model_not_found(freqai_conf, mocker, testda Backtesting(bt_config) Backtesting.cleanup() + + +def test_freqai_backtest_consistent_timerange(mocker, freqai_conf): + freqai_conf['runmode'] = 'backtest' + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['XRP/USDT:USDT'])) + + gbs = mocker.patch('freqtrade.optimize.backtesting.generate_backtest_stats') + + freqai_conf['candle_type_def'] = CandleType.FUTURES + freqai_conf.get('exchange', {}).update({'pair_whitelist': ['XRP/USDT:USDT']}) + freqai_conf.get('freqai', {}).get('feature_parameters', {}).update( + {'include_timeframes': ['5m', '1h'], 'include_corr_pairlist': []}) + freqai_conf['timerange'] = '20211120-20211121' + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.dk = FreqaiDataKitchen(freqai_conf) + + timerange = TimeRange.parse_timerange("20211115-20211122") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + + backtesting = Backtesting(deepcopy(freqai_conf)) + backtesting.start() + + gbs.call_args[1]['min_date'] == datetime(2021, 11, 20, 0, 0, tzinfo=timezone.utc) + gbs.call_args[1]['max_date'] == datetime(2021, 11, 21, 0, 0, tzinfo=timezone.utc) + Backtesting.cleanup() diff --git a/tests/freqai/test_freqai_datadrawer.py b/tests/freqai/test_freqai_datadrawer.py index 2d1b1c691..7e1a1c32e 100644 --- a/tests/freqai/test_freqai_datadrawer.py +++ b/tests/freqai/test_freqai_datadrawer.py @@ -15,6 +15,7 @@ from tests.freqai.conftest import get_patched_freqai_strategy def test_update_historic_data(mocker, freqai_conf): + freqai_conf['runmode'] = 'backtest' strategy = get_patched_freqai_strategy(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf) strategy.dp = DataProvider(freqai_conf, exchange) diff --git a/tests/freqai/test_freqai_datakitchen.py b/tests/freqai/test_freqai_datakitchen.py index cac9d9838..ba1520601 100644 --- a/tests/freqai/test_freqai_datakitchen.py +++ b/tests/freqai/test_freqai_datakitchen.py @@ -3,6 +3,7 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from unittest.mock import MagicMock +import pandas as pd import pytest from freqtrade.configuration import TimeRange @@ -75,7 +76,7 @@ def test_filter_features(mocker, freqai_conf): freqai, unfiltered_dataframe = make_unfiltered_dataframe(mocker, freqai_conf) freqai.dk.find_features(unfiltered_dataframe) - filtered_df, labels = freqai.dk.filter_features( + filtered_df, _labels = freqai.dk.filter_features( unfiltered_dataframe, freqai.dk.training_features_list, freqai.dk.label_list, @@ -135,3 +136,64 @@ def test_get_full_model_path(mocker, freqai_conf, model): model_path = freqai.dk.get_full_models_path(freqai_conf) assert model_path.is_dir() is True + + +def test_get_pair_data_for_features_with_prealoaded_data(mocker, freqai_conf): + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + + _, base_df = freqai.dd.get_base_and_corr_dataframes(timerange, "LTC/BTC", freqai.dk) + df = freqai.dk.get_pair_data_for_features("LTC/BTC", "5m", strategy, base_dataframes=base_df) + + assert df is base_df["5m"] + assert not df.empty + + +def test_get_pair_data_for_features_without_preloaded_data(mocker, freqai_conf): + freqai_conf.update({"timerange": "20180115-20180130"}) + freqai_conf['runmode'] = 'backtest' + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + + base_df = {'5m': pd.DataFrame()} + df = freqai.dk.get_pair_data_for_features("LTC/BTC", "5m", strategy, base_dataframes=base_df) + + assert df is not base_df["5m"] + assert not df.empty + assert df.iloc[0]['date'].strftime("%Y-%m-%d %H:%M:%S") == "2018-01-11 23:00:00" + assert df.iloc[-1]['date'].strftime("%Y-%m-%d %H:%M:%S") == "2018-01-30 00:00:00" + + +def test_populate_features(mocker, freqai_conf): + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180115-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + + corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(timerange, "LTC/BTC", freqai.dk) + mocker.patch.object(strategy, 'feature_engineering_expand_all', return_value=base_df["5m"]) + df = freqai.dk.populate_features(base_df["5m"], "LTC/BTC", strategy, + base_dataframes=base_df, corr_dataframes=corr_df) + + strategy.feature_engineering_expand_all.assert_called_once() + pd.testing.assert_frame_equal(base_df["5m"], + strategy.feature_engineering_expand_all.call_args[0][0]) + + assert df.iloc[0]['date'].strftime("%Y-%m-%d %H:%M:%S") == "2018-01-15 00:00:00" diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 7638c03ed..58648d97f 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -1,7 +1,6 @@ import logging import platform import shutil -import sys from pathlib import Path from unittest.mock import MagicMock @@ -16,24 +15,24 @@ from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import Trade from freqtrade.plugins.pairlistmanager import PairListManager from tests.conftest import EXMS, create_mock_trades, get_patched_exchange, log_has_re -from tests.freqai.conftest import (get_patched_freqai_strategy, is_mac, make_rl_config, +from tests.freqai.conftest import (get_patched_freqai_strategy, is_mac, is_py12, make_rl_config, mock_pytorch_mlp_model_training_parameters) -def is_py11() -> bool: - return sys.version_info >= (3, 11) - - def is_arm() -> bool: machine = platform.machine() return "arm" in machine or "aarch64" in machine def can_run_model(model: str) -> None: + is_pytorch_model = 'Reinforcement' in model or 'PyTorch' in model + + if is_py12() and ("Catboost" in model or is_pytorch_model): + pytest.skip("Model not supported on python 3.12 yet.") + if is_arm() and "Catboost" in model: pytest.skip("CatBoost is not supported on ARM.") - is_pytorch_model = 'Reinforcement' in model or 'PyTorch' in model if is_pytorch_model and is_mac() and not is_arm(): pytest.skip("Reinforcement learning / PyTorch module not available on intel based Mac OS.") @@ -176,6 +175,7 @@ def test_extract_data_and_train_model_MultiTargets(mocker, freqai_conf, model, s 'CatboostClassifier', 'XGBoostClassifier', 'XGBoostRFClassifier', + 'SKLearnRandomForestClassifier', 'PyTorchMLPClassifier', ]) def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model): @@ -298,8 +298,11 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog) def test_start_backtesting_subdaily_backtest_period(mocker, freqai_conf): freqai_conf.update({"timerange": "20180120-20180124"}) - freqai_conf.get("freqai", {}).update({"backtest_period_days": 0.5}) - freqai_conf.get("freqai", {}).update({"save_backtest_models": True}) + freqai_conf['runmode'] = 'backtest' + freqai_conf.get("freqai", {}).update({ + "backtest_period_days": 0.5, + "save_backtest_models": True, + }) freqai_conf.get("freqai", {}).get("feature_parameters", {}).update( {"indicator_periods_candles": [2]}) strategy = get_patched_freqai_strategy(mocker, freqai_conf) @@ -326,6 +329,7 @@ def test_start_backtesting_subdaily_backtest_period(mocker, freqai_conf): def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog): freqai_conf.update({"timerange": "20180120-20180130"}) + freqai_conf['runmode'] = 'backtest' freqai_conf.get("freqai", {}).update({"save_backtest_models": True}) freqai_conf.get("freqai", {}).get("feature_parameters", {}).update( {"indicator_periods_candles": [2]}) @@ -389,6 +393,7 @@ def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog): def test_backtesting_fit_live_predictions(mocker, freqai_conf, caplog): + freqai_conf['runmode'] = 'backtest' freqai_conf.get("freqai", {}).update({"fit_live_predictions_candles": 10}) strategy = get_patched_freqai_strategy(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf) @@ -522,8 +527,8 @@ def test_get_state_info(mocker, freqai_conf, dp_exists, caplog, tickers): if is_mac(): pytest.skip("Reinforcement learning module not available on intel based Mac OS") - if is_py11(): - pytest.skip("Reinforcement learning currently not available on python 3.11.") + if is_py12(): + pytest.skip("Reinforcement learning currently not available on python 3.12.") freqai_conf.update({"freqaimodel": "ReinforcementLearner"}) freqai_conf.update({"timerange": "20180110-20180130"}) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index fd594e94b..603fcc310 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -734,7 +734,7 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: 'min_rate': [0.10370188, 0.10300000000000001], 'max_rate': [0.10501, 0.1038888], 'is_open': [False, False], - 'enter_tag': [None, None], + 'enter_tag': ['', ''], "leverage": [1.0, 1.0], "is_short": [False, False], 'open_timestamp': [1517251200000, 1517283000000], @@ -742,14 +742,18 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: 'orders': [ [ {'amount': 0.00957442, 'safe_price': 0.104445, 'ft_order_side': 'buy', - 'order_filled_timestamp': 1517251200000, 'ft_is_entry': True}, + 'order_filled_timestamp': 1517251200000, 'ft_is_entry': True, + 'ft_order_tag': ''}, {'amount': 0.00957442, 'safe_price': 0.10496853383458644, 'ft_order_side': 'sell', - 'order_filled_timestamp': 1517265300000, 'ft_is_entry': False} + 'order_filled_timestamp': 1517265300000, 'ft_is_entry': False, + 'ft_order_tag': 'roi'} ], [ {'amount': 0.0097064, 'safe_price': 0.10302485, 'ft_order_side': 'buy', - 'order_filled_timestamp': 1517283000000, 'ft_is_entry': True}, + 'order_filled_timestamp': 1517283000000, 'ft_is_entry': True, + 'ft_order_tag': ''}, {'amount': 0.0097064, 'safe_price': 0.10354126528822055, 'ft_order_side': 'sell', - 'order_filled_timestamp': 1517285400000, 'ft_is_entry': False} + 'order_filled_timestamp': 1517285400000, 'ft_is_entry': False, + 'ft_order_tag': 'roi'} ] ] }) @@ -1132,6 +1136,7 @@ def test_processed(default_conf, mocker, testdatadir) -> None: def test_backtest_dataprovider_analyzed_df(default_conf, fee, mocker, testdatadir) -> None: default_conf['use_exit_signal'] = False default_conf['max_open_trades'] = 10 + default_conf['runmode'] = 'backtest' mocker.patch(f'{EXMS}.get_fee', fee) mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=0.00001) mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=100000) @@ -1298,6 +1303,7 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir): mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=float('inf')) mocker.patch(f'{EXMS}.get_fee', fee) default_conf['max_open_trades'] = 10 + default_conf['runmode'] = 'backtest' backtest_conf = _make_backtest_conf(mocker, conf=default_conf, pair='UNITTEST/BTC', datadir=testdatadir) default_conf['timeframe'] = '1m' @@ -1342,6 +1348,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) dataframe['exit_short'] = 0 return dataframe + default_conf['runmode'] = 'backtest' mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=0.00001) mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=float('inf')) mocker.patch(f'{EXMS}.get_fee', fee) diff --git a/tests/optimize/test_backtesting_adjust_position.py b/tests/optimize/test_backtesting_adjust_position.py index 9b40b3a9d..7f7bbb29f 100644 --- a/tests/optimize/test_backtesting_adjust_position.py +++ b/tests/optimize/test_backtesting_adjust_position.py @@ -72,7 +72,7 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) -> 'min_rate': [0.10370188, 0.10300000000000001], 'max_rate': [0.10481985, 0.1038888], 'is_open': [False, False], - 'enter_tag': [None, None], + 'enter_tag': ['', ''], 'leverage': [1.0, 1.0], 'is_short': [False, False], 'open_timestamp': [1517251200000, 1517283000000], @@ -148,7 +148,7 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker, levera assert pytest.approx(trade.amount) == 47.61904762 * leverage assert len(trade.orders) == 1 # Increase position by 100 - backtesting.strategy.adjust_trade_position = MagicMock(return_value=100) + backtesting.strategy.adjust_trade_position = MagicMock(return_value=(100, 'PartIncrease')) trade = backtesting._get_adjust_trade_entry_for_candle(trade, row, current_time) @@ -156,6 +156,7 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker, levera assert pytest.approx(trade.stake_amount) == 200.0 assert pytest.approx(trade.amount) == 95.23809524 * leverage assert len(trade.orders) == 2 + assert trade.orders[-1].ft_order_tag == 'PartIncrease' assert pytest.approx(trade.liquidation_price) == (0.1038916 if leverage == 1 else 1.2127791) # Reduce by more than amount - no change to trade. @@ -171,13 +172,14 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker, levera assert pytest.approx(trade.liquidation_price) == (0.1038916 if leverage == 1 else 1.2127791) # Reduce position by 50 - backtesting.strategy.adjust_trade_position = MagicMock(return_value=-100) + backtesting.strategy.adjust_trade_position = MagicMock(return_value=(-100, 'partDecrease')) trade = backtesting._get_adjust_trade_entry_for_candle(trade, row, current_time) assert trade assert pytest.approx(trade.stake_amount) == 100.0 assert pytest.approx(trade.amount) == 47.61904762 * leverage assert len(trade.orders) == 3 + assert trade.orders[-1].ft_order_tag == 'partDecrease' assert trade.nr_of_successful_entries == 2 assert trade.nr_of_successful_exits == 1 assert pytest.approx(trade.liquidation_price) == (0.1038916 if leverage == 1 else 1.2127791) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 67f776c7e..be27bb3f5 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -254,7 +254,7 @@ def test_log_results_if_loss_improves(hyperopt, capsys) -> None: 'is_best': True } ) - out, err = capsys.readouterr() + out, _err = capsys.readouterr() assert all(x in out for x in ["Best", "2/2", " 1", "0.10%", "0.00100000 BTC (1.00%)", "00:20:00"]) @@ -333,7 +333,7 @@ def test_start_calls_optimizer(mocker, hyperopt_conf, capsys) -> None: parallel.assert_called_once() - out, err = capsys.readouterr() + out, _err = capsys.readouterr() assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out # Should be called for historical candle data assert dumper.call_count == 1 @@ -577,7 +577,7 @@ def test_print_json_spaces_all(mocker, hyperopt_conf, capsys) -> None: parallel.assert_called_once() - out, err = capsys.readouterr() + out, _err = capsys.readouterr() result_str = ( '{"params":{"mfi-value":null,"sell-mfi-value":null},"minimal_roi"' ':{},"stoploss":null,"trailing_stop":null,"max_open_trades":null}' @@ -624,7 +624,7 @@ def test_print_json_spaces_default(mocker, hyperopt_conf, capsys) -> None: parallel.assert_called_once() - out, err = capsys.readouterr() + out, _err = capsys.readouterr() assert '{"params":{"mfi-value":null,"sell-mfi-value":null},"minimal_roi":{},"stoploss":null}' in out # noqa: E501 # Should be called for historical candle data assert dumper.call_count == 1 @@ -666,7 +666,7 @@ def test_print_json_spaces_roi_stoploss(mocker, hyperopt_conf, capsys) -> None: parallel.assert_called_once() - out, err = capsys.readouterr() + out, _err = capsys.readouterr() assert '{"minimal_roi":{},"stoploss":null}' in out assert dumper.call_count == 1 @@ -704,7 +704,7 @@ def test_simplified_interface_roi_stoploss(mocker, hyperopt_conf, capsys) -> Non parallel.assert_called_once() - out, err = capsys.readouterr() + out, _err = capsys.readouterr() assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out assert dumper.call_count == 1 assert dumper2.call_count == 1 @@ -777,7 +777,7 @@ def test_simplified_interface_buy(mocker, hyperopt_conf, capsys) -> None: parallel.assert_called_once() - out, err = capsys.readouterr() + out, _err = capsys.readouterr() assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out assert dumper.called assert dumper.call_count == 1 @@ -819,7 +819,7 @@ def test_simplified_interface_sell(mocker, hyperopt_conf, capsys) -> None: parallel.assert_called_once() - out, err = capsys.readouterr() + out, _err = capsys.readouterr() assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out assert dumper.called assert dumper.call_count == 1 @@ -1051,7 +1051,7 @@ def test_max_open_trades_dump(mocker, hyperopt_conf, tmp_path, fee, capsys) -> N hyperopt.start() - out, err = capsys.readouterr() + out, _err = capsys.readouterr() assert 'max_open_trades = -1' in out assert 'max_open_trades = inf' not in out @@ -1070,7 +1070,7 @@ def test_max_open_trades_dump(mocker, hyperopt_conf, tmp_path, fee, capsys) -> N hyperopt.start() - out, err = capsys.readouterr() + out, _err = capsys.readouterr() assert '"max_open_trades":-1' in out diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 8b13bddea..d7c4bc6fa 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -143,8 +143,8 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf instance = LookaheadAnalysis(lookahead_conf, strategy_obj) instance.current_analysis = analysis - table, headers, data = (LookaheadAnalysisSubFunctions. - text_table_lookahead_analysis_instances(lookahead_conf, [instance])) + _table, _headers, data = (LookaheadAnalysisSubFunctions. + text_table_lookahead_analysis_instances(lookahead_conf, [instance])) # check row contents for a try that has too few signals assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' @@ -158,14 +158,14 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf analysis.false_exit_signals = 10 instance = LookaheadAnalysis(lookahead_conf, strategy_obj) instance.current_analysis = analysis - table, headers, data = (LookaheadAnalysisSubFunctions. - text_table_lookahead_analysis_instances(lookahead_conf, [instance])) + _table, _headers, data = (LookaheadAnalysisSubFunctions. + text_table_lookahead_analysis_instances(lookahead_conf, [instance])) assert data[0][2].__contains__("error") # edit it into not showing an error instance.failed_bias_check = False - table, headers, data = (LookaheadAnalysisSubFunctions. - text_table_lookahead_analysis_instances(lookahead_conf, [instance])) + _table, _headers, data = (LookaheadAnalysisSubFunctions. + text_table_lookahead_analysis_instances(lookahead_conf, [instance])) assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' assert data[0][2] # True @@ -176,8 +176,8 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf analysis.false_indicators.append('falseIndicator1') analysis.false_indicators.append('falseIndicator2') - table, headers, data = (LookaheadAnalysisSubFunctions. - text_table_lookahead_analysis_instances(lookahead_conf, [instance])) + _table, _headers, data = (LookaheadAnalysisSubFunctions. + text_table_lookahead_analysis_instances(lookahead_conf, [instance])) assert data[0][6] == 'falseIndicator1, falseIndicator2' @@ -185,7 +185,7 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf assert len(data) == 1 # check amount of multiple rows - table, headers, data = (LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( + _table, _headers, data = (LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( lookahead_conf, [instance, instance, instance])) assert len(data) == 3 diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index a65c3f277..0f190f3f5 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -513,7 +513,7 @@ def test_show_sorted_pairlist(testdatadir, default_conf, capsys): show_sorted_pairlist(default_conf, bt_data) - out, err = capsys.readouterr() + out, _err = capsys.readouterr() assert 'Pairs for Strategy StrategyTestV3: \n[' in out assert 'TOTAL' not in out assert '"ETH/BTC", // ' in out diff --git a/tests/optimize/test_recursive_analysis.py b/tests/optimize/test_recursive_analysis.py index aad465ff3..33fae0d08 100644 --- a/tests/optimize/test_recursive_analysis.py +++ b/tests/optimize/test_recursive_analysis.py @@ -107,8 +107,8 @@ def test_recursive_helper_text_table_recursive_analysis_instances(recursive_conf instance = RecursiveAnalysis(recursive_conf, strategy_obj) instance.dict_recursive = dict_diff - table, headers, data = (RecursiveAnalysisSubFunctions. - text_table_recursive_analysis_instances([instance])) + _table, _headers, data = (RecursiveAnalysisSubFunctions. + text_table_recursive_analysis_instances([instance])) # check row contents for a try that has too few signals assert data[0][0] == 'rsi' @@ -119,8 +119,8 @@ def test_recursive_helper_text_table_recursive_analysis_instances(recursive_conf dict_diff = dict() instance = RecursiveAnalysis(recursive_conf, strategy_obj) instance.dict_recursive = dict_diff - table, headers, data = (RecursiveAnalysisSubFunctions. - text_table_recursive_analysis_instances([instance])) + _table, _headers, data = (RecursiveAnalysisSubFunctions. + text_table_recursive_analysis_instances([instance])) assert len(data) == 0 diff --git a/tests/persistence/test_db_context.py b/tests/persistence/test_db_context.py new file mode 100644 index 000000000..690006219 --- /dev/null +++ b/tests/persistence/test_db_context.py @@ -0,0 +1,24 @@ +import pytest + +from freqtrade.persistence import FtNoDBContext, PairLocks, Trade + + +@pytest.mark.parametrize('timeframe', ['', '5m', '1d']) +def test_FtNoDBContext(timeframe): + PairLocks.timeframe = '' + assert Trade.use_db is True + assert PairLocks.use_db is True + assert PairLocks.timeframe == '' + + with FtNoDBContext(timeframe): + assert Trade.use_db is False + assert PairLocks.use_db is False + assert PairLocks.timeframe == timeframe + + with FtNoDBContext(): + assert Trade.use_db is False + assert PairLocks.use_db is False + assert PairLocks.timeframe == '' + + assert Trade.use_db is True + assert PairLocks.use_db is True diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index d66a47aa6..09dcd0af3 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -18,6 +18,7 @@ from freqtrade.persistence import LocalTrade, Trade from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.resolvers import PairListResolver +from freqtrade.util.datetime_helpers import dt_now from tests.conftest import (EXMS, create_mock_trades_usdt, get_patched_exchange, get_patched_freqtradebot, log_has, log_has_re, num_log_has) @@ -1513,3 +1514,144 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None: pm.refresh_pairlist() assert pm.whitelist == [] assert log_has_re(r'Whitelist with 0 pairs: \[]', caplog) + + +@pytest.mark.parametrize('pairlists,trade_mode,result', [ + ([ + # Get 2 pairs + {"method": "StaticPairList", "allow_inactive": True}, + {"method": "MarketCapPairList", "number_assets": 2} + ], 'spot', ['BTC/USDT', 'ETH/USDT']), + ([ + # Get 6 pairs + {"method": "StaticPairList", "allow_inactive": True}, + {"method": "MarketCapPairList", "number_assets": 6} + ], 'spot', ['BTC/USDT', 'ETH/USDT', 'XRP/USDT', 'ADA/USDT']), + ([ + # Get 3 pairs within top 6 ranks + {"method": "StaticPairList", "allow_inactive": True}, + {"method": "MarketCapPairList", "max_rank": 6, "number_assets": 3} + ], 'spot', ['BTC/USDT', 'ETH/USDT', 'XRP/USDT']), + + ([ + # Get 4 pairs within top 8 ranks + {"method": "StaticPairList", "allow_inactive": True}, + {"method": "MarketCapPairList", "max_rank": 8, "number_assets": 4} + ], 'spot', ['BTC/USDT', 'ETH/USDT', 'XRP/USDT']), + ([ + # MarketCapPairList as generator + {"method": "MarketCapPairList", "number_assets": 5} + ], 'spot', ['BTC/USDT', 'ETH/USDT', 'XRP/USDT']), + ([ + # MarketCapPairList as generator - low max_rank + {"method": "MarketCapPairList", "max_rank": 2, "number_assets": 5} + ], 'spot', ['BTC/USDT', 'ETH/USDT']), + ([ + # MarketCapPairList as generator - futures - low max_rank + {"method": "MarketCapPairList", "max_rank": 2, "number_assets": 5} + ], 'futures', ['ETH/USDT:USDT']), + ([ + # MarketCapPairList as generator - futures - low number_assets + {"method": "MarketCapPairList", "number_assets": 2} + ], 'futures', ['ETH/USDT:USDT', 'ADA/USDT:USDT']), +]) +def test_MarketCapPairList_filter( + mocker, default_conf_usdt, trade_mode, markets, pairlists, result +): + test_value = [ + {"symbol": "btc"}, + {"symbol": "eth"}, + {"symbol": "usdt"}, + {"symbol": "bnb"}, + {"symbol": "sol"}, + {"symbol": "xrp"}, + {"symbol": "usdc"}, + {"symbol": "steth"}, + {"symbol": "ada"}, + {"symbol": "avax"}, + ] + + default_conf_usdt['trading_mode'] = trade_mode + if trade_mode == 'spot': + default_conf_usdt['exchange']['pair_whitelist'].extend(['BTC/USDT', 'ETC/USDT', 'ADA/USDT']) + default_conf_usdt['pairlists'] = pairlists + mocker.patch.multiple(EXMS, + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + ) + + mocker.patch("freqtrade.plugins.pairlist.MarketCapPairList.CoinGeckoAPI.get_coins_markets", + return_value=test_value) + + exchange = get_patched_exchange(mocker, default_conf_usdt) + + pm = PairListManager(exchange, default_conf_usdt) + pm.refresh_pairlist() + + assert pm.whitelist == result + + +def test_MarketCapPairList_timing(mocker, default_conf_usdt, markets, time_machine): + test_value = [ + {"symbol": "btc"}, + {"symbol": "eth"}, + {"symbol": "usdt"}, + {"symbol": "bnb"}, + {"symbol": "sol"}, + {"symbol": "xrp"}, + {"symbol": "usdc"}, + {"symbol": "steth"}, + {"symbol": "ada"}, + {"symbol": "avax"}, + ] + + default_conf_usdt['trading_mode'] = 'spot' + default_conf_usdt['exchange']['pair_whitelist'].extend(['BTC/USDT', 'ETC/USDT', 'ADA/USDT']) + default_conf_usdt['pairlists'] = [{"method": "MarketCapPairList", "number_assets": 2}] + + markets_mock = MagicMock(return_value=markets) + mocker.patch.multiple(EXMS, + get_markets=markets_mock, + exchange_has=MagicMock(return_value=True), + ) + + mocker.patch("freqtrade.plugins.pairlist.MarketCapPairList.CoinGeckoAPI.get_coins_markets", + return_value=test_value) + + start_dt = dt_now() + + exchange = get_patched_exchange(mocker, default_conf_usdt) + time_machine.move_to(start_dt) + + pm = PairListManager(exchange, default_conf_usdt) + markets_mock.reset_mock() + pm.refresh_pairlist() + assert markets_mock.call_count == 3 + markets_mock.reset_mock() + + time_machine.move_to(start_dt + timedelta(hours=20)) + pm.refresh_pairlist() + # Cached pairlist ... + assert markets_mock.call_count == 1 + + markets_mock.reset_mock() + time_machine.move_to(start_dt + timedelta(days=2)) + pm.refresh_pairlist() + # No longer cached pairlist ... + assert markets_mock.call_count == 3 + + +def test_MarketCapPairList_exceptions(mocker, default_conf_usdt, markets, time_machine): + + exchange = get_patched_exchange(mocker, default_conf_usdt) + default_conf_usdt['pairlists'] = [{"method": "MarketCapPairList"}] + with pytest.raises(OperationalException, match=r"`number_assets` not specified.*"): + # No number_assets + PairListManager(exchange, default_conf_usdt) + + default_conf_usdt['pairlists'] = [{ + "method": "MarketCapPairList", 'number_assets': 20, 'max_rank': 260 + }] + with pytest.raises(OperationalException, + match="This filter only support marketcap rank up to 250."): + PairListManager(exchange, default_conf_usdt) diff --git a/tests/plugins/test_remotepairlist.py b/tests/plugins/test_remotepairlist.py index 9814e5662..9d407de9f 100644 --- a/tests/plugins/test_remotepairlist.py +++ b/tests/plugins/test_remotepairlist.py @@ -82,7 +82,7 @@ def test_fetch_pairlist_mock_response_html(mocker, rpl_config): remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config, rpl_config['pairlists'][0], 0) - with pytest.raises(OperationalException, match='RemotePairList is not of type JSON, abort.'): + with pytest.raises(OperationalException, match='RemotePairList is not of type JSON.'): remote_pairlist.fetch_pairlist() @@ -107,9 +107,11 @@ def test_fetch_pairlist_timeout_keep_last_pairlist(mocker, rpl_config, caplog): rpl_config['pairlists'][0], 0) remote_pairlist._last_pairlist = ["BTC/USDT", "ETH/USDT", "LTC/USDT"] + remote_pairlist._init_done = True + pairlist_url = rpl_config['pairlists'][0]['pairlist_url'] + pairs, _time_elapsed = remote_pairlist.fetch_pairlist() - pairs, time_elapsed = remote_pairlist.fetch_pairlist() - assert log_has(f"Was not able to fetch pairlist from: {remote_pairlist._pairlist_url}", caplog) + assert log_has(f'Error: Was not able to fetch pairlist from: ' f'{pairlist_url}', caplog) assert log_has("Keeping last fetched pairlist", caplog) assert pairs == ["BTC/USDT", "ETH/USDT", "LTC/USDT"] @@ -281,7 +283,7 @@ def test_remote_pairlist_blacklist(mocker, rpl_config, caplog, markets, tickers) remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][1], 1) - pairs, time_elapsed = remote_pairlist.fetch_pairlist() + pairs, _time_elapsed = remote_pairlist.fetch_pairlist() assert pairs == ["XRP/USDT"] @@ -334,7 +336,7 @@ def test_remote_pairlist_whitelist(mocker, rpl_config, processing_mode, markets, remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][1], 1) - pairs, time_elapsed = remote_pairlist.fetch_pairlist() + pairs, _time_elapsed = remote_pairlist.fetch_pairlist() assert pairs == ["XRP/USDT"] diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 7ea9dae89..ca81ea0e6 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -9,7 +9,7 @@ from sqlalchemy import select from freqtrade.edge import PairInfo from freqtrade.enums import SignalDirection, State, TradingMode from freqtrade.exceptions import ExchangeError, InvalidOrderException, TemporaryError -from freqtrade.persistence import Trade +from freqtrade.persistence import Order, Trade from freqtrade.persistence.pairlock_middleware import PairLocks from freqtrade.rpc import RPC, RPCException from freqtrade.rpc.fiat_convert import CryptoToFiatConverter @@ -99,7 +99,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'order_filled_timestamp': ANY, 'order_type': 'limit', 'price': 1.098e-05, 'is_open': False, 'pair': 'ETH/BTC', 'order_id': ANY, 'remaining': ANY, 'status': ANY, 'ft_is_entry': True, 'ft_fee_base': None, - 'funding_fee': ANY, + 'funding_fee': ANY, 'ft_order_tag': None, }], } mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) @@ -355,8 +355,18 @@ def test_rpc_delete_trade(mocker, default_conf, fee, markets, caplog, is_short): rpc._rpc_delete('200') trades = Trade.session.scalars(select(Trade)).all() - trades[1].stoploss_order_id = '1234' - trades[2].stoploss_order_id = '1234' + trades[2].stoploss_order_id = '102' + trades[2].orders.append( + Order( + ft_order_side='stoploss', + ft_pair=trades[2].pair, + ft_is_open=True, + ft_amount=trades[2].amount, + ft_price=trades[2].stop_loss, + order_id='102', + status='open', + ) + ) assert len(trades) > 2 res = rpc._rpc_delete('1') @@ -369,7 +379,7 @@ def test_rpc_delete_trade(mocker, default_conf, fee, markets, caplog, is_short): cancel_mock.reset_mock() stoploss_mock.reset_mock() - res = rpc._rpc_delete('2') + res = rpc._rpc_delete('5') assert isinstance(res, dict) assert stoploss_mock.call_count == 1 assert res['cancel_order_count'] == 1 diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index e94509b40..9df26ad4e 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -112,7 +112,7 @@ def assert_response(response, expected_code=200, needs_cors=True): def test_api_not_found(botclient): - ftbot, client = botclient + _ftbot, client = botclient rc = client_get(client, f"{BASE_URI}/invalid_url") assert_response(rc, 404) @@ -120,7 +120,7 @@ def test_api_not_found(botclient): def test_api_ui_fallback(botclient, mocker): - ftbot, client = botclient + _ftbot, client = botclient rc = client_get(client, "/favicon.ico") assert rc.status_code == 200 @@ -150,7 +150,7 @@ def test_api_ui_fallback(botclient, mocker): def test_api_ui_version(botclient, mocker): - ftbot, client = botclient + _ftbot, client = botclient mocker.patch('freqtrade.commands.deploy_commands.read_ui_version', return_value='0.1.2') rc = client_get(client, "/ui_version") @@ -230,7 +230,7 @@ def test_api_unauthorized(botclient): def test_api_token_login(botclient): - ftbot, client = botclient + _ftbot, client = botclient rc = client.post(f"{BASE_URI}/token/login", data=None, headers={'Authorization': _basic_auth_str('WRONG_USER', 'WRONG_PASS'), @@ -249,7 +249,7 @@ def test_api_token_login(botclient): def test_api_token_refresh(botclient): - ftbot, client = botclient + _ftbot, client = botclient rc = client_post(client, f"{BASE_URI}/token/login") assert_response(rc) rc = client.post(f"{BASE_URI}/token/refresh", @@ -541,7 +541,7 @@ def test_api_count(botclient, mocker, ticker, fee, markets, is_short): def test_api_locks(botclient): - ftbot, client = botclient + _ftbot, client = botclient rc = client_get(client, f"{BASE_URI}/locks") assert_response(rc) @@ -728,7 +728,6 @@ def test_api_delete_trade(botclient, mocker, fee, markets, is_short): ftbot.strategy.order_types['stoploss_on_exchange'] = True trades = Trade.session.scalars(select(Trade)).all() - trades[1].stoploss_order_id = '1234' Trade.commit() assert len(trades) > 2 @@ -745,9 +744,9 @@ def test_api_delete_trade(botclient, mocker, fee, markets, is_short): assert cancel_mock.call_count == 0 assert len(trades) - 1 == len(Trade.session.scalars(select(Trade)).all()) - rc = client_delete(client, f"{BASE_URI}/trades/2") + rc = client_delete(client, f"{BASE_URI}/trades/5") assert_response(rc) - assert rc.json()['result_msg'] == 'Deleted trade 2. Closed 1 open orders.' + assert rc.json()['result_msg'] == 'Deleted trade 5. Closed 1 open orders.' assert len(trades) - 2 == len(Trade.session.scalars(select(Trade)).all()) assert stoploss_mock.call_count == 1 @@ -822,7 +821,7 @@ def test_api_trade_reload_trade(botclient, mocker, fee, markets, ticker, is_shor def test_api_logs(botclient): - ftbot, client = botclient + _ftbot, client = botclient rc = client_get(client, f"{BASE_URI}/logs") assert_response(rc) assert len(rc.json()) == 2 @@ -1229,7 +1228,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets, is_short, def test_api_version(botclient): - ftbot, client = botclient + _ftbot, client = botclient rc = client_get(client, f"{BASE_URI}/version") assert_response(rc) @@ -1237,7 +1236,7 @@ def test_api_version(botclient): def test_api_blacklist(botclient, mocker): - ftbot, client = botclient + _ftbot, client = botclient rc = client_get(client, f"{BASE_URI}/blacklist") assert_response(rc) @@ -1304,7 +1303,7 @@ def test_api_blacklist(botclient, mocker): def test_api_whitelist(botclient): - ftbot, client = botclient + _ftbot, client = botclient rc = client_get(client, f"{BASE_URI}/whitelist") assert_response(rc) @@ -1559,7 +1558,7 @@ def test_api_pair_candles(botclient, ohlcv_history): def test_api_pair_history(botclient, mocker): - ftbot, client = botclient + _ftbot, client = botclient timeframe = '5m' lfm = mocker.patch('freqtrade.strategy.interface.IStrategy.load_freqAI_model') # No pair @@ -1604,9 +1603,9 @@ def test_api_pair_history(botclient, mocker): assert 'data' in result data = result['data'] assert len(data) == 289 - # analyed DF has 28 columns - assert len(result['columns']) == 28 - assert len(data[0]) == 28 + # analyed DF has 30 columns + assert len(result['columns']) == 30 + assert len(data[0]) == 30 date_col_idx = [idx for idx, c in enumerate(result['columns']) if c == 'date'][0] rsi_col_idx = [idx for idx, c in enumerate(result['columns']) if c == 'rsi'][0] @@ -1699,7 +1698,7 @@ def test_api_strategies(botclient, tmp_path): def test_api_strategy(botclient): - ftbot, client = botclient + _ftbot, client = botclient rc = client_get(client, f"{BASE_URI}/strategy/{CURRENT_TEST_STRATEGY}") @@ -1718,7 +1717,7 @@ def test_api_strategy(botclient): def test_api_exchanges(botclient): - ftbot, client = botclient + _ftbot, client = botclient rc = client_get(client, f"{BASE_URI}/exchanges") assert_response(rc) @@ -1770,6 +1769,7 @@ def test_api_freqaimodels(botclient, tmp_path, mocker): {'name': 'LightGBMRegressorMultiTarget'}, {'name': 'ReinforcementLearner'}, {'name': 'ReinforcementLearner_multiproc'}, + {'name': 'SKlearnRandomForestClassifier'}, {'name': 'XGBoostClassifier'}, {'name': 'XGBoostRFClassifier'}, {'name': 'XGBoostRFRegressor'}, @@ -1788,6 +1788,7 @@ def test_api_freqaimodels(botclient, tmp_path, mocker): 'LightGBMRegressorMultiTarget', 'ReinforcementLearner', 'ReinforcementLearner_multiproc', + 'SKlearnRandomForestClassifier', 'XGBoostClassifier', 'XGBoostRFClassifier', 'XGBoostRFRegressor', @@ -1953,7 +1954,7 @@ def test_list_available_pairs(botclient): def test_sysinfo(botclient): - ftbot, client = botclient + _ftbot, client = botclient rc = client_get(client, f"{BASE_URI}/sysinfo") assert_response(rc) @@ -2233,7 +2234,7 @@ def test_api_patch_backtest_history_entry(botclient, tmp_path: Path): def test_health(botclient): - ftbot, client = botclient + _ftbot, client = botclient rc = client_get(client, f"{BASE_URI}/health") @@ -2244,7 +2245,7 @@ def test_health(botclient): def test_api_ws_subscribe(botclient, mocker): - ftbot, client = botclient + _ftbot, client = botclient ws_url = f"/api/v1/message/ws?token={_TEST_WS_TOKEN}" sub_mock = mocker.patch('freqtrade.rpc.api_server.ws.WebSocketChannel.set_subscriptions') @@ -2267,7 +2268,7 @@ def test_api_ws_subscribe(botclient, mocker): def test_api_ws_requests(botclient, caplog): caplog.set_level(logging.DEBUG) - ftbot, client = botclient + _ftbot, client = botclient ws_url = f"/api/v1/message/ws?token={_TEST_WS_TOKEN}" # Test whitelist request diff --git a/tests/rpc/test_rpc_emc.py b/tests/rpc/test_rpc_emc.py index 26512e30b..4cfa3e9db 100644 --- a/tests/rpc/test_rpc_emc.py +++ b/tests/rpc/test_rpc_emc.py @@ -2,7 +2,6 @@ Unit test file for rpc/external_message_consumer.py """ import asyncio -import functools import logging from datetime import datetime, timezone from unittest.mock import MagicMock @@ -302,19 +301,16 @@ async def test_emc_receive_messages_valid(default_conf, caplog, mocker): dp = DataProvider(default_conf, None, None, None) emc = ExternalMessageConsumer(default_conf, dp) - loop = asyncio.get_event_loop() - def change_running(emc): emc._running = not emc._running - class TestChannel: async def recv(self, *args, **kwargs): + emc._running = False return {"type": "whitelist", "data": ["BTC/USDT"]} async def ping(self, *args, **kwargs): return asyncio.Future() try: - change_running(emc) - loop.call_soon(functools.partial(change_running, emc=emc)) + emc._running = True await emc._receive_messages(TestChannel(), test_producer, lock) assert log_has_re(r"Received message of type `whitelist`.+", caplog) @@ -349,19 +345,16 @@ async def test_emc_receive_messages_invalid(default_conf, caplog, mocker): dp = DataProvider(default_conf, None, None, None) emc = ExternalMessageConsumer(default_conf, dp) - loop = asyncio.get_event_loop() - def change_running(emc): emc._running = not emc._running - class TestChannel: async def recv(self, *args, **kwargs): + emc._running = False return {"type": ["BTC/USDT"]} async def ping(self, *args, **kwargs): return asyncio.Future() try: - change_running(emc) - loop.call_soon(functools.partial(change_running, emc=emc)) + emc._running = True await emc._receive_messages(TestChannel(), test_producer, lock) assert log_has_re(r"Invalid message from.+", caplog) @@ -396,8 +389,8 @@ async def test_emc_receive_messages_timeout(default_conf, caplog, mocker): dp = DataProvider(default_conf, None, None, None) emc = ExternalMessageConsumer(default_conf, dp) - loop = asyncio.get_event_loop() - def change_running(emc): emc._running = not emc._running + def change_running(): + emc._running = not emc._running class TestChannel: async def recv(self, *args, **kwargs): @@ -407,8 +400,7 @@ async def test_emc_receive_messages_timeout(default_conf, caplog, mocker): return asyncio.Future() try: - change_running(emc) - loop.call_soon(functools.partial(change_running, emc=emc)) + change_running() with pytest.raises(asyncio.TimeoutError): await emc._receive_messages(TestChannel(), test_producer, lock) @@ -447,19 +439,16 @@ async def test_emc_receive_messages_handle_error(default_conf, caplog, mocker): emc.handle_producer_message = MagicMock(side_effect=Exception) - loop = asyncio.get_event_loop() - def change_running(emc): emc._running = not emc._running - class TestChannel: async def recv(self, *args, **kwargs): + emc._running = False return {"type": "whitelist", "data": ["BTC/USDT"]} async def ping(self, *args, **kwargs): return asyncio.Future() try: - change_running(emc) - loop.call_soon(functools.partial(change_running, emc=emc)) + emc._running = True await emc._receive_messages(TestChannel(), test_producer, lock) assert log_has_re(r"Error handling producer message.+", caplog) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 5d190dcc6..f85b19a01 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -599,7 +599,7 @@ async def test_daily_handle(default_conf_usdt, update, ticker, fee, mocker, time get_fee=fee, ) - telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt) + telegram, _freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt) # Move date to within day time_machine.move_to('2022-06-11 08:00:00+00:00') @@ -1154,11 +1154,11 @@ async def test_telegram_forceexit_handle(default_conf, update, ticker, fee, 'profit_amount': 6.314e-05, 'profit_ratio': 0.0629778, 'stake_currency': 'BTC', + 'quote_currency': 'BTC', 'base_currency': 'ETH', 'fiat_currency': 'USD', 'buy_tag': ANY, 'enter_tag': ANY, - 'sell_reason': ExitType.FORCE_EXIT.value, 'exit_reason': ExitType.FORCE_EXIT.value, 'open_date': ANY, 'close_date': ANY, @@ -1166,6 +1166,8 @@ async def test_telegram_forceexit_handle(default_conf, update, ticker, fee, 'stake_amount': 0.0009999999999054, 'sub_trade': False, 'cumulative_profit': 0.0, + 'is_final_exit': False, + 'final_profit_ratio': None, } == last_msg @@ -1227,11 +1229,11 @@ async def test_telegram_force_exit_down_handle(default_conf, update, ticker, fee 'profit_amount': -5.497e-05, 'profit_ratio': -0.05482878, 'stake_currency': 'BTC', + 'quote_currency': 'BTC', 'base_currency': 'ETH', 'fiat_currency': 'USD', 'buy_tag': ANY, 'enter_tag': ANY, - 'sell_reason': ExitType.FORCE_EXIT.value, 'exit_reason': ExitType.FORCE_EXIT.value, 'open_date': ANY, 'close_date': ANY, @@ -1239,6 +1241,8 @@ async def test_telegram_force_exit_down_handle(default_conf, update, ticker, fee 'stake_amount': 0.0009999999999054, 'sub_trade': False, 'cumulative_profit': 0.0, + 'is_final_exit': False, + 'final_profit_ratio': None, } == last_msg @@ -1290,11 +1294,11 @@ async def test_forceexit_all_handle(default_conf, update, ticker, fee, mocker) - 'profit_amount': -4.09e-06, 'profit_ratio': -0.00408133, 'stake_currency': 'BTC', + 'quote_currency': 'BTC', 'base_currency': 'ETH', 'fiat_currency': 'USD', 'buy_tag': ANY, 'enter_tag': ANY, - 'sell_reason': ExitType.FORCE_EXIT.value, 'exit_reason': ExitType.FORCE_EXIT.value, 'open_date': ANY, 'close_date': ANY, @@ -1302,6 +1306,8 @@ async def test_forceexit_all_handle(default_conf, update, ticker, fee, mocker) - 'stake_amount': 0.0009999999999054, 'sub_trade': False, 'cumulative_profit': 0.0, + 'is_final_exit': False, + 'final_profit_ratio': None, } == msg @@ -1474,7 +1480,7 @@ async def test_telegram_performance_handle(default_conf_usdt, update, ticker, fe fetch_ticker=ticker, get_fee=fee, ) - telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt) + telegram, _freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt) # Create some test data create_mock_trades_usdt(fee) @@ -1649,7 +1655,7 @@ async def test_telegram_lock_handle(default_conf, update, ticker, fee, mocker) - async def test_whitelist_static(default_conf, update, mocker) -> None: - telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) + telegram, _freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) await telegram._whitelist(update=update, context=MagicMock()) assert msg_mock.call_count == 1 @@ -1999,7 +2005,10 @@ def test_send_msg_enter_notification(default_conf, mocker, caplog, message_type, 'stake_amount': 0.01465333, 'stake_amount_fiat': 0.0, 'stake_currency': 'BTC', + 'quote_currency': 'BTC', + 'base_currency': 'ETH', 'fiat_currency': 'USD', + 'sub_trade': False, 'current_rate': 1.099e-05, 'amount': 1333.3333333333335, 'analyzed_candle': {'open': 1.1, 'high': 2.2, 'low': 1.0, 'close': 1.5}, @@ -2008,17 +2017,19 @@ def test_send_msg_enter_notification(default_conf, mocker, caplog, message_type, telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) telegram.send_msg(msg) - leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' + leverage_text = f' ({leverage:.1g}x)' if leverage and leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( - f'\N{LARGE BLUE CIRCLE} *Binance (dry):* {enter} ETH/BTC (#1)\n' + f'\N{LARGE BLUE CIRCLE} *Binance (dry):* New Trade (#1)\n' + f'*Pair:* `ETH/BTC`\n' '*Candle OHLC*: `1.1, 2.2, 1.0, 1.5`\n' f'*Enter Tag:* `{enter_signal}`\n' '*Amount:* `1333.33333333`\n' - f'{leverage_text}' - '*Open Rate:* `0.00001099`\n' - '*Current Rate:* `0.00001099`\n' - '*Total:* `(0.01465333 BTC, 180.895 USD)`' + f'*Direction:* `{enter}' + f'{leverage_text}`\n' + '*Open Rate:* `0.00001099 BTC`\n' + '*Current Rate:* `0.00001099 BTC`\n' + '*Total:* `0.01465333 BTC / 180.895 USD`' ) freqtradebot.config['telegram']['notification_settings'] = {'buy': 'off'} @@ -2106,20 +2117,25 @@ def test_send_msg_entry_fill_notification(default_conf, mocker, message_type, en 'leverage': leverage, 'stake_amount': 0.01465333, 'direction': entered, + 'sub_trade': False, 'stake_currency': 'BTC', + 'quote_currency': 'BTC', + 'base_currency': 'ETH', 'fiat_currency': 'USD', 'open_rate': 1.099e-05, 'amount': 1333.3333333333335, 'open_date': dt_now() - timedelta(hours=1) }) - leverage_text = f'*Leverage:* `{leverage}`\n' if leverage != 1.0 else '' + leverage_text = f' ({leverage:.1g}x)' if leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( - f'\N{CHECK MARK} *Binance (dry):* {entered}ed ETH/BTC (#1)\n' + f'\N{CHECK MARK} *Binance (dry):* New Trade filled (#1)\n' + f'*Pair:* `ETH/BTC`\n' f'*Enter Tag:* `{enter_signal}`\n' '*Amount:* `1333.33333333`\n' - f"{leverage_text}" - '*Open Rate:* `0.00001099`\n' - '*Total:* `(0.01465333 BTC, 180.895 USD)`' + f'*Direction:* `{entered}' + f"{leverage_text}`\n" + '*Open Rate:* `0.00001099 BTC`\n' + '*Total:* `0.01465333 BTC / 180.895 USD`' ) msg_mock.reset_mock() @@ -2134,6 +2150,8 @@ def test_send_msg_entry_fill_notification(default_conf, mocker, message_type, en 'sub_trade': True, 'direction': entered, 'stake_currency': 'BTC', + 'quote_currency': 'BTC', + 'base_currency': 'ETH', 'fiat_currency': 'USD', 'open_rate': 1.099e-05, 'amount': 1333.3333333333335, @@ -2141,16 +2159,18 @@ def test_send_msg_entry_fill_notification(default_conf, mocker, message_type, en }) assert msg_mock.call_args[0][0] == ( - f'\N{CHECK MARK} *Binance (dry):* {entered}ed ETH/BTC (#1)\n' + f'\N{CHECK MARK} *Binance (dry):* Position increase filled (#1)\n' + f'*Pair:* `ETH/BTC`\n' f'*Enter Tag:* `{enter_signal}`\n' '*Amount:* `1333.33333333`\n' - f"{leverage_text}" - '*Open Rate:* `0.00001099`\n' - '*Total:* `(0.01465333 BTC, 180.895 USD)`' + f'*Direction:* `{entered}' + f"{leverage_text}`\n" + '*Open Rate:* `0.00001099 BTC`\n' + '*New Total:* `0.01465333 BTC / 180.895 USD`' ) -def test_send_msg_sell_notification(default_conf, mocker) -> None: +def test_send_msg_exit_notification(default_conf, mocker) -> None: with time_machine.travel("2022-09-01 05:00:00 +00:00", tick=False): telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) @@ -2165,14 +2185,16 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'leverage': 1.0, 'direction': 'Long', 'gain': 'loss', - 'order_rate': 3.201e-05, + 'order_rate': 3.201e-04, 'amount': 1333.3333333333335, 'order_type': 'market', - 'open_rate': 7.5e-05, - 'current_rate': 3.201e-05, + 'open_rate': 7.5e-04, + 'current_rate': 3.201e-04, 'profit_amount': -0.05746268, 'profit_ratio': -0.57405275, 'stake_currency': 'ETH', + 'quote_currency': 'ETH', + 'base_currency': 'KEY', 'fiat_currency': 'USD', 'enter_tag': 'buy_signal1', 'exit_reason': ExitType.STOP_LOSS.value, @@ -2181,14 +2203,14 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: }) assert msg_mock.call_args[0][0] == ( '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' - '*Unrealized Profit:* `-57.41% (loss: -0.05746268 ETH / -24.812 USD)`\n' + '*Unrealized Profit:* `-57.41% (loss: -0.05746 ETH / -24.812 USD)`\n' '*Enter Tag:* `buy_signal1`\n' '*Exit Reason:* `stop_loss`\n' '*Direction:* `Long`\n' '*Amount:* `1333.33333333`\n' - '*Open Rate:* `0.00007500`\n' - '*Current Rate:* `0.00003201`\n' - '*Exit Rate:* `0.00003201`\n' + '*Open Rate:* `0.00075 ETH`\n' + '*Current Rate:* `0.00032 ETH`\n' + '*Exit Rate:* `0.00032 ETH`\n' '*Duration:* `1:00:00 (60.0 min)`' ) @@ -2200,15 +2222,17 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'pair': 'KEY/ETH', 'direction': 'Long', 'gain': 'loss', - 'order_rate': 3.201e-05, + 'order_rate': 3.201e-04, 'amount': 1333.3333333333335, 'order_type': 'market', - 'open_rate': 7.5e-05, - 'current_rate': 3.201e-05, + 'open_rate': 7.5e-04, + 'current_rate': 3.201e-04, 'cumulative_profit': -0.15746268, 'profit_amount': -0.05746268, 'profit_ratio': -0.57405275, 'stake_currency': 'ETH', + 'quote_currency': 'ETH', + 'base_currency': 'KEY', 'fiat_currency': 'USD', 'enter_tag': 'buy_signal1', 'exit_reason': ExitType.STOP_LOSS.value, @@ -2219,16 +2243,16 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: }) assert msg_mock.call_args[0][0] == ( '\N{WARNING SIGN} *Binance (dry):* Partially exiting KEY/ETH (#1)\n' - '*Unrealized Sub Profit:* `-57.41% (loss: -0.05746268 ETH / -24.812 USD)`\n' - '*Cumulative Profit:* (`-0.15746268 ETH / -24.812 USD`)\n' + '*Unrealized Sub Profit:* `-57.41% (loss: -0.05746 ETH / -24.812 USD)`\n' + '*Cumulative Profit:* `-0.15746 ETH / -24.812 USD`\n' '*Enter Tag:* `buy_signal1`\n' '*Exit Reason:* `stop_loss`\n' '*Direction:* `Long`\n' '*Amount:* `1333.33333333`\n' - '*Open Rate:* `0.00007500`\n' - '*Current Rate:* `0.00003201`\n' - '*Exit Rate:* `0.00003201`\n' - '*Remaining:* `(0.01 ETH, -24.812 USD)`' + '*Open Rate:* `0.00075 ETH`\n' + '*Current Rate:* `0.00032 ETH`\n' + '*Exit Rate:* `0.00032 ETH`\n' + '*Remaining:* `0.01 ETH / -24.812 USD`' ) msg_mock.reset_mock() @@ -2239,14 +2263,17 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'pair': 'KEY/ETH', 'direction': 'Long', 'gain': 'loss', - 'order_rate': 3.201e-05, + 'order_rate': 3.201e-04, 'amount': 1333.3333333333335, 'order_type': 'market', - 'open_rate': 7.5e-05, - 'current_rate': 3.201e-05, + 'open_rate': 7.5e-04, + 'current_rate': 3.201e-04, 'profit_amount': -0.05746268, 'profit_ratio': -0.57405275, 'stake_currency': 'ETH', + 'quote_currency': 'ETH', + 'base_currency': 'KEY', + 'fiat_currency': None, 'enter_tag': 'buy_signal1', 'exit_reason': ExitType.STOP_LOSS.value, 'open_date': dt_now() - timedelta(days=1, hours=2, minutes=30), @@ -2254,21 +2281,21 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: }) assert msg_mock.call_args[0][0] == ( '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' - '*Unrealized Profit:* `-57.41% (loss: -0.05746268 ETH)`\n' + '*Unrealized Profit:* `-57.41% (loss: -0.05746 ETH)`\n' '*Enter Tag:* `buy_signal1`\n' '*Exit Reason:* `stop_loss`\n' '*Direction:* `Long`\n' '*Amount:* `1333.33333333`\n' - '*Open Rate:* `0.00007500`\n' - '*Current Rate:* `0.00003201`\n' - '*Exit Rate:* `0.00003201`\n' + '*Open Rate:* `0.00075 ETH`\n' + '*Current Rate:* `0.00032 ETH`\n' + '*Exit Rate:* `0.00032 ETH`\n' '*Duration:* `1 day, 2:30:00 (1590.0 min)`' ) # Reset singleton function to avoid random breaks telegram._rpc._fiat_converter.convert_amount = old_convamount -async def test_send_msg_sell_cancel_notification(default_conf, mocker) -> None: +async def test_send_msg_exit_cancel_notification(default_conf, mocker) -> None: telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) @@ -2306,7 +2333,7 @@ async def test_send_msg_sell_cancel_notification(default_conf, mocker) -> None: ('Long', 'long_signal_01', 1.0), ('Long', 'long_signal_01', 5.0), ('Short', 'short_signal_01', 2.0)]) -def test_send_msg_sell_fill_notification(default_conf, mocker, direction, +def test_send_msg_exit_fill_notification(default_conf, mocker, direction, enter_signal, leverage) -> None: default_conf['telegram']['notification_settings']['exit_fill'] = 'on' @@ -2321,31 +2348,34 @@ def test_send_msg_sell_fill_notification(default_conf, mocker, direction, 'leverage': leverage, 'direction': direction, 'gain': 'loss', - 'limit': 3.201e-05, + 'limit': 3.201e-04, 'amount': 1333.3333333333335, 'order_type': 'market', - 'open_rate': 7.5e-05, - 'close_rate': 3.201e-05, + 'open_rate': 7.5e-04, + 'close_rate': 3.201e-04, 'profit_amount': -0.05746268, 'profit_ratio': -0.57405275, 'stake_currency': 'ETH', + 'quote_currency': 'ETH', + 'base_currency': 'KEY', + 'fiat_currency': None, 'enter_tag': enter_signal, 'exit_reason': ExitType.STOP_LOSS.value, 'open_date': dt_now() - timedelta(days=1, hours=2, minutes=30), 'close_date': dt_now(), }) - leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' + leverage_text = f' ({leverage:.1g}x)`\n' if leverage and leverage != 1.0 else '`\n' assert msg_mock.call_args[0][0] == ( '\N{WARNING SIGN} *Binance (dry):* Exited KEY/ETH (#1)\n' - '*Profit:* `-57.41% (loss: -0.05746268 ETH)`\n' + '*Profit:* `-57.41% (loss: -0.05746 ETH)`\n' f'*Enter Tag:* `{enter_signal}`\n' '*Exit Reason:* `stop_loss`\n' - f"*Direction:* `{direction}`\n" + f"*Direction:* `{direction}" f"{leverage_text}" '*Amount:* `1333.33333333`\n' - '*Open Rate:* `0.00007500`\n' - '*Exit Rate:* `0.00003201`\n' + '*Open Rate:* `0.00075 ETH`\n' + '*Exit Rate:* `0.00032 ETH`\n' '*Duration:* `1 day, 2:30:00 (1590.0 min)`' ) @@ -2416,24 +2446,29 @@ def test_send_msg_buy_notification_no_fiat( 'open_rate': 1.099e-05, 'order_type': 'limit', 'direction': enter, + 'sub_trade': False, 'stake_amount': 0.01465333, 'stake_amount_fiat': 0.0, 'stake_currency': 'BTC', + 'quote_currency': 'BTC', + 'base_currency': 'ETH', 'fiat_currency': None, 'current_rate': 1.099e-05, 'amount': 1333.3333333333335, 'open_date': dt_now() - timedelta(hours=1) }) - leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' + leverage_text = f' ({leverage:.1g}x)' if leverage and leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( - f'\N{LARGE BLUE CIRCLE} *Binance:* {enter} ETH/BTC (#1)\n' + f'\N{LARGE BLUE CIRCLE} *Binance:* New Trade (#1)\n' + '*Pair:* `ETH/BTC`\n' f'*Enter Tag:* `{enter_signal}`\n' '*Amount:* `1333.33333333`\n' - f'{leverage_text}' - '*Open Rate:* `0.00001099`\n' - '*Current Rate:* `0.00001099`\n' - '*Total:* `(0.01465333 BTC)`' + f'*Direction:* `{enter}' + f'{leverage_text}`\n' + '*Open Rate:* `0.00001099 BTC`\n' + '*Current Rate:* `0.00001099 BTC`\n' + '*Total:* `0.01465333 BTC`' ) @@ -2443,7 +2478,7 @@ def test_send_msg_buy_notification_no_fiat( ('Long', 'long_signal_01', 5.0), ('Short', 'short_signal_01', 2.0), ]) -def test_send_msg_sell_notification_no_fiat( +def test_send_msg_exit_notification_no_fiat( default_conf, mocker, direction, enter_signal, leverage, time_machine) -> None: del default_conf['fiat_display_currency'] time_machine.move_to('2022-05-02 00:00:00 +00:00', tick=False) @@ -2457,14 +2492,17 @@ def test_send_msg_sell_notification_no_fiat( 'gain': 'loss', 'leverage': leverage, 'direction': direction, - 'order_rate': 3.201e-05, + 'sub_trade': False, + 'order_rate': 3.201e-04, 'amount': 1333.3333333333335, 'order_type': 'limit', - 'open_rate': 7.5e-05, - 'current_rate': 3.201e-05, + 'open_rate': 7.5e-04, + 'current_rate': 3.201e-04, 'profit_amount': -0.05746268, 'profit_ratio': -0.57405275, 'stake_currency': 'ETH', + 'quote_currency': 'ETH', + 'base_currency': 'KEY', 'fiat_currency': 'USD', 'enter_tag': enter_signal, 'exit_reason': ExitType.STOP_LOSS.value, @@ -2472,37 +2510,37 @@ def test_send_msg_sell_notification_no_fiat( 'close_date': dt_now(), }) - leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' + leverage_text = f' ({leverage:.1g}x)' if leverage and leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' - '*Unrealized Profit:* `-57.41% (loss: -0.05746268 ETH)`\n' + '*Unrealized Profit:* `-57.41% (loss: -0.05746 ETH)`\n' f'*Enter Tag:* `{enter_signal}`\n' '*Exit Reason:* `stop_loss`\n' - f'*Direction:* `{direction}`\n' - f'{leverage_text}' + f'*Direction:* `{direction}' + f'{leverage_text}`\n' '*Amount:* `1333.33333333`\n' - '*Open Rate:* `0.00007500`\n' - '*Current Rate:* `0.00003201`\n' - '*Exit Rate:* `0.00003201`\n' + '*Open Rate:* `0.00075 ETH`\n' + '*Current Rate:* `0.00032 ETH`\n' + '*Exit Rate:* `0.00032 ETH`\n' '*Duration:* `2:35:03 (155.1 min)`' ) @pytest.mark.parametrize('msg,expected', [ - ({'profit_percent': 20.1, 'exit_reason': 'roi'}, "\N{ROCKET}"), - ({'profit_percent': 5.1, 'exit_reason': 'roi'}, "\N{ROCKET}"), - ({'profit_percent': 2.56, 'exit_reason': 'roi'}, "\N{EIGHT SPOKED ASTERISK}"), - ({'profit_percent': 1.0, 'exit_reason': 'roi'}, "\N{EIGHT SPOKED ASTERISK}"), - ({'profit_percent': 0.0, 'exit_reason': 'roi'}, "\N{EIGHT SPOKED ASTERISK}"), - ({'profit_percent': -5.0, 'exit_reason': 'stop_loss'}, "\N{WARNING SIGN}"), - ({'profit_percent': -2.0, 'exit_reason': 'sell_signal'}, "\N{CROSS MARK}"), + ({'profit_ratio': 0.201, 'exit_reason': 'roi'}, "\N{ROCKET}"), + ({'profit_ratio': 0.051, 'exit_reason': 'roi'}, "\N{ROCKET}"), + ({'profit_ratio': 0.0256, 'exit_reason': 'roi'}, "\N{EIGHT SPOKED ASTERISK}"), + ({'profit_ratio': 0.01, 'exit_reason': 'roi'}, "\N{EIGHT SPOKED ASTERISK}"), + ({'profit_ratio': 0.0, 'exit_reason': 'roi'}, "\N{EIGHT SPOKED ASTERISK}"), + ({'profit_ratio': -0.05, 'exit_reason': 'stop_loss'}, "\N{WARNING SIGN}"), + ({'profit_ratio': -0.02, 'exit_reason': 'sell_signal'}, "\N{CROSS MARK}"), ]) -def test__sell_emoji(default_conf, mocker, msg, expected): +def test__exit_emoji(default_conf, mocker, msg, expected): del default_conf['fiat_display_currency'] telegram, _, _ = get_telegram_testobject(mocker, default_conf) - assert telegram._get_sell_emoji(msg) == expected + assert telegram._get_exit_emoji(msg) == expected async def test_telegram__send_msg(default_conf, mocker, caplog) -> None: @@ -2609,7 +2647,7 @@ async def test__send_msg_keyboard(default_conf, mocker, caplog) -> None: async def test_change_market_direction(default_conf, mocker, update) -> None: - telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) + telegram, _, _msg_mock = get_telegram_testobject(mocker, default_conf) assert telegram._rpc._freqtrade.strategy.market_direction == MarketDirection.NONE context = MagicMock() context.args = ["long"] diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py index 36b96ace5..7d88056e4 100644 --- a/tests/rpc/test_rpc_webhook.py +++ b/tests/rpc/test_rpc_webhook.py @@ -1,5 +1,6 @@ # pragma pylint: disable=missing-docstring, C0103, protected-access +import logging from datetime import datetime, timedelta from unittest.mock import MagicMock @@ -331,6 +332,7 @@ def test_send_msg_webhook(default_conf, mocker): def test_exception_send_msg(default_conf, mocker, caplog): + caplog.set_level(logging.DEBUG) default_conf["webhook"] = get_webhook_dict() del default_conf["webhook"]["entry"] del default_conf["webhook"]["webhookentry"] diff --git a/tests/strategy/strats/strategy_test_v3.py b/tests/strategy/strats/strategy_test_v3.py index 571427fb1..83c7353ce 100644 --- a/tests/strategy/strats/strategy_test_v3.py +++ b/tests/strategy/strats/strategy_test_v3.py @@ -152,7 +152,7 @@ class StrategyTestV3(IStrategy): ( qtpylib.crossed_below(dataframe['rsi'], self.sell_rsi.value) ), - 'enter_short'] = 1 + ('enter_short', 'enter_tag')] = (1, 'short_Tag') return dataframe @@ -176,7 +176,7 @@ class StrategyTestV3(IStrategy): ( qtpylib.crossed_above(dataframe['rsi'], self.buy_rsi.value) ), - 'exit_short'] = 1 + ('exit_short', 'exit_tag')] = (1, 'short_Tag') return dataframe diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 226bbc7ae..790f5d255 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -105,7 +105,7 @@ def test_returns_latest_signal(ohlcv_history): _STRATEGY.config['trading_mode'] = 'spot' -def test_analyze_pair_empty(default_conf, mocker, caplog, ohlcv_history): +def test_analyze_pair_empty(mocker, caplog, ohlcv_history): mocker.patch.object(_STRATEGY.dp, 'ohlcv', return_value=ohlcv_history) mocker.patch.object( _STRATEGY, '_analyze_ticker_internal', @@ -1019,3 +1019,30 @@ def test_auto_hyperopt_interface_loadparams(default_conf, mocker, caplog): StrategyResolver.load_strategy(default_conf) assert log_has("Invalid parameter file format.", caplog) + + +@pytest.mark.parametrize('function,raises', [ + ('populate_entry_trend', True), + ('advise_entry', False), + ('populate_exit_trend', True), + ('advise_exit', False), +]) +def test_pandas_warning_direct(ohlcv_history, function, raises): + + df = _STRATEGY.populate_indicators(ohlcv_history, {'pair': 'ETH/BTC'}) + if raises: + with pytest.warns(FutureWarning): + # Test for Future warning + # FutureWarning: Setting an item of incompatible dtype is + # deprecated and will raise in a future error of pandas + # https://github.com/pandas-dev/pandas/issues/56503 + getattr(_STRATEGY, function)(df, {'pair': 'ETH/BTC'}) + else: + getattr(_STRATEGY, function)(df, {'pair': 'ETH/BTC'}) + + +def test_pandas_warning_through_analyze_pair(ohlcv_history, mocker, recwarn): + + mocker.patch.object(_STRATEGY.dp, 'ohlcv', return_value=ohlcv_history) + _STRATEGY.analyze_pair('ETH/BTC') + assert len(recwarn) == 0 diff --git a/tests/test_arguments.py b/tests/test_arguments.py index b4f6e7279..e14b09719 100644 --- a/tests/test_arguments.py +++ b/tests/test_arguments.py @@ -173,7 +173,7 @@ def test_download_data_options() -> None: def test_plot_dataframe_options() -> None: args = [ 'plot-dataframe', - '-c', 'config_examples/config_bittrex.example.json', + '-c', 'tests/testdata/testconfigs/main_test_config.json', '--indicators1', 'sma10', 'sma100', '--indicators2', 'macd', 'fastd', 'fastk', '--plot-limit', '30', diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 6472faf37..71585cac3 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -15,7 +15,7 @@ from freqtrade.configuration.deprecated_settings import (check_conflicting_setti process_deprecated_setting, process_removed_setting, process_temporary_deprecated_settings) -from freqtrade.configuration.environment_vars import flat_vars_to_nested_dict +from freqtrade.configuration.environment_vars import _flat_vars_to_nested_dict from freqtrade.configuration.load_config import (load_config_file, load_file, load_from_files, log_config_error_range) from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL, ENV_VAR_PREFIX @@ -1419,7 +1419,7 @@ def test_flat_vars_to_nested_dict(caplog): 'chat_id': '2151' } } - res = flat_vars_to_nested_dict(test_args, ENV_VAR_PREFIX) + res = _flat_vars_to_nested_dict(test_args, ENV_VAR_PREFIX) assert res == expected assert log_has("Loading variable 'FREQTRADE__EXCHANGE__SOME_SETTING'", caplog) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 70c3fee3e..8e17604ab 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -438,6 +438,7 @@ def test_enter_positions_no_pairs_left(default_conf_usdt, ticker_usdt, limit_buy create_order=MagicMock(return_value=limit_buy_order_usdt_open), get_fee=fee, ) + mocker.patch('freqtrade.configuration.config_validation._validate_whitelist') default_conf_usdt['exchange']['pair_whitelist'] = whitelist freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade) @@ -626,15 +627,16 @@ def test_process_exchange_failures(default_conf_usdt, ticker_usdt, mocker) -> No mocker.patch.multiple( EXMS, fetch_ticker=ticker_usdt, - create_order=MagicMock(side_effect=TemporaryError) + reload_markets=MagicMock(side_effect=TemporaryError), + create_order=MagicMock(side_effect=TemporaryError), ) - sleep_mock = mocker.patch('time.sleep', side_effect=lambda _: None) + sleep_mock = mocker.patch('time.sleep') worker = Worker(args=None, config=default_conf_usdt) patch_get_signal(worker.freqtrade) worker._process_running() - assert sleep_mock.has_calls() + assert sleep_mock.called is True def test_process_operational_exception(default_conf_usdt, ticker_usdt, mocker) -> None: @@ -857,7 +859,7 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, open_order['id'] = '22' freqtrade.strategy.confirm_trade_entry = MagicMock(return_value=True) assert freqtrade.execute_entry(pair, stake_amount) - assert enter_rate_mock.call_count == 1 + assert enter_rate_mock.call_count == 2 assert enter_mm.call_count == 1 call_args = enter_mm.call_args_list[0][1] assert call_args['pair'] == pair @@ -879,7 +881,7 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, fix_price = 0.06 assert freqtrade.execute_entry(pair, stake_amount, fix_price, is_short=is_short) # Make sure get_rate wasn't called again - assert enter_rate_mock.call_count == 0 + assert enter_rate_mock.call_count == 1 assert enter_mm.call_count == 2 call_args = enter_mm.call_args_list[1][1] @@ -1160,9 +1162,9 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ freqtrade.enter_positions() trade = Trade.session.scalars(select(Trade)).first() - trade.is_short = is_short - trade.is_open = True - trade.stoploss_order_id = None + assert trade.is_short == is_short + assert trade.is_open + assert trade.stoploss_order_id is None assert freqtrade.handle_stoploss_on_exchange(trade) is False assert stoploss.call_count == 1 @@ -1170,34 +1172,21 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ # Second case: when stoploss is set but it is not yet hit # should do nothing and return false - stop_order_dict.update({'id': "102"}) trade.is_open = True - trade.stoploss_order_id = "102" - trade.orders.append( - Order( - ft_order_side='stoploss', - ft_pair=trade.pair, - ft_is_open=True, - ft_amount=trade.amount, - ft_price=trade.stop_loss, - order_id='102', - status='open', - ) - ) - hanging_stoploss_order = MagicMock(return_value={'status': 'open'}) + hanging_stoploss_order = MagicMock(return_value={'id': '13434334', 'status': 'open'}) mocker.patch(f'{EXMS}.fetch_stoploss_order', hanging_stoploss_order) assert freqtrade.handle_stoploss_on_exchange(trade) is False - assert trade.stoploss_order_id == "102" + hanging_stoploss_order.assert_called_once_with('13434334', trade.pair) + assert trade.stoploss_order_id == "13434334" # Third case: when stoploss was set but it was canceled for some reason # should set a stoploss immediately and return False caplog.clear() trade.is_open = True - trade.stoploss_order_id = "102" - canceled_stoploss_order = MagicMock(return_value={'id': '103_1', 'status': 'canceled'}) + canceled_stoploss_order = MagicMock(return_value={'id': '13434334', 'status': 'canceled'}) mocker.patch(f'{EXMS}.fetch_stoploss_order', canceled_stoploss_order) stoploss.reset_mock() amount_before = trade.amount @@ -1213,25 +1202,14 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ # should unset stoploss_order_id and return true # as a trade actually happened caplog.clear() - freqtrade.enter_positions() - stop_order_dict.update({'id': "104"}) + stop_order_dict.update({'id': "103_1"}) trade = Trade.session.scalars(select(Trade)).first() trade.is_short = is_short trade.is_open = True - trade.stoploss_order_id = "104" - trade.orders.append(Order( - ft_order_side='stoploss', - order_id='104', - ft_pair=trade.pair, - ft_is_open=True, - ft_amount=trade.amount, - ft_price=0.0, - )) - assert trade stoploss_order_hit = MagicMock(return_value={ - 'id': "104", + 'id': "103_1", 'status': 'closed', 'type': 'stop_loss_limit', 'price': 3, @@ -1273,7 +1251,40 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ assert freqtrade.handle_stoploss_on_exchange(trade) is False assert stoploss.call_count == 0 - # Seventh case: emergency exit triggered + +@pytest.mark.parametrize("is_short", [False, True]) +def test_handle_stoploss_on_exchange_emergency(mocker, default_conf_usdt, fee, is_short, + limit_order) -> None: + stop_order_dict = {'id': "13434334"} + stoploss = MagicMock(return_value=stop_order_dict) + enter_order = limit_order[entry_side(is_short)] + exit_order = limit_order[exit_side(is_short)] + patch_RPCManager(mocker) + patch_exchange(mocker) + mocker.patch.multiple( + EXMS, + fetch_ticker=MagicMock(return_value={ + 'bid': 1.9, + 'ask': 2.2, + 'last': 1.9 + }), + create_order=MagicMock(side_effect=[ + enter_order, + exit_order, + ]), + get_fee=fee, + create_stoploss=stoploss + ) + freqtrade = FreqtradeBot(default_conf_usdt) + patch_get_signal(freqtrade, enter_short=is_short, enter_long=not is_short) + + freqtrade.enter_positions() + trade = Trade.session.scalars(select(Trade)).first() + assert trade.is_short == is_short + assert trade.is_open + assert trade.stoploss_order_id is None + + # emergency exit triggered # Trailing stop should not act anymore stoploss_order_cancelled = MagicMock(side_effect=[{ 'id': "107", @@ -1287,7 +1298,6 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ 'info': {'stopPrice': 22}, }]) trade.stoploss_order_id = "107" - trade.is_open = True trade.stoploss_last_update = dt_now() - timedelta(hours=1) trade.stop_loss = 24 trade.exit_reason = None @@ -1548,7 +1558,7 @@ def test_create_stoploss_order_invalid_order( # Rpc is sending first buy, then sell assert rpc_mock.call_count == 2 - assert rpc_mock.call_args_list[0][0][0]['sell_reason'] == ExitType.EMERGENCY_EXIT.value + assert rpc_mock.call_args_list[0][0][0]['exit_reason'] == ExitType.EMERGENCY_EXIT.value assert rpc_mock.call_args_list[0][0][0]['order_type'] == 'market' assert rpc_mock.call_args_list[0][0][0]['type'] == 'exit' assert rpc_mock.call_args_list[1][0][0]['type'] == 'exit_fill' @@ -1606,12 +1616,15 @@ def test_create_stoploss_order_insufficient_funds( ]) @pytest.mark.usefixtures("init_persistence") def test_handle_stoploss_on_exchange_trailing( - mocker, default_conf_usdt, fee, is_short, bid, ask, limit_order, stop_price, hang_price + mocker, default_conf_usdt, fee, is_short, bid, ask, limit_order, stop_price, hang_price, + time_machine, ) -> None: # When trailing stoploss is set enter_order = limit_order[entry_side(is_short)] exit_order = limit_order[exit_side(is_short)] - stoploss = MagicMock(return_value={'id': 13434334, 'status': 'open'}) + stoploss = MagicMock(return_value={'id': '13434334', 'status': 'open'}) + start_dt = dt_now() + time_machine.move_to(start_dt, tick=False) patch_RPCManager(mocker) mocker.patch.multiple( EXMS, @@ -1685,6 +1698,8 @@ def test_handle_stoploss_on_exchange_trailing( assert freqtrade.handle_trade(trade) is False assert freqtrade.handle_stoploss_on_exchange(trade) is False + assert trade.stoploss_order_id == '13434334' + # price jumped 2x mocker.patch( f'{EXMS}.fetch_ticker', @@ -1706,16 +1721,15 @@ def test_handle_stoploss_on_exchange_trailing( cancel_order_mock.assert_not_called() stoploss_order_mock.assert_not_called() + # Move time by 10s ... so stoploss order should be replaced. + time_machine.move_to(start_dt + timedelta(minutes=10), tick=False) + assert freqtrade.handle_trade(trade) is False assert trade.stop_loss == stop_price[1] - trade.stoploss_order_id = '100' - - # setting stoploss_on_exchange_interval to 0 seconds - freqtrade.strategy.order_types['stoploss_on_exchange_interval'] = 0 assert freqtrade.handle_stoploss_on_exchange(trade) is False - cancel_order_mock.assert_called_once_with('100', 'ETH/USDT') + cancel_order_mock.assert_called_once_with('13434334', 'ETH/USDT') stoploss_order_mock.assert_called_once_with( amount=30, pair='ETH/USDT', @@ -2228,6 +2242,7 @@ def test_update_trade_state(mocker, default_conf_usdt, limit_order, is_short, ca order = limit_order[entry_side(is_short)] mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True)) + mocker.patch('freqtrade.freqtradebot.FreqtradeBot._notify_enter') mocker.patch(f'{EXMS}.fetch_order', return_value=order) mocker.patch(f'{EXMS}.get_trades_for_order', return_value=[]) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=0.0) @@ -2297,6 +2312,7 @@ def test_update_trade_state_withorderdict( order_id = "oid_123456" order['id'] = order_id mocker.patch(f'{EXMS}.get_trades_for_order', return_value=trades_for_order) + mocker.patch('freqtrade.freqtradebot.FreqtradeBot._notify_enter') # fetch_order should not be called!! mocker.patch(f'{EXMS}.fetch_order', MagicMock(side_effect=ValueError)) patch_exchange(mocker) @@ -2340,6 +2356,7 @@ def test_update_trade_state_exception(mocker, default_conf_usdt, is_short, limit order = limit_order[entry_side(is_short)] freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mocker.patch(f'{EXMS}.fetch_order', return_value=order) + mocker.patch('freqtrade.freqtradebot.FreqtradeBot._notify_enter') # TODO: should not be magicmock trade = MagicMock() @@ -3486,7 +3503,7 @@ def test_handle_cancel_enter(mocker, caplog, default_conf_usdt, limit_order, is_ @pytest.mark.parametrize("is_short", [False, True]) -@pytest.mark.parametrize("limit_buy_order_canceled_empty", ['binance', 'kraken', 'bittrex'], +@pytest.mark.parametrize("limit_buy_order_canceled_empty", ['binance', 'kraken', 'bybit'], indirect=['limit_buy_order_canceled_empty']) def test_handle_cancel_enter_exchanges(mocker, caplog, default_conf_usdt, is_short, fee, limit_buy_order_canceled_empty) -> None: @@ -3767,9 +3784,9 @@ def test_execute_trade_exit_up(default_conf_usdt, ticker_usdt, fee, ticker_usdt_ 'profit_amount': 0.29554455 if is_short else 5.685, 'profit_ratio': 0.00493809 if is_short else 0.09451372, 'stake_currency': 'USDT', + 'quote_currency': 'USDT', 'fiat_currency': 'USD', 'base_currency': 'ETH', - 'sell_reason': ExitType.ROI.value, 'exit_reason': ExitType.ROI.value, 'open_date': ANY, 'close_date': ANY, @@ -3777,6 +3794,8 @@ def test_execute_trade_exit_up(default_conf_usdt, ticker_usdt, fee, ticker_usdt_ 'sub_trade': False, 'cumulative_profit': 0.0, 'stake_amount': pytest.approx(60), + 'is_final_exit': False, + 'final_profit_ratio': None, } == last_msg @@ -3832,9 +3851,9 @@ def test_execute_trade_exit_down(default_conf_usdt, ticker_usdt, fee, ticker_usd 'profit_amount': -5.65990099 if is_short else -0.00075, 'profit_ratio': -0.0945681 if is_short else -1.247e-05, 'stake_currency': 'USDT', + 'quote_currency': 'USDT', 'base_currency': 'ETH', 'fiat_currency': 'USD', - 'sell_reason': ExitType.STOP_LOSS.value, 'exit_reason': ExitType.STOP_LOSS.value, 'open_date': ANY, 'close_date': ANY, @@ -3842,6 +3861,8 @@ def test_execute_trade_exit_down(default_conf_usdt, ticker_usdt, fee, ticker_usd 'sub_trade': False, 'cumulative_profit': 0.0, 'stake_amount': pytest.approx(60), + 'is_final_exit': False, + 'final_profit_ratio': None, } == last_msg @@ -3918,9 +3939,9 @@ def test_execute_trade_exit_custom_exit_price( 'profit_amount': pytest.approx(profit_amount), 'profit_ratio': profit_ratio, 'stake_currency': 'USDT', + 'quote_currency': 'USDT', 'base_currency': 'ETH', 'fiat_currency': 'USD', - 'sell_reason': 'foo', 'exit_reason': 'foo', 'open_date': ANY, 'close_date': ANY, @@ -3928,6 +3949,8 @@ def test_execute_trade_exit_custom_exit_price( 'sub_trade': False, 'cumulative_profit': 0.0, 'stake_amount': pytest.approx(60), + 'is_final_exit': False, + 'final_profit_ratio': None, } == last_msg @@ -3991,9 +4014,9 @@ def test_execute_trade_exit_down_stoploss_on_exchange_dry_run( 'profit_amount': -0.3 if is_short else -0.8985, 'profit_ratio': -0.00501253 if is_short else -0.01493766, 'stake_currency': 'USDT', + 'quote_currency': 'USDT', 'fiat_currency': 'USD', 'base_currency': 'ETH', - 'sell_reason': ExitType.STOP_LOSS.value, 'exit_reason': ExitType.STOP_LOSS.value, 'open_date': ANY, 'close_date': ANY, @@ -4001,6 +4024,8 @@ def test_execute_trade_exit_down_stoploss_on_exchange_dry_run( 'sub_trade': False, 'cumulative_profit': 0.0, 'stake_amount': pytest.approx(60), + 'is_final_exit': False, + 'final_profit_ratio': None, } == last_msg @@ -4257,9 +4282,9 @@ def test_execute_trade_exit_market_order( 'profit_amount': pytest.approx(profit_amount), 'profit_ratio': profit_ratio, 'stake_currency': 'USDT', + 'quote_currency': 'USDT', 'base_currency': 'ETH', 'fiat_currency': 'USD', - 'sell_reason': ExitType.ROI.value, 'exit_reason': ExitType.ROI.value, 'open_date': ANY, 'close_date': ANY, @@ -4267,7 +4292,8 @@ def test_execute_trade_exit_market_order( 'sub_trade': False, 'cumulative_profit': 0.0, 'stake_amount': pytest.approx(60), - + 'is_final_exit': False, + 'final_profit_ratio': None, } == last_msg @@ -6699,11 +6725,15 @@ def test_check_and_call_adjust_trade_position(mocker, default_conf_usdt, fee, ca ) create_mock_trades(fee) caplog.set_level(logging.DEBUG) - freqtrade.strategy.adjust_trade_position = MagicMock(return_value=10) + freqtrade.strategy.adjust_trade_position = MagicMock(return_value=(10, 'aaaa')) freqtrade.process_open_trade_positions() assert log_has_re(r"Max adjustment entries for .* has been reached\.", caplog) + assert freqtrade.strategy.adjust_trade_position.call_count == 1 caplog.clear() - freqtrade.strategy.adjust_trade_position = MagicMock(return_value=-10) + freqtrade.strategy.adjust_trade_position = MagicMock(return_value=(-10, 'partial_exit_c')) freqtrade.process_open_trade_positions() assert log_has_re(r"LIMIT_SELL has been fulfilled.*", caplog) + assert freqtrade.strategy.adjust_trade_position.call_count == 1 + trade = Trade.get_trades(trade_filter=[Trade.id == 5]).first() + assert trade.orders[-1].ft_order_tag == 'partial_exit_c' diff --git a/tests/test_integration.py b/tests/test_integration.py index 31b742273..94253dffb 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -11,8 +11,7 @@ from freqtrade.rpc.rpc import RPC from tests.conftest import EXMS, get_patched_freqtradebot, log_has_re, patch_get_signal -def test_may_execute_exit_stoploss_on_exchange_multi(default_conf, ticker, fee, - limit_buy_order, mocker) -> None: +def test_may_execute_exit_stoploss_on_exchange_multi(default_conf, ticker, fee, mocker) -> None: """ Tests workflow of selling stoploss_on_exchange. Sells @@ -537,7 +536,7 @@ def test_dca_order_adjust_entry_replace_fails( # Create DCA order for 2nd trade (so we have 2 open orders on 2 trades) # this 2nd order won't fill. - freqtrade.strategy.adjust_trade_position = MagicMock(return_value=20) + freqtrade.strategy.adjust_trade_position = MagicMock(return_value=(20, 'PeNF')) freqtrade.process() @@ -628,12 +627,13 @@ def test_dca_exiting(default_conf_usdt, ticker_usdt, fee, mocker, caplog, levera assert log_has_re( r"Remaining amount of \d\.\d+.* would be smaller than the minimum of 10.", caplog) - freqtrade.strategy.adjust_trade_position = MagicMock(return_value=-20) + freqtrade.strategy.adjust_trade_position = MagicMock(return_value=(-20, 'PES')) freqtrade.process() trade = Trade.get_trades().first() assert len(trade.orders) == 2 assert trade.orders[-1].ft_order_side == 'sell' + assert trade.orders[-1].ft_order_tag == 'PES' assert pytest.approx(trade.stake_amount) == 40.198 assert pytest.approx(trade.amount) == 20.099 * leverage assert trade.open_rate == 2.0 @@ -650,28 +650,42 @@ def test_dca_exiting(default_conf_usdt, ticker_usdt, fee, mocker, caplog, levera caplog.clear() # Sell more than what we got (we got ~20 coins left) - # First adjusts the amount to 20 - then rejects. + # Doesn't exit, as the amount is too high. freqtrade.strategy.adjust_trade_position = MagicMock(return_value=-50) freqtrade.process() - assert log_has_re("Adjusting amount to trade.amount as it is higher.*", caplog) - assert log_has_re("Remaining amount of 0.0 would be smaller than the minimum of 10.", caplog) trade = Trade.get_trades().first() assert len(trade.orders) == 2 + + # Amount too low... + freqtrade.strategy.adjust_trade_position = MagicMock(return_value=-(trade.stake_amount * 0.99)) + freqtrade.process() + + trade = Trade.get_trades().first() + assert len(trade.orders) == 2 + + # Amount exactly comes out as exactly 0 + freqtrade.strategy.adjust_trade_position = MagicMock( + return_value=-(trade.amount / trade.leverage * 2.02)) + freqtrade.process() + + trade = Trade.get_trades().first() + assert len(trade.orders) == 3 + assert trade.orders[-1].ft_order_side == 'sell' assert pytest.approx(trade.stake_amount) == 40.198 - assert trade.is_open + assert trade.is_open is False # use amount that would trunc to 0.0 once selling mocker.patch(f"{EXMS}.amount_to_contract_precision", lambda s, p, v: round(v, 1)) freqtrade.strategy.adjust_trade_position = MagicMock(return_value=-0.01) freqtrade.process() trade = Trade.get_trades().first() - assert len(trade.orders) == 2 + assert len(trade.orders) == 3 assert trade.orders[-1].ft_order_side == 'sell' assert pytest.approx(trade.stake_amount) == 40.198 - assert trade.is_open + assert trade.is_open is False assert log_has_re('Amount to exit is 0.0 due to exchange limits - not exiting.', caplog) - expected_profit = starting_amount - 40.1980 + trade.realized_profit + expected_profit = starting_amount - 60 + trade.realized_profit assert pytest.approx(freqtrade.wallets.get_free('USDT')) == expected_profit if spot: assert pytest.approx(freqtrade.wallets.get_total('USDT')) == expected_profit diff --git a/tests/test_log_setup.py b/tests/test_log_setup.py index 182e78730..4310b8f09 100644 --- a/tests/test_log_setup.py +++ b/tests/test_log_setup.py @@ -63,9 +63,9 @@ def test_set_loggers_syslog(): setup_logging_pre() setup_logging(config) assert len(logger.handlers) == 3 - assert [x for x in logger.handlers if type(x) == logging.handlers.SysLogHandler] - assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler] - assert [x for x in logger.handlers if type(x) == FTBufferingHandler] + assert [x for x in logger.handlers if isinstance(x, logging.handlers.SysLogHandler)] + assert [x for x in logger.handlers if isinstance(x, FTStdErrStreamHandler)] + assert [x for x in logger.handlers if isinstance(x, FTBufferingHandler)] # setting up logging again should NOT cause the loggers to be added a second time. setup_logging(config) assert len(logger.handlers) == 3 @@ -86,9 +86,9 @@ def test_set_loggers_Filehandler(tmp_path): setup_logging_pre() setup_logging(config) assert len(logger.handlers) == 3 - assert [x for x in logger.handlers if type(x) == logging.handlers.RotatingFileHandler] - assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler] - assert [x for x in logger.handlers if type(x) == FTBufferingHandler] + assert [x for x in logger.handlers if isinstance(x, logging.handlers.RotatingFileHandler)] + assert [x for x in logger.handlers if isinstance(x, FTStdErrStreamHandler)] + assert [x for x in logger.handlers if isinstance(x, FTBufferingHandler)] # setting up logging again should NOT cause the loggers to be added a second time. setup_logging(config) assert len(logger.handlers) == 3 @@ -112,7 +112,7 @@ def test_set_loggers_journald(mocker): setup_logging(config) assert len(logger.handlers) == 3 assert [x for x in logger.handlers if type(x).__name__ == "JournaldLogHandler"] - assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler] + assert [x for x in logger.handlers if isinstance(x, FTStdErrStreamHandler)] # reset handlers to not break pytest logger.handlers = orig_handlers diff --git a/tests/test_main.py b/tests/test_main.py index bdb3c2bba..442e6e1d7 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -67,12 +67,12 @@ def test_main_fatal_exception(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.init_db', MagicMock()) - args = ['trade', '-c', 'config_examples/config_bittrex.example.json'] + args = ['trade', '-c', 'tests/testdata/testconfigs/main_test_config.json'] # Test Main + the KeyboardInterrupt exception with pytest.raises(SystemExit): main(args) - assert log_has('Using config: config_examples/config_bittrex.example.json ...', caplog) + assert log_has('Using config: tests/testdata/testconfigs/main_test_config.json ...', caplog) assert log_has('Fatal exception!', caplog) @@ -85,12 +85,12 @@ def test_main_keyboard_interrupt(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.wallets.Wallets.update', MagicMock()) mocker.patch('freqtrade.freqtradebot.init_db', MagicMock()) - args = ['trade', '-c', 'config_examples/config_bittrex.example.json'] + args = ['trade', '-c', 'tests/testdata/testconfigs/main_test_config.json'] # Test Main + the KeyboardInterrupt exception with pytest.raises(SystemExit): main(args) - assert log_has('Using config: config_examples/config_bittrex.example.json ...', caplog) + assert log_has('Using config: tests/testdata/testconfigs/main_test_config.json ...', caplog) assert log_has('SIGINT received, aborting ...', caplog) @@ -106,12 +106,12 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.init_db', MagicMock()) - args = ['trade', '-c', 'config_examples/config_bittrex.example.json'] + args = ['trade', '-c', 'tests/testdata/testconfigs/main_test_config.json'] # Test Main + the KeyboardInterrupt exception with pytest.raises(SystemExit): main(args) - assert log_has('Using config: config_examples/config_bittrex.example.json ...', caplog) + assert log_has('Using config: tests/testdata/testconfigs/main_test_config.json ...', caplog) assert log_has('Oh snap!', caplog) @@ -160,13 +160,13 @@ def test_main_reload_config(mocker, default_conf, caplog) -> None: args = Arguments([ 'trade', '-c', - 'config_examples/config_bittrex.example.json' + 'tests/testdata/testconfigs/main_test_config.json' ]).get_parsed_arg() worker = Worker(args=args, config=default_conf) with pytest.raises(SystemExit): - main(['trade', '-c', 'config_examples/config_bittrex.example.json']) + main(['trade', '-c', 'tests/testdata/testconfigs/main_test_config.json']) - assert log_has('Using config: config_examples/config_bittrex.example.json ...', caplog) + assert log_has('Using config: tests/testdata/testconfigs/main_test_config.json ...', caplog) assert worker_mock.call_count == 4 assert reconfigure_mock.call_count == 1 assert isinstance(worker.freqtrade, FreqtradeBot) @@ -187,7 +187,7 @@ def test_reconfigure(mocker, default_conf) -> None: args = Arguments([ 'trade', '-c', - 'config_examples/config_bittrex.example.json' + 'tests/testdata/testconfigs/main_test_config.json' ]).get_parsed_arg() worker = Worker(args=args, config=default_conf) freqtrade = worker.freqtrade diff --git a/tests/test_misc.py b/tests/test_misc.py index 7de1adbbc..c9a196259 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -7,36 +7,12 @@ from unittest.mock import MagicMock import pandas as pd import pytest -from freqtrade.misc import (dataframe_to_json, decimals_per_coin, deep_merge_dicts, file_dump_json, - file_load_json, is_file_in_dir, json_to_dataframe, pair_to_filename, - parse_db_uri_for_logging, plural, round_coin_value, safe_value_fallback, +from freqtrade.misc import (dataframe_to_json, deep_merge_dicts, file_dump_json, file_load_json, + is_file_in_dir, json_to_dataframe, pair_to_filename, + parse_db_uri_for_logging, plural, safe_value_fallback, safe_value_fallback2) -def test_decimals_per_coin(): - assert decimals_per_coin('USDT') == 3 - assert decimals_per_coin('EUR') == 3 - assert decimals_per_coin('BTC') == 8 - assert decimals_per_coin('ETH') == 5 - - -def test_round_coin_value(): - assert round_coin_value(222.222222, 'USDT') == '222.222 USDT' - assert round_coin_value(222.2, 'USDT', keep_trailing_zeros=True) == '222.200 USDT' - assert round_coin_value(222.2, 'USDT') == '222.2 USDT' - assert round_coin_value(222.12745, 'EUR') == '222.127 EUR' - assert round_coin_value(0.1274512123, 'BTC') == '0.12745121 BTC' - assert round_coin_value(0.1274512123, 'ETH') == '0.12745 ETH' - - assert round_coin_value(222.222222, 'USDT', False) == '222.222' - assert round_coin_value(222.2, 'USDT', False) == '222.2' - assert round_coin_value(222.00, 'USDT', False) == '222' - assert round_coin_value(222.12745, 'EUR', False) == '222.127' - assert round_coin_value(0.1274512123, 'BTC', False) == '0.12745121' - assert round_coin_value(0.1274512123, 'ETH', False) == '0.12745' - assert round_coin_value(222.2, 'USDT', False, True) == '222.200' - - def test_file_dump_json(mocker) -> None: file_open = mocker.patch('freqtrade.misc.Path.open', MagicMock()) json_dump = mocker.patch('rapidjson.dump', MagicMock()) diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 377caf59c..7f80a8588 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -377,7 +377,7 @@ def test_start_plot_dataframe(mocker): aup = mocker.patch("freqtrade.plot.plotting.load_and_plot_trades", MagicMock()) args = [ "plot-dataframe", - "--config", "config_examples/config_bittrex.example.json", + "--config", "tests/testdata/testconfigs/main_test_config.json", "--pairs", "ETH/BTC" ] start_plot_dataframe(get_args(args)) @@ -420,7 +420,7 @@ def test_start_plot_profit(mocker): aup = mocker.patch("freqtrade.plot.plotting.plot_profit", MagicMock()) args = [ "plot-profit", - "--config", "config_examples/config_bittrex.example.json", + "--config", "tests/testdata/testconfigs/main_test_config.json", "--pairs", "ETH/BTC" ] start_plot_profit(get_args(args)) diff --git a/config_examples/config_bittrex.example.json b/tests/testdata/testconfigs/main_test_config.json similarity index 98% rename from config_examples/config_bittrex.example.json rename to tests/testdata/testconfigs/main_test_config.json index 882b92edc..0af5816dc 100644 --- a/config_examples/config_bittrex.example.json +++ b/tests/testdata/testconfigs/main_test_config.json @@ -29,7 +29,7 @@ "order_book_top": 1 }, "exchange": { - "name": "bittrex", + "name": "binance", "key": "your_exchange_key", "secret": "your_exchange_secret", "ccxt_config": {}, diff --git a/tests/test_binance_mig.py b/tests/utils/test_binance_mig.py similarity index 75% rename from tests/test_binance_mig.py rename to tests/utils/test_binance_mig.py index b7c821a5a..b509b7320 100644 --- a/tests/test_binance_mig.py +++ b/tests/utils/test_binance_mig.py @@ -1,11 +1,10 @@ - - import shutil import pytest from freqtrade.persistence import Trade -from freqtrade.util.binance_mig import migrate_binance_futures_data, migrate_binance_futures_names +from freqtrade.util.migrations import (migrate_binance_futures_data, migrate_binance_futures_names, + migrate_data) from tests.conftest import create_mock_trades_usdt, log_has @@ -55,3 +54,13 @@ def test_binance_mig_db_conversion(default_conf_usdt, fee, caplog): default_conf_usdt['trading_mode'] = 'futures' migrate_binance_futures_names(default_conf_usdt) assert log_has('Migrating binance futures pairs in database.', caplog) + + +def test_migration_wrapper(default_conf_usdt, mocker): + default_conf_usdt['trading_mode'] = 'futures' + binmock = mocker.patch('freqtrade.util.migrations.migrate_binance_futures_data') + funding_mock = mocker.patch('freqtrade.util.migrations.migrate_funding_fee_timeframe') + migrate_data(default_conf_usdt) + + assert binmock.call_count == 1 + assert funding_mock.call_count == 1 diff --git a/tests/utils/test_formatters.py b/tests/utils/test_formatters.py new file mode 100644 index 000000000..2a989ce81 --- /dev/null +++ b/tests/utils/test_formatters.py @@ -0,0 +1,37 @@ +from freqtrade.util import decimals_per_coin, fmt_coin, round_value + + +def test_decimals_per_coin(): + assert decimals_per_coin('USDT') == 3 + assert decimals_per_coin('EUR') == 3 + assert decimals_per_coin('BTC') == 8 + assert decimals_per_coin('ETH') == 5 + + +def test_fmt_coin(): + assert fmt_coin(222.222222, 'USDT') == '222.222 USDT' + assert fmt_coin(222.2, 'USDT', keep_trailing_zeros=True) == '222.200 USDT' + assert fmt_coin(222.2, 'USDT') == '222.2 USDT' + assert fmt_coin(222.12745, 'EUR') == '222.127 EUR' + assert fmt_coin(0.1274512123, 'BTC') == '0.12745121 BTC' + assert fmt_coin(0.1274512123, 'ETH') == '0.12745 ETH' + + assert fmt_coin(222.222222, 'USDT', False) == '222.222' + assert fmt_coin(222.2, 'USDT', False) == '222.2' + assert fmt_coin(222.00, 'USDT', False) == '222' + assert fmt_coin(222.12745, 'EUR', False) == '222.127' + assert fmt_coin(0.1274512123, 'BTC', False) == '0.12745121' + assert fmt_coin(0.1274512123, 'ETH', False) == '0.12745' + assert fmt_coin(222.2, 'USDT', False, True) == '222.200' + + +def test_round_value(): + + assert round_value(222.222222, 3) == '222.222' + assert round_value(222.2, 3) == '222.2' + assert round_value(222.00, 3) == '222' + assert round_value(222.12745, 3) == '222.127' + assert round_value(0.1274512123, 8) == '0.12745121' + assert round_value(0.1274512123, 5) == '0.12745' + assert round_value(222.2, 3, True) == '222.200' + assert round_value(222.2, 0, True) == '222' diff --git a/tests/utils/test_funding_rate_migration.py b/tests/utils/test_funding_rate_migration.py new file mode 100644 index 000000000..ccb8435cf --- /dev/null +++ b/tests/utils/test_funding_rate_migration.py @@ -0,0 +1,29 @@ +from shutil import copytree + +from freqtrade.util.migrations import migrate_funding_fee_timeframe + + +def test_migrate_funding_rate_timeframe(default_conf_usdt, tmp_path, testdatadir): + + copytree(testdatadir / 'futures', tmp_path / 'futures') + file_4h = tmp_path / 'futures' / 'XRP_USDT_USDT-4h-funding_rate.feather' + file_8h = tmp_path / 'futures' / 'XRP_USDT_USDT-8h-funding_rate.feather' + file_1h = tmp_path / 'futures' / 'XRP_USDT_USDT-1h-futures.feather' + file_8h.rename(file_4h) + assert file_1h.exists() + assert file_4h.exists() + assert not file_8h.exists() + + default_conf_usdt['datadir'] = tmp_path + + # Inactive on spot trading ... + migrate_funding_fee_timeframe(default_conf_usdt, None) + + default_conf_usdt['trading_mode'] = 'futures' + + migrate_funding_fee_timeframe(default_conf_usdt, None) + + assert not file_4h.exists() + assert file_8h.exists() + # futures files is untouched. + assert file_1h.exists()