diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 20ef27f0f..7c0655b20 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,14 +2,16 @@ Thank you for sending your pull request. But first, have you included unit tests, and is your code PEP8 conformant? [More details](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) ## Summary + Explain in one sentence the goal of this PR Solve the issue: #___ ## Quick changelog -- -- +- +- ## What's new? + *Explain in details what this PR solve or improve. You can include visuals.* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb767efb1..228a60389 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,7 +87,7 @@ jobs: run: | cp config_examples/config_bittrex.example.json config.json freqtrade create-userdir --userdir user_data - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all + freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all - name: Flake8 run: | @@ -180,7 +180,7 @@ jobs: run: | cp config_examples/config_bittrex.example.json config.json freqtrade create-userdir --userdir user_data - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all + freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all - name: Flake8 run: | @@ -247,7 +247,7 @@ jobs: run: | cp config_examples/config_bittrex.example.json config.json freqtrade create-userdir --userdir user_data - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all + freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all - name: Flake8 run: | diff --git a/.travis.yml b/.travis.yml index f2a6d508d..15c174bfe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,7 @@ jobs: - script: - cp config_examples/config_bittrex.example.json config.json - freqtrade create-userdir --userdir user_data - - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily + - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily name: hyperopt - script: flake8 name: flake8 diff --git a/Dockerfile b/Dockerfile index 804b1086b..f7e26efe3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.6-slim-buster as base +FROM python:3.9.7-slim-buster as base # Setup env ENV LANG C.UTF-8 @@ -13,7 +13,7 @@ RUN mkdir /freqtrade \ && apt-get update \ && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-serial-dev \ && apt-get clean \ - && useradd -u 1000 -G sudo -U -m ftuser \ + && useradd -u 1000 -G sudo -U -m -s /bin/bash ftuser \ && chown ftuser:ftuser /freqtrade \ # Allow sudoers && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers diff --git a/README.md b/README.md index 309fab94b..01effd7bc 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ Please read the [exchange specific notes](docs/exchanges.md) to learn about even - [X] [Bittrex](https://bittrex.com/) - [X] [Kraken](https://kraken.com/) - [X] [FTX](https://ftx.com) +- [X] [Gate.io](https://www.gate.io/ref/6266643) - [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ ### Community tested @@ -78,22 +79,22 @@ For any other type of installation please refer to [Installation doc](https://ww ``` usage: freqtrade [-h] [-V] - {trade,create-userdir,new-config,new-hyperopt,new-strategy,download-data,convert-data,convert-trade-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,plot-dataframe,plot-profit} + {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver} ... Free, open source crypto trading bot positional arguments: - {trade,create-userdir,new-config,new-hyperopt,new-strategy,download-data,convert-data,convert-trade-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,plot-dataframe,plot-profit} + {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver} trade Trade module. create-userdir Create user-data directory. new-config Create new config - new-hyperopt Create new hyperopt new-strategy Create new strategy download-data Download backtesting data. convert-data Convert candle (OHLCV) data from one format to another. convert-trade-data Convert trade data from one format to another. + list-data List downloaded data. backtesting Backtesting module. edge Edge module. hyperopt Hyperopt module. @@ -107,8 +108,10 @@ positional arguments: list-timeframes Print available timeframes for the exchange. show-trades Show trades. test-pairlist Test your pairlist configuration. + install-ui Install FreqUI plot-dataframe Plot candles with indicators. plot-profit Generate plot showing profits. + webserver Webserver module. optional arguments: -h, --help show this help message and exit diff --git a/build_helpers/install_ta-lib.sh b/build_helpers/install_ta-lib.sh index dd87cf105..d12b16364 100755 --- a/build_helpers/install_ta-lib.sh +++ b/build_helpers/install_ta-lib.sh @@ -12,9 +12,12 @@ if [ ! -f "${INSTALL_LOC}/lib/libta_lib.a" ]; then && curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' -o config.sub \ && ./configure --prefix=${INSTALL_LOC}/ \ && make -j$(nproc) \ - && which sudo && sudo make install || make install \ - && cd .. + && which sudo && sudo make install || make install + if [ -x "$(command -v apt-get)" ]; then + echo "Updating library path using ldconfig" + sudo ldconfig + fi + cd .. && rm -rf ./ta-lib/ else echo "TA-lib already installed, skipping installation" fi -# && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \ diff --git a/config_examples/config_full.example.json b/config_examples/config_full.example.json index d0f3f0df6..c415d70b0 100644 --- a/config_examples/config_full.example.json +++ b/config_examples/config_full.example.json @@ -149,7 +149,9 @@ }, "sell_fill": "on", "buy_cancel": "on", - "sell_cancel": "on" + "sell_cancel": "on", + "protection_trigger": "off", + "protection_trigger_global": "on" }, "reload": true, "balance_dust_level": 0.01 diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index 5e71df67c..f5a52ff49 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -67,10 +67,10 @@ Currently, the arguments are: This function needs to return a floating point number (`float`). Smaller numbers will be interpreted as better results. The parameters and balancing for this is up to you. !!! Note - This function is called once per iteration - so please make sure to have this as optimized as possible to not slow hyperopt down unnecessarily. + This function is called once per epoch - so please make sure to have this as optimized as possible to not slow hyperopt down unnecessarily. -!!! Note - Please keep the arguments `*args` and `**kwargs` in the interface to allow us to extend this interface later. +!!! Note "`*args` and `**kwargs`" + Please keep the arguments `*args` and `**kwargs` in the interface to allow us to extend this interface in the future. ## Overriding pre-defined spaces @@ -80,10 +80,56 @@ To override a pre-defined space (`roi_space`, `generate_roi_table`, `stoploss_sp class MyAwesomeStrategy(IStrategy): class HyperOpt: # Define a custom stoploss space. - def stoploss_space(self): + def stoploss_space(): return [SKDecimal(-0.05, -0.01, decimals=3, name='stoploss')] + + # Define custom ROI space + def roi_space() -> List[Dimension]: + return [ + Integer(10, 120, name='roi_t1'), + Integer(10, 60, name='roi_t2'), + Integer(10, 40, name='roi_t3'), + SKDecimal(0.01, 0.04, decimals=3, name='roi_p1'), + SKDecimal(0.01, 0.07, decimals=3, name='roi_p2'), + SKDecimal(0.01, 0.20, decimals=3, name='roi_p3'), + ] ``` +!!! Note + All overrides are optional and can be mixed/matched as necessary. + +### Overriding Base estimator + +You can define your own estimator for Hyperopt by implementing `generate_estimator()` in the Hyperopt subclass. + +```python +class MyAwesomeStrategy(IStrategy): + class HyperOpt: + def generate_estimator(): + return "RF" + +``` + +Possible values are either one of "GP", "RF", "ET", "GBRT" (Details can be found in the [scikit-optimize documentation](https://scikit-optimize.github.io/)), or "an instance of a class that inherits from `RegressorMixin` (from sklearn) and where the `predict` method has an optional `return_std` argument, which returns `std(Y | x)` along with `E[Y | x]`". + +Some research will be necessary to find additional Regressors. + +Example for `ExtraTreesRegressor` ("ET") with additional parameters: + +```python +class MyAwesomeStrategy(IStrategy): + class HyperOpt: + def generate_estimator(): + from skopt.learning import ExtraTreesRegressor + # Corresponds to "ET" - but allows additional parameters. + return ExtraTreesRegressor(n_estimators=100) + +``` + +!!! Note + While custom estimators can be provided, it's up to you as User to do research on possible parameters and analyze / understand which ones should be used. + If you're unsure about this, best use one of the Defaults (`"ET"` has proven to be the most versatile) without further parameters. + ## Space options For the additional spaces, scikit-optimize (in combination with Freqtrade) provides the following space types: @@ -105,281 +151,3 @@ from freqtrade.optimize.space import Categorical, Dimension, Integer, SKDecimal, Assuming the definition of a rather small space (`SKDecimal(0.10, 0.15, decimals=2, name='xxx')`) - SKDecimal will have 5 possibilities (`[0.10, 0.11, 0.12, 0.13, 0.14, 0.15]`). A corresponding real space `Real(0.10, 0.15 name='xxx')` on the other hand has an almost unlimited number of possibilities (`[0.10, 0.010000000001, 0.010000000002, ... 0.014999999999, 0.01500000000]`). - ---- - -## Legacy Hyperopt - -This Section explains the configuration of an explicit Hyperopt file (separate to the strategy). - -!!! Warning "Deprecated / legacy mode" - Since the 2021.4 release you no longer have to write a separate hyperopt class, but all strategies can be hyperopted. - Please read the [main hyperopt page](hyperopt.md) for more details. - -### Prepare hyperopt file - -Configuring an explicit hyperopt file is similar to writing your own strategy, and many tasks will be similar. - -!!! Tip "About this page" - For this page, we will be using a fictional strategy called `AwesomeStrategy` - which will be optimized using the `AwesomeHyperopt` class. - -#### Create a Custom Hyperopt File - -The simplest way to get started is to use the following command, which will create a new hyperopt file from a template, which will be located under `user_data/hyperopts/AwesomeHyperopt.py`. - -Let assume you want a hyperopt file `AwesomeHyperopt.py`: - -``` bash -freqtrade new-hyperopt --hyperopt AwesomeHyperopt -``` - -#### Legacy Hyperopt checklist - -Checklist on all tasks / possibilities in hyperopt - -Depending on the space you want to optimize, only some of the below are required: - -* fill `buy_strategy_generator` - for buy signal optimization -* fill `indicator_space` - for buy signal optimization -* fill `sell_strategy_generator` - for sell signal optimization -* fill `sell_indicator_space` - for sell signal optimization - -!!! Note - `populate_indicators` needs to create all indicators any of thee spaces may use, otherwise hyperopt will not work. - -Optional in hyperopt - can also be loaded from a strategy (recommended): - -* `populate_indicators` - fallback to create indicators -* `populate_buy_trend` - fallback if not optimizing for buy space. should come from strategy -* `populate_sell_trend` - fallback if not optimizing for sell space. should come from strategy - -!!! Note - You always have to provide a strategy to Hyperopt, even if your custom Hyperopt class contains all methods. - Assuming the optional methods are not in your hyperopt file, please use `--strategy AweSomeStrategy` which contains these methods so hyperopt can use these methods instead. - -Rarely you may also need to override: - -* `roi_space` - for custom ROI optimization (if you need the ranges for the ROI parameters in the optimization hyperspace that differ from default) -* `generate_roi_table` - for custom ROI optimization (if you need the ranges for the values in the ROI table that differ from default or the number of entries (steps) in the ROI table which differs from the default 4 steps) -* `stoploss_space` - for custom stoploss optimization (if you need the range for the stoploss parameter in the optimization hyperspace that differs from default) -* `trailing_space` - for custom trailing stop optimization (if you need the ranges for the trailing stop parameters in the optimization hyperspace that differ from default) - -#### Defining a buy signal optimization - -Let's say you are curious: should you use MACD crossings or lower Bollinger -Bands to trigger your buys. And you also wonder should you use RSI or ADX to -help with those buy decisions. If you decide to use RSI or ADX, which values -should I use for them? So let's use hyperparameter optimization to solve this -mystery. - -We will start by defining a search space: - -```python - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching strategy parameters - """ - return [ - Integer(20, 40, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal'], name='trigger') - ] -``` - -Above definition says: I have five parameters I want you to randomly combine -to find the best combination. Two of them are integer values (`adx-value` and `rsi-value`) and I want you test in the range of values 20 to 40. -Then we have three category variables. First two are either `True` or `False`. -We use these to either enable or disable the ADX and RSI guards. -The last one we call `trigger` and use it to decide which buy trigger we want to use. - -So let's write the buy strategy generator using these values: - -```python - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by Hyperopt. - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - conditions = [] - # GUARDS AND TRENDS - if 'adx-enabled' in params and params['adx-enabled']: - conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi-enabled' in params and params['rsi-enabled']: - conditions.append(dataframe['rsi'] < params['rsi-value']) - - # TRIGGERS - if 'trigger' in params: - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - - # Check that volume is not 0 - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend -``` - -Hyperopt will now call `populate_buy_trend()` many times (`epochs`) with different value combinations. -It will use the given historical data and make buys based on the buy signals generated with the above function. -Based on the results, hyperopt will tell you which parameter combination produced the best results (based on the configured [loss function](#loss-functions)). - -!!! Note - The above setup expects to find ADX, RSI and Bollinger Bands in the populated indicators. - When you want to test an indicator that isn't used by the bot currently, remember to - add it to the `populate_indicators()` method in your strategy or hyperopt file. - -#### Sell optimization - -Similar to the buy-signal above, sell-signals can also be optimized. -Place the corresponding settings into the following methods - -* Inside `sell_indicator_space()` - the parameters hyperopt shall be optimizing. -* Within `sell_strategy_generator()` - populate the nested method `populate_sell_trend()` to apply the parameters. - -The configuration and rules are the same than for buy signals. -To avoid naming collisions in the search-space, please prefix all sell-spaces with `sell-`. - -### Execute Hyperopt - -Once you have updated your hyperopt configuration you can run it. -Because hyperopt tries a lot of combinations to find the best parameters it will take time to get a good result. More time usually results in better results. - -We strongly recommend to use `screen` or `tmux` to prevent any connection loss. - -```bash -freqtrade hyperopt --config config.json --hyperopt --hyperopt-loss --strategy -e 500 --spaces all -``` - -Use `` as the name of the custom hyperopt used. - -The `-e` option will set how many evaluations hyperopt will do. Since hyperopt uses Bayesian search, running too many epochs at once may not produce greater results. Experience has shown that best results are usually not improving much after 500-1000 epochs. -Doing multiple runs (executions) with a few 1000 epochs and different random state will most likely produce different results. - -The `--spaces all` option determines that all possible parameters should be optimized. Possibilities are listed below. - -!!! Note - Hyperopt will store hyperopt results with the timestamp of the hyperopt start time. - Reading commands (`hyperopt-list`, `hyperopt-show`) can use `--hyperopt-filename ` to read and display older hyperopt results. - You can find a list of filenames with `ls -l user_data/hyperopt_results/`. - -#### Running Hyperopt using methods from a strategy - -Hyperopt can reuse `populate_indicators`, `populate_buy_trend`, `populate_sell_trend` from your strategy, assuming these methods are **not** in your custom hyperopt file, and a strategy is provided. - -```bash -freqtrade hyperopt --hyperopt AwesomeHyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy AwesomeStrategy -``` - -### Understand the Hyperopt Result - -Once Hyperopt is completed you can use the result to create a new strategy. -Given the following result from hyperopt: - -``` -Best result: - - 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722%). Avg duration 180.4 mins. Objective: 1.94367 - -Buy hyperspace params: -{ 'adx-value': 44, - 'rsi-value': 29, - 'adx-enabled': False, - 'rsi-enabled': True, - 'trigger': 'bb_lower'} -``` - -You should understand this result like: - -* The buy trigger that worked best was `bb_lower`. -* You should not use ADX because `adx-enabled: False`) -* You should **consider** using the RSI indicator (`rsi-enabled: True` and the best value is `29.0` (`rsi-value: 29.0`) - -You have to look inside your strategy file into `buy_strategy_generator()` -method, what those values match to. - -So for example you had `rsi-value: 29.0` so we would look at `rsi`-block, that translates to the following code block: - -```python -(dataframe['rsi'] < 29.0) -``` - -Translating your whole hyperopt result as the new buy-signal would then look like: - -```python -def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: - dataframe.loc[ - ( - (dataframe['rsi'] < 29.0) & # rsi-value - dataframe['close'] < dataframe['bb_lowerband'] # trigger - ), - 'buy'] = 1 - return dataframe -``` - -### Validate backtesting results - -Once the optimized parameters and conditions have been implemented into your strategy, you should backtest the strategy to make sure everything is working as expected. - -To achieve same results (number of trades, their durations, profit, etc.) than during Hyperopt, please use same configuration and parameters (timerange, timeframe, ...) used for hyperopt `--dmmp`/`--disable-max-market-positions` and `--eps`/`--enable-position-stacking` for Backtesting. - -Should results don't match, please double-check to make sure you transferred all conditions correctly. -Pay special care to the stoploss (and trailing stoploss) parameters, as these are often set in configuration files, which override changes to the strategy. -You should also carefully review the log of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss` or `trailing_stop`). - -### Sharing methods with your strategy - -Hyperopt classes provide access to the Strategy via the `strategy` class attribute. -This can be a great way to reduce code duplication if used correctly, but will also complicate usage for inexperienced users. - -``` python -from pandas import DataFrame -from freqtrade.strategy.interface import IStrategy -import freqtrade.vendor.qtpylib.indicators as qtpylib - -class MyAwesomeStrategy(IStrategy): - - buy_params = { - 'rsi-value': 30, - 'adx-value': 35, - } - - def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - return self.buy_strategy_generator(self.buy_params, dataframe, metadata) - - @staticmethod - def buy_strategy_generator(params, dataframe: DataFrame, metadata: dict) -> DataFrame: - dataframe.loc[ - ( - qtpylib.crossed_above(dataframe['rsi'], params['rsi-value']) & - dataframe['adx'] > params['adx-value']) & - dataframe['volume'] > 0 - ) - , 'buy'] = 1 - return dataframe - -class MyAwesomeHyperOpt(IHyperOpt): - ... - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by Hyperopt. - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - # Call strategy's buy strategy generator - return self.StrategyClass.buy_strategy_generator(params, dataframe, metadata) - - return populate_buy_trend -``` diff --git a/docs/backtesting.md b/docs/backtesting.md index f750e0c4c..66e682745 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -18,6 +18,7 @@ usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-p PAIRS [PAIRS ...]] [--eps] [--dmmp] [--enable-protections] [--dry-run-wallet DRY_RUN_WALLET] + [--timeframe-detail TIMEFRAME_DETAIL] [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] [--export {none,trades}] [--export-filename PATH] @@ -55,6 +56,9 @@ optional arguments: --dry-run-wallet DRY_RUN_WALLET, --starting-balance DRY_RUN_WALLET Starting balance, used for backtesting / hyperopt and dry-runs. + --timeframe-detail TIMEFRAME_DETAIL + Specify detail timeframe for backtesting (`1m`, `5m`, + `30m`, `1h`, `1d`). --strategy-list STRATEGY_LIST [STRATEGY_LIST ...] Provide a space-separated list of strategies to backtest. Please note that ticker-interval needs to be @@ -425,7 +429,12 @@ It contains some useful key metrics about performance of your strategy on backte - `Drawdown Start` / `Drawdown End`: Start and end datetime for this largest drawdown (can also be visualized via the `plot-dataframe` sub-command). - `Market change`: Change of the market during the backtest period. Calculated as average of all pairs changes from the first to the last candle using the "close" column. -### Assumptions made by backtesting +### Further backtest-result analysis + +To further analyze your backtest results, you can [export the trades](#exporting-trades-to-file). +You can then load the trades to perform further analysis as shown in our [data analysis](data-analysis.md#backtesting) backtesting section. + +## Assumptions made by backtesting Since backtesting lacks some detailed information about what happens within a candle, it needs to take a few assumptions: @@ -456,10 +465,30 @@ Also, keep in mind that past results don't guarantee future success. In addition to the above assumptions, strategy authors should carefully read the [Common Mistakes](strategy-customization.md#common-mistakes-when-developing-strategies) section, to avoid using data in backtesting which is not available in real market conditions. -### Further backtest-result analysis +### Improved backtest accuracy -To further analyze your backtest results, you can [export the trades](#exporting-trades-to-file). -You can then load the trades to perform further analysis as shown in our [data analysis](data-analysis.md#backtesting) backtesting section. +One big limitation of backtesting is it's inability to know how prices moved intra-candle (was high before close, or viceversa?). +So assuming you run backtesting with a 1h timeframe, there will be 4 prices for that candle (Open, High, Low, Close). + +While backtesting does take some assumptions (read above) about this - this can never be perfect, and will always be biased in one way or the other. +To mitigate this, freqtrade can use a lower (faster) timeframe to simulate intra-candle movements. + +To utilize this, you can append `--timeframe-detail 5m` to your regular backtesting command. + +``` bash +freqtrade backtesting --strategy AwesomeStrategy --timeframe 1h --timeframe-detail 5m +``` + +This will load 1h data as well as 5m data for the timeframe. The strategy will be analyzed with the 1h timeframe - and for every "open trade candle" (candles where a trade is open) the 5m data will be used to simulate intra-candle movements. +All callback functions (`custom_sell()`, `custom_stoploss()`, ... ) will be running for each 5m candle once the trade is opened (so 12 times in the above example of 1h timeframe, and 5m detailed timeframe). + +`--timeframe-detail` must be smaller than the original timeframe, otherwise backtesting will fail to start. + +Obviously this will require more memory (5m data is bigger than 1h data), and will also impact runtime (depending on the amount of trades and trade durations). +Also, data must be available / downloaded already. + +!!! Tip + You can use this function as the last part of strategy development, to ensure your strategy is not exploiting one of the [backtesting assumptions](#assumptions-made-by-backtesting). Strategies that perform similarly well with this mode have a good chance to perform well in dry/live modes too (although only forward-testing (dry-mode) can really confirm a strategy). ## Backtesting multiple strategies diff --git a/docs/bot-basics.md b/docs/bot-basics.md index e7ff27040..80443a0bf 100644 --- a/docs/bot-basics.md +++ b/docs/bot-basics.md @@ -7,7 +7,7 @@ This page provides you some basic concepts on how Freqtrade works and operates. * **Strategy**: Your trading strategy, telling the bot what to do. * **Trade**: Open position. * **Open Order**: Order which is currently placed on the exchange, and is not yet complete. -* **Pair**: Tradable pair, usually in the format of Quote/Base (e.g. XRP/USDT). +* **Pair**: Tradable pair, usually in the format of Base/Quote (e.g. XRP/USDT). * **Timeframe**: Candle length to use (e.g. `"5m"`, `"1h"`, ...). * **Indicators**: Technical indicators (SMA, EMA, RSI, ...). * **Limit order**: Limit orders which execute at the defined limit price or better. diff --git a/docs/bot-usage.md b/docs/bot-usage.md index b65220722..c6a7f6103 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -12,22 +12,22 @@ This page explains the different parameters of the bot and how to run it. ``` usage: freqtrade [-h] [-V] - {trade,create-userdir,new-config,new-hyperopt,new-strategy,download-data,convert-data,convert-trade-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,plot-dataframe,plot-profit} + {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver} ... Free, open source crypto trading bot positional arguments: - {trade,create-userdir,new-config,new-hyperopt,new-strategy,download-data,convert-data,convert-trade-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,plot-dataframe,plot-profit} + {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver} trade Trade module. create-userdir Create user-data directory. new-config Create new config - new-hyperopt Create new hyperopt new-strategy Create new strategy download-data Download backtesting data. convert-data Convert candle (OHLCV) data from one format to another. convert-trade-data Convert trade data from one format to another. + list-data List downloaded data. backtesting Backtesting module. edge Edge module. hyperopt Hyperopt module. @@ -41,8 +41,10 @@ positional arguments: list-timeframes Print available timeframes for the exchange. show-trades Show trades. test-pairlist Test your pairlist configuration. + install-ui Install FreqUI plot-dataframe Plot candles with indicators. plot-profit Generate plot showing profits. + webserver Webserver module. optional arguments: -h, --help show this help message and exit diff --git a/docs/configuration.md b/docs/configuration.md index 09198e019..6ccea4c73 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -444,8 +444,8 @@ The possible values are: `gtc` (default), `fok` or `ioc`. ``` !!! Warning - This is ongoing work. For now, it is supported only for binance. - Please don't change the default value unless you know what you are doing and have researched the impact of using different values. + This is ongoing work. For now, it is supported only for binance and kucoin. + Please don't change the default value unless you know what you are doing and have researched the impact of using different values for your particular exchange. ### Exchange configuration diff --git a/docs/deprecated.md b/docs/deprecated.md index b7ad847e6..d86a7ac7a 100644 --- a/docs/deprecated.md +++ b/docs/deprecated.md @@ -38,3 +38,8 @@ Since only quoteVolume can be compared between assets, the other options (bidVol Using `order_book_min` and `order_book_max` used to allow stepping the orderbook and trying to find the next ROI slot - trying to place sell-orders early. As this does however increase risk and provides no benefit, it's been removed for maintainability purposes in 2021.7. + +### Legacy Hyperopt mode + +Using separate hyperopt files was deprecated in 2021.4 and was removed in 2021.9. +Please switch to the new [Parametrized Strategies](hyperopt.md) to benefit from the new hyperopt interface. diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index 1fa229225..2f350d207 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -149,6 +149,24 @@ You'll then also need to modify the `docker-compose.yml` file and uncomment the You can then run `docker-compose build` to build the docker image, and run it using the commands described above. +### Troubleshooting + +#### Docker on Windows + +* Error: `"Timestamp for this request is outside of the recvWindow."` + * The market api requests require a synchronized clock but the time in the docker container shifts a bit over time into the past. + To fix this issue temporarily you need to run `wsl --shutdown` and restart docker again (a popup on windows 10 will ask you to do so). + A permanent solution is either to host the docker container on a linux host or restart the wsl from time to time with the scheduler. + ``` + taskkill /IM "Docker Desktop.exe" /F + wsl --shutdown + start "" "C:\Program Files\Docker\Docker\Docker Desktop.exe" + ``` + +!!! Warning + Due to the above, we do not recommend the usage of docker on windows for production setups, but only for experimentation, datadownload and backtesting. + Best use a linux-VPS for running freqtrade reliably. + ## Plotting with docker-compose Commands `freqtrade plot-profit` and `freqtrade plot-dataframe` ([Documentation](plotting.md)) are available by changing the image to `*_plot` in your docker-compose.yml file. diff --git a/docs/edge.md b/docs/edge.md index 237ff36f6..4402d767f 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -3,7 +3,7 @@ The `Edge Positioning` module uses probability to calculate your win rate and risk reward ratio. It will use these statistics to control your strategy trade entry points, position size and, stoploss. !!! Warning - WHen using `Edge positioning` with a dynamic whitelist (VolumePairList), make sure to also use `AgeFilter` and set it to at least `calculate_since_number_of_days` to avoid problems with missing data. + When using `Edge positioning` with a dynamic whitelist (VolumePairList), make sure to also use `AgeFilter` and set it to at least `calculate_since_number_of_days` to avoid problems with missing data. !!! Note `Edge Positioning` only considers *its own* buy/sell/stoploss signals. It ignores the stoploss, trailing stoploss, and ROI settings in the strategy configuration file. diff --git a/docs/exchanges.md b/docs/exchanges.md index 5f54a524e..c0fbdc694 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -4,6 +4,8 @@ This page combines common gotchas and informations which are exchange-specific a ## Binance +Binance supports [time_in_force](configuration.md#understand-order_time_in_force). + !!! Tip "Stoploss on Exchange" Binance supports `stoploss_on_exchange` and uses stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it. @@ -56,6 +58,12 @@ Bittrex does not support market orders. If you have a message at the bot startup Bittrex also does not support `VolumePairlist` due to limited / split API constellation at the moment. Please use `StaticPairlist`. Other pairlists (other than `VolumePairlist`) should not be affected. +### Volume pairlist + +Bittrex does not support the direct usage of VolumePairList. This can however be worked around by using the advanced mode with `lookback_days: 1` (or more), which will emulate 24h volume. + +Read more in the [pairlist documentation](plugins.md#volumepairlist-advanced-mode). + ### Restricted markets Bittrex split its exchange into US and International versions. @@ -113,8 +121,12 @@ Kucoin requires a passphrase for each api key, you will therefore need to add th "key": "your_exchange_key", "secret": "your_exchange_secret", "password": "your_exchange_api_key_password", + // ... +} ``` +Kucoin supports [time_in_force](configuration.md#understand-order_time_in_force). + ### Kucoin Blacklists For Kucoin, please add `"KCS/"` to your blacklist to avoid issues. @@ -158,6 +170,8 @@ For example, to test the order type `FOK` with Kraken, and modify candle limit t "order_time_in_force": ["gtc", "fok"], "ohlcv_candle_limit": 200 } + //... +} ``` !!! Warning diff --git a/docs/faq.md b/docs/faq.md index b8a3a44d8..285625491 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -167,7 +167,7 @@ Since hyperopt uses Bayesian search, running for too many epochs may not produce It's therefore recommended to run between 500-1000 epochs over and over until you hit at least 10.000 epochs in total (or are satisfied with the result). You can best judge by looking at the results - if the bot keeps discovering better strategies, it's best to keep on going. ```bash -freqtrade hyperopt --hyperopt SampleHyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy SampleStrategy -e 1000 +freqtrade hyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy SampleStrategy -e 1000 ``` ### Why does it take a long time to run hyperopt? diff --git a/docs/hyperopt.md b/docs/hyperopt.md index c5a52553b..09d43939a 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -44,9 +44,8 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--data-format-ohlcv {json,jsongz,hdf5}] [--max-open-trades INT] [--stake-amount STAKE_AMOUNT] [--fee FLOAT] - [-p PAIRS [PAIRS ...]] [--hyperopt NAME] - [--hyperopt-path PATH] [--eps] [--dmmp] - [--enable-protections] + [-p PAIRS [PAIRS ...]] [--hyperopt-path PATH] + [--eps] [--dmmp] [--enable-protections] [--dry-run-wallet DRY_RUN_WALLET] [-e INT] [--spaces {all,buy,sell,roi,stoploss,trailing,protection,default} [{all,buy,sell,roi,stoploss,trailing,protection,default} ...]] [--print-all] [--no-color] [--print-json] [-j JOBS] @@ -73,10 +72,8 @@ optional arguments: -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] Limit command to these pairs. Pairs are space- separated. - --hyperopt NAME Specify hyperopt class name which will be used by the - bot. - --hyperopt-path PATH Specify additional lookup path for Hyperopt and - Hyperopt Loss functions. + --hyperopt-path PATH Specify additional lookup path for Hyperopt Loss + functions. --eps, --enable-position-stacking Allow buying the same pair multiple times (position stacking). @@ -456,7 +453,7 @@ class MyAwesomeStrategy(IStrategy): "only_per_pair": False }) - return protection + return prot def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: # ... @@ -558,7 +555,7 @@ For example, to use one month of data, pass `--timerange 20210101-20210201` (fro Full command: ```bash -freqtrade hyperopt --hyperopt --strategy --timerange 20210101-20210201 +freqtrade hyperopt --strategy --timerange 20210101-20210201 ``` ### Running Hyperopt with Smaller Search Space @@ -680,11 +677,11 @@ If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace f These ranges should be sufficient in most cases. The minutes in the steps (ROI dict keys) are scaled linearly depending on the timeframe used. The ROI values in the steps (ROI dict values) are scaled logarithmically depending on the timeframe used. -If you have the `generate_roi_table()` and `roi_space()` methods in your custom hyperopt file, remove them in order to utilize these adaptive ROI tables and the ROI hyperoptimization space generated by Freqtrade by default. +If you have the `generate_roi_table()` and `roi_space()` methods in your custom hyperopt, remove them in order to utilize these adaptive ROI tables and the ROI hyperoptimization space generated by Freqtrade by default. Override the `roi_space()` method if you need components of the ROI tables to vary in other ranges. Override the `generate_roi_table()` and `roi_space()` methods and implement your own custom approach for generation of the ROI tables during hyperoptimization if you need a different structure of the ROI tables or other amount of rows (steps). -A sample for these methods can be found in [sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py). +A sample for these methods can be found in the [overriding pre-defined spaces section](advanced-hyperopt.md#overriding-pre-defined-spaces). !!! Note "Reduced search space" To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs. @@ -726,7 +723,7 @@ If you are optimizing stoploss values, Freqtrade creates the 'stoploss' optimiza If you have the `stoploss_space()` method in your custom hyperopt file, remove it in order to utilize Stoploss hyperoptimization space generated by Freqtrade by default. -Override the `stoploss_space()` method and define the desired range in it if you need stoploss values to vary in other range during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py). +Override the `stoploss_space()` method and define the desired range in it if you need stoploss values to vary in other range during hyperoptimization. A sample for this method can be found in the [overriding pre-defined spaces section](advanced-hyperopt.md#overriding-pre-defined-spaces). !!! Note "Reduced search space" To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs. @@ -764,10 +761,10 @@ As stated in the comment, you can also use it as the values of the corresponding If you are optimizing trailing stop values, Freqtrade creates the 'trailing' optimization hyperspace for you. By default, the `trailing_stop` parameter is always set to True in that hyperspace, the value of the `trailing_only_offset_is_reached` vary between True and False, the values of the `trailing_stop_positive` and `trailing_stop_positive_offset` parameters vary in the ranges 0.02...0.35 and 0.01...0.1 correspondingly, which is sufficient in most cases. -Override the `trailing_space()` method and define the desired range in it if you need values of the trailing stop parameters to vary in other ranges during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py). +Override the `trailing_space()` method and define the desired range in it if you need values of the trailing stop parameters to vary in other ranges during hyperoptimization. A sample for this method can be found in the [overriding pre-defined spaces section](advanced-hyperopt.md#overriding-pre-defined-spaces). !!! Note "Reduced search space" - To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs. + To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#overriding-pre-defined-spaces) to change this to your needs. ### Reproducible results @@ -827,8 +824,8 @@ After you run Hyperopt for the desired amount of epochs, you can later list all Once the optimized strategy has been implemented into your strategy, you should backtest this strategy to make sure everything is working as expected. -To achieve same results (number of trades, their durations, profit, etc.) than during Hyperopt, please use same configuration and parameters (timerange, timeframe, ...) used for hyperopt `--dmmp`/`--disable-max-market-positions` and `--eps`/`--enable-position-stacking` for Backtesting. +To achieve same the results (number of trades, their durations, profit, etc.) as during Hyperopt, please use the same configuration and parameters (timerange, timeframe, ...) used for hyperopt `--dmmp`/`--disable-max-market-positions` and `--eps`/`--enable-position-stacking` for Backtesting. -Should results don't match, please double-check to make sure you transferred all conditions correctly. +Should results not match, please double-check to make sure you transferred all conditions correctly. Pay special care to the stoploss (and trailing stoploss) parameters, as these are often set in configuration files, which override changes to the strategy. You should also carefully review the log of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss` or `trailing_stop`). diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 6e23c9003..b612a4ddf 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -82,6 +82,8 @@ Filtering instances (not the first position in the list) will not apply any cach You can define a minimum volume with `min_value` - which will filter out pairs with a volume lower than the specified value in the specified timerange. +### VolumePairList Advanced mode + `VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3) and multiplies the typical price with every candle's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles. For convenience `lookback_days` can be specified, which will imply that 1d candles will be used for the lookback. In the example below the pairlist would be created based on the last 7 days: @@ -105,6 +107,24 @@ For convenience `lookback_days` can be specified, which will imply that 1d candl !!! Warning "Performance implications when using lookback range" If used in first position in combination with lookback, the computation of the range based volume can be time and resource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `VolumeFilter` to narrow the pairlist down for further range volume calculation. +??? Tip "Unsupported exchanges (Bittrex, Gemini)" + On some exchanges (like Bittrex and Gemini), regular VolumePairList does not work as the api does not natively provide 24h volume. This can be worked around by using candle data to build the volume. + To roughly simulate 24h volume, you can use the following configuration. + Please note that These pairlists will only refresh once per day. + + ```json + "pairlists": [ + { + "method": "VolumePairList", + "number_assets": 20, + "sort_key": "quoteVolume", + "min_value": 0, + "refresh_period": 86400, + "lookback_days": 1 + } + ], + ``` + More sophisticated approach can be used, by using `lookback_timeframe` for candle size and `lookback_period` which specifies the amount of candles. This example will build the volume pairs based on a rolling period of 3 days of 1h candles: ```json @@ -145,6 +165,7 @@ Example to remove the first 10 pairs from the pairlist: ```json "pairlists": [ + // ... { "method": "OffsetFilter", "offset": 10 @@ -170,6 +191,19 @@ Sorts pairs by past trade performance, as follows: Trade count is used as a tie breaker. +You can use the `minutes` parameter to only consider performance of the past X minutes (rolling window). +Not defining this parameter (or setting it to 0) will use all-time performance. + +```json +"pairlists": [ + // ... + { + "method": "PerformanceFilter", + "minutes": 1440 // rolling 24h + } +], +``` + !!! Note `PerformanceFilter` does not support backtesting mode. diff --git a/docs/index.md b/docs/index.md index fd3b8f224..7735117e2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -40,6 +40,7 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual, - [X] [Bittrex](https://bittrex.com/) - [X] [FTX](https://ftx.com) - [X] [Kraken](https://kraken.com/) +- [X] [Gate.io](https://www.gate.io/ref/6266643) - [ ] [potentially many others through ccxt](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ ### Community tested diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 8fa7341c9..9b7c12a43 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ mkdocs==1.2.2 -mkdocs-material==7.2.4 +mkdocs-material==7.3.0 mdx_truly_sane_lists==1.2 pymdown-extensions==8.2 diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 4409af6ea..b0d1937f6 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -288,6 +288,12 @@ Stoploss values returned from `custom_stoploss()` always specify a percentage re The helper function [`stoploss_from_open()`](strategy-customization.md#stoploss_from_open) can be used to convert from an open price relative stop, to a current price relative stop which can be returned from `custom_stoploss()`. +### Calculating stoploss percentage from absolute price + +Stoploss values returned from `custom_stoploss()` always specify a percentage relative to `current_rate`. In order to set a stoploss at specified absolute price level, we need to use `stop_rate` to calculate what percentage relative to the `current_rate` will give you the same result as if the percentage was specified from the open price. + +The helper function [`stoploss_from_absolute()`](strategy-customization.md#stoploss_from_absolute) can be used to convert from an absolute price, to a current price relative stop which can be returned from `custom_stoploss()`. + #### Stepped stoploss Instead of continuously trailing behind the current price, this example sets fixed stoploss price levels based on the current profit. @@ -695,3 +701,33 @@ The variable 'content', will contain the strategy file in a BASE64 encoded form. ``` Please ensure that 'NameOfStrategy' is identical to the strategy name! + +## Performance warning + +When executing a strategy, one can sometimes be greeted by the following in the logs + +> PerformanceWarning: DataFrame is highly fragmented. + +This is a warning from [`pandas`](https://github.com/pandas-dev/pandas) and as the warning continues to say: +use `pd.concat(axis=1)`. +This can have slight performance implications, which are usually only visible during hyperopt (when optimizing an indicator). + +For example: + +```python +for val in self.buy_ema_short.range: + dataframe[f'ema_short_{val}'] = ta.EMA(dataframe, timeperiod=val) +``` + +should be rewritten to + +```python +frames = [dataframe] +for val in self.buy_ema_short.range: + frames.append({ + f'ema_short_{val}': ta.EMA(dataframe, timeperiod=val) + }) + +# Append columns to existing dataframe +merged_frame = pd.concat(frames, axis=1) +``` diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index cfea60d22..110365208 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -639,6 +639,167 @@ Stoploss values returned from `custom_stoploss` must specify a percentage relati Full examples can be found in the [Custom stoploss](strategy-advanced.md#custom-stoploss) section of the Documentation. +!!! Note + Providing invalid input to `stoploss_from_open()` may produce "CustomStoploss function did not return valid stoploss" warnings. + This may happen if `current_profit` parameter is below specified `open_relative_stop`. Such situations may arise when closing trade + is blocked by `confirm_trade_exit()` method. Warnings can be solved by never blocking stop loss sells by checking `sell_reason` in + `confirm_trade_exit()`, or by using `return stoploss_from_open(...) or 1` idiom, which will request to not change stop loss when + `current_profit < open_relative_stop`. + +### *stoploss_from_absolute()* + +In some situations it may be confusing to deal with stops relative to current rate. Instead, you may define a stoploss level using an absolute price. + +??? Example "Returning a stoploss using absolute price from the custom stoploss function" + + If we want to trail a stop price at 2xATR below current proce we can call `stoploss_from_absolute(current_rate - (candle['atr'] * 2), current_rate)`. + + ``` python + + from datetime import datetime + from freqtrade.persistence import Trade + from freqtrade.strategy import IStrategy, stoploss_from_open + + class AwesomeStrategy(IStrategy): + + use_custom_stoploss = True + + def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['atr'] = ta.ATR(dataframe, timeperiod=14) + return dataframe + + def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, + current_rate: float, current_profit: float, **kwargs) -> float: + dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe) + candle = dataframe.iloc[-1].squeeze() + return stoploss_from_absolute(current_rate - (candle['atr'] * 2), current_rate) + + ``` + +### *@informative()* + +``` python +def informative(timeframe: str, asset: str = '', + fmt: Optional[Union[str, Callable[[KwArg(str)], str]]] = None, + ffill: bool = True) -> Callable[[PopulateIndicators], PopulateIndicators]: + """ + A decorator for populate_indicators_Nn(self, dataframe, metadata), allowing these functions to + define informative indicators. + + Example usage: + + @informative('1h') + def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) + return dataframe + + :param timeframe: Informative timeframe. Must always be equal or higher than strategy timeframe. + :param asset: Informative asset, for example BTC, BTC/USDT, ETH/BTC. Do not specify to use + current pair. + :param fmt: Column format (str) or column formatter (callable(name, asset, timeframe)). When not + specified, defaults to: + * {base}_{quote}_{column}_{timeframe} if asset is specified. + * {column}_{timeframe} if asset is not specified. + Format string supports these format variables: + * {asset} - full name of the asset, for example 'BTC/USDT'. + * {base} - base currency in lower case, for example 'eth'. + * {BASE} - same as {base}, except in upper case. + * {quote} - quote currency in lower case, for example 'usdt'. + * {QUOTE} - same as {quote}, except in upper case. + * {column} - name of dataframe column. + * {timeframe} - timeframe of informative dataframe. + :param ffill: ffill dataframe after merging informative pair. + """ +``` + +In most common case it is possible to easily define informative pairs by using a decorator. All decorated `populate_indicators_*` methods run in isolation, +not having access to data from other informative pairs, in the end all informative dataframes are merged and passed to main `populate_indicators()` method. +When hyperopting, use of hyperoptable parameter `.value` attribute is not supported. Please use `.range` attribute. See [optimizing an indicator parameter](hyperopt.md#optimizing-an-indicator-parameter) +for more information. + +??? Example "Fast and easy way to define informative pairs" + + Most of the time we do not need power and flexibility offered by `merge_informative_pair()`, therefore we can use a decorator to quickly define informative pairs. + + ``` python + + from datetime import datetime + from freqtrade.persistence import Trade + from freqtrade.strategy import IStrategy, informative + + class AwesomeStrategy(IStrategy): + + # This method is not required. + # def informative_pairs(self): ... + + # Define informative upper timeframe for each pair. Decorators can be stacked on same + # method. Available in populate_indicators as 'rsi_30m' and 'rsi_1h'. + @informative('30m') + @informative('1h') + def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) + return dataframe + + # Define BTC/STAKE informative pair. Available in populate_indicators and other methods as + # 'btc_rsi_1h'. Current stake currency should be specified as {stake} format variable + # instead of hardcoding actual stake currency. Available in populate_indicators and other + # methods as 'btc_usdt_rsi_1h' (when stake currency is USDT). + @informative('1h', 'BTC/{stake}') + def populate_indicators_btc_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) + return dataframe + + # Define BTC/ETH informative pair. You must specify quote currency if it is different from + # stake currency. Available in populate_indicators and other methods as 'eth_btc_rsi_1h'. + @informative('1h', 'ETH/BTC') + def populate_indicators_eth_btc_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) + return dataframe + + # Define BTC/STAKE informative pair. A custom formatter may be specified for formatting + # column names. A callable `fmt(**kwargs) -> str` may be specified, to implement custom + # formatting. Available in populate_indicators and other methods as 'rsi_upper'. + @informative('1h', 'BTC/{stake}', '{column}') + def populate_indicators_btc_1h_2(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi_upper'] = ta.RSI(dataframe, timeperiod=14) + return dataframe + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + # Strategy timeframe indicators for current pair. + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) + # Informative pairs are available in this method. + dataframe['rsi_less'] = dataframe['rsi'] < dataframe['rsi_1h'] + return dataframe + + ``` + +!!! Note + Do not use `@informative` decorator if you need to use data of one informative pair when generating another informative pair. Instead, define informative pairs + manually as described [in the DataProvider section](#complete-data-provider-sample). + +!!! Note + Use string formatting when accessing informative dataframes of other pairs. This will allow easily changing stake currency in config without having to adjust strategy code. + + ``` python + def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + stake = self.config['stake_currency'] + dataframe.loc[ + ( + (dataframe[f'btc_{stake}_rsi_1h'] < 35) + & + (dataframe['volume'] > 0) + ), + ['buy', 'buy_tag']] = (1, 'buy_signal_rsi') + + return dataframe + ``` + + Alternatively column renaming may be used to remove stake currency from column names: `@informative('1h', 'BTC/{stake}', fmt='{base}_{column}_{timeframe}')`. + +!!! Warning "Duplicate method names" + Methods tagged with `@informative()` decorator must always have unique names! Re-using same name (for example when copy-pasting already defined informative method) + will overwrite previously defined method and not produce any errors due to limitations of Python programming language. In such cases you will find that indicators + created in earlier-defined methods are not available in the dataframe. Carefully review method names and make sure they are unique! ## Additional data (Wallets) @@ -781,6 +942,8 @@ Printing more than a few rows is also possible (simply use `print(dataframe)` i ## Common mistakes when developing strategies +### Peeking into the future while backtesting + Backtesting analyzes the whole time-range at once for performance reasons. Because of this, strategy authors need to make sure that strategies do not look-ahead into the future. This is a common pain-point, which can cause huge differences between backtesting and dry/live run methods, since they all use data which is not available during dry/live runs, so these strategies will perform well during backtesting, but will fail / perform badly in real conditions. diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index b020b00db..b9d01a236 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -93,7 +93,9 @@ Example configuration showing the different settings: "buy_cancel": "silent", "sell_cancel": "on", "buy_fill": "off", - "sell_fill": "off" + "sell_fill": "off", + "protection_trigger": "off", + "protection_trigger_global": "on" }, "reload": true, "balance_dust_level": 0.01 @@ -103,6 +105,7 @@ Example configuration showing the different settings: `buy` notifications are sent when the order is placed, while `buy_fill` notifications are sent when the order is filled on the exchange. `sell` notifications are sent when the order is placed, while `sell_fill` notifications are sent when the order is filled on the exchange. `*_fill` notifications are off by default and must be explicitly enabled. +`protection_trigger` notifications are sent when a protection triggers and `protection_trigger_global` notifications trigger when global protections are triggered. `balance_dust_level` will define what the `/balance` command takes as "dust" - Currencies with a balance below this will be shown. diff --git a/docs/utils.md b/docs/utils.md index 6395fb6f9..d8fbcacb7 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -26,9 +26,7 @@ optional arguments: ├── data ├── hyperopt_results ├── hyperopts -│   ├── sample_hyperopt_advanced.py │   ├── sample_hyperopt_loss.py -│   └── sample_hyperopt.py ├── notebooks │   └── strategy_analysis_example.ipynb ├── plot @@ -111,46 +109,11 @@ Using the advanced template (populates all optional functions and methods) freqtrade new-strategy --strategy AwesomeStrategy --template advanced ``` -## Create new hyperopt +## List Strategies -Creates a new hyperopt from a template similar to SampleHyperopt. -The file will be named inline with your class name, and will not overwrite existing files. +Use the `list-strategies` subcommand to see all strategies in one particular directory. -Results will be located in `user_data/hyperopts/.py`. - -``` output -usage: freqtrade new-hyperopt [-h] [--userdir PATH] [--hyperopt NAME] - [--template {full,minimal,advanced}] - -optional arguments: - -h, --help show this help message and exit - --userdir PATH, --user-data-dir PATH - Path to userdata directory. - --hyperopt NAME Specify hyperopt class name which will be used by the - bot. - --template {full,minimal,advanced} - Use a template which is either `minimal`, `full` - (containing multiple sample indicators) or `advanced`. - Default: `full`. -``` - -### Sample usage of new-hyperopt - -```bash -freqtrade new-hyperopt --hyperopt AwesomeHyperopt -``` - -With custom user directory - -```bash -freqtrade new-hyperopt --userdir ~/.freqtrade/ --hyperopt AwesomeHyperopt -``` - -## List Strategies and List Hyperopts - -Use the `list-strategies` subcommand to see all strategies in one particular directory and the `list-hyperopts` subcommand to list custom Hyperopts. - -These subcommands are useful for finding problems in your environment with loading strategies or hyperopt classes: modules with strategies or hyperopt classes that contain errors and failed to load are printed in red (LOAD FAILED), while strategies or hyperopt classes with duplicate names are printed in yellow (DUPLICATE NAME). +This subcommand is useful for finding problems in your environment with loading strategies: modules with strategies that contain errors and failed to load are printed in red (LOAD FAILED), while strategies with duplicate names are printed in yellow (DUPLICATE NAME). ``` usage: freqtrade list-strategies [-h] [-v] [--logfile FILE] [-V] [-c PATH] @@ -164,34 +127,6 @@ optional arguments: --no-color Disable colorization of hyperopt results. May be useful if you are redirecting output to a file. -Common arguments: - -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. Special values are: - 'syslog', 'journald'. See the documentation for more - details. - -V, --version show program's version number and exit - -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to - `-` to read config from stdin. - -d PATH, --datadir PATH - Path to directory with historical backtesting data. - --userdir PATH, --user-data-dir PATH - Path to userdata directory. -``` -``` -usage: freqtrade list-hyperopts [-h] [-v] [--logfile FILE] [-V] [-c PATH] - [-d PATH] [--userdir PATH] - [--hyperopt-path PATH] [-1] [--no-color] - -optional arguments: - -h, --help show this help message and exit - --hyperopt-path PATH Specify additional lookup path for Hyperopt and - Hyperopt Loss functions. - -1, --one-column Print output in one column. - --no-color Disable colorization of hyperopt results. May be - useful if you are redirecting output to a file. - Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). --logfile FILE Log to the file specified. Special values are: @@ -211,18 +146,16 @@ Common arguments: !!! Warning Using these commands will try to load all python files from a directory. This can be a security risk if untrusted files reside in this directory, since all module-level code is executed. -Example: Search default strategies and hyperopts directories (within the default userdir). +Example: Search default strategies directories (within the default userdir). ``` bash freqtrade list-strategies -freqtrade list-hyperopts ``` -Example: Search strategies and hyperopts directory within the userdir. +Example: Search strategies directory within the userdir. ``` bash freqtrade list-strategies --userdir ~/.freqtrade/ -freqtrade list-hyperopts --userdir ~/.freqtrade/ ``` Example: Search dedicated strategy path. @@ -231,12 +164,6 @@ Example: Search dedicated strategy path. freqtrade list-strategies --strategy-path ~/.freqtrade/strategies/ ``` -Example: Search dedicated hyperopt path. - -``` bash -freqtrade list-hyperopt --hyperopt-path ~/.freqtrade/hyperopts/ -``` - ## List Exchanges Use the `list-exchanges` subcommand to see the exchanges available for the bot. diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 8b85ca56e..0b6152bbf 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2021.8' +__version__ = '2021.9' if __version__ == 'develop': @@ -22,7 +22,7 @@ if __version__ == 'develop': # subprocess.check_output( # ['git', 'log', '--format="%h"', '-n 1'], # stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"') - except Exception: + except Exception: # pragma: no cover # git not available, ignore try: # Try Fallback to freqtrade_commit file (created by CI while building docker image) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 04e46ee23..a6f14cff7 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -11,11 +11,11 @@ from freqtrade.commands.build_config_commands import start_new_config from freqtrade.commands.data_commands import (start_convert_data, start_download_data, start_list_data) from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui, - start_new_hyperopt, start_new_strategy) + start_new_strategy) from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show -from freqtrade.commands.list_commands import (start_list_exchanges, start_list_hyperopts, - start_list_markets, start_list_strategies, - start_list_timeframes, start_show_trades) +from freqtrade.commands.list_commands import (start_list_exchanges, start_list_markets, + start_list_strategies, start_list_timeframes, + start_show_trades) from freqtrade.commands.optimize_commands import start_backtesting, start_edge, start_hyperopt from freqtrade.commands.pairlist_commands import start_test_pairlist from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 1143db394..d424f3ce7 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -22,7 +22,7 @@ ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv", "max_open_trades", "stake_amount", "fee", "pairs"] ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", - "enable_protections", "dry_run_wallet", + "enable_protections", "dry_run_wallet", "timeframe_detail", "strategy_list", "export", "exportfilename"] ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", @@ -55,8 +55,6 @@ ARGS_BUILD_CONFIG = ["config"] ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] -ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"] - ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] @@ -92,10 +90,10 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", "list-markets", "list-pairs", "list-strategies", "list-data", - "list-hyperopts", "hyperopt-list", "hyperopt-show", + "hyperopt-list", "hyperopt-show", "plot-dataframe", "plot-profit", "show-trades"] -NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"] +NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] class Arguments: @@ -174,12 +172,11 @@ class Arguments: from freqtrade.commands import (start_backtesting, start_convert_data, start_create_userdir, start_download_data, start_edge, start_hyperopt, start_hyperopt_list, start_hyperopt_show, start_install_ui, - start_list_data, start_list_exchanges, start_list_hyperopts, - start_list_markets, start_list_strategies, - start_list_timeframes, start_new_config, start_new_hyperopt, - start_new_strategy, start_plot_dataframe, start_plot_profit, - start_show_trades, start_test_pairlist, start_trading, - start_webserver) + start_list_data, start_list_exchanges, start_list_markets, + start_list_strategies, start_list_timeframes, + start_new_config, start_new_strategy, start_plot_dataframe, + start_plot_profit, start_show_trades, start_test_pairlist, + start_trading, start_webserver) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added @@ -206,12 +203,6 @@ class Arguments: build_config_cmd.set_defaults(func=start_new_config) self._build_args(optionlist=ARGS_BUILD_CONFIG, parser=build_config_cmd) - # add new-hyperopt subcommand - build_hyperopt_cmd = subparsers.add_parser('new-hyperopt', - help="Create new hyperopt") - build_hyperopt_cmd.set_defaults(func=start_new_hyperopt) - self._build_args(optionlist=ARGS_BUILD_HYPEROPT, parser=build_hyperopt_cmd) - # add new-strategy subcommand build_strategy_cmd = subparsers.add_parser('new-strategy', help="Create new strategy") @@ -300,15 +291,6 @@ class Arguments: list_exchanges_cmd.set_defaults(func=start_list_exchanges) self._build_args(optionlist=ARGS_LIST_EXCHANGES, parser=list_exchanges_cmd) - # Add list-hyperopts subcommand - list_hyperopts_cmd = subparsers.add_parser( - 'list-hyperopts', - help='Print available hyperopt classes.', - parents=[_common_parser], - ) - list_hyperopts_cmd.set_defaults(func=start_list_hyperopts) - self._build_args(optionlist=ARGS_LIST_HYPEROPTS, parser=list_hyperopts_cmd) - # Add list-markets subcommand list_markets_cmd = subparsers.add_parser( 'list-markets', diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 852cab92e..faa8a98f4 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -61,21 +61,27 @@ def ask_user_config() -> Dict[str, Any]: "type": "text", "name": "stake_currency", "message": "Please insert your stake currency:", - "default": 'BTC', + "default": 'USDT', }, { "type": "text", "name": "stake_amount", - "message": "Please insert your stake amount:", - "default": "0.01", + "message": f"Please insert your stake amount (Number or '{UNLIMITED_STAKE_AMOUNT}'):", + "default": "100", "validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_float(val), + "filter": lambda val: '"' + UNLIMITED_STAKE_AMOUNT + '"' + if val == UNLIMITED_STAKE_AMOUNT + else val }, { "type": "text", "name": "max_open_trades", "message": f"Please insert max_open_trades (Integer or '{UNLIMITED_STAKE_AMOUNT}'):", "default": "3", - "validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_int(val) + "validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_int(val), + "filter": lambda val: '"' + UNLIMITED_STAKE_AMOUNT + '"' + if val == UNLIMITED_STAKE_AMOUNT + else val }, { "type": "text", @@ -99,6 +105,8 @@ def ask_user_config() -> Dict[str, Any]: "bittrex", "kraken", "ftx", + "kucoin", + "gateio", Separator(), "other", ], @@ -122,6 +130,12 @@ def ask_user_config() -> Dict[str, Any]: "message": "Insert Exchange Secret", "when": lambda x: not x['dry_run'] }, + { + "type": "password", + "name": "exchange_key_password", + "message": "Insert Exchange API Key password", + "when": lambda x: not x['dry_run'] and x['exchange_name'] == 'kucoin' + }, { "type": "confirm", "name": "telegram", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 42be5abd6..e3c7fe464 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -1,7 +1,7 @@ """ Definition of cli arguments used in arguments.py """ -from argparse import ArgumentTypeError +from argparse import SUPPRESS, ArgumentTypeError from freqtrade import __version__, constants from freqtrade.constants import HYPEROPT_LOSS_BUILTIN @@ -135,6 +135,10 @@ AVAILABLE_CLI_OPTIONS = { help='Override the value of the `stake_amount` configuration setting.', ), # Backtesting + "timeframe_detail": Arg( + '--timeframe-detail', + help='Specify detail timeframe for backtesting (`1m`, `5m`, `30m`, `1h`, `1d`).', + ), "position_stacking": Arg( '--eps', '--enable-position-stacking', help='Allow buying the same pair multiple times (position stacking).', @@ -199,13 +203,13 @@ AVAILABLE_CLI_OPTIONS = { # Hyperopt "hyperopt": Arg( '--hyperopt', - help='Specify hyperopt class name which will be used by the bot.', + help=SUPPRESS, metavar='NAME', required=False, ), "hyperopt_path": Arg( '--hyperopt-path', - help='Specify additional lookup path for Hyperopt and Hyperopt Loss functions.', + help='Specify additional lookup path for Hyperopt Loss functions.', metavar='PATH', ), "epochs": Arg( diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index c98335e0b..4f9e5bbad 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -7,7 +7,7 @@ import requests from freqtrade.configuration import setup_utils_configuration from freqtrade.configuration.directory_operations import copy_sample_files, create_userdata_dir -from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES +from freqtrade.constants import USERPATH_STRATEGIES from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.misc import render_template, render_template_with_fallback @@ -87,56 +87,6 @@ def start_new_strategy(args: Dict[str, Any]) -> None: raise OperationalException("`new-strategy` requires --strategy to be set.") -def deploy_new_hyperopt(hyperopt_name: str, hyperopt_path: Path, subtemplate: str) -> None: - """ - Deploys a new hyperopt template to hyperopt_path - """ - fallback = 'full' - buy_guards = render_template_with_fallback( - templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2", - templatefallbackfile=f"subtemplates/hyperopt_buy_guards_{fallback}.j2", - ) - sell_guards = render_template_with_fallback( - templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2", - templatefallbackfile=f"subtemplates/hyperopt_sell_guards_{fallback}.j2", - ) - buy_space = render_template_with_fallback( - templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2", - templatefallbackfile=f"subtemplates/hyperopt_buy_space_{fallback}.j2", - ) - sell_space = render_template_with_fallback( - templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2", - templatefallbackfile=f"subtemplates/hyperopt_sell_space_{fallback}.j2", - ) - - strategy_text = render_template(templatefile='base_hyperopt.py.j2', - arguments={"hyperopt": hyperopt_name, - "buy_guards": buy_guards, - "sell_guards": sell_guards, - "buy_space": buy_space, - "sell_space": sell_space, - }) - - logger.info(f"Writing hyperopt to `{hyperopt_path}`.") - hyperopt_path.write_text(strategy_text) - - -def start_new_hyperopt(args: Dict[str, Any]) -> None: - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - if 'hyperopt' in args and args['hyperopt']: - - new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args['hyperopt'] + '.py') - - if new_path.exists(): - raise OperationalException(f"`{new_path}` already exists. " - "Please choose another Hyperopt Name.") - deploy_new_hyperopt(args['hyperopt'], new_path, args['template']) - else: - raise OperationalException("`new-hyperopt` requires --hyperopt to be set.") - - def clean_ui_subdir(directory: Path): if directory.is_dir(): logger.info("Removing UI directory content.") diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 089529d15..614c4b3f5 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -53,7 +53,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: if epochs and export_csv: HyperoptTools.export_csv_file( - config, epochs, total_epochs, not config.get('hyperopt_list_best', False), export_csv + config, epochs, export_csv ) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 410b9b72b..38fb098a0 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -10,7 +10,7 @@ from colorama import init as colorama_init from tabulate import tabulate from freqtrade.configuration import setup_utils_configuration -from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES +from freqtrade.constants import USERPATH_STRATEGIES from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import market_is_active, validate_exchanges @@ -92,25 +92,6 @@ def start_list_strategies(args: Dict[str, Any]) -> None: _print_objs_tabular(strategy_objs, config.get('print_colorized', False)) -def start_list_hyperopts(args: Dict[str, Any]) -> None: - """ - Print files with HyperOpt custom classes available in the directory - """ - from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - directory = Path(config.get('hyperopt_path', config['user_data_dir'] / USERPATH_HYPEROPTS)) - hyperopt_objs = HyperOptResolver.search_all_objects(directory, not args['print_one_column']) - # Sort alphabetically - hyperopt_objs = sorted(hyperopt_objs, key=lambda x: x['name']) - - if args['print_one_column']: - print('\n'.join([s['name'] for s in hyperopt_objs])) - else: - _print_objs_tabular(hyperopt_objs, config.get('print_colorized', False)) - - def start_list_timeframes(args: Dict[str, Any]) -> None: """ Print timeframes available on Exchange diff --git a/freqtrade/configuration/PeriodicCache.py b/freqtrade/configuration/PeriodicCache.py new file mode 100644 index 000000000..25c0c47f3 --- /dev/null +++ b/freqtrade/configuration/PeriodicCache.py @@ -0,0 +1,19 @@ +from datetime import datetime, timezone + +from cachetools.ttl import TTLCache + + +class PeriodicCache(TTLCache): + """ + Special cache that expires at "straight" times + A timer with ttl of 3600 (1h) will expire at every full hour (:00). + """ + + def __init__(self, maxsize, ttl, getsizeof=None): + def local_timer(): + ts = datetime.now(timezone.utc).timestamp() + offset = (ts % ttl) + return ts - offset + + # Init with smlight offset + super().__init__(maxsize=maxsize, ttl=ttl-1e-5, timer=local_timer, getsizeof=getsizeof) diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index 607f9cdef..cf41c0ca9 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,7 +1,8 @@ # flake8: noqa: F401 -from freqtrade.configuration.check_exchange import check_exchange, remove_credentials +from freqtrade.configuration.check_exchange import check_exchange from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.configuration.config_validation import validate_config_consistency from freqtrade.configuration.configuration import Configuration +from freqtrade.configuration.PeriodicCache import PeriodicCache from freqtrade.configuration.timerange import TimeRange diff --git a/freqtrade/configuration/check_exchange.py b/freqtrade/configuration/check_exchange.py index c4f038103..fa1f47f9b 100644 --- a/freqtrade/configuration/check_exchange.py +++ b/freqtrade/configuration/check_exchange.py @@ -10,19 +10,6 @@ from freqtrade.exchange import (available_exchanges, is_exchange_known_ccxt, logger = logging.getLogger(__name__) -def remove_credentials(config: Dict[str, Any]) -> None: - """ - Removes exchange keys from the configuration and specifies dry-run - Used for backtesting / hyperopt / edge and utils. - Modifies the input dict! - """ - config['exchange']['key'] = '' - config['exchange']['secret'] = '' - config['exchange']['password'] = '' - config['exchange']['uid'] = '' - config['dry_run'] = True - - def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool: """ Check if the exchange name in the config file is supported by Freqtrade diff --git a/freqtrade/configuration/config_setup.py b/freqtrade/configuration/config_setup.py index 22836ab19..02f2d4089 100644 --- a/freqtrade/configuration/config_setup.py +++ b/freqtrade/configuration/config_setup.py @@ -3,7 +3,6 @@ from typing import Any, Dict from freqtrade.enums import RunMode -from .check_exchange import remove_credentials from .config_validation import validate_config_consistency from .configuration import Configuration @@ -21,8 +20,8 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str configuration = Configuration(args, method) config = configuration.get_config() - # Ensure we do not use Exchange credentials - remove_credentials(config) + # Ensure these modes are using Dry-run + config['dry_run'] = True validate_config_consistency(config) return config diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 4dd5b7203..94b108f2b 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -242,6 +242,9 @@ class Configuration: except ValueError: pass + self._args_to_config(config, argname='timeframe_detail', + logstring='Parameter --timeframe-detail detected, ' + 'using {} for intra-candle backtesting ...') self._args_to_config(config, argname='stake_amount', logstring='Parameter --stake-amount detected, ' 'overriding stake_amount to: {} ...') diff --git a/freqtrade/constants.py b/freqtrade/constants.py index efcd1aaca..fca319a0f 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -69,9 +69,7 @@ DUST_PER_COIN = { # Source files with destination directories within user-directory USER_DATA_FILES = { 'sample_strategy.py': USERPATH_STRATEGIES, - 'sample_hyperopt_advanced.py': USERPATH_HYPEROPTS, 'sample_hyperopt_loss.py': USERPATH_HYPEROPTS, - 'sample_hyperopt.py': USERPATH_HYPEROPTS, 'strategy_analysis_example.ipynb': USERPATH_NOTEBOOKS, } @@ -112,7 +110,7 @@ CONF_SCHEMA = { }, 'tradable_balance_ratio': { 'type': 'number', - 'minimum': 0.1, + 'minimum': 0.0, 'maximum': 1, 'default': 0.99 }, @@ -286,6 +284,15 @@ CONF_SCHEMA = { 'enum': TELEGRAM_SETTING_OPTIONS, 'default': 'off' }, + 'protection_trigger': { + 'type': 'string', + 'enum': TELEGRAM_SETTING_OPTIONS, + 'default': 'off' + }, + 'protection_trigger_global': { + 'type': 'string', + 'enum': TELEGRAM_SETTING_OPTIONS, + }, } }, 'reload': {'type': 'boolean'}, diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index cdee0f078..b197c159f 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -149,6 +149,8 @@ class DataProvider: Clear pair dataframe cache. """ self.__cached_pairs = {} + self.__cached_pairs_backtesting = {} + self.__slice_index = 0 # Exchange functions diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 6f125aaa9..e6b8db322 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -197,7 +197,8 @@ def _download_pair_history(pair: str, *, timeframe=timeframe, since_ms=since_ms if since_ms else arrow.utcnow().shift( - days=-new_pairs_days).int_timestamp * 1000 + days=-new_pairs_days).int_timestamp * 1000, + is_new_pair=data.empty ) # TODO: Maybe move parsing to exchange class (?) new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index f12b1b37d..1950f0d08 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -119,7 +119,7 @@ class Edge: ) # Download informative pairs too res = defaultdict(list) - for p, t in self.strategy.informative_pairs(): + for p, t in self.strategy.gather_informative_pairs(): res[t].append(p) for timeframe, inf_pairs in res.items(): timerange_startup = deepcopy(self._timerange) diff --git a/freqtrade/enums/rpcmessagetype.py b/freqtrade/enums/rpcmessagetype.py index 9c59f6108..4e3f693e5 100644 --- a/freqtrade/enums/rpcmessagetype.py +++ b/freqtrade/enums/rpcmessagetype.py @@ -11,6 +11,8 @@ class RPCMessageType(Enum): SELL = 'sell' SELL_FILL = 'sell_fill' SELL_CANCEL = 'sell_cancel' + PROTECTION_TRIGGER = 'protection_trigger' + PROTECTION_TRIGGER_GLOBAL = 'protection_trigger_global' def __repr__(self): return self.value diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index b0c88a51a..b08213d28 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -1,6 +1,6 @@ # flake8: noqa: F401 # isort: off -from freqtrade.exchange.common import MAP_EXCHANGE_CHILDCLASS +from freqtrade.exchange.common import remove_credentials, MAP_EXCHANGE_CHILDCLASS from freqtrade.exchange.exchange import Exchange # isort: on from freqtrade.exchange.bibox import Bibox diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 0c470cb24..8dced3894 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -1,7 +1,8 @@ """ Binance exchange subclass """ import logging -from typing import Dict +from typing import Dict, List +import arrow import ccxt from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException, @@ -18,6 +19,7 @@ class Binance(Exchange): _ft_has: Dict = { "stoploss_on_exchange": True, "order_time_in_force": ['gtc', 'fok', 'ioc'], + "time_in_force_parameter": "timeInForce", "ohlcv_candle_limit": 1000, "trades_pagination": "id", "trades_pagination_arg": "fromId", @@ -89,3 +91,20 @@ class Binance(Exchange): f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e except ccxt.BaseError as e: raise OperationalException(e) from e + + async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, + since_ms: int, is_new_pair: bool + ) -> List: + """ + Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date + Does not work for other exchanges, which don't return the earliest data when called with "0" + """ + if is_new_pair: + x = await self._async_get_candle_history(pair, timeframe, 0) + if x and x[2] and x[2][0] and x[2][0][0] > since_ms: + # Set starting date to first available candle. + since_ms = x[2][0][0] + logger.info(f"Candle-data for {pair} available starting with " + f"{arrow.get(since_ms // 1000).isoformat()}.") + return await super()._async_get_historic_ohlcv( + pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair) diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index 694aa3aa2..7b89adf06 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -51,6 +51,19 @@ EXCHANGE_HAS_OPTIONAL = [ ] +def remove_credentials(config) -> None: + """ + Removes exchange keys from the configuration and specifies dry-run + Used for backtesting / hyperopt / edge and utils. + Modifies the input dict! + """ + if config.get('dry_run', False): + config['exchange']['key'] = '' + config['exchange']['secret'] = '' + config['exchange']['password'] = '' + config['exchange']['uid'] = '' + + def calculate_backoff(retrycount, max_retries): """ Calculate backoff diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 26f034e41..2b9b08d70 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -26,9 +26,9 @@ from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFun InvalidOrderException, OperationalException, PricingError, RetryableOrderError, TemporaryError) from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, - EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, retrier, - retrier_async) -from freqtrade.misc import deep_merge_dicts, safe_value_fallback2 + EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, + remove_credentials, retrier, retrier_async) +from freqtrade.misc import chunks, deep_merge_dicts, safe_value_fallback2 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist @@ -54,12 +54,16 @@ class Exchange: # Parameters to add directly to buy/sell calls (like agreeing to trading agreement) _params: Dict = {} + # Additional headers - added to the ccxt object + _headers: Dict = {} + # Dict to specify which options each exchange implements # This defines defaults, which can be selectively overridden by subclasses using _ft_has # or by specifying them in the configuration. _ft_has_default: Dict = { "stoploss_on_exchange": False, "order_time_in_force": ["gtc"], + "time_in_force_parameter": "timeInForce", "ohlcv_params": {}, "ohlcv_candle_limit": 500, "ohlcv_partial_candle": True, @@ -100,6 +104,7 @@ class Exchange: # Holds all open sell orders for dry_run self._dry_run_open_orders: Dict[str, Any] = {} + remove_credentials(config) if config['dry_run']: logger.info('Instance is running with dry_run enabled') @@ -169,7 +174,7 @@ class Exchange: asyncio.get_event_loop().run_until_complete(self._api_async.close()) def _init_ccxt(self, exchange_config: Dict[str, Any], ccxt_module: CcxtModuleType = ccxt, - ccxt_kwargs: dict = None) -> ccxt.Exchange: + ccxt_kwargs: Dict = {}) -> ccxt.Exchange: """ Initialize ccxt with given config and return valid ccxt instance. @@ -188,6 +193,10 @@ class Exchange: } if ccxt_kwargs: logger.info('Applying additional ccxt config: %s', ccxt_kwargs) + if self._headers: + # Inject static headers after the above output to not confuse users. + ccxt_kwargs = deep_merge_dicts({'headers': self._headers}, ccxt_kwargs) + if ccxt_kwargs: ex_config.update(ccxt_kwargs) try: @@ -352,9 +361,16 @@ class Exchange: def validate_stakecurrency(self, stake_currency: str) -> None: """ Checks stake-currency against available currencies on the exchange. + Only runs on startup. If markets have not been loaded, there's been a problem with + the connection to the exchange. :param stake_currency: Stake-currency to validate :raise: OperationalException if stake-currency is not available. """ + if not self._markets: + raise OperationalException( + 'Could not load markets, therefore cannot start. ' + 'Please investigate the above error for more details.' + ) quote_currencies = self.get_quote_currencies() if stake_currency not in quote_currencies: raise OperationalException( @@ -709,7 +725,8 @@ class Exchange: params = self._params.copy() if time_in_force != 'gtc' and ordertype != 'market': - params.update({'timeInForce': time_in_force}) + param = self._ft_has.get('time_in_force_parameter', '') + params.update({param: time_in_force}) try: # Set the precision for amount and price(rate) as accepted by the exchange @@ -1178,7 +1195,7 @@ class Exchange: # Historic data def get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int) -> List: + since_ms: int, is_new_pair: bool = False) -> List: """ Get candle history using asyncio and returns the list of candles. Handles all async work for this. @@ -1190,7 +1207,7 @@ class Exchange: """ return asyncio.get_event_loop().run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, - since_ms=since_ms)) + since_ms=since_ms, is_new_pair=is_new_pair)) def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, since_ms: int) -> DataFrame: @@ -1205,11 +1222,12 @@ class Exchange: return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, drop_incomplete=self._ohlcv_partial_candle) - async def _async_get_historic_ohlcv(self, pair: str, - timeframe: str, - since_ms: int) -> List: + async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, + since_ms: int, is_new_pair: bool + ) -> List: """ Download historic ohlcv + :param is_new_pair: used by binance subclass to allow "fast" new pair downloading """ one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) @@ -1222,21 +1240,22 @@ class Exchange: pair, timeframe, since) for since in range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] - results = await asyncio.gather(*input_coroutines, return_exceptions=True) - - # Combine gathered results data: List = [] - for res in results: - if isinstance(res, Exception): - logger.warning("Async code raised an exception: %s", res.__class__.__name__) - continue - # Deconstruct tuple if it's not an exception - p, _, new_data = res - if p == pair: - data.extend(new_data) + # Chunk requests into batches of 100 to avoid overwelming ccxt Throttling + for input_coro in chunks(input_coroutines, 100): + + results = await asyncio.gather(*input_coro, return_exceptions=True) + for res in results: + if isinstance(res, Exception): + logger.warning("Async code raised an exception: %s", res.__class__.__name__) + continue + # Deconstruct tuple if it's not an exception + p, _, new_data = res + if p == pair: + data.extend(new_data) # Sort data again after extending the result - above calls return in "async order" data = sorted(data, key=lambda x: x[0]) - logger.info("Downloaded data for %s with length %s.", pair, len(data)) + logger.info(f"Downloaded data for {pair} with length {len(data)}.") return data def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, diff --git a/freqtrade/exchange/gateio.py b/freqtrade/exchange/gateio.py index 9c910a10d..e6ee01c8a 100644 --- a/freqtrade/exchange/gateio.py +++ b/freqtrade/exchange/gateio.py @@ -21,3 +21,5 @@ class Gateio(Exchange): _ft_has: Dict = { "ohlcv_candle_limit": 1000, } + + _headers = {'X-Gate-Channel-Id': 'freqtrade'} diff --git a/freqtrade/exchange/kucoin.py b/freqtrade/exchange/kucoin.py index 22886a1d8..5d818f6a2 100644 --- a/freqtrade/exchange/kucoin.py +++ b/freqtrade/exchange/kucoin.py @@ -21,4 +21,6 @@ class Kucoin(Exchange): _ft_has: Dict = { "l2_limit_range": [20, 100], "l2_limit_range_required": False, + "order_time_in_force": ['gtc', 'fok', 'ioc'], + "time_in_force_parameter": "timeInForce", } diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 259270483..3a9b21b7c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -83,10 +83,10 @@ class FreqtradeBot(LoggingMixin): self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists) - # Attach Dataprovider to Strategy baseclass - IStrategy.dp = self.dataprovider - # Attach Wallets to Strategy baseclass - IStrategy.wallets = self.wallets + # Attach Dataprovider to strategy instance + self.strategy.dp = self.dataprovider + # Attach Wallets to strategy instance + self.strategy.wallets = self.wallets # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ @@ -99,7 +99,7 @@ class FreqtradeBot(LoggingMixin): self.state = State[initial_state.upper()] if initial_state else State.STOPPED # Protect sell-logic from forcesell and vice versa - self._sell_lock = Lock() + self._exit_lock = Lock() LoggingMixin.__init__(self, logger, timeframe_to_seconds(self.strategy.timeframe)) def notify_status(self, msg: str) -> None: @@ -160,20 +160,20 @@ class FreqtradeBot(LoggingMixin): # Refreshing candles self.dataprovider.refresh(self.pairlists.create_pair_list(self.active_pair_whitelist), - self.strategy.informative_pairs()) + self.strategy.gather_informative_pairs()) strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)() self.strategy.analyze(self.active_pair_whitelist) - with self._sell_lock: + with self._exit_lock: # Check and handle any timed out open orders self.check_handle_timedout() # Protect from collisions with forcesell. # Without this, freqtrade my try to recreate stoploss_on_exchange orders # while selling is in process, since telegram messages arrive in an different thread. - with self._sell_lock: + with self._exit_lock: trades = Trade.get_open_trades() # First process current opened trades (positions) self.exit_positions(trades) @@ -296,9 +296,9 @@ class FreqtradeBot(LoggingMixin): if sell_order: self.refind_lost_order(trade) else: - self.reupdate_buy_order_fees(trade) + self.reupdate_enter_order_fees(trade) - def reupdate_buy_order_fees(self, trade: Trade): + def reupdate_enter_order_fees(self, trade: Trade): """ Get buy order from database, and try to reupdate. Handles trades where the initial fee-update did not work. @@ -476,21 +476,21 @@ class FreqtradeBot(LoggingMixin): time_in_force = self.strategy.order_time_in_force['buy'] if price: - buy_limit_requested = price + enter_limit_requested = price else: # Calculate price - proposed_buy_rate = self.exchange.get_rate(pair, refresh=True, side="buy") + proposed_enter_rate = self.exchange.get_rate(pair, refresh=True, side="buy") custom_entry_price = strategy_safe_wrapper(self.strategy.custom_entry_price, - default_retval=proposed_buy_rate)( + default_retval=proposed_enter_rate)( pair=pair, current_time=datetime.now(timezone.utc), - proposed_rate=proposed_buy_rate) + proposed_rate=proposed_enter_rate) - buy_limit_requested = self.get_valid_price(custom_entry_price, proposed_buy_rate) + enter_limit_requested = self.get_valid_price(custom_entry_price, proposed_enter_rate) - if not buy_limit_requested: + if not enter_limit_requested: raise PricingError('Could not determine buy price.') - min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, buy_limit_requested, + min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, enter_limit_requested, self.strategy.stoploss) if not self.edge: @@ -498,7 +498,7 @@ class FreqtradeBot(LoggingMixin): stake_amount = strategy_safe_wrapper(self.strategy.custom_stake_amount, default_retval=stake_amount)( pair=pair, current_time=datetime.now(timezone.utc), - current_rate=buy_limit_requested, proposed_stake=stake_amount, + current_rate=enter_limit_requested, proposed_stake=stake_amount, min_stake=min_stake_amount, max_stake=max_stake_amount) stake_amount = self.wallets._validate_stake_amount(pair, stake_amount, min_stake_amount) @@ -508,27 +508,27 @@ class FreqtradeBot(LoggingMixin): logger.info(f"Buy signal found: about create a new trade for {pair} with stake_amount: " f"{stake_amount} ...") - amount = stake_amount / buy_limit_requested + amount = stake_amount / enter_limit_requested order_type = self.strategy.order_types['buy'] if forcebuy: # Forcebuy can define a different ordertype order_type = self.strategy.order_types.get('forcebuy', order_type) if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)( - pair=pair, order_type=order_type, amount=amount, rate=buy_limit_requested, + pair=pair, order_type=order_type, amount=amount, rate=enter_limit_requested, time_in_force=time_in_force, current_time=datetime.now(timezone.utc)): logger.info(f"User requested abortion of buying {pair}") return False amount = self.exchange.amount_to_precision(pair, amount) order = self.exchange.create_order(pair=pair, ordertype=order_type, side="buy", - amount=amount, rate=buy_limit_requested, + amount=amount, rate=enter_limit_requested, time_in_force=time_in_force) order_obj = Order.parse_from_ccxt_object(order, pair, 'buy') order_id = order['id'] order_status = order.get('status', None) # we assume the order is executed at the price requested - buy_limit_filled_price = buy_limit_requested + enter_limit_filled_price = enter_limit_requested amount_requested = amount if order_status == 'expired' or order_status == 'rejected': @@ -551,13 +551,13 @@ class FreqtradeBot(LoggingMixin): ) stake_amount = order['cost'] amount = safe_value_fallback(order, 'filled', 'amount') - buy_limit_filled_price = safe_value_fallback(order, 'average', 'price') + enter_limit_filled_price = safe_value_fallback(order, 'average', 'price') # in case of FOK the order may be filled immediately and fully elif order_status == 'closed': stake_amount = order['cost'] amount = safe_value_fallback(order, 'filled', 'amount') - buy_limit_filled_price = safe_value_fallback(order, 'average', 'price') + enter_limit_filled_price = safe_value_fallback(order, 'average', 'price') # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker') @@ -569,8 +569,8 @@ class FreqtradeBot(LoggingMixin): amount_requested=amount_requested, fee_open=fee, fee_close=fee, - open_rate=buy_limit_filled_price, - open_rate_requested=buy_limit_requested, + open_rate=enter_limit_filled_price, + open_rate_requested=enter_limit_requested, open_date=datetime.utcnow(), exchange=self.exchange.id, open_order_id=order_id, @@ -590,11 +590,11 @@ class FreqtradeBot(LoggingMixin): # Updating wallets self.wallets.update() - self._notify_buy(trade, order_type) + self._notify_enter(trade, order_type) return True - def _notify_buy(self, trade: Trade, order_type: str) -> None: + def _notify_enter(self, trade: Trade, order_type: str) -> None: """ Sends rpc notification when a buy occurred. """ @@ -617,7 +617,7 @@ class FreqtradeBot(LoggingMixin): # Send the message self.rpc.send_msg(msg) - def _notify_buy_cancel(self, trade: Trade, order_type: str, reason: str) -> None: + def _notify_enter_cancel(self, trade: Trade, order_type: str, reason: str) -> None: """ Sends rpc notification when a buy cancel occurred. """ @@ -643,7 +643,7 @@ class FreqtradeBot(LoggingMixin): # Send the message self.rpc.send_msg(msg) - def _notify_buy_fill(self, trade: Trade) -> None: + def _notify_enter_fill(self, trade: Trade) -> None: msg = { 'trade_id': trade.id, 'type': RPCMessageType.BUY_FILL, @@ -713,8 +713,8 @@ class FreqtradeBot(LoggingMixin): ) logger.debug('checking sell') - sell_rate = self.exchange.get_rate(trade.pair, refresh=True, side="sell") - if self._check_and_execute_sell(trade, sell_rate, buy, sell): + exit_rate = self.exchange.get_rate(trade.pair, refresh=True, side="sell") + if self._check_and_execute_exit(trade, exit_rate, buy, sell): return True logger.debug('Found no sell signal for %s.', trade) @@ -744,7 +744,7 @@ class FreqtradeBot(LoggingMixin): except InvalidOrderException as e: trade.stoploss_order_id = None logger.error(f'Unable to place a stoploss order on exchange. {e}') - logger.warning('Selling the trade forcefully') + logger.warning('Exiting the trade forcefully') self.execute_trade_exit(trade, trade.stop_loss, sell_reason=SellCheckTuple( sell_type=SellType.EMERGENCY_SELL)) @@ -782,7 +782,7 @@ class FreqtradeBot(LoggingMixin): # Lock pair for one candle to prevent immediate rebuys self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), reason='Auto lock') - self._notify_sell(trade, "stoploss") + self._notify_exit(trade, "stoploss") return True if trade.open_order_id or not trade.is_open: @@ -851,19 +851,19 @@ class FreqtradeBot(LoggingMixin): logger.warning(f"Could not create trailing stoploss order " f"for pair {trade.pair}.") - def _check_and_execute_sell(self, trade: Trade, sell_rate: float, + def _check_and_execute_exit(self, trade: Trade, exit_rate: float, buy: bool, sell: bool) -> bool: """ - Check and execute sell + Check and execute exit """ should_sell = self.strategy.should_sell( - trade, sell_rate, datetime.now(timezone.utc), buy, sell, + trade, exit_rate, datetime.now(timezone.utc), buy, sell, force_stoploss=self.edge.stoploss(trade.pair) if self.edge else 0 ) if should_sell.sell_flag: logger.info(f'Executing Sell for {trade.pair}. Reason: {should_sell.sell_type}') - self.execute_trade_exit(trade, sell_rate, should_sell) + self.execute_trade_exit(trade, exit_rate, should_sell) return True return False @@ -906,7 +906,7 @@ class FreqtradeBot(LoggingMixin): default_retval=False)(pair=trade.pair, trade=trade, order=order))): - self.handle_cancel_buy(trade, order, constants.CANCEL_REASON['TIMEOUT']) + self.handle_cancel_enter(trade, order, constants.CANCEL_REASON['TIMEOUT']) elif (order['side'] == 'sell' and (order['status'] == 'open' or fully_cancelled) and ( fully_cancelled @@ -915,7 +915,7 @@ class FreqtradeBot(LoggingMixin): default_retval=False)(pair=trade.pair, trade=trade, order=order))): - self.handle_cancel_sell(trade, order, constants.CANCEL_REASON['TIMEOUT']) + self.handle_cancel_exit(trade, order, constants.CANCEL_REASON['TIMEOUT']) def cancel_all_open_orders(self) -> None: """ @@ -931,13 +931,13 @@ class FreqtradeBot(LoggingMixin): continue if order['side'] == 'buy': - self.handle_cancel_buy(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) + self.handle_cancel_enter(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) elif order['side'] == 'sell': - self.handle_cancel_sell(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) + self.handle_cancel_exit(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) Trade.commit() - def handle_cancel_buy(self, trade: Trade, order: Dict, reason: str) -> bool: + def handle_cancel_enter(self, trade: Trade, order: Dict, reason: str) -> bool: """ Buy cancel - cancel order :return: True if order was fully cancelled @@ -994,11 +994,11 @@ class FreqtradeBot(LoggingMixin): reason += f", {constants.CANCEL_REASON['PARTIALLY_FILLED']}" self.wallets.update() - self._notify_buy_cancel(trade, order_type=self.strategy.order_types['buy'], - reason=reason) + self._notify_enter_cancel(trade, order_type=self.strategy.order_types['buy'], + reason=reason) return was_trade_fully_canceled - def handle_cancel_sell(self, trade: Trade, order: Dict, reason: str) -> str: + def handle_cancel_exit(self, trade: Trade, order: Dict, reason: str) -> str: """ Sell cancel - cancel order and update trade :return: Reason for cancel @@ -1032,14 +1032,14 @@ class FreqtradeBot(LoggingMixin): reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] self.wallets.update() - self._notify_sell_cancel( + self._notify_exit_cancel( trade, order_type=self.strategy.order_types['sell'], reason=reason ) return reason - def _safe_sell_amount(self, pair: str, amount: float) -> float: + def _safe_exit_amount(self, pair: str, amount: float) -> float: """ Get sellable amount. Should be trade.amount - but will fall back to the available amount if necessary. @@ -1111,7 +1111,7 @@ class FreqtradeBot(LoggingMixin): # but we allow this value to be changed) order_type = self.strategy.order_types.get("forcesell", order_type) - amount = self._safe_sell_amount(trade.pair, trade.amount) + amount = self._safe_exit_amount(trade.pair, trade.amount) time_in_force = self.strategy.order_time_in_force['sell'] if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)( @@ -1150,11 +1150,11 @@ class FreqtradeBot(LoggingMixin): self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), reason='Auto lock') - self._notify_sell(trade, order_type) + self._notify_exit(trade, order_type) return True - def _notify_sell(self, trade: Trade, order_type: str, fill: bool = False) -> None: + def _notify_exit(self, trade: Trade, order_type: str, fill: bool = False) -> None: """ Sends rpc notification when a sell occurred. """ @@ -1196,7 +1196,7 @@ class FreqtradeBot(LoggingMixin): # Send the message self.rpc.send_msg(msg) - def _notify_sell_cancel(self, trade: Trade, order_type: str, reason: str) -> None: + def _notify_exit_cancel(self, trade: Trade, order_type: str, reason: str) -> None: """ Sends rpc notification when a sell cancel occurred. """ @@ -1217,7 +1217,7 @@ class FreqtradeBot(LoggingMixin): 'exchange': trade.exchange.capitalize(), 'pair': trade.pair, 'gain': gain, - 'limit': profit_rate, + 'limit': profit_rate or 0, 'order_type': order_type, 'amount': trade.amount, 'open_rate': trade.open_rate, @@ -1226,7 +1226,7 @@ class FreqtradeBot(LoggingMixin): 'profit_ratio': profit_ratio, 'sell_reason': trade.sell_reason, 'open_date': trade.open_date, - 'close_date': trade.close_date, + 'close_date': trade.close_date or datetime.now(timezone.utc), 'stake_currency': self.config['stake_currency'], 'fiat_currency': self.config.get('fiat_display_currency', None), 'reason': reason, @@ -1291,16 +1291,28 @@ class FreqtradeBot(LoggingMixin): # Updating wallets when order is closed if not trade.is_open: if not stoploss_order and not trade.open_order_id: - self._notify_sell(trade, '', True) - self.protections.stop_per_pair(trade.pair) - self.protections.global_stop() + self._notify_exit(trade, '', True) + self.handle_protections(trade.pair) self.wallets.update() elif not trade.open_order_id: # Buy fill - self._notify_buy_fill(trade) + self._notify_enter_fill(trade) return False + def handle_protections(self, pair: str) -> None: + prot_trig = self.protections.stop_per_pair(pair) + if prot_trig: + msg = {'type': RPCMessageType.PROTECTION_TRIGGER, } + msg.update(prot_trig.to_json()) + self.rpc.send_msg(msg) + + prot_trig_glb = self.protections.global_stop() + if prot_trig_glb: + msg = {'type': RPCMessageType.PROTECTION_TRIGGER_GLOBAL, } + msg.update(prot_trig_glb.to_json()) + self.rpc.send_msg(msg) + def apply_fee_conditional(self, trade: Trade, trade_base_currency: str, amount: float, fee_abs: float) -> float: """ diff --git a/freqtrade/loggers.py b/freqtrade/loggers.py index fbb05d879..5c5831695 100644 --- a/freqtrade/loggers.py +++ b/freqtrade/loggers.py @@ -87,7 +87,7 @@ def setup_logging(config: Dict[str, Any]) -> None: # syslog config. The messages should be equal for this. handler_sl.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s')) logging.root.addHandler(handler_sl) - elif s[0] == 'journald': + elif s[0] == 'journald': # pragma: no cover try: from systemd.journal import JournaldLogHandler except ImportError: diff --git a/freqtrade/main.py b/freqtrade/main.py index 2fd3d32bb..6593fbcb6 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -9,7 +9,7 @@ from typing import Any, List # check min. python version -if sys.version_info < (3, 7): +if sys.version_info < (3, 7): # pragma: no cover sys.exit("Freqtrade requires Python version >= 3.7") from freqtrade.commands import Arguments @@ -46,7 +46,7 @@ def main(sysargv: List[str] = None) -> None: "`freqtrade --help` or `freqtrade --help`." ) - except SystemExit as e: + except SystemExit as e: # pragma: no cover return_code = e except KeyboardInterrupt: logger.info('SIGINT received, aborting ...') @@ -60,5 +60,5 @@ def main(sysargv: List[str] = None) -> None: sys.exit(return_code) -if __name__ == '__main__': +if __name__ == '__main__': # pragma: no cover main() diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index eecc7af54..8328d61d3 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -11,7 +11,7 @@ from typing import Any, Dict, List, Optional, Tuple from pandas import DataFrame -from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency +from freqtrade.configuration import TimeRange, validate_config_consistency from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.data import history from freqtrade.data.btanalysis import trade_list_to_dataframe @@ -61,8 +61,7 @@ class Backtesting: self.config = config self.results: Optional[Dict[str, Any]] = None - # Reset keys for backtesting - remove_credentials(self.config) + config['dry_run'] = True self.strategylist: List[IStrategy] = [] self.all_results: Dict[str, Dict] = {} @@ -86,7 +85,7 @@ class Backtesting: "configuration or as cli argument `--timeframe 5m`") self.timeframe = str(self.config.get('timeframe')) self.timeframe_min = timeframe_to_minutes(self.timeframe) - + self.init_backtest_detail() self.pairlists = PairListManager(self.exchange, self.config) if 'VolumePairList' in self.pairlists.name_list: raise OperationalException("VolumePairList not allowed for backtesting.") @@ -109,14 +108,6 @@ class Backtesting: else: self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0]) - Trade.use_db = False - Trade.reset_trades() - PairLocks.timeframe = self.config['timeframe'] - PairLocks.use_db = False - PairLocks.reset_locks() - - self.wallets = Wallets(self.config, self.exchange, log=False) - self.timerange = TimeRange.parse_timerange( None if self.config.get('timerange') is None else str(self.config.get('timerange'))) @@ -125,9 +116,7 @@ class Backtesting: # Add maximum startup candle count to configuration for informative pairs support self.config['startup_candle_count'] = self.required_startup self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe) - - self.progress = BTProgress() - self.abort = False + self.init_backtest() def __del__(self): self.cleanup() @@ -137,6 +126,28 @@ class Backtesting: PairLocks.use_db = True Trade.use_db = True + def init_backtest_detail(self): + # Load detail timeframe if specified + self.timeframe_detail = str(self.config.get('timeframe_detail', '')) + if self.timeframe_detail: + self.timeframe_detail_min = timeframe_to_minutes(self.timeframe_detail) + if self.timeframe_min <= self.timeframe_detail_min: + raise OperationalException( + "Detail timeframe must be smaller than strategy timeframe.") + + else: + self.timeframe_detail_min = 0 + self.detail_data: Dict[str, DataFrame] = {} + + def init_backtest(self): + + self.prepare_backtest(False) + + self.wallets = Wallets(self.config, self.exchange, log=False) + + self.progress = BTProgress() + self.abort = False + def _set_strategy(self, strategy: IStrategy): """ Load strategy into backtesting @@ -144,7 +155,7 @@ class Backtesting: self.strategy: IStrategy = strategy strategy.dp = self.dataprovider # Attach Wallets to Strategy baseclass - IStrategy.wallets = self.wallets + strategy.wallets = self.wallets # Set stoploss_on_exchange to false for backtesting, # since a "perfect" stoploss-sell is assumed anyway # And the regular "stoploss" function would not apply to that case @@ -188,6 +199,23 @@ class Backtesting: self.progress.set_new_value(1) return data, self.timerange + def load_bt_data_detail(self) -> None: + """ + Loads backtest detail data (smaller timeframe) if necessary. + """ + if self.timeframe_detail: + self.detail_data = history.load_data( + datadir=self.config['datadir'], + pairs=self.pairlists.whitelist, + timeframe=self.timeframe_detail, + timerange=self.timerange, + startup_candles=0, + fail_without_data=True, + data_format=self.config.get('dataformat_ohlcv', 'json'), + ) + else: + self.detail_data = {} + def prepare_backtest(self, enable_protections): """ Backtesting setup method - called once for every call to "backtest()". @@ -199,7 +227,8 @@ class Backtesting: Trade.reset_trades() self.rejected_trades = 0 self.dataprovider.clear_cache() - self._load_protections(self.strategy) + if enable_protections: + self._load_protections(self.strategy) def check_abort(self): """ @@ -318,7 +347,8 @@ class Backtesting: else: return sell_row[OPEN_IDX] - def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]: + def _get_sell_trade_entry_for_candle(self, trade: LocalTrade, + sell_row: Tuple) -> Optional[LocalTrade]: sell_candle_time = sell_row[DATE_IDX].to_pydatetime() sell = self.strategy.should_sell(trade, sell_row[OPEN_IDX], # type: ignore sell_candle_time, sell_row[BUY_IDX], @@ -346,6 +376,32 @@ class Backtesting: return None + def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]: + if self.timeframe_detail and trade.pair in self.detail_data: + sell_candle_time = sell_row[DATE_IDX].to_pydatetime() + sell_candle_end = sell_candle_time + timedelta(minutes=self.timeframe_min) + + detail_data = self.detail_data[trade.pair] + detail_data = detail_data.loc[ + (detail_data['date'] >= sell_candle_time) & + (detail_data['date'] < sell_candle_end) + ].copy() + if len(detail_data) == 0: + # Fall back to "regular" data if no detail data was found for this candle + return self._get_sell_trade_entry_for_candle(trade, sell_row) + detail_data.loc[:, 'buy'] = sell_row[BUY_IDX] + detail_data.loc[:, 'sell'] = sell_row[SELL_IDX] + headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high'] + for det_row in detail_data[headers].values.tolist(): + res = self._get_sell_trade_entry_for_candle(trade, det_row) + if res: + return res + + return None + + else: + return self._get_sell_trade_entry_for_candle(trade, sell_row) + def _enter_trade(self, pair: str, row: List) -> Optional[LocalTrade]: try: stake_amount = self.wallets.get_trade_stake_amount(pair, None) @@ -592,6 +648,7 @@ class Backtesting: data: Dict[str, Any] = {} data, timerange = self.load_bt_data() + self.load_bt_data_detail() logger.info("Dataload complete. Calculating indicators") for strat in self.strategylist: diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index aab7def05..f211da750 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -7,7 +7,8 @@ import logging from typing import Any, Dict from freqtrade import constants -from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency +from freqtrade.configuration import TimeRange, validate_config_consistency +from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge from freqtrade.optimize.optimize_reports import generate_edge_table from freqtrade.resolvers import ExchangeResolver, StrategyResolver @@ -28,11 +29,12 @@ class EdgeCli: def __init__(self, config: Dict[str, Any]) -> None: self.config = config - # Reset keys for edge - remove_credentials(self.config) + # Ensure using dry-run + self.config['dry_run'] = True self.config['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) self.strategy = StrategyResolver.load_strategy(self.config) + self.strategy.dp = DataProvider(config, None) validate_config_consistency(self.config) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e0b35df32..9549b4054 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -22,6 +22,7 @@ from pandas import DataFrame from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN from freqtrade.data.converter import trim_dataframes from freqtrade.data.history import get_timerange +from freqtrade.exceptions import OperationalException from freqtrade.misc import deep_merge_dicts, file_dump_json, plural from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules @@ -30,7 +31,7 @@ from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401 from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401 from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_serializer from freqtrade.optimize.optimize_reports import generate_strategy_stats -from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver +from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver # Suppress scikit-learn FutureWarnings from skopt @@ -44,7 +45,7 @@ progressbar.streams.wrap_stdout() logger = logging.getLogger(__name__) -INITIAL_POINTS = 30 +INITIAL_POINTS = 5 # Keep no more than SKOPT_MODEL_QUEUE_SIZE models # in the skopt model queue, to optimize memory consumption @@ -78,10 +79,10 @@ class Hyperopt: if not self.config.get('hyperopt'): self.custom_hyperopt = HyperOptAuto(self.config) - self.auto_hyperopt = True else: - self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config) - self.auto_hyperopt = False + raise OperationalException( + "Using separate Hyperopt files has been removed in 2021.9. Please convert " + "your existing Hyperopt file to the new Hyperoptable strategy interface") self.backtesting._set_strategy(self.backtesting.strategylist[0]) self.custom_hyperopt.strategy = self.backtesting.strategy @@ -103,31 +104,6 @@ class Hyperopt: self.num_epochs_saved = 0 self.current_best_epoch: Optional[Dict[str, Any]] = None - if not self.auto_hyperopt: - # Populate "fallback" functions here - # (hasattr is slow so should not be run during "regular" operations) - if hasattr(self.custom_hyperopt, 'populate_indicators'): - logger.warning( - "DEPRECATED: Using `populate_indicators()` in the hyperopt file is deprecated. " - "Please move these methods to your strategy." - ) - self.backtesting.strategy.populate_indicators = ( # type: ignore - self.custom_hyperopt.populate_indicators) # type: ignore - if hasattr(self.custom_hyperopt, 'populate_buy_trend'): - logger.warning( - "DEPRECATED: Using `populate_buy_trend()` in the hyperopt file is deprecated. " - "Please move these methods to your strategy." - ) - self.backtesting.strategy.populate_buy_trend = ( # type: ignore - self.custom_hyperopt.populate_buy_trend) # type: ignore - if hasattr(self.custom_hyperopt, 'populate_sell_trend'): - logger.warning( - "DEPRECATED: Using `populate_sell_trend()` in the hyperopt file is deprecated. " - "Please move these methods to your strategy." - ) - self.backtesting.strategy.populate_sell_trend = ( # type: ignore - self.custom_hyperopt.populate_sell_trend) # type: ignore - # Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set if self.config.get('use_max_market_positions', True): self.max_open_trades = self.config['max_open_trades'] @@ -256,7 +232,7 @@ class Hyperopt: """ Assign the dimensions in the hyperoptimization space. """ - if self.auto_hyperopt and HyperoptTools.has_space(self.config, 'protection'): + if HyperoptTools.has_space(self.config, 'protection'): # Protections can only be optimized when using the Parameter interface logger.debug("Hyperopt has 'protection' space") # Enable Protections if protection space is selected. @@ -265,7 +241,7 @@ class Hyperopt: if HyperoptTools.has_space(self.config, 'buy'): logger.debug("Hyperopt has 'buy' space") - self.buy_space = self.custom_hyperopt.indicator_space() + self.buy_space = self.custom_hyperopt.buy_indicator_space() if HyperoptTools.has_space(self.config, 'sell'): logger.debug("Hyperopt has 'sell' space") @@ -285,6 +261,15 @@ class Hyperopt: self.dimensions = (self.buy_space + self.sell_space + self.protection_space + self.roi_space + self.stoploss_space + self.trailing_space) + def assign_params(self, params_dict: Dict, category: str) -> None: + """ + Assign hyperoptable parameters + """ + for attr_name, attr in self.backtesting.strategy.enumerate_parameters(category): + if attr.optimize: + # noinspection PyProtectedMember + attr.value = params_dict[attr_name] + def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict: """ Used Optimize function. @@ -296,18 +281,13 @@ class Hyperopt: # Apply parameters if HyperoptTools.has_space(self.config, 'buy'): - self.backtesting.strategy.advise_buy = ( # type: ignore - self.custom_hyperopt.buy_strategy_generator(params_dict)) + self.assign_params(params_dict, 'buy') if HyperoptTools.has_space(self.config, 'sell'): - self.backtesting.strategy.advise_sell = ( # type: ignore - self.custom_hyperopt.sell_strategy_generator(params_dict)) + self.assign_params(params_dict, 'sell') if HyperoptTools.has_space(self.config, 'protection'): - for attr_name, attr in self.backtesting.strategy.enumerate_parameters('protection'): - if attr.optimize: - # noinspection PyProtectedMember - attr.value = params_dict[attr_name] + self.assign_params(params_dict, 'protection') if HyperoptTools.has_space(self.config, 'roi'): self.backtesting.strategy.minimal_roi = ( # type: ignore @@ -385,10 +365,20 @@ class Hyperopt: } def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer: + estimator = self.custom_hyperopt.generate_estimator() + + acq_optimizer = "sampling" + if isinstance(estimator, str): + if estimator not in ("GP", "RF", "ET", "GBRT"): + raise OperationalException(f"Estimator {estimator} not supported.") + else: + acq_optimizer = "auto" + + logger.info(f"Using estimator {estimator}.") return Optimizer( dimensions, - base_estimator="ET", - acq_optimizer="auto", + base_estimator=estimator, + acq_optimizer=acq_optimizer, n_initial_points=INITIAL_POINTS, acq_optimizer_kwargs={'n_jobs': cpu_count}, random_state=self.random_state, @@ -517,11 +507,10 @@ class Hyperopt: f"saved to '{self.results_file}'.") if self.current_best_epoch: - if self.auto_hyperopt: - HyperoptTools.try_export_params( - self.config, - self.backtesting.strategy.get_strategy_name(), - self.current_best_epoch) + HyperoptTools.try_export_params( + self.config, + self.backtesting.strategy.get_strategy_name(), + self.current_best_epoch) HyperoptTools.show_epoch_details(self.current_best_epoch, self.total_epochs, self.print_json) diff --git a/freqtrade/optimize/hyperopt_auto.py b/freqtrade/optimize/hyperopt_auto.py index 43e92d9c6..c1c769c72 100644 --- a/freqtrade/optimize/hyperopt_auto.py +++ b/freqtrade/optimize/hyperopt_auto.py @@ -4,15 +4,23 @@ This module implements a convenience auto-hyperopt class, which can be used toge that implement IHyperStrategy interface. """ from contextlib import suppress -from typing import Any, Callable, Dict, List +from typing import Callable, Dict, List -from pandas import DataFrame +from freqtrade.exceptions import OperationalException with suppress(ImportError): from skopt.space import Dimension -from freqtrade.optimize.hyperopt_interface import IHyperOpt +from freqtrade.optimize.hyperopt_interface import EstimatorType, IHyperOpt + + +def _format_exception_message(space: str) -> str: + raise OperationalException( + f"The '{space}' space is included into the hyperoptimization " + f"but no parameter for this space was not found in your Strategy. " + f"Please make sure to have parameters for this space enabled for optimization " + f"or remove the '{space}' space from hyperoptimization.") class HyperOptAuto(IHyperOpt): @@ -22,26 +30,6 @@ class HyperOptAuto(IHyperOpt): sell_indicator_space methods, but other hyperopt methods can be overridden as well. """ - def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: - def populate_buy_trend(dataframe: DataFrame, metadata: dict): - for attr_name, attr in self.strategy.enumerate_parameters('buy'): - if attr.optimize: - # noinspection PyProtectedMember - attr.value = params[attr_name] - return self.strategy.populate_buy_trend(dataframe, metadata) - - return populate_buy_trend - - def sell_strategy_generator(self, params: Dict[str, Any]) -> Callable: - def populate_sell_trend(dataframe: DataFrame, metadata: dict): - for attr_name, attr in self.strategy.enumerate_parameters('sell'): - if attr.optimize: - # noinspection PyProtectedMember - attr.value = params[attr_name] - return self.strategy.populate_sell_trend(dataframe, metadata) - - return populate_sell_trend - def _get_func(self, name) -> Callable: """ Return a function defined in Strategy.HyperOpt class, or one defined in super() class. @@ -60,21 +48,22 @@ class HyperOptAuto(IHyperOpt): if attr.optimize: yield attr.get_space(attr_name) - def _get_indicator_space(self, category, fallback_method_name): + def _get_indicator_space(self, category): + # TODO: is this necessary, or can we call "generate_space" directly? indicator_space = list(self._generate_indicator_space(category)) if len(indicator_space) > 0: return indicator_space else: - return self._get_func(fallback_method_name)() + _format_exception_message(category) - def indicator_space(self) -> List['Dimension']: - return self._get_indicator_space('buy', 'indicator_space') + def buy_indicator_space(self) -> List['Dimension']: + return self._get_indicator_space('buy') def sell_indicator_space(self) -> List['Dimension']: - return self._get_indicator_space('sell', 'sell_indicator_space') + return self._get_indicator_space('sell') def protection_space(self) -> List['Dimension']: - return self._get_indicator_space('protection', 'protection_space') + return self._get_indicator_space('protection') def generate_roi_table(self, params: Dict) -> Dict[int, float]: return self._get_func('generate_roi_table')(params) @@ -90,3 +79,6 @@ class HyperOptAuto(IHyperOpt): def trailing_space(self) -> List['Dimension']: return self._get_func('trailing_space')() + + def generate_estimator(self) -> EstimatorType: + return self._get_func('generate_estimator')() diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index 500798627..53b4f087c 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -5,11 +5,11 @@ This module defines the interface to apply for hyperopt import logging import math from abc import ABC -from typing import Any, Callable, Dict, List +from typing import Dict, List, Union +from sklearn.base import RegressorMixin from skopt.space import Categorical, Dimension, Integer -from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import round_dict from freqtrade.optimize.space import SKDecimal @@ -18,12 +18,7 @@ from freqtrade.strategy import IStrategy logger = logging.getLogger(__name__) - -def _format_exception_message(method: str, space: str) -> str: - return (f"The '{space}' space is included into the hyperoptimization " - f"but {method}() method is not found in your " - f"custom Hyperopt class. You should either implement this " - f"method or remove the '{space}' space from hyperoptimization.") +EstimatorType = Union[RegressorMixin, str] class IHyperOpt(ABC): @@ -45,36 +40,13 @@ class IHyperOpt(ABC): IHyperOpt.ticker_interval = str(config['timeframe']) # DEPRECATED IHyperOpt.timeframe = str(config['timeframe']) - def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + def generate_estimator(self) -> EstimatorType: """ - Create a buy strategy generator. + Return base_estimator. + Can be any of "GP", "RF", "ET", "GBRT" or an instance of a class + inheriting from RegressorMixin (from sklearn). """ - raise OperationalException(_format_exception_message('buy_strategy_generator', 'buy')) - - def sell_strategy_generator(self, params: Dict[str, Any]) -> Callable: - """ - Create a sell strategy generator. - """ - raise OperationalException(_format_exception_message('sell_strategy_generator', 'sell')) - - def protection_space(self) -> List[Dimension]: - """ - Create a protection space. - Only supported by the Parameter interface. - """ - raise OperationalException(_format_exception_message('indicator_space', 'protection')) - - def indicator_space(self) -> List[Dimension]: - """ - Create an indicator space. - """ - raise OperationalException(_format_exception_message('indicator_space', 'buy')) - - def sell_indicator_space(self) -> List[Dimension]: - """ - Create a sell indicator space. - """ - raise OperationalException(_format_exception_message('sell_indicator_space', 'sell')) + return 'ET' def generate_roi_table(self, params: Dict) -> Dict[int, float]: """ diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index b2e024f65..cfbc2757e 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Any, Dict, Iterator, List, Optional, Tuple import numpy as np +import pandas as pd import rapidjson import tabulate from colorama import Fore, Style @@ -298,8 +299,8 @@ class HyperoptTools(): f"Objective: {results['loss']:.5f}") @staticmethod - def prepare_trials_columns(trials, legacy_mode: bool, has_drawdown: bool) -> str: - + def prepare_trials_columns(trials: pd.DataFrame, legacy_mode: bool, + has_drawdown: bool) -> pd.DataFrame: trials['Best'] = '' if 'results_metrics.winsdrawslosses' not in trials.columns: @@ -435,8 +436,7 @@ class HyperoptTools(): return table @staticmethod - def export_csv_file(config: dict, results: list, total_epochs: int, highlight_best: bool, - csv_file: str) -> None: + def export_csv_file(config: dict, results: list, csv_file: str) -> None: """ Log result to csv-file """ diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 7bb60228a..8bde48670 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -368,6 +368,7 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame], 'max_open_trades_setting': (config['max_open_trades'] if config['max_open_trades'] != float('inf') else -1), 'timeframe': config['timeframe'], + 'timeframe_detail': config.get('timeframe_detail', ''), 'timerange': config.get('timerange', ''), 'enable_protections': config.get('enable_protections', False), 'strategy_name': strategy, diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 8c8c1e0a9..bc5ef961a 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -2,7 +2,7 @@ This module contains the class to persist trades into SQLite """ import logging -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from decimal import Decimal from typing import Any, Dict, List, Optional @@ -832,17 +832,21 @@ class Trade(_DECL_BASE, LocalTrade): return total_open_stake_amount or 0 @staticmethod - def get_overall_performance() -> List[Dict[str, Any]]: + def get_overall_performance(minutes=None) -> List[Dict[str, Any]]: """ Returns List of dicts containing all Trades, including profit and trade count NOTE: Not supported in Backtesting. """ + filters = [Trade.is_open.is_(False)] + if minutes: + start_date = datetime.now(timezone.utc) - timedelta(minutes=minutes) + filters.append(Trade.close_date >= start_date) pair_rates = Trade.query.with_entities( Trade.pair, func.sum(Trade.close_profit).label('profit_sum'), func.sum(Trade.close_profit_abs).label('profit_sum_abs'), func.count(Trade.pair).label('count') - ).filter(Trade.is_open.is_(False))\ + ).filter(*filters)\ .group_by(Trade.pair) \ .order_by(desc('profit_sum_abs')) \ .all() diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py index af904f693..8662fc36d 100644 --- a/freqtrade/persistence/pairlock_middleware.py +++ b/freqtrade/persistence/pairlock_middleware.py @@ -30,7 +30,8 @@ class PairLocks(): PairLocks.locks = [] @staticmethod - def lock_pair(pair: str, until: datetime, reason: str = None, *, now: datetime = None) -> None: + def lock_pair(pair: str, until: datetime, reason: str = None, *, + now: datetime = None) -> PairLock: """ Create PairLock from now to "until". Uses database by default, unless PairLocks.use_db is set to False, @@ -52,6 +53,7 @@ class PairLocks(): PairLock.query.session.commit() else: PairLocks.locks.append(lock) + return lock @staticmethod def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List[PairLock]: diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index dc5cab31e..5627d82ce 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -8,6 +8,7 @@ from typing import Any, Dict, List, Optional import arrow from pandas import DataFrame +from freqtrade.configuration import PeriodicCache from freqtrade.exceptions import OperationalException from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList @@ -18,14 +19,15 @@ logger = logging.getLogger(__name__) class AgeFilter(IPairList): - # Checked symbols cache (dictionary of ticker symbol => timestamp) - _symbolsChecked: Dict[str, int] = {} - def __init__(self, exchange, pairlistmanager, config: Dict[str, Any], pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + # Checked symbols cache (dictionary of ticker symbol => timestamp) + self._symbolsChecked: Dict[str, int] = {} + self._symbolsCheckFailed = PeriodicCache(maxsize=1000, ttl=86_400) + self._min_days_listed = pairlistconfig.get('min_days_listed', 10) self._max_days_listed = pairlistconfig.get('max_days_listed', None) @@ -69,9 +71,12 @@ class AgeFilter(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new allowlist """ - needed_pairs = [(p, '1d') for p in pairlist if p not in self._symbolsChecked] + needed_pairs = [ + (p, '1d') for p in pairlist + if p not in self._symbolsChecked and p not in self._symbolsCheckFailed] if not needed_pairs: - return pairlist + # Remove pairs that have been removed before + return [p for p in pairlist if p not in self._symbolsCheckFailed] since_days = -( self._max_days_listed if self._max_days_listed else self._min_days_listed @@ -118,5 +123,6 @@ class AgeFilter(IPairList): " or more than " f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" ) if self._max_days_listed else ''), logger.info) + self._symbolsCheckFailed[pair] = arrow.utcnow().int_timestamp * 1000 return False return False diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index 46a289ae6..301ee57ab 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -2,7 +2,7 @@ Performance pair list filter """ import logging -from typing import Dict, List +from typing import Any, Dict, List import pandas as pd @@ -15,6 +15,13 @@ logger = logging.getLogger(__name__) class PerformanceFilter(IPairList): + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._minutes = pairlistconfig.get('minutes', 0) + @property def needstickers(self) -> bool: """ @@ -40,7 +47,7 @@ class PerformanceFilter(IPairList): """ # Get the trading performance for pairs from database try: - performance = pd.DataFrame(Trade.get_overall_performance()) + performance = pd.DataFrame(Trade.get_overall_performance(self._minutes)) except AttributeError: # Performancefilter does not work in backtesting. self.log_once("PerformanceFilter is not available in this mode.", logger.warning) diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index c70e4a904..0ffc8a8c8 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -123,7 +123,7 @@ class VolumePairList(IPairList): filtered_tickers = [ v for k, v in tickers.items() if (self._exchange.get_pair_quote_currency(k) == self._stake_currency - and v[self._sort_key] is not None)] + and (self._use_range or v[self._sort_key] is not None))] pairlist = [s['symbol'] for s in filtered_tickers] pairlist = self.filter_pairlist(pairlist, tickers) diff --git a/freqtrade/plugins/pairlist/pairlist_helpers.py b/freqtrade/plugins/pairlist/pairlist_helpers.py index 924bfb293..1de27fcbd 100644 --- a/freqtrade/plugins/pairlist/pairlist_helpers.py +++ b/freqtrade/plugins/pairlist/pairlist_helpers.py @@ -17,7 +17,7 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str], if keep_invalid: for pair_wc in wildcardpl: try: - comp = re.compile(pair_wc) + comp = re.compile(pair_wc, re.IGNORECASE) result_partial = [ pair for pair in available_pairs if re.fullmatch(comp, pair) ] @@ -33,7 +33,7 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str], else: for pair_wc in wildcardpl: try: - comp = re.compile(pair_wc) + comp = re.compile(pair_wc, re.IGNORECASE) result += [ pair for pair in available_pairs if re.fullmatch(comp, pair) ] diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index f33e5b4bc..2510d6fee 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -6,6 +6,7 @@ from datetime import datetime, timezone from typing import Dict, List, Optional from freqtrade.persistence import PairLocks +from freqtrade.persistence.models import PairLock from freqtrade.plugins.protections import IProtection from freqtrade.resolvers import ProtectionResolver @@ -43,30 +44,28 @@ class ProtectionManager(): """ return [{p.name: p.short_desc()} for p in self._protection_handlers] - def global_stop(self, now: Optional[datetime] = None) -> bool: + def global_stop(self, now: Optional[datetime] = None) -> Optional[PairLock]: if not now: now = datetime.now(timezone.utc) - result = False + result = None for protection_handler in self._protection_handlers: if protection_handler.has_global_stop: - result, until, reason = protection_handler.global_stop(now) + lock, until, reason = protection_handler.global_stop(now) # Early stopping - first positive result blocks further trades - if result and until: + if lock and until: if not PairLocks.is_global_lock(until): - PairLocks.lock_pair('*', until, reason, now=now) - result = True + result = PairLocks.lock_pair('*', until, reason, now=now) return result - def stop_per_pair(self, pair, now: Optional[datetime] = None) -> bool: + def stop_per_pair(self, pair, now: Optional[datetime] = None) -> Optional[PairLock]: if not now: now = datetime.now(timezone.utc) - result = False + result = None for protection_handler in self._protection_handlers: if protection_handler.has_local_stop: - result, until, reason = protection_handler.stop_per_pair(pair, now) - if result and until: + lock, until, reason = protection_handler.stop_per_pair(pair, now) + if lock and until: if not PairLocks.is_pair_locked(pair, until): - PairLocks.lock_pair(pair, until, reason, now=now) - result = True + result = PairLocks.lock_pair(pair, until, reason, now=now) return result diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index 8327a4d13..6f0263e93 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -9,7 +9,6 @@ from typing import Dict from freqtrade.constants import HYPEROPT_LOSS_BUILTIN, USERPATH_HYPEROPTS from freqtrade.exceptions import OperationalException -from freqtrade.optimize.hyperopt_interface import IHyperOpt from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss from freqtrade.resolvers import IResolver @@ -17,43 +16,6 @@ from freqtrade.resolvers import IResolver logger = logging.getLogger(__name__) -class HyperOptResolver(IResolver): - """ - This class contains all the logic to load custom hyperopt class - """ - object_type = IHyperOpt - object_type_str = "Hyperopt" - user_subdir = USERPATH_HYPEROPTS - initial_search_path = None - - @staticmethod - def load_hyperopt(config: Dict) -> IHyperOpt: - """ - Load the custom hyperopt class from config parameter - :param config: configuration dictionary - """ - if not config.get('hyperopt'): - raise OperationalException("No Hyperopt set. Please use `--hyperopt` to specify " - "the Hyperopt class to use.") - - hyperopt_name = config['hyperopt'] - - hyperopt = HyperOptResolver.load_object(hyperopt_name, config, - kwargs={'config': config}, - extra_dir=config.get('hyperopt_path')) - - if not hasattr(hyperopt, 'populate_indicators'): - logger.info("Hyperopt class does not provide populate_indicators() method. " - "Using populate_indicators from the strategy.") - if not hasattr(hyperopt, 'populate_buy_trend'): - logger.info("Hyperopt class does not provide populate_buy_trend() method. " - "Using populate_buy_trend from the strategy.") - if not hasattr(hyperopt, 'populate_sell_trend'): - logger.info("Hyperopt class does not provide populate_sell_trend() method. " - "Using populate_sell_trend from the strategy.") - return hyperopt - - class HyperOptLossResolver(IResolver): """ This class contains all the logic to load custom hyperopt loss class diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index 2fa66645b..edbc39772 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -4,6 +4,7 @@ from copy import deepcopy from fastapi import APIRouter, BackgroundTasks, Depends +from freqtrade.configuration.config_validation import validate_config_consistency from freqtrade.enums import BacktestState from freqtrade.exceptions import DependencyException from freqtrade.rpc.api_server.api_schemas import BacktestRequest, BacktestResponse @@ -42,35 +43,40 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac # Reload strategy lastconfig = ApiServer._bt_last_config strat = StrategyResolver.load_strategy(btconfig) + validate_config_consistency(btconfig) if ( not ApiServer._bt or lastconfig.get('timeframe') != strat.timeframe - or lastconfig.get('dry_run_wallet') != btconfig.get('dry_run_wallet', 0) + or lastconfig.get('timeframe_detail') != btconfig.get('timeframe_detail') or lastconfig.get('timerange') != btconfig['timerange'] ): from freqtrade.optimize.backtesting import Backtesting ApiServer._bt = Backtesting(btconfig) - + if ApiServer._bt.timeframe_detail: + ApiServer._bt.load_bt_data_detail() + else: + ApiServer._bt.config = btconfig + ApiServer._bt.init_backtest() # Only reload data if timeframe changed. if ( not ApiServer._bt_data or not ApiServer._bt_timerange - or lastconfig.get('stake_amount') != btconfig.get('stake_amount') - or lastconfig.get('enable_protections') != btconfig.get('enable_protections') - or lastconfig.get('protections') != btconfig.get('protections', []) or lastconfig.get('timeframe') != strat.timeframe + or lastconfig.get('timerange') != btconfig['timerange'] ): - lastconfig['timerange'] = btconfig['timerange'] - lastconfig['protections'] = btconfig.get('protections', []) - lastconfig['enable_protections'] = btconfig.get('enable_protections') - lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet') - lastconfig['timeframe'] = strat.timeframe ApiServer._bt_data, ApiServer._bt_timerange = ApiServer._bt.load_bt_data() + lastconfig['timerange'] = btconfig['timerange'] + lastconfig['timeframe'] = strat.timeframe + lastconfig['protections'] = btconfig.get('protections', []) + lastconfig['enable_protections'] = btconfig.get('enable_protections') + lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet') + ApiServer._bt.abort = False min_date, max_date = ApiServer._bt.backtest_one_strategy( strat, ApiServer._bt_data, ApiServer._bt_timerange) + ApiServer._bt.results = generate_backtest_stats( ApiServer._bt_data, ApiServer._bt.all_results, min_date=min_date, max_date=max_date) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 318762136..46187f571 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -46,6 +46,12 @@ class Balances(BaseModel): value: float stake: str note: str + starting_capital: float + starting_capital_ratio: float + starting_capital_pct: float + starting_capital_fiat: float + starting_capital_fiat_ratio: float + starting_capital_fiat_pct: float class Count(BaseModel): @@ -324,6 +330,7 @@ class PairHistory(BaseModel): class BacktestRequest(BaseModel): strategy: str timeframe: Optional[str] + timeframe_detail: Optional[str] timerange: Optional[str] max_open_trades: Optional[int] stake_amount: Optional[Union[float, str]] diff --git a/freqtrade/rpc/api_server/uvicorn_threaded.py b/freqtrade/rpc/api_server/uvicorn_threaded.py index b63999f51..79af659c7 100644 --- a/freqtrade/rpc/api_server/uvicorn_threaded.py +++ b/freqtrade/rpc/api_server/uvicorn_threaded.py @@ -5,6 +5,20 @@ import time import uvicorn +def asyncio_setup() -> None: # pragma: no cover + # Set eventloop for win32 setups + # Reverts a change done in uvicorn 0.15.0 - which now sets the eventloop + # via policy. + import sys + + if sys.version_info >= (3, 8) and sys.platform == "win32": + import asyncio + import selectors + selector = selectors.SelectSelector() + loop = asyncio.SelectorEventLoop(selector) + asyncio.set_event_loop(loop) + + class UvicornServer(uvicorn.Server): """ Multithreaded server - as found in https://github.com/encode/uvicorn/issues/742 @@ -28,7 +42,7 @@ class UvicornServer(uvicorn.Server): try: import uvloop # noqa except ImportError: # pragma: no cover - from uvicorn.loops.asyncio import asyncio_setup + asyncio_setup() else: asyncio.set_event_loop(uvloop.new_event_loop()) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 95a37452b..f6599b429 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -403,8 +403,11 @@ class RPC: # Doing the sum is not right - overall profit needs to be based on initial capital profit_all_ratio_sum = sum(profit_all_ratio) if profit_all_ratio else 0.0 starting_balance = self._freqtrade.wallets.get_starting_balance() - profit_closed_ratio_fromstart = profit_closed_coin_sum / starting_balance - profit_all_ratio_fromstart = profit_all_coin_sum / starting_balance + profit_closed_ratio_fromstart = 0 + profit_all_ratio_fromstart = 0 + if starting_balance: + profit_closed_ratio_fromstart = profit_closed_coin_sum / starting_balance + profit_all_ratio_fromstart = profit_all_coin_sum / starting_balance profit_all_fiat = self._fiat_converter.convert_amount( profit_all_coin_sum, @@ -455,6 +458,9 @@ class RPC: raise RPCException('Error getting current tickers.') self._freqtrade.wallets.update(require_update=False) + starting_capital = self._freqtrade.wallets.get_starting_balance() + starting_cap_fiat = self._fiat_converter.convert_amount( + starting_capital, stake_currency, fiat_display_currency) if self._fiat_converter else 0 for coin, balance in self._freqtrade.wallets.get_all_balances().items(): if not balance.total: @@ -490,15 +496,25 @@ class RPC: else: raise RPCException('All balances are zero.') - symbol = fiat_display_currency - value = self._fiat_converter.convert_amount(total, stake_currency, - symbol) if self._fiat_converter else 0 + value = self._fiat_converter.convert_amount( + total, stake_currency, fiat_display_currency) if self._fiat_converter else 0 + + starting_capital_ratio = 0.0 + starting_capital_ratio = (total / starting_capital) - 1 if starting_capital else 0.0 + starting_cap_fiat_ratio = (value / starting_cap_fiat) - 1 if starting_cap_fiat else 0.0 + return { 'currencies': output, 'total': total, - 'symbol': symbol, + 'symbol': fiat_display_currency, 'value': value, 'stake': stake_currency, + 'starting_capital': starting_capital, + 'starting_capital_ratio': starting_capital_ratio, + 'starting_capital_pct': round(starting_capital_ratio * 100, 2), + 'starting_capital_fiat': starting_cap_fiat, + 'starting_capital_fiat_ratio': starting_cap_fiat_ratio, + 'starting_capital_fiat_pct': round(starting_cap_fiat_ratio * 100, 2), 'note': 'Simulated balances' if self._freqtrade.config['dry_run'] else '' } @@ -545,12 +561,12 @@ class RPC: order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair) if order['side'] == 'buy': - fully_canceled = self._freqtrade.handle_cancel_buy( + fully_canceled = self._freqtrade.handle_cancel_enter( trade, order, CANCEL_REASON['FORCE_SELL']) if order['side'] == 'sell': # Cancel order - so it is placed anew with a fresh price. - self._freqtrade.handle_cancel_sell(trade, order, CANCEL_REASON['FORCE_SELL']) + self._freqtrade.handle_cancel_exit(trade, order, CANCEL_REASON['FORCE_SELL']) if not fully_canceled: # Get current rate and execute sell @@ -563,7 +579,7 @@ class RPC: if self._freqtrade.state != State.RUNNING: raise RPCException('trader is not running') - with self._freqtrade._sell_lock: + with self._freqtrade._exit_lock: if trade_id == 'all': # Execute sell for all open orders for trade in Trade.get_open_trades(): @@ -625,7 +641,7 @@ class RPC: Handler for delete . Delete the given trade and close eventually existing open orders. """ - with self._freqtrade._sell_lock: + with self._freqtrade._exit_lock: c_count = 0 trade = Trade.get_trades(trade_filter=[Trade.id == trade_id]).first() if not trade: diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index a988d2b60..059ba9c41 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -260,6 +260,50 @@ class Telegram(RPCHandler): return message + def compose_message(self, msg: Dict[str, Any], msg_type: RPCMessageType) -> str: + + if msg_type == RPCMessageType.BUY: + message = self._format_buy_msg(msg) + + elif msg_type in (RPCMessageType.BUY_CANCEL, RPCMessageType.SELL_CANCEL): + msg['message_side'] = 'buy' if msg_type == RPCMessageType.BUY_CANCEL else 'sell' + message = ("\N{WARNING SIGN} *{exchange}:* " + "Cancelling open {message_side} Order for {pair} (#{trade_id}). " + "Reason: {reason}.".format(**msg)) + + elif msg_type == RPCMessageType.BUY_FILL: + message = ("\N{LARGE CIRCLE} *{exchange}:* " + "Buy order for {pair} (#{trade_id}) filled " + "for {open_rate}.".format(**msg)) + elif msg_type == RPCMessageType.SELL_FILL: + message = ("\N{LARGE CIRCLE} *{exchange}:* " + "Sell order for {pair} (#{trade_id}) filled " + "for {close_rate}.".format(**msg)) + elif msg_type == RPCMessageType.SELL: + message = self._format_sell_msg(msg) + elif msg_type == RPCMessageType.PROTECTION_TRIGGER: + message = ( + "*Protection* triggered due to {reason}. " + "`{pair}` will be locked until `{lock_end_time}`." + ).format(**msg) + elif msg_type == RPCMessageType.PROTECTION_TRIGGER_GLOBAL: + message = ( + "*Protection* triggered due to {reason}. " + "*All pairs* will be locked until `{lock_end_time}`." + ).format(**msg) + elif msg_type == RPCMessageType.STATUS: + message = '*Status:* `{status}`'.format(**msg) + + elif msg_type == RPCMessageType.WARNING: + message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg) + + elif msg_type == RPCMessageType.STARTUP: + message = '{status}'.format(**msg) + + else: + raise NotImplementedError('Unknown message type: {}'.format(msg_type)) + return message + def send_msg(self, msg: Dict[str, Any]) -> None: """ Send a message to telegram channel """ @@ -284,37 +328,7 @@ class Telegram(RPCHandler): # Notification disabled return - if msg_type == RPCMessageType.BUY: - message = self._format_buy_msg(msg) - - elif msg_type in (RPCMessageType.BUY_CANCEL, RPCMessageType.SELL_CANCEL): - msg['message_side'] = 'buy' if msg_type == RPCMessageType.BUY_CANCEL else 'sell' - message = ("\N{WARNING SIGN} *{exchange}:* " - "Cancelling open {message_side} Order for {pair} (#{trade_id}). " - "Reason: {reason}.".format(**msg)) - - elif msg_type == RPCMessageType.BUY_FILL: - message = ("\N{LARGE CIRCLE} *{exchange}:* " - "Buy order for {pair} (#{trade_id}) filled " - "for {open_rate}.".format(**msg)) - elif msg_type == RPCMessageType.SELL_FILL: - message = ("\N{LARGE CIRCLE} *{exchange}:* " - "Sell order for {pair} (#{trade_id}) filled " - "for {close_rate}.".format(**msg)) - elif msg_type == RPCMessageType.SELL: - message = self._format_sell_msg(msg) - - elif msg_type == RPCMessageType.STATUS: - message = '*Status:* `{status}`'.format(**msg) - - elif msg_type == RPCMessageType.WARNING: - message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg) - - elif msg_type == RPCMessageType.STARTUP: - message = '{status}'.format(**msg) - - else: - raise NotImplementedError('Unknown message type: {}'.format(msg_type)) + message = self.compose_message(msg, msg_type) self._send_msg(message, disable_notification=(noti == 'silent')) @@ -603,12 +617,15 @@ class Telegram(RPCHandler): output = '' if self._config['dry_run']: - output += ( - f"*Warning:* Simulated balances in Dry Mode.\n" - "This mode is still experimental!\n" - "Starting capital: " - f"`{self._config['dry_run_wallet']}` {self._config['stake_currency']}.\n" - ) + output += "*Warning:* Simulated balances in Dry Mode.\n" + + output += ("Starting capital: " + f"`{result['starting_capital']}` {self._config['stake_currency']}" + ) + output += (f" `{result['starting_capital_fiat']}` " + f"{self._config['fiat_display_currency']}.\n" + ) if result['starting_capital_fiat'] > 0 else '.\n' + total_dust_balance = 0 total_dust_currencies = 0 for curr in result['currencies']: @@ -641,9 +658,12 @@ class Telegram(RPCHandler): f"{round_coin_value(total_dust_balance, result['stake'], False)}`\n") output += ("\n*Estimated Value*:\n" - f"\t`{result['stake']}: {result['total']: .8f}`\n" + f"\t`{result['stake']}: " + f"{round_coin_value(result['total'], result['stake'], False)}`" + f" `({result['starting_capital_pct']}%)`\n" f"\t`{result['symbol']}: " - f"{round_coin_value(result['value'], result['symbol'], False)}`\n") + f"{round_coin_value(result['value'], result['symbol'], False)}`" + f" `({result['starting_capital_fiat_pct']}%)`\n") self._send_msg(output, reload_able=True, callback_path="update_balance", query=update.callback_query) except RPCException as e: diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index be655fc33..2ea0ad2b4 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -3,5 +3,7 @@ from freqtrade.exchange import (timeframe_to_minutes, timeframe_to_msecs, timefr timeframe_to_prev_date, timeframe_to_seconds) from freqtrade.strategy.hyper import (BooleanParameter, CategoricalParameter, DecimalParameter, IntParameter, RealParameter) +from freqtrade.strategy.informative_decorator import informative from freqtrade.strategy.interface import IStrategy -from freqtrade.strategy.strategy_helper import merge_informative_pair, stoploss_from_open +from freqtrade.strategy.strategy_helper import (merge_informative_pair, stoploss_from_absolute, + stoploss_from_open) diff --git a/freqtrade/strategy/informative_decorator.py b/freqtrade/strategy/informative_decorator.py new file mode 100644 index 000000000..4c5f21108 --- /dev/null +++ b/freqtrade/strategy/informative_decorator.py @@ -0,0 +1,128 @@ +from typing import Any, Callable, NamedTuple, Optional, Union + +from pandas import DataFrame + +from freqtrade.exceptions import OperationalException +from freqtrade.strategy.strategy_helper import merge_informative_pair + + +PopulateIndicators = Callable[[Any, DataFrame, dict], DataFrame] + + +class InformativeData(NamedTuple): + asset: Optional[str] + timeframe: str + fmt: Union[str, Callable[[Any], str], None] + ffill: bool + + +def informative(timeframe: str, asset: str = '', + fmt: Optional[Union[str, Callable[[Any], str]]] = None, + ffill: bool = True) -> Callable[[PopulateIndicators], PopulateIndicators]: + """ + A decorator for populate_indicators_Nn(self, dataframe, metadata), allowing these functions to + define informative indicators. + + Example usage: + + @informative('1h') + def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) + return dataframe + + :param timeframe: Informative timeframe. Must always be equal or higher than strategy timeframe. + :param asset: Informative asset, for example BTC, BTC/USDT, ETH/BTC. Do not specify to use + current pair. + :param fmt: Column format (str) or column formatter (callable(name, asset, timeframe)). When not + specified, defaults to: + * {base}_{quote}_{column}_{timeframe} if asset is specified. + * {column}_{timeframe} if asset is not specified. + Format string supports these format variables: + * {asset} - full name of the asset, for example 'BTC/USDT'. + * {base} - base currency in lower case, for example 'eth'. + * {BASE} - same as {base}, except in upper case. + * {quote} - quote currency in lower case, for example 'usdt'. + * {QUOTE} - same as {quote}, except in upper case. + * {column} - name of dataframe column. + * {timeframe} - timeframe of informative dataframe. + :param ffill: ffill dataframe after merging informative pair. + """ + _asset = asset + _timeframe = timeframe + _fmt = fmt + _ffill = ffill + + def decorator(fn: PopulateIndicators): + informative_pairs = getattr(fn, '_ft_informative', []) + informative_pairs.append(InformativeData(_asset, _timeframe, _fmt, _ffill)) + setattr(fn, '_ft_informative', informative_pairs) + return fn + return decorator + + +def _format_pair_name(config, pair: str) -> str: + return pair.format(stake_currency=config['stake_currency'], + stake=config['stake_currency']).upper() + + +def _create_and_merge_informative_pair(strategy, dataframe: DataFrame, metadata: dict, + inf_data: InformativeData, + populate_indicators: PopulateIndicators): + asset = inf_data.asset or '' + timeframe = inf_data.timeframe + fmt = inf_data.fmt + config = strategy.config + + if asset: + # Insert stake currency if needed. + asset = _format_pair_name(config, asset) + else: + # Not specifying an asset will define informative dataframe for current pair. + asset = metadata['pair'] + + if '/' in asset: + base, quote = asset.split('/') + else: + # When futures are supported this may need reevaluation. + # base, quote = asset, '' + raise OperationalException('Not implemented.') + + # Default format. This optimizes for the common case: informative pairs using same stake + # currency. When quote currency matches stake currency, column name will omit base currency. + # This allows easily reconfiguring strategy to use different base currency. In a rare case + # where it is desired to keep quote currency in column name at all times user should specify + # fmt='{base}_{quote}_{column}_{timeframe}' format or similar. + if not fmt: + fmt = '{column}_{timeframe}' # Informatives of current pair + if inf_data.asset: + fmt = '{base}_{quote}_' + fmt # Informatives of other pairs + + inf_metadata = {'pair': asset, 'timeframe': timeframe} + inf_dataframe = strategy.dp.get_pair_dataframe(asset, timeframe) + inf_dataframe = populate_indicators(strategy, inf_dataframe, inf_metadata) + + formatter: Any = None + if callable(fmt): + formatter = fmt # A custom user-specified formatter function. + else: + formatter = fmt.format # A default string formatter. + + fmt_args = { + 'BASE': base.upper(), + 'QUOTE': quote.upper(), + 'base': base.lower(), + 'quote': quote.lower(), + 'asset': asset, + 'timeframe': timeframe, + } + inf_dataframe.rename(columns=lambda column: formatter(column=column, **fmt_args), + inplace=True) + + date_column = formatter(column='date', **fmt_args) + if date_column in dataframe.columns: + raise OperationalException(f'Duplicate column name {date_column} exists in ' + f'dataframe! Ensure column names are unique!') + dataframe = merge_informative_pair(dataframe, inf_dataframe, strategy.timeframe, timeframe, + ffill=inf_data.ffill, append_timeframe=False, + date_column=date_column) + return dataframe diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index c51860011..7420bd9fd 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -19,6 +19,9 @@ from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.exchange.exchange import timeframe_to_next_date from freqtrade.persistence import PairLocks, Trade from freqtrade.strategy.hyper import HyperStrategyMixin +from freqtrade.strategy.informative_decorator import (InformativeData, PopulateIndicators, + _create_and_merge_informative_pair, + _format_pair_name) from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets @@ -118,8 +121,10 @@ class IStrategy(ABC, HyperStrategyMixin): # Class level variables (intentional) containing # the dataprovider (dp) (access to other candles, historic data, ...) # and wallets - access to the current balance. - dp: Optional[DataProvider] = None + dp: Optional[DataProvider] wallets: Optional[Wallets] = None + # Filled from configuration + stake_currency: str # container variable for strategy source code __source__: str = '' @@ -132,6 +137,24 @@ class IStrategy(ABC, HyperStrategyMixin): self._last_candle_seen_per_pair: Dict[str, datetime] = {} super().__init__(config) + # Gather informative pairs from @informative-decorated methods. + self._ft_informative: List[Tuple[InformativeData, PopulateIndicators]] = [] + for attr_name in dir(self.__class__): + cls_method = getattr(self.__class__, attr_name) + if not callable(cls_method): + continue + informative_data_list = getattr(cls_method, '_ft_informative', None) + if not isinstance(informative_data_list, list): + # Type check is required because mocker would return a mock object that evaluates to + # True, confusing this code. + continue + strategy_timeframe_minutes = timeframe_to_minutes(self.timeframe) + for informative_data in informative_data_list: + if timeframe_to_minutes(informative_data.timeframe) < strategy_timeframe_minutes: + raise OperationalException('Informative timeframe must be equal or higher than ' + 'strategy timeframe!') + self._ft_informative.append((informative_data, cls_method)) + @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ @@ -375,6 +398,23 @@ class IStrategy(ABC, HyperStrategyMixin): # END - Intended to be overridden by strategy ### + def gather_informative_pairs(self) -> ListPairsWithTimeframes: + """ + Internal method which gathers all informative pairs (user or automatically defined). + """ + informative_pairs = self.informative_pairs() + for inf_data, _ in self._ft_informative: + if inf_data.asset: + pair_tf = (_format_pair_name(self.config, inf_data.asset), inf_data.timeframe) + informative_pairs.append(pair_tf) + else: + if not self.dp: + raise OperationalException('@informative decorator with unspecified asset ' + 'requires DataProvider instance.') + for pair in self.dp.current_whitelist(): + informative_pairs.append((pair, inf_data.timeframe)) + return list(set(informative_pairs)) + def get_strategy_name(self) -> str: """ Returns strategy class name @@ -775,10 +815,11 @@ class IStrategy(ABC, HyperStrategyMixin): Does not run advise_buy or advise_sell! Used by optimize operations only, not during dry / live runs. Using .copy() to get a fresh copy of the dataframe for every strategy run. + Also copy on output to avoid PerformanceWarnings pandas 1.3.0 started to show. Has positive effects on memory usage for whatever reason - also when using only one strategy. """ - return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair}) + return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair}).copy() for pair, pair_data in data.items()} def advise_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -790,6 +831,12 @@ class IStrategy(ABC, HyperStrategyMixin): :return: a Dataframe with all mandatory indicators for the strategies """ logger.debug(f"Populating indicators for pair {metadata.get('pair')}.") + + # call populate_indicators_Nm() which were tagged with @informative decorator. + for inf_data, populate_fn in self._ft_informative: + dataframe = _create_and_merge_informative_pair( + self, dataframe, metadata, inf_data, populate_fn) + if self._populate_fun_len == 2: warnings.warn("deprecated - check out the Sample strategy to see " "the current function headers!", DeprecationWarning) diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index e089ebf31..175bcaccb 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -4,7 +4,9 @@ from freqtrade.exchange import timeframe_to_minutes def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame, - timeframe: str, timeframe_inf: str, ffill: bool = True) -> pd.DataFrame: + timeframe: str, timeframe_inf: str, ffill: bool = True, + append_timeframe: bool = True, + date_column: str = 'date') -> pd.DataFrame: """ Correctly merge informative samples to the original dataframe, avoiding lookahead bias. @@ -24,6 +26,8 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame, :param timeframe: Timeframe of the original pair sample. :param timeframe_inf: Timeframe of the informative pair sample. :param ffill: Forwardfill missing values - optional but usually required + :param append_timeframe: Rename columns by appending timeframe. + :param date_column: A custom date column name. :return: Merged dataframe :raise: ValueError if the secondary timeframe is shorter than the dataframe timeframe """ @@ -32,25 +36,29 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame, minutes = timeframe_to_minutes(timeframe) if minutes == minutes_inf: # No need to forwardshift if the timeframes are identical - informative['date_merge'] = informative["date"] + informative['date_merge'] = informative[date_column] elif minutes < minutes_inf: # Subtract "small" timeframe so merging is not delayed by 1 small candle # Detailed explanation in https://github.com/freqtrade/freqtrade/issues/4073 informative['date_merge'] = ( - informative["date"] + pd.to_timedelta(minutes_inf, 'm') - pd.to_timedelta(minutes, 'm') + informative[date_column] + pd.to_timedelta(minutes_inf, 'm') - + pd.to_timedelta(minutes, 'm') ) else: raise ValueError("Tried to merge a faster timeframe to a slower timeframe." "This would create new rows, and can throw off your regular indicators.") # Rename columns to be unique - informative.columns = [f"{col}_{timeframe_inf}" for col in informative.columns] + date_merge = 'date_merge' + if append_timeframe: + date_merge = f'date_merge_{timeframe_inf}' + informative.columns = [f"{col}_{timeframe_inf}" for col in informative.columns] # Combine the 2 dataframes # all indicators on the informative sample MUST be calculated before this point dataframe = pd.merge(dataframe, informative, left_on='date', - right_on=f'date_merge_{timeframe_inf}', how='left') - dataframe = dataframe.drop(f'date_merge_{timeframe_inf}', axis=1) + right_on=date_merge, how='left') + dataframe = dataframe.drop(date_merge, axis=1) if ffill: dataframe = dataframe.ffill() @@ -83,3 +91,28 @@ def stoploss_from_open(open_relative_stop: float, current_profit: float) -> floa # negative stoploss values indicate the requested stop price is higher than the current price return max(stoploss, 0.0) + + +def stoploss_from_absolute(stop_rate: float, current_rate: float) -> float: + """ + Given current price and desired stop price, return a stop loss value that is relative to current + price. + + The requested stop can be positive for a stop above the open price, or negative for + a stop below the open price. The return value is always >= 0. + + Returns 0 if the resulting stop price would be above the current price. + + :param stop_rate: Stop loss price. + :param current_rate: Current asset price. + :return: Positive stop loss value relative to current price + """ + + # formula is undefined for current_rate 0, return maximum value + if current_rate == 0: + return 1 + + stoploss = 1 - (stop_rate / current_rate) + + # negative stoploss values indicate the requested stop price is higher than the current price + return max(stoploss, 0.0) diff --git a/freqtrade/templates/base_config.json.j2 b/freqtrade/templates/base_config.json.j2 index a5782f7cd..68eebdbd4 100644 --- a/freqtrade/templates/base_config.json.j2 +++ b/freqtrade/templates/base_config.json.j2 @@ -1,3 +1,10 @@ +{%set volume_pairlist = '{ + "method": "VolumePairList", + "number_assets": 20, + "sort_key": "quoteVolume", + "min_value": 0, + "refresh_period": 1800 + }' %} { "max_open_trades": {{ max_open_trades }}, "stake_currency": "{{ stake_currency }}", @@ -29,7 +36,7 @@ }, {{ exchange | indent(4) }}, "pairlists": [ - {"method": "StaticPairList"} + {{ '{"method": "StaticPairList"}' if exchange_name == 'bittrex' else volume_pairlist }} ], "edge": { "enabled": false, diff --git a/freqtrade/templates/base_hyperopt.py.j2 b/freqtrade/templates/base_hyperopt.py.j2 deleted file mode 100644 index f6ca1477a..000000000 --- a/freqtrade/templates/base_hyperopt.py.j2 +++ /dev/null @@ -1,137 +0,0 @@ -# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement - -# --- Do not remove these libs --- -from functools import reduce -from typing import Any, Callable, Dict, List - -import numpy as np # noqa -import pandas as pd # noqa -from pandas import DataFrame -from skopt.space import Categorical, Dimension, Integer, Real # noqa - -from freqtrade.optimize.hyperopt_interface import IHyperOpt - -# -------------------------------- -# Add your lib to import here -import talib.abstract as ta # noqa -import freqtrade.vendor.qtpylib.indicators as qtpylib - - -class {{ hyperopt }}(IHyperOpt): - """ - This is a Hyperopt template to get you started. - - More information in the documentation: https://www.freqtrade.io/en/latest/hyperopt/ - - You should: - - Add any lib you need to build your hyperopt. - - You must keep: - - The prototypes for the methods: populate_indicators, indicator_space, buy_strategy_generator. - - The methods roi_space, generate_roi_table and stoploss_space are not required - and are provided by default. - However, you may override them if you need 'roi' and 'stoploss' spaces that - differ from the defaults offered by Freqtrade. - Sample implementation of these methods will be copied to `user_data/hyperopts` when - creating the user-data directory using `freqtrade create-userdir --userdir user_data`, - or is available online under the following URL: - https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py. - """ - - @staticmethod - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching buy strategy parameters. - """ - return [ - {{ buy_space | indent(12) }} - ] - - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by Hyperopt. - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Buy strategy Hyperopt will build and use. - """ - conditions = [] - - # GUARDS AND TRENDS - {{ buy_guards | indent(12) }} - - # TRIGGERS - if 'trigger' in params: - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) - - # Check that the candle had volume - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - @staticmethod - def sell_indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching sell strategy parameters. - """ - return [ - {{ sell_space | indent(12) }} - ] - - @staticmethod - def sell_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the sell strategy parameters to be used by Hyperopt. - """ - def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Sell strategy Hyperopt will build and use. - """ - conditions = [] - - # GUARDS AND TRENDS - {{ sell_guards | indent(12) }} - - # TRIGGERS - if 'sell-trigger' in params: - if params['sell-trigger'] == 'sell-bb_upper': - conditions.append(dataframe['close'] > dataframe['bb_upperband']) - if params['sell-trigger'] == 'sell-macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macdsignal'], dataframe['macd'] - )) - if params['sell-trigger'] == 'sell-sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['sar'], dataframe['close'] - )) - - # Check that the candle had volume - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'sell'] = 1 - - return dataframe - - return populate_sell_trend - diff --git a/freqtrade/templates/sample_hyperopt.py b/freqtrade/templates/sample_hyperopt.py deleted file mode 100644 index ed1af7718..000000000 --- a/freqtrade/templates/sample_hyperopt.py +++ /dev/null @@ -1,174 +0,0 @@ -# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement -# isort: skip_file - -# --- Do not remove these libs --- -from functools import reduce -from typing import Any, Callable, Dict, List - -import numpy as np # noqa -import pandas as pd # noqa -from pandas import DataFrame -from skopt.space import Categorical, Dimension, Integer, Real # noqa - -from freqtrade.optimize.hyperopt_interface import IHyperOpt - -# -------------------------------- -# Add your lib to import here -import talib.abstract as ta # noqa -import freqtrade.vendor.qtpylib.indicators as qtpylib - - -class SampleHyperOpt(IHyperOpt): - """ - This is a sample Hyperopt to inspire you. - - More information in the documentation: https://www.freqtrade.io/en/latest/hyperopt/ - - You should: - - Rename the class name to some unique name. - - Add any methods you want to build your hyperopt. - - Add any lib you need to build your hyperopt. - - An easier way to get a new hyperopt file is by using - `freqtrade new-hyperopt --hyperopt MyCoolHyperopt`. - - You must keep: - - The prototypes for the methods: populate_indicators, indicator_space, buy_strategy_generator. - - The methods roi_space, generate_roi_table and stoploss_space are not required - and are provided by default. - However, you may override them if you need 'roi' and 'stoploss' spaces that - differ from the defaults offered by Freqtrade. - Sample implementation of these methods will be copied to `user_data/hyperopts` when - creating the user-data directory using `freqtrade create-userdir --userdir user_data`, - or is available online under the following URL: - https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py. - """ - - @staticmethod - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching buy strategy parameters. - """ - return [ - Integer(10, 25, name='mfi-value'), - Integer(15, 45, name='fastd-value'), - Integer(20, 50, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='mfi-enabled'), - Categorical([True, False], name='fastd-enabled'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') - ] - - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by Hyperopt. - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Buy strategy Hyperopt will build and use. - """ - conditions = [] - - # GUARDS AND TRENDS - if 'mfi-enabled' in params and params['mfi-enabled']: - conditions.append(dataframe['mfi'] < params['mfi-value']) - if 'fastd-enabled' in params and params['fastd-enabled']: - conditions.append(dataframe['fastd'] < params['fastd-value']) - if 'adx-enabled' in params and params['adx-enabled']: - conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi-enabled' in params and params['rsi-enabled']: - conditions.append(dataframe['rsi'] < params['rsi-value']) - - # TRIGGERS - if 'trigger' in params: - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) - - # Check that volume is not 0 - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - @staticmethod - def sell_indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching sell strategy parameters. - """ - return [ - Integer(75, 100, name='sell-mfi-value'), - Integer(50, 100, name='sell-fastd-value'), - Integer(50, 100, name='sell-adx-value'), - Integer(60, 100, name='sell-rsi-value'), - Categorical([True, False], name='sell-mfi-enabled'), - Categorical([True, False], name='sell-fastd-enabled'), - Categorical([True, False], name='sell-adx-enabled'), - Categorical([True, False], name='sell-rsi-enabled'), - Categorical(['sell-bb_upper', - 'sell-macd_cross_signal', - 'sell-sar_reversal'], name='sell-trigger') - ] - - @staticmethod - def sell_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the sell strategy parameters to be used by Hyperopt. - """ - def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Sell strategy Hyperopt will build and use. - """ - conditions = [] - - # GUARDS AND TRENDS - if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']: - conditions.append(dataframe['mfi'] > params['sell-mfi-value']) - if 'sell-fastd-enabled' in params and params['sell-fastd-enabled']: - conditions.append(dataframe['fastd'] > params['sell-fastd-value']) - if 'sell-adx-enabled' in params and params['sell-adx-enabled']: - conditions.append(dataframe['adx'] < params['sell-adx-value']) - if 'sell-rsi-enabled' in params and params['sell-rsi-enabled']: - conditions.append(dataframe['rsi'] > params['sell-rsi-value']) - - # TRIGGERS - if 'sell-trigger' in params: - if params['sell-trigger'] == 'sell-bb_upper': - conditions.append(dataframe['close'] > dataframe['bb_upperband']) - if params['sell-trigger'] == 'sell-macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macdsignal'], dataframe['macd'] - )) - if params['sell-trigger'] == 'sell-sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['sar'], dataframe['close'] - )) - - # Check that volume is not 0 - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'sell'] = 1 - - return dataframe - - return populate_sell_trend diff --git a/freqtrade/templates/sample_hyperopt_advanced.py b/freqtrade/templates/sample_hyperopt_advanced.py deleted file mode 100644 index cc13b6ba3..000000000 --- a/freqtrade/templates/sample_hyperopt_advanced.py +++ /dev/null @@ -1,269 +0,0 @@ -# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement -# isort: skip_file -# --- Do not remove these libs --- -from functools import reduce -from typing import Any, Callable, Dict, List - -import numpy as np # noqa -import pandas as pd # noqa -from pandas import DataFrame -from freqtrade.optimize.space import Categorical, Dimension, Integer, SKDecimal, Real # noqa - -from freqtrade.optimize.hyperopt_interface import IHyperOpt - -# -------------------------------- -# Add your lib to import here -import talib.abstract as ta # noqa -import freqtrade.vendor.qtpylib.indicators as qtpylib - - -class AdvancedSampleHyperOpt(IHyperOpt): - """ - This is a sample hyperopt to inspire you. - Feel free to customize it. - - More information in the documentation: https://www.freqtrade.io/en/latest/hyperopt/ - - You should: - - Rename the class name to some unique name. - - Add any methods you want to build your hyperopt. - - Add any lib you need to build your hyperopt. - - You must keep: - - The prototypes for the methods: populate_indicators, indicator_space, buy_strategy_generator. - - The methods roi_space, generate_roi_table and stoploss_space are not required - and are provided by default. - However, you may override them if you need the - 'roi' and the 'stoploss' spaces that differ from the defaults offered by Freqtrade. - - This sample illustrates how to override these methods. - """ - @staticmethod - def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - This method can also be loaded from the strategy, if it doesn't exist in the hyperopt class. - """ - dataframe['adx'] = ta.ADX(dataframe) - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - dataframe['mfi'] = ta.MFI(dataframe) - dataframe['rsi'] = ta.RSI(dataframe) - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_upperband'] = bollinger['upper'] - dataframe['sar'] = ta.SAR(dataframe) - return dataframe - - @staticmethod - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching buy strategy parameters. - """ - return [ - Integer(10, 25, name='mfi-value'), - Integer(15, 45, name='fastd-value'), - Integer(20, 50, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='mfi-enabled'), - Categorical([True, False], name='fastd-enabled'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') - ] - - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by hyperopt - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Buy strategy Hyperopt will build and use - """ - conditions = [] - # GUARDS AND TRENDS - if 'mfi-enabled' in params and params['mfi-enabled']: - conditions.append(dataframe['mfi'] < params['mfi-value']) - if 'fastd-enabled' in params and params['fastd-enabled']: - conditions.append(dataframe['fastd'] < params['fastd-value']) - if 'adx-enabled' in params and params['adx-enabled']: - conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi-enabled' in params and params['rsi-enabled']: - conditions.append(dataframe['rsi'] < params['rsi-value']) - - # TRIGGERS - if 'trigger' in params: - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) - - # Check that volume is not 0 - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - @staticmethod - def sell_indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching sell strategy parameters. - """ - return [ - Integer(75, 100, name='sell-mfi-value'), - Integer(50, 100, name='sell-fastd-value'), - Integer(50, 100, name='sell-adx-value'), - Integer(60, 100, name='sell-rsi-value'), - Categorical([True, False], name='sell-mfi-enabled'), - Categorical([True, False], name='sell-fastd-enabled'), - Categorical([True, False], name='sell-adx-enabled'), - Categorical([True, False], name='sell-rsi-enabled'), - Categorical(['sell-bb_upper', - 'sell-macd_cross_signal', - 'sell-sar_reversal'], name='sell-trigger') - ] - - @staticmethod - def sell_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the sell strategy parameters to be used by hyperopt - """ - def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Sell strategy Hyperopt will build and use - """ - # print(params) - conditions = [] - # GUARDS AND TRENDS - if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']: - conditions.append(dataframe['mfi'] > params['sell-mfi-value']) - if 'sell-fastd-enabled' in params and params['sell-fastd-enabled']: - conditions.append(dataframe['fastd'] > params['sell-fastd-value']) - if 'sell-adx-enabled' in params and params['sell-adx-enabled']: - conditions.append(dataframe['adx'] < params['sell-adx-value']) - if 'sell-rsi-enabled' in params and params['sell-rsi-enabled']: - conditions.append(dataframe['rsi'] > params['sell-rsi-value']) - - # TRIGGERS - if 'sell-trigger' in params: - if params['sell-trigger'] == 'sell-bb_upper': - conditions.append(dataframe['close'] > dataframe['bb_upperband']) - if params['sell-trigger'] == 'sell-macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macdsignal'], dataframe['macd'] - )) - if params['sell-trigger'] == 'sell-sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['sar'], dataframe['close'] - )) - - # Check that volume is not 0 - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'sell'] = 1 - - return dataframe - - return populate_sell_trend - - @staticmethod - def generate_roi_table(params: Dict) -> Dict[int, float]: - """ - Generate the ROI table that will be used by Hyperopt - - This implementation generates the default legacy Freqtrade ROI tables. - - Change it if you need different number of steps in the generated - ROI tables or other structure of the ROI tables. - - Please keep it aligned with parameters in the 'roi' optimization - hyperspace defined by the roi_space method. - """ - roi_table = {} - roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] - roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] - roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] - roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 - - return roi_table - - @staticmethod - def roi_space() -> List[Dimension]: - """ - Values to search for each ROI steps - - Override it if you need some different ranges for the parameters in the - 'roi' optimization hyperspace. - - Please keep it aligned with the implementation of the - generate_roi_table method. - """ - return [ - Integer(10, 120, name='roi_t1'), - Integer(10, 60, name='roi_t2'), - Integer(10, 40, name='roi_t3'), - SKDecimal(0.01, 0.04, decimals=3, name='roi_p1'), - SKDecimal(0.01, 0.07, decimals=3, name='roi_p2'), - SKDecimal(0.01, 0.20, decimals=3, name='roi_p3'), - ] - - @staticmethod - def stoploss_space() -> List[Dimension]: - """ - Stoploss Value to search - - Override it if you need some different range for the parameter in the - 'stoploss' optimization hyperspace. - """ - return [ - SKDecimal(-0.35, -0.02, decimals=3, name='stoploss'), - ] - - @staticmethod - def trailing_space() -> List[Dimension]: - """ - Create a trailing stoploss space. - - You may override it in your custom Hyperopt class. - """ - return [ - # It was decided to always set trailing_stop is to True if the 'trailing' hyperspace - # is used. Otherwise hyperopt will vary other parameters that won't have effect if - # trailing_stop is set False. - # This parameter is included into the hyperspace dimensions rather than assigning - # it explicitly in the code in order to have it printed in the results along with - # other 'trailing' hyperspace parameters. - Categorical([True], name='trailing_stop'), - - SKDecimal(0.01, 0.35, decimals=3, name='trailing_stop_positive'), - - # 'trailing_stop_positive_offset' should be greater than 'trailing_stop_positive', - # so this intermediate parameter is used as the value of the difference between - # them. The value of the 'trailing_stop_positive_offset' is constructed in the - # generate_trailing_params() method. - # This is similar to the hyperspace dimensions used for constructing the ROI tables. - SKDecimal(0.001, 0.1, decimals=3, name='trailing_stop_positive_offset_p1'), - - Categorical([True, False], name='trailing_only_offset_is_reached'), - ] diff --git a/freqtrade/templates/subtemplates/exchange_binance.j2 b/freqtrade/templates/subtemplates/exchange_binance.j2 index 03aa0560c..de58b6f72 100644 --- a/freqtrade/templates/subtemplates/exchange_binance.j2 +++ b/freqtrade/templates/subtemplates/exchange_binance.j2 @@ -8,34 +8,8 @@ "rateLimit": 200 }, "pair_whitelist": [ - "ALGO/BTC", - "ATOM/BTC", - "BAT/BTC", - "BCH/BTC", - "BRD/BTC", - "EOS/BTC", - "ETH/BTC", - "IOTA/BTC", - "LINK/BTC", - "LTC/BTC", - "NEO/BTC", - "NXS/BTC", - "XMR/BTC", - "XRP/BTC", - "XTZ/BTC" ], "pair_blacklist": [ - "BNB/BTC", - "BNB/BUSD", - "BNB/ETH", - "BNB/EUR", - "BNB/NGN", - "BNB/PAX", - "BNB/RUB", - "BNB/TRY", - "BNB/TUSD", - "BNB/USDC", - "BNB/USDS", - "BNB/USDT", + "BNB/.*" ] } diff --git a/freqtrade/templates/subtemplates/exchange_bittrex.j2 b/freqtrade/templates/subtemplates/exchange_bittrex.j2 index 7b27318ca..0394790ce 100644 --- a/freqtrade/templates/subtemplates/exchange_bittrex.j2 +++ b/freqtrade/templates/subtemplates/exchange_bittrex.j2 @@ -15,16 +15,6 @@ "rateLimit": 500 }, "pair_whitelist": [ - "ETH/BTC", - "LTC/BTC", - "ETC/BTC", - "DASH/BTC", - "ZEC/BTC", - "XLM/BTC", - "XRP/BTC", - "TRX/BTC", - "ADA/BTC", - "XMR/BTC" ], "pair_blacklist": [ ] diff --git a/freqtrade/templates/subtemplates/exchange_kraken.j2 b/freqtrade/templates/subtemplates/exchange_kraken.j2 index 7139a0830..4d0e4c1ff 100644 --- a/freqtrade/templates/subtemplates/exchange_kraken.j2 +++ b/freqtrade/templates/subtemplates/exchange_kraken.j2 @@ -7,28 +7,10 @@ "ccxt_async_config": { "enableRateLimit": true, "rateLimit": 1000 + // Enable the below for downoading data. + //"rateLimit": 3100 }, "pair_whitelist": [ - "ADA/EUR", - "ATOM/EUR", - "BAT/EUR", - "BCH/EUR", - "BTC/EUR", - "DAI/EUR", - "DASH/EUR", - "EOS/EUR", - "ETC/EUR", - "ETH/EUR", - "LINK/EUR", - "LTC/EUR", - "QTUM/EUR", - "REP/EUR", - "WAVES/EUR", - "XLM/EUR", - "XMR/EUR", - "XRP/EUR", - "XTZ/EUR", - "ZEC/EUR" ], "pair_blacklist": [ diff --git a/freqtrade/templates/subtemplates/exchange_kucoin.j2 b/freqtrade/templates/subtemplates/exchange_kucoin.j2 new file mode 100644 index 000000000..f9dfff663 --- /dev/null +++ b/freqtrade/templates/subtemplates/exchange_kucoin.j2 @@ -0,0 +1,18 @@ +"exchange": { + "name": "{{ exchange_name | lower }}", + "key": "{{ exchange_key }}", + "secret": "{{ exchange_secret }}", + "password": "{{ exchange_key_password }}", + "ccxt_config": { + "enableRateLimit": true + "rateLimit": 200 + }, + "ccxt_async_config": { + "enableRateLimit": true, + "rateLimit": 200 + }, + "pair_whitelist": [ + ], + "pair_blacklist": [ + ] +} diff --git a/freqtrade/templates/subtemplates/hyperopt_buy_guards_full.j2 b/freqtrade/templates/subtemplates/hyperopt_buy_guards_full.j2 deleted file mode 100644 index 5b967f4ed..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_buy_guards_full.j2 +++ /dev/null @@ -1,8 +0,0 @@ -if params.get('mfi-enabled'): - conditions.append(dataframe['mfi'] < params['mfi-value']) -if params.get('fastd-enabled'): - conditions.append(dataframe['fastd'] < params['fastd-value']) -if params.get('adx-enabled'): - conditions.append(dataframe['adx'] > params['adx-value']) -if params.get('rsi-enabled'): - conditions.append(dataframe['rsi'] < params['rsi-value']) diff --git a/freqtrade/templates/subtemplates/hyperopt_buy_guards_minimal.j2 b/freqtrade/templates/subtemplates/hyperopt_buy_guards_minimal.j2 deleted file mode 100644 index 5e1022f59..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_buy_guards_minimal.j2 +++ /dev/null @@ -1,2 +0,0 @@ -if params.get('rsi-enabled'): - conditions.append(dataframe['rsi'] < params['rsi-value']) diff --git a/freqtrade/templates/subtemplates/hyperopt_buy_space_full.j2 b/freqtrade/templates/subtemplates/hyperopt_buy_space_full.j2 deleted file mode 100644 index 29bafbd93..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_buy_space_full.j2 +++ /dev/null @@ -1,9 +0,0 @@ -Integer(10, 25, name='mfi-value'), -Integer(15, 45, name='fastd-value'), -Integer(20, 50, name='adx-value'), -Integer(20, 40, name='rsi-value'), -Categorical([True, False], name='mfi-enabled'), -Categorical([True, False], name='fastd-enabled'), -Categorical([True, False], name='adx-enabled'), -Categorical([True, False], name='rsi-enabled'), -Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') diff --git a/freqtrade/templates/subtemplates/hyperopt_buy_space_minimal.j2 b/freqtrade/templates/subtemplates/hyperopt_buy_space_minimal.j2 deleted file mode 100644 index 5ddf537fb..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_buy_space_minimal.j2 +++ /dev/null @@ -1,3 +0,0 @@ -Integer(20, 40, name='rsi-value'), -Categorical([True, False], name='rsi-enabled'), -Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') diff --git a/freqtrade/templates/subtemplates/hyperopt_sell_guards_full.j2 b/freqtrade/templates/subtemplates/hyperopt_sell_guards_full.j2 deleted file mode 100644 index bd7b499f4..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_sell_guards_full.j2 +++ /dev/null @@ -1,8 +0,0 @@ -if params.get('sell-mfi-enabled'): - conditions.append(dataframe['mfi'] > params['sell-mfi-value']) -if params.get('sell-fastd-enabled'): - conditions.append(dataframe['fastd'] > params['sell-fastd-value']) -if params.get('sell-adx-enabled'): - conditions.append(dataframe['adx'] < params['sell-adx-value']) -if params.get('sell-rsi-enabled'): - conditions.append(dataframe['rsi'] > params['sell-rsi-value']) diff --git a/freqtrade/templates/subtemplates/hyperopt_sell_guards_minimal.j2 b/freqtrade/templates/subtemplates/hyperopt_sell_guards_minimal.j2 deleted file mode 100644 index 8b4adebf6..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_sell_guards_minimal.j2 +++ /dev/null @@ -1,2 +0,0 @@ -if params.get('sell-rsi-enabled'): - conditions.append(dataframe['rsi'] > params['sell-rsi-value']) diff --git a/freqtrade/templates/subtemplates/hyperopt_sell_space_full.j2 b/freqtrade/templates/subtemplates/hyperopt_sell_space_full.j2 deleted file mode 100644 index 46469d532..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_sell_space_full.j2 +++ /dev/null @@ -1,11 +0,0 @@ -Integer(75, 100, name='sell-mfi-value'), -Integer(50, 100, name='sell-fastd-value'), -Integer(50, 100, name='sell-adx-value'), -Integer(60, 100, name='sell-rsi-value'), -Categorical([True, False], name='sell-mfi-enabled'), -Categorical([True, False], name='sell-fastd-enabled'), -Categorical([True, False], name='sell-adx-enabled'), -Categorical([True, False], name='sell-rsi-enabled'), -Categorical(['sell-bb_upper', - 'sell-macd_cross_signal', - 'sell-sar_reversal'], name='sell-trigger') diff --git a/freqtrade/templates/subtemplates/hyperopt_sell_space_minimal.j2 b/freqtrade/templates/subtemplates/hyperopt_sell_space_minimal.j2 deleted file mode 100644 index dfb110543..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_sell_space_minimal.j2 +++ /dev/null @@ -1,5 +0,0 @@ -Integer(60, 100, name='sell-rsi-value'), -Categorical([True, False], name='sell-rsi-enabled'), -Categorical(['sell-bb_upper', - 'sell-macd_cross_signal', - 'sell-sar_reversal'], name='sell-trigger') diff --git a/mkdocs.yml b/mkdocs.yml index 854939ca0..05156168f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -23,10 +23,10 @@ nav: - Hyperopt: hyperopt.md - Utility Sub-commands: utils.md - Plotting: plotting.md + - Exchange-specific Notes: exchanges.md - Data Analysis: - Jupyter Notebooks: data-analysis.md - Strategy analysis: strategy_analysis_example.md - - Exchange-specific Notes: exchanges.md - Advanced Topics: - Advanced Post-installation Tasks: advanced-setup.md - Edge Positioning: edge.md diff --git a/requirements-dev.txt b/requirements-dev.txt index 67ee0035b..2f03255a0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,18 +8,20 @@ flake8==3.9.2 flake8-type-annotations==0.1.0 flake8-tidy-imports==4.4.1 mypy==0.910 -pytest==6.2.4 +pytest==6.2.5 pytest-asyncio==0.15.1 pytest-cov==2.12.1 pytest-mock==3.6.1 pytest-random-order==1.0.4 isort==5.9.3 +# For datetime mocking +time-machine==2.4.0 # Convert jupyter notebooks to markdown documents -nbconvert==6.1.0 +nbconvert==6.2.0 # mypy types types-cachetools==4.2.0 types-filelock==0.1.5 -types-requests==2.25.6 +types-requests==2.25.9 types-tabulate==0.8.2 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index d7f22634b..9feec80f1 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -8,4 +8,4 @@ scikit-optimize==0.8.1 filelock==3.0.12 joblib==1.0.1 psutil==5.8.0 -progressbar2==3.53.1 +progressbar2==3.53.3 diff --git a/requirements-plot.txt b/requirements-plot.txt index d835ed5d9..8e17232b0 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.2.1 +plotly==5.3.1 diff --git a/requirements.txt b/requirements.txt index 1bccbd725..feeb4d942 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,17 @@ numpy==1.21.2 -pandas==1.3.2 +pandas==1.3.3 +pandas-ta==0.3.14b -ccxt==1.55.28 +ccxt==1.57.3 # Pin cryptography for now due to rust build errors with piwheels -cryptography==3.4.7 +cryptography==3.4.8 aiohttp==3.7.4.post0 -SQLAlchemy==1.4.23 +SQLAlchemy==1.4.25 python-telegram-bot==13.7 arrow==1.1.1 cachetools==4.2.2 requests==2.26.0 -urllib3==1.26.6 +urllib3==1.26.7 wrapt==1.12.1 jsonschema==3.2.0 TA-Lib==0.4.21 @@ -31,7 +32,7 @@ python-rapidjson==1.4 sdnotify==0.3.2 # API Server -fastapi==0.68.0 +fastapi==0.68.1 uvicorn==0.15.0 pyjwt==2.1.0 aiofiles==0.7.0 diff --git a/setup.py b/setup.py index 727c40c7c..cf381bdd3 100644 --- a/setup.py +++ b/setup.py @@ -54,6 +54,7 @@ setup( 'wrapt', 'jsonschema', 'TA-Lib', + 'pandas-ta', 'technical', 'tabulate', 'pycoingecko', diff --git a/setup.sh b/setup.sh index feb0241f8..aee7c80b5 100755 --- a/setup.sh +++ b/setup.sh @@ -62,7 +62,7 @@ function updateenv() { then REQUIREMENTS_PLOT="-r requirements-plot.txt" fi - if [ "${SYS_ARCH}" == "armv7l" ]; then + if [ "${SYS_ARCH}" == "armv7l" ] || [ "${SYS_ARCH}" == "armv6l" ]; then echo "Detected Raspberry, installing cython, skipping hyperopt installation." ${PYTHON} -m pip install --upgrade cython else @@ -95,19 +95,7 @@ function install_talib() { return fi - cd build_helpers - tar zxvf ta-lib-0.4.0-src.tar.gz - cd ta-lib - sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h - ./configure --prefix=/usr/local - make - sudo make install - if [ -x "$(command -v apt-get)" ]; then - echo "Updating library path using ldconfig" - sudo ldconfig - fi - cd .. && rm -rf ./ta-lib/ - cd .. + cd build_helpers && ./install_ta-lib.sh && cd .. } function install_mac_newer_python_dependencies() { @@ -119,6 +107,7 @@ function install_mac_newer_python_dependencies() { echo "-------------------------" brew install hdf5 fi + export HDF5_DIR=$(brew --prefix) if [ ! $(brew --prefix --installed c-blosc 2>/dev/null) ] then @@ -127,6 +116,7 @@ function install_mac_newer_python_dependencies() { echo "-------------------------" brew install c-blosc fi + export CBLOSC_DIR=$(brew --prefix) } # Install bot MacOS diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 1da9e5100..135510b38 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -10,10 +10,10 @@ import pytest from freqtrade.commands import (start_convert_data, start_create_userdir, start_download_data, start_hyperopt_list, start_hyperopt_show, start_install_ui, - start_list_data, start_list_exchanges, start_list_hyperopts, - start_list_markets, start_list_strategies, start_list_timeframes, - start_new_hyperopt, start_new_strategy, start_show_trades, - start_test_pairlist, start_trading, start_webserver) + start_list_data, start_list_exchanges, start_list_markets, + start_list_strategies, start_list_timeframes, start_new_strategy, + start_show_trades, start_test_pairlist, start_trading, + start_webserver) from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui, get_ui_download_url, read_ui_version) from freqtrade.configuration import setup_utils_configuration @@ -32,8 +32,6 @@ def test_setup_utils_configuration(): config = setup_utils_configuration(get_args(args), RunMode.OTHER) assert "exchange" in config assert config['dry_run'] is True - assert config['exchange']['key'] == '' - assert config['exchange']['secret'] == '' def test_start_trading_fail(mocker, caplog): @@ -519,37 +517,6 @@ def test_start_new_strategy_no_arg(mocker, caplog): start_new_strategy(get_args(args)) -def test_start_new_hyperopt(mocker, caplog): - wt_mock = mocker.patch.object(Path, "write_text", MagicMock()) - mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - - args = [ - "new-hyperopt", - "--hyperopt", - "CoolNewhyperopt" - ] - start_new_hyperopt(get_args(args)) - - assert wt_mock.call_count == 1 - assert "CoolNewhyperopt" in wt_mock.call_args_list[0][0][0] - assert log_has_re("Writing hyperopt to .*", caplog) - - mocker.patch('freqtrade.commands.deploy_commands.setup_utils_configuration') - mocker.patch.object(Path, "exists", MagicMock(return_value=True)) - with pytest.raises(OperationalException, - match=r".* already exists. Please choose another Hyperopt Name\."): - start_new_hyperopt(get_args(args)) - - -def test_start_new_hyperopt_no_arg(mocker): - args = [ - "new-hyperopt", - ] - with pytest.raises(OperationalException, - match="`new-hyperopt` requires --hyperopt to be set."): - start_new_hyperopt(get_args(args)) - - def test_start_install_ui(mocker): clean_mock = mocker.patch('freqtrade.commands.deploy_commands.clean_ui_subdir') get_url_mock = mocker.patch('freqtrade.commands.deploy_commands.get_ui_download_url', @@ -824,37 +791,20 @@ def test_start_list_strategies(mocker, caplog, capsys): assert "legacy_strategy_v1.py" in captured.out assert "StrategyTestV2" in captured.out - -def test_start_list_hyperopts(mocker, caplog, capsys): - + # Test color output args = [ - "list-hyperopts", - "--hyperopt-path", - str(Path(__file__).parent.parent / "optimize" / "hyperopts"), - "-1" + "list-strategies", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy" / "strats"), ] pargs = get_args(args) # pargs['config'] = None - start_list_hyperopts(pargs) + start_list_strategies(pargs) captured = capsys.readouterr() - assert "TestHyperoptLegacy" not in captured.out - assert "legacy_hyperopt.py" not in captured.out - assert "HyperoptTestSepFile" in captured.out - assert "test_hyperopt.py" not in captured.out - - # Test regular output - args = [ - "list-hyperopts", - "--hyperopt-path", - str(Path(__file__).parent.parent / "optimize" / "hyperopts"), - ] - pargs = get_args(args) - # pargs['config'] = None - start_list_hyperopts(pargs) - captured = capsys.readouterr() - assert "TestHyperoptLegacy" not in captured.out - assert "legacy_hyperopt.py" not in captured.out - assert "HyperoptTestSepFile" in captured.out + assert "TestStrategyLegacyV1" in captured.out + assert "legacy_strategy_v1.py" in captured.out + assert "StrategyTestV2" in captured.out + assert "LOAD FAILED" in captured.out def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): diff --git a/tests/conftest.py b/tests/conftest.py index 5e08e7097..7354c0b2c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1685,14 +1685,6 @@ def trades_for_order2(): 'fee': {'cost': 0.004, 'currency': 'LTC'}}] -@pytest.fixture(scope="function") -def trades_for_order3(trades_for_order2): - # Different fee currencies for each trade - trades_for_order = deepcopy(trades_for_order2) - trades_for_order[0]['fee'] = {'cost': 0.02, 'currency': 'BNB'} - return trades_for_order - - @pytest.fixture def buy_order_fee(): return { diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index f2b508761..dd85c3abe 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -1,3 +1,4 @@ +from datetime import datetime, timezone from random import randint from unittest.mock import MagicMock @@ -5,7 +6,7 @@ import ccxt import pytest from freqtrade.exceptions import DependencyException, InvalidOrderException, OperationalException -from tests.conftest import get_patched_exchange +from tests.conftest import get_mock_coro, get_patched_exchange, log_has_re from tests.exchange.test_exchange import ccxt_exceptionhandlers @@ -105,3 +106,35 @@ def test_stoploss_adjust_binance(mocker, default_conf): # Test with invalid order case order['type'] = 'stop_loss' assert not exchange.stoploss_adjust(1501, order) + + +@pytest.mark.asyncio +async def test__async_get_historic_ohlcv_binance(default_conf, mocker, caplog): + ohlcv = [ + [ + int((datetime.now(timezone.utc).timestamp() - 1000) * 1000), + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + + exchange = get_patched_exchange(mocker, default_conf, id='binance') + # Monkey-patch async function + exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) + + pair = 'ETH/BTC' + res = await exchange._async_get_historic_ohlcv(pair, "5m", + 1500000000000, is_new_pair=False) + # Call with very old timestamp - causes tons of requests + assert exchange._api_async.fetch_ohlcv.call_count > 400 + # assert res == ohlcv + exchange._api_async.fetch_ohlcv.reset_mock() + res = await exchange._async_get_historic_ohlcv(pair, "5m", 1500000000000, is_new_pair=True) + + # Called twice - one "init" call - and one to get the actual data. + assert exchange._api_async.fetch_ohlcv.call_count == 2 + assert res == ohlcv + assert log_has_re(r"Candle-data for ETH/BTC available starting with .*", caplog) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 3a32d108b..d71dbe015 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -54,6 +54,8 @@ EXCHANGES = { def exchange_conf(): config = get_default_conf((Path(__file__).parent / "testdata").resolve()) config['exchange']['pair_whitelist'] = [] + config['exchange']['key'] = '' + config['exchange']['secret'] = '' config['dry_run'] = False return config diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index f6ac4c459..97bc33429 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1,5 +1,6 @@ import copy import logging +from copy import deepcopy from datetime import datetime, timedelta, timezone from math import isclose from random import randint @@ -14,7 +15,7 @@ from freqtrade.exceptions import (DDosProtection, DependencyException, InvalidOr OperationalException, PricingError, TemporaryError) from freqtrade.exchange import Binance, Bittrex, Exchange, Kraken from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT, - calculate_backoff) + calculate_backoff, remove_credentials) from freqtrade.exchange.exchange import (market_is_active, timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date, timeframe_to_prev_date, timeframe_to_seconds) @@ -78,6 +79,22 @@ def test_init(default_conf, mocker, caplog): assert log_has('Instance is running with dry_run enabled', caplog) +def test_remove_credentials(default_conf, caplog) -> None: + conf = deepcopy(default_conf) + conf['dry_run'] = False + remove_credentials(conf) + + assert conf['exchange']['key'] != '' + assert conf['exchange']['secret'] != '' + + conf['dry_run'] = True + remove_credentials(conf) + assert conf['exchange']['key'] == '' + assert conf['exchange']['secret'] == '' + assert conf['exchange']['password'] == '' + assert conf['exchange']['uid'] == '' + + def test_init_ccxt_kwargs(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') @@ -108,6 +125,13 @@ def test_init_ccxt_kwargs(default_conf, mocker, caplog): assert hasattr(ex._api_async, 'TestKWARG') assert log_has("Applying additional ccxt config: {'TestKWARG': 11, 'TestKWARG44': 11}", caplog) assert log_has(asynclogmsg, caplog) + # Test additional headers case + Exchange._headers = {'hello': 'world'} + ex = Exchange(conf) + + assert log_has("Applying additional ccxt config: {'TestKWARG': 11, 'TestKWARG44': 11}", caplog) + assert ex._api.headers == {'hello': 'world'} + Exchange._headers = {} def test_destroy(default_conf, mocker, caplog): @@ -178,7 +202,7 @@ def test_exchange_resolver(default_conf, mocker, caplog): def test_validate_order_time_in_force(default_conf, mocker, caplog): caplog.set_level(logging.INFO) - # explicitly test bittrex, exchanges implementing other policies need seperate tests + # explicitly test bittrex, exchanges implementing other policies need separate tests ex = get_patched_exchange(mocker, default_conf, id="bittrex") tif = { "buy": "gtc", @@ -557,7 +581,7 @@ def test_reload_markets_exception(default_conf, mocker, caplog): @pytest.mark.parametrize("stake_currency", ['ETH', 'BTC', 'USDT']) -def test_validate_stake_currency(default_conf, stake_currency, mocker, caplog): +def test_validate_stakecurrency(default_conf, stake_currency, mocker, caplog): default_conf['stake_currency'] = stake_currency api_mock = MagicMock() type(api_mock).load_markets = MagicMock(return_value={ @@ -571,7 +595,7 @@ def test_validate_stake_currency(default_conf, stake_currency, mocker, caplog): Exchange(default_conf) -def test_validate_stake_currency_error(default_conf, mocker, caplog): +def test_validate_stakecurrency_error(default_conf, mocker, caplog): default_conf['stake_currency'] = 'XRP' api_mock = MagicMock() type(api_mock).load_markets = MagicMock(return_value={ @@ -587,6 +611,13 @@ def test_validate_stake_currency_error(default_conf, mocker, caplog): 'Available currencies are: BTC, ETH, USDT'): Exchange(default_conf) + type(api_mock).load_markets = MagicMock(side_effect=ccxt.NetworkError('No connection.')) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + + with pytest.raises(OperationalException, + match=r'Could not load markets, therefore cannot start\. Please.*'): + Exchange(default_conf) + def test_get_quote_currencies(default_conf, mocker): ex = get_patched_exchange(mocker, default_conf) @@ -1537,6 +1568,32 @@ def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name): assert 'high' in ret.columns +@pytest.mark.asyncio +@pytest.mark.parametrize("exchange_name", EXCHANGES) +async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): + ohlcv = [ + [ + int((datetime.now(timezone.utc).timestamp() - 1000) * 1000), + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + # Monkey-patch async function + exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) + + pair = 'ETH/USDT' + res = await exchange._async_get_historic_ohlcv(pair, "5m", + 1500000000000, is_new_pair=False) + # Call with very old timestamp - causes tons of requests + assert exchange._api_async.fetch_ohlcv.call_count > 200 + assert res[0] == ohlcv[0] + assert log_has_re(r'Downloaded data for .* with length .*\.', caplog) + + def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: ohlcv = [ [ @@ -2424,7 +2481,7 @@ def test_fetch_order(default_conf, mocker, exchange_name, caplog): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_fetch_stoploss_order(default_conf, mocker, exchange_name): - # Don't test FTX here - that needs a seperate test + # Don't test FTX here - that needs a separate test if exchange_name == 'ftx': return default_conf['dry_run'] = True diff --git a/tests/optimize/conftest.py b/tests/optimize/conftest.py index 95c9fef97..5c5171c3a 100644 --- a/tests/optimize/conftest.py +++ b/tests/optimize/conftest.py @@ -16,7 +16,7 @@ def hyperopt_conf(default_conf): hyperconf.update({ 'datadir': Path(default_conf['datadir']), 'runmode': RunMode.HYPEROPT, - 'hyperopt': 'HyperoptTestSepFile', + 'strategy': 'HyperoptableStrategy', 'hyperopt_loss': 'ShortTradeDurHyperOptLoss', 'hyperopt_path': str(Path(__file__).parent / 'hyperopts'), 'epochs': 1, diff --git a/tests/optimize/hyperopts/hyperopt_test_sep_file.py b/tests/optimize/hyperopts/hyperopt_test_sep_file.py deleted file mode 100644 index 0fa1e1959..000000000 --- a/tests/optimize/hyperopts/hyperopt_test_sep_file.py +++ /dev/null @@ -1,202 +0,0 @@ -# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement - -from functools import reduce -from typing import Any, Callable, Dict, List - -import talib.abstract as ta -from pandas import DataFrame -from skopt.space import Categorical, Dimension, Integer - -import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.optimize.hyperopt_interface import IHyperOpt - - -class HyperoptTestSepFile(IHyperOpt): - """ - Default hyperopt provided by the Freqtrade bot. - You can override it with your own Hyperopt - """ - @staticmethod - def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Add several indicators needed for buy and sell strategies defined below. - """ - # ADX - dataframe['adx'] = ta.ADX(dataframe) - # MACD - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - # MFI - dataframe['mfi'] = ta.MFI(dataframe) - # RSI - dataframe['rsi'] = ta.RSI(dataframe) - # Stochastic Fast - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - # Minus-DI - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_upperband'] = bollinger['upper'] - # SAR - dataframe['sar'] = ta.SAR(dataframe) - - return dataframe - - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by Hyperopt. - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Buy strategy Hyperopt will build and use. - """ - conditions = [] - - # GUARDS AND TRENDS - if 'mfi-enabled' in params and params['mfi-enabled']: - conditions.append(dataframe['mfi'] < params['mfi-value']) - if 'fastd-enabled' in params and params['fastd-enabled']: - conditions.append(dataframe['fastd'] < params['fastd-value']) - if 'adx-enabled' in params and params['adx-enabled']: - conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi-enabled' in params and params['rsi-enabled']: - conditions.append(dataframe['rsi'] < params['rsi-value']) - - # TRIGGERS - if 'trigger' in params: - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - @staticmethod - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching buy strategy parameters. - """ - return [ - Integer(10, 25, name='mfi-value'), - Integer(15, 45, name='fastd-value'), - Integer(20, 50, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='mfi-enabled'), - Categorical([True, False], name='fastd-enabled'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') - ] - - @staticmethod - def sell_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the sell strategy parameters to be used by Hyperopt. - """ - def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Sell strategy Hyperopt will build and use. - """ - conditions = [] - - # GUARDS AND TRENDS - if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']: - conditions.append(dataframe['mfi'] > params['sell-mfi-value']) - if 'sell-fastd-enabled' in params and params['sell-fastd-enabled']: - conditions.append(dataframe['fastd'] > params['sell-fastd-value']) - if 'sell-adx-enabled' in params and params['sell-adx-enabled']: - conditions.append(dataframe['adx'] < params['sell-adx-value']) - if 'sell-rsi-enabled' in params and params['sell-rsi-enabled']: - conditions.append(dataframe['rsi'] > params['sell-rsi-value']) - - # TRIGGERS - if 'sell-trigger' in params: - if params['sell-trigger'] == 'sell-bb_upper': - conditions.append(dataframe['close'] > dataframe['bb_upperband']) - if params['sell-trigger'] == 'sell-macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macdsignal'], dataframe['macd'] - )) - if params['sell-trigger'] == 'sell-sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['sar'], dataframe['close'] - )) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'sell'] = 1 - - return dataframe - - return populate_sell_trend - - @staticmethod - def sell_indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching sell strategy parameters. - """ - return [ - Integer(75, 100, name='sell-mfi-value'), - Integer(50, 100, name='sell-fastd-value'), - Integer(50, 100, name='sell-adx-value'), - Integer(60, 100, name='sell-rsi-value'), - Categorical([True, False], name='sell-mfi-enabled'), - Categorical([True, False], name='sell-fastd-enabled'), - Categorical([True, False], name='sell-adx-enabled'), - Categorical([True, False], name='sell-rsi-enabled'), - Categorical(['sell-bb_upper', - 'sell-macd_cross_signal', - 'sell-sar_reversal'], name='sell-trigger') - ] - - def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Based on TA indicators. Should be a copy of same method from strategy. - Must align to populate_indicators in this file. - Only used when --spaces does not include buy space. - """ - dataframe.loc[ - ( - (dataframe['close'] < dataframe['bb_lowerband']) & - (dataframe['mfi'] < 16) & - (dataframe['adx'] > 25) & - (dataframe['rsi'] < 21) - ), - 'buy'] = 1 - - return dataframe - - def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Based on TA indicators. Should be a copy of same method from strategy. - Must align to populate_indicators in this file. - Only used when --spaces does not include sell space. - """ - dataframe.loc[ - ( - (qtpylib.crossed_above( - dataframe['macdsignal'], dataframe['macd'] - )) & - (dataframe['fastd'] > 54) - ), - 'sell'] = 1 - - return dataframe diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 0bbd110b6..2248cd4c1 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1,7 +1,7 @@ # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument import random -from datetime import timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path from unittest.mock import MagicMock, PropertyMock @@ -441,6 +441,15 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> with pytest.raises(OperationalException, match='VolumePairList not allowed for backtesting.'): Backtesting(default_conf) + default_conf.update({ + 'pairlists': [{"method": "StaticPairList"}], + 'timeframe_detail': '1d', + }) + + with pytest.raises(OperationalException, + match='Detail timeframe must be smaller than strategy timeframe.'): + Backtesting(default_conf) + def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, tickers) -> None: mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) @@ -491,7 +500,7 @@ def test_backtest__enter_trade(default_conf, fee, mocker) -> None: pair = 'UNITTEST/BTC' row = [ pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=0), - 1, # Sell + 1, # Buy 0.001, # Open 0.0011, # Close 0, # Sell @@ -539,6 +548,88 @@ def test_backtest__enter_trade(default_conf, fee, mocker) -> None: backtesting.cleanup() +def test_backtest__get_sell_trade_entry(default_conf, fee, mocker) -> None: + default_conf['use_sell_signal'] = False + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) + patch_exchange(mocker) + default_conf['timeframe_detail'] = '1m' + default_conf['max_open_trades'] = 2 + backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) + pair = 'UNITTEST/BTC' + row = [ + pd.Timestamp(year=2020, month=1, day=1, hour=4, minute=55, tzinfo=timezone.utc), + 1, # Buy + 200, # Open + 201, # Close + 0, # Sell + 195, # Low + 201.5, # High + '', # Buy Signal Name + ] + + trade = backtesting._enter_trade(pair, row=row) + assert isinstance(trade, LocalTrade) + + row_sell = [ + pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=0, tzinfo=timezone.utc), + 0, # Buy + 200, # Open + 201, # Close + 0, # Sell + 195, # Low + 210.5, # High + '', # Buy Signal Name + ] + row_detail = pd.DataFrame( + [ + [ + pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=0, tzinfo=timezone.utc), + 1, 200, 199, 0, 197, 200.1, '', + ], [ + pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=1, tzinfo=timezone.utc), + 0, 199, 199.5, 0, 199, 199.7, '', + ], [ + pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=2, tzinfo=timezone.utc), + 0, 199.5, 200.5, 0, 199, 200.8, '', + ], [ + pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=3, tzinfo=timezone.utc), + 0, 200.5, 210.5, 0, 193, 210.5, '', # ROI sell (?) + ], [ + pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=4, tzinfo=timezone.utc), + 0, 200, 199, 0, 193, 200.1, '', + ], + ], columns=["date", "buy", "open", "close", "sell", "low", "high", "buy_tag"] + ) + + # No data available. + res = backtesting._get_sell_trade_entry(trade, row_sell) + assert res is not None + assert res.sell_reason == SellType.ROI.value + assert res.close_date_utc == datetime(2020, 1, 1, 5, 0, tzinfo=timezone.utc) + + # Enter new trade + trade = backtesting._enter_trade(pair, row=row) + assert isinstance(trade, LocalTrade) + # Assign empty ... no result. + backtesting.detail_data[pair] = pd.DataFrame( + [], columns=["date", "buy", "open", "close", "sell", "low", "high", "buy_tag"]) + + res = backtesting._get_sell_trade_entry(trade, row) + assert res is None + + # Assign backtest-detail data + backtesting.detail_data[pair] = row_detail + + res = backtesting._get_sell_trade_entry(trade, row_sell) + assert res is not None + assert res.sell_reason == SellType.ROI.value + # Sell at minute 3 (not available above!) + assert res.close_date_utc == datetime(2020, 1, 1, 5, 3, tzinfo=timezone.utc) + assert round(res.close_rate, 3) == round(209.0225, 3) + + def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: default_conf['use_sell_signal'] = False mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) @@ -1042,3 +1133,102 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat assert 'LEFT OPEN TRADES REPORT' in captured.out assert '2017-11-14 21:17:00 -> 2017-11-14 22:58:00 | Max open trades : 1' in captured.out assert 'STRATEGY SUMMARY' in captured.out + + +@pytest.mark.filterwarnings("ignore:deprecated") +def test_backtest_start_multi_strat_nomock_detail(default_conf, mocker, + caplog, testdatadir, capsys): + # Tests detail-data loading + default_conf.update({ + "use_sell_signal": True, + "sell_profit_only": False, + "sell_profit_offset": 0.0, + "ignore_roi_if_buy_signal": False, + }) + patch_exchange(mocker) + result1 = pd.DataFrame({'pair': ['XRP/BTC', 'LTC/BTC'], + 'profit_ratio': [0.0, 0.0], + 'profit_abs': [0.0, 0.0], + 'open_date': pd.to_datetime(['2018-01-29 18:40:00', + '2018-01-30 03:30:00', ], utc=True + ), + 'close_date': pd.to_datetime(['2018-01-29 20:45:00', + '2018-01-30 05:35:00', ], utc=True), + 'trade_duration': [235, 40], + 'is_open': [False, False], + 'stake_amount': [0.01, 0.01], + 'open_rate': [0.104445, 0.10302485], + 'close_rate': [0.104969, 0.103541], + 'sell_reason': [SellType.ROI, SellType.ROI] + }) + result2 = pd.DataFrame({'pair': ['XRP/BTC', 'LTC/BTC', 'ETH/BTC'], + 'profit_ratio': [0.03, 0.01, 0.1], + 'profit_abs': [0.01, 0.02, 0.2], + 'open_date': pd.to_datetime(['2018-01-29 18:40:00', + '2018-01-30 03:30:00', + '2018-01-30 05:30:00'], utc=True + ), + 'close_date': pd.to_datetime(['2018-01-29 20:45:00', + '2018-01-30 05:35:00', + '2018-01-30 08:30:00'], utc=True), + 'trade_duration': [47, 40, 20], + 'is_open': [False, False, False], + 'stake_amount': [0.01, 0.01, 0.01], + 'open_rate': [0.104445, 0.10302485, 0.122541], + 'close_rate': [0.104969, 0.103541, 0.123541], + 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] + }) + backtestmock = MagicMock(side_effect=[ + { + 'results': result1, + 'config': default_conf, + 'locks': [], + 'rejected_signals': 20, + 'final_balance': 1000, + }, + { + 'results': result2, + 'config': default_conf, + 'locks': [], + 'rejected_signals': 20, + 'final_balance': 1000, + } + ]) + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['XRP/ETH'])) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) + + patched_configuration_load_config_file(mocker, default_conf) + + args = [ + 'backtesting', + '--config', 'config.json', + '--datadir', str(testdatadir), + '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'), + '--timeframe', '5m', + '--timeframe-detail', '1m', + '--strategy-list', + 'StrategyTestV2' + ] + args = get_args(args) + start_backtesting(args) + + # check the logs, that will contain the backtest result + exists = [ + 'Parameter -i/--timeframe detected ... Using timeframe: 5m ...', + 'Parameter --timeframe-detail detected, using 1m for intra-candle backtesting ...', + f'Using data directory: {testdatadir} ...', + 'Loading data from 2019-10-11 00:00:00 ' + 'up to 2019-10-13 11:10:00 (2 days).', + 'Backtesting with data from 2019-10-11 01:40:00 ' + 'up to 2019-10-13 11:10:00 (2 days).', + 'Running backtesting for Strategy StrategyTestV2', + ] + + for line in exists: + assert log_has(line, caplog) + + captured = capsys.readouterr() + assert 'BACKTESTING REPORT' in captured.out + assert 'SELL REASON STATS' in captured.out + assert 'LEFT OPEN TRADES REPORT' in captured.out diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 565d6077a..e4ce29d44 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -17,13 +17,10 @@ from freqtrade.optimize.hyperopt_auto import HyperOptAuto from freqtrade.optimize.hyperopt_tools import HyperoptTools from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.optimize.space import SKDecimal -from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver from freqtrade.strategy.hyper import IntParameter from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) -from .hyperopts.hyperopt_test_sep_file import HyperoptTestSepFile - def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) @@ -31,7 +28,7 @@ def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, ca args = [ 'hyperopt', '--config', 'config.json', - '--hyperopt', 'HyperoptTestSepFile', + '--strategy', 'HyperoptableStrategy', ] config = setup_optimize_configuration(get_args(args), RunMode.HYPEROPT) @@ -63,7 +60,7 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo args = [ 'hyperopt', '--config', 'config.json', - '--hyperopt', 'HyperoptTestSepFile', + '--strategy', 'HyperoptableStrategy', '--datadir', '/foo/bar', '--timeframe', '1m', '--timerange', ':100', @@ -115,7 +112,7 @@ def test_setup_hyperopt_configuration_stake_amount(mocker, default_conf) -> None args = [ 'hyperopt', '--config', 'config.json', - '--hyperopt', 'HyperoptTestSepFile', + '--strategy', 'HyperoptableStrategy', '--stake-amount', '1', '--starting-balance', '2' ] @@ -133,47 +130,6 @@ def test_setup_hyperopt_configuration_stake_amount(mocker, default_conf) -> None setup_optimize_configuration(get_args(args), RunMode.HYPEROPT) -def test_hyperoptresolver(mocker, default_conf, caplog) -> None: - patched_configuration_load_config_file(mocker, default_conf) - - hyperopt = HyperoptTestSepFile - delattr(hyperopt, 'populate_indicators') - delattr(hyperopt, 'populate_buy_trend') - delattr(hyperopt, 'populate_sell_trend') - mocker.patch( - 'freqtrade.resolvers.hyperopt_resolver.HyperOptResolver.load_object', - MagicMock(return_value=hyperopt(default_conf)) - ) - default_conf.update({'hyperopt': 'HyperoptTestSepFile'}) - x = HyperOptResolver.load_hyperopt(default_conf) - assert not hasattr(x, 'populate_indicators') - assert not hasattr(x, 'populate_buy_trend') - assert not hasattr(x, 'populate_sell_trend') - assert log_has("Hyperopt class does not provide populate_indicators() method. " - "Using populate_indicators from the strategy.", caplog) - assert log_has("Hyperopt class does not provide populate_sell_trend() method. " - "Using populate_sell_trend from the strategy.", caplog) - assert log_has("Hyperopt class does not provide populate_buy_trend() method. " - "Using populate_buy_trend from the strategy.", caplog) - assert hasattr(x, "ticker_interval") # DEPRECATED - assert hasattr(x, "timeframe") - - -def test_hyperoptresolver_wrongname(default_conf) -> None: - default_conf.update({'hyperopt': "NonExistingHyperoptClass"}) - - with pytest.raises(OperationalException, match=r'Impossible to load Hyperopt.*'): - HyperOptResolver.load_hyperopt(default_conf) - - -def test_hyperoptresolver_noname(default_conf): - default_conf['hyperopt'] = '' - with pytest.raises(OperationalException, - match="No Hyperopt set. Please use `--hyperopt` to specify " - "the Hyperopt class to use."): - HyperOptResolver.load_hyperopt(default_conf) - - def test_start_not_installed(mocker, default_conf, import_fails) -> None: start_mock = MagicMock() patched_configuration_load_config_file(mocker, default_conf) @@ -184,9 +140,7 @@ def test_start_not_installed(mocker, default_conf, import_fails) -> None: args = [ 'hyperopt', '--config', 'config.json', - '--hyperopt', 'HyperoptTestSepFile', - '--hyperopt-path', - str(Path(__file__).parent / "hyperopts"), + '--strategy', 'HyperoptableStrategy', '--epochs', '5', '--hyperopt-loss', 'SharpeHyperOptLossDaily', ] @@ -196,7 +150,7 @@ def test_start_not_installed(mocker, default_conf, import_fails) -> None: start_hyperopt(pargs) -def test_start(mocker, hyperopt_conf, caplog) -> None: +def test_start_no_hyperopt_allowed(mocker, hyperopt_conf, caplog) -> None: start_mock = MagicMock() patched_configuration_load_config_file(mocker, hyperopt_conf) mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) @@ -210,10 +164,8 @@ def test_start(mocker, hyperopt_conf, caplog) -> None: '--epochs', '5' ] pargs = get_args(args) - start_hyperopt(pargs) - - assert log_has('Starting freqtrade in Hyperopt mode', caplog) - assert start_mock.call_count == 1 + with pytest.raises(OperationalException, match=r"Using separate Hyperopt files has been.*"): + start_hyperopt(pargs) def test_start_no_data(mocker, hyperopt_conf) -> None: @@ -225,11 +177,11 @@ def test_start_no_data(mocker, hyperopt_conf) -> None: ) patch_exchange(mocker) - + # TODO: migrate to strategy-based hyperopt args = [ 'hyperopt', '--config', 'config.json', - '--hyperopt', 'HyperoptTestSepFile', + '--strategy', 'HyperoptableStrategy', '--hyperopt-loss', 'SharpeHyperOptLossDaily', '--epochs', '5' ] @@ -247,7 +199,7 @@ def test_start_filelock(mocker, hyperopt_conf, caplog) -> None: args = [ 'hyperopt', '--config', 'config.json', - '--hyperopt', 'HyperoptTestSepFile', + '--strategy', 'HyperoptableStrategy', '--hyperopt-loss', 'SharpeHyperOptLossDaily', '--epochs', '5' ] @@ -427,66 +379,14 @@ def test_hyperopt_format_results(hyperopt): def test_populate_indicators(hyperopt, testdatadir) -> None: data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True) dataframes = hyperopt.backtesting.strategy.advise_all_indicators(data) - dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], - {'pair': 'UNITTEST/BTC'}) + dataframe = dataframes['UNITTEST/BTC'] # Check if some indicators are generated. We will not test all of them assert 'adx' in dataframe - assert 'mfi' in dataframe + assert 'macd' in dataframe assert 'rsi' in dataframe -def test_buy_strategy_generator(hyperopt, testdatadir) -> None: - data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True) - dataframes = hyperopt.backtesting.strategy.advise_all_indicators(data) - dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], - {'pair': 'UNITTEST/BTC'}) - - populate_buy_trend = hyperopt.custom_hyperopt.buy_strategy_generator( - { - 'adx-value': 20, - 'fastd-value': 20, - 'mfi-value': 20, - 'rsi-value': 20, - 'adx-enabled': True, - 'fastd-enabled': True, - 'mfi-enabled': True, - 'rsi-enabled': True, - 'trigger': 'bb_lower' - } - ) - result = populate_buy_trend(dataframe, {'pair': 'UNITTEST/BTC'}) - # Check if some indicators are generated. We will not test all of them - assert 'buy' in result - assert 1 in result['buy'] - - -def test_sell_strategy_generator(hyperopt, testdatadir) -> None: - data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True) - dataframes = hyperopt.backtesting.strategy.advise_all_indicators(data) - dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], - {'pair': 'UNITTEST/BTC'}) - - populate_sell_trend = hyperopt.custom_hyperopt.sell_strategy_generator( - { - 'sell-adx-value': 20, - 'sell-fastd-value': 75, - 'sell-mfi-value': 80, - 'sell-rsi-value': 20, - 'sell-adx-enabled': True, - 'sell-fastd-enabled': True, - 'sell-mfi-enabled': True, - 'sell-rsi-enabled': True, - 'sell-trigger': 'sell-bb_upper' - } - ) - result = populate_sell_trend(dataframe, {'pair': 'UNITTEST/BTC'}) - # Check if some indicators are generated. We will not test all of them - print(result) - assert 'sell' in result - assert 1 in result['sell'] - - def test_generate_optimizer(mocker, hyperopt_conf) -> None: hyperopt_conf.update({'spaces': 'all', 'hyperopt_min_trades': 1, @@ -527,24 +427,12 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: mocker.patch('freqtrade.optimize.hyperopt.load', return_value={'XRP/BTC': None}) optimizer_param = { - 'adx-value': 0, - 'fastd-value': 35, - 'mfi-value': 0, - 'rsi-value': 0, - 'adx-enabled': False, - 'fastd-enabled': True, - 'mfi-enabled': False, - 'rsi-enabled': False, - 'trigger': 'macd_cross_signal', - 'sell-adx-value': 0, - 'sell-fastd-value': 75, - 'sell-mfi-value': 0, - 'sell-rsi-value': 0, - 'sell-adx-enabled': False, - 'sell-fastd-enabled': True, - 'sell-mfi-enabled': False, - 'sell-rsi-enabled': False, - 'sell-trigger': 'macd_cross_signal', + 'buy_plusdi': 0.02, + 'buy_rsi': 35, + 'sell_minusdi': 0.02, + 'sell_rsi': 75, + 'protection_cooldown_lookback': 20, + 'protection_enabled': True, 'roi_t1': 60.0, 'roi_t2': 30.0, 'roi_t3': 20.0, @@ -564,29 +452,19 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: '0.00003100 BTC ( 0.00%). ' 'Avg duration 0:50:00 min.' ), - 'params_details': {'buy': {'adx-enabled': False, - 'adx-value': 0, - 'fastd-enabled': True, - 'fastd-value': 35, - 'mfi-enabled': False, - 'mfi-value': 0, - 'rsi-enabled': False, - 'rsi-value': 0, - 'trigger': 'macd_cross_signal'}, + 'params_details': {'buy': {'buy_plusdi': 0.02, + 'buy_rsi': 35, + }, 'roi': {"0": 0.12000000000000001, "20.0": 0.02, "50.0": 0.01, "110.0": 0}, - 'protection': {}, - 'sell': {'sell-adx-enabled': False, - 'sell-adx-value': 0, - 'sell-fastd-enabled': True, - 'sell-fastd-value': 75, - 'sell-mfi-enabled': False, - 'sell-mfi-value': 0, - 'sell-rsi-enabled': False, - 'sell-rsi-value': 0, - 'sell-trigger': 'macd_cross_signal'}, + 'protection': {'protection_cooldown_lookback': 20, + 'protection_enabled': True, + }, + 'sell': {'sell_minusdi': 0.02, + 'sell_rsi': 75, + }, 'stoploss': {'stoploss': -0.4}, 'trailing': {'trailing_only_offset_is_reached': False, 'trailing_stop': True, @@ -808,11 +686,6 @@ def test_simplified_interface_roi_stoploss(mocker, hyperopt_conf, capsys) -> Non hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) - del hyperopt.custom_hyperopt.__class__.buy_strategy_generator - del hyperopt.custom_hyperopt.__class__.sell_strategy_generator - del hyperopt.custom_hyperopt.__class__.indicator_space - del hyperopt.custom_hyperopt.__class__.sell_indicator_space - hyperopt.start() parallel.assert_called_once() @@ -843,16 +716,14 @@ def test_simplified_interface_all_failed(mocker, hyperopt_conf) -> None: hyperopt_conf.update({'spaces': 'all', }) + mocker.patch('freqtrade.optimize.hyperopt_auto.HyperOptAuto._generate_indicator_space', + return_value=[]) + hyperopt = Hyperopt(hyperopt_conf) hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) - del hyperopt.custom_hyperopt.__class__.buy_strategy_generator - del hyperopt.custom_hyperopt.__class__.sell_strategy_generator - del hyperopt.custom_hyperopt.__class__.indicator_space - del hyperopt.custom_hyperopt.__class__.sell_indicator_space - - with pytest.raises(OperationalException, match=r"The 'buy' space is included into *"): + with pytest.raises(OperationalException, match=r"The 'protection' space is included into *"): hyperopt.start() @@ -889,11 +760,6 @@ def test_simplified_interface_buy(mocker, hyperopt_conf, capsys) -> None: hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) - # TODO: sell_strategy_generator() is actually not called because - # run_optimizer_parallel() is mocked - del hyperopt.custom_hyperopt.__class__.sell_strategy_generator - del hyperopt.custom_hyperopt.__class__.sell_indicator_space - hyperopt.start() parallel.assert_called_once() @@ -943,11 +809,6 @@ def test_simplified_interface_sell(mocker, hyperopt_conf, capsys) -> None: hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) - # TODO: buy_strategy_generator() is actually not called because - # run_optimizer_parallel() is mocked - del hyperopt.custom_hyperopt.__class__.buy_strategy_generator - del hyperopt.custom_hyperopt.__class__.indicator_space - hyperopt.start() parallel.assert_called_once() @@ -964,13 +825,12 @@ def test_simplified_interface_sell(mocker, hyperopt_conf, capsys) -> None: assert hasattr(hyperopt, "position_stacking") -@pytest.mark.parametrize("method,space", [ - ('buy_strategy_generator', 'buy'), - ('indicator_space', 'buy'), - ('sell_strategy_generator', 'sell'), - ('sell_indicator_space', 'sell'), +@pytest.mark.parametrize("space", [ + ('buy'), + ('sell'), + ('protection'), ]) -def test_simplified_interface_failed(mocker, hyperopt_conf, method, space) -> None: +def test_simplified_interface_failed(mocker, hyperopt_conf, space) -> None: mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', @@ -979,6 +839,8 @@ def test_simplified_interface_failed(mocker, hyperopt_conf, method, space) -> No 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) + mocker.patch('freqtrade.optimize.hyperopt_auto.HyperOptAuto._generate_indicator_space', + return_value=[]) patch_exchange(mocker) @@ -988,8 +850,6 @@ def test_simplified_interface_failed(mocker, hyperopt_conf, method, space) -> No hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) - delattr(hyperopt.custom_hyperopt.__class__, method) - with pytest.raises(OperationalException, match=f"The '{space}' space is included into *"): hyperopt.start() @@ -999,7 +859,6 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None: mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) (Path(tmpdir) / 'hyperopt_results').mkdir(parents=True) # No hyperopt needed - del hyperopt_conf['hyperopt'] hyperopt_conf.update({ 'strategy': 'HyperoptableStrategy', 'user_data_dir': Path(tmpdir), @@ -1025,6 +884,10 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None: assert hyperopt.backtesting.strategy.buy_rsi.value != 35 assert hyperopt.backtesting.strategy.sell_rsi.value != 74 + hyperopt.custom_hyperopt.generate_estimator = lambda *args, **kwargs: 'ET1' + with pytest.raises(OperationalException, match="Estimator ET1 not supported."): + hyperopt.get_optimizer([], 2) + def test_SKDecimal(): space = SKDecimal(1, 2, decimals=2) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 5f0701a22..1ce8d172c 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -4,6 +4,7 @@ import time from unittest.mock import MagicMock, PropertyMock import pytest +import time_machine from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.exceptions import OperationalException @@ -11,7 +12,8 @@ from freqtrade.persistence import Trade from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.resolvers import PairListResolver -from tests.conftest import get_patched_exchange, get_patched_freqtradebot, log_has, log_has_re +from tests.conftest import (create_mock_trades, get_patched_exchange, get_patched_freqtradebot, + log_has, log_has_re) @pytest.fixture(scope="function") @@ -662,6 +664,31 @@ def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None: assert log_has("PerformanceFilter is not available in this mode.", caplog) +@pytest.mark.usefixtures("init_persistence") +def test_PerformanceFilter_lookback(mocker, whitelist_conf, fee) -> None: + whitelist_conf['exchange']['pair_whitelist'].append('XRP/BTC') + whitelist_conf['pairlists'] = [ + {"method": "StaticPairList"}, + {"method": "PerformanceFilter", "minutes": 60} + ] + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + exchange = get_patched_exchange(mocker, whitelist_conf) + pm = PairListManager(exchange, whitelist_conf) + pm.refresh_pairlist() + + assert pm.whitelist == ['ETH/BTC', 'TKN/BTC', 'XRP/BTC'] + + with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: + create_mock_trades(fee) + pm.refresh_pairlist() + assert pm.whitelist == ['XRP/BTC', 'ETH/BTC', 'TKN/BTC'] + + # Move to "outside" of lookback window, so original sorting is restored. + t.move_to("2021-09-01 07:00:00 +00:00") + pm.refresh_pairlist() + assert pm.whitelist == ['ETH/BTC', 'TKN/BTC', 'XRP/BTC'] + + def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}] @@ -815,32 +842,63 @@ def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tick def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history): - ohlcv_data = { - ('ETH/BTC', '1d'): ohlcv_history, - ('TKN/BTC', '1d'): ohlcv_history, - ('LTC/BTC', '1d'): ohlcv_history, - } - mocker.patch.multiple('freqtrade.exchange.Exchange', - markets=PropertyMock(return_value=markets), - exchange_has=MagicMock(return_value=True), - get_tickers=tickers - ) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), - ) + with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history, + } + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers, + refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), + ) - freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter) - assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0 - freqtrade.pairlists.refresh_pairlist() - assert len(freqtrade.pairlists.whitelist) == 3 - assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0 + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter) + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0 + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 3 + assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0 - previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count - freqtrade.pairlists.refresh_pairlist() - assert len(freqtrade.pairlists.whitelist) == 3 - # Called once for XRP/BTC - assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1 + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 3 + # Call to XRP/BTC cached + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 2 + + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history, + ('XRP/BTC', '1d'): ohlcv_history.iloc[[0]], + } + mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data) + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 3 + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1 + + # Move to next day + t.move_to("2021-09-02 01:00:00 +00:00") + mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data) + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 3 + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1 + + # Move another day with fresh mocks (now the pair is old enough) + t.move_to("2021-09-03 01:00:00 +00:00") + # Called once for XRP/BTC + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history, + ('XRP/BTC', '1d'): ohlcv_history, + } + mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data) + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 4 + # Called once (only for XRP/BTC) + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1 def test_OffsetFilter_error(mocker, whitelist_conf) -> None: diff --git a/tests/plugins/test_pairlocks.py b/tests/plugins/test_pairlocks.py index fce3a8cd1..c694fd7c1 100644 --- a/tests/plugins/test_pairlocks.py +++ b/tests/plugins/test_pairlocks.py @@ -68,7 +68,7 @@ def test_PairLocks(use_db): # Global lock PairLocks.lock_pair('*', lock_time) assert PairLocks.is_global_lock(lock_time + timedelta(minutes=-50)) - # Global lock also locks every pair seperately + # Global lock also locks every pair separately assert PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=-50)) assert PairLocks.is_pair_locked('XRP/USDT', lock_time + timedelta(minutes=-50)) diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index c0a9ae72a..a3cb29c9d 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -125,7 +125,7 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): # Test 5m after lock-period - this should try and relock the pair, but end-time # should be the previous end-time end_time = PairLocks.get_pair_longest_lock('*').lock_end_time + timedelta(minutes=5) - assert freqtrade.protections.global_stop(end_time) + freqtrade.protections.global_stop(end_time) assert not PairLocks.is_global_lock(end_time) @@ -182,7 +182,7 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair min_ago_open=180, min_ago_close=30, profit_rate=0.9, )) - assert freqtrade.protections.stop_per_pair(pair) + freqtrade.protections.stop_per_pair(pair) assert freqtrade.protections.global_stop() != only_per_pair assert PairLocks.is_pair_locked(pair) assert PairLocks.is_global_lock() != only_per_pair diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 2852486ed..7c98b2df7 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -422,20 +422,22 @@ def test_api_stopbuy(botclient): assert ftbot.config['max_open_trades'] == 0 -def test_api_balance(botclient, mocker, rpc_balance): +def test_api_balance(botclient, mocker, rpc_balance, tickers): ftbot, client = botclient ftbot.config['dry_run'] = False mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value=rpc_balance) + mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) mocker.patch('freqtrade.exchange.Exchange.get_valid_pair_combination', side_effect=lambda a, b: f"{a}/{b}") ftbot.wallets.update() rc = client_get(client, f"{BASE_URI}/balance") assert_response(rc) - assert "currencies" in rc.json() - assert len(rc.json()["currencies"]) == 5 - assert rc.json()['currencies'][0] == { + response = rc.json() + assert "currencies" in response + assert len(response["currencies"]) == 5 + assert response['currencies'][0] == { 'currency': 'BTC', 'free': 12.0, 'balance': 12.0, @@ -443,6 +445,10 @@ def test_api_balance(botclient, mocker, rpc_balance): 'est_stake': 12.0, 'stake': 'BTC', } + assert 'starting_capital' in response + assert 'starting_capital_fiat' in response + assert 'starting_capital_pct' in response + assert 'starting_capital_ratio' in response def test_api_count(botclient, mocker, ticker, fee, markets): @@ -1218,6 +1224,7 @@ def test_api_strategies(botclient): assert_response(rc) assert rc.json() == {'strategies': [ 'HyperoptableStrategy', + 'InformativeDecoratorTest', 'StrategyTestV2', 'TestStrategyLegacyV1' ]} diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 2013dad7d..7dde7b803 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -576,6 +576,8 @@ def test_balance_handle_too_large_response(default_conf, update, mocker) -> None 'total': 100.0, 'symbol': 100.0, 'value': 1000.0, + 'starting_capital': 1000, + 'starting_capital_fiat': 1000, }) telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) @@ -1311,6 +1313,34 @@ def test_send_msg_buy_cancel_notification(default_conf, mocker) -> None: 'Reason: cancelled due to timeout.') +def test_send_msg_protection_notification(default_conf, mocker, time_machine) -> None: + + default_conf['telegram']['notification_settings']['protection_trigger'] = 'on' + + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) + time_machine.move_to("2021-09-01 05:00:00 +00:00") + lock = PairLocks.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=6).datetime, 'randreason') + msg = { + 'type': RPCMessageType.PROTECTION_TRIGGER, + } + msg.update(lock.to_json()) + telegram.send_msg(msg) + assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. " + "`ETH/BTC` will be locked until `2021-09-01 05:10:00`.") + + msg_mock.reset_mock() + # Test global protection + + msg = { + 'type': RPCMessageType.PROTECTION_TRIGGER_GLOBAL, + } + lock = PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=100).datetime, 'randreason') + msg.update(lock.to_json()) + telegram.send_msg(msg) + assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. " + "*All pairs* will be locked until `2021-09-01 06:45:00`.") + + def test_send_msg_buy_fill_notification(default_conf, mocker) -> None: default_conf['telegram']['notification_settings']['buy_fill'] = 'on' diff --git a/tests/strategy/strats/informative_decorator_strategy.py b/tests/strategy/strats/informative_decorator_strategy.py new file mode 100644 index 000000000..a32ad79e8 --- /dev/null +++ b/tests/strategy/strats/informative_decorator_strategy.py @@ -0,0 +1,75 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + +from pandas import DataFrame + +from freqtrade.strategy import informative, merge_informative_pair +from freqtrade.strategy.interface import IStrategy + + +class InformativeDecoratorTest(IStrategy): + """ + Strategy used by tests freqtrade bot. + Please do not modify this strategy, it's intended for internal use only. + Please look at the SampleStrategy in the user_data/strategy directory + or strategy repository https://github.com/freqtrade/freqtrade-strategies + for samples and inspiration. + """ + INTERFACE_VERSION = 2 + stoploss = -0.10 + timeframe = '5m' + startup_candle_count: int = 20 + + def informative_pairs(self): + return [('BTC/USDT', '5m')] + + def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['buy'] = 0 + return dataframe + + def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['sell'] = 0 + return dataframe + + # Decorator stacking test. + @informative('30m') + @informative('1h') + def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = 14 + return dataframe + + # Simple informative test. + @informative('1h', 'BTC/{stake}') + def populate_indicators_btc_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = 14 + return dataframe + + # Quote currency different from stake currency test. + @informative('1h', 'ETH/BTC') + def populate_indicators_eth_btc_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = 14 + return dataframe + + # Formatting test. + @informative('30m', 'BTC/{stake}', '{column}_{BASE}_{QUOTE}_{base}_{quote}_{asset}_{timeframe}') + def populate_indicators_btc_1h_2(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = 14 + return dataframe + + # Custom formatter test + @informative('30m', 'ETH/{stake}', fmt=lambda column, **kwargs: column + '_from_callable') + def populate_indicators_eth_30m(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['rsi'] = 14 + return dataframe + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + # Strategy timeframe indicators for current pair. + dataframe['rsi'] = 14 + # Informative pairs are available in this method. + dataframe['rsi_less'] = dataframe['rsi'] < dataframe['rsi_1h'] + + # Mixing manual informative pairs with decorators. + informative = self.dp.get_pair_dataframe('BTC/USDT', '5m') + informative['rsi'] = 14 + dataframe = merge_informative_pair(dataframe, informative, self.timeframe, '5m', ffill=True) + + return dataframe diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 128599668..dcb9e3e64 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -607,7 +607,7 @@ def test_is_informative_pairs_callback(default_conf): strategy = StrategyResolver.load_strategy(default_conf) # Should return empty # Uses fallback to base implementation - assert [] == strategy.informative_pairs() + assert [] == strategy.gather_informative_pairs() @pytest.mark.parametrize('error', [ @@ -735,11 +735,16 @@ def test_auto_hyperopt_interface(default_conf): PairLocks.timeframe = default_conf['timeframe'] strategy = StrategyResolver.load_strategy(default_conf) + with pytest.raises(OperationalException): + next(strategy.enumerate_parameters('deadBeef')) + assert strategy.buy_rsi.value == strategy.buy_params['buy_rsi'] # PlusDI is NOT in the buy-params, so default should be used assert strategy.buy_plusdi.value == 0.5 assert strategy.sell_rsi.value == strategy.sell_params['sell_rsi'] + assert repr(strategy.sell_rsi) == 'IntParameter(74)' + # Parameter is disabled - so value from sell_param dict will NOT be used. assert strategy.sell_minusdi.value == 0.5 all_params = strategy.detect_all_parameters() diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index 3b84fc254..a01b55050 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -4,7 +4,9 @@ import numpy as np import pandas as pd import pytest -from freqtrade.strategy import merge_informative_pair, stoploss_from_open, timeframe_to_minutes +from freqtrade.data.dataprovider import DataProvider +from freqtrade.strategy import (merge_informative_pair, stoploss_from_absolute, stoploss_from_open, + timeframe_to_minutes) def generate_test_data(timeframe: str, size: int): @@ -132,3 +134,65 @@ def test_stoploss_from_open(): assert stoploss == 0 else: assert isclose(stop_price, expected_stop_price, rel_tol=0.00001) + + +def test_stoploss_from_absolute(): + assert stoploss_from_absolute(90, 100) == 1 - (90 / 100) + assert stoploss_from_absolute(100, 100) == 0 + assert stoploss_from_absolute(110, 100) == 0 + assert stoploss_from_absolute(100, 0) == 1 + assert stoploss_from_absolute(0, 100) == 1 + + +def test_informative_decorator(mocker, default_conf): + test_data_5m = generate_test_data('5m', 40) + test_data_30m = generate_test_data('30m', 40) + test_data_1h = generate_test_data('1h', 40) + data = { + ('XRP/USDT', '5m'): test_data_5m, + ('XRP/USDT', '30m'): test_data_30m, + ('XRP/USDT', '1h'): test_data_1h, + ('LTC/USDT', '5m'): test_data_5m, + ('LTC/USDT', '30m'): test_data_30m, + ('LTC/USDT', '1h'): test_data_1h, + ('BTC/USDT', '30m'): test_data_30m, + ('BTC/USDT', '5m'): test_data_5m, + ('BTC/USDT', '1h'): test_data_1h, + ('ETH/USDT', '1h'): test_data_1h, + ('ETH/USDT', '30m'): test_data_30m, + ('ETH/BTC', '1h'): test_data_1h, + } + from .strats.informative_decorator_strategy import InformativeDecoratorTest + default_conf['stake_currency'] = 'USDT' + strategy = InformativeDecoratorTest(config=default_conf) + strategy.dp = DataProvider({}, None, None) + mocker.patch.object(strategy.dp, 'current_whitelist', return_value=[ + 'XRP/USDT', 'LTC/USDT', 'BTC/USDT' + ]) + + assert len(strategy._ft_informative) == 6 # Equal to number of decorators used + informative_pairs = [('XRP/USDT', '1h'), ('LTC/USDT', '1h'), ('XRP/USDT', '30m'), + ('LTC/USDT', '30m'), ('BTC/USDT', '1h'), ('BTC/USDT', '30m'), + ('BTC/USDT', '5m'), ('ETH/BTC', '1h'), ('ETH/USDT', '30m')] + for inf_pair in informative_pairs: + assert inf_pair in strategy.gather_informative_pairs() + + def test_historic_ohlcv(pair, timeframe): + return data[(pair, timeframe or strategy.timeframe)].copy() + mocker.patch('freqtrade.data.dataprovider.DataProvider.historic_ohlcv', + side_effect=test_historic_ohlcv) + + analyzed = strategy.advise_all_indicators( + {p: data[(p, strategy.timeframe)] for p in ('XRP/USDT', 'LTC/USDT')}) + expected_columns = [ + 'rsi_1h', 'rsi_30m', # Stacked informative decorators + 'btc_usdt_rsi_1h', # BTC 1h informative + 'rsi_BTC_USDT_btc_usdt_BTC/USDT_30m', # Column formatting + 'rsi_from_callable', # Custom column formatter + 'eth_btc_rsi_1h', # Quote currency not matching stake currency + 'rsi', 'rsi_less', # Non-informative columns + 'rsi_5m', # Manual informative dataframe + ] + for _, dataframe in analyzed.items(): + for col in expected_columns: + assert col in dataframe.columns diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index 2cbc9d0c6..3a30a824a 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -35,7 +35,7 @@ def test_search_all_strategies_no_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver.search_all_objects(directory, enum_failed=False) assert isinstance(strategies, list) - assert len(strategies) == 3 + assert len(strategies) == 4 assert isinstance(strategies[0], dict) @@ -43,10 +43,10 @@ def test_search_all_strategies_with_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver.search_all_objects(directory, enum_failed=True) assert isinstance(strategies, list) - assert len(strategies) == 4 + assert len(strategies) == 5 # with enum_failed=True search_all_objects() shall find 2 good strategies # and 1 which fails to load - assert len([x for x in strategies if x['class'] is not None]) == 3 + assert len([x for x in strategies if x['class'] is not None]) == 4 assert len([x for x in strategies if x['class'] is None]) == 1 diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 9aea4fa11..1ce45e4d5 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -11,8 +11,7 @@ import pytest from jsonschema import ValidationError from freqtrade.commands import Arguments -from freqtrade.configuration import (Configuration, check_exchange, remove_credentials, - validate_config_consistency) +from freqtrade.configuration import Configuration, check_exchange, validate_config_consistency from freqtrade.configuration.config_validation import validate_config_schema from freqtrade.configuration.deprecated_settings import (check_conflicting_settings, process_deprecated_setting, @@ -617,18 +616,6 @@ def test_check_exchange(default_conf, caplog) -> None: check_exchange(default_conf) -def test_remove_credentials(default_conf, caplog) -> None: - conf = deepcopy(default_conf) - conf['dry_run'] = False - remove_credentials(conf) - - assert conf['dry_run'] is True - assert conf['exchange']['key'] == '' - assert conf['exchange']['secret'] == '' - assert conf['exchange']['password'] == '' - assert conf['exchange']['uid'] == '' - - def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) diff --git a/tests/test_directory_operations.py b/tests/test_directory_operations.py index a11200526..905b078f9 100644 --- a/tests/test_directory_operations.py +++ b/tests/test_directory_operations.py @@ -74,16 +74,12 @@ def test_copy_sample_files(mocker, default_conf, caplog) -> None: copymock = mocker.patch('shutil.copy', MagicMock()) copy_sample_files(Path('/tmp/bar')) - assert copymock.call_count == 5 + assert copymock.call_count == 3 assert copymock.call_args_list[0][0][1] == str( Path('/tmp/bar') / 'strategies/sample_strategy.py') assert copymock.call_args_list[1][0][1] == str( - Path('/tmp/bar') / 'hyperopts/sample_hyperopt_advanced.py') - assert copymock.call_args_list[2][0][1] == str( Path('/tmp/bar') / 'hyperopts/sample_hyperopt_loss.py') - assert copymock.call_args_list[3][0][1] == str( - Path('/tmp/bar') / 'hyperopts/sample_hyperopt.py') - assert copymock.call_args_list[4][0][1] == str( + assert copymock.call_args_list[2][0][1] == str( Path('/tmp/bar') / 'notebooks/strategy_analysis_example.ipynb') diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 75b67e59c..760a9dee7 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -78,11 +78,15 @@ def test_bot_cleanup(mocker, default_conf, caplog) -> None: assert coo_mock.call_count == 1 -def test_order_dict_dry_run(default_conf, mocker, caplog) -> None: +@pytest.mark.parametrize('runmode', [ + RunMode.DRY_RUN, + RunMode.LIVE +]) +def test_order_dict(default_conf, mocker, runmode, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) conf = default_conf.copy() - conf['runmode'] = RunMode.DRY_RUN + conf['runmode'] = runmode conf['order_types'] = { 'buy': 'market', 'sell': 'limit', @@ -92,45 +96,14 @@ def test_order_dict_dry_run(default_conf, mocker, caplog) -> None: conf['bid_strategy']['price_side'] = 'ask' freqtrade = FreqtradeBot(conf) + if runmode == RunMode.LIVE: + assert not log_has_re(".*stoploss_on_exchange .* dry-run", caplog) assert freqtrade.strategy.order_types['stoploss_on_exchange'] caplog.clear() # is left untouched conf = default_conf.copy() - conf['runmode'] = RunMode.DRY_RUN - conf['order_types'] = { - 'buy': 'market', - 'sell': 'limit', - 'stoploss': 'limit', - 'stoploss_on_exchange': False, - } - freqtrade = FreqtradeBot(conf) - assert not freqtrade.strategy.order_types['stoploss_on_exchange'] - assert not log_has_re(".*stoploss_on_exchange .* dry-run", caplog) - - -def test_order_dict_live(default_conf, mocker, caplog) -> None: - patch_RPCManager(mocker) - patch_exchange(mocker) - - conf = default_conf.copy() - conf['runmode'] = RunMode.LIVE - conf['order_types'] = { - 'buy': 'market', - 'sell': 'limit', - 'stoploss': 'limit', - 'stoploss_on_exchange': True, - } - conf['bid_strategy']['price_side'] = 'ask' - - freqtrade = FreqtradeBot(conf) - assert not log_has_re(".*stoploss_on_exchange .* dry-run", caplog) - assert freqtrade.strategy.order_types['stoploss_on_exchange'] - - caplog.clear() - # is left untouched - conf = default_conf.copy() - conf['runmode'] = RunMode.LIVE + conf['runmode'] = runmode conf['order_types'] = { 'buy': 'market', 'sell': 'limit', @@ -219,8 +192,14 @@ def test_edge_overrides_stake_amount(mocker, edge_conf) -> None: 'LTC/BTC', freqtrade.edge) == (999.9 * 0.5 * 0.01) / 0.21 -def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, edge_conf) -> None: - +@pytest.mark.parametrize('buy_price_mult,ignore_strat_sl', [ + # Override stoploss + (0.79, False), + # Override strategy stoploss + (0.85, True) +]) +def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, + buy_price_mult, ignore_strat_sl, edge_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) patch_edge(mocker) @@ -234,9 +213,9 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, edge_conf mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ - 'bid': buy_price * 0.79, - 'ask': buy_price * 0.79, - 'last': buy_price * 0.79 + 'bid': buy_price * buy_price_mult, + 'ask': buy_price * buy_price_mult, + 'last': buy_price * buy_price_mult, }), get_fee=fee, ) @@ -253,46 +232,10 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, edge_conf ############################################# # stoploss shoud be hit - assert freqtrade.handle_trade(trade) is True - assert log_has('Executing Sell for NEO/BTC. Reason: stop_loss', caplog) - assert trade.sell_reason == SellType.STOP_LOSS.value - - -def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, - mocker, edge_conf) -> None: - patch_RPCManager(mocker) - patch_exchange(mocker) - patch_edge(mocker) - edge_conf['max_open_trades'] = float('inf') - - # Strategy stoploss is -0.1 but Edge imposes a stoploss at -0.2 - # Thus, if price falls 15%, stoploss should not be triggered - # - # mocking the ticker: price is falling ... - buy_price = limit_buy_order['price'] - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=MagicMock(return_value={ - 'bid': buy_price * 0.85, - 'ask': buy_price * 0.85, - 'last': buy_price * 0.85 - }), - get_fee=fee, - ) - ############################################# - - # Create a trade with "limit_buy_order" price - freqtrade = FreqtradeBot(edge_conf) - freqtrade.active_pair_whitelist = ['NEO/BTC'] - patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.enter_positions() - trade = Trade.query.first() - trade.update(limit_buy_order) - ############################################# - - # stoploss shoud not be hit - assert freqtrade.handle_trade(trade) is False + assert freqtrade.handle_trade(trade) is not ignore_strat_sl + if not ignore_strat_sl: + assert log_has('Executing Sell for NEO/BTC. Reason: stop_loss', caplog) + assert trade.sell_reason == SellType.STOP_LOSS.value def test_total_open_trades_stakes(mocker, default_conf, ticker, fee) -> None: @@ -376,8 +319,16 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, freqtrade.create_trade('ETH/BTC') -def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order_open, - fee, mocker) -> None: +@pytest.mark.parametrize('stake_amount,create,amount_enough,max_open_trades', [ + (0.0005, True, True, 99), + (0.000000005, True, False, 99), + (0, False, True, 99), + (UNLIMITED_STAKE_AMOUNT, False, True, 0), +]) +def test_create_trade_minimal_amount( + default_conf, ticker, limit_buy_order_open, fee, mocker, + stake_amount, create, amount_enough, max_open_trades, caplog +) -> None: patch_RPCManager(mocker) patch_exchange(mocker) buy_mock = MagicMock(return_value=limit_buy_order_open) @@ -387,78 +338,33 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order_open, create_order=buy_mock, get_fee=fee, ) - default_conf['stake_amount'] = 0.0005 + default_conf['max_open_trades'] = max_open_trades freqtrade = FreqtradeBot(default_conf) + freqtrade.config['stake_amount'] = stake_amount patch_get_signal(freqtrade) - freqtrade.create_trade('ETH/BTC') - rate, amount = buy_mock.call_args[1]['rate'], buy_mock.call_args[1]['amount'] - assert rate * amount <= default_conf['stake_amount'] - - -def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_order_open, - fee, mocker, caplog) -> None: - patch_RPCManager(mocker) - patch_exchange(mocker) - buy_mock = MagicMock(return_value=limit_buy_order_open) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, - create_order=buy_mock, - get_fee=fee, - ) - - freqtrade = FreqtradeBot(default_conf) - freqtrade.config['stake_amount'] = 0.000000005 - - patch_get_signal(freqtrade) - - assert freqtrade.create_trade('ETH/BTC') - assert log_has_re(r"Stake amount for pair .* is too small.*", caplog) - - -def test_create_trade_zero_stake_amount(default_conf, ticker, limit_buy_order_open, - fee, mocker) -> None: - patch_RPCManager(mocker) - patch_exchange(mocker) - buy_mock = MagicMock(return_value=limit_buy_order_open) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, - create_order=buy_mock, - get_fee=fee, - ) - - freqtrade = FreqtradeBot(default_conf) - freqtrade.config['stake_amount'] = 0 - - patch_get_signal(freqtrade) - - assert not freqtrade.create_trade('ETH/BTC') - - -def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order_open, - fee, mocker) -> None: - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, - create_order=MagicMock(return_value=limit_buy_order_open), - get_fee=fee, - ) - default_conf['max_open_trades'] = 0 - default_conf['stake_amount'] = UNLIMITED_STAKE_AMOUNT - - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) - - assert not freqtrade.create_trade('ETH/BTC') - assert freqtrade.wallets.get_trade_stake_amount('ETH/BTC', freqtrade.edge) == 0 + if create: + assert freqtrade.create_trade('ETH/BTC') + if amount_enough: + rate, amount = buy_mock.call_args[1]['rate'], buy_mock.call_args[1]['amount'] + assert rate * amount <= default_conf['stake_amount'] + else: + assert log_has_re( + r"Stake amount for pair .* is too small.*", + caplog + ) + else: + assert not freqtrade.create_trade('ETH/BTC') + if not max_open_trades: + assert freqtrade.wallets.get_trade_stake_amount('ETH/BTC', freqtrade.edge) == 0 +@pytest.mark.parametrize('whitelist,positions', [ + (["ETH/BTC"], 1), # No pairs left + ([], 0), # No pairs in whitelist +]) def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order_open, fee, - mocker, caplog) -> None: + whitelist, positions, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -467,36 +373,20 @@ def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order_ope create_order=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) - - default_conf['exchange']['pair_whitelist'] = ["ETH/BTC"] + default_conf['exchange']['pair_whitelist'] = whitelist freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) n = freqtrade.enter_positions() - assert n == 1 - assert not log_has_re(r"No currency pair in active pair whitelist.*", caplog) - n = freqtrade.enter_positions() - assert n == 0 - assert log_has_re(r"No currency pair in active pair whitelist.*", caplog) - - -def test_enter_positions_no_pairs_in_whitelist(default_conf, ticker, limit_buy_order, fee, - mocker, caplog) -> None: - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, - create_order=MagicMock(return_value={'id': limit_buy_order['id']}), - get_fee=fee, - ) - default_conf['exchange']['pair_whitelist'] = [] - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) - - n = freqtrade.enter_positions() - assert n == 0 - assert log_has("Active pair whitelist is empty.", caplog) + assert n == positions + if positions: + assert not log_has_re(r"No currency pair in active pair whitelist.*", caplog) + n = freqtrade.enter_positions() + assert n == 0 + assert log_has_re(r"No currency pair in active pair whitelist.*", caplog) + else: + assert n == 0 + assert log_has("Active pair whitelist is empty.", caplog) @pytest.mark.usefixtures("init_persistence") @@ -518,6 +408,7 @@ def test_enter_positions_global_pairlock(default_conf, ticker, limit_buy_order, # 0 trades, but it's not because of pairlock. assert n == 0 assert not log_has_re(message, caplog) + caplog.clear() PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=20).datetime, 'Just because') n = freqtrade.enter_positions() @@ -525,6 +416,29 @@ def test_enter_positions_global_pairlock(default_conf, ticker, limit_buy_order, assert log_has_re(message, caplog) +def test_handle_protections(mocker, default_conf, fee): + default_conf['protections'] = [ + {"method": "CooldownPeriod", "stop_duration": 60}, + { + "method": "StoplossGuard", + "lookback_period_candles": 24, + "trade_limit": 4, + "stop_duration_candles": 4, + "only_per_pair": False + } + ] + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade.protections._protection_handlers[1].global_stop = MagicMock( + return_value=(True, arrow.utcnow().shift(hours=1).datetime, "asdf")) + create_mock_trades(fee) + freqtrade.handle_protections('ETC/BTC') + send_msg_mock = freqtrade.rpc.send_msg + assert send_msg_mock.call_count == 2 + assert send_msg_mock.call_args_list[0][0][0]['type'] == RPCMessageType.PROTECTION_TRIGGER + assert send_msg_mock.call_args_list[1][0][0]['type'] == RPCMessageType.PROTECTION_TRIGGER_GLOBAL + + def test_create_trade_no_signal(default_conf, fee, mocker) -> None: default_conf['dry_run'] = True @@ -1086,6 +1000,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, assert log_has_re(r'STOP_LOSS_LIMIT is hit for Trade\(id=1, .*\)\.', caplog) assert trade.stoploss_order_id is None assert trade.is_open is False + caplog.clear() mocker.patch( 'freqtrade.exchange.Binance.stoploss', @@ -1190,7 +1105,7 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, assert trade.stoploss_order_id is None assert trade.sell_reason == SellType.EMERGENCY_SELL.value assert log_has("Unable to place a stoploss order on exchange. ", caplog) - assert log_has("Selling the trade forcefully", caplog) + assert log_has("Exiting the trade forcefully", caplog) # Should call a market sell assert create_order_mock.call_count == 2 @@ -1651,30 +1566,27 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, stop_price=0.00002346 * 0.99) -def test_enter_positions(mocker, default_conf, caplog) -> None: +@pytest.mark.parametrize('return_value,side_effect,log_message', [ + (False, None, 'Found no buy signals for whitelisted currencies. Trying again...'), + (None, DependencyException, 'Unable to create trade for ETH/BTC: ') +]) +def test_enter_positions(mocker, default_conf, return_value, side_effect, + log_message, caplog) -> None: caplog.set_level(logging.DEBUG) freqtrade = get_patched_freqtradebot(mocker, default_conf) - mock_ct = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trade', - MagicMock(return_value=False)) - n = freqtrade.enter_positions() - assert n == 0 - assert log_has('Found no buy signals for whitelisted currencies. Trying again...', caplog) - # create_trade should be called once for every pair in the whitelist. - assert mock_ct.call_count == len(default_conf['exchange']['pair_whitelist']) - - -def test_enter_positions_exception(mocker, default_conf, caplog) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) - mock_ct = mocker.patch( 'freqtrade.freqtradebot.FreqtradeBot.create_trade', - MagicMock(side_effect=DependencyException) + MagicMock( + return_value=return_value, + side_effect=side_effect + ) ) n = freqtrade.enter_positions() assert n == 0 + assert log_has(log_message, caplog) + # create_trade should be called once for every pair in the whitelist. assert mock_ct.call_count == len(default_conf['exchange']['pair_whitelist']) - assert log_has('Unable to create trade for ETH/BTC: ', caplog) def test_exit_positions(mocker, default_conf, limit_buy_order, caplog) -> None: @@ -1743,10 +1655,12 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No ) assert not freqtrade.update_trade_state(trade, None) assert log_has_re(r'Orderid for trade .* is empty.', caplog) + caplog.clear() # Add datetime explicitly since sqlalchemy defaults apply only once written to database freqtrade.update_trade_state(trade, '123') # Test amount not modified by fee-logic assert not log_has_re(r'Applying fee to .*', caplog) + caplog.clear() assert trade.open_order_id is None assert trade.amount == limit_buy_order['amount'] @@ -1766,8 +1680,13 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No assert log_has_re('Found open order for.*', caplog) +@pytest.mark.parametrize('initial_amount,has_rounding_fee', [ + (90.99181073 + 1e-14, True), + (8.0, False) +]) def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_buy_order, fee, - mocker): + mocker, initial_amount, has_rounding_fee, caplog): + trades_for_order[0]['amount'] = initial_amount mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) # fetch_order should not be called!! mocker.patch('freqtrade.exchange.Exchange.fetch_order', MagicMock(side_effect=ValueError)) @@ -1788,32 +1707,8 @@ def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_ freqtrade.update_trade_state(trade, '123456', limit_buy_order) assert trade.amount != amount assert trade.amount == limit_buy_order['amount'] - - -def test_update_trade_state_withorderdict_rounding_fee(default_conf, trades_for_order, fee, - limit_buy_order, mocker, caplog): - trades_for_order[0]['amount'] = limit_buy_order['amount'] + 1e-14 - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) - # fetch_order should not be called!! - mocker.patch('freqtrade.exchange.Exchange.fetch_order', MagicMock(side_effect=ValueError)) - patch_exchange(mocker) - amount = sum(x['amount'] for x in trades_for_order) - freqtrade = get_patched_freqtradebot(mocker, default_conf) - trade = Trade( - pair='LTC/ETH', - amount=amount, - exchange='binance', - open_rate=0.245441, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_order_id='123456', - is_open=True, - open_date=arrow.utcnow().datetime, - ) - freqtrade.update_trade_state(trade, '123456', limit_buy_order) - assert trade.amount != amount - assert trade.amount == limit_buy_order['amount'] - assert log_has_re(r'Applying fee on amount for .*', caplog) + if has_rounding_fee: + assert log_has_re(r'Applying fee on amount for .*', caplog) def test_update_trade_state_exception(mocker, default_conf, @@ -2453,8 +2348,8 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', - handle_cancel_buy=MagicMock(), - handle_cancel_sell=MagicMock(), + handle_cancel_enter=MagicMock(), + handle_cancel_exit=MagicMock(), ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -2474,7 +2369,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke caplog) -def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> None: +def test_handle_cancel_enter(mocker, caplog, default_conf, limit_buy_order) -> None: patch_RPCManager(mocker) patch_exchange(mocker) cancel_buy_order = deepcopy(limit_buy_order) @@ -2485,7 +2380,7 @@ def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> Non mocker.patch('freqtrade.exchange.Exchange.cancel_order_with_result', cancel_order_mock) freqtrade = FreqtradeBot(default_conf) - freqtrade._notify_buy_cancel = MagicMock() + freqtrade._notify_enter_cancel = MagicMock() trade = MagicMock() trade.pair = 'LTC/USDT' @@ -2493,46 +2388,46 @@ def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> Non limit_buy_order['filled'] = 0.0 limit_buy_order['status'] = 'open' reason = CANCEL_REASON['TIMEOUT'] - assert freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) + assert freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 cancel_order_mock.reset_mock() caplog.clear() limit_buy_order['filled'] = 0.01 - assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) + assert not freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 0 assert log_has_re("Order .* for .* not cancelled, as the filled amount.* unsellable.*", caplog) caplog.clear() cancel_order_mock.reset_mock() limit_buy_order['filled'] = 2 - assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) + assert not freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 # Order remained open for some reason (cancel failed) cancel_buy_order['status'] = 'open' cancel_order_mock = MagicMock(return_value=cancel_buy_order) mocker.patch('freqtrade.exchange.Exchange.cancel_order_with_result', cancel_order_mock) - assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) + assert not freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) assert log_has_re(r"Order .* for .* not cancelled.", caplog) @pytest.mark.parametrize("limit_buy_order_canceled_empty", ['binance', 'ftx', 'kraken', 'bittrex'], indirect=['limit_buy_order_canceled_empty']) -def test_handle_cancel_buy_exchanges(mocker, caplog, default_conf, - limit_buy_order_canceled_empty) -> None: +def test_handle_cancel_enter_exchanges(mocker, caplog, default_conf, + limit_buy_order_canceled_empty) -> None: patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = mocker.patch( 'freqtrade.exchange.Exchange.cancel_order_with_result', return_value=limit_buy_order_canceled_empty) - nofiy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot._notify_buy_cancel') + nofiy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot._notify_enter_cancel') freqtrade = FreqtradeBot(default_conf) reason = CANCEL_REASON['TIMEOUT'] trade = MagicMock() trade.pair = 'LTC/ETH' - assert freqtrade.handle_cancel_buy(trade, limit_buy_order_canceled_empty, reason) + assert freqtrade.handle_cancel_enter(trade, limit_buy_order_canceled_empty, reason) assert cancel_order_mock.call_count == 0 assert log_has_re(r'Buy order fully cancelled. Removing .* from database\.', caplog) assert nofiy_mock.call_count == 1 @@ -2544,8 +2439,8 @@ def test_handle_cancel_buy_exchanges(mocker, caplog, default_conf, 'String Return value', 123 ]) -def test_handle_cancel_buy_corder_empty(mocker, default_conf, limit_buy_order, - cancelorder) -> None: +def test_handle_cancel_enter_corder_empty(mocker, default_conf, limit_buy_order, + cancelorder) -> None: patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = MagicMock(return_value=cancelorder) @@ -2555,7 +2450,7 @@ def test_handle_cancel_buy_corder_empty(mocker, default_conf, limit_buy_order, ) freqtrade = FreqtradeBot(default_conf) - freqtrade._notify_buy_cancel = MagicMock() + freqtrade._notify_enter_cancel = MagicMock() trade = MagicMock() trade.pair = 'LTC/USDT' @@ -2563,16 +2458,16 @@ def test_handle_cancel_buy_corder_empty(mocker, default_conf, limit_buy_order, limit_buy_order['filled'] = 0.0 limit_buy_order['status'] = 'open' reason = CANCEL_REASON['TIMEOUT'] - assert freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) + assert freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 cancel_order_mock.reset_mock() limit_buy_order['filled'] = 1.0 - assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) + assert not freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 -def test_handle_cancel_sell_limit(mocker, default_conf, fee) -> None: +def test_handle_cancel_exit_limit(mocker, default_conf, fee) -> None: send_msg_mock = patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = MagicMock() @@ -2598,26 +2493,26 @@ def test_handle_cancel_sell_limit(mocker, default_conf, fee) -> None: 'amount': 1, 'status': "open"} reason = CANCEL_REASON['TIMEOUT'] - assert freqtrade.handle_cancel_sell(trade, order, reason) + assert freqtrade.handle_cancel_exit(trade, order, reason) assert cancel_order_mock.call_count == 1 assert send_msg_mock.call_count == 1 send_msg_mock.reset_mock() order['amount'] = 2 - assert freqtrade.handle_cancel_sell(trade, order, reason + assert freqtrade.handle_cancel_exit(trade, order, reason ) == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] # Assert cancel_order was not called (callcount remains unchanged) assert cancel_order_mock.call_count == 1 assert send_msg_mock.call_count == 1 - assert freqtrade.handle_cancel_sell(trade, order, reason + assert freqtrade.handle_cancel_exit(trade, order, reason ) == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] # Message should not be iterated again assert trade.sell_order_status == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] assert send_msg_mock.call_count == 1 -def test_handle_cancel_sell_cancel_exception(mocker, default_conf) -> None: +def test_handle_cancel_exit_cancel_exception(mocker, default_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch( @@ -2630,7 +2525,7 @@ def test_handle_cancel_sell_cancel_exception(mocker, default_conf) -> None: order = {'remaining': 1, 'amount': 1, 'status': "open"} - assert freqtrade.handle_cancel_sell(trade, order, reason) == 'error cancelling order' + assert freqtrade.handle_cancel_exit(trade, order, reason) == 'error cancelling order' def test_execute_trade_exit_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> None: @@ -3124,16 +3019,28 @@ def test_execute_trade_exit_insufficient_funds_error(default_conf, ticker, fee, assert mock_insuf.call_count == 1 -def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, limit_buy_order_open, - fee, mocker) -> None: +@pytest.mark.parametrize('profit_only,bid,ask,handle_first,handle_second,sell_type', [ + # Enable profit + (True, 0.00001172, 0.00001173, False, True, SellType.SELL_SIGNAL.value), + # Disable profit + (False, 0.00002172, 0.00002173, True, False, SellType.SELL_SIGNAL.value), + # Enable loss + # * Shouldn't this be SellType.STOP_LOSS.value + (True, 0.00000172, 0.00000173, False, False, None), + # Disable loss + (False, 0.00000172, 0.00000173, True, False, SellType.SELL_SIGNAL.value), +]) +def test_sell_profit_only( + default_conf, limit_buy_order, limit_buy_order_open, + fee, mocker, profit_only, bid, ask, handle_first, handle_second, sell_type) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172 + 'bid': bid, + 'ask': ask, + 'last': bid }), create_order=MagicMock(side_effect=[ limit_buy_order_open, @@ -3143,128 +3050,29 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, limit_buy ) default_conf.update({ 'use_sell_signal': True, - 'sell_profit_only': True, + 'sell_profit_only': profit_only, 'sell_profit_offset': 0.1, }) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - + if sell_type == SellType.SELL_SIGNAL.value: + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) + else: + freqtrade.strategy.stop_loss_reached = MagicMock(return_value=SellCheckTuple( + sell_type=SellType.NONE)) freqtrade.enter_positions() trade = Trade.query.first() trade.update(limit_buy_order) freqtrade.wallets.update() patch_get_signal(freqtrade, value=(False, True, None)) - assert freqtrade.handle_trade(trade) is False + assert freqtrade.handle_trade(trade) is handle_first - freqtrade.strategy.sell_profit_offset = 0.0 - assert freqtrade.handle_trade(trade) is True + if handle_second: + freqtrade.strategy.sell_profit_offset = 0.0 + assert freqtrade.handle_trade(trade) is True - assert trade.sell_reason == SellType.SELL_SIGNAL.value - - -def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, limit_buy_order_open, - fee, mocker) -> None: - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=MagicMock(return_value={ - 'bid': 0.00002172, - 'ask': 0.00002173, - 'last': 0.00002172 - }), - create_order=MagicMock(side_effect=[ - limit_buy_order_open, - {'id': 1234553382}, - ]), - get_fee=fee, - ) - default_conf.update({ - 'use_sell_signal': True, - 'sell_profit_only': False, - }) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.enter_positions() - - trade = Trade.query.first() - trade.update(limit_buy_order) - freqtrade.wallets.update() - patch_get_signal(freqtrade, value=(False, True, None)) - assert freqtrade.handle_trade(trade) is True - assert trade.sell_reason == SellType.SELL_SIGNAL.value - - -def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, limit_buy_order_open, - fee, mocker) -> None: - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=MagicMock(return_value={ - 'bid': 0.00000172, - 'ask': 0.00000173, - 'last': 0.00000172 - }), - create_order=MagicMock(side_effect=[ - limit_buy_order_open, - {'id': 1234553382}, - ]), - get_fee=fee, - ) - default_conf.update({ - 'use_sell_signal': True, - 'sell_profit_only': True, - }) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) - freqtrade.strategy.stop_loss_reached = MagicMock(return_value=SellCheckTuple( - sell_type=SellType.NONE)) - freqtrade.enter_positions() - - trade = Trade.query.first() - trade.update(limit_buy_order) - patch_get_signal(freqtrade, value=(False, True, None)) - assert freqtrade.handle_trade(trade) is False - - -def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, limit_buy_order_open, - fee, mocker) -> None: - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=MagicMock(return_value={ - 'bid': 0.0000172, - 'ask': 0.0000173, - 'last': 0.0000172 - }), - create_order=MagicMock(side_effect=[ - limit_buy_order_open, - {'id': 1234553382}, - ]), - get_fee=fee, - ) - default_conf.update({ - 'use_sell_signal': True, - 'sell_profit_only': False, - }) - - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - - freqtrade.enter_positions() - - trade = Trade.query.first() - trade.update(limit_buy_order) - freqtrade.wallets.update() - patch_get_signal(freqtrade, value=(False, True, None)) - assert freqtrade.handle_trade(trade) is True - assert trade.sell_reason == SellType.SELL_SIGNAL.value + assert trade.sell_reason == sell_type def test_sell_not_enough_balance(default_conf, limit_buy_order, limit_buy_order_open, @@ -3302,11 +3110,15 @@ def test_sell_not_enough_balance(default_conf, limit_buy_order, limit_buy_order_ assert trade.amount != amnt -def test__safe_sell_amount(default_conf, fee, caplog, mocker): +@pytest.mark.parametrize('amount_wallet,has_err', [ + (95.29, False), + (91.29, True) +]) +def test__safe_exit_amount(default_conf, fee, caplog, mocker, amount_wallet, has_err): patch_RPCManager(mocker) patch_exchange(mocker) amount = 95.33 - amount_wallet = 95.29 + amount_wallet = amount_wallet mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=amount_wallet)) wallet_update = mocker.patch('freqtrade.wallets.Wallets.update') trade = Trade( @@ -3320,37 +3132,19 @@ def test__safe_sell_amount(default_conf, fee, caplog, mocker): ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - - wallet_update.reset_mock() - assert freqtrade._safe_sell_amount(trade.pair, trade.amount) == amount_wallet - assert log_has_re(r'.*Falling back to wallet-amount.', caplog) - assert wallet_update.call_count == 1 - caplog.clear() - wallet_update.reset_mock() - assert freqtrade._safe_sell_amount(trade.pair, amount_wallet) == amount_wallet - assert not log_has_re(r'.*Falling back to wallet-amount.', caplog) - assert wallet_update.call_count == 1 - - -def test__safe_sell_amount_error(default_conf, fee, caplog, mocker): - patch_RPCManager(mocker) - patch_exchange(mocker) - amount = 95.33 - amount_wallet = 91.29 - mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=amount_wallet)) - trade = Trade( - pair='LTC/ETH', - amount=amount, - exchange='binance', - open_rate=0.245441, - open_order_id="123456", - fee_open=fee.return_value, - fee_close=fee.return_value, - ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) - with pytest.raises(DependencyException, match=r"Not enough amount to sell."): - assert freqtrade._safe_sell_amount(trade.pair, trade.amount) + if has_err: + with pytest.raises(DependencyException, match=r"Not enough amount to sell."): + assert freqtrade._safe_exit_amount(trade.pair, trade.amount) + else: + wallet_update.reset_mock() + assert freqtrade._safe_exit_amount(trade.pair, trade.amount) == amount_wallet + assert log_has_re(r'.*Falling back to wallet-amount.', caplog) + assert wallet_update.call_count == 1 + caplog.clear() + wallet_update.reset_mock() + assert freqtrade._safe_exit_amount(trade.pair, amount_wallet) == amount_wallet + assert not log_has_re(r'.*Falling back to wallet-amount.', caplog) + assert wallet_update.call_count == 1 def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplog) -> None: @@ -3524,6 +3318,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, limit_buy_or assert log_has("ETH/BTC - Using positive stoploss: 0.01 offset: 0 profit: 0.2666%", caplog) assert log_has("ETH/BTC - Adjusting stoploss...", caplog) assert trade.stop_loss == 0.0000138501 + caplog.clear() mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ @@ -3584,6 +3379,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, limit_buy_orde assert log_has("ETH/BTC - Using positive stoploss: 0.01 offset: 0.011 profit: 0.2666%", caplog) assert log_has("ETH/BTC - Adjusting stoploss...", caplog) assert trade.stop_loss == 0.0000138501 + caplog.clear() mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ @@ -3648,6 +3444,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, limit_buy_order_ assert not log_has("ETH/BTC - Adjusting stoploss...", caplog) assert trade.stop_loss == 0.0000098910 + caplog.clear() # price rises above the offset (rises 12% when the offset is 5.5%) mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', @@ -3771,8 +3568,33 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f caplog) -def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fee, mocker): - trades_for_order[0]['fee']['currency'] = 'ETH' +@pytest.mark.parametrize( + 'fee_par,fee_reduction_amount,use_ticker_rate,expected_log', [ + # basic, amount does not change + ({'cost': 0.008, 'currency': 'ETH'}, 0, False, None), + # no currency in fee + ({'cost': 0.004, 'currency': None}, 0, True, None), + # BNB no rate + ({'cost': 0.00094518, 'currency': 'BNB'}, 0, True, ( + 'Fee for Trade Trade(id=None, pair=LTC/ETH, amount=8.00000000, open_rate=0.24544100,' + ' open_since=closed) [buy]: 0.00094518 BNB - rate: None' + )), + # from order + ({'cost': 0.004, 'currency': 'LTC'}, 0.004, False, ( + 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).' + )), + # invalid, no currency in from fee dict + ({'cost': 0.008, 'currency': None}, 0, True, None), + ]) +def test_get_real_amount( + default_conf, trades_for_order, buy_order_fee, fee, mocker, caplog, + fee_par, fee_reduction_amount, use_ticker_rate, expected_log +): + + buy_order = deepcopy(buy_order_fee) + buy_order['fee'] = fee_par + trades_for_order[0]['fee'] = fee_par mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) @@ -3787,19 +3609,38 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fe ) freqtrade = get_patched_freqtradebot(mocker, default_conf) - # Amount does not change - assert freqtrade.get_real_amount(trade, buy_order_fee) == amount + if not use_ticker_rate: + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', side_effect=ExchangeError) + + caplog.clear() + assert freqtrade.get_real_amount(trade, buy_order) == amount - fee_reduction_amount + + if expected_log: + assert log_has(expected_log, caplog) -def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_order_fee, - fee, mocker): +@pytest.mark.parametrize( + 'fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log_amount', [ + # basic, amount is reduced by fee + (None, None, 0.001, 0.001, 7.992), + # different fee currency on both trades, fee is average of both trade's fee + (0.02, 'BNB', 0.0005, 0.001518575, 7.996), + ]) +def test_get_real_amount_multi( + default_conf, trades_for_order2, buy_order_fee, caplog, fee, mocker, markets, + fee_cost, fee_currency, fee_reduction_amount, expected_fee, expected_log_amount, +): - limit_buy_order = deepcopy(buy_order_fee) - limit_buy_order['fee'] = {'cost': 0.004, 'currency': None} - trades_for_order[0]['fee']['currency'] = None + trades_for_order = deepcopy(trades_for_order2) + if fee_cost: + trades_for_order[0]['fee']['cost'] = fee_cost + if fee_currency: + trades_for_order[0]['fee']['currency'] = fee_currency mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) - amount = sum(x['amount'] for x in trades_for_order) + amount = float(sum(x['amount'] for x in trades_for_order)) + default_conf['stake_currency'] = "ETH" + trade = Trade( pair='LTC/ETH', amount=amount, @@ -3809,76 +3650,7 @@ def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_ open_rate=0.245441, open_order_id="123456" ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) - # Amount does not change - assert freqtrade.get_real_amount(trade, limit_buy_order) == amount - - -def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, fee, mocker): - trades_for_order[0]['fee']['currency'] = 'BNB' - trades_for_order[0]['fee']['cost'] = 0.00094518 - - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) - amount = sum(x['amount'] for x in trades_for_order) - trade = Trade( - pair='LTC/ETH', - amount=amount, - exchange='binance', - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.245441, - open_order_id="123456" - ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) - - # Amount does not change - assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - - -def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, caplog, fee, mocker): - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order2) - amount = float(sum(x['amount'] for x in trades_for_order2)) - trade = Trade( - pair='LTC/ETH', - amount=amount, - exchange='binance', - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.245441, - open_order_id="123456" - ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) - - # Amount is reduced by "fee" - assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) - assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).', - caplog) - - assert trade.fee_open == 0.001 - assert trade.fee_close == 0.001 - assert trade.fee_open_cost is not None - assert trade.fee_open_currency is not None - assert trade.fee_close_cost is None - assert trade.fee_close_currency is None - - -def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee, caplog, fee, - mocker, markets): - # Different fee currency on both trades - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order3) - amount = float(sum(x['amount'] for x in trades_for_order3)) - default_conf['stake_currency'] = 'ETH' - trade = Trade( - pair='LTC/ETH', - amount=amount, - exchange='binance', - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.245441, - open_order_id="123456" - ) # Fake markets entry to enable fee parsing markets['BNB/ETH'] = markets['ETH/BTC'] freqtrade = get_patched_freqtradebot(mocker, default_conf) @@ -3887,46 +3659,24 @@ def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee, return_value={'ask': 0.19, 'last': 0.2}) # Amount is reduced by "fee" - assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.0005) - assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).', - caplog) - # Overall fee is average of both trade's fee - assert trade.fee_open == 0.001518575 + expected_amount = amount - (amount * fee_reduction_amount) + assert freqtrade.get_real_amount(trade, buy_order_fee) == expected_amount + assert log_has( + ( + 'Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' + f'open_rate=0.24544100, open_since=closed) (from 8.0 to {expected_log_amount}).' + ), + caplog + ) + + assert trade.fee_open == expected_fee + assert trade.fee_close == expected_fee assert trade.fee_open_cost is not None assert trade.fee_open_currency is not None assert trade.fee_close_cost is None assert trade.fee_close_currency is None -def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee, fee, - caplog, mocker): - limit_buy_order = deepcopy(buy_order_fee) - limit_buy_order['fee'] = {'cost': 0.004, 'currency': 'LTC'} - - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', - return_value=[trades_for_order]) - amount = float(sum(x['amount'] for x in trades_for_order)) - trade = Trade( - pair='LTC/ETH', - amount=amount, - exchange='binance', - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.245441, - open_order_id="123456" - ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) - # Ticker rate cannot be found for this to work. - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', side_effect=ExchangeError) - - # Amount is reduced by "fee" - assert freqtrade.get_real_amount(trade, limit_buy_order) == amount - 0.004 - assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).', - caplog) - - def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order_fee, fee, mocker): limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['fee'] = {'cost': 0.004} @@ -3994,27 +3744,6 @@ def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, b abs_tol=MATH_CLOSE_PREC,) -def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, fee, mocker): - # Remove "Currency" from fee dict - trades_for_order[0]['fee'] = {'cost': 0.008} - - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) - amount = sum(x['amount'] for x in trades_for_order) - trade = Trade( - pair='LTC/ETH', - amount=amount, - exchange='binance', - open_rate=0.245441, - fee_open=fee.return_value, - fee_close=fee.return_value, - - open_order_id="123456" - ) - freqtrade = get_patched_freqtradebot(mocker, default_conf) - # Amount does not change - assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - - def test_get_real_amount_open_trade(default_conf, fee, mocker): amount = 12345 trade = Trade( @@ -4065,10 +3794,14 @@ def test_apply_fee_conditional(default_conf, fee, caplog, mocker, assert walletmock.call_count == 1 +@pytest.mark.parametrize("delta, is_high_delta", [ + (0.1, False), + (100, True), +]) def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_open, limit_buy_order, - fee, mocker, order_book_l2): + fee, mocker, order_book_l2, delta, is_high_delta): default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True - default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 0.1 + default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = delta patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) @@ -4086,88 +3819,55 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_open, freqtrade.enter_positions() trade = Trade.query.first() - assert trade is not None - assert trade.stake_amount == 0.001 - assert trade.is_open - assert trade.open_date is not None - assert trade.exchange == 'binance' + if is_high_delta: + assert trade is None + else: + assert trade is not None + assert trade.stake_amount == 0.001 + assert trade.is_open + assert trade.open_date is not None + assert trade.exchange == 'binance' - assert len(Trade.query.all()) == 1 + assert len(Trade.query.all()) == 1 - # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) + # Simulate fulfilled LIMIT_BUY order for trade + trade.update(limit_buy_order) - assert trade.open_rate == 0.00001099 - assert whitelist == default_conf['exchange']['pair_whitelist'] + assert trade.open_rate == 0.00001099 + assert whitelist == default_conf['exchange']['pair_whitelist'] -def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_order, - fee, mocker, order_book_l2): - default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True - # delta is 100 which is impossible to reach. hence check_depth_of_market will return false - default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 100 - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, - create_order=MagicMock(return_value={'id': limit_buy_order['id']}), - get_fee=fee, - ) - # Save state of current whitelist - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) - freqtrade.enter_positions() - - trade = Trade.query.first() - assert trade is None - - -def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: +@pytest.mark.parametrize('exception_thrown,ask,last,order_book_top,order_book', [ + (False, 0.045, 0.046, 2, None), + (True, 0.042, 0.046, 1, {'bids': [[]], 'asks': [[]]}) +]) +def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2, exception_thrown, + ask, last, order_book_top, order_book, caplog) -> None: """ - test if function get_rate will return the order book price - instead of the ask rate + test if function get_rate will return the order book price instead of the ask rate """ patch_exchange(mocker) - ticker_mock = MagicMock(return_value={'ask': 0.045, 'last': 0.046}) + ticker_mock = MagicMock(return_value={'ask': ask, 'last': last}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - fetch_l2_order_book=order_book_l2, + fetch_l2_order_book=MagicMock(return_value=order_book) if order_book else order_book_l2, fetch_ticker=ticker_mock, - ) default_conf['exchange']['name'] = 'binance' default_conf['bid_strategy']['use_order_book'] = True - default_conf['bid_strategy']['order_book_top'] = 2 + default_conf['bid_strategy']['order_book_top'] = order_book_top default_conf['bid_strategy']['ask_last_balance'] = 0 default_conf['telegram']['enabled'] = False freqtrade = FreqtradeBot(default_conf) - assert freqtrade.exchange.get_rate('ETH/BTC', refresh=True, side="buy") == 0.043935 - assert ticker_mock.call_count == 0 - - -def test_order_book_bid_strategy_exception(mocker, default_conf, caplog) -> None: - patch_exchange(mocker) - ticker_mock = MagicMock(return_value={'ask': 0.042, 'last': 0.046}) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_l2_order_book=MagicMock(return_value={'bids': [[]], 'asks': [[]]}), - fetch_ticker=ticker_mock, - - ) - default_conf['exchange']['name'] = 'binance' - default_conf['bid_strategy']['use_order_book'] = True - default_conf['bid_strategy']['order_book_top'] = 1 - default_conf['bid_strategy']['ask_last_balance'] = 0 - default_conf['telegram']['enabled'] = False - - freqtrade = FreqtradeBot(default_conf) - # orderbook shall be used even if tickers would be lower. - with pytest.raises(PricingError): - freqtrade.exchange.get_rate('ETH/BTC', refresh=True, side="buy") - assert log_has_re(r'Buy Price at location 1 from orderbook could not be determined.', caplog) + if exception_thrown: + with pytest.raises(PricingError): + freqtrade.exchange.get_rate('ETH/BTC', refresh=True, side="buy") + assert log_has_re( + r'Buy Price at location 1 from orderbook could not be determined.', caplog) + else: + assert freqtrade.exchange.get_rate('ETH/BTC', refresh=True, side="buy") == 0.043935 + assert ticker_mock.call_count == 0 def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None: @@ -4304,8 +4004,8 @@ def test_cancel_all_open_orders(mocker, default_conf, fee, limit_buy_order, limi mocker.patch('freqtrade.exchange.Exchange.fetch_order', side_effect=[ ExchangeError(), limit_sell_order, limit_buy_order, limit_sell_order]) - buy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_buy') - sell_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_sell') + buy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_enter') + sell_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_exit') freqtrade = get_patched_freqtradebot(mocker, default_conf) create_mock_trades(fee) @@ -4339,6 +4039,7 @@ def test_update_open_orders(mocker, default_conf, fee, caplog): freqtrade.update_open_orders() assert not log_has_re(r"Error updating Order .*", caplog) + caplog.clear() freqtrade.config['dry_run'] = False freqtrade.update_open_orders() @@ -4420,14 +4121,14 @@ def test_update_closed_trades_without_assigned_fees(mocker, default_conf, fee): @pytest.mark.usefixtures("init_persistence") -def test_reupdate_buy_order_fees(mocker, default_conf, fee, caplog): +def test_reupdate_enter_order_fees(mocker, default_conf, fee, caplog): freqtrade = get_patched_freqtradebot(mocker, default_conf) mock_uts = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.update_trade_state') create_mock_trades(fee) trades = Trade.get_trades().all() - freqtrade.reupdate_buy_order_fees(trades[0]) + freqtrade.reupdate_enter_order_fees(trades[0]) assert log_has_re(r"Trying to reupdate buy fees for .*", caplog) assert mock_uts.call_count == 1 assert mock_uts.call_args_list[0][0][0] == trades[0] @@ -4450,7 +4151,7 @@ def test_reupdate_buy_order_fees(mocker, default_conf, fee, caplog): ) Trade.query.session.add(trade) - freqtrade.reupdate_buy_order_fees(trade) + freqtrade.reupdate_enter_order_fees(trade) assert log_has_re(r"Trying to reupdate buy fees for .*", caplog) assert mock_uts.call_count == 0 assert not log_has_re(r"Updating buy-fee on trade .* for order .*\.", caplog) @@ -4460,7 +4161,7 @@ def test_reupdate_buy_order_fees(mocker, default_conf, fee, caplog): def test_handle_insufficient_funds(mocker, default_conf, fee): freqtrade = get_patched_freqtradebot(mocker, default_conf) mock_rlo = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.refind_lost_order') - mock_bof = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.reupdate_buy_order_fees') + mock_bof = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.reupdate_enter_order_fees') create_mock_trades(fee) trades = Trade.get_trades().all() diff --git a/tests/test_integration.py b/tests/test_integration.py index 215927098..a3484d438 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -70,7 +70,7 @@ def test_may_execute_exit_stoploss_on_exchange_multi(default_conf, ticker, fee, mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', create_stoploss_order=MagicMock(return_value=True), - _notify_sell=MagicMock(), + _notify_exit=MagicMock(), ) mocker.patch("freqtrade.strategy.interface.IStrategy.should_sell", should_sell_mock) wallets_mock = mocker.patch("freqtrade.wallets.Wallets.update", MagicMock()) @@ -154,7 +154,7 @@ def test_forcebuy_last_unlimited(default_conf, ticker, fee, limit_buy_order, moc mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', create_stoploss_order=MagicMock(return_value=True), - _notify_sell=MagicMock(), + _notify_exit=MagicMock(), ) should_sell_mock = MagicMock(side_effect=[ SellCheckTuple(sell_type=SellType.NONE), diff --git a/tests/test_periodiccache.py b/tests/test_periodiccache.py new file mode 100644 index 000000000..f874f9041 --- /dev/null +++ b/tests/test_periodiccache.py @@ -0,0 +1,32 @@ +import time_machine + +from freqtrade.configuration import PeriodicCache + + +def test_ttl_cache(): + + with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: + + cache = PeriodicCache(5, ttl=60) + cache1h = PeriodicCache(5, ttl=3600) + + assert cache.timer() == 1630472400.0 + cache['a'] = 1235 + cache1h['a'] = 555123 + assert 'a' in cache + assert 'a' in cache1h + + t.move_to("2021-09-01 05:00:59 +00:00") + assert 'a' in cache + assert 'a' in cache1h + + # Cache expired + t.move_to("2021-09-01 05:01:00 +00:00") + assert 'a' not in cache + assert 'a' in cache1h + + t.move_to("2021-09-01 05:59:59 +00:00") + assert 'a' in cache1h + + t.move_to("2021-09-01 06:00:00 +00:00") + assert 'a' not in cache1h