From bb758da9408e15eed3327f3122eb576674d3f4b1 Mon Sep 17 00:00:00 2001 From: froggleston Date: Tue, 17 May 2022 22:05:33 +0100 Subject: [PATCH 01/11] Add support for fudging unavailable funding rates, allowing backtesting of timeranges where futures candles are available, but rates are not --- docs/configuration.md | 1 + docs/leverage.md | 6 ++++++ freqtrade/data/history/history_utils.py | 7 ++++++- freqtrade/optimize/backtesting.py | 23 +++++++++++++++++++++-- 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 80cd52c5b..5a6d5849a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -230,6 +230,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `dataformat_trades` | Data format to use to store historical trades data.
*Defaults to `jsongz`*.
**Datatype:** String | `position_adjustment_enable` | Enables the strategy to use position adjustments (additional buys or sells). [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean | `max_entry_position_adjustment` | Maximum additional order(s) for each open trade on top of the first entry Order. Set it to `-1` for unlimited additional orders. [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `-1`.*
**Datatype:** Positive Integer or -1 +| `futures_funding_rate` | User-specified funding rate to be used when historical funding rates are not available from the exchange. This does not overwrite real historical rates. It is recommended that this be set to 0 unless you are testing a specific coin and you understand how the funding rate will affect freqtrade's profit calculations. [More information here](configuration.md)
*Defaults to None.*
**Datatype:** Float ### Parameters in the strategy diff --git a/docs/leverage.md b/docs/leverage.md index 79d3c9842..d8a9c8032 100644 --- a/docs/leverage.md +++ b/docs/leverage.md @@ -101,6 +101,12 @@ Possible values are any floats between 0.0 and 0.99 !!! Danger "A `liquidation_buffer` of 0.0, or a low `liquidation_buffer` is likely to result in liquidations, and liquidation fees" Currently Freqtrade is able to calculate liquidation prices, but does not calculate liquidation fees. Setting your `liquidation_buffer` to 0.0, or using a low `liquidation_buffer` could result in your positions being liquidated. Freqtrade does not track liquidation fees, so liquidations will result in inaccurate profit/loss results for your bot. If you use a low `liquidation_buffer`, it is recommended to use `stoploss_on_exchange` if your exchange supports this. +## Unavailable funding rates + +For futures data, exchanges commonly provide the futures candles, the marks, and the funding rates. However, it is common that whilst candles and marks might be available, the funding rates are not. This can affect backtesting timeranges, i.e. you may only be able to test recent timeranges and not earlier, experiencing the `No data found. Terminating.` error. To get around this, add the `futures_funding_rate` config option as listed in [configuration.md](configuration.md), and it is recommended that you set this to `0`, unless you know a given specific funding rate for your pair, exchange and timerange. Setting this to anything other than `0` can have drastic effects on your profit calculations within strategy, e.g. within the `custom_exit`, `custom_stoploss`, etc functions. + +!!! This will not overwrite funding rates that are available from the exchange. + ### Developer #### Margin mode diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index eb36d2042..b589001ca 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -68,7 +68,8 @@ def load_data(datadir: Path, startup_candles: int = 0, fail_without_data: bool = False, data_format: str = 'json', - candle_type: CandleType = CandleType.SPOT + candle_type: CandleType = CandleType.SPOT, + user_futures_funding_rate = None, ) -> Dict[str, DataFrame]: """ Load ohlcv history data for a list of pairs. @@ -100,6 +101,10 @@ def load_data(datadir: Path, ) if not hist.empty: result[pair] = hist + else: + if candle_type is CandleType.FUNDING_RATE and user_futures_funding_rate is not None: + logger.warn(f"{pair} using user specified [{user_futures_funding_rate}]") + result[pair] = DataFrame(columns=["open","close","high","low","volume"]) if fail_without_data and not result: raise OperationalException("No data found. Terminating.") diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4e604898f..49b085ca1 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -275,8 +275,27 @@ class Backtesting: if pair not in self.exchange._leverage_tiers: unavailable_pairs.append(pair) continue - self.futures_data[pair] = funding_rates_dict[pair].merge( - mark_rates_dict[pair], on='date', how="inner", suffixes=["_fund", "_mark"]) + + if (pair in mark_rates_dict + and len(funding_rates_dict[pair]) == 0 + and "futures_funding_rate" in self.config): + mark_rates_dict[pair]["open_fund"] = self.config.get('futures_funding_rate') + mark_rates_dict[pair]["close_fund"] = 0.0 + mark_rates_dict[pair]["high_fund"] = 0.0 + mark_rates_dict[pair]["low_fund"] = 0.0 + mark_rates_dict[pair]["volume_fund"] = 0.0 + mark_rates_dict[pair].rename( + columns = {'open':'open_mark', + 'close':'close_mark', + 'high':'high_mark', + 'low':'low_mark', + 'volume':'volume_mark'}, + inplace = True) + + self.futures_data[pair] = mark_rates_dict[pair] + else: + self.futures_data[pair] = mark_rates_dict[pair].merge( + funding_rates_dict[pair], on='date', how="inner", suffixes=["_fund", "_mark"]) if unavailable_pairs: raise OperationalException( From 37e4ede65c674c898193a828d72feb90a92c5ea4 Mon Sep 17 00:00:00 2001 From: froggleston Date: Tue, 17 May 2022 22:32:17 +0100 Subject: [PATCH 02/11] Fix flake issues --- docs/leverage.md | 5 +++-- freqtrade/data/history/history_utils.py | 4 ++-- freqtrade/optimize/backtesting.py | 19 ++++++++++--------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/docs/leverage.md b/docs/leverage.md index d8a9c8032..0c8139ad3 100644 --- a/docs/leverage.md +++ b/docs/leverage.md @@ -105,8 +105,9 @@ Possible values are any floats between 0.0 and 0.99 For futures data, exchanges commonly provide the futures candles, the marks, and the funding rates. However, it is common that whilst candles and marks might be available, the funding rates are not. This can affect backtesting timeranges, i.e. you may only be able to test recent timeranges and not earlier, experiencing the `No data found. Terminating.` error. To get around this, add the `futures_funding_rate` config option as listed in [configuration.md](configuration.md), and it is recommended that you set this to `0`, unless you know a given specific funding rate for your pair, exchange and timerange. Setting this to anything other than `0` can have drastic effects on your profit calculations within strategy, e.g. within the `custom_exit`, `custom_stoploss`, etc functions. -!!! This will not overwrite funding rates that are available from the exchange. - +!!! Warning This will mean your backtests are inaccurate. + This will not overwrite funding rates that are available from the exchange, but bear in mind that setting a false funding rate will mean backtesting results will be inaccurate for historical timeranges where funding rates are not available. + ### Developer #### Margin mode diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index b589001ca..4600d6ab4 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -69,7 +69,7 @@ def load_data(datadir: Path, fail_without_data: bool = False, data_format: str = 'json', candle_type: CandleType = CandleType.SPOT, - user_futures_funding_rate = None, + user_futures_funding_rate: int = None, ) -> Dict[str, DataFrame]: """ Load ohlcv history data for a list of pairs. @@ -104,7 +104,7 @@ def load_data(datadir: Path, else: if candle_type is CandleType.FUNDING_RATE and user_futures_funding_rate is not None: logger.warn(f"{pair} using user specified [{user_futures_funding_rate}]") - result[pair] = DataFrame(columns=["open","close","high","low","volume"]) + result[pair] = DataFrame(columns=["open", "close", "high", "low", "volume"]) if fail_without_data and not result: raise OperationalException("No data found. Terminating.") diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 49b085ca1..8d5a5fcea 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -277,25 +277,26 @@ class Backtesting: continue if (pair in mark_rates_dict - and len(funding_rates_dict[pair]) == 0 - and "futures_funding_rate" in self.config): + and len(funding_rates_dict[pair]) == 0 + and "futures_funding_rate" in self.config): mark_rates_dict[pair]["open_fund"] = self.config.get('futures_funding_rate') mark_rates_dict[pair]["close_fund"] = 0.0 mark_rates_dict[pair]["high_fund"] = 0.0 mark_rates_dict[pair]["low_fund"] = 0.0 mark_rates_dict[pair]["volume_fund"] = 0.0 mark_rates_dict[pair].rename( - columns = {'open':'open_mark', - 'close':'close_mark', - 'high':'high_mark', - 'low':'low_mark', - 'volume':'volume_mark'}, - inplace = True) + columns={'open': 'open_mark', + 'close': 'close_mark', + 'high': 'high_mark', + 'low': 'low_mark', + 'volume': 'volume_mark'}, + inplace=True) self.futures_data[pair] = mark_rates_dict[pair] else: self.futures_data[pair] = mark_rates_dict[pair].merge( - funding_rates_dict[pair], on='date', how="inner", suffixes=["_fund", "_mark"]) + funding_rates_dict[pair], on='date', + how="inner", suffixes=["_fund", "_mark"]) if unavailable_pairs: raise OperationalException( From c41d4c4f45ba06deb2b70923469ce9dfb2701e6f Mon Sep 17 00:00:00 2001 From: froggleston Date: Tue, 17 May 2022 22:37:48 +0100 Subject: [PATCH 03/11] Fix leverage docs --- docs/leverage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/leverage.md b/docs/leverage.md index 0c8139ad3..58e7cc778 100644 --- a/docs/leverage.md +++ b/docs/leverage.md @@ -105,7 +105,7 @@ Possible values are any floats between 0.0 and 0.99 For futures data, exchanges commonly provide the futures candles, the marks, and the funding rates. However, it is common that whilst candles and marks might be available, the funding rates are not. This can affect backtesting timeranges, i.e. you may only be able to test recent timeranges and not earlier, experiencing the `No data found. Terminating.` error. To get around this, add the `futures_funding_rate` config option as listed in [configuration.md](configuration.md), and it is recommended that you set this to `0`, unless you know a given specific funding rate for your pair, exchange and timerange. Setting this to anything other than `0` can have drastic effects on your profit calculations within strategy, e.g. within the `custom_exit`, `custom_stoploss`, etc functions. -!!! Warning This will mean your backtests are inaccurate. +!!! Warning "This will mean your backtests are inaccurate." This will not overwrite funding rates that are available from the exchange, but bear in mind that setting a false funding rate will mean backtesting results will be inaccurate for historical timeranges where funding rates are not available. ### Developer From 736f9f4972add41a066bfc23705d1664915aa9b4 Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 18 May 2022 12:47:37 +0100 Subject: [PATCH 04/11] Fix docs and add outer join support for merging funding rates across full timerange --- docs/configuration.md | 2 +- freqtrade/optimize/backtesting.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 5a6d5849a..4a05ad3d4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -230,7 +230,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `dataformat_trades` | Data format to use to store historical trades data.
*Defaults to `jsongz`*.
**Datatype:** String | `position_adjustment_enable` | Enables the strategy to use position adjustments (additional buys or sells). [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean | `max_entry_position_adjustment` | Maximum additional order(s) for each open trade on top of the first entry Order. Set it to `-1` for unlimited additional orders. [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `-1`.*
**Datatype:** Positive Integer or -1 -| `futures_funding_rate` | User-specified funding rate to be used when historical funding rates are not available from the exchange. This does not overwrite real historical rates. It is recommended that this be set to 0 unless you are testing a specific coin and you understand how the funding rate will affect freqtrade's profit calculations. [More information here](configuration.md)
*Defaults to None.*
**Datatype:** Float +| `futures_funding_rate` | User-specified funding rate to be used when historical funding rates are not available from the exchange. This does not overwrite real historical rates. It is recommended that this be set to 0 unless you are testing a specific coin and you understand how the funding rate will affect freqtrade's profit calculations. [More information here](leverage.md)
*Defaults to None.*
**Datatype:** Float ### Parameters in the strategy diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 8d5a5fcea..78faf65be 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -294,9 +294,15 @@ class Backtesting: self.futures_data[pair] = mark_rates_dict[pair] else: - self.futures_data[pair] = mark_rates_dict[pair].merge( - funding_rates_dict[pair], on='date', - how="inner", suffixes=["_fund", "_mark"]) + if "futures_funding_rate" in self.config: + self.futures_data[pair] = mark_rates_dict[pair].merge( + funding_rates_dict[pair], on='date', + how="outer", suffixes=["_fund", "_mark"]).fillna( + self.config.get('futures_funding_rate')) + else: + self.futures_data[pair] = mark_rates_dict[pair].merge( + funding_rates_dict[pair], on='date', + how="inner", suffixes=["_fund", "_mark"]) if unavailable_pairs: raise OperationalException( From 363098d32dcb729f4b6f3e3e2aadae492c97138e Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 18 May 2022 12:56:43 +0100 Subject: [PATCH 05/11] Fix reversed makr/funding_rate columns --- freqtrade/optimize/backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 78faf65be..a80266b2c 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -297,12 +297,12 @@ class Backtesting: if "futures_funding_rate" in self.config: self.futures_data[pair] = mark_rates_dict[pair].merge( funding_rates_dict[pair], on='date', - how="outer", suffixes=["_fund", "_mark"]).fillna( + how="outer", suffixes=["_mark", "_fund"]).fillna( self.config.get('futures_funding_rate')) else: self.futures_data[pair] = mark_rates_dict[pair].merge( funding_rates_dict[pair], on='date', - how="inner", suffixes=["_fund", "_mark"]) + how="inner", suffixes=["_mark", "_fund"]) if unavailable_pairs: raise OperationalException( From c499a92f57cccf520f3d6f19941857af87fac5aa Mon Sep 17 00:00:00 2001 From: froggleston Date: Fri, 20 May 2022 11:48:53 +0100 Subject: [PATCH 06/11] Remove surplus mark columns, and make fillna on funding rate only --- freqtrade/optimize/backtesting.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index a80266b2c..99bddbf8a 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -280,10 +280,6 @@ class Backtesting: and len(funding_rates_dict[pair]) == 0 and "futures_funding_rate" in self.config): mark_rates_dict[pair]["open_fund"] = self.config.get('futures_funding_rate') - mark_rates_dict[pair]["close_fund"] = 0.0 - mark_rates_dict[pair]["high_fund"] = 0.0 - mark_rates_dict[pair]["low_fund"] = 0.0 - mark_rates_dict[pair]["volume_fund"] = 0.0 mark_rates_dict[pair].rename( columns={'open': 'open_mark', 'close': 'close_mark', @@ -297,7 +293,7 @@ class Backtesting: if "futures_funding_rate" in self.config: self.futures_data[pair] = mark_rates_dict[pair].merge( funding_rates_dict[pair], on='date', - how="outer", suffixes=["_mark", "_fund"]).fillna( + how="outer", suffixes=["_mark", "_fund"])['open_fund'].fillna( self.config.get('futures_funding_rate')) else: self.futures_data[pair] = mark_rates_dict[pair].merge( From 0e158b66b0e089f12f79a1217cce431ec3fc7a4f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 21 May 2022 08:26:44 +0200 Subject: [PATCH 07/11] Update docs link --- docs/configuration.md | 2 +- docs/leverage.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 4a05ad3d4..949cac91d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -230,7 +230,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `dataformat_trades` | Data format to use to store historical trades data.
*Defaults to `jsongz`*.
**Datatype:** String | `position_adjustment_enable` | Enables the strategy to use position adjustments (additional buys or sells). [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean | `max_entry_position_adjustment` | Maximum additional order(s) for each open trade on top of the first entry Order. Set it to `-1` for unlimited additional orders. [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `-1`.*
**Datatype:** Positive Integer or -1 -| `futures_funding_rate` | User-specified funding rate to be used when historical funding rates are not available from the exchange. This does not overwrite real historical rates. It is recommended that this be set to 0 unless you are testing a specific coin and you understand how the funding rate will affect freqtrade's profit calculations. [More information here](leverage.md)
*Defaults to None.*
**Datatype:** Float +| `futures_funding_rate` | User-specified funding rate to be used when historical funding rates are not available from the exchange. This does not overwrite real historical rates. It is recommended that this be set to 0 unless you are testing a specific coin and you understand how the funding rate will affect freqtrade's profit calculations. [More information here](leverage.md#unavailable-funding-rates)
*Defaults to None.*
**Datatype:** Float ### Parameters in the strategy diff --git a/docs/leverage.md b/docs/leverage.md index 58e7cc778..2ee6f8444 100644 --- a/docs/leverage.md +++ b/docs/leverage.md @@ -107,7 +107,7 @@ For futures data, exchanges commonly provide the futures candles, the marks, and !!! Warning "This will mean your backtests are inaccurate." This will not overwrite funding rates that are available from the exchange, but bear in mind that setting a false funding rate will mean backtesting results will be inaccurate for historical timeranges where funding rates are not available. - + ### Developer #### Margin mode From 6bd5535d6c3476d36bafdf73d02926c562546fc6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 21 May 2022 08:31:34 +0200 Subject: [PATCH 08/11] Use exchange method to combine funding and mark candles --- freqtrade/exchange/exchange.py | 2 +- freqtrade/optimize/backtesting.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index d2766cd6d..65d9909c6 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2420,7 +2420,7 @@ class Exchange: :param mark_rates: Dataframe containing Mark rates (Type mark_ohlcv_price) """ - return funding_rates.merge(mark_rates, on='date', how="inner", suffixes=["_fund", "_mark"]) + return mark_rates.merge(funding_rates, on='date', how="inner", suffixes=["_mark", "_fund"]) def calculate_funding_fees( self, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 99bddbf8a..3041136a3 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -296,9 +296,10 @@ class Backtesting: how="outer", suffixes=["_mark", "_fund"])['open_fund'].fillna( self.config.get('futures_funding_rate')) else: - self.futures_data[pair] = mark_rates_dict[pair].merge( - funding_rates_dict[pair], on='date', - how="inner", suffixes=["_mark", "_fund"]) + self.futures_data[pair] = self.exchange.combine_funding_and_mark( + funding_rates=funding_rates_dict[pair], + mark_rates=mark_rates_dict[pair] + ) if unavailable_pairs: raise OperationalException( From 2df42a3035902028dfbe50839cd685be78f2e0c9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 21 May 2022 08:50:39 +0200 Subject: [PATCH 09/11] Move "funding fillup" logic to exchange class --- freqtrade/exchange/exchange.py | 23 +++++++++++++++++++++-- freqtrade/optimize/backtesting.py | 29 +++++------------------------ 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 65d9909c6..9372c77b7 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2413,14 +2413,33 @@ class Exchange: ) @staticmethod - def combine_funding_and_mark(funding_rates: DataFrame, mark_rates: DataFrame) -> DataFrame: + def combine_funding_and_mark(funding_rates: DataFrame, mark_rates: DataFrame, + futures_funding_rate: Optional[int] = None) -> DataFrame: """ Combine funding-rates and mark-rates dataframes :param funding_rates: Dataframe containing Funding rates (Type FUNDING_RATE) :param mark_rates: Dataframe containing Mark rates (Type mark_ohlcv_price) + :param futures_funding_rate: Fake funding rate to use if funding_rates are not available """ + if futures_funding_rate is None: + return mark_rates.merge( + funding_rates, on='date', how="inner", suffixes=["_mark", "_fund"]) + else: + if len(funding_rates) == 0: + # No funding rate candles - full fillup with fallback variable + mark_rates['open_fund'] = futures_funding_rate + return mark_rates.rename( + columns={'open': 'open_mark', + 'close': 'close_mark', + 'high': 'high_mark', + 'low': 'low_mark', + 'volume': 'volume_mark'}) - return mark_rates.merge(funding_rates, on='date', how="inner", suffixes=["_mark", "_fund"]) + else: + # Fill up missing funding_rate candles with fallback value + return mark_rates.merge( + funding_rates, on='date', how="outer", suffixes=["_mark", "_fund"] + )['open_fund'].fillna(futures_funding_rate) def calculate_funding_fees( self, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3041136a3..2c34e29b0 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -276,30 +276,11 @@ class Backtesting: unavailable_pairs.append(pair) continue - if (pair in mark_rates_dict - and len(funding_rates_dict[pair]) == 0 - and "futures_funding_rate" in self.config): - mark_rates_dict[pair]["open_fund"] = self.config.get('futures_funding_rate') - mark_rates_dict[pair].rename( - columns={'open': 'open_mark', - 'close': 'close_mark', - 'high': 'high_mark', - 'low': 'low_mark', - 'volume': 'volume_mark'}, - inplace=True) - - self.futures_data[pair] = mark_rates_dict[pair] - else: - if "futures_funding_rate" in self.config: - self.futures_data[pair] = mark_rates_dict[pair].merge( - funding_rates_dict[pair], on='date', - how="outer", suffixes=["_mark", "_fund"])['open_fund'].fillna( - self.config.get('futures_funding_rate')) - else: - self.futures_data[pair] = self.exchange.combine_funding_and_mark( - funding_rates=funding_rates_dict[pair], - mark_rates=mark_rates_dict[pair] - ) + self.futures_data[pair] = self.exchange.combine_funding_and_mark( + funding_rates=funding_rates_dict[pair], + mark_rates=mark_rates_dict[pair], + futures_funding_rate=self.config.get('futures_funding_rate'), + ) if unavailable_pairs: raise OperationalException( From 0d388b561bca0af4f2b0ee958632cea3b61824f2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 21 May 2022 09:03:30 +0200 Subject: [PATCH 10/11] Add test for "combine_funding_and_mark", fix bug --- freqtrade/exchange/exchange.py | 6 ++-- tests/exchange/test_exchange.py | 64 +++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 9372c77b7..d30c5fc2f 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2437,9 +2437,11 @@ class Exchange: else: # Fill up missing funding_rate candles with fallback value - return mark_rates.merge( + combined = mark_rates.merge( funding_rates, on='date', how="outer", suffixes=["_mark", "_fund"] - )['open_fund'].fillna(futures_funding_rate) + ) + combined['open_fund'] = combined['open_fund'].fillna(futures_funding_rate) + return combined def calculate_funding_fees( self, diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 53e6cc3f3..37f4dedbe 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -3912,6 +3912,70 @@ def test_calculate_funding_fees( ) == kraken_fee +@pytest.mark.parametrize( + 'mark_price,funding_rate,futures_funding_rate', [ + (1000, 0.001, None), + (1000, 0.001, 0.01), + (1000, 0.001, 0.0), + (1000, 0.001, -0.01), + ]) +def test_combine_funding_and_mark( + default_conf, + mocker, + funding_rate, + mark_price, + futures_funding_rate, +): + exchange = get_patched_exchange(mocker, default_conf) + prior2_date = timeframe_to_prev_date('1h', datetime.now(timezone.utc) - timedelta(hours=2)) + prior_date = timeframe_to_prev_date('1h', datetime.now(timezone.utc) - timedelta(hours=1)) + trade_date = timeframe_to_prev_date('1h', datetime.now(timezone.utc)) + funding_rates = DataFrame([ + {'date': prior2_date, 'open': funding_rate}, + {'date': prior_date, 'open': funding_rate}, + {'date': trade_date, 'open': funding_rate}, + ]) + mark_rates = DataFrame([ + {'date': prior2_date, 'open': mark_price}, + {'date': prior_date, 'open': mark_price}, + {'date': trade_date, 'open': mark_price}, + ]) + + df = exchange.combine_funding_and_mark(funding_rates, mark_rates, futures_funding_rate) + assert 'open_mark' in df.columns + assert 'open_fund' in df.columns + assert len(df) == 3 + + funding_rates = DataFrame([ + {'date': trade_date, 'open': funding_rate}, + ]) + mark_rates = DataFrame([ + {'date': prior2_date, 'open': mark_price}, + {'date': prior_date, 'open': mark_price}, + {'date': trade_date, 'open': mark_price}, + ]) + df = exchange.combine_funding_and_mark(funding_rates, mark_rates, futures_funding_rate) + + if futures_funding_rate is not None: + assert len(df) == 3 + assert df.iloc[0]['open_fund'] == futures_funding_rate + assert df.iloc[1]['open_fund'] == futures_funding_rate + assert df.iloc[2]['open_fund'] == funding_rate + else: + assert len(df) == 1 + + # Empty funding rates + funding_rates = DataFrame([], columns=['date', 'open']) + df = exchange.combine_funding_and_mark(funding_rates, mark_rates, futures_funding_rate) + if futures_funding_rate is not None: + assert len(df) == 3 + assert df.iloc[0]['open_fund'] == futures_funding_rate + assert df.iloc[1]['open_fund'] == futures_funding_rate + assert df.iloc[2]['open_fund'] == futures_funding_rate + else: + assert len(df) == 0 + + def test_get_or_calculate_liquidation_price(mocker, default_conf): api_mock = MagicMock() From f006978caf8d10df218c8643d4ed7f1c795b94ca Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 21 May 2022 17:35:49 +0200 Subject: [PATCH 11/11] Be more explicit in default value --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 2c34e29b0..3a3660c39 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -279,7 +279,7 @@ class Backtesting: self.futures_data[pair] = self.exchange.combine_funding_and_mark( funding_rates=funding_rates_dict[pair], mark_rates=mark_rates_dict[pair], - futures_funding_rate=self.config.get('futures_funding_rate'), + futures_funding_rate=self.config.get('futures_funding_rate', None), ) if unavailable_pairs: