From 3c8387ab611df10c8a327b5196f4798541f67b10 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Fri, 1 Apr 2022 20:48:13 +0900 Subject: [PATCH 01/15] Add exchange id for binance Futures --- docs/exchanges.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index b808096d2..420061050 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -87,7 +87,7 @@ When trading on Binance Futures market, orderbook must be used because there is Binance has been split into 2, and users must use the correct ccxt exchange ID for their exchange, otherwise API keys are not recognized. -* [binance.com](https://www.binance.com/) - International users. Use exchange id: `binance`. +* [binance.com](https://www.binance.com/) - International users. Use exchange id: `binance` for spot market, and use `binanceusdm` or `binancecoinm` for Futures market. * [binance.us](https://www.binance.us/) - US based users. Use exchange id: `binanceus`. ## Kraken From da0688b6aa26f6eee36acef9ede4186b48d876c6 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sat, 2 Apr 2022 03:20:21 +0900 Subject: [PATCH 02/15] Revert "Add exchange id for binance Futures" This reverts commit 3c8387ab611df10c8a327b5196f4798541f67b10. --- docs/exchanges.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index 420061050..b808096d2 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -87,7 +87,7 @@ When trading on Binance Futures market, orderbook must be used because there is Binance has been split into 2, and users must use the correct ccxt exchange ID for their exchange, otherwise API keys are not recognized. -* [binance.com](https://www.binance.com/) - International users. Use exchange id: `binance` for spot market, and use `binanceusdm` or `binancecoinm` for Futures market. +* [binance.com](https://www.binance.com/) - International users. Use exchange id: `binance`. * [binance.us](https://www.binance.us/) - US based users. Use exchange id: `binanceus`. ## Kraken From 71a80cab3a19771628e3c8b936f3479003894c1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=AE=AE=E0=AE=A9=E0=AF=8B=E0=AE=9C=E0=AF=8D=E0=AE=95?= =?UTF-8?q?=E0=AF=81=E0=AE=AE=E0=AE=BE=E0=AE=B0=E0=AF=8D=20=E0=AE=AA?= =?UTF-8?q?=E0=AE=B4=E0=AE=A9=E0=AE=BF=E0=AE=9A=E0=AF=8D=E0=AE=9A=E0=AE=BE?= =?UTF-8?q?=E0=AE=AE=E0=AE=BF?= Date: Fri, 13 May 2022 21:19:40 +0530 Subject: [PATCH 03/15] fixed variable naming style --- freqtrade/optimize/backtesting.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 45300b744..3f2c35cd5 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -538,33 +538,33 @@ class Backtesting: trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60) try: - closerate = self._get_close_rate(row, trade, exit_, trade_dur) + close_rate = self._get_close_rate(row, trade, exit_, trade_dur) except ValueError: return None - # call the custom exit price,with default value as previous closerate - current_profit = trade.calc_profit_ratio(closerate) + # call the custom exit price,with default value as previous close_rate + current_profit = trade.calc_profit_ratio(close_rate) order_type = self.strategy.order_types['exit'] if exit_.exit_type in (ExitType.EXIT_SIGNAL, ExitType.CUSTOM_EXIT): # Custom exit pricing only for exit-signals if order_type == 'limit': - closerate = strategy_safe_wrapper(self.strategy.custom_exit_price, - default_retval=closerate)( + close_rate = strategy_safe_wrapper(self.strategy.custom_exit_price, + default_retval=close_rate)( pair=trade.pair, trade=trade, current_time=exit_candle_time, - proposed_rate=closerate, current_profit=current_profit, + proposed_rate=close_rate, current_profit=current_profit, exit_tag=exit_.exit_reason) # We can't place orders lower than current low. # freqtrade does not support this in live, and the order would fill immediately if trade.is_short: - closerate = min(closerate, row[HIGH_IDX]) + close_rate = min(close_rate, row[HIGH_IDX]) else: - closerate = max(closerate, row[LOW_IDX]) + close_rate = max(close_rate, row[LOW_IDX]) # Confirm trade exit: time_in_force = self.strategy.order_time_in_force['exit'] if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)( pair=trade.pair, trade=trade, order_type='limit', amount=trade.amount, - rate=closerate, + rate=close_rate, time_in_force=time_in_force, sell_reason=exit_.exit_reason, # deprecated exit_reason=exit_.exit_reason, @@ -597,12 +597,12 @@ class Backtesting: side=trade.exit_side, order_type=order_type, status="open", - price=closerate, - average=closerate, + price=close_rate, + average=close_rate, amount=trade.amount, filled=0, remaining=trade.amount, - cost=trade.amount * closerate, + cost=trade.amount * close_rate, ) trade.orders.append(order) return trade From 64670726a61e8f56593a734d86ba24c10dbbc25f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=AE=AE=E0=AE=A9=E0=AF=8B=E0=AE=9C=E0=AF=8D=E0=AE=95?= =?UTF-8?q?=E0=AF=81=E0=AE=AE=E0=AE=BE=E0=AE=B0=E0=AF=8D=20=E0=AE=AA?= =?UTF-8?q?=E0=AE=B4=E0=AE=A9=E0=AE=BF=E0=AE=9A=E0=AF=8D=E0=AE=9A=E0=AE=BE?= =?UTF-8?q?=E0=AE=AE=E0=AE=BF?= Date: Fri, 13 May 2022 21:52:26 +0530 Subject: [PATCH 04/15] flake8 fix --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3f2c35cd5..20f34a0bb 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -548,7 +548,7 @@ class Backtesting: # Custom exit pricing only for exit-signals if order_type == 'limit': close_rate = strategy_safe_wrapper(self.strategy.custom_exit_price, - default_retval=close_rate)( + default_retval=close_rate)( pair=trade.pair, trade=trade, current_time=exit_candle_time, proposed_rate=close_rate, current_profit=current_profit, From e21f6a7787091e2c3831f872bf26aaed3df0184c Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 16 May 2022 07:28:40 +0900 Subject: [PATCH 05/15] missing newline --- freqtrade/rpc/telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 1a9be4503..259d2c831 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1417,7 +1417,7 @@ class Telegram(RPCHandler): "*/stopbuy:* `Stops buying, but handles open trades gracefully` \n" "*/forceexit |all:* `Instantly exits the given trade or all trades, " "regardless of profit`\n" - "*/fe |all:* `Alias to /forceexit`" + "*/fe |all:* `Alias to /forceexit`\n" f"{force_enter_text if self._config.get('force_entry_enable', False) else ''}" "*/delete :* `Instantly delete the given trade in the database`\n" "*/whitelist:* `Show current whitelist` \n" From 2cb8eecf18eed273df41ee2702e54def89831b03 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 16 May 2022 07:43:36 +0900 Subject: [PATCH 06/15] add space --- freqtrade/rpc/telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 259d2c831..f26de8b5c 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1410,7 +1410,7 @@ class Telegram(RPCHandler): "Optionally takes a rate at which to sell " "(only applies to limit orders).` \n") message = ( - "_BotControl_\n" + "_Bot Control_\n" "------------\n" "*/start:* `Starts the trader`\n" "*/stop:* Stops the trader\n" From 748055892cbc0f32a6610960ec0e58521e6b45fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 03:01:33 +0000 Subject: [PATCH 07/15] Bump plotly from 5.7.0 to 5.8.0 Bumps [plotly](https://github.com/plotly/plotly.py) from 5.7.0 to 5.8.0. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v5.7.0...v5.8.0) --- updated-dependencies: - dependency-name: plotly dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-plot.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-plot.txt b/requirements-plot.txt index d9faed301..e17efbc71 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,4 +1,4 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.7.0 +plotly==5.8.0 From 528509f809b2474218e24ac5442d889b4ca1fce9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 May 2022 19:18:13 +0200 Subject: [PATCH 08/15] Extract get_price_side from get_rate --- freqtrade/exchange/exchange.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index ee804aa68..156216557 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1457,6 +1457,23 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + def _get_price_side(self, side: str, is_short: bool, conf_strategy: Dict) -> str: + price_side = conf_strategy['price_side'] + + if price_side in ('same', 'other'): + price_map = { + ('entry', 'long', 'same'): 'bid', + ('entry', 'long', 'other'): 'ask', + ('entry', 'short', 'same'): 'ask', + ('entry', 'short', 'other'): 'bid', + ('exit', 'long', 'same'): 'ask', + ('exit', 'long', 'other'): 'bid', + ('exit', 'short', 'same'): 'bid', + ('exit', 'short', 'other'): 'ask', + } + price_side = price_map[(side, 'short' if is_short else 'long', price_side)] + return price_side + def get_rate(self, pair: str, refresh: bool, side: EntryExit, is_short: bool) -> float: """ @@ -1483,20 +1500,7 @@ class Exchange: conf_strategy = self._config.get(strat_name, {}) - price_side = conf_strategy['price_side'] - - if price_side in ('same', 'other'): - price_map = { - ('entry', 'long', 'same'): 'bid', - ('entry', 'long', 'other'): 'ask', - ('entry', 'short', 'same'): 'ask', - ('entry', 'short', 'other'): 'bid', - ('exit', 'long', 'same'): 'ask', - ('exit', 'long', 'other'): 'bid', - ('exit', 'short', 'same'): 'bid', - ('exit', 'short', 'other'): 'ask', - } - price_side = price_map[(side, 'short' if is_short else 'long', price_side)] + price_side = self._get_price_side(side, is_short, conf_strategy) price_side_word = price_side.capitalize() From a793cf8f05975767432d6b12bd31307e21923eb6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Mar 2022 07:10:46 +0200 Subject: [PATCH 09/15] Use ccxt's "precise" to do precise math --- freqtrade/exchange/exchange.py | 10 ++++++---- tests/exchange/test_exchange.py | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 156216557..d17c84f5c 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -16,6 +16,7 @@ import arrow import ccxt import ccxt.async_support as ccxt_async from cachetools import TTLCache +from ccxt import Precise from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, decimal_to_precision) from pandas import DataFrame @@ -704,10 +705,11 @@ class Exchange: # counting_mode=self.precisionMode, # )) if self.precisionMode == TICK_SIZE: - precision = self.markets[pair]['precision']['price'] - missing = price % precision - if missing != 0: - price = round(price - missing + precision, 10) + precision = Precise(str(self.markets[pair]['precision']['price'])) + price_str = Precise(str(price)) + missing = price_str.mod(precision) + if not missing.equals(Precise("0")): + price = round(float(str(price_str.sub(missing).add(precision))), 14) else: symbol_prec = self.markets[pair]['precision']['price'] big_price = price * pow(10, symbol_prec) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index e580c82d3..53e6cc3f3 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -305,6 +305,7 @@ def test_amount_to_precision( (234.53, 4, 0.5, 235.0), (0.891534, 4, 0.0001, 0.8916), (64968.89, 4, 0.01, 64968.89), + (0.000000003483, 4, 1e-12, 0.000000003483), ]) def test_price_to_precision(default_conf, mocker, price, precision_mode, precision, expected): From c8e0fc926d756f9cd5b5eff539653b2f9e332c06 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Apr 2022 12:00:41 +0200 Subject: [PATCH 10/15] Update to do Builtin Precise math --- freqtrade/exchange/exchange.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index d17c84f5c..8bbbf6d4d 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -707,9 +707,9 @@ class Exchange: if self.precisionMode == TICK_SIZE: precision = Precise(str(self.markets[pair]['precision']['price'])) price_str = Precise(str(price)) - missing = price_str.mod(precision) - if not missing.equals(Precise("0")): - price = round(float(str(price_str.sub(missing).add(precision))), 14) + missing = price_str % precision + if not missing == Precise("0"): + price = round(float(str(price_str - missing + precision)), 14) else: symbol_prec = self.markets[pair]['precision']['price'] big_price = price * pow(10, symbol_prec) From d09b462930adf105d3f6a074d1f2b9f3d58e3ab4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Apr 2022 19:46:55 +0200 Subject: [PATCH 11/15] Add rudimentary tests for Precise "builtin operator" workings --- tests/exchange/test_ccxt_precise.py | 75 +++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 tests/exchange/test_ccxt_precise.py diff --git a/tests/exchange/test_ccxt_precise.py b/tests/exchange/test_ccxt_precise.py new file mode 100644 index 000000000..026adb4c1 --- /dev/null +++ b/tests/exchange/test_ccxt_precise.py @@ -0,0 +1,75 @@ +from ccxt import Precise + + +ws = Precise('-1.123e-6') +ws = Precise('-1.123e-6') +xs = Precise('0.00000002') +ys = Precise('69696900000') +zs = Precise('0') + + +def test_precise(): + assert ys * xs == '1393.938' + assert xs * ys == '1393.938' + + assert ys + xs == '69696900000.00000002' + assert xs + ys == '69696900000.00000002' + assert xs - ys == '-69696899999.99999998' + assert ys - xs == '69696899999.99999998' + assert xs / ys == '0' + assert ys / xs == '3484845000000000000' + + assert ws * xs == '-0.00000000000002246' + assert xs * ws == '-0.00000000000002246' + + assert ws + xs == '-0.000001103' + assert xs + ws == '-0.000001103' + + assert xs - ws == '0.000001143' + assert ws - xs == '-0.000001143' + + assert xs / ws == '-0.017809439002671415' + assert ws / xs == '-56.15' + + assert zs * ws == '0' + assert zs * xs == '0' + assert zs * ys == '0' + assert ws * zs == '0' + assert xs * zs == '0' + assert ys * zs == '0' + + assert zs + ws == '-0.000001123' + assert zs + xs == '0.00000002' + assert zs + ys == '69696900000' + assert ws + zs == '-0.000001123' + assert xs + zs == '0.00000002' + assert ys + zs == '69696900000' + + assert abs(Precise('-500.1')) == '500.1' + assert abs(Precise('213')) == '213' + + assert abs(Precise('-500.1')) == '500.1' + assert -Precise('213') == '-213' + + assert Precise('10.1') % Precise('0.5') == '0.1' + assert Precise('5550') % Precise('120') == '30' + + assert Precise('-0.0') == Precise('0') + assert Precise('5.534000') == Precise('5.5340') + + assert min(Precise('-3.1415'), Precise('-2')) == '-3.1415' + + assert max(Precise('3.1415'), Precise('-2')) == '3.1415' + + assert Precise('2') > Precise('1.2345') + assert not Precise('-3.1415') > Precise('-2') + assert not Precise('3.1415') > Precise('3.1415') + assert Precise.string_gt('3.14150000000000000000001', '3.1415') + + assert Precise('3.1415') >= Precise('3.1415') + assert Precise('3.14150000000000000000001') >= Precise('3.1415') + + assert not Precise('3.1415') < Precise('3.1415') + + assert Precise('3.1415') <= Precise('3.1415') + assert Precise('3.1415') <= Precise('3.14150000000000000000001') From 9607d0427903a05334576f7ce33e3d4e92282a51 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Apr 2022 09:55:35 +0200 Subject: [PATCH 12/15] Improve ccxt imports --- freqtrade/exchange/exchange.py | 4 +--- setup.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 8bbbf6d4d..d2766cd6d 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -16,9 +16,7 @@ import arrow import ccxt import ccxt.async_support as ccxt_async from cachetools import TTLCache -from ccxt import Precise -from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, - decimal_to_precision) +from ccxt import ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, Precise, decimal_to_precision from pandas import DataFrame from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES, BuySell, diff --git a/setup.py b/setup.py index c5e418d0d..fadd4629f 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ setup( ], install_requires=[ # from requirements.txt - 'ccxt>=1.79.69', + 'ccxt>=1.80.67', 'SQLAlchemy', 'python-telegram-bot>=13.4', 'arrow>=0.17.0', From a1048fb619e186d83ff83ad25ba0e02e818d6fd1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 1 May 2022 17:00:00 +0200 Subject: [PATCH 13/15] Store monthly candles as "Mo" --- freqtrade/data/history/hdf5datahandler.py | 2 +- freqtrade/data/history/idatahandler.py | 16 ++++++++++++++-- freqtrade/data/history/jsondatahandler.py | 2 +- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 23120a4ba..165685960 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -40,7 +40,7 @@ class HDF5DataHandler(IDataHandler): return [ ( cls.rebuild_pair_from_filename(match[1]), - match[2], + cls.rebuild_timeframe_from_filename(match[2]), CandleType.from_string(match[3]) ) for match in _tmp if match and len(match.groups()) > 1] diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 2e6b070ca..bd795f480 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) class IDataHandler(ABC): - _OHLCV_REGEX = r'^([a-zA-Z_-]+)\-(\d+\S)\-?([a-zA-Z_]*)?(?=\.)' + _OHLCV_REGEX = r'^([a-zA-Z_-]+)\-(\d+[a-zA-Z]{1,2})\-?([a-zA-Z_]*)?(?=\.)' def __init__(self, datadir: Path) -> None: self._datadir = datadir @@ -201,7 +201,7 @@ class IDataHandler(ABC): datadir = datadir.joinpath('futures') candle = f"-{candle_type}" filename = datadir.joinpath( - f'{pair_s}-{timeframe}{candle}.{cls._get_file_extension()}') + f'{pair_s}-{cls.timeframe_to_file(timeframe)}{candle}.{cls._get_file_extension()}') return filename @classmethod @@ -210,6 +210,18 @@ class IDataHandler(ABC): filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}') return filename + @staticmethod + def timeframe_to_file(timeframe: str): + return timeframe.replace('M', 'Mo') + + @staticmethod + def rebuild_timeframe_from_filename(timeframe: str) -> str: + """ + converts timeframe from disk to file + Replaces mo with M (to avoid problems on case-insensitive filesystems) + """ + return re.sub('mo', 'M', timeframe, flags=re.IGNORECASE) + @staticmethod def rebuild_pair_from_filename(pair: str) -> str: """ diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 23054ac51..fa02c770b 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -41,7 +41,7 @@ class JsonDataHandler(IDataHandler): return [ ( cls.rebuild_pair_from_filename(match[1]), - match[2], + cls.rebuild_timeframe_from_filename(match[2]), CandleType.from_string(match[3]) ) for match in _tmp if match and len(match.groups()) > 1] From 2e65a1793d086ecbdf904d74a0dc3dc3b4ddeac1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 1 May 2022 19:51:25 +0200 Subject: [PATCH 14/15] Add fallback to load 1M files as well as 1Mo files --- freqtrade/data/history/hdf5datahandler.py | 11 +++++++--- freqtrade/data/history/idatahandler.py | 7 ++++--- freqtrade/data/history/jsondatahandler.py | 12 ++++++++--- tests/data/test_history.py | 25 ++++++++++++----------- 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 165685960..6099c22bc 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -77,7 +77,8 @@ class HDF5DataHandler(IDataHandler): key = self._pair_ohlcv_key(pair, timeframe) _data = data.copy() - filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) + filename = self._pair_data_filename( + self._datadir, pair, self.timeframe_to_file(timeframe), candle_type) self.create_dir_if_needed(filename) _data.loc[:, self._columns].to_hdf( @@ -104,12 +105,16 @@ class HDF5DataHandler(IDataHandler): filename = self._pair_data_filename( self._datadir, pair, - timeframe, + self.timeframe_to_file(timeframe), candle_type=candle_type ) if not filename.exists(): - return pd.DataFrame(columns=self._columns) + # Fallback mode for 1M files + filename = self._pair_data_filename( + self._datadir, pair, timeframe, candle_type=candle_type) + if not filename.exists(): + return pd.DataFrame(columns=self._columns) where = [] if timerange: if timerange.starttype == 'date': diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index bd795f480..69d6212ee 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -98,7 +98,8 @@ class IDataHandler(ABC): :param candle_type: Any of the enum CandleType (must match trading mode!) :return: True when deleted, false if file did not exist. """ - filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) + filename = self._pair_data_filename( + self._datadir, pair, self.timeframe_to_file(timeframe), candle_type) if filename.exists(): filename.unlink() return True @@ -201,7 +202,7 @@ class IDataHandler(ABC): datadir = datadir.joinpath('futures') candle = f"-{candle_type}" filename = datadir.joinpath( - f'{pair_s}-{cls.timeframe_to_file(timeframe)}{candle}.{cls._get_file_extension()}') + f'{pair_s}-{timeframe}{candle}.{cls._get_file_extension()}') return filename @classmethod @@ -220,7 +221,7 @@ class IDataHandler(ABC): converts timeframe from disk to file Replaces mo with M (to avoid problems on case-insensitive filesystems) """ - return re.sub('mo', 'M', timeframe, flags=re.IGNORECASE) + return re.sub('1mo', '1M', timeframe, flags=re.IGNORECASE) @staticmethod def rebuild_pair_from_filename(pair: str) -> str: diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index fa02c770b..38402a113 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -77,7 +77,8 @@ class JsonDataHandler(IDataHandler): :param candle_type: Any of the enum CandleType (must match trading mode!) :return: None """ - filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) + filename = self._pair_data_filename( + self._datadir, pair, self.timeframe_to_file(timeframe), candle_type) self.create_dir_if_needed(filename) _data = data.copy() # Convert date to int @@ -103,9 +104,14 @@ class JsonDataHandler(IDataHandler): :param candle_type: Any of the enum CandleType (must match trading mode!) :return: DataFrame with ohlcv data, or empty DataFrame """ - filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type=candle_type) + filename = self._pair_data_filename( + self._datadir, pair, self.timeframe_to_file(timeframe), candle_type=candle_type) if not filename.exists(): - return DataFrame(columns=self._columns) + # Fallback mode for 1M files + filename = self._pair_data_filename( + self._datadir, pair, timeframe, candle_type=candle_type) + if not filename.exists(): + return DataFrame(columns=self._columns) try: pairdata = read_json(filename, orient='values') pairdata.columns = self._columns diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 82d4a841c..1e7d8855e 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -158,21 +158,22 @@ def test_testdata_path(testdatadir) -> None: assert str(Path('tests') / 'testdata') in str(testdatadir) -@pytest.mark.parametrize("pair,expected_result,candle_type", [ - ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-5m.json', ""), - ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-5m.json', ""), - ("ETHH20", 'freqtrade/hello/world/ETHH20-5m.json', ""), - (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-5m.json', ""), - ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-5m.json', ""), - ("ACC_OLD/BTC", 'freqtrade/hello/world/ACC_OLD_BTC-5m.json', ""), - ("ETH/BTC", 'freqtrade/hello/world/futures/ETH_BTC-5m-mark.json', "mark"), - ("ACC_OLD/BTC", 'freqtrade/hello/world/futures/ACC_OLD_BTC-5m-index.json', "index"), +@pytest.mark.parametrize("pair,timeframe,expected_result,candle_type", [ + ("ETH/BTC", "5m", "freqtrade/hello/world/ETH_BTC-5m.json", ""), + ("ETH/USDT", "1M", "freqtrade/hello/world/ETH_USDT-1Mo.json", ""), + ("Fabric Token/ETH", "5m", "freqtrade/hello/world/Fabric_Token_ETH-5m.json", ""), + ("ETHH20", "5m", "freqtrade/hello/world/ETHH20-5m.json", ""), + (".XBTBON2H", "5m", "freqtrade/hello/world/_XBTBON2H-5m.json", ""), + ("ETHUSD.d", "5m", "freqtrade/hello/world/ETHUSD_d-5m.json", ""), + ("ACC_OLD/BTC", "5m", "freqtrade/hello/world/ACC_OLD_BTC-5m.json", ""), + ("ETH/BTC", "5m", "freqtrade/hello/world/futures/ETH_BTC-5m-mark.json", "mark"), + ("ACC_OLD/BTC", "5m", "freqtrade/hello/world/futures/ACC_OLD_BTC-5m-index.json", "index"), ]) -def test_json_pair_data_filename(pair, expected_result, candle_type): +def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type): fn = JsonDataHandler._pair_data_filename( Path('freqtrade/hello/world'), pair, - '5m', + JsonDataHandler.timeframe_to_file(timeframe), CandleType.from_string(candle_type) ) assert isinstance(fn, Path) @@ -180,7 +181,7 @@ def test_json_pair_data_filename(pair, expected_result, candle_type): fn = JsonGzDataHandler._pair_data_filename( Path('freqtrade/hello/world'), pair, - '5m', + JsonGzDataHandler.timeframe_to_file(timeframe), candle_type=CandleType.from_string(candle_type) ) assert isinstance(fn, Path) From 76637d3939994219e1ec15e1cdf7e513217536a5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 May 2022 19:53:01 +0200 Subject: [PATCH 15/15] Simplify timeframe-transition --- freqtrade/data/history/hdf5datahandler.py | 7 +++---- freqtrade/data/history/idatahandler.py | 9 ++++++--- freqtrade/data/history/jsondatahandler.py | 7 +++---- tests/data/test_history.py | 4 ++-- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 6099c22bc..dadc9c7e6 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -77,8 +77,7 @@ class HDF5DataHandler(IDataHandler): key = self._pair_ohlcv_key(pair, timeframe) _data = data.copy() - filename = self._pair_data_filename( - self._datadir, pair, self.timeframe_to_file(timeframe), candle_type) + filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) self.create_dir_if_needed(filename) _data.loc[:, self._columns].to_hdf( @@ -105,14 +104,14 @@ class HDF5DataHandler(IDataHandler): filename = self._pair_data_filename( self._datadir, pair, - self.timeframe_to_file(timeframe), + timeframe, candle_type=candle_type ) if not filename.exists(): # Fallback mode for 1M files filename = self._pair_data_filename( - self._datadir, pair, timeframe, candle_type=candle_type) + self._datadir, pair, timeframe, candle_type=candle_type, no_timeframe_modify=True) if not filename.exists(): return pd.DataFrame(columns=self._columns) where = [] diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 69d6212ee..07dc7c763 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -98,8 +98,7 @@ class IDataHandler(ABC): :param candle_type: Any of the enum CandleType (must match trading mode!) :return: True when deleted, false if file did not exist. """ - filename = self._pair_data_filename( - self._datadir, pair, self.timeframe_to_file(timeframe), candle_type) + filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) if filename.exists(): filename.unlink() return True @@ -194,10 +193,14 @@ class IDataHandler(ABC): datadir: Path, pair: str, timeframe: str, - candle_type: CandleType + candle_type: CandleType, + no_timeframe_modify: bool = False ) -> Path: pair_s = misc.pair_to_filename(pair) candle = "" + if not no_timeframe_modify: + timeframe = cls.timeframe_to_file(timeframe) + if candle_type != CandleType.SPOT: datadir = datadir.joinpath('futures') candle = f"-{candle_type}" diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 38402a113..83ec183df 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -77,8 +77,7 @@ class JsonDataHandler(IDataHandler): :param candle_type: Any of the enum CandleType (must match trading mode!) :return: None """ - filename = self._pair_data_filename( - self._datadir, pair, self.timeframe_to_file(timeframe), candle_type) + filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) self.create_dir_if_needed(filename) _data = data.copy() # Convert date to int @@ -105,11 +104,11 @@ class JsonDataHandler(IDataHandler): :return: DataFrame with ohlcv data, or empty DataFrame """ filename = self._pair_data_filename( - self._datadir, pair, self.timeframe_to_file(timeframe), candle_type=candle_type) + self._datadir, pair, timeframe, candle_type=candle_type) if not filename.exists(): # Fallback mode for 1M files filename = self._pair_data_filename( - self._datadir, pair, timeframe, candle_type=candle_type) + self._datadir, pair, timeframe, candle_type=candle_type, no_timeframe_modify=True) if not filename.exists(): return DataFrame(columns=self._columns) try: diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 1e7d8855e..9709e7ad0 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -173,7 +173,7 @@ def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type): fn = JsonDataHandler._pair_data_filename( Path('freqtrade/hello/world'), pair, - JsonDataHandler.timeframe_to_file(timeframe), + timeframe, CandleType.from_string(candle_type) ) assert isinstance(fn, Path) @@ -181,7 +181,7 @@ def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type): fn = JsonGzDataHandler._pair_data_filename( Path('freqtrade/hello/world'), pair, - JsonGzDataHandler.timeframe_to_file(timeframe), + timeframe, candle_type=CandleType.from_string(candle_type) ) assert isinstance(fn, Path)