diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 23120a4ba..dadc9c7e6 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -40,7 +40,7 @@ class HDF5DataHandler(IDataHandler): return [ ( cls.rebuild_pair_from_filename(match[1]), - match[2], + cls.rebuild_timeframe_from_filename(match[2]), CandleType.from_string(match[3]) ) for match in _tmp if match and len(match.groups()) > 1] @@ -109,7 +109,11 @@ class HDF5DataHandler(IDataHandler): ) if not filename.exists(): - return pd.DataFrame(columns=self._columns) + # Fallback mode for 1M files + filename = self._pair_data_filename( + self._datadir, pair, timeframe, candle_type=candle_type, no_timeframe_modify=True) + if not filename.exists(): + return pd.DataFrame(columns=self._columns) where = [] if timerange: if timerange.starttype == 'date': diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 2e6b070ca..07dc7c763 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) class IDataHandler(ABC): - _OHLCV_REGEX = r'^([a-zA-Z_-]+)\-(\d+\S)\-?([a-zA-Z_]*)?(?=\.)' + _OHLCV_REGEX = r'^([a-zA-Z_-]+)\-(\d+[a-zA-Z]{1,2})\-?([a-zA-Z_]*)?(?=\.)' def __init__(self, datadir: Path) -> None: self._datadir = datadir @@ -193,10 +193,14 @@ class IDataHandler(ABC): datadir: Path, pair: str, timeframe: str, - candle_type: CandleType + candle_type: CandleType, + no_timeframe_modify: bool = False ) -> Path: pair_s = misc.pair_to_filename(pair) candle = "" + if not no_timeframe_modify: + timeframe = cls.timeframe_to_file(timeframe) + if candle_type != CandleType.SPOT: datadir = datadir.joinpath('futures') candle = f"-{candle_type}" @@ -210,6 +214,18 @@ class IDataHandler(ABC): filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}') return filename + @staticmethod + def timeframe_to_file(timeframe: str): + return timeframe.replace('M', 'Mo') + + @staticmethod + def rebuild_timeframe_from_filename(timeframe: str) -> str: + """ + converts timeframe from disk to file + Replaces mo with M (to avoid problems on case-insensitive filesystems) + """ + return re.sub('1mo', '1M', timeframe, flags=re.IGNORECASE) + @staticmethod def rebuild_pair_from_filename(pair: str) -> str: """ diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 23054ac51..83ec183df 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -41,7 +41,7 @@ class JsonDataHandler(IDataHandler): return [ ( cls.rebuild_pair_from_filename(match[1]), - match[2], + cls.rebuild_timeframe_from_filename(match[2]), CandleType.from_string(match[3]) ) for match in _tmp if match and len(match.groups()) > 1] @@ -103,9 +103,14 @@ class JsonDataHandler(IDataHandler): :param candle_type: Any of the enum CandleType (must match trading mode!) :return: DataFrame with ohlcv data, or empty DataFrame """ - filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type=candle_type) + filename = self._pair_data_filename( + self._datadir, pair, timeframe, candle_type=candle_type) if not filename.exists(): - return DataFrame(columns=self._columns) + # Fallback mode for 1M files + filename = self._pair_data_filename( + self._datadir, pair, timeframe, candle_type=candle_type, no_timeframe_modify=True) + if not filename.exists(): + return DataFrame(columns=self._columns) try: pairdata = read_json(filename, orient='values') pairdata.columns = self._columns diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index ee804aa68..d2766cd6d 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -16,8 +16,7 @@ import arrow import ccxt import ccxt.async_support as ccxt_async from cachetools import TTLCache -from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, - decimal_to_precision) +from ccxt import ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, Precise, decimal_to_precision from pandas import DataFrame from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES, BuySell, @@ -704,10 +703,11 @@ class Exchange: # counting_mode=self.precisionMode, # )) if self.precisionMode == TICK_SIZE: - precision = self.markets[pair]['precision']['price'] - missing = price % precision - if missing != 0: - price = round(price - missing + precision, 10) + precision = Precise(str(self.markets[pair]['precision']['price'])) + price_str = Precise(str(price)) + missing = price_str % precision + if not missing == Precise("0"): + price = round(float(str(price_str - missing + precision)), 14) else: symbol_prec = self.markets[pair]['precision']['price'] big_price = price * pow(10, symbol_prec) @@ -1457,6 +1457,23 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + def _get_price_side(self, side: str, is_short: bool, conf_strategy: Dict) -> str: + price_side = conf_strategy['price_side'] + + if price_side in ('same', 'other'): + price_map = { + ('entry', 'long', 'same'): 'bid', + ('entry', 'long', 'other'): 'ask', + ('entry', 'short', 'same'): 'ask', + ('entry', 'short', 'other'): 'bid', + ('exit', 'long', 'same'): 'ask', + ('exit', 'long', 'other'): 'bid', + ('exit', 'short', 'same'): 'bid', + ('exit', 'short', 'other'): 'ask', + } + price_side = price_map[(side, 'short' if is_short else 'long', price_side)] + return price_side + def get_rate(self, pair: str, refresh: bool, side: EntryExit, is_short: bool) -> float: """ @@ -1483,20 +1500,7 @@ class Exchange: conf_strategy = self._config.get(strat_name, {}) - price_side = conf_strategy['price_side'] - - if price_side in ('same', 'other'): - price_map = { - ('entry', 'long', 'same'): 'bid', - ('entry', 'long', 'other'): 'ask', - ('entry', 'short', 'same'): 'ask', - ('entry', 'short', 'other'): 'bid', - ('exit', 'long', 'same'): 'ask', - ('exit', 'long', 'other'): 'bid', - ('exit', 'short', 'same'): 'bid', - ('exit', 'short', 'other'): 'ask', - } - price_side = price_map[(side, 'short' if is_short else 'long', price_side)] + price_side = self._get_price_side(side, is_short, conf_strategy) price_side_word = price_side.capitalize() diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9aee1215f..4e604898f 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -542,11 +542,11 @@ class Backtesting: trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60) try: - closerate = self._get_close_rate(row, trade, exit_, trade_dur) + close_rate = self._get_close_rate(row, trade, exit_, trade_dur) except ValueError: return None - # call the custom exit price,with default value as previous closerate - current_profit = trade.calc_profit_ratio(closerate) + # call the custom exit price,with default value as previous close_rate + current_profit = trade.calc_profit_ratio(close_rate) order_type = self.strategy.order_types['exit'] if exit_.exit_type in (ExitType.EXIT_SIGNAL, ExitType.CUSTOM_EXIT): # Checks and adds an exit tag, after checking that the length of the @@ -560,24 +560,24 @@ class Backtesting: exit_reason = row[EXIT_TAG_IDX] # Custom exit pricing only for exit-signals if order_type == 'limit': - closerate = strategy_safe_wrapper(self.strategy.custom_exit_price, - default_retval=closerate)( + close_rate = strategy_safe_wrapper(self.strategy.custom_exit_price, + default_retval=close_rate)( pair=trade.pair, trade=trade, current_time=exit_candle_time, - proposed_rate=closerate, current_profit=current_profit, + proposed_rate=close_rate, current_profit=current_profit, exit_tag=exit_reason) # We can't place orders lower than current low. # freqtrade does not support this in live, and the order would fill immediately if trade.is_short: - closerate = min(closerate, row[HIGH_IDX]) + close_rate = min(close_rate, row[HIGH_IDX]) else: - closerate = max(closerate, row[LOW_IDX]) + close_rate = max(close_rate, row[LOW_IDX]) # Confirm trade exit: time_in_force = self.strategy.order_time_in_force['exit'] if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)( pair=trade.pair, trade=trade, order_type='limit', amount=trade.amount, - rate=closerate, + rate=close_rate, time_in_force=time_in_force, sell_reason=exit_reason, # deprecated exit_reason=exit_reason, @@ -600,12 +600,12 @@ class Backtesting: side=trade.exit_side, order_type=order_type, status="open", - price=closerate, - average=closerate, + price=close_rate, + average=close_rate, amount=trade.amount, filled=0, remaining=trade.amount, - cost=trade.amount * closerate, + cost=trade.amount * close_rate, ) trade.orders.append(order) return trade diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 1a9be4503..f26de8b5c 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1410,14 +1410,14 @@ class Telegram(RPCHandler): "Optionally takes a rate at which to sell " "(only applies to limit orders).` \n") message = ( - "_BotControl_\n" + "_Bot Control_\n" "------------\n" "*/start:* `Starts the trader`\n" "*/stop:* Stops the trader\n" "*/stopbuy:* `Stops buying, but handles open trades gracefully` \n" "*/forceexit |all:* `Instantly exits the given trade or all trades, " "regardless of profit`\n" - "*/fe |all:* `Alias to /forceexit`" + "*/fe |all:* `Alias to /forceexit`\n" f"{force_enter_text if self._config.get('force_entry_enable', False) else ''}" "*/delete :* `Instantly delete the given trade in the database`\n" "*/whitelist:* `Show current whitelist` \n" diff --git a/requirements-plot.txt b/requirements-plot.txt index d9faed301..e17efbc71 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,4 +1,4 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.7.0 +plotly==5.8.0 diff --git a/setup.py b/setup.py index c5e418d0d..fadd4629f 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ setup( ], install_requires=[ # from requirements.txt - 'ccxt>=1.79.69', + 'ccxt>=1.80.67', 'SQLAlchemy', 'python-telegram-bot>=13.4', 'arrow>=0.17.0', diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 82d4a841c..9709e7ad0 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -158,21 +158,22 @@ def test_testdata_path(testdatadir) -> None: assert str(Path('tests') / 'testdata') in str(testdatadir) -@pytest.mark.parametrize("pair,expected_result,candle_type", [ - ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-5m.json', ""), - ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-5m.json', ""), - ("ETHH20", 'freqtrade/hello/world/ETHH20-5m.json', ""), - (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-5m.json', ""), - ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-5m.json', ""), - ("ACC_OLD/BTC", 'freqtrade/hello/world/ACC_OLD_BTC-5m.json', ""), - ("ETH/BTC", 'freqtrade/hello/world/futures/ETH_BTC-5m-mark.json', "mark"), - ("ACC_OLD/BTC", 'freqtrade/hello/world/futures/ACC_OLD_BTC-5m-index.json', "index"), +@pytest.mark.parametrize("pair,timeframe,expected_result,candle_type", [ + ("ETH/BTC", "5m", "freqtrade/hello/world/ETH_BTC-5m.json", ""), + ("ETH/USDT", "1M", "freqtrade/hello/world/ETH_USDT-1Mo.json", ""), + ("Fabric Token/ETH", "5m", "freqtrade/hello/world/Fabric_Token_ETH-5m.json", ""), + ("ETHH20", "5m", "freqtrade/hello/world/ETHH20-5m.json", ""), + (".XBTBON2H", "5m", "freqtrade/hello/world/_XBTBON2H-5m.json", ""), + ("ETHUSD.d", "5m", "freqtrade/hello/world/ETHUSD_d-5m.json", ""), + ("ACC_OLD/BTC", "5m", "freqtrade/hello/world/ACC_OLD_BTC-5m.json", ""), + ("ETH/BTC", "5m", "freqtrade/hello/world/futures/ETH_BTC-5m-mark.json", "mark"), + ("ACC_OLD/BTC", "5m", "freqtrade/hello/world/futures/ACC_OLD_BTC-5m-index.json", "index"), ]) -def test_json_pair_data_filename(pair, expected_result, candle_type): +def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type): fn = JsonDataHandler._pair_data_filename( Path('freqtrade/hello/world'), pair, - '5m', + timeframe, CandleType.from_string(candle_type) ) assert isinstance(fn, Path) @@ -180,7 +181,7 @@ def test_json_pair_data_filename(pair, expected_result, candle_type): fn = JsonGzDataHandler._pair_data_filename( Path('freqtrade/hello/world'), pair, - '5m', + timeframe, candle_type=CandleType.from_string(candle_type) ) assert isinstance(fn, Path) diff --git a/tests/exchange/test_ccxt_precise.py b/tests/exchange/test_ccxt_precise.py new file mode 100644 index 000000000..026adb4c1 --- /dev/null +++ b/tests/exchange/test_ccxt_precise.py @@ -0,0 +1,75 @@ +from ccxt import Precise + + +ws = Precise('-1.123e-6') +ws = Precise('-1.123e-6') +xs = Precise('0.00000002') +ys = Precise('69696900000') +zs = Precise('0') + + +def test_precise(): + assert ys * xs == '1393.938' + assert xs * ys == '1393.938' + + assert ys + xs == '69696900000.00000002' + assert xs + ys == '69696900000.00000002' + assert xs - ys == '-69696899999.99999998' + assert ys - xs == '69696899999.99999998' + assert xs / ys == '0' + assert ys / xs == '3484845000000000000' + + assert ws * xs == '-0.00000000000002246' + assert xs * ws == '-0.00000000000002246' + + assert ws + xs == '-0.000001103' + assert xs + ws == '-0.000001103' + + assert xs - ws == '0.000001143' + assert ws - xs == '-0.000001143' + + assert xs / ws == '-0.017809439002671415' + assert ws / xs == '-56.15' + + assert zs * ws == '0' + assert zs * xs == '0' + assert zs * ys == '0' + assert ws * zs == '0' + assert xs * zs == '0' + assert ys * zs == '0' + + assert zs + ws == '-0.000001123' + assert zs + xs == '0.00000002' + assert zs + ys == '69696900000' + assert ws + zs == '-0.000001123' + assert xs + zs == '0.00000002' + assert ys + zs == '69696900000' + + assert abs(Precise('-500.1')) == '500.1' + assert abs(Precise('213')) == '213' + + assert abs(Precise('-500.1')) == '500.1' + assert -Precise('213') == '-213' + + assert Precise('10.1') % Precise('0.5') == '0.1' + assert Precise('5550') % Precise('120') == '30' + + assert Precise('-0.0') == Precise('0') + assert Precise('5.534000') == Precise('5.5340') + + assert min(Precise('-3.1415'), Precise('-2')) == '-3.1415' + + assert max(Precise('3.1415'), Precise('-2')) == '3.1415' + + assert Precise('2') > Precise('1.2345') + assert not Precise('-3.1415') > Precise('-2') + assert not Precise('3.1415') > Precise('3.1415') + assert Precise.string_gt('3.14150000000000000000001', '3.1415') + + assert Precise('3.1415') >= Precise('3.1415') + assert Precise('3.14150000000000000000001') >= Precise('3.1415') + + assert not Precise('3.1415') < Precise('3.1415') + + assert Precise('3.1415') <= Precise('3.1415') + assert Precise('3.1415') <= Precise('3.14150000000000000000001') diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index e580c82d3..53e6cc3f3 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -305,6 +305,7 @@ def test_amount_to_precision( (234.53, 4, 0.5, 235.0), (0.891534, 4, 0.0001, 0.8916), (64968.89, 4, 0.01, 64968.89), + (0.000000003483, 4, 1e-12, 0.000000003483), ]) def test_price_to_precision(default_conf, mocker, price, precision_mode, precision, expected):