Merge branch 'develop' into feat/sort_volatility

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11.7-slim-bookworm as base
+FROM python:3.11.8-slim-bookworm as base
 
 # Setup env
 ENV LANG C.UTF-8

docker/Dockerfile.armhf

@@ -1,4 +1,4 @@
-FROM python:3.11.7-slim-bookworm as base
+FROM python:3.11.8-slim-bookworm as base
 
 # Setup env
 ENV LANG C.UTF-8

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
 markdown==3.5.2
 mkdocs==1.5.3
-mkdocs-material==9.5.9
+mkdocs-material==9.5.11
 mdx_truly_sane_lists==1.3
 pymdown-extensions==10.7
 jinja2==3.1.3

docs/strategy-callbacks.md

@@ -791,7 +791,7 @@ Returning a value more than the above (so remaining stake_amount would become ne
     If you wish to buy additional orders with DCA, then make sure to leave enough funds in the wallet for that.
     Using 'unlimited' stake amount with DCA orders requires you to also implement the `custom_stake_amount()` callback to avoid allocating all funds to the initial order.
 
-!!! Warning
+!!! Warning "Stoploss calculation"
     Stoploss is still calculated from the initial opening price, not averaged price.
     Regular stoploss rules still apply (cannot move down).
 
@@ -801,6 +801,11 @@ Returning a value more than the above (so remaining stake_amount would become ne
     During backtesting this callback is called for each candle in `timeframe` or `timeframe_detail`, so run-time performance will be affected.
     This can also cause deviating results between live and backtesting, since backtesting can adjust the trade only once per candle, whereas live could adjust the trade multiple times per candle.
 
+!!! Warning "Performance with many position adjustments"
+    Position adjustments can be a good approach to increase a strategy's output - but it can also have drawbacks if using this feature extensively.  
+    Each of the orders will be attached to the trade object for the duration of the trade - hence increasing memory usage.
+    Trades with long duration and 10s or even 100ds of position adjustments are therefore not recommended, and should be closed at regular intervals to not affect performance.
+
 ``` python
 from freqtrade.persistence import Trade
 

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ Freqtrade bot """
-__version__ = '2024.2-dev'
+__version__ = '2024.3-dev'
 
 if 'dev' in __version__:
     from pathlib import Path

freqtrade/data/history/hdf5datahandler.py

@@ -35,7 +35,7 @@ class HDF5DataHandler(IDataHandler):
         self.create_dir_if_needed(filename)
 
         _data.loc[:, self._columns].to_hdf(
-            filename, key, mode='a', complevel=9, complib='blosc',
+            filename, key=key, mode='a', complevel=9, complib='blosc',
             format='table', data_columns=['date']
         )
 
@@ -110,7 +110,7 @@ class HDF5DataHandler(IDataHandler):
         key = self._pair_trades_key(pair)
 
         data.to_hdf(
-            self._pair_trades_filename(self._datadir, pair), key,
+            self._pair_trades_filename(self._datadir, pair), key=key,
             mode='a', complevel=9, complib='blosc',
             format='table', data_columns=['timestamp']
         )

freqtrade/data/history/jsondatahandler.py

@@ -37,7 +37,7 @@ class JsonDataHandler(IDataHandler):
         self.create_dir_if_needed(filename)
         _data = data.copy()
         # Convert date to int
-        _data['date'] = _data['date'].view(np.int64) // 1000 // 1000
+        _data['date'] = _data['date'].astype(np.int64) // 1000 // 1000
 
         # Reset index, select only appropriate columns and save as json
         _data.reset_index(drop=True).loc[:, self._columns].to_json(

freqtrade/freqtradebot.py

@@ -128,8 +128,9 @@ class FreqtradeBot(LoggingMixin):
                 self.update_funding_fees()
                 self.wallets.update()
 
-            # TODO: This would be more efficient if scheduled in utc time, and performed at each
+            # This would be more efficient if scheduled in utc time, and performed at each
-            # TODO: funding interval, specified by funding_fee_times on the exchange classes
+            # funding interval, specified by funding_fee_times on the exchange classes
+            # However, this reduces the precision - and might therefore lead to problems.
             for time_slot in range(0, 24):
                 for minutes in [1, 31]:
                     t = str(time(time_slot, minutes, 2))

freqtrade/optimize/analysis/lookahead_helpers.py

@@ -107,9 +107,9 @@ class LookaheadAnalysisSubFunctions:
                 csv_df = add_or_update_row(csv_df, new_row_data)
 
         # Fill NaN values with a default value (e.g., 0)
-        csv_df['total_signals'] = csv_df['total_signals'].fillna(0)
+        csv_df['total_signals'] = csv_df['total_signals'].astype(int).fillna(0)
-        csv_df['biased_entry_signals'] = csv_df['biased_entry_signals'].fillna(0)
+        csv_df['biased_entry_signals'] = csv_df['biased_entry_signals'].astype(int).fillna(0)
-        csv_df['biased_exit_signals'] = csv_df['biased_exit_signals'].fillna(0)
+        csv_df['biased_exit_signals'] = csv_df['biased_exit_signals'].astype(int).fillna(0)
 
         # Convert columns to integers
         csv_df['total_signals'] = csv_df['total_signals'].astype(int)

freqtrade/optimize/backtesting.py

@@ -201,7 +201,7 @@ class Backtesting:
 
         self.prepare_backtest(False)
 
-        self.wallets = Wallets(self.config, self.exchange, log=False)
+        self.wallets = Wallets(self.config, self.exchange, is_backtest=True)
 
         self.progress = BTProgress()
         self.abort = False

freqtrade/optimize/optimize_reports/optimize_reports.py

@@ -215,7 +215,7 @@ def _get_resample_from_period(period: str) -> str:
         # Weekly defaulting to Monday.
         return '1W-MON'
     if period == 'month':
-        return '1M'
+        return '1ME'
     raise ValueError(f"Period {period} is not supported.")
 
 

freqtrade/rpc/rpc.py

@@ -1155,7 +1155,7 @@ class RPC:
         }
         if has_content:
 
-            dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].view(int64) // 1000 // 1000
+            dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].astype(int64) // 1000 // 1000
             # Move signal close to separate column when signal for easy plotting
             for sig_type in signals.keys():
                 if sig_type in dataframe.columns:

freqtrade/wallets.py

@@ -36,9 +36,9 @@ class PositionWallet(NamedTuple):
 
 class Wallets:
 
-    def __init__(self, config: Config, exchange: Exchange, log: bool = True) -> None:
+    def __init__(self, config: Config, exchange: Exchange, is_backtest: bool = False) -> None:
         self._config = config
-        self._log = log
+        self._is_backtest = is_backtest
         self._exchange = exchange
         self._wallets: Dict[str, Wallet] = {}
         self._positions: Dict[str, PositionWallet] = {}
@@ -78,11 +78,11 @@ class Wallets:
         _wallets = {}
         _positions = {}
         open_trades = Trade.get_trades_proxy(is_open=True)
-        # If not backtesting...
+        if not self._is_backtest:
-        # TODO: potentially remove the ._log workaround to determine backtest mode.
+            # Live / Dry-run mode
-        if self._log:
             tot_profit = Trade.get_total_closed_profit()
         else:
+            # Backtest mode
             tot_profit = LocalTrade.total_profit
         tot_profit += sum(trade.realized_profit for trade in open_trades)
         tot_in_trades = sum(trade.stake_amount for trade in open_trades)
@@ -177,7 +177,7 @@ class Wallets:
                 self._update_live()
             else:
                 self._update_dry()
-            if self._log:
+            if not self._is_backtest:
                 logger.info('Wallets synced.')
             self._last_wallet_refresh = dt_now()
 
@@ -341,19 +341,19 @@ class Wallets:
             max_allowed_stake = min(max_allowed_stake, max_stake_amount - trade_amount)
 
         if min_stake_amount is not None and min_stake_amount > max_allowed_stake:
-            if self._log:
+            if not self._is_backtest:
                 logger.warning("Minimum stake amount > available balance. "
                                f"{min_stake_amount} > {max_allowed_stake}")
             return 0
         if min_stake_amount is not None and stake_amount < min_stake_amount:
-            if self._log:
+            if not self._is_backtest:
                 logger.info(
                     f"Stake amount for pair {pair} is too small "
                     f"({stake_amount} < {min_stake_amount}), adjusting to {min_stake_amount}."
                 )
             if stake_amount * 1.3 < min_stake_amount:
                 # Top-cap stake-amount adjustments to +30%.
-                if self._log:
+                if not self._is_backtest:
                     logger.info(
                         f"Adjusted stake amount for pair {pair} is more than 30% bigger than "
                         f"the desired stake amount of ({stake_amount:.8f} * 1.3 = "
@@ -363,7 +363,7 @@ class Wallets:
             stake_amount = min_stake_amount
 
         if stake_amount > max_allowed_stake:
-            if self._log:
+            if not self._is_backtest:
                 logger.info(
                     f"Stake amount for pair {pair} is too big "
                     f"({stake_amount} > {max_allowed_stake}), adjusting to {max_allowed_stake}."

requirements-dev.txt

@@ -10,7 +10,7 @@ coveralls==3.3.1
 ruff==0.2.2
 mypy==1.8.0
 pre-commit==3.6.2
-pytest==8.0.1
+pytest==8.0.2
 pytest-asyncio==0.23.5
 pytest-cov==4.1.0
 pytest-mock==3.12.0
@@ -21,7 +21,7 @@ isort==5.13.2
 time-machine==2.13.0
 
 # Convert jupyter notebooks to markdown documents
-nbconvert==7.16.0
+nbconvert==7.16.1
 
 # mypy types
 types-cachetools==5.3.0.7

requirements.txt

@@ -1,9 +1,9 @@
 numpy==1.26.4
-pandas==2.1.4
+pandas==2.2.1
 pandas-ta==0.3.14b
 
-ccxt==4.2.47
+ccxt==4.2.51
-cryptography==42.0.4
+cryptography==42.0.5
 aiohttp==3.9.3
 SQLAlchemy==2.0.27
 python-telegram-bot==20.8
@@ -30,14 +30,14 @@ py_find_1st==1.1.6
 # Load ticker files 30% faster
 python-rapidjson==1.14
 # Properly format api responses
-orjson==3.9.14
+orjson==3.9.15
 
 # Notify systemd
 sdnotify==0.3.2
 
 # API Server
-fastapi==0.109.2
+fastapi==0.110.0
-pydantic==2.6.1
+pydantic==2.6.2
 uvicorn==0.27.1
 pyjwt==2.8.0
 aiofiles==23.2.1
@@ -50,6 +50,7 @@ questionary==2.0.1
 prompt-toolkit==3.0.36
 # Extensions to datetime library
 python-dateutil==2.8.2
+pytz==2024.1
 
 #Futures
 schedule==1.2.1

setup.py

@@ -110,6 +110,7 @@ setup(
         'cryptography',
         'sdnotify',
         'python-dateutil',
+        'pytz',
         'packaging',
     ],
     extras_require={

tests/conftest.py

@@ -177,7 +177,7 @@ def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05', ran
 def generate_test_data_raw(timeframe: str, size: int, start: str = '2020-07-05', random_seed=42):
     """ Generates data in the ohlcv format used by ccxt """
     df = generate_test_data(timeframe, size, start, random_seed)
-    df['date'] = df.loc[:, 'date'].view(np.int64) // 1000 // 1000
+    df['date'] = df.loc[:, 'date'].astype(np.int64) // 1000 // 1000
     return list(list(x) for x in zip(*(df[x].values.tolist() for x in df.columns)))
 
 

tests/freqtradebot/test_freqtradebot.py

@@ -4680,9 +4680,14 @@ def test_get_valid_price(mocker, default_conf_usdt) -> None:
     ('futures', 17, "2021-08-31 23:59:59", "2021-09-01 08:01:07"),
     ('futures', 17, "2021-08-31 23:59:58", "2021-09-01 08:01:07"),
 ])
+@pytest.mark.parametrize('tzoffset', [
+    '+00:00',
+    '+01:00',
+    '-02:00',
+])
 def test_update_funding_fees_schedule(mocker, default_conf, trading_mode, calls, time_machine,
-                                      t1, t2):
+                                      t1, t2, tzoffset):
-    time_machine.move_to(f"{t1} +00:00", tick=False)
+    time_machine.move_to(f"{t1} {tzoffset}", tick=False)
 
     patch_RPCManager(mocker)
     patch_exchange(mocker)
@@ -4691,7 +4696,7 @@ def test_update_funding_fees_schedule(mocker, default_conf, trading_mode, calls,
     default_conf['margin_mode'] = 'isolated'
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
-    time_machine.move_to(f"{t2} +00:00", tick=False)
+    time_machine.move_to(f"{t2} {tzoffset}", tick=False)
     # Check schedule jobs in debugging with freqtrade._schedule.jobs
     freqtrade._schedule.run_pending()
 

tests/optimize/test_backtesting_adjust_position.py

@@ -57,28 +57,30 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) ->
                                      ),
          'close_date': pd.to_datetime([dt_utc(2018, 1, 29, 22, 00, 0),
                                        dt_utc(2018, 1, 30, 4, 10, 0)], utc=True),
-         'open_rate': [0.10401764894444211, 0.10302485],
+         'open_rate': [0.10401764891917063, 0.10302485],
-         'close_rate': [0.10453904066847439, 0.103541],
+         'close_rate': [0.10453904064307624, 0.10354126528822055],
          'fee_open': [0.0025, 0.0025],
          'fee_close': [0.0025, 0.0025],
          'trade_duration': [200, 40],
          'profit_ratio': [0.0, 0.0],
          'profit_abs': [0.0, 0.0],
          'exit_reason': [ExitType.ROI.value, ExitType.ROI.value],
-         'initial_stop_loss_abs': [0.0940005, 0.09272236],
+         'initial_stop_loss_abs': [0.0940005, 0.092722365],
          'initial_stop_loss_ratio': [-0.1, -0.1],
-         'stop_loss_abs': [0.0940005, 0.09272236],
+         'stop_loss_abs': [0.0940005, 0.092722365],
          'stop_loss_ratio': [-0.1, -0.1],
          'min_rate': [0.10370188, 0.10300000000000001],
-         'max_rate': [0.10481985, 0.1038888],
+         'max_rate': [0.10481985, 0.10388887000000001],
          'is_open': [False, False],
          'enter_tag': ['', ''],
          'leverage': [1.0, 1.0],
          'is_short': [False, False],
          'open_timestamp': [1517251200000, 1517283000000],
-         'close_timestamp': [1517265300000, 1517285400000],
+         'close_timestamp': [1517263200000, 1517285400000],
          })
-    pd.testing.assert_frame_equal(results.drop(columns=['orders']), expected)
+    results_no = results.drop(columns=['orders'])
+    pd.testing.assert_frame_equal(results_no, expected, check_exact=True)
+
     data_pair = processed[pair]
     assert len(results.iloc[0]['orders']) == 6
     assert len(results.iloc[1]['orders']) == 2

tests/optimize/test_optimize_reports.py

@@ -498,7 +498,7 @@ def test__get_resample_from_period():
 
     assert _get_resample_from_period('day') == '1d'
     assert _get_resample_from_period('week') == '1W-MON'
-    assert _get_resample_from_period('month') == '1M'
+    assert _get_resample_from_period('month') == '1ME'
     with pytest.raises(ValueError, match=r"Period noooo is not supported."):
         _get_resample_from_period('noooo')
 

tests/strategy/test_interface.py

@@ -1022,22 +1022,22 @@ def test_auto_hyperopt_interface_loadparams(default_conf, mocker, caplog):
 
 
 @pytest.mark.parametrize('function,raises', [
-    ('populate_entry_trend', True),
+    ('populate_entry_trend', False),
     ('advise_entry', False),
-    ('populate_exit_trend', True),
+    ('populate_exit_trend', False),
     ('advise_exit', False),
 ])
-def test_pandas_warning_direct(ohlcv_history, function, raises):
+def test_pandas_warning_direct(ohlcv_history, function, raises, recwarn):
 
     df = _STRATEGY.populate_indicators(ohlcv_history, {'pair': 'ETH/BTC'})
     if raises:
-        with pytest.warns(FutureWarning):
+        assert len(recwarn) == 1
-            # Test for Future warning
-            # FutureWarning: Setting an item of incompatible dtype is
-            # deprecated and will raise in a future error of pandas
         # https://github.com/pandas-dev/pandas/issues/56503
+        # Fixed in 2.2.x
         getattr(_STRATEGY, function)(df, {'pair': 'ETH/BTC'})
     else:
+        assert len(recwarn) == 0
+
         getattr(_STRATEGY, function)(df, {'pair': 'ETH/BTC'})