Merge pull request #10143 from freqtrade/feat/pairhistory_advanced

RPC: Advanced pairhistory endpoint

freqtrade/rpc/api_server/api_schemas.py

@@ -489,12 +489,26 @@ class AvailablePairs(BaseModel):
     pair_interval: List[List[str]]
 
 
+class PairCandlesRequest(BaseModel):
+    pair: str
+    timeframe: str
+    limit: Optional[int] = None
+    columns: Optional[List[str]] = None
+
+
+class PairHistoryRequest(PairCandlesRequest):
+    timerange: str
+    strategy: str
+    freqaimodel: Optional[str] = None
+
+
 class PairHistory(BaseModel):
     strategy: str
     pair: str
     timeframe: str
     timeframe_ms: int
     columns: List[str]
+    all_columns: List[str] = []
     data: SerializeAsAny[List[Any]]
     length: int
     buy_signals: int

freqtrade/rpc/api_server/api_v1.py

@@ -17,10 +17,11 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac
                                                   ForceEnterResponse, ForceExitPayload,
                                                   FreqAIModelListResponse, Health, Locks,
                                                   LocksPayload, Logs, MixTag, OpenTradeSchema,
-                                                  PairHistory, PerformanceEntry, Ping, PlotConfig,
+                                                  PairCandlesRequest, PairHistory,
-                                                  Profit, ResultMsg, ShowConfig, Stats, StatusMsg,
+                                                  PairHistoryRequest, PerformanceEntry, Ping,
-                                                  StrategyListResponse, StrategyResponse, SysInfo,
+                                                  PlotConfig, Profit, ResultMsg, ShowConfig, Stats,
-                                                  Version, WhitelistResponse)
+                                                  StatusMsg, StrategyListResponse, StrategyResponse,
+                                                  SysInfo, Version, WhitelistResponse)
 from freqtrade.rpc.api_server.deps import get_config, get_exchange, get_rpc, get_rpc_optional
 from freqtrade.rpc.rpc import RPCException
 
@@ -53,7 +54,8 @@ logger = logging.getLogger(__name__)
 # 2.32: new /backtest/history/ patch endpoint
 # 2.33: Additional weekly/monthly metrics
 # 2.34: new entries/exits/mix_tags endpoints
-API_VERSION = 2.34
+# 2.35: pair_candles and pair_history endpoints as Post variant
+API_VERSION = 2.35
 
 # Public API, requires no auth.
 router_public = APIRouter()
@@ -291,7 +293,14 @@ def reload_config(rpc: RPC = Depends(get_rpc)):
 @router.get('/pair_candles', response_model=PairHistory, tags=['candle data'])
 def pair_candles(
         pair: str, timeframe: str, limit: Optional[int] = None, rpc: RPC = Depends(get_rpc)):
-    return rpc._rpc_analysed_dataframe(pair, timeframe, limit)
+    return rpc._rpc_analysed_dataframe(pair, timeframe, limit, None)
+
+
+@router.post('/pair_candles', response_model=PairHistory, tags=['candle data'])
+def pair_candles_filtered(payload: PairCandlesRequest, rpc: RPC = Depends(get_rpc)):
+    # Advanced pair_candles endpoint with column filtering
+    return rpc._rpc_analysed_dataframe(
+        payload.pair, payload.timeframe, payload.limit, payload.columns)
 
 
 @router.get('/pair_history', response_model=PairHistory, tags=['candle data'])
@@ -307,7 +316,25 @@ def pair_history(pair: str, timeframe: str, timerange: str, strategy: str,
         'freqaimodel': freqaimodel if freqaimodel else config.get('freqaimodel'),
     })
     try:
-        return RPC._rpc_analysed_history_full(config, pair, timeframe, exchange)
+        return RPC._rpc_analysed_history_full(config, pair, timeframe, exchange, None)
+    except Exception as e:
+        raise HTTPException(status_code=502, detail=str(e))
+
+
+@router.post('/pair_history', response_model=PairHistory, tags=['candle data'])
+def pair_history_filtered(payload: PairHistoryRequest,
+                          config=Depends(get_config), exchange=Depends(get_exchange)):
+    # The initial call to this endpoint can be slow, as it may need to initialize
+    # the exchange class.
+    config = deepcopy(config)
+    config.update({
+        'strategy': payload.strategy,
+        'timerange': payload.timerange,
+        'freqaimodel': payload.freqaimodel if payload.freqaimodel else config.get('freqaimodel'),
+    })
+    try:
+        return RPC._rpc_analysed_history_full(
+            config, payload.pair, payload.timeframe, exchange, payload.columns)
     except Exception as e:
         raise HTTPException(status_code=502, detail=str(e))
 

freqtrade/rpc/rpc.py

@@ -16,7 +16,7 @@ from sqlalchemy import func, select
 
 from freqtrade import __version__
 from freqtrade.configuration.timerange import TimeRange
-from freqtrade.constants import CANCEL_REASON, Config
+from freqtrade.constants import CANCEL_REASON, DEFAULT_DATAFRAME_COLUMNS, Config
 from freqtrade.data.history import load_data
 from freqtrade.data.metrics import calculate_expectancy, calculate_max_drawdown
 from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, MarketDirection, SignalDirection,
@@ -1190,9 +1190,11 @@ class RPC:
         return self._freqtrade.edge.accepted_pairs()
 
     @staticmethod
-    def _convert_dataframe_to_dict(strategy: str, pair: str, timeframe: str, dataframe: DataFrame,
+    def _convert_dataframe_to_dict(
-                                   last_analyzed: datetime) -> Dict[str, Any]:
+            strategy: str, pair: str, timeframe: str, dataframe: DataFrame,
+            last_analyzed: datetime, selected_cols: Optional[List[str]]) -> Dict[str, Any]:
         has_content = len(dataframe) != 0
+        dataframe_columns = list(dataframe.columns)
         signals = {
             'enter_long': 0,
             'exit_long': 0,
@@ -1200,6 +1202,11 @@ class RPC:
             'exit_short': 0,
         }
         if has_content:
+            if selected_cols is not None:
+                # Ensure OHLCV columns are always present
+                cols_set = set(DEFAULT_DATAFRAME_COLUMNS + list(signals.keys()) + selected_cols)
+                df_cols = [col for col in dataframe_columns if col in cols_set]
+                dataframe = dataframe.loc[:, df_cols]
 
             dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].astype(int64) // 1000 // 1000
             # Move signal close to separate column when signal for easy plotting
@@ -1224,6 +1231,7 @@ class RPC:
             'timeframe': timeframe,
             'timeframe_ms': timeframe_to_msecs(timeframe),
             'strategy': strategy,
+            'all_columns': dataframe_columns,
             'columns': list(dataframe.columns),
             'data': dataframe.values.tolist(),
             'length': len(dataframe),
@@ -1249,13 +1257,16 @@ class RPC:
             })
         return res
 
-    def _rpc_analysed_dataframe(self, pair: str, timeframe: str,
+    def _rpc_analysed_dataframe(
-                                limit: Optional[int]) -> Dict[str, Any]:
+            self, pair: str, timeframe: str, limit: Optional[int],
+            selected_cols: Optional[List[str]]) -> Dict[str, Any]:
         """ Analyzed dataframe in Dict form """
 
         _data, last_analyzed = self.__rpc_analysed_dataframe_raw(pair, timeframe, limit)
-        return RPC._convert_dataframe_to_dict(self._freqtrade.config['strategy'],
+        return RPC._convert_dataframe_to_dict(
-                                              pair, timeframe, _data, last_analyzed)
+            self._freqtrade.config['strategy'], pair, timeframe, _data, last_analyzed,
+            selected_cols
+        )
 
     def __rpc_analysed_dataframe_raw(
         self,
@@ -1322,7 +1333,7 @@ class RPC:
 
     @staticmethod
     def _rpc_analysed_history_full(config: Config, pair: str, timeframe: str,
-                                   exchange) -> Dict[str, Any]:
+                                   exchange, selected_cols: Optional[List[str]]) -> Dict[str, Any]:
         timerange_parsed = TimeRange.parse_timerange(config.get('timerange'))
 
         from freqtrade.data.converter import trim_dataframe
@@ -1352,7 +1363,8 @@ class RPC:
         df_analyzed = trim_dataframe(df_analyzed, timerange_parsed, startup_candles=startup_candles)
 
         return RPC._convert_dataframe_to_dict(strategy.get_strategy_name(), pair, timeframe,
-                                              df_analyzed.copy(), dt_now())
+                                              df_analyzed.copy(), dt_now(),
+                                              selected_cols)
 
     def _rpc_plot_config(self) -> Dict[str, Any]:
         if (self._freqtrade.strategy.plot_config and

tests/rpc/test_rpc_apiserver.py

@@ -1511,6 +1511,7 @@ def test_api_pair_candles(botclient, ohlcv_history):
     assert 'data_stop_ts' in rc.json()
     assert len(rc.json()['data']) == 0
     ohlcv_history['sma'] = ohlcv_history['close'].rolling(2).mean()
+    ohlcv_history['sma2'] = ohlcv_history['close'].rolling(2).mean()
     ohlcv_history['enter_long'] = 0
     ohlcv_history.loc[1, 'enter_long'] = 1
     ohlcv_history['exit_long'] = 0
@@ -1518,44 +1519,83 @@ def test_api_pair_candles(botclient, ohlcv_history):
     ohlcv_history['exit_short'] = 0
 
     ftbot.dataprovider._set_cached_df("XRP/BTC", timeframe, ohlcv_history, CandleType.SPOT)
+    for call in ('get', 'post'):
+        if call == 'get':
+            rc = client_get(
+                client,
+                f"{BASE_URI}/pair_candles?limit={amount}&pair=XRP%2FBTC&timeframe={timeframe}")
+        else:
+            rc = client_post(
+                client,
+                f"{BASE_URI}/pair_candles",
+                data={
+                    "pair": "XRP/BTC",
+                    "timeframe": timeframe,
+                    "limit": amount,
+                    "columns": ['sma'],
+                }
+            )
+        assert_response(rc)
+        resp = rc.json()
+        assert 'strategy' in resp
+        assert resp['strategy'] == CURRENT_TEST_STRATEGY
+        assert 'columns' in resp
+        assert 'data_start_ts' in resp
+        assert 'data_start' in resp
+        assert 'data_stop' in resp
+        assert 'data_stop_ts' in resp
+        assert resp['data_start'] == '2017-11-26 08:50:00+00:00'
+        assert resp['data_start_ts'] == 1511686200000
+        assert resp['data_stop'] == '2017-11-26 09:00:00+00:00'
+        assert resp['data_stop_ts'] == 1511686800000
+        assert isinstance(resp['columns'], list)
+        base_cols = {
+                'date', 'open', 'high', 'low', 'close', 'volume',
+                'sma', 'enter_long', 'exit_long', 'enter_short', 'exit_short', '__date_ts',
+                '_enter_long_signal_close', '_exit_long_signal_close',
+                '_enter_short_signal_close', '_exit_short_signal_close'
+            }
+        if call == 'get':
+            assert set(resp['columns']) == base_cols.union({'sma2'})
+        else:
+            assert set(resp['columns']) == base_cols
 
-    rc = client_get(client,
+        # All columns doesn't include the internal columns
-                    f"{BASE_URI}/pair_candles?limit={amount}&pair=XRP%2FBTC&timeframe={timeframe}")
+        assert set(resp['all_columns']) == {
-    assert_response(rc)
+            'date', 'open', 'high', 'low', 'close', 'volume',
-    assert 'strategy' in rc.json()
+            'sma', 'sma2', 'enter_long', 'exit_long', 'enter_short', 'exit_short'
-    assert rc.json()['strategy'] == CURRENT_TEST_STRATEGY
+        }
-    assert 'columns' in rc.json()
+        assert 'pair' in resp
-    assert 'data_start_ts' in rc.json()
+        assert resp['pair'] == 'XRP/BTC'
-    assert 'data_start' in rc.json()
-    assert 'data_stop' in rc.json()
-    assert 'data_stop_ts' in rc.json()
-    assert rc.json()['data_start'] == '2017-11-26 08:50:00+00:00'
-    assert rc.json()['data_start_ts'] == 1511686200000
-    assert rc.json()['data_stop'] == '2017-11-26 09:00:00+00:00'
-    assert rc.json()['data_stop_ts'] == 1511686800000
-    assert isinstance(rc.json()['columns'], list)
-    assert set(rc.json()['columns']) == {
-        'date', 'open', 'high', 'low', 'close', 'volume',
-        'sma', 'enter_long', 'exit_long', 'enter_short', 'exit_short', '__date_ts',
-        '_enter_long_signal_close', '_exit_long_signal_close',
-        '_enter_short_signal_close', '_exit_short_signal_close'
-    }
-    assert 'pair' in rc.json()
-    assert rc.json()['pair'] == 'XRP/BTC'
 
-    assert 'data' in rc.json()
+        assert 'data' in resp
-    assert len(rc.json()['data']) == amount
+        assert len(resp['data']) == amount
+        if call == 'get':
+            assert len(resp['data'][0]) == 17
+            assert resp['data'] == [
+                ['2017-11-26T08:50:00Z', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05,
+                 0.0877869, None, None, 0, 0, 0, 0, 1511686200000, None, None, None, None],
+                ['2017-11-26T08:55:00Z', 8.88e-05, 8.942e-05, 8.88e-05, 8.893e-05, 0.05874751,
+                 8.886500000000001e-05, 8.886500000000001e-05, 1, 0, 0, 0, 1511686500000,
+                 8.893e-05, None, None, None],
+                ['2017-11-26T09:00:00Z', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05,
+                 0.7039405, 8.885e-05, 8.885e-05, 0, 0, 0, 0, 1511686800000, None, None, None, None
+                 ]
+            ]
+        else:
+            assert len(resp['data'][0]) == 16
+            assert resp['data'] == [
+                ['2017-11-26T08:50:00Z', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05,
+                 0.0877869, None, 0, 0, 0, 0, 1511686200000, None, None, None, None],
+                ['2017-11-26T08:55:00Z', 8.88e-05, 8.942e-05, 8.88e-05, 8.893e-05, 0.05874751,
+                 8.886500000000001e-05, 1, 0, 0, 0, 1511686500000,
+                 8.893e-05, None, None, None],
+                ['2017-11-26T09:00:00Z', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05,
+                 0.7039405, 8.885e-05, 0, 0, 0, 0, 1511686800000, None, None, None, None
+                 ]
+            ]
 
-    assert (rc.json()['data'] ==
+    # prep for next test
-            [['2017-11-26T08:50:00Z', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, 0.0877869,
-              None, 0, 0, 0, 0, 1511686200000, None, None, None, None],
-             ['2017-11-26T08:55:00Z', 8.88e-05, 8.942e-05, 8.88e-05,
-                 8.893e-05, 0.05874751, 8.886500000000001e-05, 1, 0, 0, 0, 1511686500000, 8.893e-05,
-                 None, None, None],
-             ['2017-11-26T09:00:00Z', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05,
-                 0.7039405, 8.885e-05, 0, 0, 0, 0, 1511686800000, None, None, None, None]
-
-             ])
     ohlcv_history['exit_long'] = ohlcv_history['exit_long'].astype('float64')
     ohlcv_history.at[0, 'exit_long'] = float('inf')
     ohlcv_history['date1'] = ohlcv_history['date']
@@ -1567,13 +1607,13 @@ def test_api_pair_candles(botclient, ohlcv_history):
     assert_response(rc)
     assert (rc.json()['data'] ==
             [['2017-11-26T08:50:00Z', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, 0.0877869,
-              None, 0, None, 0, 0, None, 1511686200000, None, None, None, None],
+              None, None, 0, None, 0, 0, None, 1511686200000, None, None, None, None],
              ['2017-11-26T08:55:00Z', 8.88e-05, 8.942e-05, 8.88e-05,
-                 8.893e-05, 0.05874751, 8.886500000000001e-05, 1, 0.0, 0, 0, '2017-11-26T08:55:00Z',
+                 8.893e-05, 0.05874751, 8.886500000000001e-05, 8.886500000000001e-05, 1, 0.0, 0,
-                 1511686500000, 8.893e-05, None, None, None],
+                 0, '2017-11-26T08:55:00Z', 1511686500000, 8.893e-05, None, None, None],
              ['2017-11-26T09:00:00Z', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05,
-                 0.7039405, 8.885e-05, 0, 0.0, 0, 0, '2017-11-26T09:00:00Z', 1511686800000,
+                 0.7039405, 8.885e-05, 8.885e-05, 0, 0.0, 0, 0, '2017-11-26T09:00:00Z',
-                 None, None, None, None]
+                 1511686800000, None, None, None, None]
              ])
 
 
@@ -1614,40 +1654,69 @@ def test_api_pair_history(botclient, tmp_path, mocker):
     assert_response(rc, 502)
 
     # Working
-    rc = client_get(client,
+    for call in ('get', 'post'):
-                    f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC&timeframe={timeframe}"
+        if call == 'get':
-                    f"&timerange=20180111-20180112&strategy={CURRENT_TEST_STRATEGY}")
+            rc = client_get(client,
-    assert_response(rc, 200)
+                            f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC&timeframe={timeframe}"
-    result = rc.json()
+                            f"&timerange=20180111-20180112&strategy={CURRENT_TEST_STRATEGY}")
-    assert result['length'] == 289
+        else:
-    assert len(result['data']) == result['length']
+            rc = client_post(
-    assert 'columns' in result
+                client,
-    assert 'data' in result
+                f"{BASE_URI}/pair_history",
-    data = result['data']
+                data={
-    assert len(data) == 289
+                    "pair": "UNITTEST/BTC",
-    # analyzed DF has 30 columns
+                    "timeframe": timeframe,
-    assert len(result['columns']) == 30
+                    "timerange": "20180111-20180112",
-    assert len(data[0]) == 30
+                    "strategy": CURRENT_TEST_STRATEGY,
-    date_col_idx = [idx for idx, c in enumerate(result['columns']) if c == 'date'][0]
+                    "columns": ['rsi', 'fastd', 'fastk'],
-    rsi_col_idx = [idx for idx, c in enumerate(result['columns']) if c == 'rsi'][0]
+                })
 
-    assert data[0][date_col_idx] == '2018-01-11T00:00:00Z'
+        assert_response(rc, 200)
-    assert data[0][rsi_col_idx] is not None
+        result = rc.json()
-    assert data[0][rsi_col_idx] > 0
+        assert result['length'] == 289
-    assert lfm.call_count == 1
+        assert len(result['data']) == result['length']
-    assert result['pair'] == 'UNITTEST/BTC'
+        assert 'columns' in result
-    assert result['strategy'] == CURRENT_TEST_STRATEGY
+        assert 'data' in result
-    assert result['data_start'] == '2018-01-11 00:00:00+00:00'
+        data = result['data']
-    assert result['data_start_ts'] == 1515628800000
+        assert len(data) == 289
-    assert result['data_stop'] == '2018-01-12 00:00:00+00:00'
+        col_count = 30 if call == 'get' else 18
-    assert result['data_stop_ts'] == 1515715200000
+        # analyzed DF has 30 columns
+        assert len(result['columns']) == col_count
+        assert len(result['all_columns']) == 25
+        assert len(data[0]) == col_count
+        date_col_idx = [idx for idx, c in enumerate(result['columns']) if c == 'date'][0]
+        rsi_col_idx = [idx for idx, c in enumerate(result['columns']) if c == 'rsi'][0]
 
-    # No data found
+        assert data[0][date_col_idx] == '2018-01-11T00:00:00Z'
-    rc = client_get(client,
+        assert data[0][rsi_col_idx] is not None
-                    f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC&timeframe={timeframe}"
+        assert data[0][rsi_col_idx] > 0
-                    f"&timerange=20200111-20200112&strategy={CURRENT_TEST_STRATEGY}")
+        assert lfm.call_count == 1
-    assert_response(rc, 502)
+        assert result['pair'] == 'UNITTEST/BTC'
-    assert rc.json()['detail'] == ("No data for UNITTEST/BTC, 5m in 20200111-20200112 found.")
+        assert result['strategy'] == CURRENT_TEST_STRATEGY
+        assert result['data_start'] == '2018-01-11 00:00:00+00:00'
+        assert result['data_start_ts'] == 1515628800000
+        assert result['data_stop'] == '2018-01-12 00:00:00+00:00'
+        assert result['data_stop_ts'] == 1515715200000
+        lfm.reset_mock()
+
+        # No data found
+        if call == 'get':
+            rc = client_get(client,
+                            f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC&timeframe={timeframe}"
+                            f"&timerange=20200111-20200112&strategy={CURRENT_TEST_STRATEGY}")
+        else:
+            rc = client_post(
+                client,
+                f"{BASE_URI}/pair_history",
+                data={
+                    "pair": "UNITTEST/BTC",
+                    "timeframe": timeframe,
+                    "timerange": "20200111-20200112",
+                    "strategy": CURRENT_TEST_STRATEGY,
+                    "columns": ['rsi', 'fastd', 'fastk'],
+                })
+        assert_response(rc, 502)
+        assert rc.json()['detail'] == ("No data for UNITTEST/BTC, 5m in 20200111-20200112 found.")
 
 
 def test_api_plot_config(botclient, mocker, tmp_path):