From 31ae2b30606f3c52fb7ee1e3d170e59a8bc06b31 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Tue, 24 May 2022 14:46:16 +0200 Subject: [PATCH] alleviate FutureWarning in sklearn about ensuring svm model features are passed with identical order --- docs/freqai.md | 41 ++++++++++++++++++-- freqtrade/freqai/data_kitchen.py | 8 +++- freqtrade/strategy/interface.py | 2 +- freqtrade/templates/FreqaiExampleStrategy.py | 21 +++++++--- 4 files changed, 61 insertions(+), 11 deletions(-) diff --git a/docs/freqai.md b/docs/freqai.md index 606b88912..27d393d0a 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -105,11 +105,11 @@ config setup includes: ### Building the feature set -Most of these parameters are controlling the feature data set. Features are added by the user -inside the `populate_any_indicators()` method of the strategy by prepending indicators with `%`: +Features are added by the user inside the `populate_any_indicators()` method of the strategy +by prepending indicators with `%`: ```python - def populate_any_indicators(self, pair, df, tf, informative=None, coin=""): + def populate_any_indicators(self, metadata, pair, df, tf, informative=None, coin=""): informative['%-''%-' + coin + "rsi"] = ta.RSI(informative, timeperiod=14) informative['%-' + coin + "mfi"] = ta.MFI(informative, timeperiod=25) informative['%-' + coin + "adx"] = ta.ADX(informative, window=20) @@ -120,11 +120,46 @@ inside the `populate_any_indicators()` method of the strategy by prepending indi informative['%-' + coin + "bb_width"] = ( informative[coin + "bb_upperband"] - informative[coin + "bb_lowerband"] ) / informative[coin + "bb_middleband"] + + + + # The following code automatically adds features according to the `shift` parameter passed + # in the config. Do not remove + indicators = [col for col in informative if col.startswith('%')] + for n in range(self.freqai_info["feature_parameters"]["shift"] + 1): + if n == 0: + continue + informative_shift = informative[indicators].shift(n) + informative_shift = informative_shift.add_suffix("_shift-" + str(n)) + informative = pd.concat((informative, informative_shift), axis=1) + + # The following code safely merges into the base timeframe. + # Do not remove. + df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) + skip_columns = [(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]] + df = df.drop(columns=skip_columns) ``` The user of the present example does not want to pass the `bb_lowerband` as a feature to the model, and has therefore not prepended it with `%`. The user does, however, wish to pass `bb_width` to the model for training/prediction and has therfore prepended it with `%`._ +Note: features **must** be defined in `populate_any_indicators()`. Making features in `populate_indicators()` +will fail in live/dry. If the user wishes to add generalized features that are not associated with +a specific pair or timeframe, they should use the following structure inside `populate_any_indicators()` +(as exemplified in `freqtrade/templates/FreqaiExampleStrategy.py`: + +```python + def populate_any_indicators(self, metadata, pair, df, tf, informative=None, coin=""): + + + # Add generalized indicators here (because in live, it will call only this function to populate + # indicators for retraining). Notice how we ensure not to add them multiple times by associating + # these generalized indicators to the basepair/timeframe + if pair == metadata['pair'] and tf == self.timeframe: + df['%-day_of_week'] = (df["date"].dt.dayofweek + 1) / 7 + df['%-hour_of_day'] = (df['date'].dt.hour + 1) / 25 + + (Please see the example script located in `freqtrade/templates/FreqaiExampleStrategy.py` for a full example of `populate_any_indicators()`) The `timeframes` from the example config above are the timeframes of each `populate_any_indicator()` diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index a4867d7eb..765c58a37 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -823,7 +823,9 @@ class FreqaiDataKitchen: pairs = self.freqai_config.get("corr_pairlist", []) for tf in self.freqai_config.get("timeframes"): - dataframe = strategy.populate_any_indicators(metadata['pair'], + dataframe = strategy.populate_any_indicators( + metadata, + metadata['pair'], dataframe.copy(), tf, base_dataframes[tf], @@ -833,7 +835,9 @@ class FreqaiDataKitchen: for i in pairs: if metadata['pair'] in i: continue # dont repeat anything from whitelist - dataframe = strategy.populate_any_indicators(i, + dataframe = strategy.populate_any_indicators( + metadata, + i, dataframe.copy(), tf, corr_dataframes[i][tf], diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index e681d70bd..6237e3397 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -532,7 +532,7 @@ class IStrategy(ABC, HyperStrategyMixin): """ return None - def populate_any_indicators(self, pair: str, df: DataFrame, tf: str, + def populate_any_indicators(self, metadata: dict, pair: str, df: DataFrame, tf: str, informative: DataFrame = None, coin: str = "") -> DataFrame: """ Function designed to automatically generate, name and merge features diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index 690532c10..d2eb2c306 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -63,7 +63,7 @@ class FreqaiExampleStrategy(IStrategy): def bot_start(self): self.model = CustomModel(self.config) - def populate_any_indicators(self, pair, df, tf, informative=None, coin=""): + def populate_any_indicators(self, metadata, pair, df, tf, informative=None, coin=""): """ Function designed to automatically generate, name and merge features from user indicated timeframes in the configuration file. User controls the indicators @@ -124,8 +124,9 @@ class FreqaiExampleStrategy(IStrategy): informative[coin + "pct-change"] = informative["close"].pct_change() + # The following code automatically adds features according to the `shift` parameter passed + # in the config. Do not remove indicators = [col for col in informative if col.startswith('%')] - for n in range(self.freqai_info["feature_parameters"]["shift"] + 1): if n == 0: continue @@ -133,28 +134,38 @@ class FreqaiExampleStrategy(IStrategy): informative_shift = informative_shift.add_suffix("_shift-" + str(n)) informative = pd.concat((informative, informative_shift), axis=1) + # The following code safely merges into the base timeframe. + # Do not remove. df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) skip_columns = [(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]] df = df.drop(columns=skip_columns) + # Add generalized indicators (not associated to any individual coin or timeframe) here + # because in live, it will call this function to populate + # indicators during training. Notice how we ensure not to add them multiple times + if pair == metadata['pair'] and tf == self.timeframe: + df['%-day_of_week'] = (df["date"].dt.dayofweek + 1) / 7 + df['%-hour_of_day'] = (df['date'].dt.hour + 1) / 25 + return df def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - # the configuration file parameters are stored here self.freqai_info = self.config["freqai"] self.pair = metadata['pair'] # the following loops are necessary for building the features # indicated by the user in the configuration file. + # All indicators must be populated by populate_any_indicators() for live functionality + # to work correctly. for tf in self.freqai_info["timeframes"]: - dataframe = self.populate_any_indicators(self.pair, dataframe.copy(), tf, + dataframe = self.populate_any_indicators(metadata, self.pair, dataframe.copy(), tf, coin=self.pair.split("/")[0] + "-") for pair in self.freqai_info["corr_pairlist"]: if metadata['pair'] in pair: continue # do not include whitelisted pair twice if it is in corr_pairlist dataframe = self.populate_any_indicators( - pair, dataframe.copy(), tf, coin=pair.split("/")[0] + "-" + metadata, pair, dataframe.copy(), tf, coin=pair.split("/")[0] + "-" ) # the model will return 4 values, its prediction, an indication of whether or not the