From 583d70ec9c535d6b2ad6275818096bcb2b9fac49 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 14:18:30 +0200 Subject: [PATCH 01/24] add plot module proto --- freqtrade/plot/__init__.py | 0 freqtrade/plot/plotting.py | 13 +++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 freqtrade/plot/__init__.py create mode 100644 freqtrade/plot/plotting.py diff --git a/freqtrade/plot/__init__.py b/freqtrade/plot/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py new file mode 100644 index 000000000..e04b51726 --- /dev/null +++ b/freqtrade/plot/plotting.py @@ -0,0 +1,13 @@ +import logging + + +logger = logging.getLogger(__name__) + + +try: + from plotly import tools + from plotly.offline import plot + import plotly.graph_objs as go +except ImportError: + logger.exception("Module plotly not found \n Please install using `pip install plotly`") + exit() From 68af6d415157183a33472eb8dfd2070d3db2e3f3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 28 May 2019 07:00:57 +0200 Subject: [PATCH 02/24] Move plot-functions to plotting module --- freqtrade/plot/plotting.py | 177 +++++++++++++++++++++++++++++++++++++ scripts/plot_dataframe.py | 170 ++--------------------------------- 2 files changed, 185 insertions(+), 162 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index e04b51726..7e815bdd7 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -1,5 +1,7 @@ import logging +from typing import List +import pandas as pd logger = logging.getLogger(__name__) @@ -11,3 +13,178 @@ try: except ImportError: logger.exception("Module plotly not found \n Please install using `pip install plotly`") exit() + + +def generate_row(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.make_subplots: + """ + Generator all the indicator selected by the user for a specific row + :param fig: Plot figure to append to + :param row: row number for this plot + :param indicators: List of indicators present in the dataframe + :param data: candlestick DataFrame + """ + for indicator in indicators: + if indicator in data: + # TODO: Replace all Scatter with Scattergl for performance!! + scattergl = go.Scatter( + x=data['date'], + y=data[indicator], + mode='lines', + name=indicator + ) + fig.append_trace(scattergl, row, 1) + else: + logger.info( + 'Indicator "%s" ignored. Reason: This indicator is not found ' + 'in your strategy.', + indicator + ) + + return fig + + +def plot_trades(fig, trades: pd.DataFrame): + """ + Plot trades to "fig" + """ + # Trades can be empty + if trades is not None: + trade_buys = go.Scatter( + x=trades["open_time"], + y=trades["open_rate"], + mode='markers', + name='trade_buy', + marker=dict( + symbol='square-open', + size=11, + line=dict(width=2), + color='green' + ) + ) + trade_sells = go.Scatter( + x=trades["close_time"], + y=trades["close_rate"], + mode='markers', + name='trade_sell', + marker=dict( + symbol='square-open', + size=11, + line=dict(width=2), + color='red' + ) + ) + fig.append_trace(trade_buys, 1, 1) + fig.append_trace(trade_sells, 1, 1) + return fig + + +def generate_graph( + pair: str, + data: pd.DataFrame, + trades: pd.DataFrame = None, + indicators1: List[str] = [], + indicators2: List[str] = [], +) -> tools.make_subplots: + """ + Generate the graph from the data generated by Backtesting or from DB + Volume will always be ploted in row2, so Row 1 and are to our disposal for custom indicators + :param pair: Pair to Display on the graph + :param data: OHLCV DataFrame containing indicators and buy/sell signals + :param trades: All trades created + :param indicators1: List containing Main plot indicators + :param indicators2: List containing Sub plot indicators + :return: None + """ + + # Define the graph + fig = tools.make_subplots( + rows=3, + cols=1, + shared_xaxes=True, + row_width=[1, 1, 4], + vertical_spacing=0.0001, + ) + fig['layout'].update(title=pair) + fig['layout']['yaxis1'].update(title='Price') + fig['layout']['yaxis2'].update(title='Volume') + fig['layout']['yaxis3'].update(title='Other') + fig['layout']['xaxis']['rangeslider'].update(visible=False) + + # Common information + candles = go.Candlestick( + x=data.date, + open=data.open, + high=data.high, + low=data.low, + close=data.close, + name='Price' + ) + fig.append_trace(candles, 1, 1) + + if 'buy' in data.columns: + df_buy = data[data['buy'] == 1] + buys = go.Scatter( + x=df_buy.date, + y=df_buy.close, + mode='markers', + name='buy', + marker=dict( + symbol='triangle-up-dot', + size=9, + line=dict(width=1), + color='green', + ) + ) + fig.append_trace(buys, 1, 1) + + if 'sell' in data.columns: + df_sell = data[data['sell'] == 1] + sells = go.Scatter( + x=df_sell.date, + y=df_sell.close, + mode='markers', + name='sell', + marker=dict( + symbol='triangle-down-dot', + size=9, + line=dict(width=1), + color='red', + ) + ) + fig.append_trace(sells, 1, 1) + + if 'bb_lowerband' in data and 'bb_upperband' in data: + bb_lower = go.Scatter( + x=data.date, + y=data.bb_lowerband, + name='BB lower', + line={'color': 'rgba(255,255,255,0)'}, + ) + bb_upper = go.Scatter( + x=data.date, + y=data.bb_upperband, + name='BB upper', + fill="tonexty", + fillcolor="rgba(0,176,246,0.2)", + line={'color': 'rgba(255,255,255,0)'}, + ) + fig.append_trace(bb_lower, 1, 1) + fig.append_trace(bb_upper, 1, 1) + + # Add indicators to main plot + fig = generate_row(fig=fig, row=1, indicators=indicators1, data=data) + + fig = plot_trades(fig, trades) + + # Volume goes to row 2 + volume = go.Bar( + x=data['date'], + y=data['volume'], + name='Volume' + ) + fig.append_trace(volume, 2, 1) + + # Add indicators to seperate row + fig = generate_row(fig=fig, row=3, indicators=indicators2, data=data) + + return fig diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 4f8ffb32b..897b0c917 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -31,15 +31,14 @@ from pathlib import Path from typing import Any, Dict, List import pandas as pd -import plotly.graph_objs as go import pytz -from plotly import tools from plotly.offline import plot from freqtrade import persistence from freqtrade.arguments import Arguments, TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data +from freqtrade.plot.plotting import generate_graph from freqtrade.exchange import Exchange from freqtrade.optimize import setup_configuration from freqtrade.persistence import Trade @@ -96,7 +95,10 @@ def generate_plot_file(fig, pair, ticker_interval, is_last) -> None: Path("user_data/plots").mkdir(parents=True, exist_ok=True) - plot(fig, filename=str(Path('user_data/plots').joinpath(file_name)), auto_open=False) + plot(fig, filename=str(Path('user_data/plots').joinpath(file_name)), + auto_open=False, + include_plotlyjs='https://cdn.plot.ly/plotly-1.47.4.min.js' + ) if is_last: plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html')), auto_open=False) @@ -186,162 +188,6 @@ def extract_trades_of_period(dataframe, trades) -> pd.DataFrame: return trades -def generate_graph( - pair: str, - trades: pd.DataFrame, - data: pd.DataFrame, - indicators1: str, - indicators2: str - ) -> tools.make_subplots: - """ - Generate the graph from the data generated by Backtesting or from DB - :param pair: Pair to Display on the graph - :param trades: All trades created - :param data: Dataframe - :indicators1: String Main plot indicators - :indicators2: String Sub plot indicators - :return: None - """ - - # Define the graph - fig = tools.make_subplots( - rows=3, - cols=1, - shared_xaxes=True, - row_width=[1, 1, 4], - vertical_spacing=0.0001, - ) - fig['layout'].update(title=pair) - fig['layout']['yaxis1'].update(title='Price') - fig['layout']['yaxis2'].update(title='Volume') - fig['layout']['yaxis3'].update(title='Other') - fig['layout']['xaxis']['rangeslider'].update(visible=False) - - # Common information - candles = go.Candlestick( - x=data.date, - open=data.open, - high=data.high, - low=data.low, - close=data.close, - name='Price' - ) - - df_buy = data[data['buy'] == 1] - buys = go.Scattergl( - x=df_buy.date, - y=df_buy.close, - mode='markers', - name='buy', - marker=dict( - symbol='triangle-up-dot', - size=9, - line=dict(width=1), - color='green', - ) - ) - df_sell = data[data['sell'] == 1] - sells = go.Scattergl( - x=df_sell.date, - y=df_sell.close, - mode='markers', - name='sell', - marker=dict( - symbol='triangle-down-dot', - size=9, - line=dict(width=1), - color='red', - ) - ) - - trade_buys = go.Scattergl( - x=trades["open_time"], - y=trades["open_rate"], - mode='markers', - name='trade_buy', - marker=dict( - symbol='square-open', - size=11, - line=dict(width=2), - color='green' - ) - ) - trade_sells = go.Scattergl( - x=trades["close_time"], - y=trades["close_rate"], - mode='markers', - name='trade_sell', - marker=dict( - symbol='square-open', - size=11, - line=dict(width=2), - color='red' - ) - ) - - # Row 1 - fig.append_trace(candles, 1, 1) - - if 'bb_lowerband' in data and 'bb_upperband' in data: - bb_lower = go.Scatter( - x=data.date, - y=data.bb_lowerband, - name='BB lower', - line={'color': 'rgba(255,255,255,0)'}, - ) - bb_upper = go.Scatter( - x=data.date, - y=data.bb_upperband, - name='BB upper', - fill="tonexty", - fillcolor="rgba(0,176,246,0.2)", - line={'color': 'rgba(255,255,255,0)'}, - ) - fig.append_trace(bb_lower, 1, 1) - fig.append_trace(bb_upper, 1, 1) - - fig = generate_row(fig=fig, row=1, raw_indicators=indicators1, data=data) - fig.append_trace(buys, 1, 1) - fig.append_trace(sells, 1, 1) - fig.append_trace(trade_buys, 1, 1) - fig.append_trace(trade_sells, 1, 1) - - # Row 2 - volume = go.Bar( - x=data['date'], - y=data['volume'], - name='Volume' - ) - fig.append_trace(volume, 2, 1) - - # Row 3 - fig = generate_row(fig=fig, row=3, raw_indicators=indicators2, data=data) - - return fig - - -def generate_row(fig, row, raw_indicators, data) -> tools.make_subplots: - """ - Generator all the indicator selected by the user for a specific row - """ - for indicator in raw_indicators.split(','): - if indicator in data: - scattergl = go.Scattergl( - x=data['date'], - y=data[indicator], - name=indicator - ) - fig.append_trace(scattergl, row, 1) - else: - logger.info( - 'Indicator "%s" ignored. Reason: This indicator is not found ' - 'in your strategy.', - indicator - ) - - return fig - - def plot_parse_args(args: List[str]) -> Namespace: """ Parse args passed to the script @@ -411,10 +257,10 @@ def analyse_and_plot_pairs(args: Namespace): fig = generate_graph( pair=pair, - trades=trades, data=dataframe, - indicators1=args.indicators1, - indicators2=args.indicators2 + trades=trades, + indicators1=args.indicators1.split(","), + indicators2=args.indicators2.split(",") ) is_last = (False, True)[pair_counter == len(tickers)] From 6df0b39f8151c31b89d4acbf06a9654fa6c56976 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 28 May 2019 20:02:17 +0200 Subject: [PATCH 03/24] Cleanup plot_dataframe a bit --- scripts/plot_dataframe.py | 99 +++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 51 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 897b0c917..1d3b24449 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -51,7 +51,7 @@ _CONF: Dict[str, Any] = {} timeZone = pytz.UTC -def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFrame: +def load_trades(args: Namespace, pair: str) -> pd.DataFrame: trades: pd.DataFrame = pd.DataFrame() if args.db_url: persistence.init(args.db_url, clean_open_orders=False) @@ -96,9 +96,7 @@ def generate_plot_file(fig, pair, ticker_interval, is_last) -> None: Path("user_data/plots").mkdir(parents=True, exist_ok=True) plot(fig, filename=str(Path('user_data/plots').joinpath(file_name)), - auto_open=False, - include_plotlyjs='https://cdn.plot.ly/plotly-1.47.4.min.js' - ) + auto_open=False) if is_last: plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html')), auto_open=False) @@ -133,14 +131,13 @@ def get_trading_env(args: Namespace): return [strategy, exchange, pairs] -def get_tickers_data(strategy, exchange, pairs: List[str], args): +def get_tickers_data(strategy, exchange, pairs: List[str], timerange: TimeRange, live: bool): """ Get tickers data for each pairs on live or local, option defined in args - :return: dictinnary of tickers. output format: {'pair': tickersdata} + :return: dictionary of tickers. output format: {'pair': tickersdata} """ ticker_interval = strategy.ticker_interval - timerange = Arguments.parse_timerange(args.timerange) tickers = history.load_data( datadir=Path(str(_CONF.get("datadir"))), @@ -184,10 +181,53 @@ def extract_trades_of_period(dataframe, trades) -> pd.DataFrame: Compare trades and backtested pair DataFrames to get trades performed on backtested period :return: the DataFrame of a trades of period """ - trades = trades.loc[trades['open_time'] >= dataframe.iloc[0]['date']] + trades = trades.loc[(trades['open_time'] >= dataframe.iloc[0]['date']) & + (trades['close_time'] <= dataframe.iloc[-1]['date'])] return trades +def analyse_and_plot_pairs(args: Namespace): + """ + From arguments provided in cli: + -Initialise backtest env + -Get tickers data + -Generate Dafaframes populated with indicators and signals + -Load trades excecuted on same periods + -Generate Plotly plot objects + -Generate plot files + :return: None + """ + strategy, exchange, pairs = get_trading_env(args) + # Set timerange to use + timerange = Arguments.parse_timerange(args.timerange) + ticker_interval = strategy.ticker_interval + + tickers = get_tickers_data(strategy, exchange, pairs, timerange, args.live) + pair_counter = 0 + for pair, data in tickers.items(): + pair_counter += 1 + logger.info("analyse pair %s", pair) + tickers = {} + tickers[pair] = data + dataframe = generate_dataframe(strategy, tickers, pair) + + trades = load_trades(args, pair) + trades = extract_trades_of_period(dataframe, trades) + + fig = generate_graph( + pair=pair, + data=dataframe, + trades=trades, + indicators1=args.indicators1.split(","), + indicators2=args.indicators2.split(",") + ) + + is_last = (False, True)[pair_counter == len(tickers)] + generate_plot_file(fig, pair, ticker_interval, is_last) + + logger.info('End of ploting process %s plots generated', pair_counter) + + def plot_parse_args(args: List[str]) -> Namespace: """ Parse args passed to the script @@ -226,49 +266,6 @@ def plot_parse_args(args: List[str]) -> Namespace: arguments.backtesting_options(arguments.parser) return arguments.parse_args() - -def analyse_and_plot_pairs(args: Namespace): - """ - From arguments provided in cli: - -Initialise backtest env - -Get tickers data - -Generate Dafaframes populated with indicators and signals - -Load trades excecuted on same periods - -Generate Plotly plot objects - -Generate plot files - :return: None - """ - strategy, exchange, pairs = get_trading_env(args) - # Set timerange to use - timerange = Arguments.parse_timerange(args.timerange) - ticker_interval = strategy.ticker_interval - - tickers = get_tickers_data(strategy, exchange, pairs, args) - pair_counter = 0 - for pair, data in tickers.items(): - pair_counter += 1 - logger.info("analyse pair %s", pair) - tickers = {} - tickers[pair] = data - dataframe = generate_dataframe(strategy, tickers, pair) - - trades = load_trades(args, pair, timerange) - trades = extract_trades_of_period(dataframe, trades) - - fig = generate_graph( - pair=pair, - data=dataframe, - trades=trades, - indicators1=args.indicators1.split(","), - indicators2=args.indicators2.split(",") - ) - - is_last = (False, True)[pair_counter == len(tickers)] - generate_plot_file(fig, pair, ticker_interval, is_last) - - logger.info('End of ploting process %s plots generated', pair_counter) - - def main(sysargv: List[str]) -> None: """ This function will initiate the bot and start the trading loop. From e0a1e5417fbb8e36bc947e29398f05754b565959 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 28 May 2019 20:23:16 +0200 Subject: [PATCH 04/24] sanity checks before plotting, cleanup --- freqtrade/plot/plotting.py | 63 ++++++++++++++++++++------------------ scripts/plot_dataframe.py | 29 +++++++++++------- 2 files changed, 52 insertions(+), 40 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 7e815bdd7..a18b7bf70 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -25,8 +25,7 @@ def generate_row(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.m """ for indicator in indicators: if indicator in data: - # TODO: Replace all Scatter with Scattergl for performance!! - scattergl = go.Scatter( + scattergl = go.Scattergl( x=data['date'], y=data[indicator], mode='lines', @@ -48,7 +47,7 @@ def plot_trades(fig, trades: pd.DataFrame): Plot trades to "fig" """ # Trades can be empty - if trades is not None: + if trades is not None and len(trades) > 0: trade_buys = go.Scatter( x=trades["open_time"], y=trades["open_rate"], @@ -123,44 +122,50 @@ def generate_graph( if 'buy' in data.columns: df_buy = data[data['buy'] == 1] - buys = go.Scatter( - x=df_buy.date, - y=df_buy.close, - mode='markers', - name='buy', - marker=dict( - symbol='triangle-up-dot', - size=9, - line=dict(width=1), - color='green', + if len(df_buy) > 0: + buys = go.Scattergl( + x=df_buy.date, + y=df_buy.close, + mode='markers', + name='buy', + marker=dict( + symbol='triangle-up-dot', + size=9, + line=dict(width=1), + color='green', + ) ) - ) - fig.append_trace(buys, 1, 1) + fig.append_trace(buys, 1, 1) + else: + logger.warning("No buy-signals found.") if 'sell' in data.columns: df_sell = data[data['sell'] == 1] - sells = go.Scatter( - x=df_sell.date, - y=df_sell.close, - mode='markers', - name='sell', - marker=dict( - symbol='triangle-down-dot', - size=9, - line=dict(width=1), - color='red', + if len(df_sell) > 0: + sells = go.Scattergl( + x=df_sell.date, + y=df_sell.close, + mode='markers', + name='sell', + marker=dict( + symbol='triangle-down-dot', + size=9, + line=dict(width=1), + color='red', + ) ) - ) - fig.append_trace(sells, 1, 1) + fig.append_trace(sells, 1, 1) + else: + logger.warning("No sell-signals found.") if 'bb_lowerband' in data and 'bb_upperband' in data: - bb_lower = go.Scatter( + bb_lower = go.Scattergl( x=data.date, y=data.bb_lowerband, name='BB lower', line={'color': 'rgba(255,255,255,0)'}, ) - bb_upper = go.Scatter( + bb_upper = go.Scattergl( x=data.date, y=data.bb_upperband, name='BB upper', diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 1d3b24449..84e18e5cd 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -51,11 +51,18 @@ _CONF: Dict[str, Any] = {} timeZone = pytz.UTC -def load_trades(args: Namespace, pair: str) -> pd.DataFrame: - trades: pd.DataFrame = pd.DataFrame() - if args.db_url: - persistence.init(args.db_url, clean_open_orders=False) +def load_trades(db_url: str = None, exportfilename: str = None) -> pd.DataFrame: + """ + Load trades, either from a DB (using dburl) or via a backtest export file. + :param db_url: Sqlite url (default format sqlite:///tradesv3.dry-run.sqlite) + :param exportfilename: Path to a file exported from backtesting + :returns: Dataframe containing Trades + """ + # TODO: Document and move to btanalysis + trades: pd.DataFrame = pd.DataFrame([], columns=BT_DATA_COLUMNS) + if db_url: + persistence.init(db_url, clean_open_orders=False) columns = ["pair", "profit", "open_time", "close_time", "open_rate", "close_rate", "duration"] @@ -68,18 +75,15 @@ def load_trades(args: Namespace, pair: str) -> pd.DataFrame: t.open_rate, t.close_rate, t.close_date.timestamp() - t.open_date.timestamp() if t.close_date else None) - for t in Trade.query.filter(Trade.pair.is_(pair)).all()], + for t in Trade.query.all()], columns=columns) - elif args.exportfilename: + elif exportfilename: - file = Path(args.exportfilename) + file = Path(exportfilename) if file.exists(): trades = load_backtest_data(file) - else: - trades = pd.DataFrame([], columns=BT_DATA_COLUMNS) - return trades @@ -181,6 +185,7 @@ def extract_trades_of_period(dataframe, trades) -> pd.DataFrame: Compare trades and backtested pair DataFrames to get trades performed on backtested period :return: the DataFrame of a trades of period """ + # TODO: Document and move to btanalysis (?) trades = trades.loc[(trades['open_time'] >= dataframe.iloc[0]['date']) & (trades['close_time'] <= dataframe.iloc[-1]['date'])] return trades @@ -211,7 +216,9 @@ def analyse_and_plot_pairs(args: Namespace): tickers[pair] = data dataframe = generate_dataframe(strategy, tickers, pair) - trades = load_trades(args, pair) + trades = load_trades(pair, db_url=args.db_url, + exportfilename=args.exportfilename) + trades = trades.loc[trades['pair'] == pair] trades = extract_trades_of_period(dataframe, trades) fig = generate_graph( From b1a01345f9db4bf45b128e31464498fd14af003e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 May 2019 07:19:21 +0200 Subject: [PATCH 05/24] Add better hover tip --- freqtrade/plot/plotting.py | 19 +++++++++++++------ scripts/plot_dataframe.py | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index a18b7bf70..a8bd032ab 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -1,6 +1,7 @@ import logging - from typing import List + +import arrow import pandas as pd logger = logging.getLogger(__name__) @@ -12,7 +13,7 @@ try: import plotly.graph_objs as go except ImportError: logger.exception("Module plotly not found \n Please install using `pip install plotly`") - exit() + exit(1) def generate_row(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.make_subplots: @@ -25,9 +26,10 @@ def generate_row(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.m """ for indicator in indicators: if indicator in data: - scattergl = go.Scattergl( + # TODO: Figure out why scattergl causes problems + scattergl = go.Scatter( x=data['date'], - y=data[indicator], + y=data[indicator].values, mode='lines', name=indicator ) @@ -60,9 +62,14 @@ def plot_trades(fig, trades: pd.DataFrame): color='green' ) ) + # Create description for sell summarizing the trade + desc = trades.apply(lambda row: f"{round(row['profitperc'], 3)}%, {row['sell_reason']}, " + f"{row['duration']}min", + axis=1) trade_sells = go.Scatter( x=trades["close_time"], y=trades["close_rate"], + text=desc, mode='markers', name='trade_sell', marker=dict( @@ -123,7 +130,7 @@ def generate_graph( if 'buy' in data.columns: df_buy = data[data['buy'] == 1] if len(df_buy) > 0: - buys = go.Scattergl( + buys = go.Scatter( x=df_buy.date, y=df_buy.close, mode='markers', @@ -142,7 +149,7 @@ def generate_graph( if 'sell' in data.columns: df_sell = data[data['sell'] == 1] if len(df_sell) > 0: - sells = go.Scattergl( + sells = go.Scatter( x=df_sell.date, y=df_sell.close, mode='markers', diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 84e18e5cd..7457aadc4 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -216,7 +216,7 @@ def analyse_and_plot_pairs(args: Namespace): tickers[pair] = data dataframe = generate_dataframe(strategy, tickers, pair) - trades = load_trades(pair, db_url=args.db_url, + trades = load_trades(db_url=args.db_url, exportfilename=args.exportfilename) trades = trades.loc[trades['pair'] == pair] trades = extract_trades_of_period(dataframe, trades) From 6347161975591e6cb0d9721df52de98003c13ef2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 30 May 2019 20:26:46 +0200 Subject: [PATCH 06/24] don't use print in plot_dataframe --- scripts/plot_dataframe.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 7457aadc4..a20d8abae 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -67,7 +67,7 @@ def load_trades(db_url: str = None, exportfilename: str = None) -> pd.DataFrame: "open_rate", "close_rate", "duration"] for x in Trade.query.all(): - print("date: {}".format(x.open_date)) + logger.info("date: {}".format(x.open_date)) trades = pd.DataFrame([(t.pair, t.calc_profit(), t.open_date.replace(tzinfo=timeZone), @@ -114,7 +114,6 @@ def get_trading_env(args: Namespace): # Load the configuration _CONF.update(setup_configuration(args, RunMode.BACKTEST)) - print(_CONF) pairs = args.pairs.split(',') if pairs is None: @@ -161,7 +160,7 @@ def get_tickers_data(strategy, exchange, pairs: List[str], timerange: TimeRange, if data.empty: del tickers[pair] logger.info( - 'An issue occured while retreiving datas of %s pair, please retry ' + 'An issue occured while retreiving data of %s pair, please retry ' 'using -l option for live or --refresh-pairs-cached', pair) return tickers @@ -273,6 +272,7 @@ def plot_parse_args(args: List[str]) -> Namespace: arguments.backtesting_options(arguments.parser) return arguments.parse_args() + def main(sysargv: List[str]) -> None: """ This function will initiate the bot and start the trading loop. From cae218546087552ba890cb39425e6f6820058ad7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 31 May 2019 06:41:55 +0200 Subject: [PATCH 07/24] Move generate_plot to plotting.py --- freqtrade/plot/plotting.py | 20 ++++++++++++++++++++ scripts/plot_dataframe.py | 22 ++-------------------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index a8bd032ab..037516a27 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -3,6 +3,7 @@ from typing import List import arrow import pandas as pd +from pathlib import Path logger = logging.getLogger(__name__) @@ -200,3 +201,22 @@ def generate_graph( fig = generate_row(fig=fig, row=3, indicators=indicators2, data=data) return fig + + +def generate_plot_file(fig, pair, ticker_interval) -> None: + """ + Generate a plot html file from pre populated fig plotly object + :param fig: Plotly Figure to plot + :param pair: Pair to plot (used as filename and Plot title) + :param ticker_interval: Used as part of the filename + :return: None + """ + logger.info('Generate plot file for %s', pair) + + pair_name = pair.replace("/", "_") + file_name = 'freqtrade-plot-' + pair_name + '-' + ticker_interval + '.html' + + Path("user_data/plots").mkdir(parents=True, exist_ok=True) + + plot(fig, filename=str(Path('user_data/plots').joinpath(file_name)), + auto_open=False) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index a20d8abae..bccf98261 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -38,7 +38,7 @@ from freqtrade import persistence from freqtrade.arguments import Arguments, TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data -from freqtrade.plot.plotting import generate_graph +from freqtrade.plot.plotting import generate_graph, generate_plot_file from freqtrade.exchange import Exchange from freqtrade.optimize import setup_configuration from freqtrade.persistence import Trade @@ -87,23 +87,6 @@ def load_trades(db_url: str = None, exportfilename: str = None) -> pd.DataFrame: return trades -def generate_plot_file(fig, pair, ticker_interval, is_last) -> None: - """ - Generate a plot html file from pre populated fig plotly object - :return: None - """ - logger.info('Generate plot file for %s', pair) - - pair_name = pair.replace("/", "_") - file_name = 'freqtrade-plot-' + pair_name + '-' + ticker_interval + '.html' - - Path("user_data/plots").mkdir(parents=True, exist_ok=True) - - plot(fig, filename=str(Path('user_data/plots').joinpath(file_name)), - auto_open=False) - if is_last: - plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html')), auto_open=False) - def get_trading_env(args: Namespace): """ @@ -228,8 +211,7 @@ def analyse_and_plot_pairs(args: Namespace): indicators2=args.indicators2.split(",") ) - is_last = (False, True)[pair_counter == len(tickers)] - generate_plot_file(fig, pair, ticker_interval, is_last) + generate_plot_file(fig, pair, ticker_interval) logger.info('End of ploting process %s plots generated', pair_counter) From 2891d7cccbc61835cc28d40f62fb9cce3bda8720 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 10 Jun 2019 20:17:23 +0200 Subject: [PATCH 08/24] Add initial plotting test --- freqtrade/plot/plotting.py | 2 +- freqtrade/tests/test_plotting.py | 52 ++++++++++++++++++++++++++++++++ scripts/plot_dataframe.py | 4 +-- 3 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 freqtrade/tests/test_plotting.py diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 037516a27..b1e32c4fb 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -91,7 +91,7 @@ def generate_graph( trades: pd.DataFrame = None, indicators1: List[str] = [], indicators2: List[str] = [], -) -> tools.make_subplots: +) -> go.Figure: """ Generate the graph from the data generated by Backtesting or from DB Volume will always be ploted in row2, so Row 1 and are to our disposal for custom indicators diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py new file mode 100644 index 000000000..1f4873f4a --- /dev/null +++ b/freqtrade/tests/test_plotting.py @@ -0,0 +1,52 @@ + +from unittest.mock import MagicMock + +import plotly.graph_objs as go + +from freqtrade.arguments import Arguments, TimeRange +from freqtrade.data import history +from freqtrade.plot.plotting import (generate_graph, generate_plot_file, + generate_row, plot_trades) + + +def fig_generating_mock(fig, *args, **kwargs): + """ Return Fig - used to mock generate_row and plot_trades""" + return fig + + +def test_generate_row(): + # TODO: implement me + pass + + +def test_plot_trades(): + # TODO: implement me + pass + + +def test_generate_graph(default_conf, mocker): + row_mock = mocker.patch('freqtrade.plot.plotting.generate_row', + MagicMock(side_effect=fig_generating_mock)) + trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades', + MagicMock(side_effect=fig_generating_mock)) + + timerange = TimeRange(None, 'line', 0, -100) + data = history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='1m', + datadir=None, timerange=timerange) + + indicators1 = [] + indicators2 = [] + fig = generate_graph(pair="UNITTEST/BTC", data=data, trades=None, + indicators1=indicators1, indicators2=indicators2) + assert isinstance(fig, go.Figure) + assert fig.layout.title.text == "UNITTEST/BTC" + figure = fig.layout.figure + # Candlesticks are plotted first + assert isinstance(figure.data[0], go.Candlestick) + assert figure.data[0].name == "Price" + + assert isinstance(figure.data[1], go.Bar) + assert figure.data[1].name == "Volume" + + assert row_mock.call_count == 2 + assert trades_mock.call_count == 1 diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index bccf98261..8bed81985 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -32,7 +32,6 @@ from typing import Any, Dict, List import pandas as pd import pytz -from plotly.offline import plot from freqtrade import persistence from freqtrade.arguments import Arguments, TimeRange @@ -87,7 +86,6 @@ def load_trades(db_url: str = None, exportfilename: str = None) -> pd.DataFrame: return trades - def get_trading_env(args: Namespace): """ Initalize freqtrade Exchange and Strategy, split pairs recieved in parameter @@ -132,7 +130,7 @@ def get_tickers_data(strategy, exchange, pairs: List[str], timerange: TimeRange, refresh_pairs=_CONF.get('refresh_pairs', False), timerange=timerange, exchange=Exchange(_CONF), - live=args.live, + live=live, ) # No ticker found, impossible to download, len mismatch From 6db4e05aef7a97531041b20417448c0551974753 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 11 Jun 2019 06:45:36 +0200 Subject: [PATCH 09/24] Improve plotting tests --- freqtrade/tests/test_plotting.py | 83 +++++++++++++++++++++++++++----- requirements-dev.txt | 1 + 2 files changed, 73 insertions(+), 11 deletions(-) diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py index 1f4873f4a..d9fb3b338 100644 --- a/freqtrade/tests/test_plotting.py +++ b/freqtrade/tests/test_plotting.py @@ -7,13 +7,19 @@ from freqtrade.arguments import Arguments, TimeRange from freqtrade.data import history from freqtrade.plot.plotting import (generate_graph, generate_plot_file, generate_row, plot_trades) - +from freqtrade.strategy.default_strategy import DefaultStrategy +from freqtrade.tests.conftest import log_has, log_has_re def fig_generating_mock(fig, *args, **kwargs): """ Return Fig - used to mock generate_row and plot_trades""" return fig +def find_trace_in_fig_data(data, search_string: str): + matches = filter(lambda x: x.name == search_string, data) + return next(matches) + + def test_generate_row(): # TODO: implement me pass @@ -24,29 +30,84 @@ def test_plot_trades(): pass -def test_generate_graph(default_conf, mocker): +def test_generate_graph_no_signals_no_trades(default_conf, mocker, caplog): row_mock = mocker.patch('freqtrade.plot.plotting.generate_row', MagicMock(side_effect=fig_generating_mock)) trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades', MagicMock(side_effect=fig_generating_mock)) - timerange = TimeRange(None, 'line', 0, -100) - data = history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='1m', + pair = "UNITTEST/BTC" + timerange = TimeRange(None, 'line', 0, -1000) + data = history.load_pair_history(pair=pair, ticker_interval='1m', datadir=None, timerange=timerange) + data['buy'] = 0 + data['sell'] = 0 indicators1 = [] indicators2 = [] - fig = generate_graph(pair="UNITTEST/BTC", data=data, trades=None, + fig = generate_graph(pair=pair, data=data, trades=None, indicators1=indicators1, indicators2=indicators2) assert isinstance(fig, go.Figure) - assert fig.layout.title.text == "UNITTEST/BTC" + assert fig.layout.title.text == pair figure = fig.layout.figure - # Candlesticks are plotted first - assert isinstance(figure.data[0], go.Candlestick) - assert figure.data[0].name == "Price" - assert isinstance(figure.data[1], go.Bar) - assert figure.data[1].name == "Volume" + assert len(figure.data) == 2 + # Candlesticks are plotted first + candles = find_trace_in_fig_data(figure.data, "Price") + assert isinstance(candles, go.Candlestick) + + volume = find_trace_in_fig_data(figure.data, "Volume") + assert isinstance(volume, go.Bar) + + assert row_mock.call_count == 2 + assert trades_mock.call_count == 1 + + assert log_has("No buy-signals found.", caplog.record_tuples) + assert log_has("No sell-signals found.", caplog.record_tuples) + + +def test_generate_graph_no_trades(default_conf, mocker): + row_mock = mocker.patch('freqtrade.plot.plotting.generate_row', + MagicMock(side_effect=fig_generating_mock)) + trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades', + MagicMock(side_effect=fig_generating_mock)) + pair = 'UNITTEST/BTC' + timerange = TimeRange(None, 'line', 0, -1000) + data = history.load_pair_history(pair=pair, ticker_interval='1m', + datadir=None, timerange=timerange) + + # Generate buy/sell signals and indicators + strat = DefaultStrategy(default_conf) + data = strat.analyze_ticker(data, {'pair': pair}) + + indicators1 = [] + indicators2 = [] + fig = generate_graph(pair=pair, data=data, trades=None, + indicators1=indicators1, indicators2=indicators2) + assert isinstance(fig, go.Figure) + assert fig.layout.title.text == pair + figure = fig.layout.figure + + assert len(figure.data) == 6 + # Candlesticks are plotted first + candles = find_trace_in_fig_data(figure.data, "Price") + assert isinstance(candles, go.Candlestick) + + volume = find_trace_in_fig_data(figure.data, "Volume") + assert isinstance(volume, go.Bar) + + buy = find_trace_in_fig_data(figure.data, "buy") + assert isinstance(buy, go.Scatter) + # All buy-signals should be plotted + assert int(data.buy.sum()) == len(buy.x) + + sell = find_trace_in_fig_data(figure.data, "sell") + assert isinstance(sell, go.Scatter) + # All buy-signals should be plotted + assert int(data.sell.sum()) == len(sell.x) + + assert find_trace_in_fig_data(figure.data, "BB lower") + assert find_trace_in_fig_data(figure.data, "BB upper") assert row_mock.call_count == 2 assert trades_mock.call_count == 1 diff --git a/requirements-dev.txt b/requirements-dev.txt index 315033847..c8dd8b0b9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,6 @@ # Include all requirements to run the bot. -r requirements.txt +-r requirements-plot.txt flake8==3.7.7 flake8-type-annotations==0.1.0 From 9f5ca82f485bece4452b46c5a1c4ea7d2315cdb6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Jun 2019 10:32:12 +0200 Subject: [PATCH 10/24] Add more tests --- freqtrade/tests/test_plotting.py | 54 +++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py index d9fb3b338..e264ef6b3 100644 --- a/freqtrade/tests/test_plotting.py +++ b/freqtrade/tests/test_plotting.py @@ -1,7 +1,9 @@ from unittest.mock import MagicMock +from plotly import tools import plotly.graph_objs as go +from copy import deepcopy from freqtrade.arguments import Arguments, TimeRange from freqtrade.data import history @@ -10,6 +12,7 @@ from freqtrade.plot.plotting import (generate_graph, generate_plot_file, from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.tests.conftest import log_has, log_has_re + def fig_generating_mock(fig, *args, **kwargs): """ Return Fig - used to mock generate_row and plot_trades""" return fig @@ -20,14 +23,55 @@ def find_trace_in_fig_data(data, search_string: str): return next(matches) -def test_generate_row(): - # TODO: implement me - pass +def generage_empty_figure(): + return tools.make_subplots( + rows=3, + cols=1, + shared_xaxes=True, + row_width=[1, 1, 4], + vertical_spacing=0.0001, + ) + +def test_generate_row(default_conf, caplog): + pair = "UNITTEST/BTC" + timerange = TimeRange(None, 'line', 0, -1000) + + data = history.load_pair_history(pair=pair, ticker_interval='1m', + datadir=None, timerange=timerange) + indicators1 = ["ema10"] + indicators2 = ["macd"] + + # Generate buy/sell signals and indicators + strat = DefaultStrategy(default_conf) + data = strat.analyze_ticker(data, {'pair': pair}) + fig = generage_empty_figure() + + # Row 1 + fig1 = generate_row(fig=deepcopy(fig), row=1, indicators=indicators1, data=data) + figure = fig1.layout.figure + ema10 = find_trace_in_fig_data(figure.data, "ema10") + assert isinstance(ema10, go.Scatter) + assert ema10.yaxis == "y" + + fig2 = generate_row(fig=deepcopy(fig), row=3, indicators=indicators2, data=data) + figure = fig2.layout.figure + macd = find_trace_in_fig_data(figure.data, "macd") + assert isinstance(macd, go.Scatter) + assert macd.yaxis == "y3" + + # No indicator found + fig3 = generate_row(fig=deepcopy(fig), row=3, indicators=['no_indicator'], data=data) + assert fig == fig3 + assert log_has_re(r'Indicator "no_indicator" ignored\..*', caplog.record_tuples) def test_plot_trades(): - # TODO: implement me - pass + fig1 = generage_empty_figure() + # nothing happens when no trades are available + fig = plot_trades(fig1, None) + assert fig == fig1 + + # TODO: implement tests that do something def test_generate_graph_no_signals_no_trades(default_conf, mocker, caplog): From c7643e142b7e6ba0c7e9cbc422409caa38608952 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Jun 2019 10:41:05 +0200 Subject: [PATCH 11/24] Move load_trades to bt_anlaysis --- freqtrade/data/btanalysis.py | 43 ++++++++++++++++++++++++++++++++++++ scripts/plot_dataframe.py | 43 +----------------------------------- 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 6fce4361b..fb120e521 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -1,12 +1,18 @@ """ Helpers when analyzing backtest data """ +import logging from pathlib import Path import numpy as np import pandas as pd +import pytz +from freqtrade import persistence from freqtrade.misc import json_load +from freqtrade.persistence import Trade + +logger = logging.getLogger(__name__) # must align with columns in backtest.py BT_DATA_COLUMNS = ["pair", "profitperc", "open_time", "close_time", "index", "duration", @@ -65,3 +71,40 @@ def evaluate_result_multi(results: pd.DataFrame, freq: str, max_open_trades: int df2 = df2.set_index('date') df_final = df2.resample(freq)[['pair']].count() return df_final[df_final['pair'] > max_open_trades] + + +def load_trades(db_url: str = None, exportfilename: str = None) -> pd.DataFrame: + """ + Load trades, either from a DB (using dburl) or via a backtest export file. + :param db_url: Sqlite url (default format sqlite:///tradesv3.dry-run.sqlite) + :param exportfilename: Path to a file exported from backtesting + :returns: Dataframe containing Trades + """ + timeZone = pytz.UTC + + trades: pd.DataFrame = pd.DataFrame([], columns=BT_DATA_COLUMNS) + + if db_url: + persistence.init(db_url, clean_open_orders=False) + columns = ["pair", "profit", "open_time", "close_time", + "open_rate", "close_rate", "duration"] + + for x in Trade.query.all(): + logger.info("date: {}".format(x.open_date)) + + trades = pd.DataFrame([(t.pair, t.calc_profit(), + t.open_date.replace(tzinfo=timeZone), + t.close_date.replace(tzinfo=timeZone) if t.close_date else None, + t.open_rate, t.close_rate, + t.close_date.timestamp() - t.open_date.timestamp() + if t.close_date else None) + for t in Trade.query.all()], + columns=columns) + + elif exportfilename: + + file = Path(exportfilename) + if file.exists(): + trades = load_backtest_data(file) + + return trades diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 8bed81985..005148d3b 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -31,60 +31,19 @@ from pathlib import Path from typing import Any, Dict, List import pandas as pd -import pytz -from freqtrade import persistence from freqtrade.arguments import Arguments, TimeRange from freqtrade.data import history -from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data +from freqtrade.data.btanalysis import load_trades from freqtrade.plot.plotting import generate_graph, generate_plot_file from freqtrade.exchange import Exchange from freqtrade.optimize import setup_configuration -from freqtrade.persistence import Trade from freqtrade.resolvers import StrategyResolver from freqtrade.state import RunMode logger = logging.getLogger(__name__) _CONF: Dict[str, Any] = {} -timeZone = pytz.UTC - - -def load_trades(db_url: str = None, exportfilename: str = None) -> pd.DataFrame: - """ - Load trades, either from a DB (using dburl) or via a backtest export file. - :param db_url: Sqlite url (default format sqlite:///tradesv3.dry-run.sqlite) - :param exportfilename: Path to a file exported from backtesting - :returns: Dataframe containing Trades - """ - # TODO: Document and move to btanalysis - trades: pd.DataFrame = pd.DataFrame([], columns=BT_DATA_COLUMNS) - - if db_url: - persistence.init(db_url, clean_open_orders=False) - columns = ["pair", "profit", "open_time", "close_time", - "open_rate", "close_rate", "duration"] - - for x in Trade.query.all(): - logger.info("date: {}".format(x.open_date)) - - trades = pd.DataFrame([(t.pair, t.calc_profit(), - t.open_date.replace(tzinfo=timeZone), - t.close_date.replace(tzinfo=timeZone) if t.close_date else None, - t.open_rate, t.close_rate, - t.close_date.timestamp() - t.open_date.timestamp() - if t.close_date else None) - for t in Trade.query.all()], - columns=columns) - - elif exportfilename: - - file = Path(exportfilename) - if file.exists(): - trades = load_backtest_data(file) - - return trades - def get_trading_env(args: Namespace): """ From 1c53aa5687c7b714b5bd7043a3e977f82e9f5ea2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Jun 2019 10:57:21 +0200 Subject: [PATCH 12/24] Add tests for load_trades --- freqtrade/tests/data/test_btanalysis.py | 28 ++++++++- freqtrade/tests/test_persistence.py | 84 +++++++++++++------------ 2 files changed, 72 insertions(+), 40 deletions(-) diff --git a/freqtrade/tests/data/test_btanalysis.py b/freqtrade/tests/data/test_btanalysis.py index dd7cbe0d9..4a8babf1d 100644 --- a/freqtrade/tests/data/test_btanalysis.py +++ b/freqtrade/tests/data/test_btanalysis.py @@ -1,8 +1,11 @@ import pytest +from unittest.mock import MagicMock from pandas import DataFrame -from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data +from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data, load_trades from freqtrade.data.history import make_testdata_path +from freqtrade.persistence import init, Trade +from freqtrade.tests.test_persistence import init_persistence, create_mock_trades def test_load_backtest_data(): @@ -19,3 +22,26 @@ def test_load_backtest_data(): with pytest.raises(ValueError, match=r"File .* does not exist\."): load_backtest_data(str("filename") + "nofile") + + +def test_load_trades_file(default_conf, fee, mocker): + # Real testing of load_backtest_data is done in test_load_backtest_data + lbt = mocker.patch("freqtrade.data.btanalysis.load_backtest_data", MagicMock()) + filename = make_testdata_path(None) / "backtest-result_test.json" + load_trades(db_url=None, exportfilename=filename) + assert lbt.call_count == 1 + + +@pytest.mark.usefixtures("init_persistence") +def test_load_trades_db(default_conf, fee, mocker): + + create_mock_trades(fee) + # remove init so it does not init again + init_mock = mocker.patch('freqtrade.persistence.init', MagicMock()) + + trades = load_trades(db_url=default_conf['db_url'], exportfilename=None) + assert init_mock.call_count == 1 + assert len(trades) == 3 + assert isinstance(trades, DataFrame) + assert "pair" in trades.columns + assert "open_time" in trades.columns diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index bb00fa8f4..381f04bd1 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -16,6 +16,50 @@ def init_persistence(default_conf): init(default_conf['db_url'], default_conf['dry_run']) +def create_mock_trades(fee): + """ + Create some fake trades ... + """ + # Simulate dry_run entries + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + amount=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + exchange='bittrex', + open_order_id='dry_run_buy_12345' + ) + Trade.session.add(trade) + + trade = Trade( + pair='ETC/BTC', + stake_amount=0.001, + amount=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + exchange='bittrex', + is_open=False, + open_order_id='dry_run_sell_12345' + ) + Trade.session.add(trade) + + # Simulate prod entry + trade = Trade( + pair='ETC/BTC', + stake_amount=0.001, + amount=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + exchange='bittrex', + open_order_id='prod_buy_12345' + ) + Trade.session.add(trade) + + def test_init_create_session(default_conf): # Check if init create a session init(default_conf['db_url'], default_conf['dry_run']) @@ -671,45 +715,7 @@ def test_adjust_min_max_rates(fee): @pytest.mark.usefixtures("init_persistence") def test_get_open(default_conf, fee): - # Simulate dry_run entries - trade = Trade( - pair='ETH/BTC', - stake_amount=0.001, - amount=123.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.123, - exchange='bittrex', - open_order_id='dry_run_buy_12345' - ) - Trade.session.add(trade) - - trade = Trade( - pair='ETC/BTC', - stake_amount=0.001, - amount=123.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.123, - exchange='bittrex', - is_open=False, - open_order_id='dry_run_sell_12345' - ) - Trade.session.add(trade) - - # Simulate prod entry - trade = Trade( - pair='ETC/BTC', - stake_amount=0.001, - amount=123.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.123, - exchange='bittrex', - open_order_id='prod_buy_12345' - ) - Trade.session.add(trade) - + create_mock_trades(fee) assert len(Trade.get_open_trades()) == 2 From 1cd84157230c4416a106215460f7da8a0192cb28 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Jun 2019 11:12:19 +0200 Subject: [PATCH 13/24] Move extract_trades_of_period to btanlaysis --- freqtrade/data/btanalysis.py | 10 +++++ freqtrade/tests/data/test_btanalysis.py | 53 ++++++++++++++++++++++--- scripts/plot_dataframe.py | 14 +------ 3 files changed, 59 insertions(+), 18 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index fb120e521..e1ddd1638 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -108,3 +108,13 @@ def load_trades(db_url: str = None, exportfilename: str = None) -> pd.DataFrame: trades = load_backtest_data(file) return trades + + +def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> pd.DataFrame: + """ + Compare trades and backtested pair DataFrames to get trades performed on backtested period + :return: the DataFrame of a trades of period + """ + trades = trades.loc[(trades['open_time'] >= dataframe.iloc[0]['date']) & + (trades['close_time'] <= dataframe.iloc[-1]['date'])] + return trades diff --git a/freqtrade/tests/data/test_btanalysis.py b/freqtrade/tests/data/test_btanalysis.py index 4a8babf1d..aa066557b 100644 --- a/freqtrade/tests/data/test_btanalysis.py +++ b/freqtrade/tests/data/test_btanalysis.py @@ -1,11 +1,18 @@ -import pytest from unittest.mock import MagicMock -from pandas import DataFrame -from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data, load_trades -from freqtrade.data.history import make_testdata_path -from freqtrade.persistence import init, Trade -from freqtrade.tests.test_persistence import init_persistence, create_mock_trades +from arrow import Arrow +import pytest +from pandas import DataFrame, to_datetime + +from freqtrade.arguments import Arguments, TimeRange +from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, + extract_trades_of_period, + load_backtest_data, load_trades) +from freqtrade.data.history import load_pair_history, make_testdata_path +from freqtrade.persistence import Trade, init +from freqtrade.strategy.interface import SellType +from freqtrade.tests.test_persistence import (create_mock_trades, + init_persistence) def test_load_backtest_data(): @@ -45,3 +52,37 @@ def test_load_trades_db(default_conf, fee, mocker): assert isinstance(trades, DataFrame) assert "pair" in trades.columns assert "open_time" in trades.columns + + +def test_extract_trades_of_period(): + pair = "UNITTEST/BTC" + timerange = TimeRange(None, 'line', 0, -1000) + + data = load_pair_history(pair=pair, ticker_interval='1m', + datadir=None, timerange=timerange) + + # timerange = 2017-11-14 06:07 - 2017-11-14 22:58:00 + trades = DataFrame( + {'pair': [pair, pair, pair, pair], + 'profit_percent': [0.0, 0.1, -0.2, -0.5], + 'profit_abs': [0.0, 1, -2, -5], + 'open_time': to_datetime([Arrow(2017, 11, 13, 15, 40, 0).datetime, + Arrow(2017, 11, 14, 9, 41, 0).datetime, + Arrow(2017, 11, 14, 14, 20, 0).datetime, + Arrow(2017, 11, 15, 3, 40, 0).datetime, + ], utc=True + ), + 'close_time': to_datetime([Arrow(2017, 11, 13, 16, 40, 0).datetime, + Arrow(2017, 11, 14, 10, 41, 0).datetime, + Arrow(2017, 11, 14, 15, 25, 0).datetime, + Arrow(2017, 11, 15, 3, 55, 0).datetime, + ], utc=True) + }) + trades1 = extract_trades_of_period(data, trades) + # First and last trade are dropped as they are out of range + assert len(trades1) == 2 + assert trades1.iloc[0].open_time == Arrow(2017, 11, 14, 9, 41, 0).datetime + assert trades1.iloc[0].close_time == Arrow(2017, 11, 14, 10, 41, 0).datetime + assert trades1.iloc[-1].open_time == Arrow(2017, 11, 14, 14, 20, 0).datetime + assert trades1.iloc[-1].close_time == Arrow(2017, 11, 14, 15, 25, 0).datetime + diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 005148d3b..6d2e545ce 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -35,9 +35,10 @@ import pandas as pd from freqtrade.arguments import Arguments, TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import load_trades -from freqtrade.plot.plotting import generate_graph, generate_plot_file from freqtrade.exchange import Exchange from freqtrade.optimize import setup_configuration +from freqtrade.plot.plotting import (extract_trades_of_period, generate_graph, + generate_plot_file) from freqtrade.resolvers import StrategyResolver from freqtrade.state import RunMode @@ -119,17 +120,6 @@ def generate_dataframe(strategy, tickers, pair) -> pd.DataFrame: return dataframe -def extract_trades_of_period(dataframe, trades) -> pd.DataFrame: - """ - Compare trades and backtested pair DataFrames to get trades performed on backtested period - :return: the DataFrame of a trades of period - """ - # TODO: Document and move to btanalysis (?) - trades = trades.loc[(trades['open_time'] >= dataframe.iloc[0]['date']) & - (trades['close_time'] <= dataframe.iloc[-1]['date'])] - return trades - - def analyse_and_plot_pairs(args: Namespace): """ From arguments provided in cli: From bf2c0390e7cc01020dbd10d2990a3606f75adaa1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Jun 2019 12:54:27 +0200 Subject: [PATCH 14/24] Adjust some imports --- freqtrade/tests/conftest.py | 5 +++++ freqtrade/tests/data/test_btanalysis.py | 8 ++------ freqtrade/tests/test_persistence.py | 5 ----- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 034cb5f8b..e956d89c4 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -151,6 +151,11 @@ def patch_coinmarketcap(mocker) -> None: ) +@pytest.fixture(scope='function') +def init_persistence(default_conf): + persistence.init(default_conf['db_url'], default_conf['dry_run']) + + @pytest.fixture(scope="function") def default_conf(): """ Returns validated configuration suitable for most tests """ diff --git a/freqtrade/tests/data/test_btanalysis.py b/freqtrade/tests/data/test_btanalysis.py index aa066557b..6fa529394 100644 --- a/freqtrade/tests/data/test_btanalysis.py +++ b/freqtrade/tests/data/test_btanalysis.py @@ -4,15 +4,12 @@ from arrow import Arrow import pytest from pandas import DataFrame, to_datetime -from freqtrade.arguments import Arguments, TimeRange +from freqtrade.arguments import TimeRange from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, extract_trades_of_period, load_backtest_data, load_trades) from freqtrade.data.history import load_pair_history, make_testdata_path -from freqtrade.persistence import Trade, init -from freqtrade.strategy.interface import SellType -from freqtrade.tests.test_persistence import (create_mock_trades, - init_persistence) +from freqtrade.tests.test_persistence import create_mock_trades def test_load_backtest_data(): @@ -85,4 +82,3 @@ def test_extract_trades_of_period(): assert trades1.iloc[0].close_time == Arrow(2017, 11, 14, 10, 41, 0).datetime assert trades1.iloc[-1].open_time == Arrow(2017, 11, 14, 14, 20, 0).datetime assert trades1.iloc[-1].close_time == Arrow(2017, 11, 14, 15, 25, 0).datetime - diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 381f04bd1..32425ef7b 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -11,11 +11,6 @@ from freqtrade.persistence import Trade, clean_dry_run_db, init from freqtrade.tests.conftest import log_has -@pytest.fixture(scope='function') -def init_persistence(default_conf): - init(default_conf['db_url'], default_conf['dry_run']) - - def create_mock_trades(fee): """ Create some fake trades ... From 0300128cb82f4a109902ac78770345474a41793a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Jun 2019 19:35:15 +0200 Subject: [PATCH 15/24] Move plot-options to arguments.py --- freqtrade/arguments.py | 32 ++++++++++++++++++++++++++++++- freqtrade/tests/test_arguments.py | 14 ++++++++++++++ scripts/plot_dataframe.py | 30 +++-------------------------- 3 files changed, 48 insertions(+), 28 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 35d388432..fde372b63 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -55,7 +55,7 @@ class Arguments(object): # Workaround issue in argparse with action='append' and default value # (see https://bugs.python.org/issue16399) - if parsed_arg.config is None and not no_default_config: + if not no_default_config and parsed_arg.config is None: parsed_arg.config = [constants.DEFAULT_CONFIG] return parsed_arg @@ -514,3 +514,33 @@ class Arguments(object): dest='erase', action='store_true' ) + + def plot_dataframe_options(self) -> None: + """ + Parses given arguments for plot_dataframe + """ + self.parser.add_argument( + '--indicators1', + help='Set indicators from your strategy you want in the first row of the graph. Separate ' + 'them with a comma. E.g: ema3,ema5 (default: %(default)s)', + type=str, + default='sma,ema3,ema5', + dest='indicators1', + ) + + self.parser.add_argument( + '--indicators2', + help='Set indicators from your strategy you want in the third row of the graph. Separate ' + 'them with a comma. E.g: macd,fastd,fastk (default: %(default)s)', + type=str, + default='macd,macdsignal', + dest='indicators2', + ) + self.parser.add_argument( + '--plot-limit', + help='Specify tick limit for plotting - too high values cause huge files - ' + 'Default: %(default)s', + dest='plot_limit', + default=750, + type=int, + ) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index afa42f287..d1b96a923 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -186,6 +186,20 @@ def test_download_data_options() -> None: assert args.exchange == 'binance' +def test_plot_dataframe_options() -> None: + args = [ + '--indicators1', 'sma10,sma100', + '--indicators2', 'macd,fastd,fastk', + '--plot-limit', '30', + ] + arguments = Arguments(args, '') + arguments.plot_dataframe_options() + args = arguments.parse_args(True) + assert args.indicators1 == "sma10,sma100" + assert args.indicators2 == "macd,fastd,fastk" + assert args.plot_limit == 30 + + def test_check_int_positive() -> None: assert Arguments.check_int_positive("3") == 3 diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 6d2e545ce..d37d63c32 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -34,10 +34,10 @@ import pandas as pd from freqtrade.arguments import Arguments, TimeRange from freqtrade.data import history -from freqtrade.data.btanalysis import load_trades +from freqtrade.data.btanalysis import load_trades, extract_trades_of_period from freqtrade.exchange import Exchange from freqtrade.optimize import setup_configuration -from freqtrade.plot.plotting import (extract_trades_of_period, generate_graph, +from freqtrade.plot.plotting import (generate_graph, generate_plot_file) from freqtrade.resolvers import StrategyResolver from freqtrade.state import RunMode @@ -171,31 +171,7 @@ def plot_parse_args(args: List[str]) -> Namespace: """ arguments = Arguments(args, 'Graph dataframe') arguments.scripts_options() - arguments.parser.add_argument( - '--indicators1', - help='Set indicators from your strategy you want in the first row of the graph. Separate ' - 'them with a coma. E.g: ema3,ema5 (default: %(default)s)', - type=str, - default='sma,ema3,ema5', - dest='indicators1', - ) - - arguments.parser.add_argument( - '--indicators2', - help='Set indicators from your strategy you want in the third row of the graph. Separate ' - 'them with a coma. E.g: fastd,fastk (default: %(default)s)', - type=str, - default='macd,macdsignal', - dest='indicators2', - ) - arguments.parser.add_argument( - '--plot-limit', - help='Specify tick limit for plotting - too high values cause huge files - ' - 'Default: %(default)s', - dest='plot_limit', - default=750, - type=int, - ) + arguments.plot_dataframe_options() arguments.common_args_parser() arguments.optimizer_shared_options(arguments.parser) arguments.backtesting_options(arguments.parser) From 3f04930f383272f588bd33d455b10f832cf16c18 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Jun 2019 13:19:06 +0200 Subject: [PATCH 16/24] Require pairs argument --- freqtrade/arguments.py | 7 +++++++ freqtrade/tests/test_arguments.py | 17 +++++++++++++---- scripts/plot_dataframe.py | 5 +++-- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index fde372b63..cef38784d 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -519,6 +519,13 @@ class Arguments(object): """ Parses given arguments for plot_dataframe """ + self.parser.add_argument( + '-p', '--pairs', + help='Show profits for only this pairs. Pairs are comma-separated.', + dest='pairs', + required=True, + default=None + ) self.parser.add_argument( '--indicators1', help='Set indicators from your strategy you want in the first row of the graph. Separate ' diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index d1b96a923..d584a9e01 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -191,13 +191,22 @@ def test_plot_dataframe_options() -> None: '--indicators1', 'sma10,sma100', '--indicators2', 'macd,fastd,fastk', '--plot-limit', '30', + '-p', 'UNITTEST/BTC', ] arguments = Arguments(args, '') arguments.plot_dataframe_options() - args = arguments.parse_args(True) - assert args.indicators1 == "sma10,sma100" - assert args.indicators2 == "macd,fastd,fastk" - assert args.plot_limit == 30 + pargs = arguments.parse_args(True) + assert pargs.indicators1 == "sma10,sma100" + assert pargs.indicators2 == "macd,fastd,fastk" + assert pargs.plot_limit == 30 + assert pargs.pairs == "UNITTEST/BTC" + + # Pop pairs argument + args = args[:-2] + arguments = Arguments(args, '') + arguments.plot_dataframe_options() + with pytest.raises(SystemExit, match=r'2'): + arguments.parse_args(True) def test_check_int_positive() -> None: diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index d37d63c32..c44f2aa4b 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -89,7 +89,7 @@ def get_tickers_data(strategy, exchange, pairs: List[str], timerange: TimeRange, ticker_interval=ticker_interval, refresh_pairs=_CONF.get('refresh_pairs', False), timerange=timerange, - exchange=Exchange(_CONF), + exchange=exchange, live=live, ) @@ -132,6 +132,8 @@ def analyse_and_plot_pairs(args: Namespace): :return: None """ strategy, exchange, pairs = get_trading_env(args) + pairs = args.pairs.split(',') + # Set timerange to use timerange = Arguments.parse_timerange(args.timerange) ticker_interval = strategy.ticker_interval @@ -170,7 +172,6 @@ def plot_parse_args(args: List[str]) -> Namespace: :return: args: Array with all arguments """ arguments = Arguments(args, 'Graph dataframe') - arguments.scripts_options() arguments.plot_dataframe_options() arguments.common_args_parser() arguments.optimizer_shared_options(arguments.parser) From 907c2f1e6b5a8ed53649640d09f62b5939d71236 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Jun 2019 13:31:24 +0200 Subject: [PATCH 17/24] Copy plot options to config --- freqtrade/configuration.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 82d96313d..c63e99318 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -98,6 +98,9 @@ class Configuration(object): # Load Optimize configurations config = self._load_optimize_config(config) + # Add plotting options if available + config = self._load_plot_config(config) + # Set runmode if not self.runmode: # Handle real mode, infer dry/live from config @@ -332,6 +335,26 @@ class Configuration(object): return config + def _load_plot_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + """ + Extract information for sys.argv Plotting configuration + :return: configuration as dictionary + """ + + self._args_to_config(config, argname='pairs', + logstring='Using pairs {}') + + self._args_to_config(config, argname='indicators1', + logstring='Using indicators1: {}') + + self._args_to_config(config, argname='indicators2', + logstring='Using indicators2: {}') + + self._args_to_config(config, argname='plot_limit', + logstring='Limiting plot to: {}') + + return config + def _validate_config_schema(self, conf: Dict[str, Any]) -> Dict[str, Any]: """ Validate the configuration follow the Config Schema From 488bb971ffa25a6a277e4d494b098401d9c9d870 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Jun 2019 13:41:36 +0200 Subject: [PATCH 18/24] Get rid of global conf object --- scripts/plot_dataframe.py | 70 ++++++++++++++------------------------- 1 file changed, 25 insertions(+), 45 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index c44f2aa4b..e54a78124 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -35,47 +35,17 @@ import pandas as pd from freqtrade.arguments import Arguments, TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import load_trades, extract_trades_of_period -from freqtrade.exchange import Exchange from freqtrade.optimize import setup_configuration from freqtrade.plot.plotting import (generate_graph, generate_plot_file) -from freqtrade.resolvers import StrategyResolver +from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.state import RunMode logger = logging.getLogger(__name__) -_CONF: Dict[str, Any] = {} -def get_trading_env(args: Namespace): - """ - Initalize freqtrade Exchange and Strategy, split pairs recieved in parameter - :return: Strategy - """ - global _CONF - - # Load the configuration - _CONF.update(setup_configuration(args, RunMode.BACKTEST)) - - pairs = args.pairs.split(',') - if pairs is None: - logger.critical('Parameter --pairs mandatory;. E.g --pairs ETH/BTC,XRP/BTC') - exit() - - # Load the strategy - try: - strategy = StrategyResolver(_CONF).strategy - exchange = Exchange(_CONF) - except AttributeError: - logger.critical( - 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', - args.strategy - ) - exit() - - return [strategy, exchange, pairs] - - -def get_tickers_data(strategy, exchange, pairs: List[str], timerange: TimeRange, live: bool): +def get_tickers_data(strategy, exchange, pairs: List[str], timerange: TimeRange, + datadir: Path, refresh_pairs: bool, live: bool): """ Get tickers data for each pairs on live or local, option defined in args :return: dictionary of tickers. output format: {'pair': tickersdata} @@ -84,10 +54,10 @@ def get_tickers_data(strategy, exchange, pairs: List[str], timerange: TimeRange, ticker_interval = strategy.ticker_interval tickers = history.load_data( - datadir=Path(str(_CONF.get("datadir"))), + datadir=datadir, pairs=pairs, ticker_interval=ticker_interval, - refresh_pairs=_CONF.get('refresh_pairs', False), + refresh_pairs=refresh_pairs, timerange=timerange, exchange=exchange, live=live, @@ -120,7 +90,7 @@ def generate_dataframe(strategy, tickers, pair) -> pd.DataFrame: return dataframe -def analyse_and_plot_pairs(args: Namespace): +def analyse_and_plot_pairs(config: Dict[str, Any]): """ From arguments provided in cli: -Initialise backtest env @@ -131,14 +101,20 @@ def analyse_and_plot_pairs(args: Namespace): -Generate plot files :return: None """ - strategy, exchange, pairs = get_trading_env(args) - pairs = args.pairs.split(',') + exchange_name = config.get('exchange', {}).get('name').title() + exchange = ExchangeResolver(exchange_name, config).exchange + + strategy = StrategyResolver(config).strategy + pairs = config["pairs"].split(',') # Set timerange to use - timerange = Arguments.parse_timerange(args.timerange) + timerange = Arguments.parse_timerange(config["timerange"]) ticker_interval = strategy.ticker_interval - tickers = get_tickers_data(strategy, exchange, pairs, timerange, args.live) + tickers = get_tickers_data(strategy, exchange, pairs, timerange, + datadir=Path(str(config.get("datadir"))), + refresh_pairs=config.get('refresh_pairs', False), + live=config.get("live", False)) pair_counter = 0 for pair, data in tickers.items(): pair_counter += 1 @@ -147,8 +123,8 @@ def analyse_and_plot_pairs(args: Namespace): tickers[pair] = data dataframe = generate_dataframe(strategy, tickers, pair) - trades = load_trades(db_url=args.db_url, - exportfilename=args.exportfilename) + trades = load_trades(db_url=config["db_url"], + exportfilename=config["exportfilename"]) trades = trades.loc[trades['pair'] == pair] trades = extract_trades_of_period(dataframe, trades) @@ -156,8 +132,8 @@ def analyse_and_plot_pairs(args: Namespace): pair=pair, data=dataframe, trades=trades, - indicators1=args.indicators1.split(","), - indicators2=args.indicators2.split(",") + indicators1=config["indicators1"].split(","), + indicators2=config["indicators2"].split(",") ) generate_plot_file(fig, pair, ticker_interval) @@ -176,7 +152,11 @@ def plot_parse_args(args: List[str]) -> Namespace: arguments.common_args_parser() arguments.optimizer_shared_options(arguments.parser) arguments.backtesting_options(arguments.parser) - return arguments.parse_args() + parsed_args = arguments.parse_args() + + # Load the configuration + config = setup_configuration(parsed_args, RunMode.BACKTEST) + return config def main(sysargv: List[str]) -> None: From 4b7dfc64c66afa33f2723be1214668b16f42f327 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Jun 2019 14:03:55 +0200 Subject: [PATCH 19/24] Add test for generate_plot_file --- freqtrade/plot/plotting.py | 1 - freqtrade/tests/test_plotting.py | 12 ++++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index b1e32c4fb..dcea3291f 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -1,7 +1,6 @@ import logging from typing import List -import arrow import pandas as pd from pathlib import Path diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py index e264ef6b3..b1dad5d5c 100644 --- a/freqtrade/tests/test_plotting.py +++ b/freqtrade/tests/test_plotting.py @@ -32,6 +32,7 @@ def generage_empty_figure(): vertical_spacing=0.0001, ) + def test_generate_row(default_conf, caplog): pair = "UNITTEST/BTC" timerange = TimeRange(None, 'line', 0, -1000) @@ -155,3 +156,14 @@ def test_generate_graph_no_trades(default_conf, mocker): assert row_mock.call_count == 2 assert trades_mock.call_count == 1 + + +def test_generate_plot_file(mocker, caplog): + fig = generage_empty_figure() + plot_mock = mocker.patch("freqtrade.plot.plotting.plot", MagicMock()) + generate_plot_file(fig, "UNITTEST/BTC", "5m") + + assert plot_mock.call_count == 1 + assert plot_mock.call_args[0][0] == fig + assert (plot_mock.call_args_list[0][1]['filename'] + == "user_data/plots/freqtrade-plot-UNITTEST_BTC-5m.html") From fc3e3c468c3a79dacb05259f8b4c2cdeedfb3212 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Jun 2019 19:26:43 +0200 Subject: [PATCH 20/24] File existence is checked in load_backtest_data --- freqtrade/data/btanalysis.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index e1ddd1638..4aecf8ecf 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -103,9 +103,7 @@ def load_trades(db_url: str = None, exportfilename: str = None) -> pd.DataFrame: elif exportfilename: - file = Path(exportfilename) - if file.exists(): - trades = load_backtest_data(file) + trades = load_backtest_data(Path(exportfilename)) return trades From 0eb109f8f7a01d8962f76cd89a111c9aa8ab3a7a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Jun 2019 19:53:48 +0200 Subject: [PATCH 21/24] Improve some tests --- freqtrade/arguments.py | 8 ++++---- freqtrade/tests/test_plotting.py | 23 +++++++++++++++++++++-- scripts/plot_dataframe.py | 2 +- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index cef38784d..3f21709c9 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -528,8 +528,8 @@ class Arguments(object): ) self.parser.add_argument( '--indicators1', - help='Set indicators from your strategy you want in the first row of the graph. Separate ' - 'them with a comma. E.g: ema3,ema5 (default: %(default)s)', + help='Set indicators from your strategy you want in the first row of the graph. ' + 'Separate them with a comma. E.g: ema3,ema5 (default: %(default)s)', type=str, default='sma,ema3,ema5', dest='indicators1', @@ -537,8 +537,8 @@ class Arguments(object): self.parser.add_argument( '--indicators2', - help='Set indicators from your strategy you want in the third row of the graph. Separate ' - 'them with a comma. E.g: macd,fastd,fastk (default: %(default)s)', + help='Set indicators from your strategy you want in the third row of the graph. ' + 'Separate them with a comma. E.g: macd,fastd,fastk (default: %(default)s)', type=str, default='macd,macdsignal', dest='indicators2', diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py index b1dad5d5c..15ab698d8 100644 --- a/freqtrade/tests/test_plotting.py +++ b/freqtrade/tests/test_plotting.py @@ -5,8 +5,9 @@ from plotly import tools import plotly.graph_objs as go from copy import deepcopy -from freqtrade.arguments import Arguments, TimeRange +from freqtrade.arguments import TimeRange from freqtrade.data import history +from freqtrade.data.btanalysis import load_backtest_data from freqtrade.plot.plotting import (generate_graph, generate_plot_file, generate_row, plot_trades) from freqtrade.strategy.default_strategy import DefaultStrategy @@ -71,8 +72,26 @@ def test_plot_trades(): # nothing happens when no trades are available fig = plot_trades(fig1, None) assert fig == fig1 + pair = "ADA/BTC" + filename = history.make_testdata_path(None) / "backtest-result_test.json" + trades = load_backtest_data(filename) + trades = trades.loc[trades['pair'] == pair] - # TODO: implement tests that do something + fig = plot_trades(fig, trades) + figure = fig1.layout.figure + + # Check buys - color, should be in first graph, ... + trade_buy = find_trace_in_fig_data(figure.data, "trade_buy") + assert isinstance(trade_buy, go.Scatter) + assert trade_buy.yaxis == 'y' + assert len(trades) == len(trade_buy.x) + assert trade_buy.marker.color == 'green' + + trade_sell = find_trace_in_fig_data(figure.data, "trade_sell") + assert isinstance(trade_sell, go.Scatter) + assert trade_sell.yaxis == 'y' + assert len(trades) == len(trade_sell.x) + assert trade_sell.marker.color == 'red' def test_generate_graph_no_signals_no_trades(default_conf, mocker, caplog): diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index e54a78124..54ce199f4 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -141,7 +141,7 @@ def analyse_and_plot_pairs(config: Dict[str, Any]): logger.info('End of ploting process %s plots generated', pair_counter) -def plot_parse_args(args: List[str]) -> Namespace: +def plot_parse_args(args: List[str]) -> Dict[str, Any]: """ Parse args passed to the script :param args: Cli arguments From 765eff23f018c6532fa169505d354f175f0f08b3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Jun 2019 20:14:31 +0200 Subject: [PATCH 22/24] Fix typo --- freqtrade/plot/plotting.py | 2 +- scripts/plot_dataframe.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index dcea3291f..94c0830bf 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -93,7 +93,7 @@ def generate_graph( ) -> go.Figure: """ Generate the graph from the data generated by Backtesting or from DB - Volume will always be ploted in row2, so Row 1 and are to our disposal for custom indicators + Volume will always be ploted in row2, so Row 1 and 3 are to our disposal for custom indicators :param pair: Pair to Display on the graph :param data: OHLCV DataFrame containing indicators and buy/sell signals :param trades: All trades created diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 54ce199f4..eebe20be2 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -26,7 +26,6 @@ Example of usage: """ import logging import sys -from argparse import Namespace from pathlib import Path from typing import Any, Dict, List From a581ca66bf33e0e9fa4b3a36eea0b56804fd33df Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 21 Jun 2019 19:31:18 +0200 Subject: [PATCH 23/24] Adapt test after merging develop --- freqtrade/arguments.py | 1 + freqtrade/tests/test_arguments.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 3075dd0fe..59123a14b 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -444,6 +444,7 @@ class Arguments(object): '-p', '--pairs', help='Show profits for only this pairs. Pairs are comma-separated.', dest='pairs', + required=True ) def download_data_options(self) -> None: diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 78ca9d055..4c2bf708d 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -195,6 +195,7 @@ def test_plot_dataframe_options() -> None: '-p', 'UNITTEST/BTC', ] arguments = Arguments(args, '') + arguments.common_scripts_options() arguments.plot_dataframe_options() pargs = arguments.parse_args(True) assert pargs.indicators1 == "sma10,sma100" @@ -205,6 +206,7 @@ def test_plot_dataframe_options() -> None: # Pop pairs argument args = args[:-2] arguments = Arguments(args, '') + arguments.common_scripts_options() arguments.plot_dataframe_options() with pytest.raises(SystemExit, match=r'2'): arguments.parse_args(True) From db17b20e26a563048838d98ca5c6cc4f35381fc1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 21 Jun 2019 20:21:03 +0200 Subject: [PATCH 24/24] Don't require pairs but fall back to pair_whitelist instead --- freqtrade/arguments.py | 1 - freqtrade/tests/test_arguments.py | 8 -------- scripts/plot_dataframe.py | 5 ++++- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 59123a14b..3075dd0fe 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -444,7 +444,6 @@ class Arguments(object): '-p', '--pairs', help='Show profits for only this pairs. Pairs are comma-separated.', dest='pairs', - required=True ) def download_data_options(self) -> None: diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 4c2bf708d..d9292bdb5 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -203,14 +203,6 @@ def test_plot_dataframe_options() -> None: assert pargs.plot_limit == 30 assert pargs.pairs == "UNITTEST/BTC" - # Pop pairs argument - args = args[:-2] - arguments = Arguments(args, '') - arguments.common_scripts_options() - arguments.plot_dataframe_options() - with pytest.raises(SystemExit, match=r'2'): - arguments.parse_args(True) - def test_check_int_positive() -> None: diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index a17076085..4aacc99dd 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -104,7 +104,10 @@ def analyse_and_plot_pairs(config: Dict[str, Any]): exchange = ExchangeResolver(exchange_name, config).exchange strategy = StrategyResolver(config).strategy - pairs = config["pairs"].split(',') + if "pairs" in config: + pairs = config["pairs"].split(',') + else: + pairs = config["exchange"]["pair_whitelist"] # Set timerange to use timerange = Arguments.parse_timerange(config["timerange"])