diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 967957fe2..3075dd0fe 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -56,7 +56,7 @@ class Arguments(object): # Workaround issue in argparse with action='append' and default value # (see https://bugs.python.org/issue16399) - if parsed_arg.config is None and not no_default_config: + if not no_default_config and parsed_arg.config is None: parsed_arg.config = [constants.DEFAULT_CONFIG] return parsed_arg diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 47b4bb8f7..b2c35c977 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -103,6 +103,9 @@ class Configuration(object): # Load Optimize configurations config = self._load_optimize_config(config) + # Add plotting options if available + config = self._load_plot_config(config) + # Set runmode if not self.runmode: # Handle real mode, infer dry/live from config @@ -338,6 +341,26 @@ class Configuration(object): return config + def _load_plot_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + """ + Extract information for sys.argv Plotting configuration + :return: configuration as dictionary + """ + + self._args_to_config(config, argname='pairs', + logstring='Using pairs {}') + + self._args_to_config(config, argname='indicators1', + logstring='Using indicators1: {}') + + self._args_to_config(config, argname='indicators2', + logstring='Using indicators2: {}') + + self._args_to_config(config, argname='plot_limit', + logstring='Limiting plot to: {}') + + return config + def _validate_config_schema(self, conf: Dict[str, Any]) -> Dict[str, Any]: """ Validate the configuration follow the Config Schema diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 6fce4361b..4aecf8ecf 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -1,12 +1,18 @@ """ Helpers when analyzing backtest data """ +import logging from pathlib import Path import numpy as np import pandas as pd +import pytz +from freqtrade import persistence from freqtrade.misc import json_load +from freqtrade.persistence import Trade + +logger = logging.getLogger(__name__) # must align with columns in backtest.py BT_DATA_COLUMNS = ["pair", "profitperc", "open_time", "close_time", "index", "duration", @@ -65,3 +71,48 @@ def evaluate_result_multi(results: pd.DataFrame, freq: str, max_open_trades: int df2 = df2.set_index('date') df_final = df2.resample(freq)[['pair']].count() return df_final[df_final['pair'] > max_open_trades] + + +def load_trades(db_url: str = None, exportfilename: str = None) -> pd.DataFrame: + """ + Load trades, either from a DB (using dburl) or via a backtest export file. + :param db_url: Sqlite url (default format sqlite:///tradesv3.dry-run.sqlite) + :param exportfilename: Path to a file exported from backtesting + :returns: Dataframe containing Trades + """ + timeZone = pytz.UTC + + trades: pd.DataFrame = pd.DataFrame([], columns=BT_DATA_COLUMNS) + + if db_url: + persistence.init(db_url, clean_open_orders=False) + columns = ["pair", "profit", "open_time", "close_time", + "open_rate", "close_rate", "duration"] + + for x in Trade.query.all(): + logger.info("date: {}".format(x.open_date)) + + trades = pd.DataFrame([(t.pair, t.calc_profit(), + t.open_date.replace(tzinfo=timeZone), + t.close_date.replace(tzinfo=timeZone) if t.close_date else None, + t.open_rate, t.close_rate, + t.close_date.timestamp() - t.open_date.timestamp() + if t.close_date else None) + for t in Trade.query.all()], + columns=columns) + + elif exportfilename: + + trades = load_backtest_data(Path(exportfilename)) + + return trades + + +def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> pd.DataFrame: + """ + Compare trades and backtested pair DataFrames to get trades performed on backtested period + :return: the DataFrame of a trades of period + """ + trades = trades.loc[(trades['open_time'] >= dataframe.iloc[0]['date']) & + (trades['close_time'] <= dataframe.iloc[-1]['date'])] + return trades diff --git a/freqtrade/plot/__init__.py b/freqtrade/plot/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py new file mode 100644 index 000000000..94c0830bf --- /dev/null +++ b/freqtrade/plot/plotting.py @@ -0,0 +1,221 @@ +import logging +from typing import List + +import pandas as pd +from pathlib import Path + +logger = logging.getLogger(__name__) + + +try: + from plotly import tools + from plotly.offline import plot + import plotly.graph_objs as go +except ImportError: + logger.exception("Module plotly not found \n Please install using `pip install plotly`") + exit(1) + + +def generate_row(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.make_subplots: + """ + Generator all the indicator selected by the user for a specific row + :param fig: Plot figure to append to + :param row: row number for this plot + :param indicators: List of indicators present in the dataframe + :param data: candlestick DataFrame + """ + for indicator in indicators: + if indicator in data: + # TODO: Figure out why scattergl causes problems + scattergl = go.Scatter( + x=data['date'], + y=data[indicator].values, + mode='lines', + name=indicator + ) + fig.append_trace(scattergl, row, 1) + else: + logger.info( + 'Indicator "%s" ignored. Reason: This indicator is not found ' + 'in your strategy.', + indicator + ) + + return fig + + +def plot_trades(fig, trades: pd.DataFrame): + """ + Plot trades to "fig" + """ + # Trades can be empty + if trades is not None and len(trades) > 0: + trade_buys = go.Scatter( + x=trades["open_time"], + y=trades["open_rate"], + mode='markers', + name='trade_buy', + marker=dict( + symbol='square-open', + size=11, + line=dict(width=2), + color='green' + ) + ) + # Create description for sell summarizing the trade + desc = trades.apply(lambda row: f"{round(row['profitperc'], 3)}%, {row['sell_reason']}, " + f"{row['duration']}min", + axis=1) + trade_sells = go.Scatter( + x=trades["close_time"], + y=trades["close_rate"], + text=desc, + mode='markers', + name='trade_sell', + marker=dict( + symbol='square-open', + size=11, + line=dict(width=2), + color='red' + ) + ) + fig.append_trace(trade_buys, 1, 1) + fig.append_trace(trade_sells, 1, 1) + return fig + + +def generate_graph( + pair: str, + data: pd.DataFrame, + trades: pd.DataFrame = None, + indicators1: List[str] = [], + indicators2: List[str] = [], +) -> go.Figure: + """ + Generate the graph from the data generated by Backtesting or from DB + Volume will always be ploted in row2, so Row 1 and 3 are to our disposal for custom indicators + :param pair: Pair to Display on the graph + :param data: OHLCV DataFrame containing indicators and buy/sell signals + :param trades: All trades created + :param indicators1: List containing Main plot indicators + :param indicators2: List containing Sub plot indicators + :return: None + """ + + # Define the graph + fig = tools.make_subplots( + rows=3, + cols=1, + shared_xaxes=True, + row_width=[1, 1, 4], + vertical_spacing=0.0001, + ) + fig['layout'].update(title=pair) + fig['layout']['yaxis1'].update(title='Price') + fig['layout']['yaxis2'].update(title='Volume') + fig['layout']['yaxis3'].update(title='Other') + fig['layout']['xaxis']['rangeslider'].update(visible=False) + + # Common information + candles = go.Candlestick( + x=data.date, + open=data.open, + high=data.high, + low=data.low, + close=data.close, + name='Price' + ) + fig.append_trace(candles, 1, 1) + + if 'buy' in data.columns: + df_buy = data[data['buy'] == 1] + if len(df_buy) > 0: + buys = go.Scatter( + x=df_buy.date, + y=df_buy.close, + mode='markers', + name='buy', + marker=dict( + symbol='triangle-up-dot', + size=9, + line=dict(width=1), + color='green', + ) + ) + fig.append_trace(buys, 1, 1) + else: + logger.warning("No buy-signals found.") + + if 'sell' in data.columns: + df_sell = data[data['sell'] == 1] + if len(df_sell) > 0: + sells = go.Scatter( + x=df_sell.date, + y=df_sell.close, + mode='markers', + name='sell', + marker=dict( + symbol='triangle-down-dot', + size=9, + line=dict(width=1), + color='red', + ) + ) + fig.append_trace(sells, 1, 1) + else: + logger.warning("No sell-signals found.") + + if 'bb_lowerband' in data and 'bb_upperband' in data: + bb_lower = go.Scattergl( + x=data.date, + y=data.bb_lowerband, + name='BB lower', + line={'color': 'rgba(255,255,255,0)'}, + ) + bb_upper = go.Scattergl( + x=data.date, + y=data.bb_upperband, + name='BB upper', + fill="tonexty", + fillcolor="rgba(0,176,246,0.2)", + line={'color': 'rgba(255,255,255,0)'}, + ) + fig.append_trace(bb_lower, 1, 1) + fig.append_trace(bb_upper, 1, 1) + + # Add indicators to main plot + fig = generate_row(fig=fig, row=1, indicators=indicators1, data=data) + + fig = plot_trades(fig, trades) + + # Volume goes to row 2 + volume = go.Bar( + x=data['date'], + y=data['volume'], + name='Volume' + ) + fig.append_trace(volume, 2, 1) + + # Add indicators to seperate row + fig = generate_row(fig=fig, row=3, indicators=indicators2, data=data) + + return fig + + +def generate_plot_file(fig, pair, ticker_interval) -> None: + """ + Generate a plot html file from pre populated fig plotly object + :param fig: Plotly Figure to plot + :param pair: Pair to plot (used as filename and Plot title) + :param ticker_interval: Used as part of the filename + :return: None + """ + logger.info('Generate plot file for %s', pair) + + pair_name = pair.replace("/", "_") + file_name = 'freqtrade-plot-' + pair_name + '-' + ticker_interval + '.html' + + Path("user_data/plots").mkdir(parents=True, exist_ok=True) + + plot(fig, filename=str(Path('user_data/plots').joinpath(file_name)), + auto_open=False) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 034cb5f8b..e956d89c4 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -151,6 +151,11 @@ def patch_coinmarketcap(mocker) -> None: ) +@pytest.fixture(scope='function') +def init_persistence(default_conf): + persistence.init(default_conf['db_url'], default_conf['dry_run']) + + @pytest.fixture(scope="function") def default_conf(): """ Returns validated configuration suitable for most tests """ diff --git a/freqtrade/tests/data/test_btanalysis.py b/freqtrade/tests/data/test_btanalysis.py index dd7cbe0d9..6fa529394 100644 --- a/freqtrade/tests/data/test_btanalysis.py +++ b/freqtrade/tests/data/test_btanalysis.py @@ -1,8 +1,15 @@ -import pytest -from pandas import DataFrame +from unittest.mock import MagicMock -from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data -from freqtrade.data.history import make_testdata_path +from arrow import Arrow +import pytest +from pandas import DataFrame, to_datetime + +from freqtrade.arguments import TimeRange +from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, + extract_trades_of_period, + load_backtest_data, load_trades) +from freqtrade.data.history import load_pair_history, make_testdata_path +from freqtrade.tests.test_persistence import create_mock_trades def test_load_backtest_data(): @@ -19,3 +26,59 @@ def test_load_backtest_data(): with pytest.raises(ValueError, match=r"File .* does not exist\."): load_backtest_data(str("filename") + "nofile") + + +def test_load_trades_file(default_conf, fee, mocker): + # Real testing of load_backtest_data is done in test_load_backtest_data + lbt = mocker.patch("freqtrade.data.btanalysis.load_backtest_data", MagicMock()) + filename = make_testdata_path(None) / "backtest-result_test.json" + load_trades(db_url=None, exportfilename=filename) + assert lbt.call_count == 1 + + +@pytest.mark.usefixtures("init_persistence") +def test_load_trades_db(default_conf, fee, mocker): + + create_mock_trades(fee) + # remove init so it does not init again + init_mock = mocker.patch('freqtrade.persistence.init', MagicMock()) + + trades = load_trades(db_url=default_conf['db_url'], exportfilename=None) + assert init_mock.call_count == 1 + assert len(trades) == 3 + assert isinstance(trades, DataFrame) + assert "pair" in trades.columns + assert "open_time" in trades.columns + + +def test_extract_trades_of_period(): + pair = "UNITTEST/BTC" + timerange = TimeRange(None, 'line', 0, -1000) + + data = load_pair_history(pair=pair, ticker_interval='1m', + datadir=None, timerange=timerange) + + # timerange = 2017-11-14 06:07 - 2017-11-14 22:58:00 + trades = DataFrame( + {'pair': [pair, pair, pair, pair], + 'profit_percent': [0.0, 0.1, -0.2, -0.5], + 'profit_abs': [0.0, 1, -2, -5], + 'open_time': to_datetime([Arrow(2017, 11, 13, 15, 40, 0).datetime, + Arrow(2017, 11, 14, 9, 41, 0).datetime, + Arrow(2017, 11, 14, 14, 20, 0).datetime, + Arrow(2017, 11, 15, 3, 40, 0).datetime, + ], utc=True + ), + 'close_time': to_datetime([Arrow(2017, 11, 13, 16, 40, 0).datetime, + Arrow(2017, 11, 14, 10, 41, 0).datetime, + Arrow(2017, 11, 14, 15, 25, 0).datetime, + Arrow(2017, 11, 15, 3, 55, 0).datetime, + ], utc=True) + }) + trades1 = extract_trades_of_period(data, trades) + # First and last trade are dropped as they are out of range + assert len(trades1) == 2 + assert trades1.iloc[0].open_time == Arrow(2017, 11, 14, 9, 41, 0).datetime + assert trades1.iloc[0].close_time == Arrow(2017, 11, 14, 10, 41, 0).datetime + assert trades1.iloc[-1].open_time == Arrow(2017, 11, 14, 14, 20, 0).datetime + assert trades1.iloc[-1].close_time == Arrow(2017, 11, 14, 15, 25, 0).datetime diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 89c910a28..d9292bdb5 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -187,6 +187,23 @@ def test_download_data_options() -> None: assert args.exchange == 'binance' +def test_plot_dataframe_options() -> None: + args = [ + '--indicators1', 'sma10,sma100', + '--indicators2', 'macd,fastd,fastk', + '--plot-limit', '30', + '-p', 'UNITTEST/BTC', + ] + arguments = Arguments(args, '') + arguments.common_scripts_options() + arguments.plot_dataframe_options() + pargs = arguments.parse_args(True) + assert pargs.indicators1 == "sma10,sma100" + assert pargs.indicators2 == "macd,fastd,fastk" + assert pargs.plot_limit == 30 + assert pargs.pairs == "UNITTEST/BTC" + + def test_check_int_positive() -> None: assert Arguments.check_int_positive("3") == 3 diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index bb00fa8f4..32425ef7b 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -11,9 +11,48 @@ from freqtrade.persistence import Trade, clean_dry_run_db, init from freqtrade.tests.conftest import log_has -@pytest.fixture(scope='function') -def init_persistence(default_conf): - init(default_conf['db_url'], default_conf['dry_run']) +def create_mock_trades(fee): + """ + Create some fake trades ... + """ + # Simulate dry_run entries + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + amount=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + exchange='bittrex', + open_order_id='dry_run_buy_12345' + ) + Trade.session.add(trade) + + trade = Trade( + pair='ETC/BTC', + stake_amount=0.001, + amount=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + exchange='bittrex', + is_open=False, + open_order_id='dry_run_sell_12345' + ) + Trade.session.add(trade) + + # Simulate prod entry + trade = Trade( + pair='ETC/BTC', + stake_amount=0.001, + amount=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + exchange='bittrex', + open_order_id='prod_buy_12345' + ) + Trade.session.add(trade) def test_init_create_session(default_conf): @@ -671,45 +710,7 @@ def test_adjust_min_max_rates(fee): @pytest.mark.usefixtures("init_persistence") def test_get_open(default_conf, fee): - # Simulate dry_run entries - trade = Trade( - pair='ETH/BTC', - stake_amount=0.001, - amount=123.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.123, - exchange='bittrex', - open_order_id='dry_run_buy_12345' - ) - Trade.session.add(trade) - - trade = Trade( - pair='ETC/BTC', - stake_amount=0.001, - amount=123.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.123, - exchange='bittrex', - is_open=False, - open_order_id='dry_run_sell_12345' - ) - Trade.session.add(trade) - - # Simulate prod entry - trade = Trade( - pair='ETC/BTC', - stake_amount=0.001, - amount=123.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.123, - exchange='bittrex', - open_order_id='prod_buy_12345' - ) - Trade.session.add(trade) - + create_mock_trades(fee) assert len(Trade.get_open_trades()) == 2 diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py new file mode 100644 index 000000000..15ab698d8 --- /dev/null +++ b/freqtrade/tests/test_plotting.py @@ -0,0 +1,188 @@ + +from unittest.mock import MagicMock + +from plotly import tools +import plotly.graph_objs as go +from copy import deepcopy + +from freqtrade.arguments import TimeRange +from freqtrade.data import history +from freqtrade.data.btanalysis import load_backtest_data +from freqtrade.plot.plotting import (generate_graph, generate_plot_file, + generate_row, plot_trades) +from freqtrade.strategy.default_strategy import DefaultStrategy +from freqtrade.tests.conftest import log_has, log_has_re + + +def fig_generating_mock(fig, *args, **kwargs): + """ Return Fig - used to mock generate_row and plot_trades""" + return fig + + +def find_trace_in_fig_data(data, search_string: str): + matches = filter(lambda x: x.name == search_string, data) + return next(matches) + + +def generage_empty_figure(): + return tools.make_subplots( + rows=3, + cols=1, + shared_xaxes=True, + row_width=[1, 1, 4], + vertical_spacing=0.0001, + ) + + +def test_generate_row(default_conf, caplog): + pair = "UNITTEST/BTC" + timerange = TimeRange(None, 'line', 0, -1000) + + data = history.load_pair_history(pair=pair, ticker_interval='1m', + datadir=None, timerange=timerange) + indicators1 = ["ema10"] + indicators2 = ["macd"] + + # Generate buy/sell signals and indicators + strat = DefaultStrategy(default_conf) + data = strat.analyze_ticker(data, {'pair': pair}) + fig = generage_empty_figure() + + # Row 1 + fig1 = generate_row(fig=deepcopy(fig), row=1, indicators=indicators1, data=data) + figure = fig1.layout.figure + ema10 = find_trace_in_fig_data(figure.data, "ema10") + assert isinstance(ema10, go.Scatter) + assert ema10.yaxis == "y" + + fig2 = generate_row(fig=deepcopy(fig), row=3, indicators=indicators2, data=data) + figure = fig2.layout.figure + macd = find_trace_in_fig_data(figure.data, "macd") + assert isinstance(macd, go.Scatter) + assert macd.yaxis == "y3" + + # No indicator found + fig3 = generate_row(fig=deepcopy(fig), row=3, indicators=['no_indicator'], data=data) + assert fig == fig3 + assert log_has_re(r'Indicator "no_indicator" ignored\..*', caplog.record_tuples) + + +def test_plot_trades(): + fig1 = generage_empty_figure() + # nothing happens when no trades are available + fig = plot_trades(fig1, None) + assert fig == fig1 + pair = "ADA/BTC" + filename = history.make_testdata_path(None) / "backtest-result_test.json" + trades = load_backtest_data(filename) + trades = trades.loc[trades['pair'] == pair] + + fig = plot_trades(fig, trades) + figure = fig1.layout.figure + + # Check buys - color, should be in first graph, ... + trade_buy = find_trace_in_fig_data(figure.data, "trade_buy") + assert isinstance(trade_buy, go.Scatter) + assert trade_buy.yaxis == 'y' + assert len(trades) == len(trade_buy.x) + assert trade_buy.marker.color == 'green' + + trade_sell = find_trace_in_fig_data(figure.data, "trade_sell") + assert isinstance(trade_sell, go.Scatter) + assert trade_sell.yaxis == 'y' + assert len(trades) == len(trade_sell.x) + assert trade_sell.marker.color == 'red' + + +def test_generate_graph_no_signals_no_trades(default_conf, mocker, caplog): + row_mock = mocker.patch('freqtrade.plot.plotting.generate_row', + MagicMock(side_effect=fig_generating_mock)) + trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades', + MagicMock(side_effect=fig_generating_mock)) + + pair = "UNITTEST/BTC" + timerange = TimeRange(None, 'line', 0, -1000) + data = history.load_pair_history(pair=pair, ticker_interval='1m', + datadir=None, timerange=timerange) + data['buy'] = 0 + data['sell'] = 0 + + indicators1 = [] + indicators2 = [] + fig = generate_graph(pair=pair, data=data, trades=None, + indicators1=indicators1, indicators2=indicators2) + assert isinstance(fig, go.Figure) + assert fig.layout.title.text == pair + figure = fig.layout.figure + + assert len(figure.data) == 2 + # Candlesticks are plotted first + candles = find_trace_in_fig_data(figure.data, "Price") + assert isinstance(candles, go.Candlestick) + + volume = find_trace_in_fig_data(figure.data, "Volume") + assert isinstance(volume, go.Bar) + + assert row_mock.call_count == 2 + assert trades_mock.call_count == 1 + + assert log_has("No buy-signals found.", caplog.record_tuples) + assert log_has("No sell-signals found.", caplog.record_tuples) + + +def test_generate_graph_no_trades(default_conf, mocker): + row_mock = mocker.patch('freqtrade.plot.plotting.generate_row', + MagicMock(side_effect=fig_generating_mock)) + trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades', + MagicMock(side_effect=fig_generating_mock)) + pair = 'UNITTEST/BTC' + timerange = TimeRange(None, 'line', 0, -1000) + data = history.load_pair_history(pair=pair, ticker_interval='1m', + datadir=None, timerange=timerange) + + # Generate buy/sell signals and indicators + strat = DefaultStrategy(default_conf) + data = strat.analyze_ticker(data, {'pair': pair}) + + indicators1 = [] + indicators2 = [] + fig = generate_graph(pair=pair, data=data, trades=None, + indicators1=indicators1, indicators2=indicators2) + assert isinstance(fig, go.Figure) + assert fig.layout.title.text == pair + figure = fig.layout.figure + + assert len(figure.data) == 6 + # Candlesticks are plotted first + candles = find_trace_in_fig_data(figure.data, "Price") + assert isinstance(candles, go.Candlestick) + + volume = find_trace_in_fig_data(figure.data, "Volume") + assert isinstance(volume, go.Bar) + + buy = find_trace_in_fig_data(figure.data, "buy") + assert isinstance(buy, go.Scatter) + # All buy-signals should be plotted + assert int(data.buy.sum()) == len(buy.x) + + sell = find_trace_in_fig_data(figure.data, "sell") + assert isinstance(sell, go.Scatter) + # All buy-signals should be plotted + assert int(data.sell.sum()) == len(sell.x) + + assert find_trace_in_fig_data(figure.data, "BB lower") + assert find_trace_in_fig_data(figure.data, "BB upper") + + assert row_mock.call_count == 2 + assert trades_mock.call_count == 1 + + +def test_generate_plot_file(mocker, caplog): + fig = generage_empty_figure() + plot_mock = mocker.patch("freqtrade.plot.plotting.plot", MagicMock()) + generate_plot_file(fig, "UNITTEST/BTC", "5m") + + assert plot_mock.call_count == 1 + assert plot_mock.call_args[0][0] == fig + assert (plot_mock.call_args_list[0][1]['filename'] + == "user_data/plots/freqtrade-plot-UNITTEST_BTC-5m.html") diff --git a/requirements-dev.txt b/requirements-dev.txt index 21bb899b3..ce05b47f0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,6 @@ # Include all requirements to run the bot. -r requirements.txt +-r requirements-plot.txt flake8==3.7.7 flake8-type-annotations==0.1.0 diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 19a4d3506..4aacc99dd 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -26,128 +26,40 @@ Example of usage: """ import logging import sys -from argparse import Namespace from pathlib import Path from typing import Any, Dict, List import pandas as pd -import plotly.graph_objs as go -import pytz -from plotly import tools -from plotly.offline import plot -from freqtrade import persistence from freqtrade.arguments import Arguments, TimeRange from freqtrade.data import history -from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data -from freqtrade.exchange import Exchange +from freqtrade.data.btanalysis import load_trades, extract_trades_of_period from freqtrade.optimize import setup_configuration -from freqtrade.persistence import Trade -from freqtrade.resolvers import StrategyResolver +from freqtrade.plot.plotting import (generate_graph, + generate_plot_file) +from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.state import RunMode logger = logging.getLogger(__name__) -_CONF: Dict[str, Any] = {} - -timeZone = pytz.UTC -def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFrame: - trades: pd.DataFrame = pd.DataFrame() - if args.db_url: - persistence.init(args.db_url, clean_open_orders=False) - - columns = ["pair", "profit", "open_time", "close_time", - "open_rate", "close_rate", "duration"] - - for x in Trade.query.all(): - print("date: {}".format(x.open_date)) - - trades = pd.DataFrame([(t.pair, t.calc_profit(), - t.open_date.replace(tzinfo=timeZone), - t.close_date.replace(tzinfo=timeZone) if t.close_date else None, - t.open_rate, t.close_rate, - t.close_date.timestamp() - t.open_date.timestamp() - if t.close_date else None) - for t in Trade.query.filter(Trade.pair.is_(pair)).all()], - columns=columns) - - elif args.exportfilename: - - file = Path(args.exportfilename) - if file.exists(): - trades = load_backtest_data(file) - - else: - trades = pd.DataFrame([], columns=BT_DATA_COLUMNS) - - return trades - - -def generate_plot_file(fig, pair, ticker_interval, is_last) -> None: - """ - Generate a plot html file from pre populated fig plotly object - :return: None - """ - logger.info('Generate plot file for %s', pair) - - pair_name = pair.replace("/", "_") - file_name = 'freqtrade-plot-' + pair_name + '-' + ticker_interval + '.html' - - Path("user_data/plots").mkdir(parents=True, exist_ok=True) - - plot(fig, filename=str(Path('user_data/plots').joinpath(file_name)), auto_open=False) - if is_last: - plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html')), auto_open=False) - - -def get_trading_env(args: Namespace): - """ - Initalize freqtrade Exchange and Strategy, split pairs recieved in parameter - :return: Strategy - """ - global _CONF - - # Load the configuration - _CONF.update(setup_configuration(args, RunMode.BACKTEST)) - print(_CONF) - - pairs = args.pairs.split(',') - if pairs is None: - logger.critical('Parameter --pairs mandatory;. E.g --pairs ETH/BTC,XRP/BTC') - exit() - - # Load the strategy - try: - strategy = StrategyResolver(_CONF).strategy - exchange = Exchange(_CONF) - except AttributeError: - logger.critical( - 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', - args.strategy - ) - exit() - - return [strategy, exchange, pairs] - - -def get_tickers_data(strategy, exchange, pairs: List[str], args): +def get_tickers_data(strategy, exchange, pairs: List[str], timerange: TimeRange, + datadir: Path, refresh_pairs: bool, live: bool): """ Get tickers data for each pairs on live or local, option defined in args - :return: dictinnary of tickers. output format: {'pair': tickersdata} + :return: dictionary of tickers. output format: {'pair': tickersdata} """ ticker_interval = strategy.ticker_interval - timerange = Arguments.parse_timerange(args.timerange) tickers = history.load_data( - datadir=Path(str(_CONF.get("datadir"))), + datadir=datadir, pairs=pairs, ticker_interval=ticker_interval, - refresh_pairs=_CONF.get('refresh_pairs', False), + refresh_pairs=refresh_pairs, timerange=timerange, - exchange=Exchange(_CONF), - live=args.live, + exchange=exchange, + live=live, ) # No ticker found, impossible to download, len mismatch @@ -158,7 +70,7 @@ def get_tickers_data(strategy, exchange, pairs: List[str], args): if data.empty: del tickers[pair] logger.info( - 'An issue occured while retreiving datas of %s pair, please retry ' + 'An issue occured while retreiving data of %s pair, please retry ' 'using -l option for live or --refresh-pairs-cached', pair) return tickers @@ -177,172 +89,61 @@ def generate_dataframe(strategy, tickers, pair) -> pd.DataFrame: return dataframe -def extract_trades_of_period(dataframe, trades) -> pd.DataFrame: +def analyse_and_plot_pairs(config: Dict[str, Any]): """ - Compare trades and backtested pair DataFrames to get trades performed on backtested period - :return: the DataFrame of a trades of period - """ - trades = trades.loc[trades['open_time'] >= dataframe.iloc[0]['date']] - return trades - - -def generate_graph( - pair: str, - trades: pd.DataFrame, - data: pd.DataFrame, - indicators1: str, - indicators2: str - ) -> tools.make_subplots: - """ - Generate the graph from the data generated by Backtesting or from DB - :param pair: Pair to Display on the graph - :param trades: All trades created - :param data: Dataframe - :indicators1: String Main plot indicators - :indicators2: String Sub plot indicators + From arguments provided in cli: + -Initialise backtest env + -Get tickers data + -Generate Dafaframes populated with indicators and signals + -Load trades excecuted on same periods + -Generate Plotly plot objects + -Generate plot files :return: None """ + exchange_name = config.get('exchange', {}).get('name').title() + exchange = ExchangeResolver(exchange_name, config).exchange - # Define the graph - fig = tools.make_subplots( - rows=3, - cols=1, - shared_xaxes=True, - row_width=[1, 1, 4], - vertical_spacing=0.0001, - ) - fig['layout'].update(title=pair) - fig['layout']['yaxis1'].update(title='Price') - fig['layout']['yaxis2'].update(title='Volume') - fig['layout']['yaxis3'].update(title='Other') - fig['layout']['xaxis']['rangeslider'].update(visible=False) + strategy = StrategyResolver(config).strategy + if "pairs" in config: + pairs = config["pairs"].split(',') + else: + pairs = config["exchange"]["pair_whitelist"] - # Common information - candles = go.Candlestick( - x=data.date, - open=data.open, - high=data.high, - low=data.low, - close=data.close, - name='Price' - ) + # Set timerange to use + timerange = Arguments.parse_timerange(config["timerange"]) + ticker_interval = strategy.ticker_interval - df_buy = data[data['buy'] == 1] - buys = go.Scattergl( - x=df_buy.date, - y=df_buy.close, - mode='markers', - name='buy', - marker=dict( - symbol='triangle-up-dot', - size=9, - line=dict(width=1), - color='green', + tickers = get_tickers_data(strategy, exchange, pairs, timerange, + datadir=Path(str(config.get("datadir"))), + refresh_pairs=config.get('refresh_pairs', False), + live=config.get("live", False)) + pair_counter = 0 + for pair, data in tickers.items(): + pair_counter += 1 + logger.info("analyse pair %s", pair) + tickers = {} + tickers[pair] = data + dataframe = generate_dataframe(strategy, tickers, pair) + + trades = load_trades(db_url=config["db_url"], + exportfilename=config["exportfilename"]) + trades = trades.loc[trades['pair'] == pair] + trades = extract_trades_of_period(dataframe, trades) + + fig = generate_graph( + pair=pair, + data=dataframe, + trades=trades, + indicators1=config["indicators1"].split(","), + indicators2=config["indicators2"].split(",") ) - ) - df_sell = data[data['sell'] == 1] - sells = go.Scattergl( - x=df_sell.date, - y=df_sell.close, - mode='markers', - name='sell', - marker=dict( - symbol='triangle-down-dot', - size=9, - line=dict(width=1), - color='red', - ) - ) - trade_buys = go.Scattergl( - x=trades["open_time"], - y=trades["open_rate"], - mode='markers', - name='trade_buy', - marker=dict( - symbol='square-open', - size=11, - line=dict(width=2), - color='green' - ) - ) - trade_sells = go.Scattergl( - x=trades["close_time"], - y=trades["close_rate"], - mode='markers', - name='trade_sell', - marker=dict( - symbol='square-open', - size=11, - line=dict(width=2), - color='red' - ) - ) + generate_plot_file(fig, pair, ticker_interval) - # Row 1 - fig.append_trace(candles, 1, 1) - - if 'bb_lowerband' in data and 'bb_upperband' in data: - bb_lower = go.Scatter( - x=data.date, - y=data.bb_lowerband, - name='BB lower', - line={'color': 'rgba(255,255,255,0)'}, - ) - bb_upper = go.Scatter( - x=data.date, - y=data.bb_upperband, - name='BB upper', - fill="tonexty", - fillcolor="rgba(0,176,246,0.2)", - line={'color': 'rgba(255,255,255,0)'}, - ) - fig.append_trace(bb_lower, 1, 1) - fig.append_trace(bb_upper, 1, 1) - - fig = generate_row(fig=fig, row=1, raw_indicators=indicators1, data=data) - fig.append_trace(buys, 1, 1) - fig.append_trace(sells, 1, 1) - fig.append_trace(trade_buys, 1, 1) - fig.append_trace(trade_sells, 1, 1) - - # Row 2 - volume = go.Bar( - x=data['date'], - y=data['volume'], - name='Volume' - ) - fig.append_trace(volume, 2, 1) - - # Row 3 - fig = generate_row(fig=fig, row=3, raw_indicators=indicators2, data=data) - - return fig + logger.info('End of ploting process %s plots generated', pair_counter) -def generate_row(fig, row, raw_indicators, data) -> tools.make_subplots: - """ - Generator all the indicator selected by the user for a specific row - """ - for indicator in raw_indicators.split(','): - if indicator in data: - scattergl = go.Scattergl( - x=data['date'], - y=data[indicator], - name=indicator - ) - fig.append_trace(scattergl, row, 1) - else: - logger.info( - 'Indicator "%s" ignored. Reason: This indicator is not found ' - 'in your strategy.', - indicator - ) - - return fig - - -def plot_parse_args(args: List[str]) -> Namespace: +def plot_parse_args(args: List[str]) -> Dict[str, Any]: """ Parse args passed to the script :param args: Cli arguments @@ -355,49 +156,11 @@ def plot_parse_args(args: List[str]) -> Namespace: arguments.backtesting_options() arguments.common_scripts_options() arguments.plot_dataframe_options() - return arguments.parse_args() + parsed_args = arguments.parse_args() - -def analyse_and_plot_pairs(args: Namespace): - """ - From arguments provided in cli: - -Initialise backtest env - -Get tickers data - -Generate Dafaframes populated with indicators and signals - -Load trades excecuted on same periods - -Generate Plotly plot objects - -Generate plot files - :return: None - """ - strategy, exchange, pairs = get_trading_env(args) - # Set timerange to use - timerange = Arguments.parse_timerange(args.timerange) - ticker_interval = strategy.ticker_interval - - tickers = get_tickers_data(strategy, exchange, pairs, args) - pair_counter = 0 - for pair, data in tickers.items(): - pair_counter += 1 - logger.info("analyse pair %s", pair) - tickers = {} - tickers[pair] = data - dataframe = generate_dataframe(strategy, tickers, pair) - - trades = load_trades(args, pair, timerange) - trades = extract_trades_of_period(dataframe, trades) - - fig = generate_graph( - pair=pair, - trades=trades, - data=dataframe, - indicators1=args.indicators1, - indicators2=args.indicators2 - ) - - is_last = (False, True)[pair_counter == len(tickers)] - generate_plot_file(fig, pair, ticker_interval, is_last) - - logger.info('End of ploting process %s plots generated', pair_counter) + # Load the configuration + config = setup_configuration(parsed_args, RunMode.BACKTEST) + return config def main(sysargv: List[str]) -> None: