From da9c3e7d7d6b2271e3c16503e26f8b2ab6ef6119 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 29 Oct 2017 09:36:03 +0200 Subject: [PATCH 1/5] remove leftover dates from removing date filtering --- freqtrade/analyze.py | 4 ++-- freqtrade/tests/test_analyze.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 3b9a870c3..7151208d9 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -15,7 +15,7 @@ logging.basicConfig(level=logging.DEBUG, logger = logging.getLogger(__name__) -def parse_ticker_dataframe(ticker: list, minimum_date: arrow.Arrow) -> DataFrame: +def parse_ticker_dataframe(ticker: list) -> DataFrame: """ Analyses the trend for the given pair :param pair: pair as str in format BTC_ETH or BTC-ETH @@ -73,7 +73,7 @@ def analyze_ticker(pair: str) -> DataFrame: """ minimum_date = arrow.utcnow().shift(hours=-24) data = get_ticker_history(pair, minimum_date) - dataframe = parse_ticker_dataframe(data['result'], minimum_date) + dataframe = parse_ticker_dataframe(data['result']) if dataframe.empty: logger.warning('Empty dataframe for pair %s', pair) diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index d063de775..0c363e4a3 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -1,7 +1,7 @@ # pragma pylint: disable=missing-docstring +from datetime import datetime import json import pytest -import arrow from pandas import DataFrame from freqtrade.analyze import parse_ticker_dataframe, populate_buy_trend, populate_indicators, \ @@ -12,7 +12,7 @@ def result(): with open('freqtrade/tests/testdata/btc-eth.json') as data_file: data = json.load(data_file) - return parse_ticker_dataframe(data['result'], arrow.get('2017-08-30T10:00:00')) + return parse_ticker_dataframe(data['result']) def test_dataframe_has_correct_columns(result): assert result.columns.tolist() == \ @@ -27,10 +27,10 @@ def test_populates_buy_trend(result): assert 'buy_price' in dataframe.columns def test_returns_latest_buy_signal(mocker): - buydf = DataFrame([{'buy': 1, 'date': arrow.utcnow()}]) + buydf = DataFrame([{'buy': 1, 'date': datetime.today()}]) mocker.patch('freqtrade.analyze.analyze_ticker', return_value=buydf) assert get_buy_signal('BTC-ETH') - buydf = DataFrame([{'buy': 0, 'date': arrow.utcnow()}]) + buydf = DataFrame([{'buy': 0, 'date': datetime.today()}]) mocker.patch('freqtrade.analyze.analyze_ticker', return_value=buydf) assert not get_buy_signal('BTC-ETH') From 54987fd9cacbfad9d88d2499707957a55618380a Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 29 Oct 2017 10:16:53 +0200 Subject: [PATCH 2/5] do date parsing while loading json, not later --- freqtrade/analyze.py | 7 ++++--- freqtrade/tests/test_backtesting.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 7151208d9..399c5cd0b 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -4,7 +4,7 @@ from datetime import timedelta import arrow import talib.abstract as ta -from pandas import DataFrame +from pandas import DataFrame, to_datetime from qtpylib.indicators import awesome_oscillator, crossed_above from freqtrade import exchange @@ -23,8 +23,9 @@ def parse_ticker_dataframe(ticker: list) -> DataFrame: """ df = DataFrame(ticker) \ .drop('BV', 1) \ - .rename(columns={'C':'close', 'V':'volume', 'O':'open', 'H':'high', 'L':'low', 'T':'date'}) \ - .sort_values('date') + .rename(columns={'C':'close', 'V':'volume', 'O':'open', 'H':'high', 'L':'low', 'T':'date'}) + df['date'] = to_datetime(df['date'], utc=True, infer_datetime_format=True) + df.sort_values('date', inplace=True) return df diff --git a/freqtrade/tests/test_backtesting.py b/freqtrade/tests/test_backtesting.py index fa2af3511..4929676f5 100644 --- a/freqtrade/tests/test_backtesting.py +++ b/freqtrade/tests/test_backtesting.py @@ -56,12 +56,12 @@ def backtest(conf, pairs, mocker): for index, row in ticker[ticker.buy == 1].iterrows(): trade = Trade( open_rate=row['close'], - open_date=arrow.get(row['date']).datetime, + open_date=row['date'], amount=1, ) # calculate win/lose forwards from buy point for index2, row2 in ticker[index:].iterrows(): - if should_sell(trade, row2['close'], arrow.get(row2['date']).datetime): + if should_sell(trade, row2['close'], row2['date']): current_profit = (row2['close'] - trade.open_rate) / trade.open_rate trades.append((pair, current_profit, index2 - index)) From 4edf8f2079e63c0078e0fd455d73d3733f2c54a2 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 29 Oct 2017 15:10:18 +0200 Subject: [PATCH 3/5] copy only needed columns before iterating over them --- freqtrade/tests/test_backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/test_backtesting.py b/freqtrade/tests/test_backtesting.py index 4929676f5..c32337f18 100644 --- a/freqtrade/tests/test_backtesting.py +++ b/freqtrade/tests/test_backtesting.py @@ -51,7 +51,7 @@ def backtest(conf, pairs, mocker): mocker.patch('freqtrade.analyze.get_ticker_history', return_value=data) mocker.patch('arrow.utcnow', return_value=arrow.get('2017-08-20T14:50:00')) - ticker = analyze_ticker(pair) + ticker = analyze_ticker(pair)[['close', 'date', 'buy']].copy() # for each buy point for index, row in ticker[ticker.buy == 1].iterrows(): trade = Trade( From 3b1dc36d8a42da913cf5ef764df651953f41996c Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 29 Oct 2017 15:13:02 +0200 Subject: [PATCH 4/5] switch to using itertuples instead of iterrows as it's a lot faster --- freqtrade/tests/test_backtesting.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/freqtrade/tests/test_backtesting.py b/freqtrade/tests/test_backtesting.py index c32337f18..29a9627a3 100644 --- a/freqtrade/tests/test_backtesting.py +++ b/freqtrade/tests/test_backtesting.py @@ -53,18 +53,14 @@ def backtest(conf, pairs, mocker): mocker.patch('arrow.utcnow', return_value=arrow.get('2017-08-20T14:50:00')) ticker = analyze_ticker(pair)[['close', 'date', 'buy']].copy() # for each buy point - for index, row in ticker[ticker.buy == 1].iterrows(): - trade = Trade( - open_rate=row['close'], - open_date=row['date'], - amount=1, - ) + for row in ticker[ticker.buy == 1].itertuples(index=True): + trade = Trade(open_rate=row.close, open_date=row.date, amount=1) # calculate win/lose forwards from buy point - for index2, row2 in ticker[index:].iterrows(): - if should_sell(trade, row2['close'], row2['date']): - current_profit = (row2['close'] - trade.open_rate) / trade.open_rate + for row2 in ticker[row.Index:].itertuples(index=True): + if should_sell(trade, row2.close, row2.date): + current_profit = (row2.close - trade.open_rate) / trade.open_rate - trades.append((pair, current_profit, index2 - index)) + trades.append((pair, current_profit, row2.Index - row.Index)) break labels = ['currency', 'profit', 'duration'] results = DataFrame.from_records(trades, columns=labels) From 9b00fc34748115475e8adafcf013f53039839469 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 29 Oct 2017 16:16:23 +0200 Subject: [PATCH 5/5] use .ix instead of .loc for small perf boost --- freqtrade/analyze.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 399c5cd0b..1eeb8408d 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -54,14 +54,14 @@ def populate_buy_trend(dataframe: DataFrame) -> DataFrame: :param dataframe: DataFrame :return: DataFrame with buy column """ - dataframe.loc[ + dataframe.ix[ (dataframe['close'] < dataframe['sma']) & (dataframe['tema'] <= dataframe['blower']) & (dataframe['mfi'] < 25) & (dataframe['fastd'] < 25) & (dataframe['adx'] > 30), 'buy'] = 1 - dataframe.loc[dataframe['buy'] == 1, 'buy_price'] = dataframe['close'] + dataframe.ix[dataframe['buy'] == 1, 'buy_price'] = dataframe['close'] return dataframe