From d3fb2e4516e245ac8feff0b959470449e58f0eaa Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Wed, 25 Apr 2018 20:16:36 +0200 Subject: [PATCH 01/12] Add open_rate_requested and close_rate_requested for slippage detection --- freqtrade/freqtradebot.py | 2 ++ freqtrade/persistence.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0a332b952..56ace2ae2 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -330,6 +330,7 @@ class FreqtradeBot(object): fee_open=fee, fee_close=fee, open_rate=buy_limit, + open_rate_requested=buy_limit, open_date=datetime.utcnow(), exchange=exchange.get_id(), open_order_id=order_id @@ -538,6 +539,7 @@ class FreqtradeBot(object): # Execute sell and update trade record order_id = exchange.sell(str(trade.pair), limit, trade.amount)['id'] trade.open_order_id = order_id + trade.close_rate_requested = limit fmt_exp_profit = round(trade.calc_profit_percent(rate=limit) * 100, 2) profit_trade = trade.calc_profit(rate=limit) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index ed81ad2ec..1da551c7d 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -88,7 +88,9 @@ class Trade(_DECL_BASE): fee_open = Column(Float, nullable=False, default=0.0) fee_close = Column(Float, nullable=False, default=0.0) open_rate = Column(Float) + open_rate_requested = Column(Float) close_rate = Column(Float) + close_rate_requested = Column(Float) close_profit = Column(Float) stake_amount = Column(Float, nullable=False) amount = Column(Float) From ccf1c894b459b03a9b7a6272b9c0079afd57df0c Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Sun, 6 May 2018 09:09:53 +0200 Subject: [PATCH 02/12] Inital try mirate --- freqtrade/persistence.py | 48 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 1da551c7d..68ec923ea 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -15,6 +15,8 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker from sqlalchemy.pool import StaticPool +from sqlalchemy import inspect + logger = logging.getLogger(__name__) @@ -50,12 +52,58 @@ def init(config: dict, engine: Optional[Engine] = None) -> None: Trade.session = session() Trade.query = session.query_property() _DECL_BASE.metadata.create_all(engine) + check_migrate(engine) # Clean dry_run DB if _CONF.get('dry_run', False) and _CONF.get('dry_run_db', False): clean_dry_run_db() +def has_column(columns, searchname: str) -> bool: + return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1 + + +def check_migrate(engine) -> None: + """ + Checks if migration is necessary and migrates if necessary + """ + inspector = inspect(engine) + + cols = inspector.get_columns('trades') + + if not has_column(cols, 'fee_open'): + # Schema migration necessary + engine.execute("create table trades_bak as select * from trades;") + + # let SQLAlchemy create the schema as required + _DECL_BASE.metadata.create_all(engine) + + # Copy data back - following the correct schema + engine.execute("""insert into trades + (id, exchange, pair, is_open, fee_open, fee_close, open_rate, + open_rate_requested, close_rate, close_rate_requested, close_profit, + stake_amount, amount, open_date, close_date, open_order_id) + select id, exchange, pair, is_open, fee fee_open, fee fee_close, + open_rate, null open_rate_requested, close_rate, + null close_rate_requested, close_profit, + stake_amount, amount, open_date, close_date, open_order_id + from trades_bak + """) + + # engine.execute("alter table trades add fee_open float not null") + # engine.execute("alter table trades add fee_close float not null") + # # Update fee_open and fee_close with "fee" column values + # engine.execute("update trades set fee_open = fee, fee_close = fee where fee_open is null") + # engine.execute("alter table trades drop fee_close float not null") + + if not has_column(cols, 'open_rate_requested'): + engine.execute("alter table trades add open_rate_requested float") + if not has_column(cols, 'close_rate_requested'): + engine.execute("alter table trades add close_rate_requested float") + + + + def cleanup() -> None: """ Flushes all pending operations to disk. From ab4e2bd5a9b4dd2c377b154b502d6ddb54033024 Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Sat, 12 May 2018 10:04:41 +0200 Subject: [PATCH 03/12] Fix migrate script --- freqtrade/persistence.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 68ec923ea..7170ab4ff 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -73,8 +73,9 @@ def check_migrate(engine) -> None: if not has_column(cols, 'fee_open'): # Schema migration necessary - engine.execute("create table trades_bak as select * from trades;") - + engine.execute("drop table if exists trades_bak") + engine.execute("create table trades_bak as select * from trades") + engine.execute("drop table if exists trades") # let SQLAlchemy create the schema as required _DECL_BASE.metadata.create_all(engine) @@ -90,11 +91,9 @@ def check_migrate(engine) -> None: from trades_bak """) - # engine.execute("alter table trades add fee_open float not null") - # engine.execute("alter table trades add fee_close float not null") - # # Update fee_open and fee_close with "fee" column values - # engine.execute("update trades set fee_open = fee, fee_close = fee where fee_open is null") - # engine.execute("alter table trades drop fee_close float not null") + # Reread columns - the above recreated the table! + inspector = inspect(engine) + cols = inspector.get_columns('trades') if not has_column(cols, 'open_rate_requested'): engine.execute("alter table trades add open_rate_requested float") @@ -102,8 +101,6 @@ def check_migrate(engine) -> None: engine.execute("alter table trades add close_rate_requested float") - - def cleanup() -> None: """ Flushes all pending operations to disk. From 81ee6f82658ac5116ce4488174f5e283dbc89bd5 Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Sat, 12 May 2018 10:19:52 +0200 Subject: [PATCH 04/12] Update sql docs to new schema --- docs/sql_cheatsheet.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/sql_cheatsheet.md b/docs/sql_cheatsheet.md index 065f264f1..141eaeafe 100644 --- a/docs/sql_cheatsheet.md +++ b/docs/sql_cheatsheet.md @@ -32,9 +32,12 @@ CREATE TABLE trades ( exchange VARCHAR NOT NULL, pair VARCHAR NOT NULL, is_open BOOLEAN NOT NULL, - fee FLOAT NOT NULL, + fee_open FLOAT NOT NULL, + fee_close FLOAT NOT NULL, open_rate FLOAT, + open_rate_requested FLOAT, close_rate FLOAT, + close_rate_requested FLOAT, close_profit FLOAT, stake_amount FLOAT NOT NULL, amount FLOAT, @@ -71,13 +74,13 @@ WHERE id=31; ```sql INSERT -INTO trades (exchange, pair, is_open, fee, open_rate, stake_amount, amount, open_date) -VALUES ('BITTREX', 'BTC_', 1, 0.0025, , , , '') +INTO trades (exchange, pair, is_open, fee_open, fee_close, open_rate, stake_amount, amount, open_date) +VALUES ('BITTREX', 'BTC_', 1, 0.0025, 0.0025, , , , '') ``` **Example:** ```sql -INSERT INTO trades (exchange, pair, is_open, fee, open_rate, stake_amount, amount, open_date) VALUES ('BITTREX', 'BTC_ETC', 1, 0.0025, 0.00258580, 0.002, 0.7715262081, '2017-11-28 12:44:24.000000') +INSERT INTO trades (exchange, pair, is_open, fee_open, fee_close, open_rate, stake_amount, amount, open_date) VALUES ('BITTREX', 'BTC_ETC', 1, 0.0025, 0.0025, 0.00258580, 0.002, 0.7715262081, '2017-11-28 12:44:24.000000') ``` ## Fix wrong fees in the table @@ -87,4 +90,4 @@ If your DB was created before ```sql UPDATE trades SET fee=0.0025 WHERE fee=0.005; -``` \ No newline at end of file +``` From f5ff6ceead76c07779e27126caf76621c12e1a11 Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Sat, 12 May 2018 10:29:10 +0200 Subject: [PATCH 05/12] Rename instead of drop/create --- freqtrade/persistence.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 7170ab4ff..c6a185188 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -73,9 +73,7 @@ def check_migrate(engine) -> None: if not has_column(cols, 'fee_open'): # Schema migration necessary - engine.execute("drop table if exists trades_bak") - engine.execute("create table trades_bak as select * from trades") - engine.execute("drop table if exists trades") + engine.execute("alter table trades rename to trades_bak") # let SQLAlchemy create the schema as required _DECL_BASE.metadata.create_all(engine) From 49266fc4b8a9478700ec7040d7fb153035951fdc Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Sat, 12 May 2018 10:29:26 +0200 Subject: [PATCH 06/12] Add migration test --- freqtrade/tests/test_persistence.py | 39 ++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index db8a5e9bd..8c3ffeb6a 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -4,7 +4,7 @@ import os import pytest from sqlalchemy import create_engine -from freqtrade.persistence import Trade, init, clean_dry_run_db +from freqtrade.persistence import Trade, init, clean_dry_run_db, check_migrate @pytest.fixture(scope='function') @@ -375,3 +375,40 @@ def test_clean_dry_run_db(default_conf, fee): # We have now only the prod assert len(Trade.query.filter(Trade.open_order_id.isnot(None)).all()) == 1 + + +def test_migrate(default_conf, fee): + create_table_old = """CREATE TABLE IF NOT EXISTS "trades" ( + id INTEGER NOT NULL, + exchange VARCHAR NOT NULL, + pair VARCHAR NOT NULL, + is_open BOOLEAN NOT NULL, + fee FLOAT NOT NULL, + open_rate FLOAT, + close_rate FLOAT, + close_profit FLOAT, + stake_amount FLOAT NOT NULL, + amount FLOAT, + open_date DATETIME NOT NULL, + close_date DATETIME, + open_order_id VARCHAR, + PRIMARY KEY (id), + CHECK (is_open IN (0, 1)) + );""" + insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee, + open_rate, stake_amount, amount, open_date) + VALUES ('BITTREX', 'BTC_ETC', 1, {}, + 0.00258580, 0.002, 0.7715262081, + '2017-11-28 12:44:24.000000')""".format(fee.return_value) + engine = create_engine('sqlite://') + engine.execute(create_table_old) + engine.execute(insert_table_old) + init(default_conf, engine) + assert len(Trade.query.filter(Trade.id == 1).all()) == 1 + trade = Trade.query.filter(Trade.id == 1).first() + assert trade.fee_open == fee.return_value + assert trade.fee_close == fee.return_value + assert trade.open_rate_requested is None + assert trade.close_rate_requested is None + + From ada98abfee3f674a21d62344b25d077886f6fa26 Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Sat, 12 May 2018 10:30:30 +0200 Subject: [PATCH 07/12] fix flake --- freqtrade/tests/test_persistence.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 8c3ffeb6a..26c2e385a 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -4,7 +4,7 @@ import os import pytest from sqlalchemy import create_engine -from freqtrade.persistence import Trade, init, clean_dry_run_db, check_migrate +from freqtrade.persistence import Trade, init, clean_dry_run_db @pytest.fixture(scope='function') @@ -410,5 +410,3 @@ def test_migrate(default_conf, fee): assert trade.fee_close == fee.return_value assert trade.open_rate_requested is None assert trade.close_rate_requested is None - - From 8e3ff8235fb673171dc20061648146de0ecb08ac Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Sat, 12 May 2018 10:31:24 +0200 Subject: [PATCH 08/12] add explaining comments --- freqtrade/tests/test_persistence.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 26c2e385a..90cf7dad5 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -401,9 +401,12 @@ def test_migrate(default_conf, fee): 0.00258580, 0.002, 0.7715262081, '2017-11-28 12:44:24.000000')""".format(fee.return_value) engine = create_engine('sqlite://') + # Create table using the old format engine.execute(create_table_old) engine.execute(insert_table_old) + # Run init to test migration init(default_conf, engine) + assert len(Trade.query.filter(Trade.id == 1).all()) == 1 trade = Trade.query.filter(Trade.id == 1).first() assert trade.fee_open == fee.return_value From 631081a2b2b7e2234bd6e0bb90ab4ced78ace31e Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Sat, 12 May 2018 10:37:17 +0200 Subject: [PATCH 09/12] Add additional tests --- freqtrade/tests/test_persistence.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 90cf7dad5..27a09fa5a 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -378,6 +378,7 @@ def test_clean_dry_run_db(default_conf, fee): def test_migrate(default_conf, fee): + amount = 103.223 create_table_old = """CREATE TABLE IF NOT EXISTS "trades" ( id INTEGER NOT NULL, exchange VARCHAR NOT NULL, @@ -397,9 +398,13 @@ def test_migrate(default_conf, fee): );""" insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee, open_rate, stake_amount, amount, open_date) - VALUES ('BITTREX', 'BTC_ETC', 1, {}, - 0.00258580, 0.002, 0.7715262081, - '2017-11-28 12:44:24.000000')""".format(fee.return_value) + VALUES ('BITTREX', 'BTC_ETC', 1, {fee}, + 0.00258580, {stake}, {amount}, + '2017-11-28 12:44:24.000000') + """.format(fee=fee.return_value, + stake=default_conf.get("stake_amount"), + amount=amount + ) engine = create_engine('sqlite://') # Create table using the old format engine.execute(create_table_old) @@ -413,3 +418,6 @@ def test_migrate(default_conf, fee): assert trade.fee_close == fee.return_value assert trade.open_rate_requested is None assert trade.close_rate_requested is None + assert trade.is_open == 1 + assert trade.amount == amount + assert trade.stake_amount == default_conf.get("stake_amount") From 40c581e5a8426aeba04c7a91bfdff8f6f7841e6c Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Sat, 12 May 2018 13:37:42 +0200 Subject: [PATCH 10/12] Convert pair-format to new format --- freqtrade/persistence.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index c6a185188..6c89fd737 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -82,7 +82,15 @@ def check_migrate(engine) -> None: (id, exchange, pair, is_open, fee_open, fee_close, open_rate, open_rate_requested, close_rate, close_rate_requested, close_profit, stake_amount, amount, open_date, close_date, open_order_id) - select id, exchange, pair, is_open, fee fee_open, fee fee_close, + select id, exchange, + case + when instr(pair, '_') != 0 then + substr(pair, instr(pair, '_') + 1) || '/' || + substr(pair, 1, instr(pair, '_') - 1) + else pair + end + pair, + is_open, fee fee_open, fee fee_close, open_rate, null open_rate_requested, close_rate, null close_rate_requested, close_profit, stake_amount, amount, open_date, close_date, open_order_id From e3ae1c6c2f974fc90793fc961b729a8a395d4ff7 Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Sat, 12 May 2018 13:39:16 +0200 Subject: [PATCH 11/12] Convert exchange-name to new format --- freqtrade/persistence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 6c89fd737..2d497662e 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -82,7 +82,7 @@ def check_migrate(engine) -> None: (id, exchange, pair, is_open, fee_open, fee_close, open_rate, open_rate_requested, close_rate, close_rate_requested, close_profit, stake_amount, amount, open_date, close_date, open_order_id) - select id, exchange, + select id, lower(exchange), case when instr(pair, '_') != 0 then substr(pair, instr(pair, '_') + 1) || '/' || From 58425993daa73558bc13abccf76a6d9458d6e79a Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Sat, 12 May 2018 13:39:29 +0200 Subject: [PATCH 12/12] Adapt tests to verify pair-conversion and exchange conversion --- freqtrade/tests/test_persistence.py | 58 ++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 27a09fa5a..3e0f50fbb 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -377,7 +377,10 @@ def test_clean_dry_run_db(default_conf, fee): assert len(Trade.query.filter(Trade.open_order_id.isnot(None)).all()) == 1 -def test_migrate(default_conf, fee): +def test_migrate_old(default_conf, fee): + """ + Test Database migration(starting with old pairformat) + """ amount = 103.223 create_table_old = """CREATE TABLE IF NOT EXISTS "trades" ( id INTEGER NOT NULL, @@ -421,3 +424,56 @@ def test_migrate(default_conf, fee): assert trade.is_open == 1 assert trade.amount == amount assert trade.stake_amount == default_conf.get("stake_amount") + assert trade.pair == "ETC/BTC" + assert trade.exchange == "bittrex" + + +def test_migrate_new(default_conf, fee): + """ + Test Database migration (starting with new pairformat) + """ + amount = 103.223 + create_table_old = """CREATE TABLE IF NOT EXISTS "trades" ( + id INTEGER NOT NULL, + exchange VARCHAR NOT NULL, + pair VARCHAR NOT NULL, + is_open BOOLEAN NOT NULL, + fee FLOAT NOT NULL, + open_rate FLOAT, + close_rate FLOAT, + close_profit FLOAT, + stake_amount FLOAT NOT NULL, + amount FLOAT, + open_date DATETIME NOT NULL, + close_date DATETIME, + open_order_id VARCHAR, + PRIMARY KEY (id), + CHECK (is_open IN (0, 1)) + );""" + insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee, + open_rate, stake_amount, amount, open_date) + VALUES ('binance', 'ETC/BTC', 1, {fee}, + 0.00258580, {stake}, {amount}, + '2019-11-28 12:44:24.000000') + """.format(fee=fee.return_value, + stake=default_conf.get("stake_amount"), + amount=amount + ) + engine = create_engine('sqlite://') + # Create table using the old format + engine.execute(create_table_old) + engine.execute(insert_table_old) + # Run init to test migration + init(default_conf, engine) + + assert len(Trade.query.filter(Trade.id == 1).all()) == 1 + trade = Trade.query.filter(Trade.id == 1).first() + assert trade.fee_open == fee.return_value + assert trade.fee_close == fee.return_value + assert trade.open_rate_requested is None + assert trade.close_rate_requested is None + assert trade.is_open == 1 + assert trade.amount == amount + assert trade.stake_amount == default_conf.get("stake_amount") + assert trade.pair == "ETC/BTC" + assert trade.exchange == "binance"