diff --git a/config_full.json.example b/config_full.json.example index c6b229ea3..5789e49ac 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -119,7 +119,8 @@ "initial_state": "running", "forcebuy_enable": false, "internals": { - "process_throttle_secs": 5 + "process_throttle_secs": 5, + "heartbeat_interval": 60 }, "strategy": "DefaultStrategy", "strategy_path": "user_data/strategies/" diff --git a/docs/advanced-setup.md b/docs/advanced-setup.md new file mode 100644 index 000000000..e6334d2c1 --- /dev/null +++ b/docs/advanced-setup.md @@ -0,0 +1,33 @@ +# Advanced Post-installation Tasks + +This page explains some advanced tasks and configuration options that can be performed after the bot installation and may be uselful in some environments. + +If you do not know what things mentioned here mean, you probably do not need it. + +## Configure the bot running as a systemd service + +Copy the `freqtrade.service` file to your systemd user directory (usually `~/.config/systemd/user`) and update `WorkingDirectory` and `ExecStart` to match your setup. + +After that you can start the daemon with: + +```bash +systemctl --user start freqtrade +``` + +For this to be persistent (run when user is logged out) you'll need to enable `linger` for your freqtrade user. + +```bash +sudo loginctl enable-linger "$USER" +``` + +If you run the bot as a service, you can use systemd service manager as a software watchdog monitoring freqtrade bot +state and restarting it in the case of failures. If the `internals.sd_notify` parameter is set to true in the +configuration or the `--sd-notify` command line option is used, the bot will send keep-alive ping messages to systemd +using the sd_notify (systemd notifications) protocol and will also tell systemd its current state (Running or Stopped) +when it changes. + +The `freqtrade.service.watchdog` file contains an example of the service unit configuration file which uses systemd +as the watchdog. + +!!! Note + The sd_notify communication between the bot and the systemd service manager will not work if the bot runs in a Docker container. diff --git a/docs/configuration.md b/docs/configuration.md index 0eff4da88..ff40b1750 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -98,6 +98,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `strategy` | DefaultStrategy | Defines Strategy class to use. | `strategy_path` | null | Adds an additional strategy lookup path (must be a directory). | `internals.process_throttle_secs` | 5 | **Required.** Set the process throttle. Value in second. +| `internals.heartbeat_interval` | 60 | Print heartbeat message every X seconds. Set to 0 to disable heartbeat messages. | `internals.sd_notify` | false | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details. | `logfile` | | Specify Logfile. Uses a rolling strategy of 10 files, with 1Mb per file. | `user_data_dir` | cwd()/user_data | Directory containing user data. Defaults to `./user_data/`. @@ -330,7 +331,7 @@ This configuration enables binance, as well as rate limiting to avoid bans from Optimal settings for rate limiting depend on the exchange and the size of the whitelist, so an ideal parameter will vary on many other settings. We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step. -#### Advanced FreqTrade Exchange configuration +#### Advanced Freqtrade Exchange configuration Advanced options can be configured using the `_ft_has_params` setting, which will override Defaults and exchange-specific behaviours. @@ -350,6 +351,13 @@ For example, to test the order type `FOK` with Kraken, and modify candle_limit t !!! Warning Please make sure to fully understand the impacts of these settings before modifying them. +#### Random notes for other exchanges + +* The Ocean (ccxt id: 'theocean') exchange uses Web3 functionality and requires web3 package to be installed: +```shell +$ pip3 install web3 +``` + ### What values can be used for fiat_display_currency? The `fiat_display_currency` configuration parameter sets the base currency to use for the diff --git a/docs/installation.md b/docs/installation.md index 68348d4b0..fcbce571e 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -151,7 +151,7 @@ python3 -m venv .env source .env/bin/activate ``` -#### 3. Install FreqTrade +#### 3. Install Freqtrade Clone the git repository: @@ -192,33 +192,9 @@ freqtrade -c config.json *Note*: If you run the bot on a server, you should consider using [Docker](docker.md) or a terminal multiplexer like `screen` or [`tmux`](https://en.wikipedia.org/wiki/Tmux) to avoid that the bot is stopped on logout. -#### 7. [Optional] Configure `freqtrade` as a `systemd` service +#### 7. (Optional) Post-installation Tasks -From the freqtrade repo... copy `freqtrade.service` to your systemd user directory (usually `~/.config/systemd/user`) and update `WorkingDirectory` and `ExecStart` to match your setup. - -After that you can start the daemon with: - -```bash -systemctl --user start freqtrade -``` - -For this to be persistent (run when user is logged out) you'll need to enable `linger` for your freqtrade user. - -```bash -sudo loginctl enable-linger "$USER" -``` - -If you run the bot as a service, you can use systemd service manager as a software watchdog monitoring freqtrade bot -state and restarting it in the case of failures. If the `internals.sd_notify` parameter is set to true in the -configuration or the `--sd-notify` command line option is used, the bot will send keep-alive ping messages to systemd -using the sd_notify (systemd notifications) protocol and will also tell systemd its current state (Running or Stopped) -when it changes. - -The `freqtrade.service.watchdog` file contains an example of the service unit configuration file which uses systemd -as the watchdog. - -!!! Note - The sd_notify communication between the bot and the systemd service manager will not work if the bot runs in a Docker container. +On Linux, as an optional post-installation task, you can setup the bot to run as a `systemd` service. See [Advanced Post-installation Tasks](advanced-setup.md) for details. ------ diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ed5116b08..7251715a7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -6,26 +6,27 @@ import logging import traceback from datetime import datetime from math import isclose +from os import getpid from typing import Any, Dict, List, Optional, Tuple import arrow from requests.exceptions import RequestException -from freqtrade import (DependencyException, InvalidOrderException, - __version__, constants, persistence) +from freqtrade import (DependencyException, InvalidOrderException, __version__, + constants, persistence) +from freqtrade.configuration import validate_config_consistency from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge -from freqtrade.configuration import validate_config_consistency from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date from freqtrade.persistence import Trade +from freqtrade.resolvers import (ExchangeResolver, PairListResolver, + StrategyResolver) from freqtrade.rpc import RPCManager, RPCMessageType -from freqtrade.resolvers import ExchangeResolver, StrategyResolver, PairListResolver from freqtrade.state import State -from freqtrade.strategy.interface import SellType, IStrategy +from freqtrade.strategy.interface import IStrategy, SellType from freqtrade.wallets import Wallets - logger = logging.getLogger(__name__) @@ -50,13 +51,15 @@ class FreqtradeBot: # Init objects self.config = config + self._heartbeat_msg = 0 + + self.heartbeat_interval = self.config.get('internals', {}).get('heartbeat_interval', 60) + self.strategy: IStrategy = StrategyResolver(self.config).strategy # Check config consistency here since strategies can set certain options validate_config_consistency(config) - self.rpc: RPCManager = RPCManager(self) - self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange self.wallets = Wallets(self.config, self.exchange) @@ -83,6 +86,13 @@ class FreqtradeBot: initial_state = self.config.get('initial_state') self.state = State[initial_state.upper()] if initial_state else State.STOPPED + # RPC runs in separate threads, can start handling external commands just after + # initialization, even before Freqtradebot has a chance to start its throttling, + # so anything in the Freqtradebot instance should be ready (initialized), including + # the initial state of the bot. + # Keep this at the end of this initialization method. + self.rpc: RPCManager = RPCManager(self) + def cleanup(self) -> None: """ Cleanup pending resources on an already stopped bot @@ -145,6 +155,11 @@ class FreqtradeBot: self.check_handle_timedout() Trade.session.flush() + if (self.heartbeat_interval + and (arrow.utcnow().timestamp - self._heartbeat_msg > self.heartbeat_interval)): + logger.info(f"Bot heartbeat. PID={getpid()}") + self._heartbeat_msg = arrow.utcnow().timestamp + def _extend_whitelist_with_trades(self, whitelist: List[str], trades: List[Any]): """ Extend whitelist with pairs from open trades @@ -438,7 +453,7 @@ class FreqtradeBot: try: # Create entity and execute trade if not self.create_trades(): - logger.info('Found no buy signals for whitelisted currencies. Trying again...') + logger.debug('Found no buy signals for whitelisted currencies. Trying again...') except DependencyException as exception: logger.warning('Unable to create trade: %s', exception) diff --git a/mkdocs.yml b/mkdocs.yml index 863731873..2c3f70191 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -22,6 +22,7 @@ nav: - Strategy analysis: strategy_analysis_example.md - Plotting: plotting.md - SQL Cheatsheet: sql_cheatsheet.md + - Advanced Post-installation Tasks: advanced-setup.md - Sandbox Testing: sandbox-testing.md - Deprecated Features: deprecated.md - Contributors Guide: developer.md diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py deleted file mode 100755 index a8f919a10..000000000 --- a/scripts/download_backtest_data.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python3 - -import sys - - -print("This script has been integrated into freqtrade " - "and its functionality is available by calling `freqtrade download-data`.") -print("Please check the documentation on https://www.freqtrade.io/en/latest/backtesting/ " - "for details.") - -sys.exit(1) diff --git a/scripts/get_market_pairs.py b/scripts/get_market_pairs.py deleted file mode 100644 index cd38bf2fa..000000000 --- a/scripts/get_market_pairs.py +++ /dev/null @@ -1,103 +0,0 @@ -""" -This script was adapted from ccxt here: -https://github.com/ccxt/ccxt/blob/master/examples/py/arbitrage-pairs.py -""" -import os -import sys -import traceback - -root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -sys.path.append(root + '/python') - -import ccxt # noqa: E402 - - -def style(s, style): - return style + s + '\033[0m' - - -def green(s): - return style(s, '\033[92m') - - -def blue(s): - return style(s, '\033[94m') - - -def yellow(s): - return style(s, '\033[93m') - - -def red(s): - return style(s, '\033[91m') - - -def pink(s): - return style(s, '\033[95m') - - -def bold(s): - return style(s, '\033[1m') - - -def underline(s): - return style(s, '\033[4m') - - -def dump(*args): - print(' '.join([str(arg) for arg in args])) - - -def print_supported_exchanges(): - dump('Supported exchanges:', green(', '.join(ccxt.exchanges))) - - -try: - - if len(sys.argv) < 2: - dump("Usage: python " + sys.argv[0], green('id')) - print_supported_exchanges() - sys.exit(1) - - id = sys.argv[1] # get exchange id from command line arguments - - # check if the exchange is supported by ccxt - exchange_found = id in ccxt.exchanges - - if exchange_found: - dump('Instantiating', green(id), 'exchange') - - # instantiate the exchange by id - exchange = getattr(ccxt, id)({ - # 'proxy':'https://cors-anywhere.herokuapp.com/', - }) - - # load all markets from the exchange - markets = exchange.load_markets() - - # output a list of all market symbols - dump(green(id), 'has', len(exchange.symbols), 'symbols:', exchange.symbols) - - tuples = list(ccxt.Exchange.keysort(markets).items()) - - # debug - for (k, v) in tuples: - print(v) - - # output a table of all markets - dump(pink('{:<15} {:<15} {:<15} {:<15}'.format('id', 'symbol', 'base', 'quote'))) - - for (k, v) in tuples: - dump('{:<15} {:<15} {:<15} {:<15}'.format(v['id'], v['symbol'], v['base'], v['quote'])) - - else: - - dump('Exchange ' + red(id) + ' not found') - print_supported_exchanges() - -except Exception as e: - dump('[' + type(e).__name__ + ']', str(e)) - dump(traceback.format_exc()) - dump("Usage: python " + sys.argv[0], green('id')) - print_supported_exchanges() - sys.exit(1) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py deleted file mode 100755 index 62c4bc39f..000000000 --- a/scripts/plot_dataframe.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python3 - -import sys - - -print("This script has been integrated into freqtrade " - "and its functionality is available by calling `freqtrade plot-dataframe`.") -print("Please check the documentation on https://www.freqtrade.io/en/latest/plotting/ " - "for details.") - -sys.exit(1) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py deleted file mode 100755 index c9a23c1ee..000000000 --- a/scripts/plot_profit.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python3 - -import sys - - -print("This script has been integrated into freqtrade " - "and its functionality is available by calling `freqtrade plot-profit`.") -print("Please check the documentation on https://www.freqtrade.io/en/latest/plotting/ " - "for details.") - -sys.exit(1) diff --git a/scripts/pytest.sh b/tests/pytest.sh similarity index 100% rename from scripts/pytest.sh rename to tests/pytest.sh diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index f1533d867..8aefaba17 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1521,6 +1521,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, def test_process_maybe_execute_buys(mocker, default_conf, caplog) -> None: + caplog.set_level(logging.DEBUG) freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trades', MagicMock(return_value=False)) @@ -3647,3 +3648,27 @@ def test_startup_trade_reinit(default_conf, edge_conf, mocker): ftbot = get_patched_freqtradebot(mocker, edge_conf) ftbot.startup() assert reinit_mock.call_count == 0 + + +def test_process_i_am_alive(default_conf, mocker, caplog): + patch_RPCManager(mocker) + patch_exchange(mocker) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + + ftbot = get_patched_freqtradebot(mocker, default_conf) + message = r"Bot heartbeat\. PID=.*" + ftbot.process() + assert log_has_re(message, caplog) + assert ftbot._heartbeat_msg != 0 + + caplog.clear() + # Message is not shown before interval is up + ftbot.process() + assert not log_has_re(message, caplog) + + caplog.clear() + # Set clock - 70 seconds + ftbot._heartbeat_msg -= 70 + + ftbot.process() + assert log_has_re(message, caplog)