From 6fbf60d84cbdf349023c53a98d029b875b2b2739 Mon Sep 17 00:00:00 2001 From: Marcin Jachymiak Date: Tue, 3 Mar 2020 15:49:09 -0500 Subject: [PATCH 1/8] Start health check script --- setup-scripts/health-checker.py | 107 ++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100755 setup-scripts/health-checker.py diff --git a/setup-scripts/health-checker.py b/setup-scripts/health-checker.py new file mode 100755 index 00000000..54a49edf --- /dev/null +++ b/setup-scripts/health-checker.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 + +from urllib.request import urlopen, Request +import urllib +import json +import sys + + +# sc_precision is the number of hastings per siacoin +sc_precision = 10 ** 24 + +api_endpoint = "http://localhost:9980" # TODO get port from env/param + + +# TODO: get values from a param +min_balance = 10_000 * sc_precision +min_unspent_renter_funds = 5_000 * sc_precision +default_renter_funds = 20_000 * sc_precision + +#siac class provides wrappers for the necessary siac commands. +class siac: + # initializes values for using the API (password and + # user-agent) so that all calls to urllib.request.urlopen have these set. + @staticmethod + def initialize(): + # Setup a handler with the API password + username = "" + password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() + password_mgr.add_password(None, api_endpoint, username, siac.get_password()) + handler = urllib.request.HTTPBasicAuthHandler(password_mgr) + + # Setup an opener with the correct user agent + opener = urllib.request.build_opener(handler) + opener.addheaders = [('User-agent', 'Sia-Agent')] + + # Install the opener. + # Now all calls to urllib.request.urlopen use our opener. + urllib.request.install_opener(opener) + + @staticmethod + def get_password(): + password_file = open("/home/marcin/.sia/apipassword") + password = password_file.readlines() + return password[0].strip() + + # load_json reads the http response and decodes the JSON value + @staticmethod + def load_json(resp): + return json.loads(resp.decode("utf-8")) + + @staticmethod + def get_wallet(): + resp = urllib.request.urlopen(api_endpoint + "/wallet").read() + return siac.load_json(resp) + + @staticmethod + def get_renter(): + resp = urllib.request.urlopen(api_endpoint + "/renter").read() + return siac.load_json(resp) + +# Initialize the API opener +siac.initialize() + +# fail should ping discord for failures in health checks that can't be managed automatically. +def fail(err_msg): + print("Error message: ", err_msg) + + +# check_wallet_health_check checks that the wallet is unlocked and has at least min_balance SC. +def check_wallet_health(): + wallet_get = siac.get_wallet() + + if not wallet_get['unlocked']: + fail("Wallet not unlocked") + + confirmed_coins = int(wallet_get['confirmedsiacoinbalance']) + unconfirmed_coins = int(wallet_get['unconfirmedincomingsiacoins']) + unconfirmed_outgoing_coins = int(wallet_get['unconfirmedoutgoingsiacoins']) + balance = confirmed_coins + unconfirmed_coins - unconfirmed_outgoing_coins + + print("Balance: ", balance / sc_precision) + print("Min balance: ", min_balance / sc_precision) + + if balance < min_balance: + fail("Wallet balance too low: {}".format(balance)) + +def check_renter_health(): + renter_get = siac.get_renter() + + allowance = renter_get['settings']['allowance'] + allowance_funds = int(allowance['funds']) + unspent_funds = int(renter_get['financialmetrics']['unspent']) + + # if funds are under a certain amount, re-balance the wallet. + if unspent_funds < min_unspent_renter_funds: + #siac.set_renter_funds(default_renter_funds) + #TODO + +check_renter_health() + +try: + check_wallet_health() + check_renter_health() + +except: # catch all exceptions + e = sys.exc_info()[0] + fail(e) From cbe73e14c7b746e002be9d6c086a3828ef822463 Mon Sep 17 00:00:00 2001 From: Marcin Jachymiak Date: Tue, 3 Mar 2020 18:26:51 -0500 Subject: [PATCH 2/8] Cleanup health check script --- setup-scripts/health-checker.py | 108 +++++++++++++++++++++----------- 1 file changed, 73 insertions(+), 35 deletions(-) diff --git a/setup-scripts/health-checker.py b/setup-scripts/health-checker.py index 54a49edf..d1dc57aa 100755 --- a/setup-scripts/health-checker.py +++ b/setup-scripts/health-checker.py @@ -1,21 +1,45 @@ #!/usr/bin/env python3 from urllib.request import urlopen, Request -import urllib -import json -import sys +import urllib, json, os, traceback, discord +portal_name = os.environ["PORTAL_NAME"] # sc_precision is the number of hastings per siacoin sc_precision = 10 ** 24 - api_endpoint = "http://localhost:9980" # TODO get port from env/param +# Discord bot initialization +bot_token = os.environ["DISCORD_BOT_TOKEN"] +client = discord.Client() +channel_name = "skynet-portal-health-check" -# TODO: get values from a param -min_balance = 10_000 * sc_precision -min_unspent_renter_funds = 5_000 * sc_precision -default_renter_funds = 20_000 * sc_precision +@client.event +async def on_ready(): + await run_checks() + await client.close() + +# send_msg sends the msg to the specified discord channel. If force_notify is set to true it adds "@here". +async def send_msg(msg, force_notify=False): + await client.wait_until_ready() + + guild = client.guilds[0] + channels = guild.channels + + chan = None + for c in channels: + if c.name == channel_name: + chan = c + + if chan is None: + print("Can't find channel {}".format(channel_name)) + + # Add the portal name. + msg = "`{}`: {}".format(portal_name, msg) + + if force_notify: + msg = "@here: \n{}".format(msg) + await chan.send(msg) #siac class provides wrappers for the necessary siac commands. class siac: @@ -48,60 +72,74 @@ class siac: def load_json(resp): return json.loads(resp.decode("utf-8")) + @staticmethod def get_wallet(): resp = urllib.request.urlopen(api_endpoint + "/wallet").read() return siac.load_json(resp) + @staticmethod def get_renter(): resp = urllib.request.urlopen(api_endpoint + "/renter").read() return siac.load_json(resp) -# Initialize the API opener -siac.initialize() -# fail should ping discord for failures in health checks that can't be managed automatically. -def fail(err_msg): - print("Error message: ", err_msg) + @staticmethod + def get_renter_contracts(): + resp = urllib.request.urlopen(api_endpoint + "/renter/contracts").read() + return siac.load_json(resp) -# check_wallet_health_check checks that the wallet is unlocked and has at least min_balance SC. -def check_wallet_health(): +# check_health checks that the wallet is unlocked, that it has at least 1 +# allowance worth of money left, and if more than hald the allowance is spent. If +# all checks pass it sends a informational message. +async def check_health(): + print("\nChecking health...") wallet_get = siac.get_wallet() + renter_get = siac.get_renter() if not wallet_get['unlocked']: - fail("Wallet not unlocked") + await send_msg("Wallet locked", force_notify=True) + return confirmed_coins = int(wallet_get['confirmedsiacoinbalance']) unconfirmed_coins = int(wallet_get['unconfirmedincomingsiacoins']) unconfirmed_outgoing_coins = int(wallet_get['unconfirmedoutgoingsiacoins']) balance = confirmed_coins + unconfirmed_coins - unconfirmed_outgoing_coins - print("Balance: ", balance / sc_precision) - print("Min balance: ", min_balance / sc_precision) - - if balance < min_balance: - fail("Wallet balance too low: {}".format(balance)) - -def check_renter_health(): - renter_get = siac.get_renter() allowance = renter_get['settings']['allowance'] allowance_funds = int(allowance['funds']) unspent_funds = int(renter_get['financialmetrics']['unspent']) + spent_funds = allowance_funds - unspent_funds - # if funds are under a certain amount, re-balance the wallet. - if unspent_funds < min_unspent_renter_funds: - #siac.set_renter_funds(default_renter_funds) - #TODO + # Send an alert if there is less than 1 allowance worth of money left. + if balance < allowance_funds: + await send_msg("Wallet balance running low: {}".format(balance), force_notify=True) + return -check_renter_health() + # Alert devs when 1/2 the allowance is gone + if spent_funds >= unspent_funds: + await send_msg("Allowance half spent: \nUnspent: {}\nSpent: {}".format(unspent_funds/sc_precision, spent_funds/sc_precision), force_notify=True) + return -try: - check_wallet_health() - check_renter_health() + # Send an informational heartbeat if all checks passed. + pretty_renter_get = json.dumps(siac.get_renter(), indent=4) + pretty_renter_contracts = json.dumps(siac.get_renter_contracts(), indent=4) + await send_msg("Health checks passed:\n\nRenter Info:\n```\n{}\n```\n\nContract Info:\n```\n{}\n```\n".format(pretty_renter_get, pretty_renter_contracts)) -except: # catch all exceptions - e = sys.exc_info()[0] - fail(e) + +async def run_checks(): + # Initialize the siac API helper. + siac.initialize() + + print("Running Skynet portal health checks") + try: + await check_health() + + except: # catch all exceptions + trace = traceback.format_exc() + await send_msg("```\n{}\n```".format(trace), force_notify=True) + +client.run(bot_token) From 64681bf19dac4f2636a2114b0e61b04f2a088953 Mon Sep 17 00:00:00 2001 From: Marcin Jachymiak Date: Tue, 3 Mar 2020 18:29:17 -0500 Subject: [PATCH 3/8] Fixup port env var --- setup-scripts/health-checker.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/setup-scripts/health-checker.py b/setup-scripts/health-checker.py index d1dc57aa..64b6335b 100755 --- a/setup-scripts/health-checker.py +++ b/setup-scripts/health-checker.py @@ -3,11 +3,16 @@ from urllib.request import urlopen, Request import urllib, json, os, traceback, discord -portal_name = os.environ["PORTAL_NAME"] +# Get a port or use default +port = os.getenv("API_PORT") +if not port: + port = "9980" + +api_endpoint = "http://localhost:{}".format(port) +portal_name = os.getenv("PORTAL_NAME") # sc_precision is the number of hastings per siacoin sc_precision = 10 ** 24 -api_endpoint = "http://localhost:9980" # TODO get port from env/param # Discord bot initialization bot_token = os.environ["DISCORD_BOT_TOKEN"] From d9b27d416c69b0802040efa5cffeb125e2849450 Mon Sep 17 00:00:00 2001 From: Marcin Jachymiak Date: Thu, 5 Mar 2020 10:03:04 -0500 Subject: [PATCH 4/8] Cleanup Discord msgs and add password var --- setup-scripts/health-checker.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/setup-scripts/health-checker.py b/setup-scripts/health-checker.py index 64b6335b..4a62f1bd 100755 --- a/setup-scripts/health-checker.py +++ b/setup-scripts/health-checker.py @@ -68,9 +68,14 @@ class siac: @staticmethod def get_password(): - password_file = open("/home/marcin/.sia/apipassword") - password = password_file.readlines() - return password[0].strip() + # Get a port or use default + password = os.getenv("API_PASSWORD") + if not password: + home = os.getenv("HOME") + password_file = open(home+"/.sia/apipassword") + password = password_file.readlines()[0].strip() + + return password # load_json reads the http response and decodes the JSON value @staticmethod @@ -121,7 +126,7 @@ async def check_health(): # Send an alert if there is less than 1 allowance worth of money left. if balance < allowance_funds: - await send_msg("Wallet balance running low: {}".format(balance), force_notify=True) + await send_msg("Wallet balance running low. Balance: `{}SC` Allowance Funds: `{}SC`".format(balance/sc_precision, allowance_funds/sc_precision), force_notify=True) return # Alert devs when 1/2 the allowance is gone @@ -131,8 +136,7 @@ async def check_health(): # Send an informational heartbeat if all checks passed. pretty_renter_get = json.dumps(siac.get_renter(), indent=4) - pretty_renter_contracts = json.dumps(siac.get_renter_contracts(), indent=4) - await send_msg("Health checks passed:\n\nRenter Info:\n```\n{}\n```\n\nContract Info:\n```\n{}\n```\n".format(pretty_renter_get, pretty_renter_contracts)) + await send_msg("Health checks passed:\n\nWallet Balance: `{}SC`\n\n Renter Info:\n```\n{}\n```".format(balance/sc_precision, pretty_renter_get)) async def run_checks(): From 2af43ed49950ca6275f956179a50871a341f6404 Mon Sep 17 00:00:00 2001 From: Marcin Jachymiak Date: Thu, 5 Mar 2020 10:07:03 -0500 Subject: [PATCH 5/8] fix api pass env --- setup-scripts/health-checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-scripts/health-checker.py b/setup-scripts/health-checker.py index 4a62f1bd..13fb8982 100755 --- a/setup-scripts/health-checker.py +++ b/setup-scripts/health-checker.py @@ -69,7 +69,7 @@ class siac: @staticmethod def get_password(): # Get a port or use default - password = os.getenv("API_PASSWORD") + password = os.getenv("SIA_API_PASSWORD") if not password: home = os.getenv("HOME") password_file = open(home+"/.sia/apipassword") From 29a73e59636be5042bb4f4740d4a78b26b01b66c Mon Sep 17 00:00:00 2001 From: Marcin Jachymiak Date: Thu, 5 Mar 2020 12:07:37 -0500 Subject: [PATCH 6/8] Add health-check setup script --- setup-scripts/health-checker.py | 24 ++++++++++++++++----- setup-scripts/setup-health-check-scripts.sh | 13 +++++++++++ setup-scripts/sia.env | 3 +++ 3 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 setup-scripts/setup-health-check-scripts.sh diff --git a/setup-scripts/health-checker.py b/setup-scripts/health-checker.py index 13fb8982..59bab3aa 100755 --- a/setup-scripts/health-checker.py +++ b/setup-scripts/health-checker.py @@ -1,7 +1,25 @@ #!/usr/bin/env python3 from urllib.request import urlopen, Request -import urllib, json, os, traceback, discord +from dotenv import load_dotenv +from pathlib import Path + +import urllib, json, os, traceback, discord, sys + + +# sc_precision is the number of hastings per siacoin +sc_precision = 10 ** 24 + +# Environment variable globals +api_endpoint, port, portal_name, bot_token, password = None, None, None, None, None + +# Load dotenv file if possible. +if len(sys.argv) > 1: + env_path = Path(sys.argv[1]) + load_dotenv(dotenv_path=env_path) + +bot_token = os.environ["DISCORD_BOT_TOKEN"] +portal_name = os.getenv("PORTAL_NAME") # Get a port or use default port = os.getenv("API_PORT") @@ -9,13 +27,9 @@ if not port: port = "9980" api_endpoint = "http://localhost:{}".format(port) -portal_name = os.getenv("PORTAL_NAME") -# sc_precision is the number of hastings per siacoin -sc_precision = 10 ** 24 # Discord bot initialization -bot_token = os.environ["DISCORD_BOT_TOKEN"] client = discord.Client() channel_name = "skynet-portal-health-check" diff --git a/setup-scripts/setup-health-check-scripts.sh b/setup-scripts/setup-health-check-scripts.sh new file mode 100644 index 00000000..f635c68a --- /dev/null +++ b/setup-scripts/setup-health-check-scripts.sh @@ -0,0 +1,13 @@ +#! /usr/bin/env bash +set -e + +sudo apt-get update +sudo apt-get -y install python3-pip + +pip3 install discord.py + +downloadCheck="0 0,8,16 * * * ~/skynet-webportal/setup-scripts/health-checker.py ~/.sia/sia.env" +uploadCheck="0 0,8,16 * * * ~/skynet-webportal/setup-scripts/health-checker.py ~/.sia/sia-upload.env" + +(crontab -u userhere -l; echo "$downloadCheck" ) | crontab -u userhere - +(crontab -u userhere -l; echo "$uploadCheck" ) | crontab -u userhere - diff --git a/setup-scripts/sia.env b/setup-scripts/sia.env index cc986b46..5b971918 100644 --- a/setup-scripts/sia.env +++ b/setup-scripts/sia.env @@ -1,3 +1,6 @@ SIA_DATA_DIR="" SIA_API_PASSWORD="" SIA_WALLET_PASSWORD="" +API_PORT="" +PORTAL_NAME="" +DISCORD_BOT_TOKEN="" From 3244c1cd899261e7565be44ac6fe7abdfade4538 Mon Sep 17 00:00:00 2001 From: Marcin Jachymiak Date: Thu, 5 Mar 2020 12:09:30 -0500 Subject: [PATCH 7/8] make script executable --- setup-scripts/setup-health-check-scripts.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 setup-scripts/setup-health-check-scripts.sh diff --git a/setup-scripts/setup-health-check-scripts.sh b/setup-scripts/setup-health-check-scripts.sh old mode 100644 new mode 100755 From 2ba2ac4b9fb39d87b8538c7bc295ba22e70b1f5e Mon Sep 17 00:00:00 2001 From: Marcin Jachymiak Date: Thu, 5 Mar 2020 12:10:35 -0500 Subject: [PATCH 8/8] Fixup --- setup-scripts/setup-health-check-scripts.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup-scripts/setup-health-check-scripts.sh b/setup-scripts/setup-health-check-scripts.sh index f635c68a..36a7cd2f 100755 --- a/setup-scripts/setup-health-check-scripts.sh +++ b/setup-scripts/setup-health-check-scripts.sh @@ -9,5 +9,5 @@ pip3 install discord.py downloadCheck="0 0,8,16 * * * ~/skynet-webportal/setup-scripts/health-checker.py ~/.sia/sia.env" uploadCheck="0 0,8,16 * * * ~/skynet-webportal/setup-scripts/health-checker.py ~/.sia/sia-upload.env" -(crontab -u userhere -l; echo "$downloadCheck" ) | crontab -u userhere - -(crontab -u userhere -l; echo "$uploadCheck" ) | crontab -u userhere - +(crontab -u user -l; echo "$downloadCheck" ) | crontab -u user - +(crontab -u user- l; echo "$uploadCheck" ) | crontab -u user -