Cleanup health check script

This commit is contained in:
Marcin Jachymiak 2020-03-03 18:26:51 -05:00
parent 6fbf60d84c
commit cbe73e14c7
1 changed files with 73 additions and 35 deletions

View File

@ -1,21 +1,45 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
from urllib.request import urlopen, Request from urllib.request import urlopen, Request
import urllib import urllib, json, os, traceback, discord
import json
import sys
portal_name = os.environ["PORTAL_NAME"]
# sc_precision is the number of hastings per siacoin # sc_precision is the number of hastings per siacoin
sc_precision = 10 ** 24 sc_precision = 10 ** 24
api_endpoint = "http://localhost:9980" # TODO get port from env/param api_endpoint = "http://localhost:9980" # TODO get port from env/param
# Discord bot initialization
bot_token = os.environ["DISCORD_BOT_TOKEN"]
client = discord.Client()
channel_name = "skynet-portal-health-check"
# TODO: get values from a param @client.event
min_balance = 10_000 * sc_precision async def on_ready():
min_unspent_renter_funds = 5_000 * sc_precision await run_checks()
default_renter_funds = 20_000 * sc_precision await client.close()
# send_msg sends the msg to the specified discord channel. If force_notify is set to true it adds "@here".
async def send_msg(msg, force_notify=False):
await client.wait_until_ready()
guild = client.guilds[0]
channels = guild.channels
chan = None
for c in channels:
if c.name == channel_name:
chan = c
if chan is None:
print("Can't find channel {}".format(channel_name))
# Add the portal name.
msg = "`{}`: {}".format(portal_name, msg)
if force_notify:
msg = "@here: \n{}".format(msg)
await chan.send(msg)
#siac class provides wrappers for the necessary siac commands. #siac class provides wrappers for the necessary siac commands.
class siac: class siac:
@ -48,60 +72,74 @@ class siac:
def load_json(resp): def load_json(resp):
return json.loads(resp.decode("utf-8")) return json.loads(resp.decode("utf-8"))
@staticmethod @staticmethod
def get_wallet(): def get_wallet():
resp = urllib.request.urlopen(api_endpoint + "/wallet").read() resp = urllib.request.urlopen(api_endpoint + "/wallet").read()
return siac.load_json(resp) return siac.load_json(resp)
@staticmethod @staticmethod
def get_renter(): def get_renter():
resp = urllib.request.urlopen(api_endpoint + "/renter").read() resp = urllib.request.urlopen(api_endpoint + "/renter").read()
return siac.load_json(resp) return siac.load_json(resp)
# Initialize the API opener
siac.initialize()
# fail should ping discord for failures in health checks that can't be managed automatically. @staticmethod
def fail(err_msg): def get_renter_contracts():
print("Error message: ", err_msg) resp = urllib.request.urlopen(api_endpoint + "/renter/contracts").read()
return siac.load_json(resp)
# check_wallet_health_check checks that the wallet is unlocked and has at least min_balance SC. # check_health checks that the wallet is unlocked, that it has at least 1
def check_wallet_health(): # allowance worth of money left, and if more than hald the allowance is spent. If
# all checks pass it sends a informational message.
async def check_health():
print("\nChecking health...")
wallet_get = siac.get_wallet() wallet_get = siac.get_wallet()
renter_get = siac.get_renter()
if not wallet_get['unlocked']: if not wallet_get['unlocked']:
fail("Wallet not unlocked") await send_msg("Wallet locked", force_notify=True)
return
confirmed_coins = int(wallet_get['confirmedsiacoinbalance']) confirmed_coins = int(wallet_get['confirmedsiacoinbalance'])
unconfirmed_coins = int(wallet_get['unconfirmedincomingsiacoins']) unconfirmed_coins = int(wallet_get['unconfirmedincomingsiacoins'])
unconfirmed_outgoing_coins = int(wallet_get['unconfirmedoutgoingsiacoins']) unconfirmed_outgoing_coins = int(wallet_get['unconfirmedoutgoingsiacoins'])
balance = confirmed_coins + unconfirmed_coins - unconfirmed_outgoing_coins balance = confirmed_coins + unconfirmed_coins - unconfirmed_outgoing_coins
print("Balance: ", balance / sc_precision) print("Balance: ", balance / sc_precision)
print("Min balance: ", min_balance / sc_precision)
if balance < min_balance:
fail("Wallet balance too low: {}".format(balance))
def check_renter_health():
renter_get = siac.get_renter()
allowance = renter_get['settings']['allowance'] allowance = renter_get['settings']['allowance']
allowance_funds = int(allowance['funds']) allowance_funds = int(allowance['funds'])
unspent_funds = int(renter_get['financialmetrics']['unspent']) unspent_funds = int(renter_get['financialmetrics']['unspent'])
spent_funds = allowance_funds - unspent_funds
# if funds are under a certain amount, re-balance the wallet. # Send an alert if there is less than 1 allowance worth of money left.
if unspent_funds < min_unspent_renter_funds: if balance < allowance_funds:
#siac.set_renter_funds(default_renter_funds) await send_msg("Wallet balance running low: {}".format(balance), force_notify=True)
#TODO return
check_renter_health() # Alert devs when 1/2 the allowance is gone
if spent_funds >= unspent_funds:
await send_msg("Allowance half spent: \nUnspent: {}\nSpent: {}".format(unspent_funds/sc_precision, spent_funds/sc_precision), force_notify=True)
return
try: # Send an informational heartbeat if all checks passed.
check_wallet_health() pretty_renter_get = json.dumps(siac.get_renter(), indent=4)
check_renter_health() pretty_renter_contracts = json.dumps(siac.get_renter_contracts(), indent=4)
await send_msg("Health checks passed:\n\nRenter Info:\n```\n{}\n```\n\nContract Info:\n```\n{}\n```\n".format(pretty_renter_get, pretty_renter_contracts))
except: # catch all exceptions
e = sys.exc_info()[0] async def run_checks():
fail(e) # Initialize the siac API helper.
siac.initialize()
print("Running Skynet portal health checks")
try:
await check_health()
except: # catch all exceptions
trace = traceback.format_exc()
await send_msg("```\n{}\n```".format(trace), force_notify=True)
client.run(bot_token)