Merge pull request #74 from NebulousLabs/add-log-checker
Add log checker
This commit is contained in:
Marcin Jachymiak
GitHub
commit
e0e0e7949f
|
|
@ -115,3 +115,20 @@ You can check that with `node -v` and `yarn -v` commands respectively.
|
||||||
- run `yarn build` to build the client package
|
- run `yarn build` to build the client package
|
||||||
|
|
||||||
Client package will be outputted to `/public` and nginx configuration will pick it up automatically.
|
Client package will be outputted to `/public` and nginx configuration will pick it up automatically.
|
||||||
|
|
||||||
|
## Health Check Scripts.
|
||||||
|
|
||||||
|
There are 2 optional health check scripts that can be setup using
|
||||||
|
`setup-health-check-scripts.sh`. That command will install the necesary Python
|
||||||
|
dependencies and setup 2 cronjobs for each script: one for a downloading `siad`
|
||||||
|
and for an uploading `siad` service.
|
||||||
|
|
||||||
|
To use the scripts you must setup a Discord bot and provide a bot token. The bot
|
||||||
|
scripts take in 1 or more arguments, the first always being the path to an
|
||||||
|
`.env` file.
|
||||||
|
|
||||||
|
`funds-checker` checks that the wallet balance and allowance settings are
|
||||||
|
sufficient for portal usage.
|
||||||
|
|
||||||
|
`log-checker` checks if there are any critical warnings in the journal for the
|
||||||
|
running services.
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,127 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
from urllib.request import urlopen, Request
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import urllib, json, os, traceback, discord, sys
|
||||||
|
|
||||||
|
# sc_precision is the number of hastings per siacoin
|
||||||
|
sc_precision = 10 ** 24
|
||||||
|
|
||||||
|
channel_name = "skynet-portal-health-check"
|
||||||
|
|
||||||
|
# Environment variable globals
|
||||||
|
api_endpoint, port, portal_name, bot_token, password = None, None, None, None, None
|
||||||
|
discord_client = None
|
||||||
|
setup_done = False
|
||||||
|
|
||||||
|
def setup():
|
||||||
|
# Load dotenv file if possible.
|
||||||
|
if len(sys.argv) > 1:
|
||||||
|
env_path = Path(sys.argv[1])
|
||||||
|
load_dotenv(dotenv_path=env_path, override=True)
|
||||||
|
|
||||||
|
global bot_token
|
||||||
|
bot_token = os.environ["DISCORD_BOT_TOKEN"]
|
||||||
|
|
||||||
|
global portal_name
|
||||||
|
portal_name = os.getenv("PORTAL_NAME")
|
||||||
|
|
||||||
|
# Get a port or use default
|
||||||
|
global port
|
||||||
|
port = os.getenv("API_PORT")
|
||||||
|
if not port:
|
||||||
|
port = "9980"
|
||||||
|
|
||||||
|
global api_endpoint
|
||||||
|
api_endpoint = "http://localhost:{}".format(port)
|
||||||
|
|
||||||
|
siad.initialize()
|
||||||
|
|
||||||
|
global setup_done
|
||||||
|
setup_done = True
|
||||||
|
|
||||||
|
return bot_token
|
||||||
|
|
||||||
|
# send_msg sends the msg to the specified discord channel. If force_notify is set to true it adds "@here".
|
||||||
|
async def send_msg(client, msg, force_notify=False, file=None):
|
||||||
|
await client.wait_until_ready()
|
||||||
|
|
||||||
|
guild = client.guilds[0]
|
||||||
|
channels = guild.channels
|
||||||
|
|
||||||
|
chan = None
|
||||||
|
for c in channels:
|
||||||
|
if c.name == channel_name:
|
||||||
|
chan = c
|
||||||
|
|
||||||
|
if chan is None:
|
||||||
|
print("Can't find channel {}".format(channel_name))
|
||||||
|
|
||||||
|
# Add the portal name.
|
||||||
|
msg = "`{}`: {}".format(portal_name, msg)
|
||||||
|
|
||||||
|
if force_notify:
|
||||||
|
msg = "@here: \n{}".format(msg)
|
||||||
|
await chan.send(msg, file=file)
|
||||||
|
|
||||||
|
|
||||||
|
#siad class provides wrappers for the necessary siad commands.
|
||||||
|
class siad:
|
||||||
|
# initializes values for using the API (password and
|
||||||
|
# user-agent) so that all calls to urllib.request.urlopen have these set.
|
||||||
|
@staticmethod
|
||||||
|
def initialize():
|
||||||
|
# Setup a handler with the API password
|
||||||
|
username = ""
|
||||||
|
password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
|
||||||
|
password_mgr.add_password(None, api_endpoint, username, siad.get_password())
|
||||||
|
handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
|
||||||
|
|
||||||
|
# Setup an opener with the correct user agent
|
||||||
|
opener = urllib.request.build_opener(handler)
|
||||||
|
opener.addheaders = [('User-agent', 'Sia-Agent')]
|
||||||
|
|
||||||
|
# Install the opener.
|
||||||
|
# Now all calls to urllib.request.urlopen use our opener.
|
||||||
|
urllib.request.install_opener(opener)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_password():
|
||||||
|
# Get a port or use default
|
||||||
|
password = os.getenv("SIA_API_PASSWORD")
|
||||||
|
if not password:
|
||||||
|
home = os.getenv("HOME")
|
||||||
|
password_file = open(home+"/.sia/apipassword")
|
||||||
|
password = password_file.readlines()[0].strip()
|
||||||
|
return password
|
||||||
|
|
||||||
|
# load_json reads the http response and decodes the JSON value
|
||||||
|
@staticmethod
|
||||||
|
def load_json(resp):
|
||||||
|
return json.loads(resp.decode("utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_wallet():
|
||||||
|
if not setup_done: setup()
|
||||||
|
|
||||||
|
resp = urllib.request.urlopen(api_endpoint + "/wallet").read()
|
||||||
|
return siad.load_json(resp)
|
||||||
|
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_renter():
|
||||||
|
if not setup_done: setup()
|
||||||
|
|
||||||
|
resp = urllib.request.urlopen(api_endpoint + "/renter").read()
|
||||||
|
return siad.load_json(resp)
|
||||||
|
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_renter_contracts():
|
||||||
|
if not setup_done: setup()
|
||||||
|
|
||||||
|
resp = urllib.request.urlopen(api_endpoint + "/renter/contracts").read()
|
||||||
|
return siad.load_json(resp)
|
||||||
|
|
@ -0,0 +1,70 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
"""
|
||||||
|
health-checker runs simple health checks on a portal node using the siad API and
|
||||||
|
dispatches messages to a Discord channel.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import discord, traceback
|
||||||
|
from bot_utils import setup, send_msg, siad, sc_precision
|
||||||
|
|
||||||
|
bot_token = setup()
|
||||||
|
client = discord.Client()
|
||||||
|
|
||||||
|
@client.event
|
||||||
|
async def on_ready():
|
||||||
|
await run_checks()
|
||||||
|
await client.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def run_checks():
|
||||||
|
print("Running Skynet portal health checks")
|
||||||
|
try:
|
||||||
|
await check_health()
|
||||||
|
|
||||||
|
except: # catch all exceptions
|
||||||
|
trace = traceback.format_exc()
|
||||||
|
await send_msg(client, "```\n{}\n```".format(trace), force_notify=True)
|
||||||
|
|
||||||
|
|
||||||
|
# check_health checks that the wallet is unlocked, that it has at least 1
|
||||||
|
# allowance worth of money left, and if more than hald the allowance is spent. If
|
||||||
|
# all checks pass it sends a informational message.
|
||||||
|
async def check_health():
|
||||||
|
print("\nChecking wallet/funds health...")
|
||||||
|
wallet_get = siad.get_wallet()
|
||||||
|
renter_get = siad.get_renter()
|
||||||
|
|
||||||
|
if not wallet_get['unlocked']:
|
||||||
|
await send_msg(client, "Wallet locked", force_notify=True)
|
||||||
|
return
|
||||||
|
|
||||||
|
confirmed_coins = int(wallet_get['confirmedsiacoinbalance'])
|
||||||
|
unconfirmed_coins = int(wallet_get['unconfirmedincomingsiacoins'])
|
||||||
|
unconfirmed_outgoing_coins = int(wallet_get['unconfirmedoutgoingsiacoins'])
|
||||||
|
balance = confirmed_coins + unconfirmed_coins - unconfirmed_outgoing_coins
|
||||||
|
print("Balance: ", balance / sc_precision)
|
||||||
|
|
||||||
|
allowance = renter_get['settings']['allowance']
|
||||||
|
allowance_funds = int(allowance['funds'])
|
||||||
|
allocated_funds = int(renter_get['financialmetrics']['totalallocated'])
|
||||||
|
unallocated_funds = allowance_funds - allocated_funds
|
||||||
|
|
||||||
|
|
||||||
|
balance_msg = "Balance: `{} SC` Allowance Funds: `{} SC`".format(round(balance/sc_precision), round(allowance_funds/sc_precision))
|
||||||
|
alloc_msg = "Unallocated: `{} SC`\nAllocated: `{} SC`".format(round(unallocated_funds/sc_precision), round(allocated_funds/sc_precision))
|
||||||
|
|
||||||
|
# Send an alert if there is less than 1 allowance worth of money left.
|
||||||
|
if balance < allowance_funds:
|
||||||
|
await send_msg(client, "Wallet balance running low. \n{}`".format(balance_msg), force_notify=True)
|
||||||
|
return
|
||||||
|
|
||||||
|
# Alert devs when 1/2 the allowance is gone
|
||||||
|
if allocated_funds >= unallocated_funds:
|
||||||
|
await send_msg(client, "Allowance half spent: \n{}".format(alloc_msg), force_notify=True)
|
||||||
|
return
|
||||||
|
|
||||||
|
# Send an informational heartbeat if all checks passed.
|
||||||
|
await send_msg(client, "Health checks passed:\n{} \n{}".format(balance_msg, alloc_msg))
|
||||||
|
|
||||||
|
client.run(bot_token)
|
||||||
|
|
@ -1,168 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
from urllib.request import urlopen, Request
|
|
||||||
from dotenv import load_dotenv
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import urllib, json, os, traceback, discord, sys
|
|
||||||
|
|
||||||
|
|
||||||
# sc_precision is the number of hastings per siacoin
|
|
||||||
sc_precision = 10 ** 24
|
|
||||||
|
|
||||||
# Environment variable globals
|
|
||||||
api_endpoint, port, portal_name, bot_token, password = None, None, None, None, None
|
|
||||||
|
|
||||||
# Load dotenv file if possible.
|
|
||||||
if len(sys.argv) > 1:
|
|
||||||
env_path = Path(sys.argv[1])
|
|
||||||
load_dotenv(dotenv_path=env_path, override=True)
|
|
||||||
|
|
||||||
bot_token = os.environ["DISCORD_BOT_TOKEN"]
|
|
||||||
portal_name = os.getenv("PORTAL_NAME")
|
|
||||||
|
|
||||||
# Get a port or use default
|
|
||||||
port = os.getenv("API_PORT")
|
|
||||||
if not port:
|
|
||||||
port = "9980"
|
|
||||||
|
|
||||||
api_endpoint = "http://localhost:{}".format(port)
|
|
||||||
|
|
||||||
|
|
||||||
# Discord bot initialization
|
|
||||||
client = discord.Client()
|
|
||||||
channel_name = "skynet-portal-health-check"
|
|
||||||
|
|
||||||
@client.event
|
|
||||||
async def on_ready():
|
|
||||||
await run_checks()
|
|
||||||
await client.close()
|
|
||||||
|
|
||||||
# send_msg sends the msg to the specified discord channel. If force_notify is set to true it adds "@here".
|
|
||||||
async def send_msg(msg, force_notify=False):
|
|
||||||
await client.wait_until_ready()
|
|
||||||
|
|
||||||
guild = client.guilds[0]
|
|
||||||
channels = guild.channels
|
|
||||||
|
|
||||||
chan = None
|
|
||||||
for c in channels:
|
|
||||||
if c.name == channel_name:
|
|
||||||
chan = c
|
|
||||||
|
|
||||||
if chan is None:
|
|
||||||
print("Can't find channel {}".format(channel_name))
|
|
||||||
|
|
||||||
# Add the portal name.
|
|
||||||
msg = "`{}`: {}".format(portal_name, msg)
|
|
||||||
|
|
||||||
if force_notify:
|
|
||||||
msg = "@here: \n{}".format(msg)
|
|
||||||
await chan.send(msg)
|
|
||||||
|
|
||||||
#siac class provides wrappers for the necessary siac commands.
|
|
||||||
class siac:
|
|
||||||
# initializes values for using the API (password and
|
|
||||||
# user-agent) so that all calls to urllib.request.urlopen have these set.
|
|
||||||
@staticmethod
|
|
||||||
def initialize():
|
|
||||||
# Setup a handler with the API password
|
|
||||||
username = ""
|
|
||||||
password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
|
|
||||||
password_mgr.add_password(None, api_endpoint, username, siac.get_password())
|
|
||||||
handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
|
|
||||||
|
|
||||||
# Setup an opener with the correct user agent
|
|
||||||
opener = urllib.request.build_opener(handler)
|
|
||||||
opener.addheaders = [('User-agent', 'Sia-Agent')]
|
|
||||||
|
|
||||||
# Install the opener.
|
|
||||||
# Now all calls to urllib.request.urlopen use our opener.
|
|
||||||
urllib.request.install_opener(opener)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def get_password():
|
|
||||||
# Get a port or use default
|
|
||||||
password = os.getenv("SIA_API_PASSWORD")
|
|
||||||
if not password:
|
|
||||||
home = os.getenv("HOME")
|
|
||||||
password_file = open(home+"/.sia/apipassword")
|
|
||||||
password = password_file.readlines()[0].strip()
|
|
||||||
|
|
||||||
return password
|
|
||||||
|
|
||||||
# load_json reads the http response and decodes the JSON value
|
|
||||||
@staticmethod
|
|
||||||
def load_json(resp):
|
|
||||||
return json.loads(resp.decode("utf-8"))
|
|
||||||
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def get_wallet():
|
|
||||||
resp = urllib.request.urlopen(api_endpoint + "/wallet").read()
|
|
||||||
return siac.load_json(resp)
|
|
||||||
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def get_renter():
|
|
||||||
resp = urllib.request.urlopen(api_endpoint + "/renter").read()
|
|
||||||
return siac.load_json(resp)
|
|
||||||
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def get_renter_contracts():
|
|
||||||
resp = urllib.request.urlopen(api_endpoint + "/renter/contracts").read()
|
|
||||||
return siac.load_json(resp)
|
|
||||||
|
|
||||||
|
|
||||||
# check_health checks that the wallet is unlocked, that it has at least 1
|
|
||||||
# allowance worth of money left, and if more than hald the allowance is spent. If
|
|
||||||
# all checks pass it sends a informational message.
|
|
||||||
async def check_health():
|
|
||||||
print("\nChecking health...")
|
|
||||||
wallet_get = siac.get_wallet()
|
|
||||||
renter_get = siac.get_renter()
|
|
||||||
|
|
||||||
if not wallet_get['unlocked']:
|
|
||||||
await send_msg("Wallet locked", force_notify=True)
|
|
||||||
return
|
|
||||||
|
|
||||||
confirmed_coins = int(wallet_get['confirmedsiacoinbalance'])
|
|
||||||
unconfirmed_coins = int(wallet_get['unconfirmedincomingsiacoins'])
|
|
||||||
unconfirmed_outgoing_coins = int(wallet_get['unconfirmedoutgoingsiacoins'])
|
|
||||||
balance = confirmed_coins + unconfirmed_coins - unconfirmed_outgoing_coins
|
|
||||||
print("Balance: ", balance / sc_precision)
|
|
||||||
|
|
||||||
allowance = renter_get['settings']['allowance']
|
|
||||||
allowance_funds = int(allowance['funds'])
|
|
||||||
allocated_funds = int(renter_get['financialmetrics']['totalallocated'])
|
|
||||||
unallocated_funds = allowance_funds - allocated_funds
|
|
||||||
|
|
||||||
# Send an alert if there is less than 1 allowance worth of money left.
|
|
||||||
if balance < allowance_funds:
|
|
||||||
await send_msg("Wallet balance running low. Balance: `{} SC` Allowance Funds: `{} SC`".format(round(balance/sc_precision), round(allowance_funds/sc_precision)), force_notify=True)
|
|
||||||
return
|
|
||||||
|
|
||||||
# Alert devs when 1/2 the allowance is gone
|
|
||||||
if allocated_funds >= unallocated_funds:
|
|
||||||
await send_msg("Allowance half spent: \nUnallocated: `{} SC`\nAllocated: `{} SC`".format(round(unallocated_funds/sc_precision), round(allocated_funds/sc_precision)), force_notify=True)
|
|
||||||
return
|
|
||||||
|
|
||||||
# Send an informational heartbeat if all checks passed.
|
|
||||||
pretty_renter_get = json.dumps(siac.get_renter(), indent=4)
|
|
||||||
await send_msg("Health checks passed:\n\nWallet Balance: `{} SC`\n\n Renter Info:\n```\n{}\n```".format(round(balance/sc_precision), pretty_renter_get))
|
|
||||||
|
|
||||||
|
|
||||||
async def run_checks():
|
|
||||||
# Initialize the siac API helper.
|
|
||||||
siac.initialize()
|
|
||||||
|
|
||||||
print("Running Skynet portal health checks")
|
|
||||||
try:
|
|
||||||
await check_health()
|
|
||||||
|
|
||||||
except: # catch all exceptions
|
|
||||||
trace = traceback.format_exc()
|
|
||||||
await send_msg("```\n{}\n```".format(trace), force_notify=True)
|
|
||||||
|
|
||||||
client.run(bot_token)
|
|
||||||
|
|
@ -0,0 +1,70 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import discord, sys, traceback, io
|
||||||
|
from bot_utils import setup, send_msg, sc_precision
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from subprocess import Popen, PIPE
|
||||||
|
|
||||||
|
"""
|
||||||
|
log-checker checks journal logs for siad.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
1. path to a .env file (default is none so env variables can already be
|
||||||
|
preset)
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
DEFAULT_CHECK_INTERVAL = timedelta(hours=1)
|
||||||
|
|
||||||
|
bot_token = setup()
|
||||||
|
client = discord.Client()
|
||||||
|
|
||||||
|
@client.event
|
||||||
|
async def on_ready():
|
||||||
|
await run_checks()
|
||||||
|
await client.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def run_checks():
|
||||||
|
print("Running Skynet portal log checks")
|
||||||
|
try:
|
||||||
|
await check_journal()
|
||||||
|
|
||||||
|
except: # catch all exceptions
|
||||||
|
trace = traceback.format_exc()
|
||||||
|
await send_msg(client, "```\n{}\n```".format(trace), force_notify=False)
|
||||||
|
|
||||||
|
|
||||||
|
# check_journal checks the journal
|
||||||
|
async def check_journal():
|
||||||
|
print("\nChecking journal...")
|
||||||
|
|
||||||
|
now = datetime.now()
|
||||||
|
time = now - DEFAULT_CHECK_INTERVAL
|
||||||
|
time_string = "{}-{}-{} {}:{}:{}".format(time.year, time.month, time.day, time.hour, time.minute, time.second)
|
||||||
|
|
||||||
|
# Get the systemd service name as an argument, or use "siad" as default.
|
||||||
|
service_name = "siad"
|
||||||
|
if len(sys.argv) > 2:
|
||||||
|
service_name = sys.argv[2]
|
||||||
|
|
||||||
|
# Open the journal.
|
||||||
|
proc = Popen(["journalctl", "--user-unit", service_name, "--since", time_string], stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True)
|
||||||
|
std_out, std_err = proc.communicate()
|
||||||
|
|
||||||
|
if len(std_err) > 0:
|
||||||
|
await send_msg(client, "Error reading journalctl output: {}".format(std_err), force_notify=True)
|
||||||
|
return
|
||||||
|
|
||||||
|
# If there are any critical errors. upload the whole log file.
|
||||||
|
if "Critical" in std_out:
|
||||||
|
upload_name = "{}-{}-{}-{}-{}:{}:{}.log".format(service_name, time.year, time.month, time.day, time.hour, time.minute, time.second)
|
||||||
|
await send_msg(client, "Critical error found in log!", file=discord.File(io.BytesIO(std_out.encode()), filename=upload_name), force_notify=True)
|
||||||
|
return
|
||||||
|
|
||||||
|
# No critical errors, return a heartbeat type message.
|
||||||
|
await send_msg(client, "No critical warnings in log (size of log portion checked: {})".format(len(std_out)))
|
||||||
|
|
||||||
|
|
||||||
|
client.run(bot_token)
|
||||||
|
|
@ -7,8 +7,14 @@ sudo apt-get -y install python3-pip
|
||||||
pip3 install discord.py
|
pip3 install discord.py
|
||||||
pip3 install python-dotenv
|
pip3 install python-dotenv
|
||||||
|
|
||||||
downloadCheck="0 0,8,16 * * * ~/skynet-webportal/setup-scripts/health-checker.py ~/.sia/sia.env"
|
downloadCheck="0 0,8,16 * * * ~/skynet-webportal/setup-scripts/funds-checker.py ~/.sia/sia.env"
|
||||||
uploadCheck="0 0,8,16 * * * ~/skynet-webportal/setup-scripts/health-checker.py ~/.sia/sia-upload.env"
|
uploadCheck="0 0,8,16 * * * ~/skynet-webportal/setup-scripts/funds-checker.py ~/.sia/sia-upload.env"
|
||||||
|
|
||||||
|
logCheck1="0 * * * * ~/skynet-webportal/setup-scripts/log-checker.py ~/.sia/sia.env"
|
||||||
|
logCheck2="0 * * * * ~/skynet-webportal/setup-scripts/log-checker.py ~/.sia/sia-upload.env siad-upload"
|
||||||
|
|
||||||
(crontab -u user -l; echo "$downloadCheck" ) | crontab -u user -
|
(crontab -u user -l; echo "$downloadCheck" ) | crontab -u user -
|
||||||
(crontab -u user -l; echo "$uploadCheck" ) | crontab -u user -
|
(crontab -u user -l; echo "$uploadCheck" ) | crontab -u user -
|
||||||
|
|
||||||
|
(crontab -u user -l; echo "$logCheck1" ) | crontab -u user -
|
||||||
|
(crontab -u user -l; echo "$logCheck2" ) | crontab -u user -
|
||||||
|
|
|
||||||
Reference in New Issue