diff --git a/setup-scripts/log-checker.py b/setup-scripts/log-checker.py index e8d9b5e3..ff8cf03a 100755 --- a/setup-scripts/log-checker.py +++ b/setup-scripts/log-checker.py @@ -1,26 +1,25 @@ #!/usr/bin/env python3 -import discord, sys, traceback, io +import discord, sys, traceback, io, os from bot_utils import setup, send_msg, sc_precision - from datetime import datetime, timedelta from subprocess import Popen, PIPE """ -log-checker checks journal logs for siad. +log-checker checks the docker logs for siad. Arguments: 1. path to a .env file (default is none so env variables can already be preset) - 2. systemd service name (default: "siad") + 2. docker container name name (default: "sia") - 3. number of hours to look back in log (used as --since value in journalctl - command) (default: 1 hour) + 3. number of hours to look back in log (default: 1 hour) """ -DEFAULT_CHECK_INTERVAL = timedelta(hours=1) +# The default check interval in hours. +DEFAULT_CHECK_INTERVAL = 1 bot_token = setup() client = discord.Client() @@ -34,7 +33,7 @@ async def on_ready(): async def run_checks(): print("Running Skynet portal log checks") try: - await check_journal() + await check_docker_logs() except: # catch all exceptions trace = traceback.format_exc() @@ -51,7 +50,7 @@ async def check_journal(): service_name = sys.argv[2] # Get the systemd service name as an argument, or use "siad" as default. - check_interval = DEFAULT_CHECK_INTERVAL + check_interval = timedelta(hours=DEFAULT_CHECK_INTERVAL) if len(sys.argv) > 3: check_interval = timedelta(hours=int(sys.argv[3])) @@ -73,10 +72,51 @@ async def check_journal(): await send_msg(client, "Critical error found in log!", file=discord.File(io.BytesIO(std_out.encode()), filename=upload_name), force_notify=True) return - # No critical errors, return a heartbeat type messagej + # No critical errors, return a heartbeat type message pretty_before = time.strftime("%I:%M%p") pretty_now = now.strftime("%I:%M%p") await send_msg(client, "No critical warnings in log from `{}` to `{}`".format(pretty_before, pretty_now)) +# check_docker_logs checks the docker logs by filtering on the docker image name +async def check_docker_logs(): + print("\nChecking docker logs...") + + # Get the container name as an argument or use "sia" as default. + container_name = "sia" + if len(sys.argv) > 2: + container_name = sys.argv[2] + + # Get the container id for siad. + stream = os.popen('docker ps -q --filter name=^{}$'.format(container_name)) + image_id = stream.read().strip() + + # Get the number of hours to look back in the logs or use 1 as default. + check_hours = DEFAULT_CHECK_INTERVAL + if len(sys.argv) > 3: + check_hours = int(sys.argv[3]) + + now = datetime.now() + time = now - timedelta(hours=check_hours) + time_string = "{}h".format(check_hours) + + # Read the logs. + proc = Popen(["docker", "logs", "--since", time_string, image_id], stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True) + std_out, std_err = proc.communicate() + + if len(std_err) > 0: + await send_msg(client, "Error reading docker logs output: {}".format(std_err), force_notify=True) + return + + # If there are any critical errors. upload the whole log file. + if "Critical" in std_out or "panic" in std_out: + upload_name = "{}-{}-{}-{}-{}:{}:{}.log".format(container_name, time.year, time.month, time.day, time.hour, time.minute, time.second) + await send_msg(client, "Critical error found in log!", file=discord.File(io.BytesIO(std_out.encode()), filename=upload_name), force_notify=True) + return + + # No critical errors, return a heartbeat type message + pretty_before = time.strftime("%I:%M%p") + pretty_now = now.strftime("%I:%M%p") + await send_msg(client, "No critical warnings in log from `{}` to `{}`".format(pretty_before, pretty_now)) + client.run(bot_token) diff --git a/setup-scripts/setup-health-check-scripts.sh b/setup-scripts/setup-health-check-scripts.sh index 81ec5376..6f56e8b7 100755 --- a/setup-scripts/setup-health-check-scripts.sh +++ b/setup-scripts/setup-health-check-scripts.sh @@ -9,7 +9,7 @@ pip3 install discord.py pip3 install python-dotenv fundsCheck="0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/.sia/sia.env" -logsCheck="0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/.sia/sia.env siad 8" +logsCheck="0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/.sia/sia.env sia 8" (crontab -u user -l; echo "$fundsCheck" ) | crontab -u user - (crontab -u user -l; echo "$logsCheck" ) | crontab -u user -