This repository has been archived on 2022-10-07. You can view files and clone it, but cannot push or open issues or pull requests.
skynet-webportal/setup-scripts/log-checker.py

96 lines
2.6 KiB
Python
Raw Normal View History

2020-03-09 18:54:41 +00:00
#!/usr/bin/env python3
from bot_utils import setup, send_msg
2020-03-09 18:54:41 +00:00
from subprocess import Popen, PIPE
import sys
import traceback
import asyncio
2020-03-09 18:54:41 +00:00
"""
2020-08-18 14:27:01 +00:00
log-checker checks the docker logs for siad.
2020-03-09 18:54:41 +00:00
Arguments:
1. path to a .env file (default is none so env variables can already be
preset)
2020-08-18 14:27:01 +00:00
2. docker container name name (default: "sia")
2020-03-10 15:08:54 +00:00
2020-08-18 14:27:01 +00:00
3. number of hours to look back in log (default: 1 hour)
2020-03-10 15:08:54 +00:00
2020-03-09 18:54:41 +00:00
"""
# Get the container name as an argument or use "sia" as default.
CONTAINER_NAME = "sia"
if len(sys.argv) > 2:
CONTAINER_NAME = sys.argv[2]
# Get the number of hours to look back in the logs or use 1 as default.
CHECK_HOURS = 1
if len(sys.argv) > 3:
CHECK_HOURS = int(sys.argv[3])
# Discord messages have a limit on their length set at 2000 bytes. We use
# a lower limit in order to leave some space for additional message text.
DISCORD_MAX_MESSAGE_LENGTH = 1900
2020-03-09 18:54:41 +00:00
setup()
2020-03-09 18:54:41 +00:00
async def run_checks():
print("Running Skynet portal log checks")
try:
2020-08-18 14:27:01 +00:00
await check_docker_logs()
2020-09-29 10:32:45 +00:00
except: # catch all exceptions
2020-03-09 18:54:41 +00:00
trace = traceback.format_exc()
await send_msg("```\n{}\n```".format(trace), force_notify=False)
2020-03-09 18:54:41 +00:00
2020-08-18 14:27:01 +00:00
# check_docker_logs checks the docker logs by filtering on the docker image name
async def check_docker_logs():
print("\nChecking docker logs...")
2020-09-29 10:32:45 +00:00
since_string = "{}h".format(CHECK_HOURS)
2020-08-18 14:27:01 +00:00
# Read the logs.
2020-09-29 10:32:45 +00:00
print(
"[DEBUG] Will run `docker logs --since {} {}`".format(
since_string, CONTAINER_NAME
)
)
proc = Popen(
["docker", "logs", "--since", since_string, CONTAINER_NAME],
stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
text=True,
)
2020-08-18 14:27:01 +00:00
std_out, std_err = proc.communicate()
if len(std_err) > 0:
# Trim the error log to under 1MB.
2020-09-29 10:32:45 +00:00
one_mb = 1024 * 1024
if len(std_err) > one_mb:
pos = std_err.find("\n", -one_mb)
2020-09-29 10:32:45 +00:00
std_err = std_err[pos + 1 :]
return await send_msg(
"Error(s) found in log!", file=std_err, force_notify=True
2020-09-29 10:32:45 +00:00
)
2020-08-18 14:27:01 +00:00
2020-09-03 14:50:43 +00:00
# If there are any critical or severe errors. upload the whole log file.
2020-09-29 10:32:45 +00:00
if "Critical" in std_out or "Severe" in std_out or "panic" in std_out:
return await send_msg(
"Critical or Severe error found in log!",
file=std_out,
force_notify=True,
)
2020-08-18 14:27:01 +00:00
2020-09-03 14:50:43 +00:00
# No critical or severe errors, return a heartbeat type message
2020-09-29 10:32:45 +00:00
return await send_msg(
2020-09-29 13:46:40 +00:00
"No critical or severe warnings in log since {} hours".format(CHECK_HOURS),
2020-09-29 10:32:45 +00:00
)
2020-08-18 14:27:01 +00:00
2020-08-31 11:27:49 +00:00
loop = asyncio.get_event_loop()
loop.run_until_complete(run_checks())