diff --git a/setup-scripts/health-checker.py b/setup-scripts/health-checker.py index b8df8b71..f28d8469 100755 --- a/setup-scripts/health-checker.py +++ b/setup-scripts/health-checker.py @@ -17,8 +17,13 @@ health-checker reads the /health-check endpoint of the portal and dispatches messages to a Discord channel. """ -# The default check interval in hours. -DEFAULT_CHECK_INTERVAL = 1 +# Get the number of hours to look back in the logs or use 1 as default. +CHECK_HOURS = 1 +if len(sys.argv) > 3: + CHECK_HOURS = int(sys.argv[3]) + +# Discord messages have a limit on their length set at 2000 bytes. We use +# a lower limit in order to leave some space for additional message text. DISCORD_MAX_MESSAGE_LENGTH = 1900 bot_token = setup() @@ -69,17 +74,12 @@ async def check_health(): force_notify=True) return - # Get the number of hours to look back in the logs or use 1 as default. - check_hours = DEFAULT_CHECK_INTERVAL - if len(sys.argv) > 3: - check_hours = int(sys.argv[3]) - # Check the health records. failed_records = [] failed_checks = 0 failed_critical = 0 passed_checks_counter = 0 - time_limit = datetime.now() - timedelta(hours=check_hours) + time_limit = datetime.now() - timedelta(hours=CHECK_HOURS) for rec in res.json(): time = datetime.strptime(rec['date'], '%Y-%m-%dT%H:%M:%S.%fZ') if time < time_limit: diff --git a/setup-scripts/log-checker.py b/setup-scripts/log-checker.py index 50363e88..029b9a07 100755 --- a/setup-scripts/log-checker.py +++ b/setup-scripts/log-checker.py @@ -18,8 +18,16 @@ Arguments: """ -# The default check interval in hours. -DEFAULT_CHECK_INTERVAL = 1 +# Get the container name as an argument or use "sia" as default. +CONTAINER_NAME = "sia" +if len(sys.argv) > 2: + CONTAINER_NAME = sys.argv[2] + +# Get the number of hours to look back in the logs or use 1 as default. +CHECK_HOURS = 1 +if len(sys.argv) > 3: + CHECK_HOURS = int(sys.argv[3]) + # Discord messages have a limit on their length set at 2000 bytes. We use # a lower limit in order to leave some space for additional message text. DISCORD_MAX_MESSAGE_LENGTH = 1900 @@ -68,23 +76,13 @@ async def check_load_average(): async def check_docker_logs(): print("\nChecking docker logs...") - # Get the container name as an argument or use "sia" as default. - container_name = "sia" - if len(sys.argv) > 2: - container_name = sys.argv[2] - - # Get the number of hours to look back in the logs or use 1 as default. - check_hours = DEFAULT_CHECK_INTERVAL - if len(sys.argv) > 3: - check_hours = int(sys.argv[3]) - now = datetime.now() - time = now - timedelta(hours=check_hours) - time_string = "{}h".format(check_hours) + time = now - timedelta(hours=CHECK_HOURS) + time_string = "{}h".format(CHECK_HOURS) # Read the logs. - print("[DEBUG] Will run `docker logs --since {} {}`".format(time_string, container_name)) - proc = Popen(["docker", "logs", "--since", time_string, container_name], stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True) + print("[DEBUG] Will run `docker logs --since {} {}`".format(time_string, CONTAINER_NAME)) + proc = Popen(["docker", "logs", "--since", time_string, CONTAINER_NAME], stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True) std_out, std_err = proc.communicate() if len(std_err) > 0: @@ -93,7 +91,7 @@ async def check_docker_logs(): if len(std_err) > one_mb: pos = std_err.find("\n", -one_mb) std_err = std_err[pos+1:] - upload_name = "{}-{}-{}-{}-{}:{}:{}_err.log".format(container_name, time.year, time.month, time.day, time.hour, time.minute, time.second) + upload_name = "{}-{}-{}-{}-{}:{}:{}_err.log".format(CONTAINER_NAME, time.year, time.month, time.day, time.hour, time.minute, time.second) await send_msg(client, "Error(s) found in log!", file=discord.File(io.BytesIO(std_err.encode()), filename=upload_name), force_notify=True) # Send at most DISCORD_MAX_MESSAGE_LENGTH characters of logs, rounded # down to the nearest new line. This is a limitation in the size of @@ -107,7 +105,7 @@ async def check_docker_logs(): # If there are any critical or severe errors. upload the whole log file. if 'Critical' in std_out or 'Severe' in std_out or 'panic' in std_out: - upload_name = "{}-{}-{}-{}-{}:{}:{}.log".format(container_name, time.year, time.month, time.day, time.hour, time.minute, time.second) + upload_name = "{}-{}-{}-{}-{}:{}:{}.log".format(CONTAINER_NAME, time.year, time.month, time.day, time.hour, time.minute, time.second) await send_msg(client, "Critical or Severe error found in log!", file=discord.File(io.BytesIO(std_out.encode()), filename=upload_name), force_notify=True) return