run python format

This commit is contained in:
Matthew Sevey 2020-11-20 11:33:07 -07:00
parent efc6060924
commit a337b754a8
1 changed files with 45 additions and 28 deletions

View File

@ -1,6 +1,14 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import asyncio, json, os, re, sys, traceback, discord, requests, time import asyncio
import json
import os
import re
import sys
import traceback
import discord
import requests
import time
from datetime import datetime, timedelta from datetime import datetime, timedelta
from bot_utils import setup, send_msg from bot_utils import setup, send_msg
@ -69,7 +77,8 @@ async def check_load_average():
pattern = "^.*load average: \d*\.\d*, \d*\.\d*, (\d*\.\d*)$" pattern = "^.*load average: \d*\.\d*, \d*\.\d*, (\d*\.\d*)$"
load_av = re.match(pattern, uptime_string).group(1) load_av = re.match(pattern, uptime_string).group(1)
if float(load_av) > 10: if float(load_av) > 10:
message = "High system load detected in uptime output: {}".format(uptime_string) message = "High system load detected in uptime output: {}".format(
uptime_string)
await send_msg(client, message, force_notify=True) await send_msg(client, message, force_notify=True)
@ -101,11 +110,13 @@ async def check_disk():
# and shut down sia container so it doesn't get corrupted # and shut down sia container so it doesn't get corrupted
if int(volumes[vol]) < FREE_DISK_SPACE_THRESHOLD_CRITICAL: if int(volumes[vol]) < FREE_DISK_SPACE_THRESHOLD_CRITICAL:
free_space_gb = "{:.2f}".format(int(volumes[vol]) / GB) free_space_gb = "{:.2f}".format(int(volumes[vol]) / GB)
message = "CRITICAL! Very low disk space: {}GiB, **siad stopped**!".format(free_space_gb) message = "CRITICAL! Very low disk space: {}GiB, **siad stopped**!".format(
free_space_gb)
inspect = os.popen("docker inspect sia").read().strip() inspect = os.popen("docker inspect sia").read().strip()
inspect_json = json.loads(inspect) inspect_json = json.loads(inspect)
if inspect_json[0]["State"]["Running"] == True: if inspect_json[0]["State"]["Running"] == True:
os.popen("docker exec health-check cli/disable") # mark portal as unhealthy # mark portal as unhealthy
os.popen("docker exec health-check cli/disable")
time.sleep(300) # wait 5 minutes to propagate dns changes time.sleep(300) # wait 5 minutes to propagate dns changes
os.popen("docker stop sia") # stop sia container os.popen("docker stop sia") # stop sia container
return await send_msg(client, message, force_notify=True) return await send_msg(client, message, force_notify=True)
@ -175,7 +186,7 @@ async def check_health():
failed_records.append(verbose) failed_records.append(verbose)
################################################################################ ################################################################################
################ create a message # create a message
################################################################################ ################################################################################
message = "" message = ""
@ -194,7 +205,8 @@ async def check_health():
) )
force_notify = True force_notify = True
else: else:
message += "All {} critical checks passed. ".format(critical_checks_total) message += "All {} critical checks passed. ".format(
critical_checks_total)
if verbose_checks_failed: if verbose_checks_failed:
message += "{}/{} verbose checks failed over the last {} hours! ".format( message += "{}/{} verbose checks failed over the last {} hours! ".format(
@ -202,7 +214,8 @@ async def check_health():
) )
force_notify = True force_notify = True
else: else:
message += "All {} verbose checks passed. ".format(verbose_checks_total) message += "All {} verbose checks passed. ".format(
verbose_checks_total)
if len(failed_records): if len(failed_records):
failed_records_file = json.dumps(failed_records, indent=2) failed_records_file = json.dumps(failed_records, indent=2)
@ -220,6 +233,8 @@ def contains_string(string_to_check, string_to_find):
return string_to_find in string_to_check return string_to_find in string_to_check
# check_alerts checks the alerts returned from siad's daemon/alerts API # check_alerts checks the alerts returned from siad's daemon/alerts API
async def check_alerts(): async def check_alerts():
print("\nChecking portal siad alerts...") print("\nChecking portal siad alerts...")
@ -234,7 +249,7 @@ async def check_alerts():
num_critical_alerts = 0 num_critical_alerts = 0
num_error_alerts = 0 num_error_alerts = 0
num_warning_alerts = 0 num_warning_alerts = 0
num_siafile_alerts =0 num_siafile_alerts = 0
siafile_alerts = [] siafile_alerts = []
# Pattern strings to search for # Pattern strings to search for
@ -247,22 +262,22 @@ async def check_alerts():
# Split the output by line and check for type of alert and siafile alerts # Split the output by line and check for type of alert and siafile alerts
for line in siac_alert_output.split("\n"): for line in siac_alert_output.split("\n"):
# Check for the type of alert # Check for the type of alert
if contains_string(lin,critical): if contains_string(lin, critical):
num_critical_alerts++ num_critical_alerts++
if contains_string(lin,error): if contains_string(lin, error):
num_error_alerts++ num_error_alerts++
if contains_string(lin,warning): if contains_string(lin, warning):
num_warning_alerts++ num_warning_alerts++
# Check for siafile alerts in alerts. This is so that the alert # Check for siafile alerts in alerts. This is so that the alert
# severity can change and this doesn't need to be updated # severity can change and this doesn't need to be updated
if contains_string(line,siafile_alert_message): if contains_string(line, siafile_alert_message):
num_siafile_alerts++ num_siafile_alerts++
if contains_string(line,health_of) if contains_string(line, health_of)
siafile_alerts.append(line) siafile_alerts.append(line)
################################################################################ ################################################################################
################ create a message # create a message
################################################################################ ################################################################################
message = "" message = ""
@ -287,6 +302,8 @@ async def check_alerts():
# check_portal_size checks the number of files that the portal is managing to # check_portal_size checks the number of files that the portal is managing to
# determine if it is time to rotate it out # determine if it is time to rotate it out
async def check_portal_size(): async def check_portal_size():
print("\nChecking portal size...") print("\nChecking portal size...")
@ -303,13 +320,13 @@ async def check_portal_size():
max_files = 250000 max_files = 250000
files_text = "Files" files_text = "Files"
for line in siac_renter_output.split("\n"): for line in siac_renter_output.split("\n"):
if contains_string(line,files_text): if contains_string(line, files_text):
for el in line.split(): for el in line.split():
if el.isdigit(): if el.isdigit():
num_files = int(el) num_files = int(el)
################################################################################ ################################################################################
################ create a message # create a message
################################################################################ ################################################################################
message = "" message = ""