run python format
This commit is contained in:
parent
efc6060924
commit
a337b754a8
|
@ -1,6 +1,14 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
import asyncio, json, os, re, sys, traceback, discord, requests, time
|
import asyncio
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import traceback
|
||||||
|
import discord
|
||||||
|
import requests
|
||||||
|
import time
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from bot_utils import setup, send_msg
|
from bot_utils import setup, send_msg
|
||||||
|
|
||||||
|
@ -69,7 +77,8 @@ async def check_load_average():
|
||||||
pattern = "^.*load average: \d*\.\d*, \d*\.\d*, (\d*\.\d*)$"
|
pattern = "^.*load average: \d*\.\d*, \d*\.\d*, (\d*\.\d*)$"
|
||||||
load_av = re.match(pattern, uptime_string).group(1)
|
load_av = re.match(pattern, uptime_string).group(1)
|
||||||
if float(load_av) > 10:
|
if float(load_av) > 10:
|
||||||
message = "High system load detected in uptime output: {}".format(uptime_string)
|
message = "High system load detected in uptime output: {}".format(
|
||||||
|
uptime_string)
|
||||||
await send_msg(client, message, force_notify=True)
|
await send_msg(client, message, force_notify=True)
|
||||||
|
|
||||||
|
|
||||||
|
@ -97,17 +106,19 @@ async def check_disk():
|
||||||
message = "Failed to check free disk space! Didn't find a suitable mount point to check."
|
message = "Failed to check free disk space! Didn't find a suitable mount point to check."
|
||||||
return await send_msg(client, message, file=df)
|
return await send_msg(client, message, file=df)
|
||||||
|
|
||||||
# if we've reached a critical free disk space threshold we need to send proper notice
|
# if we've reached a critical free disk space threshold we need to send proper notice
|
||||||
# and shut down sia container so it doesn't get corrupted
|
# and shut down sia container so it doesn't get corrupted
|
||||||
if int(volumes[vol]) < FREE_DISK_SPACE_THRESHOLD_CRITICAL:
|
if int(volumes[vol]) < FREE_DISK_SPACE_THRESHOLD_CRITICAL:
|
||||||
free_space_gb = "{:.2f}".format(int(volumes[vol]) / GB)
|
free_space_gb = "{:.2f}".format(int(volumes[vol]) / GB)
|
||||||
message = "CRITICAL! Very low disk space: {}GiB, **siad stopped**!".format(free_space_gb)
|
message = "CRITICAL! Very low disk space: {}GiB, **siad stopped**!".format(
|
||||||
|
free_space_gb)
|
||||||
inspect = os.popen("docker inspect sia").read().strip()
|
inspect = os.popen("docker inspect sia").read().strip()
|
||||||
inspect_json = json.loads(inspect)
|
inspect_json = json.loads(inspect)
|
||||||
if inspect_json[0]["State"]["Running"] == True:
|
if inspect_json[0]["State"]["Running"] == True:
|
||||||
os.popen("docker exec health-check cli/disable") # mark portal as unhealthy
|
# mark portal as unhealthy
|
||||||
time.sleep(300) # wait 5 minutes to propagate dns changes
|
os.popen("docker exec health-check cli/disable")
|
||||||
os.popen("docker stop sia") # stop sia container
|
time.sleep(300) # wait 5 minutes to propagate dns changes
|
||||||
|
os.popen("docker stop sia") # stop sia container
|
||||||
return await send_msg(client, message, force_notify=True)
|
return await send_msg(client, message, force_notify=True)
|
||||||
|
|
||||||
# if we're reached a free disk space threshold we need to send proper notice
|
# if we're reached a free disk space threshold we need to send proper notice
|
||||||
|
@ -175,7 +186,7 @@ async def check_health():
|
||||||
failed_records.append(verbose)
|
failed_records.append(verbose)
|
||||||
|
|
||||||
################################################################################
|
################################################################################
|
||||||
################ create a message
|
# create a message
|
||||||
################################################################################
|
################################################################################
|
||||||
|
|
||||||
message = ""
|
message = ""
|
||||||
|
@ -194,7 +205,8 @@ async def check_health():
|
||||||
)
|
)
|
||||||
force_notify = True
|
force_notify = True
|
||||||
else:
|
else:
|
||||||
message += "All {} critical checks passed. ".format(critical_checks_total)
|
message += "All {} critical checks passed. ".format(
|
||||||
|
critical_checks_total)
|
||||||
|
|
||||||
if verbose_checks_failed:
|
if verbose_checks_failed:
|
||||||
message += "{}/{} verbose checks failed over the last {} hours! ".format(
|
message += "{}/{} verbose checks failed over the last {} hours! ".format(
|
||||||
|
@ -202,7 +214,8 @@ async def check_health():
|
||||||
)
|
)
|
||||||
force_notify = True
|
force_notify = True
|
||||||
else:
|
else:
|
||||||
message += "All {} verbose checks passed. ".format(verbose_checks_total)
|
message += "All {} verbose checks passed. ".format(
|
||||||
|
verbose_checks_total)
|
||||||
|
|
||||||
if len(failed_records):
|
if len(failed_records):
|
||||||
failed_records_file = json.dumps(failed_records, indent=2)
|
failed_records_file = json.dumps(failed_records, indent=2)
|
||||||
|
@ -217,9 +230,11 @@ async def check_health():
|
||||||
# contains_string is a simple helper to check if a string contains a string.
|
# contains_string is a simple helper to check if a string contains a string.
|
||||||
# This is faster and easier than regex for word comparisons
|
# This is faster and easier than regex for word comparisons
|
||||||
def contains_string(string_to_check, string_to_find):
|
def contains_string(string_to_check, string_to_find):
|
||||||
return string_to_find in string_to_check
|
return string_to_find in string_to_check
|
||||||
|
|
||||||
# check_alerts checks the alerts returned from siad's daemon/alerts API
|
# check_alerts checks the alerts returned from siad's daemon/alerts API
|
||||||
|
|
||||||
|
|
||||||
async def check_alerts():
|
async def check_alerts():
|
||||||
print("\nChecking portal siad alerts...")
|
print("\nChecking portal siad alerts...")
|
||||||
|
|
||||||
|
@ -231,10 +246,10 @@ async def check_alerts():
|
||||||
siac_alert_output = os.popen(cmd_string).read().strip()
|
siac_alert_output = os.popen(cmd_string).read().strip()
|
||||||
|
|
||||||
# Initialize variables
|
# Initialize variables
|
||||||
num_critical_alerts = 0
|
num_critical_alerts = 0
|
||||||
num_error_alerts = 0
|
num_error_alerts = 0
|
||||||
num_warning_alerts = 0
|
num_warning_alerts = 0
|
||||||
num_siafile_alerts =0
|
num_siafile_alerts = 0
|
||||||
siafile_alerts = []
|
siafile_alerts = []
|
||||||
|
|
||||||
# Pattern strings to search for
|
# Pattern strings to search for
|
||||||
|
@ -243,26 +258,26 @@ async def check_alerts():
|
||||||
warning = 'Severity: warning'
|
warning = 'Severity: warning'
|
||||||
health_of = 'has a health of'
|
health_of = 'has a health of'
|
||||||
siafile_alert_message = "The SiaFile mentioned in the 'Cause' is below 75% redundancy"
|
siafile_alert_message = "The SiaFile mentioned in the 'Cause' is below 75% redundancy"
|
||||||
|
|
||||||
# Split the output by line and check for type of alert and siafile alerts
|
# Split the output by line and check for type of alert and siafile alerts
|
||||||
for line in siac_alert_output.split("\n"):
|
for line in siac_alert_output.split("\n"):
|
||||||
# Check for the type of alert
|
# Check for the type of alert
|
||||||
if contains_string(lin,critical):
|
if contains_string(lin, critical):
|
||||||
num_critical_alerts++
|
num_critical_alerts++
|
||||||
if contains_string(lin,error):
|
if contains_string(lin, error):
|
||||||
num_error_alerts++
|
num_error_alerts++
|
||||||
if contains_string(lin,warning):
|
if contains_string(lin, warning):
|
||||||
num_warning_alerts++
|
num_warning_alerts++
|
||||||
|
|
||||||
# Check for siafile alerts in alerts. This is so that the alert
|
# Check for siafile alerts in alerts. This is so that the alert
|
||||||
# severity can change and this doesn't need to be updated
|
# severity can change and this doesn't need to be updated
|
||||||
if contains_string(line,siafile_alert_message):
|
if contains_string(line, siafile_alert_message):
|
||||||
num_siafile_alerts++
|
num_siafile_alerts++
|
||||||
if contains_string(line,health_of)
|
if contains_string(line, health_of)
|
||||||
siafile_alerts.append(line)
|
siafile_alerts.append(line)
|
||||||
|
|
||||||
################################################################################
|
################################################################################
|
||||||
################ create a message
|
# create a message
|
||||||
################################################################################
|
################################################################################
|
||||||
|
|
||||||
message = ""
|
message = ""
|
||||||
|
@ -274,7 +289,7 @@ async def check_alerts():
|
||||||
if num_error_alerts > 0:
|
if num_error_alerts > 0:
|
||||||
message += "{} Error Alerts found! ".format(num_error_alerts)
|
message += "{} Error Alerts found! ".format(num_error_alerts)
|
||||||
force_notify = True
|
force_notify = True
|
||||||
|
|
||||||
message += "{} Warning Alerts found. ".format(num_warning_alerts)
|
message += "{} Warning Alerts found. ".format(num_warning_alerts)
|
||||||
message += "{} SiaFiles with bad health found. ".format(num_siafile_alerts)
|
message += "{} SiaFiles with bad health found. ".format(num_siafile_alerts)
|
||||||
|
|
||||||
|
@ -287,6 +302,8 @@ async def check_alerts():
|
||||||
|
|
||||||
# check_portal_size checks the number of files that the portal is managing to
|
# check_portal_size checks the number of files that the portal is managing to
|
||||||
# determine if it is time to rotate it out
|
# determine if it is time to rotate it out
|
||||||
|
|
||||||
|
|
||||||
async def check_portal_size():
|
async def check_portal_size():
|
||||||
print("\nChecking portal size...")
|
print("\nChecking portal size...")
|
||||||
|
|
||||||
|
@ -303,13 +320,13 @@ async def check_portal_size():
|
||||||
max_files = 250000
|
max_files = 250000
|
||||||
files_text = "Files"
|
files_text = "Files"
|
||||||
for line in siac_renter_output.split("\n"):
|
for line in siac_renter_output.split("\n"):
|
||||||
if contains_string(line,files_text):
|
if contains_string(line, files_text):
|
||||||
for el in line.split():
|
for el in line.split():
|
||||||
if el.isdigit():
|
if el.isdigit():
|
||||||
num_files = int(el)
|
num_files = int(el)
|
||||||
|
|
||||||
################################################################################
|
################################################################################
|
||||||
################ create a message
|
# create a message
|
||||||
################################################################################
|
################################################################################
|
||||||
|
|
||||||
message = ""
|
message = ""
|
||||||
|
@ -318,7 +335,7 @@ async def check_portal_size():
|
||||||
if num_files > max_files:
|
if num_files > max_files:
|
||||||
message += "Portal has {} files! Consider rotating! ".format(num_files)
|
message += "Portal has {} files! Consider rotating! ".format(num_files)
|
||||||
force_notify = True
|
force_notify = True
|
||||||
else:
|
else:
|
||||||
message += "Portal has {} files. ".format(num_files)
|
message += "Portal has {} files. ".format(num_files)
|
||||||
|
|
||||||
# send a message if we force notification, or just once daily (heartbeat) on 1 AM
|
# send a message if we force notification, or just once daily (heartbeat) on 1 AM
|
||||||
|
|
Reference in New Issue