LumeWeb
/
skynet-webportal
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

stop sia container on critical disk space threshold

This commit is contained in:
Karol Wypchlo 2020-10-06 11:24:18 +02:00
parent 70640ce375
commit 60f8371170
1 changed files with 20 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
setup-scripts/health-checker.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import asyncio, json, os, re, sys, traceback, discord, requests import asyncio, json, os, re, sys, traceback, discord, requests, time, subprocess
from datetime import datetime, timedelta from datetime import datetime, timedelta
from bot_utils import setup, send_msg from bot_utils import setup, send_msg
@ -19,9 +19,10 @@ if len(sys.argv) > 3:
DISCORD_MAX_MESSAGE_LENGTH = 1900 DISCORD_MAX_MESSAGE_LENGTH = 1900
GB = 1 << 30 # 1 GiB in bytes GB = 1 << 30 # 1 GiB in bytes
# We are going to issue Discord warnings if the free space on a server falls
# under this threshold. # Free disk space threshold used for notices and shutting down siad.
FREE_DISK_SPACE_THRESHOLD = 50 * GB FREE_DISK_SPACE_THRESHOLD = 50 * GB
FREE_DISK_SPACE_THRESHOLD_CRITICAL = 20 * GB
bot_token = setup() bot_token = setup()
client = discord.Client() client = discord.Client()
@ -92,6 +93,22 @@ async def check_disk():
if vol == "": if vol == "":
message = "Failed to check free disk space! Didn't find a suitable mount point to check." message = "Failed to check free disk space! Didn't find a suitable mount point to check."
return await send_msg(client, message, file=df) return await send_msg(client, message, file=df)
# if we've reached a critical free disk space threshold we need to send proper notice
# and shut down sia container so it doesn't get corrupted
if int(volumes[vol]) < FREE_DISK_SPACE_THRESHOLD_CRITICAL:
free_space_gb = "{:.2f}".format(int(volumes[vol]) / GB)
message = "CRITICAL! Very low disk space: {}GiB".format(free_space_gb)
inspect = os.popen("docker inspect sia").read().strip()
inspect_json = json.loads(inspect)
if inspect_json[0]["State"]["Running"] == True:
message += ", **stopping siad container**!"
subprocess.Popen('/home/user/skynet-webportal/scripts/portal-down.sh')
else:
message += ", siad container is already stopped!"
return await send_msg(client, message, force_notify=True)
# if we're reached a free disk space threshold we need to send proper notice
if int(volumes[vol]) < FREE_DISK_SPACE_THRESHOLD: if int(volumes[vol]) < FREE_DISK_SPACE_THRESHOLD:
free_space_gb = "{:.2f}".format(int(volumes[vol]) / GB) free_space_gb = "{:.2f}".format(int(volumes[vol]) / GB)
message = "WARNING! Low disk space: {}GiB".format(free_space_gb) message = "WARNING! Low disk space: {}GiB".format(free_space_gb)