LumeWeb
/
skynet-webportal
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #446 from NebulousLabs/stop-sia-on-critical-disk-space-threshold 

stop sia container on critical disk space threshold
This commit is contained in:
Karol Wypchło 2020-10-06 14:54:59 +02:00 committed by GitHub
parent 6b379d4343 1922c4cd98
commit c229d78401
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 19 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
setup-scripts/health-checker.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import asyncio, json, os, re, sys, traceback, discord, requests import asyncio, json, os, re, sys, traceback, discord, requests, time
from datetime import datetime, timedelta from datetime import datetime, timedelta
from bot_utils import setup, send_msg from bot_utils import setup, send_msg
@ -19,9 +19,10 @@ if len(sys.argv) > 3:
DISCORD_MAX_MESSAGE_LENGTH = 1900 DISCORD_MAX_MESSAGE_LENGTH = 1900
GB = 1 << 30 # 1 GiB in bytes GB = 1 << 30 # 1 GiB in bytes
# We are going to issue Discord warnings if the free space on a server falls
# under this threshold. # Free disk space threshold used for notices and shutting down siad.
FREE_DISK_SPACE_THRESHOLD = 50 * GB FREE_DISK_SPACE_THRESHOLD = 50 * GB
FREE_DISK_SPACE_THRESHOLD_CRITICAL = 20 * GB
bot_token = setup() bot_token = setup()
client = discord.Client() client = discord.Client()
@ -92,6 +93,21 @@ async def check_disk():
if vol == "": if vol == "":
message = "Failed to check free disk space! Didn't find a suitable mount point to check." message = "Failed to check free disk space! Didn't find a suitable mount point to check."
return await send_msg(client, message, file=df) return await send_msg(client, message, file=df)
# if we've reached a critical free disk space threshold we need to send proper notice
# and shut down sia container so it doesn't get corrupted
if int(volumes[vol]) < FREE_DISK_SPACE_THRESHOLD_CRITICAL:
free_space_gb = "{:.2f}".format(int(volumes[vol]) / GB)
message = "CRITICAL! Very low disk space: {}GiB, **siad stopped**!".format(free_space_gb)
inspect = os.popen("docker inspect sia").read().strip()
inspect_json = json.loads(inspect)
if inspect_json[0]["State"]["Running"] == True:
os.popen("docker exec health-check cli/disable") # mark portal as unhealthy
time.sleep(300) # wait 5 minutes to propagate dns changes
os.popen("docker stop sia") # stop sia container
return await send_msg(client, message, force_notify=True)
# if we're reached a free disk space threshold we need to send proper notice
if int(volumes[vol]) < FREE_DISK_SPACE_THRESHOLD: if int(volumes[vol]) < FREE_DISK_SPACE_THRESHOLD:
free_space_gb = "{:.2f}".format(int(volumes[vol]) / GB) free_space_gb = "{:.2f}".format(int(volumes[vol]) / GB)
message = "WARNING! Low disk space: {}GiB".format(free_space_gb) message = "WARNING! Low disk space: {}GiB".format(free_space_gb)