use webhook instead of discrod bot to send messages (#979)
* initial refactor * do not use before define * forgot to remove client * test notification * add /cc * fix /cc * fix /cc role * fix /cc * test file upload * test file upload * test file upload * default to no mentions * unformat * replace discord with DiscordWebhook * add readme * don't fail on failures in message send
This commit is contained in:
parent
3e516d8a46
commit
71f9d5280e
|
@ -92,7 +92,9 @@ At this point we have almost everything running, we just need to set up your wal
|
||||||
- `AWS_ACCESS_KEY_ID` (optional) if using route53 as a dns loadbalancer
|
- `AWS_ACCESS_KEY_ID` (optional) if using route53 as a dns loadbalancer
|
||||||
- `AWS_SECRET_ACCESS_KEY` (optional) if using route53 as a dns loadbalancer
|
- `AWS_SECRET_ACCESS_KEY` (optional) if using route53 as a dns loadbalancer
|
||||||
- `PORTAL_NAME` a string representing name of your portal e.g. `siasky.xyz` or `my skynet portal`
|
- `PORTAL_NAME` a string representing name of your portal e.g. `siasky.xyz` or `my skynet portal`
|
||||||
- `DISCORD_BOT_TOKEN` (optional) if you're using Discord notifications for health checks and such
|
- `DISCORD_WEBHOOK_URL` (required if using Discord notifications) discord webhook url (generate from discord app)
|
||||||
|
- `DISCORD_MENTION_USER_ID` (optional) add `/cc @user` mention to important messages from webhook (has to be id not user name)
|
||||||
|
- `DISCORD_MENTION_ROLE_ID` (optional) add `/cc @role` mention to important messages from webhook (has to be id not role name)
|
||||||
- `SKYNET_DB_USER` (optional) if using `accounts` this is the MongoDB username
|
- `SKYNET_DB_USER` (optional) if using `accounts` this is the MongoDB username
|
||||||
- `SKYNET_DB_PASS` (optional) if using `accounts` this is the MongoDB password
|
- `SKYNET_DB_PASS` (optional) if using `accounts` this is the MongoDB password
|
||||||
- `SKYNET_DB_HOST` (optional) if using `accounts` this is the MongoDB address or container name
|
- `SKYNET_DB_HOST` (optional) if using `accounts` this is the MongoDB address or container name
|
||||||
|
|
|
@ -3,14 +3,20 @@
|
||||||
import traceback, os, re, asyncio, requests, json, discord
|
import traceback, os, re, asyncio, requests, json, discord
|
||||||
from bot_utils import setup, send_msg
|
from bot_utils import setup, send_msg
|
||||||
|
|
||||||
bot_token = setup()
|
setup()
|
||||||
client = discord.Client()
|
|
||||||
|
|
||||||
AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY")
|
AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY")
|
||||||
AIRTABLE_BASE = os.getenv("AIRTABLE_BASE", "app89plJvA9EqTJEc")
|
AIRTABLE_BASE = os.getenv("AIRTABLE_BASE", "app89plJvA9EqTJEc")
|
||||||
AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE", "Table%201")
|
AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE", "Table%201")
|
||||||
AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD", "Link")
|
AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD", "Link")
|
||||||
|
|
||||||
|
async def run_checks():
|
||||||
|
try:
|
||||||
|
await block_skylinks_from_airtable()
|
||||||
|
except: # catch all exceptions
|
||||||
|
trace = traceback.format_exc()
|
||||||
|
await send_msg("```\n{}\n```".format(trace), force_notify=True)
|
||||||
|
|
||||||
|
|
||||||
def exec(command):
|
def exec(command):
|
||||||
return os.popen(command).read().strip()
|
return os.popen(command).read().strip()
|
||||||
|
@ -33,7 +39,7 @@ async def block_skylinks_from_airtable():
|
||||||
status_code = str(response.status_code)
|
status_code = str(response.status_code)
|
||||||
response_text = response.text or "empty response"
|
response_text = response.text or "empty response"
|
||||||
message = "Airtable blocklist integration responded with code " + status_code + ": " + response_text
|
message = "Airtable blocklist integration responded with code " + status_code + ": " + response_text
|
||||||
return print(message) or await send_msg(client, message, force_notify=False)
|
return await send_msg(message, force_notify=False)
|
||||||
|
|
||||||
data = response.json()
|
data = response.json()
|
||||||
|
|
||||||
|
@ -53,7 +59,7 @@ async def block_skylinks_from_airtable():
|
||||||
if len(skylinks_returned) != len(skylinks):
|
if len(skylinks_returned) != len(skylinks):
|
||||||
invalid_skylinks = [str(skylink) for skylink in list(set(skylinks_returned) - set(skylinks))]
|
invalid_skylinks = [str(skylink) for skylink in list(set(skylinks_returned) - set(skylinks))]
|
||||||
message = str(len(invalid_skylinks)) + " of the skylinks returned from Airtable are not valid"
|
message = str(len(invalid_skylinks)) + " of the skylinks returned from Airtable are not valid"
|
||||||
print(message) or await send_msg(client, message, file=("\n".join(invalid_skylinks)))
|
await send_msg(message, file=("\n".join(invalid_skylinks)))
|
||||||
|
|
||||||
apipassword = exec("docker exec sia cat /sia-data/apipassword")
|
apipassword = exec("docker exec sia cat /sia-data/apipassword")
|
||||||
ipaddress = exec("docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia")
|
ipaddress = exec("docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia")
|
||||||
|
@ -72,7 +78,7 @@ async def block_skylinks_from_airtable():
|
||||||
status_code = str(response.status_code)
|
status_code = str(response.status_code)
|
||||||
response_text = response.text or "empty response"
|
response_text = response.text or "empty response"
|
||||||
message = "Siad blocklist endpoint responded with code " + status_code + ": " + response_text
|
message = "Siad blocklist endpoint responded with code " + status_code + ": " + response_text
|
||||||
return print(message) or await send_msg(client, message, force_notify=False)
|
return await send_msg(message, force_notify=False)
|
||||||
|
|
||||||
print("Searching nginx cache for blocked files")
|
print("Searching nginx cache for blocked files")
|
||||||
cached_files_count = 0
|
cached_files_count = 0
|
||||||
|
@ -89,25 +95,11 @@ async def block_skylinks_from_airtable():
|
||||||
|
|
||||||
exec('docker exec -it nginx bash -c "' + cached_files_command + ' | xargs rm"')
|
exec('docker exec -it nginx bash -c "' + cached_files_command + ' | xargs rm"')
|
||||||
message = "Purged " + str(cached_files_count) + " blocklisted files from nginx cache"
|
message = "Purged " + str(cached_files_count) + " blocklisted files from nginx cache"
|
||||||
return print(message) or await send_msg(client, message)
|
return await send_msg(message)
|
||||||
|
|
||||||
|
|
||||||
async def exit_after(delay):
|
loop = asyncio.get_event_loop()
|
||||||
await asyncio.sleep(delay)
|
loop.run_until_complete(run_checks())
|
||||||
os._exit(0)
|
|
||||||
|
|
||||||
|
|
||||||
@client.event
|
|
||||||
async def on_ready():
|
|
||||||
try:
|
|
||||||
await block_skylinks_from_airtable()
|
|
||||||
except: # catch all exceptions
|
|
||||||
message = "```\n{}\n```".format(traceback.format_exc())
|
|
||||||
await send_msg(client, message, force_notify=False)
|
|
||||||
asyncio.create_task(exit_after(3))
|
|
||||||
|
|
||||||
|
|
||||||
client.run(bot_token)
|
|
||||||
|
|
||||||
# --- BASH EQUIVALENT
|
# --- BASH EQUIVALENT
|
||||||
# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")
|
# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")
|
||||||
|
|
|
@ -4,22 +4,27 @@ from urllib.request import urlopen, Request
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
from discord_webhook import DiscordWebhook
|
||||||
|
|
||||||
import urllib, json, os, traceback, discord, sys, re, subprocess, requests, io
|
import urllib, json, os, traceback, discord, sys, re, subprocess, requests, io
|
||||||
|
|
||||||
# sc_precision is the number of hastings per siacoin
|
# Load dotenv file if possible.
|
||||||
sc_precision = 10 ** 24
|
# TODO: change all scripts to use named flags/params
|
||||||
|
if len(sys.argv) > 1:
|
||||||
# Environment variable globals
|
env_path = Path(sys.argv[1])
|
||||||
api_endpoint, port, portal_name, bot_token, password = None, None, None, None, None
|
load_dotenv(dotenv_path=env_path, override=True)
|
||||||
discord_client = None
|
|
||||||
setup_done = False
|
|
||||||
|
|
||||||
# Get the container name as an argument or use "sia" as default.
|
# Get the container name as an argument or use "sia" as default.
|
||||||
CONTAINER_NAME = "sia"
|
CONTAINER_NAME = "sia"
|
||||||
if len(sys.argv) > 2:
|
if len(sys.argv) > 2:
|
||||||
CONTAINER_NAME = sys.argv[2]
|
CONTAINER_NAME = sys.argv[2]
|
||||||
|
|
||||||
|
# sc_precision is the number of hastings per siacoin
|
||||||
|
sc_precision = 10 ** 24
|
||||||
|
|
||||||
|
# Environment variable globals
|
||||||
|
setup_done = False
|
||||||
|
|
||||||
# find out local siad ip by inspecting its docker container
|
# find out local siad ip by inspecting its docker container
|
||||||
def get_docker_container_ip(container_name):
|
def get_docker_container_ip(container_name):
|
||||||
ip_regex = re.compile(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$")
|
ip_regex = re.compile(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$")
|
||||||
|
@ -31,6 +36,11 @@ def get_docker_container_ip(container_name):
|
||||||
return ip_regex.findall(output)[0]
|
return ip_regex.findall(output)[0]
|
||||||
|
|
||||||
|
|
||||||
|
# sia deamon local ip address with port
|
||||||
|
api_endpoint = "http://{}:{}".format(
|
||||||
|
get_docker_container_ip(CONTAINER_NAME), os.getenv("API_PORT", "9980")
|
||||||
|
)
|
||||||
|
|
||||||
# find siad api password by getting it out of the docker container
|
# find siad api password by getting it out of the docker container
|
||||||
def get_api_password():
|
def get_api_password():
|
||||||
api_password_regex = re.compile(r"^\w+$")
|
api_password_regex = re.compile(r"^\w+$")
|
||||||
|
@ -40,83 +50,54 @@ def get_api_password():
|
||||||
|
|
||||||
|
|
||||||
def setup():
|
def setup():
|
||||||
# Load dotenv file if possible.
|
|
||||||
# TODO: change all scripts to use named flags/params
|
|
||||||
if len(sys.argv) > 1:
|
|
||||||
env_path = Path(sys.argv[1])
|
|
||||||
load_dotenv(dotenv_path=env_path, override=True)
|
|
||||||
|
|
||||||
global bot_token
|
|
||||||
bot_token = os.getenv("DISCORD_BOT_TOKEN")
|
|
||||||
|
|
||||||
global bot_channel
|
|
||||||
bot_channel = os.getenv("DISCORD_BOT_CHANNEL", "skynet-server-health")
|
|
||||||
|
|
||||||
global bot_notify_role
|
|
||||||
bot_notify_role = os.getenv("DISCORD_BOT_NOTIFY_ROLE", "skynet-prod")
|
|
||||||
|
|
||||||
global portal_name
|
|
||||||
portal_name = os.getenv("SKYNET_SERVER_API")
|
|
||||||
|
|
||||||
global port
|
|
||||||
port = os.getenv("API_PORT", "9980")
|
|
||||||
|
|
||||||
global api_endpoint
|
|
||||||
api_endpoint = "http://{}:{}".format(get_docker_container_ip(CONTAINER_NAME), port)
|
|
||||||
|
|
||||||
siad.initialize()
|
siad.initialize()
|
||||||
|
|
||||||
global setup_done
|
global setup_done
|
||||||
setup_done = True
|
setup_done = True
|
||||||
|
|
||||||
return bot_token
|
|
||||||
|
|
||||||
|
|
||||||
# send_msg sends the msg to the specified discord channel. If force_notify is set to true it adds "@here".
|
# send_msg sends the msg to the specified discord channel. If force_notify is set to true it adds "@here".
|
||||||
async def send_msg(client, msg, force_notify=False, file=None):
|
async def send_msg(msg, force_notify=False, file=None):
|
||||||
await client.wait_until_ready()
|
try:
|
||||||
|
webhook_url = os.getenv("DISCORD_WEBHOOK_URL")
|
||||||
|
webhook_mention_user_id = os.getenv("DISCORD_MENTION_USER_ID")
|
||||||
|
webhook_mention_role_id = os.getenv("DISCORD_MENTION_ROLE_ID")
|
||||||
|
webhook = DiscordWebhook(url=webhook_url, rate_limit_retry=True)
|
||||||
|
|
||||||
guild = client.guilds[0]
|
# Add the portal name.
|
||||||
|
msg = "**{}**: {}".format(os.getenv("SKYNET_SERVER_API"), msg)
|
||||||
|
|
||||||
chan = None
|
if file and isinstance(file, str):
|
||||||
for c in guild.channels:
|
is_json = is_json_string(file)
|
||||||
if c.name == bot_channel:
|
content_type = "application/json" if is_json else "text/plain"
|
||||||
chan = c
|
ext = "json" if is_json else "txt"
|
||||||
break
|
filename = "{}-{}.{}".format(
|
||||||
|
CONTAINER_NAME, datetime.utcnow().strftime("%Y-%m-%d-%H:%M:%S"), ext
|
||||||
|
)
|
||||||
|
skylink = upload_to_skynet(file, filename, content_type=content_type)
|
||||||
|
if skylink:
|
||||||
|
msg = "{} {}".format(msg, skylink) # append skylink to message
|
||||||
|
else:
|
||||||
|
webhook.add_file(file=io.BytesIO(file.encode()), filename=filename)
|
||||||
|
|
||||||
if chan is None:
|
if force_notify and (webhook_mention_user_id or webhook_mention_role_id):
|
||||||
print("Can't find channel {}".format(bot_channel))
|
webhook.allowed_mentions = {
|
||||||
|
"users": [webhook_mention_user_id],
|
||||||
|
"roles": [webhook_mention_role_id],
|
||||||
|
}
|
||||||
|
msg = "{} /cc".format(msg) # separate message from mentions
|
||||||
|
if webhook_mention_role_id:
|
||||||
|
msg = "{} <@&{}>".format(msg, webhook_mention_role_id)
|
||||||
|
if webhook_mention_user_id:
|
||||||
|
msg = "{} <@{}>".format(msg, webhook_mention_user_id)
|
||||||
|
|
||||||
# Get the prod team role
|
webhook.content = msg
|
||||||
role = None
|
webhook.execute()
|
||||||
for r in guild.roles:
|
|
||||||
if r.name == bot_notify_role:
|
|
||||||
role = r
|
|
||||||
break
|
|
||||||
|
|
||||||
# Add the portal name.
|
print("msg > " + msg) # print message to std output for debugging purposes
|
||||||
msg = "**{}**: {}".format(portal_name, msg)
|
except:
|
||||||
|
print("Failed to send message!")
|
||||||
if file and isinstance(file, str):
|
print(traceback.format_exc())
|
||||||
is_json = is_json_string(file)
|
|
||||||
content_type = "application/json" if is_json else "text/plain"
|
|
||||||
ext = "json" if is_json else "txt"
|
|
||||||
filename = "{}-{}.{}".format(
|
|
||||||
CONTAINER_NAME, datetime.utcnow().strftime("%Y-%m-%d-%H:%M:%S"), ext
|
|
||||||
)
|
|
||||||
skylink = upload_to_skynet(file, filename, content_type=content_type)
|
|
||||||
if skylink:
|
|
||||||
msg = "{} {}".format(msg, skylink) # append skylink to message
|
|
||||||
file = None # clean file reference, we're using a skylink
|
|
||||||
else:
|
|
||||||
file = discord.File(
|
|
||||||
io.BytesIO(file.encode()), filename=filename
|
|
||||||
) # wrap text into discord file wrapper
|
|
||||||
|
|
||||||
if force_notify and role:
|
|
||||||
msg = "{} /cc {}".format(msg, role.mention)
|
|
||||||
|
|
||||||
await chan.send(msg, file=file)
|
|
||||||
|
|
||||||
|
|
||||||
def upload_to_skynet(contents, filename="file.txt", content_type="text/plain"):
|
def upload_to_skynet(contents, filename="file.txt", content_type="text/plain"):
|
||||||
|
|
|
@ -8,19 +8,7 @@ dispatches messages to a Discord channel.
|
||||||
import discord, traceback, asyncio, os
|
import discord, traceback, asyncio, os
|
||||||
from bot_utils import setup, send_msg, siad, sc_precision
|
from bot_utils import setup, send_msg, siad, sc_precision
|
||||||
|
|
||||||
bot_token = setup()
|
setup()
|
||||||
client = discord.Client()
|
|
||||||
|
|
||||||
|
|
||||||
async def exit_after(delay):
|
|
||||||
await asyncio.sleep(delay)
|
|
||||||
os._exit(0)
|
|
||||||
|
|
||||||
|
|
||||||
@client.event
|
|
||||||
async def on_ready():
|
|
||||||
await run_checks()
|
|
||||||
asyncio.create_task(exit_after(3))
|
|
||||||
|
|
||||||
|
|
||||||
async def run_checks():
|
async def run_checks():
|
||||||
|
@ -29,7 +17,7 @@ async def run_checks():
|
||||||
await check_funds()
|
await check_funds()
|
||||||
except: # catch all exceptions
|
except: # catch all exceptions
|
||||||
trace = traceback.format_exc()
|
trace = traceback.format_exc()
|
||||||
await send_msg(client, "```\n{}\n```".format(trace), force_notify=True)
|
await send_msg("```\n{}\n```".format(trace), force_notify=True)
|
||||||
|
|
||||||
|
|
||||||
# check_funds checks that the wallet is unlocked, that it has at least 1
|
# check_funds checks that the wallet is unlocked, that it has at least 1
|
||||||
|
@ -41,7 +29,7 @@ async def check_funds():
|
||||||
renter_get = siad.get_renter()
|
renter_get = siad.get_renter()
|
||||||
|
|
||||||
if not wallet_get["unlocked"]:
|
if not wallet_get["unlocked"]:
|
||||||
await send_msg(client, "Wallet locked", force_notify=True)
|
await send_msg("Wallet locked", force_notify=True)
|
||||||
return
|
return
|
||||||
|
|
||||||
confirmed_coins = int(wallet_get["confirmedsiacoinbalance"])
|
confirmed_coins = int(wallet_get["confirmedsiacoinbalance"])
|
||||||
|
@ -67,8 +55,10 @@ async def check_funds():
|
||||||
if balance < allowance_funds * WALLET_ALLOWANCE_THRESHOLD:
|
if balance < allowance_funds * WALLET_ALLOWANCE_THRESHOLD:
|
||||||
wallet_address_res = siad.get("/wallet/address")
|
wallet_address_res = siad.get("/wallet/address")
|
||||||
wallet_msg = "Address: {}".format(wallet_address_res["address"])
|
wallet_msg = "Address: {}".format(wallet_address_res["address"])
|
||||||
message = "__Wallet balance running low!__ {} {}".format(balance_msg, wallet_msg)
|
message = "__Wallet balance running low!__ {} {}".format(
|
||||||
return await send_msg(client, message, force_notify=True)
|
balance_msg, wallet_msg
|
||||||
|
)
|
||||||
|
return await send_msg(message, force_notify=True)
|
||||||
|
|
||||||
# Alert devs when only a fraction of the allowance is remaining.
|
# Alert devs when only a fraction of the allowance is remaining.
|
||||||
SPEND_THRESHOLD = 0.9
|
SPEND_THRESHOLD = 0.9
|
||||||
|
@ -76,10 +66,11 @@ async def check_funds():
|
||||||
message = "__More than {:.0%} of allowance spent!__ {}".format(
|
message = "__More than {:.0%} of allowance spent!__ {}".format(
|
||||||
SPEND_THRESHOLD, alloc_msg
|
SPEND_THRESHOLD, alloc_msg
|
||||||
)
|
)
|
||||||
return await send_msg(client, message, force_notify=True)
|
return await send_msg(message, force_notify=True)
|
||||||
|
|
||||||
# Send an informational heartbeat if all checks passed.
|
# Send an informational heartbeat if all checks passed.
|
||||||
await send_msg(client, "Funds checks passed. {} {}".format(balance_msg, alloc_msg))
|
await send_msg("Funds checks passed. {} {}".format(balance_msg, alloc_msg))
|
||||||
|
|
||||||
|
|
||||||
client.run(bot_token)
|
loop = asyncio.get_event_loop()
|
||||||
|
loop.run_until_complete(run_checks())
|
||||||
|
|
|
@ -38,20 +38,7 @@ GB = 1 << 30 # 1 GiB in bytes
|
||||||
FREE_DISK_SPACE_THRESHOLD = 50 * GB
|
FREE_DISK_SPACE_THRESHOLD = 50 * GB
|
||||||
FREE_DISK_SPACE_THRESHOLD_CRITICAL = 20 * GB
|
FREE_DISK_SPACE_THRESHOLD_CRITICAL = 20 * GB
|
||||||
|
|
||||||
bot_token = setup()
|
setup()
|
||||||
client = discord.Client()
|
|
||||||
|
|
||||||
|
|
||||||
# exit_after kills the script if it hasn't exited on its own after `delay` seconds
|
|
||||||
async def exit_after(delay):
|
|
||||||
await asyncio.sleep(delay)
|
|
||||||
os._exit(0)
|
|
||||||
|
|
||||||
|
|
||||||
@client.event
|
|
||||||
async def on_ready():
|
|
||||||
await run_checks()
|
|
||||||
asyncio.create_task(exit_after(3))
|
|
||||||
|
|
||||||
|
|
||||||
async def run_checks():
|
async def run_checks():
|
||||||
|
@ -66,7 +53,6 @@ async def run_checks():
|
||||||
trace = traceback.format_exc()
|
trace = traceback.format_exc()
|
||||||
print("[DEBUG] run_checks() failed.")
|
print("[DEBUG] run_checks() failed.")
|
||||||
await send_msg(
|
await send_msg(
|
||||||
client,
|
|
||||||
"Failed to run the portal health checks!",
|
"Failed to run the portal health checks!",
|
||||||
file=trace,
|
file=trace,
|
||||||
force_notify=True,
|
force_notify=True,
|
||||||
|
@ -84,7 +70,7 @@ async def check_load_average():
|
||||||
load_av = re.match(pattern, uptime_string).group(1)
|
load_av = re.match(pattern, uptime_string).group(1)
|
||||||
if float(load_av) > 10:
|
if float(load_av) > 10:
|
||||||
message = "High system load detected in uptime output: {}".format(uptime_string)
|
message = "High system load detected in uptime output: {}".format(uptime_string)
|
||||||
await send_msg(client, message, force_notify=True)
|
await send_msg(message, force_notify=True)
|
||||||
|
|
||||||
|
|
||||||
# check_disk checks the amount of free space on the /home partition and issues
|
# check_disk checks the amount of free space on the /home partition and issues
|
||||||
|
@ -109,7 +95,7 @@ async def check_disk():
|
||||||
break
|
break
|
||||||
if vol == "":
|
if vol == "":
|
||||||
message = "Failed to check free disk space! Didn't find a suitable mount point to check."
|
message = "Failed to check free disk space! Didn't find a suitable mount point to check."
|
||||||
return await send_msg(client, message, file=df)
|
return await send_msg(message, file=df)
|
||||||
|
|
||||||
# if we've reached a critical free disk space threshold we need to send proper notice
|
# if we've reached a critical free disk space threshold we need to send proper notice
|
||||||
# and shut down sia container so it doesn't get corrupted
|
# and shut down sia container so it doesn't get corrupted
|
||||||
|
@ -125,13 +111,13 @@ async def check_disk():
|
||||||
os.popen("docker exec health-check cli/disable")
|
os.popen("docker exec health-check cli/disable")
|
||||||
time.sleep(300) # wait 5 minutes to propagate dns changes
|
time.sleep(300) # wait 5 minutes to propagate dns changes
|
||||||
os.popen("docker stop sia") # stop sia container
|
os.popen("docker stop sia") # stop sia container
|
||||||
return await send_msg(client, message, force_notify=True)
|
return await send_msg(message, force_notify=True)
|
||||||
|
|
||||||
# if we're reached a free disk space threshold we need to send proper notice
|
# if we're reached a free disk space threshold we need to send proper notice
|
||||||
if int(volumes[vol]) < FREE_DISK_SPACE_THRESHOLD:
|
if int(volumes[vol]) < FREE_DISK_SPACE_THRESHOLD:
|
||||||
free_space_gb = "{:.2f}".format(int(volumes[vol]) / GB)
|
free_space_gb = "{:.2f}".format(int(volumes[vol]) / GB)
|
||||||
message = "WARNING! Low disk space: {}GiB".format(free_space_gb)
|
message = "WARNING! Low disk space: {}GiB".format(free_space_gb)
|
||||||
return await send_msg(client, message, force_notify=True)
|
return await send_msg(message, force_notify=True)
|
||||||
|
|
||||||
|
|
||||||
# check_health checks /health-check endpoint and reports recent issues
|
# check_health checks /health-check endpoint and reports recent issues
|
||||||
|
@ -142,7 +128,7 @@ async def check_health():
|
||||||
endpoint = "http://{}:{}".format(get_docker_container_ip("health-check"), 3100)
|
endpoint = "http://{}:{}".format(get_docker_container_ip("health-check"), 3100)
|
||||||
except:
|
except:
|
||||||
message = "Could not get health check service endpoint api!"
|
message = "Could not get health check service endpoint api!"
|
||||||
return await send_msg(client, message, force_notify=True)
|
return await send_msg(message, force_notify=True)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
res = requests.get(endpoint + "/health-check", verify=False)
|
res = requests.get(endpoint + "/health-check", verify=False)
|
||||||
|
@ -158,10 +144,12 @@ async def check_health():
|
||||||
except:
|
except:
|
||||||
message = traceback.format_exc()
|
message = traceback.format_exc()
|
||||||
message += "\n" + "Request url: " + res.url if res.url else "-"
|
message += "\n" + "Request url: " + res.url if res.url else "-"
|
||||||
message += "\n" + "Status code: " + str(res.status_code) if res.status_code else "-"
|
message += (
|
||||||
|
"\n" + "Status code: " + str(res.status_code) if res.status_code else "-"
|
||||||
|
)
|
||||||
message += "\n" + "Response body: " + res.text if res.text else "-"
|
message += "\n" + "Response body: " + res.text if res.text else "-"
|
||||||
return await send_msg(
|
return await send_msg(
|
||||||
client, "Failed to run health checks!", file=message, force_notify=True
|
"Failed to run health checks!", file=message, force_notify=True
|
||||||
)
|
)
|
||||||
|
|
||||||
critical_checks_total = 0
|
critical_checks_total = 0
|
||||||
|
@ -241,7 +229,7 @@ async def check_health():
|
||||||
or datetime.utcnow().hour == 1
|
or datetime.utcnow().hour == 1
|
||||||
):
|
):
|
||||||
return await send_msg(
|
return await send_msg(
|
||||||
client, message, file=failed_records_file, force_notify=force_notify
|
message, file=failed_records_file, force_notify=force_notify
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -340,7 +328,7 @@ async def check_alerts():
|
||||||
# on 1 AM
|
# on 1 AM
|
||||||
if force_notify or datetime.utcnow().hour == 1:
|
if force_notify or datetime.utcnow().hour == 1:
|
||||||
return await send_msg(
|
return await send_msg(
|
||||||
client, message, file=siac_alert_output, force_notify=force_notify
|
message, file=siac_alert_output, force_notify=force_notify
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -383,7 +371,8 @@ async def check_portal_size():
|
||||||
|
|
||||||
# send a message if we force notification, or just once daily (heartbeat) on 1 AM
|
# send a message if we force notification, or just once daily (heartbeat) on 1 AM
|
||||||
if force_notify or datetime.utcnow().hour == 1:
|
if force_notify or datetime.utcnow().hour == 1:
|
||||||
return await send_msg(client, message, force_notify=force_notify)
|
return await send_msg(message, force_notify=force_notify)
|
||||||
|
|
||||||
|
|
||||||
client.run(bot_token)
|
loop = asyncio.get_event_loop()
|
||||||
|
loop.run_until_complete(run_checks())
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
import discord, sys, traceback, io, os, asyncio
|
import sys, traceback, io, os, asyncio
|
||||||
from bot_utils import setup, send_msg, upload_to_skynet
|
from bot_utils import setup, send_msg, upload_to_skynet
|
||||||
from subprocess import Popen, PIPE
|
from subprocess import Popen, PIPE
|
||||||
|
|
||||||
|
@ -31,20 +31,7 @@ if len(sys.argv) > 3:
|
||||||
# a lower limit in order to leave some space for additional message text.
|
# a lower limit in order to leave some space for additional message text.
|
||||||
DISCORD_MAX_MESSAGE_LENGTH = 1900
|
DISCORD_MAX_MESSAGE_LENGTH = 1900
|
||||||
|
|
||||||
bot_token = setup()
|
setup()
|
||||||
client = discord.Client()
|
|
||||||
|
|
||||||
|
|
||||||
# exit_after kills the script if it hasn't exited on its own after `delay` seconds
|
|
||||||
async def exit_after(delay):
|
|
||||||
await asyncio.sleep(delay)
|
|
||||||
os._exit(0)
|
|
||||||
|
|
||||||
|
|
||||||
@client.event
|
|
||||||
async def on_ready():
|
|
||||||
await run_checks()
|
|
||||||
asyncio.create_task(exit_after(3))
|
|
||||||
|
|
||||||
|
|
||||||
async def run_checks():
|
async def run_checks():
|
||||||
|
@ -53,7 +40,7 @@ async def run_checks():
|
||||||
await check_docker_logs()
|
await check_docker_logs()
|
||||||
except: # catch all exceptions
|
except: # catch all exceptions
|
||||||
trace = traceback.format_exc()
|
trace = traceback.format_exc()
|
||||||
await send_msg(client, "```\n{}\n```".format(trace), force_notify=False)
|
await send_msg("```\n{}\n```".format(trace), force_notify=False)
|
||||||
|
|
||||||
|
|
||||||
# check_docker_logs checks the docker logs by filtering on the docker image name
|
# check_docker_logs checks the docker logs by filtering on the docker image name
|
||||||
|
@ -84,13 +71,12 @@ async def check_docker_logs():
|
||||||
pos = std_err.find("\n", -one_mb)
|
pos = std_err.find("\n", -one_mb)
|
||||||
std_err = std_err[pos + 1 :]
|
std_err = std_err[pos + 1 :]
|
||||||
return await send_msg(
|
return await send_msg(
|
||||||
client, "Error(s) found in log!", file=std_err, force_notify=True
|
"Error(s) found in log!", file=std_err, force_notify=True
|
||||||
)
|
)
|
||||||
|
|
||||||
# If there are any critical or severe errors. upload the whole log file.
|
# If there are any critical or severe errors. upload the whole log file.
|
||||||
if "Critical" in std_out or "Severe" in std_out or "panic" in std_out:
|
if "Critical" in std_out or "Severe" in std_out or "panic" in std_out:
|
||||||
return await send_msg(
|
return await send_msg(
|
||||||
client,
|
|
||||||
"Critical or Severe error found in log!",
|
"Critical or Severe error found in log!",
|
||||||
file=std_out,
|
file=std_out,
|
||||||
force_notify=True,
|
force_notify=True,
|
||||||
|
@ -98,9 +84,9 @@ async def check_docker_logs():
|
||||||
|
|
||||||
# No critical or severe errors, return a heartbeat type message
|
# No critical or severe errors, return a heartbeat type message
|
||||||
return await send_msg(
|
return await send_msg(
|
||||||
client,
|
|
||||||
"No critical or severe warnings in log since {} hours".format(CHECK_HOURS),
|
"No critical or severe warnings in log since {} hours".format(CHECK_HOURS),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
client.run(bot_token)
|
loop = asyncio.get_event_loop()
|
||||||
|
loop.run_until_complete(run_checks())
|
||||||
|
|
|
@ -32,7 +32,9 @@ docker-compose --version # sanity check
|
||||||
# * AWS_SECRET_ACCESS_KEY - (optional) if using route53 as a dns loadbalancer
|
# * AWS_SECRET_ACCESS_KEY - (optional) if using route53 as a dns loadbalancer
|
||||||
# * API_PORT - (optional) the port on which siad is listening, defaults to 9980
|
# * API_PORT - (optional) the port on which siad is listening, defaults to 9980
|
||||||
# * PORTAL_NAME - a string representing name of your portal e.g. `siasky.xyz` or `my skynet portal` (internal use only)
|
# * PORTAL_NAME - a string representing name of your portal e.g. `siasky.xyz` or `my skynet portal` (internal use only)
|
||||||
# * DISCORD_BOT_TOKEN - (optional) only required if you're using the discord notifications integration
|
# * DISCORD_WEBHOOK_URL - (required if using Discord notifications) discord webhook url (generate from discord app)
|
||||||
|
# * DISCORD_MENTION_USER_ID - (optional) add `/cc @user` mention to important messages from webhook (has to be id not user name)
|
||||||
|
# * DISCORD_MENTION_ROLE_ID - (optional) add `/cc @role` mention to important messages from webhook (has to be id not role name)
|
||||||
# * SKYNET_DB_USER - (optional) if using `accounts` this is the MongoDB username
|
# * SKYNET_DB_USER - (optional) if using `accounts` this is the MongoDB username
|
||||||
# * SKYNET_DB_PASS - (optional) if using `accounts` this is the MongoDB password
|
# * SKYNET_DB_PASS - (optional) if using `accounts` this is the MongoDB password
|
||||||
# * SKYNET_DB_HOST - (optional) if using `accounts` this is the MongoDB address or container name
|
# * SKYNET_DB_HOST - (optional) if using `accounts` this is the MongoDB address or container name
|
||||||
|
@ -44,7 +46,7 @@ docker-compose --version # sanity check
|
||||||
# * CR_CLUSTER_NODES - (optional) if using `accounts` the list of servers (with ports) which make up your CockroachDB cluster, e.g. `helsinki.siasky.net:26257,germany.siasky.net:26257,us-east.siasky.net:26257`
|
# * CR_CLUSTER_NODES - (optional) if using `accounts` the list of servers (with ports) which make up your CockroachDB cluster, e.g. `helsinki.siasky.net:26257,germany.siasky.net:26257,us-east.siasky.net:26257`
|
||||||
if ! [ -f /home/user/skynet-webportal/.env ]; then
|
if ! [ -f /home/user/skynet-webportal/.env ]; then
|
||||||
HSD_API_KEY=$(openssl rand -base64 32) # generate safe random key for handshake
|
HSD_API_KEY=$(openssl rand -base64 32) # generate safe random key for handshake
|
||||||
printf "SSL_CERTIFICATE_STRING=siasky.net, *.siasky.net, *.hns.siasky.net\nSKYNET_PORTAL_API=https://siasky.net\nSKYNET_SERVER_API=https://eu-dc-1.siasky.net\nSKYNET_DASHBOARD_URL=https://account.example.com\nEMAIL_ADDRESS=email@example.com\nSIA_WALLET_PASSWORD=\nHSD_API_KEY=${HSD_API_KEY}\nCLOUDFLARE_AUTH_TOKEN=\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\nPORTAL_NAME=\nDISCORD_BOT_TOKEN=\n" > /home/user/skynet-webportal/.env
|
printf "SSL_CERTIFICATE_STRING=siasky.net, *.siasky.net, *.hns.siasky.net\nSKYNET_PORTAL_API=https://siasky.net\nSKYNET_SERVER_API=https://eu-dc-1.siasky.net\nSKYNET_DASHBOARD_URL=https://account.example.com\nEMAIL_ADDRESS=email@example.com\nSIA_WALLET_PASSWORD=\nHSD_API_KEY=${HSD_API_KEY}\nCLOUDFLARE_AUTH_TOKEN=\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\nPORTAL_NAME=\DISCORD_WEBHOOK_URL=\nDISCORD_MENTION_USER_ID=\nDISCORD_MENTION_ROLE_ID=\n" > /home/user/skynet-webportal/.env
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Start docker container with nginx and client
|
# Start docker container with nginx and client
|
||||||
|
|
|
@ -5,7 +5,7 @@ set -e # exit on first error
|
||||||
sudo apt-get update
|
sudo apt-get update
|
||||||
sudo apt-get -y install python3-pip
|
sudo apt-get -y install python3-pip
|
||||||
|
|
||||||
pip3 install discord.py python-dotenv requests elasticsearch-curator
|
pip3 install DiscordWebhook python-dotenv requests elasticsearch-curator
|
||||||
|
|
||||||
# add cron entries to user crontab
|
# add cron entries to user crontab
|
||||||
crontab -u user /home/user/skynet-webportal/setup-scripts/support/crontab
|
crontab -u user /home/user/skynet-webportal/setup-scripts/support/crontab
|
||||||
|
|
|
@ -7,4 +7,8 @@ SIA_WALLET_PASSWORD=""
|
||||||
# portal specific environment variables
|
# portal specific environment variables
|
||||||
API_PORT="9980"
|
API_PORT="9980"
|
||||||
PORTAL_NAME=""
|
PORTAL_NAME=""
|
||||||
DISCORD_BOT_TOKEN=""
|
|
||||||
|
# discord integration
|
||||||
|
DISCORD_WEBHOOK_URL=""
|
||||||
|
DISCORD_MENTION_USER_ID=""
|
||||||
|
DISCORD_MENTION_ROLE_ID=""
|
||||||
|
|
Reference in New Issue