use webhook instead of discrod bot to send messages (#979)

* initial refactor

* do not use before define

* forgot to remove client

* test notification

* add /cc

* fix /cc

* fix /cc role

* fix /cc

* test file upload

* test file upload

* test file upload

* default to no mentions

* unformat

* replace discord with DiscordWebhook

* add readme

* don't fail on failures in message send
This commit is contained in:
Karol Wypchło 2021-07-16 13:12:58 +02:00 committed by GitHub
parent 3e516d8a46
commit 71f9d5280e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 112 additions and 165 deletions

View File

@ -92,7 +92,9 @@ At this point we have almost everything running, we just need to set up your wal
- `AWS_ACCESS_KEY_ID` (optional) if using route53 as a dns loadbalancer
- `AWS_SECRET_ACCESS_KEY` (optional) if using route53 as a dns loadbalancer
- `PORTAL_NAME` a string representing name of your portal e.g. `siasky.xyz` or `my skynet portal`
- `DISCORD_BOT_TOKEN` (optional) if you're using Discord notifications for health checks and such
- `DISCORD_WEBHOOK_URL` (required if using Discord notifications) discord webhook url (generate from discord app)
- `DISCORD_MENTION_USER_ID` (optional) add `/cc @user` mention to important messages from webhook (has to be id not user name)
- `DISCORD_MENTION_ROLE_ID` (optional) add `/cc @role` mention to important messages from webhook (has to be id not role name)
- `SKYNET_DB_USER` (optional) if using `accounts` this is the MongoDB username
- `SKYNET_DB_PASS` (optional) if using `accounts` this is the MongoDB password
- `SKYNET_DB_HOST` (optional) if using `accounts` this is the MongoDB address or container name

View File

@ -3,14 +3,20 @@
import traceback, os, re, asyncio, requests, json, discord
from bot_utils import setup, send_msg
bot_token = setup()
client = discord.Client()
setup()
AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY")
AIRTABLE_BASE = os.getenv("AIRTABLE_BASE", "app89plJvA9EqTJEc")
AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE", "Table%201")
AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD", "Link")
async def run_checks():
try:
await block_skylinks_from_airtable()
except: # catch all exceptions
trace = traceback.format_exc()
await send_msg("```\n{}\n```".format(trace), force_notify=True)
def exec(command):
return os.popen(command).read().strip()
@ -33,7 +39,7 @@ async def block_skylinks_from_airtable():
status_code = str(response.status_code)
response_text = response.text or "empty response"
message = "Airtable blocklist integration responded with code " + status_code + ": " + response_text
return print(message) or await send_msg(client, message, force_notify=False)
return await send_msg(message, force_notify=False)
data = response.json()
@ -53,7 +59,7 @@ async def block_skylinks_from_airtable():
if len(skylinks_returned) != len(skylinks):
invalid_skylinks = [str(skylink) for skylink in list(set(skylinks_returned) - set(skylinks))]
message = str(len(invalid_skylinks)) + " of the skylinks returned from Airtable are not valid"
print(message) or await send_msg(client, message, file=("\n".join(invalid_skylinks)))
await send_msg(message, file=("\n".join(invalid_skylinks)))
apipassword = exec("docker exec sia cat /sia-data/apipassword")
ipaddress = exec("docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia")
@ -72,7 +78,7 @@ async def block_skylinks_from_airtable():
status_code = str(response.status_code)
response_text = response.text or "empty response"
message = "Siad blocklist endpoint responded with code " + status_code + ": " + response_text
return print(message) or await send_msg(client, message, force_notify=False)
return await send_msg(message, force_notify=False)
print("Searching nginx cache for blocked files")
cached_files_count = 0
@ -89,25 +95,11 @@ async def block_skylinks_from_airtable():
exec('docker exec -it nginx bash -c "' + cached_files_command + ' | xargs rm"')
message = "Purged " + str(cached_files_count) + " blocklisted files from nginx cache"
return print(message) or await send_msg(client, message)
return await send_msg(message)
async def exit_after(delay):
await asyncio.sleep(delay)
os._exit(0)
@client.event
async def on_ready():
try:
await block_skylinks_from_airtable()
except: # catch all exceptions
message = "```\n{}\n```".format(traceback.format_exc())
await send_msg(client, message, force_notify=False)
asyncio.create_task(exit_after(3))
client.run(bot_token)
loop = asyncio.get_event_loop()
loop.run_until_complete(run_checks())
# --- BASH EQUIVALENT
# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")

View File

@ -4,22 +4,27 @@ from urllib.request import urlopen, Request
from dotenv import load_dotenv
from pathlib import Path
from datetime import datetime
from discord_webhook import DiscordWebhook
import urllib, json, os, traceback, discord, sys, re, subprocess, requests, io
# sc_precision is the number of hastings per siacoin
sc_precision = 10 ** 24
# Environment variable globals
api_endpoint, port, portal_name, bot_token, password = None, None, None, None, None
discord_client = None
setup_done = False
# Load dotenv file if possible.
# TODO: change all scripts to use named flags/params
if len(sys.argv) > 1:
env_path = Path(sys.argv[1])
load_dotenv(dotenv_path=env_path, override=True)
# Get the container name as an argument or use "sia" as default.
CONTAINER_NAME = "sia"
if len(sys.argv) > 2:
CONTAINER_NAME = sys.argv[2]
# sc_precision is the number of hastings per siacoin
sc_precision = 10 ** 24
# Environment variable globals
setup_done = False
# find out local siad ip by inspecting its docker container
def get_docker_container_ip(container_name):
ip_regex = re.compile(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$")
@ -31,6 +36,11 @@ def get_docker_container_ip(container_name):
return ip_regex.findall(output)[0]
# sia deamon local ip address with port
api_endpoint = "http://{}:{}".format(
get_docker_container_ip(CONTAINER_NAME), os.getenv("API_PORT", "9980")
)
# find siad api password by getting it out of the docker container
def get_api_password():
api_password_regex = re.compile(r"^\w+$")
@ -40,83 +50,54 @@ def get_api_password():
def setup():
# Load dotenv file if possible.
# TODO: change all scripts to use named flags/params
if len(sys.argv) > 1:
env_path = Path(sys.argv[1])
load_dotenv(dotenv_path=env_path, override=True)
global bot_token
bot_token = os.getenv("DISCORD_BOT_TOKEN")
global bot_channel
bot_channel = os.getenv("DISCORD_BOT_CHANNEL", "skynet-server-health")
global bot_notify_role
bot_notify_role = os.getenv("DISCORD_BOT_NOTIFY_ROLE", "skynet-prod")
global portal_name
portal_name = os.getenv("SKYNET_SERVER_API")
global port
port = os.getenv("API_PORT", "9980")
global api_endpoint
api_endpoint = "http://{}:{}".format(get_docker_container_ip(CONTAINER_NAME), port)
siad.initialize()
global setup_done
setup_done = True
return bot_token
# send_msg sends the msg to the specified discord channel. If force_notify is set to true it adds "@here".
async def send_msg(client, msg, force_notify=False, file=None):
await client.wait_until_ready()
async def send_msg(msg, force_notify=False, file=None):
try:
webhook_url = os.getenv("DISCORD_WEBHOOK_URL")
webhook_mention_user_id = os.getenv("DISCORD_MENTION_USER_ID")
webhook_mention_role_id = os.getenv("DISCORD_MENTION_ROLE_ID")
webhook = DiscordWebhook(url=webhook_url, rate_limit_retry=True)
guild = client.guilds[0]
# Add the portal name.
msg = "**{}**: {}".format(os.getenv("SKYNET_SERVER_API"), msg)
chan = None
for c in guild.channels:
if c.name == bot_channel:
chan = c
break
if file and isinstance(file, str):
is_json = is_json_string(file)
content_type = "application/json" if is_json else "text/plain"
ext = "json" if is_json else "txt"
filename = "{}-{}.{}".format(
CONTAINER_NAME, datetime.utcnow().strftime("%Y-%m-%d-%H:%M:%S"), ext
)
skylink = upload_to_skynet(file, filename, content_type=content_type)
if skylink:
msg = "{} {}".format(msg, skylink) # append skylink to message
else:
webhook.add_file(file=io.BytesIO(file.encode()), filename=filename)
if chan is None:
print("Can't find channel {}".format(bot_channel))
if force_notify and (webhook_mention_user_id or webhook_mention_role_id):
webhook.allowed_mentions = {
"users": [webhook_mention_user_id],
"roles": [webhook_mention_role_id],
}
msg = "{} /cc".format(msg) # separate message from mentions
if webhook_mention_role_id:
msg = "{} <@&{}>".format(msg, webhook_mention_role_id)
if webhook_mention_user_id:
msg = "{} <@{}>".format(msg, webhook_mention_user_id)
# Get the prod team role
role = None
for r in guild.roles:
if r.name == bot_notify_role:
role = r
break
webhook.content = msg
webhook.execute()
# Add the portal name.
msg = "**{}**: {}".format(portal_name, msg)
if file and isinstance(file, str):
is_json = is_json_string(file)
content_type = "application/json" if is_json else "text/plain"
ext = "json" if is_json else "txt"
filename = "{}-{}.{}".format(
CONTAINER_NAME, datetime.utcnow().strftime("%Y-%m-%d-%H:%M:%S"), ext
)
skylink = upload_to_skynet(file, filename, content_type=content_type)
if skylink:
msg = "{} {}".format(msg, skylink) # append skylink to message
file = None # clean file reference, we're using a skylink
else:
file = discord.File(
io.BytesIO(file.encode()), filename=filename
) # wrap text into discord file wrapper
if force_notify and role:
msg = "{} /cc {}".format(msg, role.mention)
await chan.send(msg, file=file)
print("msg > " + msg) # print message to std output for debugging purposes
except:
print("Failed to send message!")
print(traceback.format_exc())
def upload_to_skynet(contents, filename="file.txt", content_type="text/plain"):

View File

@ -8,19 +8,7 @@ dispatches messages to a Discord channel.
import discord, traceback, asyncio, os
from bot_utils import setup, send_msg, siad, sc_precision
bot_token = setup()
client = discord.Client()
async def exit_after(delay):
await asyncio.sleep(delay)
os._exit(0)
@client.event
async def on_ready():
await run_checks()
asyncio.create_task(exit_after(3))
setup()
async def run_checks():
@ -29,7 +17,7 @@ async def run_checks():
await check_funds()
except: # catch all exceptions
trace = traceback.format_exc()
await send_msg(client, "```\n{}\n```".format(trace), force_notify=True)
await send_msg("```\n{}\n```".format(trace), force_notify=True)
# check_funds checks that the wallet is unlocked, that it has at least 1
@ -41,7 +29,7 @@ async def check_funds():
renter_get = siad.get_renter()
if not wallet_get["unlocked"]:
await send_msg(client, "Wallet locked", force_notify=True)
await send_msg("Wallet locked", force_notify=True)
return
confirmed_coins = int(wallet_get["confirmedsiacoinbalance"])
@ -67,8 +55,10 @@ async def check_funds():
if balance < allowance_funds * WALLET_ALLOWANCE_THRESHOLD:
wallet_address_res = siad.get("/wallet/address")
wallet_msg = "Address: {}".format(wallet_address_res["address"])
message = "__Wallet balance running low!__ {} {}".format(balance_msg, wallet_msg)
return await send_msg(client, message, force_notify=True)
message = "__Wallet balance running low!__ {} {}".format(
balance_msg, wallet_msg
)
return await send_msg(message, force_notify=True)
# Alert devs when only a fraction of the allowance is remaining.
SPEND_THRESHOLD = 0.9
@ -76,10 +66,11 @@ async def check_funds():
message = "__More than {:.0%} of allowance spent!__ {}".format(
SPEND_THRESHOLD, alloc_msg
)
return await send_msg(client, message, force_notify=True)
return await send_msg(message, force_notify=True)
# Send an informational heartbeat if all checks passed.
await send_msg(client, "Funds checks passed. {} {}".format(balance_msg, alloc_msg))
await send_msg("Funds checks passed. {} {}".format(balance_msg, alloc_msg))
client.run(bot_token)
loop = asyncio.get_event_loop()
loop.run_until_complete(run_checks())

View File

@ -38,20 +38,7 @@ GB = 1 << 30 # 1 GiB in bytes
FREE_DISK_SPACE_THRESHOLD = 50 * GB
FREE_DISK_SPACE_THRESHOLD_CRITICAL = 20 * GB
bot_token = setup()
client = discord.Client()
# exit_after kills the script if it hasn't exited on its own after `delay` seconds
async def exit_after(delay):
await asyncio.sleep(delay)
os._exit(0)
@client.event
async def on_ready():
await run_checks()
asyncio.create_task(exit_after(3))
setup()
async def run_checks():
@ -66,7 +53,6 @@ async def run_checks():
trace = traceback.format_exc()
print("[DEBUG] run_checks() failed.")
await send_msg(
client,
"Failed to run the portal health checks!",
file=trace,
force_notify=True,
@ -84,7 +70,7 @@ async def check_load_average():
load_av = re.match(pattern, uptime_string).group(1)
if float(load_av) > 10:
message = "High system load detected in uptime output: {}".format(uptime_string)
await send_msg(client, message, force_notify=True)
await send_msg(message, force_notify=True)
# check_disk checks the amount of free space on the /home partition and issues
@ -109,7 +95,7 @@ async def check_disk():
break
if vol == "":
message = "Failed to check free disk space! Didn't find a suitable mount point to check."
return await send_msg(client, message, file=df)
return await send_msg(message, file=df)
# if we've reached a critical free disk space threshold we need to send proper notice
# and shut down sia container so it doesn't get corrupted
@ -125,13 +111,13 @@ async def check_disk():
os.popen("docker exec health-check cli/disable")
time.sleep(300) # wait 5 minutes to propagate dns changes
os.popen("docker stop sia") # stop sia container
return await send_msg(client, message, force_notify=True)
return await send_msg(message, force_notify=True)
# if we're reached a free disk space threshold we need to send proper notice
if int(volumes[vol]) < FREE_DISK_SPACE_THRESHOLD:
free_space_gb = "{:.2f}".format(int(volumes[vol]) / GB)
message = "WARNING! Low disk space: {}GiB".format(free_space_gb)
return await send_msg(client, message, force_notify=True)
return await send_msg(message, force_notify=True)
# check_health checks /health-check endpoint and reports recent issues
@ -142,7 +128,7 @@ async def check_health():
endpoint = "http://{}:{}".format(get_docker_container_ip("health-check"), 3100)
except:
message = "Could not get health check service endpoint api!"
return await send_msg(client, message, force_notify=True)
return await send_msg(message, force_notify=True)
try:
res = requests.get(endpoint + "/health-check", verify=False)
@ -158,10 +144,12 @@ async def check_health():
except:
message = traceback.format_exc()
message += "\n" + "Request url: " + res.url if res.url else "-"
message += "\n" + "Status code: " + str(res.status_code) if res.status_code else "-"
message += (
"\n" + "Status code: " + str(res.status_code) if res.status_code else "-"
)
message += "\n" + "Response body: " + res.text if res.text else "-"
return await send_msg(
client, "Failed to run health checks!", file=message, force_notify=True
"Failed to run health checks!", file=message, force_notify=True
)
critical_checks_total = 0
@ -241,7 +229,7 @@ async def check_health():
or datetime.utcnow().hour == 1
):
return await send_msg(
client, message, file=failed_records_file, force_notify=force_notify
message, file=failed_records_file, force_notify=force_notify
)
@ -340,7 +328,7 @@ async def check_alerts():
# on 1 AM
if force_notify or datetime.utcnow().hour == 1:
return await send_msg(
client, message, file=siac_alert_output, force_notify=force_notify
message, file=siac_alert_output, force_notify=force_notify
)
@ -383,7 +371,8 @@ async def check_portal_size():
# send a message if we force notification, or just once daily (heartbeat) on 1 AM
if force_notify or datetime.utcnow().hour == 1:
return await send_msg(client, message, force_notify=force_notify)
return await send_msg(message, force_notify=force_notify)
client.run(bot_token)
loop = asyncio.get_event_loop()
loop.run_until_complete(run_checks())

View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3
import discord, sys, traceback, io, os, asyncio
import sys, traceback, io, os, asyncio
from bot_utils import setup, send_msg, upload_to_skynet
from subprocess import Popen, PIPE
@ -31,20 +31,7 @@ if len(sys.argv) > 3:
# a lower limit in order to leave some space for additional message text.
DISCORD_MAX_MESSAGE_LENGTH = 1900
bot_token = setup()
client = discord.Client()
# exit_after kills the script if it hasn't exited on its own after `delay` seconds
async def exit_after(delay):
await asyncio.sleep(delay)
os._exit(0)
@client.event
async def on_ready():
await run_checks()
asyncio.create_task(exit_after(3))
setup()
async def run_checks():
@ -53,7 +40,7 @@ async def run_checks():
await check_docker_logs()
except: # catch all exceptions
trace = traceback.format_exc()
await send_msg(client, "```\n{}\n```".format(trace), force_notify=False)
await send_msg("```\n{}\n```".format(trace), force_notify=False)
# check_docker_logs checks the docker logs by filtering on the docker image name
@ -84,13 +71,12 @@ async def check_docker_logs():
pos = std_err.find("\n", -one_mb)
std_err = std_err[pos + 1 :]
return await send_msg(
client, "Error(s) found in log!", file=std_err, force_notify=True
"Error(s) found in log!", file=std_err, force_notify=True
)
# If there are any critical or severe errors. upload the whole log file.
if "Critical" in std_out or "Severe" in std_out or "panic" in std_out:
return await send_msg(
client,
"Critical or Severe error found in log!",
file=std_out,
force_notify=True,
@ -98,9 +84,9 @@ async def check_docker_logs():
# No critical or severe errors, return a heartbeat type message
return await send_msg(
client,
"No critical or severe warnings in log since {} hours".format(CHECK_HOURS),
)
client.run(bot_token)
loop = asyncio.get_event_loop()
loop.run_until_complete(run_checks())

View File

@ -32,7 +32,9 @@ docker-compose --version # sanity check
# * AWS_SECRET_ACCESS_KEY - (optional) if using route53 as a dns loadbalancer
# * API_PORT - (optional) the port on which siad is listening, defaults to 9980
# * PORTAL_NAME - a string representing name of your portal e.g. `siasky.xyz` or `my skynet portal` (internal use only)
# * DISCORD_BOT_TOKEN - (optional) only required if you're using the discord notifications integration
# * DISCORD_WEBHOOK_URL - (required if using Discord notifications) discord webhook url (generate from discord app)
# * DISCORD_MENTION_USER_ID - (optional) add `/cc @user` mention to important messages from webhook (has to be id not user name)
# * DISCORD_MENTION_ROLE_ID - (optional) add `/cc @role` mention to important messages from webhook (has to be id not role name)
# * SKYNET_DB_USER - (optional) if using `accounts` this is the MongoDB username
# * SKYNET_DB_PASS - (optional) if using `accounts` this is the MongoDB password
# * SKYNET_DB_HOST - (optional) if using `accounts` this is the MongoDB address or container name
@ -44,7 +46,7 @@ docker-compose --version # sanity check
# * CR_CLUSTER_NODES - (optional) if using `accounts` the list of servers (with ports) which make up your CockroachDB cluster, e.g. `helsinki.siasky.net:26257,germany.siasky.net:26257,us-east.siasky.net:26257`
if ! [ -f /home/user/skynet-webportal/.env ]; then
HSD_API_KEY=$(openssl rand -base64 32) # generate safe random key for handshake
printf "SSL_CERTIFICATE_STRING=siasky.net, *.siasky.net, *.hns.siasky.net\nSKYNET_PORTAL_API=https://siasky.net\nSKYNET_SERVER_API=https://eu-dc-1.siasky.net\nSKYNET_DASHBOARD_URL=https://account.example.com\nEMAIL_ADDRESS=email@example.com\nSIA_WALLET_PASSWORD=\nHSD_API_KEY=${HSD_API_KEY}\nCLOUDFLARE_AUTH_TOKEN=\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\nPORTAL_NAME=\nDISCORD_BOT_TOKEN=\n" > /home/user/skynet-webportal/.env
printf "SSL_CERTIFICATE_STRING=siasky.net, *.siasky.net, *.hns.siasky.net\nSKYNET_PORTAL_API=https://siasky.net\nSKYNET_SERVER_API=https://eu-dc-1.siasky.net\nSKYNET_DASHBOARD_URL=https://account.example.com\nEMAIL_ADDRESS=email@example.com\nSIA_WALLET_PASSWORD=\nHSD_API_KEY=${HSD_API_KEY}\nCLOUDFLARE_AUTH_TOKEN=\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\nPORTAL_NAME=\DISCORD_WEBHOOK_URL=\nDISCORD_MENTION_USER_ID=\nDISCORD_MENTION_ROLE_ID=\n" > /home/user/skynet-webportal/.env
fi
# Start docker container with nginx and client

View File

@ -5,7 +5,7 @@ set -e # exit on first error
sudo apt-get update
sudo apt-get -y install python3-pip
pip3 install discord.py python-dotenv requests elasticsearch-curator
pip3 install DiscordWebhook python-dotenv requests elasticsearch-curator
# add cron entries to user crontab
crontab -u user /home/user/skynet-webportal/setup-scripts/support/crontab

View File

@ -7,4 +7,8 @@ SIA_WALLET_PASSWORD=""
# portal specific environment variables
API_PORT="9980"
PORTAL_NAME=""
DISCORD_BOT_TOKEN=""
# discord integration
DISCORD_WEBHOOK_URL=""
DISCORD_MENTION_USER_ID=""
DISCORD_MENTION_ROLE_ID=""