use webhook instead of discrod bot to send messages (#979)

* initial refactor

* do not use before define

* forgot to remove client

* test notification

* add /cc

* fix /cc

* fix /cc role

* fix /cc

* test file upload

* test file upload

* test file upload

* default to no mentions

* unformat

* replace discord with DiscordWebhook

* add readme

* don't fail on failures in message send

setup-scripts/README.md

@@ -92,7 +92,9 @@ At this point we have almost everything running, we just need to set up your wal
    - `AWS_ACCESS_KEY_ID` (optional) if using route53 as a dns loadbalancer
    - `AWS_SECRET_ACCESS_KEY` (optional) if using route53 as a dns loadbalancer
    - `PORTAL_NAME` a string representing name of your portal e.g. `siasky.xyz` or `my skynet portal`
-   - `DISCORD_BOT_TOKEN` (optional) if you're using Discord notifications for health checks and such
+   - `DISCORD_WEBHOOK_URL` (required if using Discord notifications) discord webhook url (generate from discord app)
+   - `DISCORD_MENTION_USER_ID` (optional) add `/cc @user` mention to important messages from webhook (has to be id not user name)
+   - `DISCORD_MENTION_ROLE_ID` (optional) add `/cc @role` mention to important messages from webhook (has to be id not role name)
    - `SKYNET_DB_USER` (optional) if using `accounts` this is the MongoDB username
    - `SKYNET_DB_PASS` (optional) if using `accounts` this is the MongoDB password
    - `SKYNET_DB_HOST` (optional) if using `accounts` this is the MongoDB address or container name

setup-scripts/blocklist-airtable.py

@@ -3,14 +3,20 @@
 import traceback, os, re, asyncio, requests, json, discord
 from bot_utils import setup, send_msg
 
-bot_token = setup()
+setup()
-client = discord.Client()
 
 AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY")
 AIRTABLE_BASE = os.getenv("AIRTABLE_BASE", "app89plJvA9EqTJEc")
 AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE", "Table%201")
 AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD", "Link")
 
+async def run_checks():
+    try:
+        await block_skylinks_from_airtable()
+    except:  # catch all exceptions
+        trace = traceback.format_exc()
+        await send_msg("```\n{}\n```".format(trace), force_notify=True)
+
 
 def exec(command):
     return os.popen(command).read().strip()
@@ -33,7 +39,7 @@ async def block_skylinks_from_airtable():
             status_code = str(response.status_code)
             response_text = response.text or "empty response"
             message = "Airtable blocklist integration responded with code " + status_code + ": " + response_text
-            return print(message) or await send_msg(client, message, force_notify=False)
+            return await send_msg(message, force_notify=False)
 
         data = response.json()
 
@@ -53,7 +59,7 @@ async def block_skylinks_from_airtable():
     if len(skylinks_returned) != len(skylinks):
         invalid_skylinks = [str(skylink) for skylink in list(set(skylinks_returned) - set(skylinks))]
         message = str(len(invalid_skylinks)) + " of the skylinks returned from Airtable are not valid"
-        print(message) or await send_msg(client, message, file=("\n".join(invalid_skylinks)))
+        await send_msg(message, file=("\n".join(invalid_skylinks)))
 
     apipassword = exec("docker exec sia cat /sia-data/apipassword")
     ipaddress = exec("docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia")
@@ -72,7 +78,7 @@ async def block_skylinks_from_airtable():
         status_code = str(response.status_code)
         response_text = response.text or "empty response"
         message = "Siad blocklist endpoint responded with code " + status_code + ": " + response_text
-        return print(message) or await send_msg(client, message, force_notify=False)
+        return await send_msg(message, force_notify=False)
 
     print("Searching nginx cache for blocked files")
     cached_files_count = 0
@@ -89,25 +95,11 @@ async def block_skylinks_from_airtable():
 
     exec('docker exec -it nginx bash -c "' + cached_files_command + ' | xargs rm"')
     message = "Purged " + str(cached_files_count) + " blocklisted files from nginx cache"
-    return print(message) or await send_msg(client, message)
+    return await send_msg(message)
 
 
-async def exit_after(delay):
+loop = asyncio.get_event_loop()
-    await asyncio.sleep(delay)
+loop.run_until_complete(run_checks())
-    os._exit(0)
-
-
-@client.event
-async def on_ready():
-    try:
-        await block_skylinks_from_airtable()
-    except:  # catch all exceptions
-        message = "```\n{}\n```".format(traceback.format_exc())
-        await send_msg(client, message, force_notify=False)
-    asyncio.create_task(exit_after(3))
-
-
-client.run(bot_token)
 
 # --- BASH EQUIVALENT
 # skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")

setup-scripts/bot_utils.py

@@ -4,22 +4,27 @@ from urllib.request import urlopen, Request
 from dotenv import load_dotenv
 from pathlib import Path
 from datetime import datetime
+from discord_webhook import DiscordWebhook
 
 import urllib, json, os, traceback, discord, sys, re, subprocess, requests, io
 
-# sc_precision is the number of hastings per siacoin
+# Load dotenv file if possible.
-sc_precision = 10 ** 24
+# TODO: change all scripts to use named flags/params
-
+if len(sys.argv) > 1:
-# Environment variable globals
+    env_path = Path(sys.argv[1])
-api_endpoint, port, portal_name, bot_token, password = None, None, None, None, None
+    load_dotenv(dotenv_path=env_path, override=True)
-discord_client = None
-setup_done = False
 
 # Get the container name as an argument or use "sia" as default.
 CONTAINER_NAME = "sia"
 if len(sys.argv) > 2:
     CONTAINER_NAME = sys.argv[2]
 
+# sc_precision is the number of hastings per siacoin
+sc_precision = 10 ** 24
+
+# Environment variable globals
+setup_done = False
+
 # find out local siad ip by inspecting its docker container
 def get_docker_container_ip(container_name):
     ip_regex = re.compile(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$")
@@ -31,6 +36,11 @@ def get_docker_container_ip(container_name):
     return ip_regex.findall(output)[0]
 
 
+# sia deamon local ip address with port
+api_endpoint = "http://{}:{}".format(
+    get_docker_container_ip(CONTAINER_NAME), os.getenv("API_PORT", "9980")
+)
+
 # find siad api password by getting it out of the docker container
 def get_api_password():
     api_password_regex = re.compile(r"^\w+$")
@@ -40,62 +50,22 @@ def get_api_password():
 
 
 def setup():
-    # Load dotenv file if possible.
-    # TODO: change all scripts to use named flags/params
-    if len(sys.argv) > 1:
-        env_path = Path(sys.argv[1])
-        load_dotenv(dotenv_path=env_path, override=True)
-
-    global bot_token
-    bot_token = os.getenv("DISCORD_BOT_TOKEN")
-
-    global bot_channel
-    bot_channel = os.getenv("DISCORD_BOT_CHANNEL", "skynet-server-health")
-
-    global bot_notify_role
-    bot_notify_role = os.getenv("DISCORD_BOT_NOTIFY_ROLE", "skynet-prod")
-
-    global portal_name
-    portal_name = os.getenv("SKYNET_SERVER_API")
-
-    global port
-    port = os.getenv("API_PORT", "9980")
-
-    global api_endpoint
-    api_endpoint = "http://{}:{}".format(get_docker_container_ip(CONTAINER_NAME), port)
-
     siad.initialize()
 
     global setup_done
     setup_done = True
 
-    return bot_token
-
 
 # send_msg sends the msg to the specified discord channel. If force_notify is set to true it adds "@here".
-async def send_msg(client, msg, force_notify=False, file=None):
+async def send_msg(msg, force_notify=False, file=None):
-    await client.wait_until_ready()
+    try:
-
+        webhook_url = os.getenv("DISCORD_WEBHOOK_URL")
-    guild = client.guilds[0]
+        webhook_mention_user_id = os.getenv("DISCORD_MENTION_USER_ID")
-
+        webhook_mention_role_id = os.getenv("DISCORD_MENTION_ROLE_ID")
-    chan = None
+        webhook = DiscordWebhook(url=webhook_url, rate_limit_retry=True)
-    for c in guild.channels:
-        if c.name == bot_channel:
-            chan = c
-            break
-
-    if chan is None:
-        print("Can't find channel {}".format(bot_channel))
-
-    # Get the prod team role
-    role = None
-    for r in guild.roles:
-        if r.name == bot_notify_role:
-            role = r
-            break
 
         # Add the portal name.
-    msg = "**{}**: {}".format(portal_name, msg)
+        msg = "**{}**: {}".format(os.getenv("SKYNET_SERVER_API"), msg)
 
         if file and isinstance(file, str):
             is_json = is_json_string(file)
@@ -107,16 +77,27 @@ async def send_msg(client, msg, force_notify=False, file=None):
             skylink = upload_to_skynet(file, filename, content_type=content_type)
             if skylink:
                 msg = "{} {}".format(msg, skylink)  # append skylink to message
-            file = None  # clean file reference, we're using a skylink
             else:
-            file = discord.File(
+                webhook.add_file(file=io.BytesIO(file.encode()), filename=filename)
-                io.BytesIO(file.encode()), filename=filename
-            )  # wrap text into discord file wrapper
 
-    if force_notify and role:
+        if force_notify and (webhook_mention_user_id or webhook_mention_role_id):
-        msg = "{} /cc {}".format(msg, role.mention)
+            webhook.allowed_mentions = {
+                "users": [webhook_mention_user_id],
+                "roles": [webhook_mention_role_id],
+            }
+            msg = "{} /cc".format(msg)  # separate message from mentions
+            if webhook_mention_role_id:
+                msg = "{} <@&{}>".format(msg, webhook_mention_role_id)
+            if webhook_mention_user_id:
+                msg = "{} <@{}>".format(msg, webhook_mention_user_id)
 
-    await chan.send(msg, file=file)
+        webhook.content = msg
+        webhook.execute()
+
+        print("msg > " + msg)  # print message to std output for debugging purposes
+    except:
+        print("Failed to send message!")
+        print(traceback.format_exc())
 
 
 def upload_to_skynet(contents, filename="file.txt", content_type="text/plain"):

setup-scripts/funds-checker.py

@@ -8,19 +8,7 @@ dispatches messages to a Discord channel.
 import discord, traceback, asyncio, os
 from bot_utils import setup, send_msg, siad, sc_precision
 
-bot_token = setup()
+setup()
-client = discord.Client()
-
-
-async def exit_after(delay):
-    await asyncio.sleep(delay)
-    os._exit(0)
-
-
-@client.event
-async def on_ready():
-    await run_checks()
-    asyncio.create_task(exit_after(3))
 
 
 async def run_checks():
@@ -29,7 +17,7 @@ async def run_checks():
         await check_funds()
     except:  # catch all exceptions
         trace = traceback.format_exc()
-        await send_msg(client, "```\n{}\n```".format(trace), force_notify=True)
+        await send_msg("```\n{}\n```".format(trace), force_notify=True)
 
 
 # check_funds checks that the wallet is unlocked, that it has at least 1
@@ -41,7 +29,7 @@ async def check_funds():
     renter_get = siad.get_renter()
 
     if not wallet_get["unlocked"]:
-        await send_msg(client, "Wallet locked", force_notify=True)
+        await send_msg("Wallet locked", force_notify=True)
         return
 
     confirmed_coins = int(wallet_get["confirmedsiacoinbalance"])
@@ -67,8 +55,10 @@ async def check_funds():
     if balance < allowance_funds * WALLET_ALLOWANCE_THRESHOLD:
         wallet_address_res = siad.get("/wallet/address")
         wallet_msg = "Address: {}".format(wallet_address_res["address"])
-        message = "__Wallet balance running low!__ {} {}".format(balance_msg, wallet_msg)
+        message = "__Wallet balance running low!__ {} {}".format(
-        return await send_msg(client, message, force_notify=True)
+            balance_msg, wallet_msg
+        )
+        return await send_msg(message, force_notify=True)
 
     # Alert devs when only a fraction of the allowance is remaining.
     SPEND_THRESHOLD = 0.9
@@ -76,10 +66,11 @@ async def check_funds():
         message = "__More than {:.0%} of allowance spent!__ {}".format(
             SPEND_THRESHOLD, alloc_msg
         )
-        return await send_msg(client, message, force_notify=True)
+        return await send_msg(message, force_notify=True)
 
     # Send an informational heartbeat if all checks passed.
-    await send_msg(client, "Funds checks passed. {} {}".format(balance_msg, alloc_msg))
+    await send_msg("Funds checks passed. {} {}".format(balance_msg, alloc_msg))
 
 
-client.run(bot_token)
+loop = asyncio.get_event_loop()
+loop.run_until_complete(run_checks())

setup-scripts/health-checker.py

@@ -38,20 +38,7 @@ GB = 1 << 30  # 1 GiB in bytes
 FREE_DISK_SPACE_THRESHOLD = 50 * GB
 FREE_DISK_SPACE_THRESHOLD_CRITICAL = 20 * GB
 
-bot_token = setup()
+setup()
-client = discord.Client()
-
-
-# exit_after kills the script if it hasn't exited on its own after `delay` seconds
-async def exit_after(delay):
-    await asyncio.sleep(delay)
-    os._exit(0)
-
-
-@client.event
-async def on_ready():
-    await run_checks()
-    asyncio.create_task(exit_after(3))
 
 
 async def run_checks():
@@ -66,7 +53,6 @@ async def run_checks():
         trace = traceback.format_exc()
         print("[DEBUG] run_checks() failed.")
         await send_msg(
-            client,
             "Failed to run the portal health checks!",
             file=trace,
             force_notify=True,
@@ -84,7 +70,7 @@ async def check_load_average():
     load_av = re.match(pattern, uptime_string).group(1)
     if float(load_av) > 10:
         message = "High system load detected in uptime output: {}".format(uptime_string)
-        await send_msg(client, message, force_notify=True)
+        await send_msg(message, force_notify=True)
 
 
 # check_disk checks the amount of free space on the /home partition and issues
@@ -109,7 +95,7 @@ async def check_disk():
             break
     if vol == "":
         message = "Failed to check free disk space! Didn't find a suitable mount point to check."
-        return await send_msg(client, message, file=df)
+        return await send_msg(message, file=df)
 
     # if we've reached a critical free disk space threshold we need to send proper notice
     # and shut down sia container so it doesn't get corrupted
@@ -125,13 +111,13 @@ async def check_disk():
             os.popen("docker exec health-check cli/disable")
             time.sleep(300)  # wait 5 minutes to propagate dns changes
             os.popen("docker stop sia")  # stop sia container
-        return await send_msg(client, message, force_notify=True)
+        return await send_msg(message, force_notify=True)
 
     # if we're reached a free disk space threshold we need to send proper notice
     if int(volumes[vol]) < FREE_DISK_SPACE_THRESHOLD:
         free_space_gb = "{:.2f}".format(int(volumes[vol]) / GB)
         message = "WARNING! Low disk space: {}GiB".format(free_space_gb)
-        return await send_msg(client, message, force_notify=True)
+        return await send_msg(message, force_notify=True)
 
 
 # check_health checks /health-check endpoint and reports recent issues
@@ -142,7 +128,7 @@ async def check_health():
         endpoint = "http://{}:{}".format(get_docker_container_ip("health-check"), 3100)
     except:
         message = "Could not get health check service endpoint api!"
-        return await send_msg(client, message, force_notify=True)
+        return await send_msg(message, force_notify=True)
 
     try:
         res = requests.get(endpoint + "/health-check", verify=False)
@@ -158,10 +144,12 @@ async def check_health():
     except:
         message = traceback.format_exc()
         message += "\n" + "Request url: " + res.url if res.url else "-"
-        message += "\n" + "Status code: " + str(res.status_code) if res.status_code else "-"
+        message += (
+            "\n" + "Status code: " + str(res.status_code) if res.status_code else "-"
+        )
         message += "\n" + "Response body: " + res.text if res.text else "-"
         return await send_msg(
-            client, "Failed to run health checks!", file=message, force_notify=True
+            "Failed to run health checks!", file=message, force_notify=True
         )
 
     critical_checks_total = 0
@@ -241,7 +229,7 @@ async def check_health():
         or datetime.utcnow().hour == 1
     ):
         return await send_msg(
-            client, message, file=failed_records_file, force_notify=force_notify
+            message, file=failed_records_file, force_notify=force_notify
         )
 
 
@@ -340,7 +328,7 @@ async def check_alerts():
     # on 1 AM
     if force_notify or datetime.utcnow().hour == 1:
         return await send_msg(
-            client, message, file=siac_alert_output, force_notify=force_notify
+            message, file=siac_alert_output, force_notify=force_notify
         )
 
 
@@ -383,7 +371,8 @@ async def check_portal_size():
 
     # send a message if we force notification, or just once daily (heartbeat) on 1 AM
     if force_notify or datetime.utcnow().hour == 1:
-        return await send_msg(client, message, force_notify=force_notify)
+        return await send_msg(message, force_notify=force_notify)
 
 
-client.run(bot_token)
+loop = asyncio.get_event_loop()
+loop.run_until_complete(run_checks())

setup-scripts/log-checker.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-import discord, sys, traceback, io, os, asyncio
+import sys, traceback, io, os, asyncio
 from bot_utils import setup, send_msg, upload_to_skynet
 from subprocess import Popen, PIPE
 
@@ -31,20 +31,7 @@ if len(sys.argv) > 3:
 # a lower limit in order to leave some space for additional message text.
 DISCORD_MAX_MESSAGE_LENGTH = 1900
 
-bot_token = setup()
+setup()
-client = discord.Client()
-
-
-# exit_after kills the script if it hasn't exited on its own after `delay` seconds
-async def exit_after(delay):
-    await asyncio.sleep(delay)
-    os._exit(0)
-
-
-@client.event
-async def on_ready():
-    await run_checks()
-    asyncio.create_task(exit_after(3))
 
 
 async def run_checks():
@@ -53,7 +40,7 @@ async def run_checks():
         await check_docker_logs()
     except:  # catch all exceptions
         trace = traceback.format_exc()
-        await send_msg(client, "```\n{}\n```".format(trace), force_notify=False)
+        await send_msg("```\n{}\n```".format(trace), force_notify=False)
 
 
 # check_docker_logs checks the docker logs by filtering on the docker image name
@@ -84,13 +71,12 @@ async def check_docker_logs():
             pos = std_err.find("\n", -one_mb)
             std_err = std_err[pos + 1 :]
             return await send_msg(
-                client, "Error(s) found in log!", file=std_err, force_notify=True
+                "Error(s) found in log!", file=std_err, force_notify=True
             )
 
     # If there are any critical or severe errors. upload the whole log file.
     if "Critical" in std_out or "Severe" in std_out or "panic" in std_out:
         return await send_msg(
-            client,
             "Critical or Severe error found in log!",
             file=std_out,
             force_notify=True,
@@ -98,9 +84,9 @@ async def check_docker_logs():
 
     # No critical or severe errors, return a heartbeat type message
     return await send_msg(
-        client,
         "No critical or severe warnings in log since {} hours".format(CHECK_HOURS),
     )
 
 
-client.run(bot_token)
+loop = asyncio.get_event_loop()
+loop.run_until_complete(run_checks())

setup-scripts/setup-docker-services.sh

@@ -32,7 +32,9 @@ docker-compose --version # sanity check
 # * AWS_SECRET_ACCESS_KEY - (optional) if using route53 as a dns loadbalancer
 # * API_PORT - (optional) the port on which siad is listening, defaults to 9980
 # * PORTAL_NAME - a string representing name of your portal e.g. `siasky.xyz` or `my skynet portal` (internal use only)
-# * DISCORD_BOT_TOKEN - (optional) only required if you're using the discord notifications integration
+# * DISCORD_WEBHOOK_URL - (required if using Discord notifications) discord webhook url (generate from discord app)
+# * DISCORD_MENTION_USER_ID - (optional) add `/cc @user` mention to important messages from webhook (has to be id not user name)
+# * DISCORD_MENTION_ROLE_ID - (optional) add `/cc @role` mention to important messages from webhook (has to be id not role name)
 # * SKYNET_DB_USER - (optional) if using `accounts` this is the MongoDB username
 # * SKYNET_DB_PASS - (optional) if using `accounts` this is the MongoDB password
 # * SKYNET_DB_HOST - (optional) if using `accounts` this is the MongoDB address or container name
@@ -44,7 +46,7 @@ docker-compose --version # sanity check
 # * CR_CLUSTER_NODES - (optional) if using `accounts` the list of servers (with ports) which make up your CockroachDB cluster, e.g. `helsinki.siasky.net:26257,germany.siasky.net:26257,us-east.siasky.net:26257`
 if ! [ -f /home/user/skynet-webportal/.env ]; then
     HSD_API_KEY=$(openssl rand -base64 32) # generate safe random key for handshake
-    printf "SSL_CERTIFICATE_STRING=siasky.net, *.siasky.net, *.hns.siasky.net\nSKYNET_PORTAL_API=https://siasky.net\nSKYNET_SERVER_API=https://eu-dc-1.siasky.net\nSKYNET_DASHBOARD_URL=https://account.example.com\nEMAIL_ADDRESS=email@example.com\nSIA_WALLET_PASSWORD=\nHSD_API_KEY=${HSD_API_KEY}\nCLOUDFLARE_AUTH_TOKEN=\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\nPORTAL_NAME=\nDISCORD_BOT_TOKEN=\n" > /home/user/skynet-webportal/.env
+    printf "SSL_CERTIFICATE_STRING=siasky.net, *.siasky.net, *.hns.siasky.net\nSKYNET_PORTAL_API=https://siasky.net\nSKYNET_SERVER_API=https://eu-dc-1.siasky.net\nSKYNET_DASHBOARD_URL=https://account.example.com\nEMAIL_ADDRESS=email@example.com\nSIA_WALLET_PASSWORD=\nHSD_API_KEY=${HSD_API_KEY}\nCLOUDFLARE_AUTH_TOKEN=\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\nPORTAL_NAME=\DISCORD_WEBHOOK_URL=\nDISCORD_MENTION_USER_ID=\nDISCORD_MENTION_ROLE_ID=\n" > /home/user/skynet-webportal/.env
 fi
 
 # Start docker container with nginx and client

setup-scripts/setup-health-check-scripts.sh

@@ -5,7 +5,7 @@ set -e # exit on first error
 sudo apt-get update
 sudo apt-get -y install python3-pip
 
-pip3 install discord.py python-dotenv requests elasticsearch-curator
+pip3 install DiscordWebhook python-dotenv requests elasticsearch-curator
 
 # add cron entries to user crontab
 crontab -u user /home/user/skynet-webportal/setup-scripts/support/crontab

setup-scripts/support/sia.env

@@ -7,4 +7,8 @@ SIA_WALLET_PASSWORD=""
 # portal specific environment variables
 API_PORT="9980"
 PORTAL_NAME=""
-DISCORD_BOT_TOKEN=""
+
+# discord integration
+DISCORD_WEBHOOK_URL=""
+DISCORD_MENTION_USER_ID=""
+DISCORD_MENTION_ROLE_ID=""