Merge pull request #633 from NebulousLabs/airtabl-blocklist

Airtable blocklist

setup-scripts/blocklist-airtable.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+
+import traceback, os, re, asyncio, requests, json, discord
+from bot_utils import setup, send_msg
+
+bot_token = setup()
+client = discord.Client()
+
+AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY")
+AIRTABLE_BASE = os.getenv("AIRTABLE_BASE", "app89plJvA9EqTJEc")
+AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE", "Table%201")
+AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD", "Link")
+
+
+def exec(command):
+    return os.popen(command).read().strip()
+
+
+async def block_skylinks_from_airtable():
+    print("Pulling blocked skylinks from Airtable via api integration")
+    headers = {"Authorization": "Bearer " + AIRTABLE_API_KEY}
+    skylinks = []
+    offset = None
+    while len(skylinks) == 0 or offset:
+        print("Requesting a batch of records from Airtable with " + (offset if offset else "empty") + " offset")
+        query = "&".join(["fields%5B%5D=" + AIRTABLE_FIELD, ("offset=" + offset) if offset else ""])
+        response = requests.get(
+            "https://api.airtable.com/v0/" + AIRTABLE_BASE + "/" + AIRTABLE_TABLE + "?" + query,
+            headers=headers,
+        )
+
+        if response.status_code != 200:
+            status_code = str(response.status_code)
+            response_text = response.text or "empty response"
+            message = "Airtable blocklist integration responded with code " + status_code + ": " + response_text
+            return print(message) or await send_msg(client, message, force_notify=False)
+
+        data = response.json()
+        skylinks = skylinks + [entry["fields"][AIRTABLE_FIELD] for entry in data["records"]]
+
+        if len(skylinks) == 0:
+            return print("Airtable returned 0 skylinks - make sure your configuration is correct")
+
+        offset = data.get("offset")
+
+    print("Airtable returned total " + str(len(skylinks)) + " skylinks to block")
+
+    skylinks_returned = skylinks
+    skylinks = [skylink for skylink in skylinks if re.search("^[a-zA-Z0-9_-]{46}$", skylink)]
+
+    if len(skylinks_returned) != len(skylinks):
+        invalid_skylinks = [str(skylink) for skylink in list(set(skylinks_returned) - set(skylinks))]
+        message = str(len(invalid_skylinks)) + " of the skylinks returned from Airtable are not valid"
+        print(message) or await send_msg(client, message, file=("\n".join(invalid_skylinks)))
+
+    apipassword = exec("docker exec sia cat /sia-data/apipassword")
+    ipaddress = exec("docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia")
+
+    print("Sending blocklist request to siad")
+    response = requests.post(
+        "http://" + ipaddress + ":9980/skynet/blocklist",
+        auth=("", apipassword),
+        headers={"user-agent": "Sia-Agent"},
+        data=json.dumps({"add": skylinks}),
+    )
+
+    if response.status_code == 204:
+        print("Siad blocklist successfully updated with provided skylink")
+    else:
+        status_code = str(response.status_code)
+        response_text = response.text or "empty response"
+        message = "Siad blocklist endpoint responded with code " + status_code + ": " + response_text
+        return print(message) or await send_msg(client, message, force_notify=False)
+
+    print("Searching nginx cache for blocked files")
+    cached_files_command = (
+        "/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -Els '^KEY: .*("
+        + "|".join(skylinks)
+        + ")'"
+    )
+    cached_files_count = int(exec('docker exec -it nginx bash -c "' + cached_files_command + ' | wc -l"'))
+
+    if cached_files_count == 0:
+        return print("No nginx cached files matching blocked skylinks were found")
+
+    exec('docker exec -it nginx bash -c "' + cached_files_command + ' | xargs rm"')
+    message = "Purged " + str(cached_files_count) + " blocklisted files from nginx cache"
+    return print(message) or await send_msg(client, message)
+
+
+async def exit_after(delay):
+    await asyncio.sleep(delay)
+    os._exit(0)
+
+
+@client.event
+async def on_ready():
+    try:
+        await block_skylinks_from_airtable()
+    except:  # catch all exceptions
+        message = "```\n{}\n```".format(traceback.format_exc())
+        await send_msg(client, message, force_notify=False)
+    asyncio.create_task(exit_after(3))
+
+
+client.run(bot_token)
+
+# --- BASH EQUIVALENT
+# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")
+# apipassword=$(docker exec sia cat /sia-data/apipassword)
+# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia)
+# curl -A "Sia-Agent" --user "":"${apipassword}" --data "{\"add\" : ${skylinks}}" "${ipaddress}:9980/skynet/blocklist"

setup-scripts/support/crontab

@@ -1,3 +1,4 @@
 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/skynet-webportal/.env
 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/skynet-webportal/.env sia 8
 0 * * * * /home/user/skynet-webportal/setup-scripts/health-checker.py /home/user/skynet-webportal/.env sia 1
+30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env