Revert "remove outdated blocklist scripts"

This reverts commit 0a26516bdf.

scripts/README.md

@@ -4,6 +4,10 @@ This package contains useful scripts for managing a Skynet Webportal.
 
 ## Available Scripts
 
+**blocklist-skylink.sh**\
+The `blocklist-skylink.sh` script adds a skylink to the blocklist on all
+servers.
+
 **maintenance-upgrade.sh**\
 The `maintenance-upgrade.sh` script upgrades the docker images for nodes on
 a maintenance server.

scripts/blocklist-skylink.sh

@@ -0,0 +1,49 @@
+#! /usr/bin/env bash
+
+# This script is for manual skylink blocking. It accepts either a single 
+# skylink or a file containing list of skylinks. The script is intented
+# for manual use and it should be run locally on each skynet webportal server. 
+# The automatic script that is used to continuously sync an Airtable sheet 
+# list with the blocklist on the web portals is /setup-scripts/blocklist-airtable.py 
+
+set -e # exit on first error
+
+if [ -z "$1" ]; then
+    echo "Please provide either a skylink or a file with skylinks separated by new lines" && exit 1
+fi
+
+#########################################################
+# read either a file containing skylinks separated by new
+# lines or a single skylink and put them in an array
+#########################################################
+skylinks=()
+if test -f "$1"; then
+    line_number=1
+
+    # Read file including the last line even when it doesn't end with newline
+    while IFS="" read -r line || [ -n "$line" ];
+    do
+        if [[ $line =~ (^[a-zA-Z0-9_-]{46}$) ]]; then
+            skylinks+=("$line")
+        else
+            echo "Incorrect skylink at line ${line_number}: $line" && exit 1
+        fi
+        let line_number+=1
+    done < $1;
+else
+    skylinks=("$1") # just single skylink passed as input argument
+fi
+
+# get local skyd ip adress
+ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia)
+
+# get sia api password either from env variable if exists or from apipassword file in sia-data directory
+apipassword=$(docker exec sia sh -c '[ ! -z "${SIA_API_PASSWORD}" ] && echo ${SIA_API_PASSWORD} || $(cat /sia-data/apipassword | tr -d '\n')')
+
+# iterate over provided skylinks and block them one by one
+for skylink in "${skylinks[@]}"; do
+    echo "> Blocking ${skylink} ... "
+
+    # POST /skynet/blocklist always returns 200 and in case of failure print error message
+    curl -A Sia-Agent -u "":${apipassword} --data "{\"add\":[\"$skylink\"]}" "http://${ipaddress}:9980/skynet/blocklist"
+done

setup-scripts/README.md

@@ -1,6 +1,8 @@
 # Skynet Portal Setup Scripts
 
-> :warning: This documentation is outdated and should be used for reference only. Portal setup documentation is located at https://portal-docs.skynetlabs.com/.
+> :warning: This documentation is outdated and should be used for reference
+only. Portal setup documentation is located at
+https://portal-docs.skynetlabs.com/.
 
 This directory contains a setup guide and scripts that will install and
 configure some basic requirements for running a Skynet Portal. The assumption is
@@ -33,6 +35,7 @@ You may want to fork this repository and replace ssh keys in
   - [funds-checker](funds-checker.py): script that checks wallet balance and sends status messages to discord periodically
   - [health-checker](health-checker.py): script that monitors health-check service for server health issues and reports them to discord periodically
   - [log-checker](log-checker.py): script that scans siad logs for critical errors and reports them to discord periodically
+- [blocklist-skylink](../scripts/blocklist-skylink.sh): script that can be run locally from a machine that has access to all your skynet portal servers that blocklists provided skylink and prunes nginx cache to ensure it's not available any more (that is a bit much but that's the best we can do right now without paid nginx version) - if you want to use it, make sure to adjust the server addresses
 
 ### Step 1: setting up server user
 

setup-scripts/blocklist-airtable.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+
+from bot_utils import get_api_password, setup, send_msg
+from random import randint
+from time import sleep
+
+import traceback
+import os
+import sys
+import asyncio
+import requests
+import json
+
+from requests.auth import HTTPBasicAuth
+
+setup()
+
+
+AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY")
+AIRTABLE_BASE = os.getenv("AIRTABLE_BASE")
+AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE")
+AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD")
+
+# Check environment variables are defined
+for value in [AIRTABLE_API_KEY, AIRTABLE_BASE, AIRTABLE_TABLE, AIRTABLE_FIELD]:
+    if not value:
+        sys.exit("Configuration error: Missing AirTable environment variable.")
+
+
+async def run_checks():
+    try:
+        await block_skylinks_from_airtable()
+    except:  # catch all exceptions
+        trace = traceback.format_exc()
+        await send_msg("```\n{}\n```".format(trace), force_notify=True)
+
+
+def exec(command):
+    return os.popen(command).read().strip()
+
+
+async def block_skylinks_from_airtable():
+    # Get sia IP before doing anything else. If this step fails we don't
+    # need to continue with the execution of the script.
+    ipaddress = exec(
+        "docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia"
+    )
+
+    if ipaddress == "":
+        print("Skyd IP could not be detected. Exiting.")
+        return
+
+    print("Pulling blocked skylinks from Airtable via api integration")
+    headers = {"Authorization": "Bearer " + AIRTABLE_API_KEY}
+    skylinks = []
+    offset = None
+    retry = 0
+    while len(skylinks) == 0 or offset:
+        print(
+            "Requesting a batch of records from Airtable with "
+            + (offset if offset else "empty")
+            + " offset"
+            + (" (retry " + str(retry) + ")" if retry else "")
+        )
+        query = "&".join(
+            ["fields%5B%5D=" + AIRTABLE_FIELD, ("offset=" + offset) if offset else ""]
+        )
+        response = requests.get(
+            "https://api.airtable.com/v0/"
+            + AIRTABLE_BASE
+            + "/"
+            + AIRTABLE_TABLE
+            + "?"
+            + query,
+            headers=headers,
+        )
+
+        # rate limited - sleep for 2-10 secs and retry (up to 100 times, ~10 minutes)
+        # https://support.airtable.com/hc/en-us/articles/203313985-Public-REST-API
+        # > 5 requests per second, per base
+        if response.status_code == 429:
+            if retry < 100:
+                retry = retry + 1
+                sleep(randint(1, 10))
+                continue
+            else:
+                return await send_msg(
+                    "Airtable: too many retries, aborting!", force_notify=True
+                )
+        retry = 0  # reset retry counter
+
+        if response.status_code != 200:
+            status_code = str(response.status_code)
+            response_text = response.text or "empty response"
+            message = (
+                "Airtable blocklist integration responded with code "
+                + status_code
+                + ": "
+                + response_text
+            )
+            return await send_msg(message, force_notify=False)
+
+        data = response.json()
+
+        if len(data["records"]) == 0:
+            return print(
+                "Airtable returned 0 records - make sure your configuration is correct"
+            )
+
+        skylinks = skylinks + [
+            entry["fields"].get(AIRTABLE_FIELD, "") for entry in data["records"]
+        ]
+        skylinks = [
+            skylink.strip() for skylink in skylinks if skylink
+        ]  # filter empty skylinks, most likely empty rows, trim whitespace
+
+        offset = data.get("offset")
+
+    print(
+        "Sending /skynet/blocklist request with "
+        + str(len(skylinks))
+        + " skylinks to siad"
+    )
+    response = requests.post(
+        "http://" + ipaddress + ":9980/skynet/blocklist",
+        data=json.dumps({"add": skylinks}),
+        headers={"User-Agent": "Sia-Agent"},
+        auth=HTTPBasicAuth("", get_api_password()),
+    )
+
+    if response.status_code != 200:
+        status_code = str(response.status_code)
+        response_text = response.text or "empty response"
+        message = (
+            "Airtable blocklist request responded with code "
+            + status_code
+            + ": "
+            + response_text
+        )
+        return await send_msg(message, force_notify=False)
+
+    response_json = json.loads(response.text)
+    invalid_skylinks = response_json["invalids"]
+
+    if invalid_skylinks is None:
+        return await send_msg("Blocklist successfully updated all skylinks")
+    return await send_msg(
+        "Blocklist responded ok but failed to update "
+        + str(len(invalid_skylinks))
+        + " skylinks: "
+        + json.dumps(invalid_skylinks)
+    )
+
+
+loop = asyncio.get_event_loop()
+loop.run_until_complete(run_checks())
+
+# --- BASH EQUIVALENT
+# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")
+# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia)
+# curl --data "{\"add\" : ${skylinks}}" "${ipaddress}:8000/skynet/blocklist"

setup-scripts/support/crontab

@@ -1,6 +1,7 @@
 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/skynet-webportal/.env
 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/skynet-webportal/.env sia 8
 0 * * * * /home/user/skynet-webportal/setup-scripts/health-checker.py /home/user/skynet-webportal/.env sia 1
+30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env
 44 5 * * * /home/user/skynet-webportal/scripts/backup-aws-s3.sh 1>>/home/user/skynet-webportal/logs/backup-aws-s3.log 2>>/home/user/skynet-webportal/logs/backup-aws-s3.log
 6 13 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1>>/home/user/skynet-webportal/logs/db_backup.log 2>>/home/user/skynet-webportal/logs/db_backup.log
 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200