diff --git a/scripts/README.md b/scripts/README.md index 2bc32fbd..e7b909b4 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -4,6 +4,10 @@ This package contains useful scripts for managing a Skynet Webportal. ## Available Scripts +**blocklist-skylink.sh**\ +The `blocklist-skylink.sh` script adds a skylink to the blocklist on all +servers. + **maintenance-upgrade.sh**\ The `maintenance-upgrade.sh` script upgrades the docker images for nodes on a maintenance server. diff --git a/scripts/blocklist-skylink.sh b/scripts/blocklist-skylink.sh new file mode 100755 index 00000000..cb60bbdb --- /dev/null +++ b/scripts/blocklist-skylink.sh @@ -0,0 +1,49 @@ +#! /usr/bin/env bash + +# This script is for manual skylink blocking. It accepts either a single +# skylink or a file containing list of skylinks. The script is intented +# for manual use and it should be run locally on each skynet webportal server. +# The automatic script that is used to continuously sync an Airtable sheet +# list with the blocklist on the web portals is /setup-scripts/blocklist-airtable.py + +set -e # exit on first error + +if [ -z "$1" ]; then + echo "Please provide either a skylink or a file with skylinks separated by new lines" && exit 1 +fi + +######################################################### +# read either a file containing skylinks separated by new +# lines or a single skylink and put them in an array +######################################################### +skylinks=() +if test -f "$1"; then + line_number=1 + + # Read file including the last line even when it doesn't end with newline + while IFS="" read -r line || [ -n "$line" ]; + do + if [[ $line =~ (^[a-zA-Z0-9_-]{46}$) ]]; then + skylinks+=("$line") + else + echo "Incorrect skylink at line ${line_number}: $line" && exit 1 + fi + let line_number+=1 + done < $1; +else + skylinks=("$1") # just single skylink passed as input argument +fi + +# get local skyd ip adress +ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia) + +# get sia api password either from env variable if exists or from apipassword file in sia-data directory +apipassword=$(docker exec sia sh -c '[ ! -z "${SIA_API_PASSWORD}" ] && echo ${SIA_API_PASSWORD} || $(cat /sia-data/apipassword | tr -d '\n')') + +# iterate over provided skylinks and block them one by one +for skylink in "${skylinks[@]}"; do + echo "> Blocking ${skylink} ... " + + # POST /skynet/blocklist always returns 200 and in case of failure print error message + curl -A Sia-Agent -u "":${apipassword} --data "{\"add\":[\"$skylink\"]}" "http://${ipaddress}:9980/skynet/blocklist" +done diff --git a/setup-scripts/README.md b/setup-scripts/README.md index 6fba56b1..d5237e09 100644 --- a/setup-scripts/README.md +++ b/setup-scripts/README.md @@ -1,6 +1,8 @@ # Skynet Portal Setup Scripts -> :warning: This documentation is outdated and should be used for reference only. Portal setup documentation is located at https://portal-docs.skynetlabs.com/. +> :warning: This documentation is outdated and should be used for reference +only. Portal setup documentation is located at +https://portal-docs.skynetlabs.com/. This directory contains a setup guide and scripts that will install and configure some basic requirements for running a Skynet Portal. The assumption is @@ -33,6 +35,7 @@ You may want to fork this repository and replace ssh keys in - [funds-checker](funds-checker.py): script that checks wallet balance and sends status messages to discord periodically - [health-checker](health-checker.py): script that monitors health-check service for server health issues and reports them to discord periodically - [log-checker](log-checker.py): script that scans siad logs for critical errors and reports them to discord periodically +- [blocklist-skylink](../scripts/blocklist-skylink.sh): script that can be run locally from a machine that has access to all your skynet portal servers that blocklists provided skylink and prunes nginx cache to ensure it's not available any more (that is a bit much but that's the best we can do right now without paid nginx version) - if you want to use it, make sure to adjust the server addresses ### Step 1: setting up server user diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py new file mode 100755 index 00000000..8bd9d2dc --- /dev/null +++ b/setup-scripts/blocklist-airtable.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 + +from bot_utils import get_api_password, setup, send_msg +from random import randint +from time import sleep + +import traceback +import os +import sys +import asyncio +import requests +import json + +from requests.auth import HTTPBasicAuth + +setup() + + +AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY") +AIRTABLE_BASE = os.getenv("AIRTABLE_BASE") +AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE") +AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD") + +# Check environment variables are defined +for value in [AIRTABLE_API_KEY, AIRTABLE_BASE, AIRTABLE_TABLE, AIRTABLE_FIELD]: + if not value: + sys.exit("Configuration error: Missing AirTable environment variable.") + + +async def run_checks(): + try: + await block_skylinks_from_airtable() + except: # catch all exceptions + trace = traceback.format_exc() + await send_msg("```\n{}\n```".format(trace), force_notify=True) + + +def exec(command): + return os.popen(command).read().strip() + + +async def block_skylinks_from_airtable(): + # Get sia IP before doing anything else. If this step fails we don't + # need to continue with the execution of the script. + ipaddress = exec( + "docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia" + ) + + if ipaddress == "": + print("Skyd IP could not be detected. Exiting.") + return + + print("Pulling blocked skylinks from Airtable via api integration") + headers = {"Authorization": "Bearer " + AIRTABLE_API_KEY} + skylinks = [] + offset = None + retry = 0 + while len(skylinks) == 0 or offset: + print( + "Requesting a batch of records from Airtable with " + + (offset if offset else "empty") + + " offset" + + (" (retry " + str(retry) + ")" if retry else "") + ) + query = "&".join( + ["fields%5B%5D=" + AIRTABLE_FIELD, ("offset=" + offset) if offset else ""] + ) + response = requests.get( + "https://api.airtable.com/v0/" + + AIRTABLE_BASE + + "/" + + AIRTABLE_TABLE + + "?" + + query, + headers=headers, + ) + + # rate limited - sleep for 2-10 secs and retry (up to 100 times, ~10 minutes) + # https://support.airtable.com/hc/en-us/articles/203313985-Public-REST-API + # > 5 requests per second, per base + if response.status_code == 429: + if retry < 100: + retry = retry + 1 + sleep(randint(1, 10)) + continue + else: + return await send_msg( + "Airtable: too many retries, aborting!", force_notify=True + ) + retry = 0 # reset retry counter + + if response.status_code != 200: + status_code = str(response.status_code) + response_text = response.text or "empty response" + message = ( + "Airtable blocklist integration responded with code " + + status_code + + ": " + + response_text + ) + return await send_msg(message, force_notify=False) + + data = response.json() + + if len(data["records"]) == 0: + return print( + "Airtable returned 0 records - make sure your configuration is correct" + ) + + skylinks = skylinks + [ + entry["fields"].get(AIRTABLE_FIELD, "") for entry in data["records"] + ] + skylinks = [ + skylink.strip() for skylink in skylinks if skylink + ] # filter empty skylinks, most likely empty rows, trim whitespace + + offset = data.get("offset") + + print( + "Sending /skynet/blocklist request with " + + str(len(skylinks)) + + " skylinks to siad" + ) + response = requests.post( + "http://" + ipaddress + ":9980/skynet/blocklist", + data=json.dumps({"add": skylinks}), + headers={"User-Agent": "Sia-Agent"}, + auth=HTTPBasicAuth("", get_api_password()), + ) + + if response.status_code != 200: + status_code = str(response.status_code) + response_text = response.text or "empty response" + message = ( + "Airtable blocklist request responded with code " + + status_code + + ": " + + response_text + ) + return await send_msg(message, force_notify=False) + + response_json = json.loads(response.text) + invalid_skylinks = response_json["invalids"] + + if invalid_skylinks is None: + return await send_msg("Blocklist successfully updated all skylinks") + return await send_msg( + "Blocklist responded ok but failed to update " + + str(len(invalid_skylinks)) + + " skylinks: " + + json.dumps(invalid_skylinks) + ) + + +loop = asyncio.get_event_loop() +loop.run_until_complete(run_checks()) + +# --- BASH EQUIVALENT +# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')") +# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia) +# curl --data "{\"add\" : ${skylinks}}" "${ipaddress}:8000/skynet/blocklist" diff --git a/setup-scripts/support/crontab b/setup-scripts/support/crontab index e8909134..7fffdfb9 100644 --- a/setup-scripts/support/crontab +++ b/setup-scripts/support/crontab @@ -1,6 +1,7 @@ 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/skynet-webportal/.env 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/skynet-webportal/.env sia 8 0 * * * * /home/user/skynet-webportal/setup-scripts/health-checker.py /home/user/skynet-webportal/.env sia 1 +30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env 44 5 * * * /home/user/skynet-webportal/scripts/backup-aws-s3.sh 1>>/home/user/skynet-webportal/logs/backup-aws-s3.log 2>>/home/user/skynet-webportal/logs/backup-aws-s3.log 6 13 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1>>/home/user/skynet-webportal/logs/db_backup.log 2>>/home/user/skynet-webportal/logs/db_backup.log 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200