From 0a26516bdff050bde36f1865797ee4222b1fe7b9 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 9 Aug 2022 14:23:06 +0200 Subject: [PATCH] remove outdated blocklist scripts --- scripts/README.md | 4 - scripts/blocklist-skylink.sh | 49 --------- setup-scripts/README.md | 5 +- setup-scripts/blocklist-airtable.py | 161 ---------------------------- setup-scripts/support/crontab | 1 - 5 files changed, 1 insertion(+), 219 deletions(-) delete mode 100755 scripts/blocklist-skylink.sh delete mode 100755 setup-scripts/blocklist-airtable.py diff --git a/scripts/README.md b/scripts/README.md index e7b909b4..2bc32fbd 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -4,10 +4,6 @@ This package contains useful scripts for managing a Skynet Webportal. ## Available Scripts -**blocklist-skylink.sh**\ -The `blocklist-skylink.sh` script adds a skylink to the blocklist on all -servers. - **maintenance-upgrade.sh**\ The `maintenance-upgrade.sh` script upgrades the docker images for nodes on a maintenance server. diff --git a/scripts/blocklist-skylink.sh b/scripts/blocklist-skylink.sh deleted file mode 100755 index cb60bbdb..00000000 --- a/scripts/blocklist-skylink.sh +++ /dev/null @@ -1,49 +0,0 @@ -#! /usr/bin/env bash - -# This script is for manual skylink blocking. It accepts either a single -# skylink or a file containing list of skylinks. The script is intented -# for manual use and it should be run locally on each skynet webportal server. -# The automatic script that is used to continuously sync an Airtable sheet -# list with the blocklist on the web portals is /setup-scripts/blocklist-airtable.py - -set -e # exit on first error - -if [ -z "$1" ]; then - echo "Please provide either a skylink or a file with skylinks separated by new lines" && exit 1 -fi - -######################################################### -# read either a file containing skylinks separated by new -# lines or a single skylink and put them in an array -######################################################### -skylinks=() -if test -f "$1"; then - line_number=1 - - # Read file including the last line even when it doesn't end with newline - while IFS="" read -r line || [ -n "$line" ]; - do - if [[ $line =~ (^[a-zA-Z0-9_-]{46}$) ]]; then - skylinks+=("$line") - else - echo "Incorrect skylink at line ${line_number}: $line" && exit 1 - fi - let line_number+=1 - done < $1; -else - skylinks=("$1") # just single skylink passed as input argument -fi - -# get local skyd ip adress -ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia) - -# get sia api password either from env variable if exists or from apipassword file in sia-data directory -apipassword=$(docker exec sia sh -c '[ ! -z "${SIA_API_PASSWORD}" ] && echo ${SIA_API_PASSWORD} || $(cat /sia-data/apipassword | tr -d '\n')') - -# iterate over provided skylinks and block them one by one -for skylink in "${skylinks[@]}"; do - echo "> Blocking ${skylink} ... " - - # POST /skynet/blocklist always returns 200 and in case of failure print error message - curl -A Sia-Agent -u "":${apipassword} --data "{\"add\":[\"$skylink\"]}" "http://${ipaddress}:9980/skynet/blocklist" -done diff --git a/setup-scripts/README.md b/setup-scripts/README.md index d5237e09..6fba56b1 100644 --- a/setup-scripts/README.md +++ b/setup-scripts/README.md @@ -1,8 +1,6 @@ # Skynet Portal Setup Scripts -> :warning: This documentation is outdated and should be used for reference -only. Portal setup documentation is located at -https://portal-docs.skynetlabs.com/. +> :warning: This documentation is outdated and should be used for reference only. Portal setup documentation is located at https://portal-docs.skynetlabs.com/. This directory contains a setup guide and scripts that will install and configure some basic requirements for running a Skynet Portal. The assumption is @@ -35,7 +33,6 @@ You may want to fork this repository and replace ssh keys in - [funds-checker](funds-checker.py): script that checks wallet balance and sends status messages to discord periodically - [health-checker](health-checker.py): script that monitors health-check service for server health issues and reports them to discord periodically - [log-checker](log-checker.py): script that scans siad logs for critical errors and reports them to discord periodically -- [blocklist-skylink](../scripts/blocklist-skylink.sh): script that can be run locally from a machine that has access to all your skynet portal servers that blocklists provided skylink and prunes nginx cache to ensure it's not available any more (that is a bit much but that's the best we can do right now without paid nginx version) - if you want to use it, make sure to adjust the server addresses ### Step 1: setting up server user diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py deleted file mode 100755 index 8bd9d2dc..00000000 --- a/setup-scripts/blocklist-airtable.py +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env python3 - -from bot_utils import get_api_password, setup, send_msg -from random import randint -from time import sleep - -import traceback -import os -import sys -import asyncio -import requests -import json - -from requests.auth import HTTPBasicAuth - -setup() - - -AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY") -AIRTABLE_BASE = os.getenv("AIRTABLE_BASE") -AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE") -AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD") - -# Check environment variables are defined -for value in [AIRTABLE_API_KEY, AIRTABLE_BASE, AIRTABLE_TABLE, AIRTABLE_FIELD]: - if not value: - sys.exit("Configuration error: Missing AirTable environment variable.") - - -async def run_checks(): - try: - await block_skylinks_from_airtable() - except: # catch all exceptions - trace = traceback.format_exc() - await send_msg("```\n{}\n```".format(trace), force_notify=True) - - -def exec(command): - return os.popen(command).read().strip() - - -async def block_skylinks_from_airtable(): - # Get sia IP before doing anything else. If this step fails we don't - # need to continue with the execution of the script. - ipaddress = exec( - "docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia" - ) - - if ipaddress == "": - print("Skyd IP could not be detected. Exiting.") - return - - print("Pulling blocked skylinks from Airtable via api integration") - headers = {"Authorization": "Bearer " + AIRTABLE_API_KEY} - skylinks = [] - offset = None - retry = 0 - while len(skylinks) == 0 or offset: - print( - "Requesting a batch of records from Airtable with " - + (offset if offset else "empty") - + " offset" - + (" (retry " + str(retry) + ")" if retry else "") - ) - query = "&".join( - ["fields%5B%5D=" + AIRTABLE_FIELD, ("offset=" + offset) if offset else ""] - ) - response = requests.get( - "https://api.airtable.com/v0/" - + AIRTABLE_BASE - + "/" - + AIRTABLE_TABLE - + "?" - + query, - headers=headers, - ) - - # rate limited - sleep for 2-10 secs and retry (up to 100 times, ~10 minutes) - # https://support.airtable.com/hc/en-us/articles/203313985-Public-REST-API - # > 5 requests per second, per base - if response.status_code == 429: - if retry < 100: - retry = retry + 1 - sleep(randint(1, 10)) - continue - else: - return await send_msg( - "Airtable: too many retries, aborting!", force_notify=True - ) - retry = 0 # reset retry counter - - if response.status_code != 200: - status_code = str(response.status_code) - response_text = response.text or "empty response" - message = ( - "Airtable blocklist integration responded with code " - + status_code - + ": " - + response_text - ) - return await send_msg(message, force_notify=False) - - data = response.json() - - if len(data["records"]) == 0: - return print( - "Airtable returned 0 records - make sure your configuration is correct" - ) - - skylinks = skylinks + [ - entry["fields"].get(AIRTABLE_FIELD, "") for entry in data["records"] - ] - skylinks = [ - skylink.strip() for skylink in skylinks if skylink - ] # filter empty skylinks, most likely empty rows, trim whitespace - - offset = data.get("offset") - - print( - "Sending /skynet/blocklist request with " - + str(len(skylinks)) - + " skylinks to siad" - ) - response = requests.post( - "http://" + ipaddress + ":9980/skynet/blocklist", - data=json.dumps({"add": skylinks}), - headers={"User-Agent": "Sia-Agent"}, - auth=HTTPBasicAuth("", get_api_password()), - ) - - if response.status_code != 200: - status_code = str(response.status_code) - response_text = response.text or "empty response" - message = ( - "Airtable blocklist request responded with code " - + status_code - + ": " - + response_text - ) - return await send_msg(message, force_notify=False) - - response_json = json.loads(response.text) - invalid_skylinks = response_json["invalids"] - - if invalid_skylinks is None: - return await send_msg("Blocklist successfully updated all skylinks") - return await send_msg( - "Blocklist responded ok but failed to update " - + str(len(invalid_skylinks)) - + " skylinks: " - + json.dumps(invalid_skylinks) - ) - - -loop = asyncio.get_event_loop() -loop.run_until_complete(run_checks()) - -# --- BASH EQUIVALENT -# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')") -# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia) -# curl --data "{\"add\" : ${skylinks}}" "${ipaddress}:8000/skynet/blocklist" diff --git a/setup-scripts/support/crontab b/setup-scripts/support/crontab index 7fffdfb9..e8909134 100644 --- a/setup-scripts/support/crontab +++ b/setup-scripts/support/crontab @@ -1,7 +1,6 @@ 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/skynet-webportal/.env 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/skynet-webportal/.env sia 8 0 * * * * /home/user/skynet-webportal/setup-scripts/health-checker.py /home/user/skynet-webportal/.env sia 1 -30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env 44 5 * * * /home/user/skynet-webportal/scripts/backup-aws-s3.sh 1>>/home/user/skynet-webportal/logs/backup-aws-s3.log 2>>/home/user/skynet-webportal/logs/backup-aws-s3.log 6 13 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1>>/home/user/skynet-webportal/logs/db_backup.log 2>>/home/user/skynet-webportal/logs/db_backup.log 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200