From 0a26516bdff050bde36f1865797ee4222b1fe7b9 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 9 Aug 2022 14:23:06 +0200 Subject: [PATCH 1/3] remove outdated blocklist scripts --- scripts/README.md | 4 - scripts/blocklist-skylink.sh | 49 --------- setup-scripts/README.md | 5 +- setup-scripts/blocklist-airtable.py | 161 ---------------------------- setup-scripts/support/crontab | 1 - 5 files changed, 1 insertion(+), 219 deletions(-) delete mode 100755 scripts/blocklist-skylink.sh delete mode 100755 setup-scripts/blocklist-airtable.py diff --git a/scripts/README.md b/scripts/README.md index e7b909b4..2bc32fbd 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -4,10 +4,6 @@ This package contains useful scripts for managing a Skynet Webportal. ## Available Scripts -**blocklist-skylink.sh**\ -The `blocklist-skylink.sh` script adds a skylink to the blocklist on all -servers. - **maintenance-upgrade.sh**\ The `maintenance-upgrade.sh` script upgrades the docker images for nodes on a maintenance server. diff --git a/scripts/blocklist-skylink.sh b/scripts/blocklist-skylink.sh deleted file mode 100755 index cb60bbdb..00000000 --- a/scripts/blocklist-skylink.sh +++ /dev/null @@ -1,49 +0,0 @@ -#! /usr/bin/env bash - -# This script is for manual skylink blocking. It accepts either a single -# skylink or a file containing list of skylinks. The script is intented -# for manual use and it should be run locally on each skynet webportal server. -# The automatic script that is used to continuously sync an Airtable sheet -# list with the blocklist on the web portals is /setup-scripts/blocklist-airtable.py - -set -e # exit on first error - -if [ -z "$1" ]; then - echo "Please provide either a skylink or a file with skylinks separated by new lines" && exit 1 -fi - -######################################################### -# read either a file containing skylinks separated by new -# lines or a single skylink and put them in an array -######################################################### -skylinks=() -if test -f "$1"; then - line_number=1 - - # Read file including the last line even when it doesn't end with newline - while IFS="" read -r line || [ -n "$line" ]; - do - if [[ $line =~ (^[a-zA-Z0-9_-]{46}$) ]]; then - skylinks+=("$line") - else - echo "Incorrect skylink at line ${line_number}: $line" && exit 1 - fi - let line_number+=1 - done < $1; -else - skylinks=("$1") # just single skylink passed as input argument -fi - -# get local skyd ip adress -ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia) - -# get sia api password either from env variable if exists or from apipassword file in sia-data directory -apipassword=$(docker exec sia sh -c '[ ! -z "${SIA_API_PASSWORD}" ] && echo ${SIA_API_PASSWORD} || $(cat /sia-data/apipassword | tr -d '\n')') - -# iterate over provided skylinks and block them one by one -for skylink in "${skylinks[@]}"; do - echo "> Blocking ${skylink} ... " - - # POST /skynet/blocklist always returns 200 and in case of failure print error message - curl -A Sia-Agent -u "":${apipassword} --data "{\"add\":[\"$skylink\"]}" "http://${ipaddress}:9980/skynet/blocklist" -done diff --git a/setup-scripts/README.md b/setup-scripts/README.md index d5237e09..6fba56b1 100644 --- a/setup-scripts/README.md +++ b/setup-scripts/README.md @@ -1,8 +1,6 @@ # Skynet Portal Setup Scripts -> :warning: This documentation is outdated and should be used for reference -only. Portal setup documentation is located at -https://portal-docs.skynetlabs.com/. +> :warning: This documentation is outdated and should be used for reference only. Portal setup documentation is located at https://portal-docs.skynetlabs.com/. This directory contains a setup guide and scripts that will install and configure some basic requirements for running a Skynet Portal. The assumption is @@ -35,7 +33,6 @@ You may want to fork this repository and replace ssh keys in - [funds-checker](funds-checker.py): script that checks wallet balance and sends status messages to discord periodically - [health-checker](health-checker.py): script that monitors health-check service for server health issues and reports them to discord periodically - [log-checker](log-checker.py): script that scans siad logs for critical errors and reports them to discord periodically -- [blocklist-skylink](../scripts/blocklist-skylink.sh): script that can be run locally from a machine that has access to all your skynet portal servers that blocklists provided skylink and prunes nginx cache to ensure it's not available any more (that is a bit much but that's the best we can do right now without paid nginx version) - if you want to use it, make sure to adjust the server addresses ### Step 1: setting up server user diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py deleted file mode 100755 index 8bd9d2dc..00000000 --- a/setup-scripts/blocklist-airtable.py +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env python3 - -from bot_utils import get_api_password, setup, send_msg -from random import randint -from time import sleep - -import traceback -import os -import sys -import asyncio -import requests -import json - -from requests.auth import HTTPBasicAuth - -setup() - - -AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY") -AIRTABLE_BASE = os.getenv("AIRTABLE_BASE") -AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE") -AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD") - -# Check environment variables are defined -for value in [AIRTABLE_API_KEY, AIRTABLE_BASE, AIRTABLE_TABLE, AIRTABLE_FIELD]: - if not value: - sys.exit("Configuration error: Missing AirTable environment variable.") - - -async def run_checks(): - try: - await block_skylinks_from_airtable() - except: # catch all exceptions - trace = traceback.format_exc() - await send_msg("```\n{}\n```".format(trace), force_notify=True) - - -def exec(command): - return os.popen(command).read().strip() - - -async def block_skylinks_from_airtable(): - # Get sia IP before doing anything else. If this step fails we don't - # need to continue with the execution of the script. - ipaddress = exec( - "docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia" - ) - - if ipaddress == "": - print("Skyd IP could not be detected. Exiting.") - return - - print("Pulling blocked skylinks from Airtable via api integration") - headers = {"Authorization": "Bearer " + AIRTABLE_API_KEY} - skylinks = [] - offset = None - retry = 0 - while len(skylinks) == 0 or offset: - print( - "Requesting a batch of records from Airtable with " - + (offset if offset else "empty") - + " offset" - + (" (retry " + str(retry) + ")" if retry else "") - ) - query = "&".join( - ["fields%5B%5D=" + AIRTABLE_FIELD, ("offset=" + offset) if offset else ""] - ) - response = requests.get( - "https://api.airtable.com/v0/" - + AIRTABLE_BASE - + "/" - + AIRTABLE_TABLE - + "?" - + query, - headers=headers, - ) - - # rate limited - sleep for 2-10 secs and retry (up to 100 times, ~10 minutes) - # https://support.airtable.com/hc/en-us/articles/203313985-Public-REST-API - # > 5 requests per second, per base - if response.status_code == 429: - if retry < 100: - retry = retry + 1 - sleep(randint(1, 10)) - continue - else: - return await send_msg( - "Airtable: too many retries, aborting!", force_notify=True - ) - retry = 0 # reset retry counter - - if response.status_code != 200: - status_code = str(response.status_code) - response_text = response.text or "empty response" - message = ( - "Airtable blocklist integration responded with code " - + status_code - + ": " - + response_text - ) - return await send_msg(message, force_notify=False) - - data = response.json() - - if len(data["records"]) == 0: - return print( - "Airtable returned 0 records - make sure your configuration is correct" - ) - - skylinks = skylinks + [ - entry["fields"].get(AIRTABLE_FIELD, "") for entry in data["records"] - ] - skylinks = [ - skylink.strip() for skylink in skylinks if skylink - ] # filter empty skylinks, most likely empty rows, trim whitespace - - offset = data.get("offset") - - print( - "Sending /skynet/blocklist request with " - + str(len(skylinks)) - + " skylinks to siad" - ) - response = requests.post( - "http://" + ipaddress + ":9980/skynet/blocklist", - data=json.dumps({"add": skylinks}), - headers={"User-Agent": "Sia-Agent"}, - auth=HTTPBasicAuth("", get_api_password()), - ) - - if response.status_code != 200: - status_code = str(response.status_code) - response_text = response.text or "empty response" - message = ( - "Airtable blocklist request responded with code " - + status_code - + ": " - + response_text - ) - return await send_msg(message, force_notify=False) - - response_json = json.loads(response.text) - invalid_skylinks = response_json["invalids"] - - if invalid_skylinks is None: - return await send_msg("Blocklist successfully updated all skylinks") - return await send_msg( - "Blocklist responded ok but failed to update " - + str(len(invalid_skylinks)) - + " skylinks: " - + json.dumps(invalid_skylinks) - ) - - -loop = asyncio.get_event_loop() -loop.run_until_complete(run_checks()) - -# --- BASH EQUIVALENT -# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')") -# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia) -# curl --data "{\"add\" : ${skylinks}}" "${ipaddress}:8000/skynet/blocklist" diff --git a/setup-scripts/support/crontab b/setup-scripts/support/crontab index 7fffdfb9..e8909134 100644 --- a/setup-scripts/support/crontab +++ b/setup-scripts/support/crontab @@ -1,7 +1,6 @@ 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/skynet-webportal/.env 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/skynet-webportal/.env sia 8 0 * * * * /home/user/skynet-webportal/setup-scripts/health-checker.py /home/user/skynet-webportal/.env sia 1 -30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env 44 5 * * * /home/user/skynet-webportal/scripts/backup-aws-s3.sh 1>>/home/user/skynet-webportal/logs/backup-aws-s3.log 2>>/home/user/skynet-webportal/logs/backup-aws-s3.log 6 13 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1>>/home/user/skynet-webportal/logs/db_backup.log 2>>/home/user/skynet-webportal/logs/db_backup.log 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200 From 9e1a8600a5f458309f26dfd0bef1ed3b1d82a334 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 9 Aug 2022 15:05:11 +0200 Subject: [PATCH 2/3] Revert "remove outdated blocklist scripts" This reverts commit 0a26516bdff050bde36f1865797ee4222b1fe7b9. --- scripts/README.md | 4 + scripts/blocklist-skylink.sh | 49 +++++++++ setup-scripts/README.md | 5 +- setup-scripts/blocklist-airtable.py | 161 ++++++++++++++++++++++++++++ setup-scripts/support/crontab | 1 + 5 files changed, 219 insertions(+), 1 deletion(-) create mode 100755 scripts/blocklist-skylink.sh create mode 100755 setup-scripts/blocklist-airtable.py diff --git a/scripts/README.md b/scripts/README.md index 2bc32fbd..e7b909b4 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -4,6 +4,10 @@ This package contains useful scripts for managing a Skynet Webportal. ## Available Scripts +**blocklist-skylink.sh**\ +The `blocklist-skylink.sh` script adds a skylink to the blocklist on all +servers. + **maintenance-upgrade.sh**\ The `maintenance-upgrade.sh` script upgrades the docker images for nodes on a maintenance server. diff --git a/scripts/blocklist-skylink.sh b/scripts/blocklist-skylink.sh new file mode 100755 index 00000000..cb60bbdb --- /dev/null +++ b/scripts/blocklist-skylink.sh @@ -0,0 +1,49 @@ +#! /usr/bin/env bash + +# This script is for manual skylink blocking. It accepts either a single +# skylink or a file containing list of skylinks. The script is intented +# for manual use and it should be run locally on each skynet webportal server. +# The automatic script that is used to continuously sync an Airtable sheet +# list with the blocklist on the web portals is /setup-scripts/blocklist-airtable.py + +set -e # exit on first error + +if [ -z "$1" ]; then + echo "Please provide either a skylink or a file with skylinks separated by new lines" && exit 1 +fi + +######################################################### +# read either a file containing skylinks separated by new +# lines or a single skylink and put them in an array +######################################################### +skylinks=() +if test -f "$1"; then + line_number=1 + + # Read file including the last line even when it doesn't end with newline + while IFS="" read -r line || [ -n "$line" ]; + do + if [[ $line =~ (^[a-zA-Z0-9_-]{46}$) ]]; then + skylinks+=("$line") + else + echo "Incorrect skylink at line ${line_number}: $line" && exit 1 + fi + let line_number+=1 + done < $1; +else + skylinks=("$1") # just single skylink passed as input argument +fi + +# get local skyd ip adress +ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia) + +# get sia api password either from env variable if exists or from apipassword file in sia-data directory +apipassword=$(docker exec sia sh -c '[ ! -z "${SIA_API_PASSWORD}" ] && echo ${SIA_API_PASSWORD} || $(cat /sia-data/apipassword | tr -d '\n')') + +# iterate over provided skylinks and block them one by one +for skylink in "${skylinks[@]}"; do + echo "> Blocking ${skylink} ... " + + # POST /skynet/blocklist always returns 200 and in case of failure print error message + curl -A Sia-Agent -u "":${apipassword} --data "{\"add\":[\"$skylink\"]}" "http://${ipaddress}:9980/skynet/blocklist" +done diff --git a/setup-scripts/README.md b/setup-scripts/README.md index 6fba56b1..d5237e09 100644 --- a/setup-scripts/README.md +++ b/setup-scripts/README.md @@ -1,6 +1,8 @@ # Skynet Portal Setup Scripts -> :warning: This documentation is outdated and should be used for reference only. Portal setup documentation is located at https://portal-docs.skynetlabs.com/. +> :warning: This documentation is outdated and should be used for reference +only. Portal setup documentation is located at +https://portal-docs.skynetlabs.com/. This directory contains a setup guide and scripts that will install and configure some basic requirements for running a Skynet Portal. The assumption is @@ -33,6 +35,7 @@ You may want to fork this repository and replace ssh keys in - [funds-checker](funds-checker.py): script that checks wallet balance and sends status messages to discord periodically - [health-checker](health-checker.py): script that monitors health-check service for server health issues and reports them to discord periodically - [log-checker](log-checker.py): script that scans siad logs for critical errors and reports them to discord periodically +- [blocklist-skylink](../scripts/blocklist-skylink.sh): script that can be run locally from a machine that has access to all your skynet portal servers that blocklists provided skylink and prunes nginx cache to ensure it's not available any more (that is a bit much but that's the best we can do right now without paid nginx version) - if you want to use it, make sure to adjust the server addresses ### Step 1: setting up server user diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py new file mode 100755 index 00000000..8bd9d2dc --- /dev/null +++ b/setup-scripts/blocklist-airtable.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 + +from bot_utils import get_api_password, setup, send_msg +from random import randint +from time import sleep + +import traceback +import os +import sys +import asyncio +import requests +import json + +from requests.auth import HTTPBasicAuth + +setup() + + +AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY") +AIRTABLE_BASE = os.getenv("AIRTABLE_BASE") +AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE") +AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD") + +# Check environment variables are defined +for value in [AIRTABLE_API_KEY, AIRTABLE_BASE, AIRTABLE_TABLE, AIRTABLE_FIELD]: + if not value: + sys.exit("Configuration error: Missing AirTable environment variable.") + + +async def run_checks(): + try: + await block_skylinks_from_airtable() + except: # catch all exceptions + trace = traceback.format_exc() + await send_msg("```\n{}\n```".format(trace), force_notify=True) + + +def exec(command): + return os.popen(command).read().strip() + + +async def block_skylinks_from_airtable(): + # Get sia IP before doing anything else. If this step fails we don't + # need to continue with the execution of the script. + ipaddress = exec( + "docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia" + ) + + if ipaddress == "": + print("Skyd IP could not be detected. Exiting.") + return + + print("Pulling blocked skylinks from Airtable via api integration") + headers = {"Authorization": "Bearer " + AIRTABLE_API_KEY} + skylinks = [] + offset = None + retry = 0 + while len(skylinks) == 0 or offset: + print( + "Requesting a batch of records from Airtable with " + + (offset if offset else "empty") + + " offset" + + (" (retry " + str(retry) + ")" if retry else "") + ) + query = "&".join( + ["fields%5B%5D=" + AIRTABLE_FIELD, ("offset=" + offset) if offset else ""] + ) + response = requests.get( + "https://api.airtable.com/v0/" + + AIRTABLE_BASE + + "/" + + AIRTABLE_TABLE + + "?" + + query, + headers=headers, + ) + + # rate limited - sleep for 2-10 secs and retry (up to 100 times, ~10 minutes) + # https://support.airtable.com/hc/en-us/articles/203313985-Public-REST-API + # > 5 requests per second, per base + if response.status_code == 429: + if retry < 100: + retry = retry + 1 + sleep(randint(1, 10)) + continue + else: + return await send_msg( + "Airtable: too many retries, aborting!", force_notify=True + ) + retry = 0 # reset retry counter + + if response.status_code != 200: + status_code = str(response.status_code) + response_text = response.text or "empty response" + message = ( + "Airtable blocklist integration responded with code " + + status_code + + ": " + + response_text + ) + return await send_msg(message, force_notify=False) + + data = response.json() + + if len(data["records"]) == 0: + return print( + "Airtable returned 0 records - make sure your configuration is correct" + ) + + skylinks = skylinks + [ + entry["fields"].get(AIRTABLE_FIELD, "") for entry in data["records"] + ] + skylinks = [ + skylink.strip() for skylink in skylinks if skylink + ] # filter empty skylinks, most likely empty rows, trim whitespace + + offset = data.get("offset") + + print( + "Sending /skynet/blocklist request with " + + str(len(skylinks)) + + " skylinks to siad" + ) + response = requests.post( + "http://" + ipaddress + ":9980/skynet/blocklist", + data=json.dumps({"add": skylinks}), + headers={"User-Agent": "Sia-Agent"}, + auth=HTTPBasicAuth("", get_api_password()), + ) + + if response.status_code != 200: + status_code = str(response.status_code) + response_text = response.text or "empty response" + message = ( + "Airtable blocklist request responded with code " + + status_code + + ": " + + response_text + ) + return await send_msg(message, force_notify=False) + + response_json = json.loads(response.text) + invalid_skylinks = response_json["invalids"] + + if invalid_skylinks is None: + return await send_msg("Blocklist successfully updated all skylinks") + return await send_msg( + "Blocklist responded ok but failed to update " + + str(len(invalid_skylinks)) + + " skylinks: " + + json.dumps(invalid_skylinks) + ) + + +loop = asyncio.get_event_loop() +loop.run_until_complete(run_checks()) + +# --- BASH EQUIVALENT +# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')") +# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia) +# curl --data "{\"add\" : ${skylinks}}" "${ipaddress}:8000/skynet/blocklist" diff --git a/setup-scripts/support/crontab b/setup-scripts/support/crontab index e8909134..7fffdfb9 100644 --- a/setup-scripts/support/crontab +++ b/setup-scripts/support/crontab @@ -1,6 +1,7 @@ 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/skynet-webportal/.env 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/skynet-webportal/.env sia 8 0 * * * * /home/user/skynet-webportal/setup-scripts/health-checker.py /home/user/skynet-webportal/.env sia 1 +30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env 44 5 * * * /home/user/skynet-webportal/scripts/backup-aws-s3.sh 1>>/home/user/skynet-webportal/logs/backup-aws-s3.log 2>>/home/user/skynet-webportal/logs/backup-aws-s3.log 6 13 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1>>/home/user/skynet-webportal/logs/db_backup.log 2>>/home/user/skynet-webportal/logs/db_backup.log 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200 From e53a8ba46a4bf98120fd7fd1265e7966bfffb4a0 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 9 Aug 2022 15:06:20 +0200 Subject: [PATCH 3/3] remove airtable blocklist scripts --- scripts/blocklist-skylink.sh | 2 - setup-scripts/blocklist-airtable.py | 161 ---------------------------- setup-scripts/support/crontab | 1 - 3 files changed, 164 deletions(-) delete mode 100755 setup-scripts/blocklist-airtable.py diff --git a/scripts/blocklist-skylink.sh b/scripts/blocklist-skylink.sh index cb60bbdb..385e7c2e 100755 --- a/scripts/blocklist-skylink.sh +++ b/scripts/blocklist-skylink.sh @@ -3,8 +3,6 @@ # This script is for manual skylink blocking. It accepts either a single # skylink or a file containing list of skylinks. The script is intented # for manual use and it should be run locally on each skynet webportal server. -# The automatic script that is used to continuously sync an Airtable sheet -# list with the blocklist on the web portals is /setup-scripts/blocklist-airtable.py set -e # exit on first error diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py deleted file mode 100755 index 8bd9d2dc..00000000 --- a/setup-scripts/blocklist-airtable.py +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env python3 - -from bot_utils import get_api_password, setup, send_msg -from random import randint -from time import sleep - -import traceback -import os -import sys -import asyncio -import requests -import json - -from requests.auth import HTTPBasicAuth - -setup() - - -AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY") -AIRTABLE_BASE = os.getenv("AIRTABLE_BASE") -AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE") -AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD") - -# Check environment variables are defined -for value in [AIRTABLE_API_KEY, AIRTABLE_BASE, AIRTABLE_TABLE, AIRTABLE_FIELD]: - if not value: - sys.exit("Configuration error: Missing AirTable environment variable.") - - -async def run_checks(): - try: - await block_skylinks_from_airtable() - except: # catch all exceptions - trace = traceback.format_exc() - await send_msg("```\n{}\n```".format(trace), force_notify=True) - - -def exec(command): - return os.popen(command).read().strip() - - -async def block_skylinks_from_airtable(): - # Get sia IP before doing anything else. If this step fails we don't - # need to continue with the execution of the script. - ipaddress = exec( - "docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia" - ) - - if ipaddress == "": - print("Skyd IP could not be detected. Exiting.") - return - - print("Pulling blocked skylinks from Airtable via api integration") - headers = {"Authorization": "Bearer " + AIRTABLE_API_KEY} - skylinks = [] - offset = None - retry = 0 - while len(skylinks) == 0 or offset: - print( - "Requesting a batch of records from Airtable with " - + (offset if offset else "empty") - + " offset" - + (" (retry " + str(retry) + ")" if retry else "") - ) - query = "&".join( - ["fields%5B%5D=" + AIRTABLE_FIELD, ("offset=" + offset) if offset else ""] - ) - response = requests.get( - "https://api.airtable.com/v0/" - + AIRTABLE_BASE - + "/" - + AIRTABLE_TABLE - + "?" - + query, - headers=headers, - ) - - # rate limited - sleep for 2-10 secs and retry (up to 100 times, ~10 minutes) - # https://support.airtable.com/hc/en-us/articles/203313985-Public-REST-API - # > 5 requests per second, per base - if response.status_code == 429: - if retry < 100: - retry = retry + 1 - sleep(randint(1, 10)) - continue - else: - return await send_msg( - "Airtable: too many retries, aborting!", force_notify=True - ) - retry = 0 # reset retry counter - - if response.status_code != 200: - status_code = str(response.status_code) - response_text = response.text or "empty response" - message = ( - "Airtable blocklist integration responded with code " - + status_code - + ": " - + response_text - ) - return await send_msg(message, force_notify=False) - - data = response.json() - - if len(data["records"]) == 0: - return print( - "Airtable returned 0 records - make sure your configuration is correct" - ) - - skylinks = skylinks + [ - entry["fields"].get(AIRTABLE_FIELD, "") for entry in data["records"] - ] - skylinks = [ - skylink.strip() for skylink in skylinks if skylink - ] # filter empty skylinks, most likely empty rows, trim whitespace - - offset = data.get("offset") - - print( - "Sending /skynet/blocklist request with " - + str(len(skylinks)) - + " skylinks to siad" - ) - response = requests.post( - "http://" + ipaddress + ":9980/skynet/blocklist", - data=json.dumps({"add": skylinks}), - headers={"User-Agent": "Sia-Agent"}, - auth=HTTPBasicAuth("", get_api_password()), - ) - - if response.status_code != 200: - status_code = str(response.status_code) - response_text = response.text or "empty response" - message = ( - "Airtable blocklist request responded with code " - + status_code - + ": " - + response_text - ) - return await send_msg(message, force_notify=False) - - response_json = json.loads(response.text) - invalid_skylinks = response_json["invalids"] - - if invalid_skylinks is None: - return await send_msg("Blocklist successfully updated all skylinks") - return await send_msg( - "Blocklist responded ok but failed to update " - + str(len(invalid_skylinks)) - + " skylinks: " - + json.dumps(invalid_skylinks) - ) - - -loop = asyncio.get_event_loop() -loop.run_until_complete(run_checks()) - -# --- BASH EQUIVALENT -# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')") -# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia) -# curl --data "{\"add\" : ${skylinks}}" "${ipaddress}:8000/skynet/blocklist" diff --git a/setup-scripts/support/crontab b/setup-scripts/support/crontab index 7fffdfb9..e8909134 100644 --- a/setup-scripts/support/crontab +++ b/setup-scripts/support/crontab @@ -1,7 +1,6 @@ 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/skynet-webportal/.env 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/skynet-webportal/.env sia 8 0 * * * * /home/user/skynet-webportal/setup-scripts/health-checker.py /home/user/skynet-webportal/.env sia 1 -30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env 44 5 * * * /home/user/skynet-webportal/scripts/backup-aws-s3.sh 1>>/home/user/skynet-webportal/logs/backup-aws-s3.log 2>>/home/user/skynet-webportal/logs/backup-aws-s3.log 6 13 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1>>/home/user/skynet-webportal/logs/db_backup.log 2>>/home/user/skynet-webportal/logs/db_backup.log 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200