Merge pull request #2192 from SkynetLabs/remove-outdated-blocklist-scripts

remove outdated blocklist scripts
This commit is contained in:
Ivaylo Novakov 2022-08-15 10:30:47 +02:00 committed by GitHub
commit fd41aa0518
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 0 additions and 164 deletions

View File

@ -3,8 +3,6 @@
# This script is for manual skylink blocking. It accepts either a single
# skylink or a file containing list of skylinks. The script is intented
# for manual use and it should be run locally on each skynet webportal server.
# The automatic script that is used to continuously sync an Airtable sheet
# list with the blocklist on the web portals is /setup-scripts/blocklist-airtable.py
set -e # exit on first error

View File

@ -1,161 +0,0 @@
#!/usr/bin/env python3
from bot_utils import get_api_password, setup, send_msg
from random import randint
from time import sleep
import traceback
import os
import sys
import asyncio
import requests
import json
from requests.auth import HTTPBasicAuth
setup()
AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY")
AIRTABLE_BASE = os.getenv("AIRTABLE_BASE")
AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE")
AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD")
# Check environment variables are defined
for value in [AIRTABLE_API_KEY, AIRTABLE_BASE, AIRTABLE_TABLE, AIRTABLE_FIELD]:
if not value:
sys.exit("Configuration error: Missing AirTable environment variable.")
async def run_checks():
try:
await block_skylinks_from_airtable()
except: # catch all exceptions
trace = traceback.format_exc()
await send_msg("```\n{}\n```".format(trace), force_notify=True)
def exec(command):
return os.popen(command).read().strip()
async def block_skylinks_from_airtable():
# Get sia IP before doing anything else. If this step fails we don't
# need to continue with the execution of the script.
ipaddress = exec(
"docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia"
)
if ipaddress == "":
print("Skyd IP could not be detected. Exiting.")
return
print("Pulling blocked skylinks from Airtable via api integration")
headers = {"Authorization": "Bearer " + AIRTABLE_API_KEY}
skylinks = []
offset = None
retry = 0
while len(skylinks) == 0 or offset:
print(
"Requesting a batch of records from Airtable with "
+ (offset if offset else "empty")
+ " offset"
+ (" (retry " + str(retry) + ")" if retry else "")
)
query = "&".join(
["fields%5B%5D=" + AIRTABLE_FIELD, ("offset=" + offset) if offset else ""]
)
response = requests.get(
"https://api.airtable.com/v0/"
+ AIRTABLE_BASE
+ "/"
+ AIRTABLE_TABLE
+ "?"
+ query,
headers=headers,
)
# rate limited - sleep for 2-10 secs and retry (up to 100 times, ~10 minutes)
# https://support.airtable.com/hc/en-us/articles/203313985-Public-REST-API
# > 5 requests per second, per base
if response.status_code == 429:
if retry < 100:
retry = retry + 1
sleep(randint(1, 10))
continue
else:
return await send_msg(
"Airtable: too many retries, aborting!", force_notify=True
)
retry = 0 # reset retry counter
if response.status_code != 200:
status_code = str(response.status_code)
response_text = response.text or "empty response"
message = (
"Airtable blocklist integration responded with code "
+ status_code
+ ": "
+ response_text
)
return await send_msg(message, force_notify=False)
data = response.json()
if len(data["records"]) == 0:
return print(
"Airtable returned 0 records - make sure your configuration is correct"
)
skylinks = skylinks + [
entry["fields"].get(AIRTABLE_FIELD, "") for entry in data["records"]
]
skylinks = [
skylink.strip() for skylink in skylinks if skylink
] # filter empty skylinks, most likely empty rows, trim whitespace
offset = data.get("offset")
print(
"Sending /skynet/blocklist request with "
+ str(len(skylinks))
+ " skylinks to siad"
)
response = requests.post(
"http://" + ipaddress + ":9980/skynet/blocklist",
data=json.dumps({"add": skylinks}),
headers={"User-Agent": "Sia-Agent"},
auth=HTTPBasicAuth("", get_api_password()),
)
if response.status_code != 200:
status_code = str(response.status_code)
response_text = response.text or "empty response"
message = (
"Airtable blocklist request responded with code "
+ status_code
+ ": "
+ response_text
)
return await send_msg(message, force_notify=False)
response_json = json.loads(response.text)
invalid_skylinks = response_json["invalids"]
if invalid_skylinks is None:
return await send_msg("Blocklist successfully updated all skylinks")
return await send_msg(
"Blocklist responded ok but failed to update "
+ str(len(invalid_skylinks))
+ " skylinks: "
+ json.dumps(invalid_skylinks)
)
loop = asyncio.get_event_loop()
loop.run_until_complete(run_checks())
# --- BASH EQUIVALENT
# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")
# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia)
# curl --data "{\"add\" : ${skylinks}}" "${ipaddress}:8000/skynet/blocklist"

View File

@ -1,7 +1,6 @@
0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/skynet-webportal/.env
0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/skynet-webportal/.env sia 8
0 * * * * /home/user/skynet-webportal/setup-scripts/health-checker.py /home/user/skynet-webportal/.env sia 1
30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env
44 5 * * * /home/user/skynet-webportal/scripts/backup-aws-s3.sh 1>>/home/user/skynet-webportal/logs/backup-aws-s3.log 2>>/home/user/skynet-webportal/logs/backup-aws-s3.log
6 13 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1>>/home/user/skynet-webportal/logs/db_backup.log 2>>/home/user/skynet-webportal/logs/db_backup.log
0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200