This repository has been archived on 2022-10-07. You can view files and clone it, but cannot push or open issues or pull requests.
skynet-webportal/setup-scripts/blocklist-airtable.py

158 lines
5.1 KiB
Python
Raw Normal View History

2021-03-01 15:47:06 +00:00
#!/usr/bin/env python3
2021-03-01 15:59:38 +00:00
from bot_utils import setup, send_msg
2021-07-18 06:22:32 +00:00
from random import randint
from time import sleep
2021-03-01 15:47:06 +00:00
import traceback
import os
2021-11-10 12:46:43 +00:00
import sys
import re
import asyncio
import requests
import json
setup()
2021-03-01 16:13:00 +00:00
2021-11-10 12:46:43 +00:00
2021-11-10 13:22:00 +00:00
AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY")
AIRTABLE_BASE = os.getenv("AIRTABLE_BASE")
AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE")
AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD")
# Check environment variables are defined
2021-11-10 14:13:14 +00:00
for value in [AIRTABLE_API_KEY, AIRTABLE_BASE, AIRTABLE_TABLE, AIRTABLE_FIELD]:
if not value:
2021-11-10 13:22:00 +00:00
sys.exit("Configuration error: Missing AirTable environment variable.")
2021-03-01 22:39:28 +00:00
async def run_checks():
try:
await block_skylinks_from_airtable()
except: # catch all exceptions
trace = traceback.format_exc()
await send_msg("```\n{}\n```".format(trace), force_notify=True)
2021-03-01 22:39:28 +00:00
def exec(command):
return os.popen(command).read().strip()
2021-03-01 15:47:06 +00:00
async def block_skylinks_from_airtable():
2021-03-01 22:39:28 +00:00
print("Pulling blocked skylinks from Airtable via api integration")
headers = {"Authorization": "Bearer " + AIRTABLE_API_KEY}
2021-03-01 22:08:38 +00:00
skylinks = []
2021-03-01 22:15:54 +00:00
offset = None
2021-07-18 06:22:32 +00:00
retry = 0
2021-03-01 22:08:38 +00:00
while len(skylinks) == 0 or offset:
print(
"Requesting a batch of records from Airtable with "
+ (offset if offset else "empty")
+ " offset"
+ (" (retry " + str(retry) + ")" if retry else "")
)
query = "&".join(
["fields%5B%5D=" + AIRTABLE_FIELD, ("offset=" + offset) if offset else ""]
)
2021-03-01 22:39:28 +00:00
response = requests.get(
"https://api.airtable.com/v0/"
+ AIRTABLE_BASE
+ "/"
+ AIRTABLE_TABLE
+ "?"
+ query,
2021-03-01 22:39:28 +00:00
headers=headers,
2021-03-01 22:08:38 +00:00
)
2021-03-01 16:19:13 +00:00
2021-07-18 06:22:32 +00:00
# rate limited - sleep for 2-10 secs and retry (up to 100 times, ~10 minutes)
# https://support.airtable.com/hc/en-us/articles/203313985-Public-REST-API
# > 5 requests per second, per base
if response.status_code == 429:
if retry < 100:
retry = retry + 1
sleep(randint(1, 10))
2021-07-18 06:22:32 +00:00
continue
else:
return await send_msg(
"Airtable: too many retries, aborting!", force_notify=True
)
2021-07-18 06:22:32 +00:00
retry = 0 # reset retry counter
2021-03-01 22:39:28 +00:00
if response.status_code != 200:
status_code = str(response.status_code)
response_text = response.text or "empty response"
message = (
"Airtable blocklist integration responded with code "
+ status_code
+ ": "
+ response_text
)
return await send_msg(message, force_notify=False)
2021-03-01 22:39:28 +00:00
data = response.json()
2021-03-01 16:56:30 +00:00
2021-03-03 13:07:29 +00:00
if len(data["records"]) == 0:
return print(
"Airtable returned 0 records - make sure your configuration is correct"
)
2021-03-03 13:07:29 +00:00
skylinks = skylinks + [
entry["fields"].get(AIRTABLE_FIELD, "") for entry in data["records"]
]
skylinks = [
skylink.strip() for skylink in skylinks if skylink
] # filter empty skylinks, most likely empty rows, trim whitespace
2021-03-01 22:39:28 +00:00
offset = data.get("offset")
2021-03-01 22:22:53 +00:00
print("Airtable returned total " + str(len(skylinks)) + " skylinks to block")
2021-03-01 22:39:28 +00:00
2021-03-02 12:40:18 +00:00
skylinks_returned = skylinks
skylinks = [
skylink for skylink in skylinks if re.search("^[a-zA-Z0-9_-]{46}$", skylink)
]
2021-03-02 12:40:18 +00:00
if len(skylinks_returned) != len(skylinks):
invalid_skylinks = [
str(skylink) for skylink in list(set(skylinks_returned) - set(skylinks))
]
message = (
str(len(invalid_skylinks))
+ " of the skylinks returned from Airtable are not valid"
)
await send_msg(message, file=("\n".join(invalid_skylinks)))
2021-03-02 12:40:18 +00:00
ipaddress = exec(
2021-12-23 11:55:47 +00:00
"docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx"
)
2021-03-01 22:39:28 +00:00
2021-12-23 11:55:47 +00:00
print("Sending blocklist request to siad through nginx")
2021-03-01 22:39:28 +00:00
response = requests.post(
2021-12-23 11:55:47 +00:00
"http://" + ipaddress + ":8000/skynet/blocklist",
2021-03-01 22:39:28 +00:00
data=json.dumps({"add": skylinks}),
)
2021-12-23 11:55:47 +00:00
print(json.dumps({"add": skylinks}))
if response.status_code != 204:
2021-03-01 22:39:28 +00:00
status_code = str(response.status_code)
response_text = response.text or "empty response"
message = (
2021-12-23 11:55:47 +00:00
"Airtable blocklist request responded with code "
+ status_code
+ ": "
+ response_text
)
return await send_msg(message, force_notify=False)
2021-03-01 15:47:06 +00:00
2021-12-23 11:55:47 +00:00
return await send_msg("Siad blocklist successfully updated with provided skylink")
2021-03-01 22:39:28 +00:00
loop = asyncio.get_event_loop()
loop.run_until_complete(run_checks())
2021-03-01 16:09:18 +00:00
2021-03-01 15:47:06 +00:00
# --- BASH EQUIVALENT
2021-03-02 11:43:23 +00:00
# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")
2021-12-23 11:55:47 +00:00
# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx)
# curl --data "{\"add\" : ${skylinks}}" "${ipaddress}:8000/skynet/blocklist"