This repository has been archived on 2022-10-07. You can view files and clone it, but cannot push or open issues or pull requests.
skynet-webportal/setup-scripts/blocklist-airtable.py

115 lines
4.9 KiB
Python
Raw Normal View History

2021-03-01 15:47:06 +00:00
#!/usr/bin/env python3
2021-03-02 12:41:23 +00:00
import traceback, os, re, asyncio, requests, json, discord
2021-03-01 15:59:38 +00:00
from bot_utils import setup, send_msg
2021-03-01 15:47:06 +00:00
2021-03-01 16:13:00 +00:00
bot_token = setup()
client = discord.Client()
2021-03-01 22:39:28 +00:00
AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY")
AIRTABLE_BASE = os.getenv("AIRTABLE_BASE", "app89plJvA9EqTJEc")
AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE", "Table%201")
AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD", "Link")
def exec(command):
return os.popen(command).read().strip()
2021-03-01 15:47:06 +00:00
async def block_skylinks_from_airtable():
2021-03-01 22:39:28 +00:00
print("Pulling blocked skylinks from Airtable via api integration")
headers = {"Authorization": "Bearer " + AIRTABLE_API_KEY}
2021-03-01 22:08:38 +00:00
skylinks = []
2021-03-01 22:15:54 +00:00
offset = None
2021-03-01 22:08:38 +00:00
while len(skylinks) == 0 or offset:
2021-03-01 22:39:28 +00:00
print("Requesting a batch of records from Airtable with " + (offset if offset else "empty") + " offset")
query = "&".join(["fields%5B%5D=" + AIRTABLE_FIELD, ("offset=" + offset) if offset else ""])
response = requests.get(
"https://api.airtable.com/v0/" + AIRTABLE_BASE + "/" + AIRTABLE_TABLE + "?" + query,
headers=headers,
2021-03-01 22:08:38 +00:00
)
2021-03-01 16:19:13 +00:00
2021-03-01 22:39:28 +00:00
if response.status_code != 200:
status_code = str(response.status_code)
response_text = response.text or "empty response"
message = "Airtable blocklist integration responded with code " + status_code + ": " + response_text
2021-03-01 22:13:28 +00:00
return print(message) or await send_msg(client, message, force_notify=False)
2021-03-01 22:39:28 +00:00
data = response.json()
2021-03-01 16:56:30 +00:00
2021-03-03 13:07:29 +00:00
if len(data["records"]) == 0:
return print("Airtable returned 0 records - make sure your configuration is correct")
skylinks = skylinks + [entry["fields"].get(AIRTABLE_FIELD, "") for entry in data["records"]]
skylinks = [skylink for skylink in skylinks if skylink] # filter empty skylinks, most likely empty rows
2021-03-01 22:39:28 +00:00
offset = data.get("offset")
2021-03-01 22:22:53 +00:00
print("Airtable returned total " + str(len(skylinks)) + " skylinks to block")
2021-03-01 22:39:28 +00:00
2021-03-02 12:40:18 +00:00
skylinks_returned = skylinks
skylinks = [skylink for skylink in skylinks if re.search("^[a-zA-Z0-9_-]{46}$", skylink)]
if len(skylinks_returned) != len(skylinks):
invalid_skylinks = [str(skylink) for skylink in list(set(skylinks_returned) - set(skylinks))]
2021-03-02 12:44:27 +00:00
message = str(len(invalid_skylinks)) + " of the skylinks returned from Airtable are not valid"
2021-03-02 12:40:18 +00:00
print(message) or await send_msg(client, message, file=("\n".join(invalid_skylinks)))
2021-03-01 22:39:28 +00:00
apipassword = exec("docker exec sia cat /sia-data/apipassword")
ipaddress = exec("docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia")
2021-03-01 16:08:10 +00:00
print("Sending blocklist request to siad")
2021-03-01 22:39:28 +00:00
response = requests.post(
"http://" + ipaddress + ":9980/skynet/blocklist",
auth=("", apipassword),
headers={"user-agent": "Sia-Agent"},
data=json.dumps({"add": skylinks}),
)
2021-03-01 16:08:10 +00:00
if response.status_code == 204:
2021-03-01 17:50:01 +00:00
print("Siad blocklist successfully updated with provided skylink")
2021-03-01 16:08:10 +00:00
else:
2021-03-01 22:39:28 +00:00
status_code = str(response.status_code)
response_text = response.text or "empty response"
message = "Siad blocklist endpoint responded with code " + status_code + ": " + response_text
2021-03-01 22:13:28 +00:00
return print(message) or await send_msg(client, message, force_notify=False)
2021-03-01 15:47:06 +00:00
2021-03-01 17:41:41 +00:00
print("Searching nginx cache for blocked files")
2021-03-01 22:39:28 +00:00
cached_files_command = (
"/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -Els '^KEY: .*("
+ "|".join(skylinks)
+ ")'"
)
2021-03-02 15:44:56 +00:00
cached_files_count = int(exec('docker exec -it nginx bash -c "' + cached_files_command + ' | wc -l"') or 0)
2021-03-01 17:39:43 +00:00
if cached_files_count == 0:
return print("No nginx cached files matching blocked skylinks were found")
2021-03-01 22:39:28 +00:00
exec('docker exec -it nginx bash -c "' + cached_files_command + ' | xargs rm"')
message = "Purged " + str(cached_files_count) + " blocklisted files from nginx cache"
2021-03-01 17:39:43 +00:00
return print(message) or await send_msg(client, message)
2021-03-01 16:56:30 +00:00
2021-03-01 22:39:28 +00:00
2021-03-01 15:47:06 +00:00
async def exit_after(delay):
await asyncio.sleep(delay)
os._exit(0)
2021-03-01 22:39:28 +00:00
2021-03-01 15:59:38 +00:00
@client.event
2021-03-01 15:47:06 +00:00
async def on_ready():
2021-03-01 15:59:38 +00:00
try:
await block_skylinks_from_airtable()
except: # catch all exceptions
2021-03-01 22:39:28 +00:00
message = "```\n{}\n```".format(traceback.format_exc())
await send_msg(client, message, force_notify=False)
2021-03-01 15:47:06 +00:00
asyncio.create_task(exit_after(3))
2021-03-01 22:39:28 +00:00
2021-03-01 16:09:18 +00:00
client.run(bot_token)
2021-03-01 15:47:06 +00:00
# --- BASH EQUIVALENT
2021-03-02 11:43:23 +00:00
# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")
2021-03-01 15:47:06 +00:00
# apipassword=$(docker exec sia cat /sia-data/apipassword)
# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia)
# curl -A "Sia-Agent" --user "":"${apipassword}" --data "{\"add\" : ${skylinks}}" "${ipaddress}:9980/skynet/blocklist"