do not prune blocklist manually

This commit is contained in:
Karol Wypchlo 2021-12-23 12:55:47 +01:00
parent 25bd31d679
commit 5cf5acb4e2
No known key found for this signature in database
GPG Key ID: C92C016317A964D0
2 changed files with 13 additions and 44 deletions

View File

@ -1,6 +1,9 @@
include /etc/nginx/conf.d/include/init-optional-variables; include /etc/nginx/conf.d/include/init-optional-variables;
location /skynet/blocklist { location /skynet/blocklist {
client_max_body_size 10m; # increase max body size to account for large lists
client_body_buffer_size 10m; # force whole body to memory so we can read it
content_by_lua_block { content_by_lua_block {
local httpc = require("resty.http").new() local httpc = require("resty.http").new()

View File

@ -122,63 +122,30 @@ async def block_skylinks_from_airtable():
) )
await send_msg(message, file=("\n".join(invalid_skylinks))) await send_msg(message, file=("\n".join(invalid_skylinks)))
apipassword = exec("docker exec sia cat /sia-data/apipassword")
ipaddress = exec( ipaddress = exec(
"docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia" "docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx"
) )
print("Sending blocklist request to siad") print("Sending blocklist request to siad through nginx")
response = requests.post( response = requests.post(
"http://" + ipaddress + ":9980/skynet/blocklist", "http://" + ipaddress + ":8000/skynet/blocklist",
auth=("", apipassword),
headers={"user-agent": "Sia-Agent"},
data=json.dumps({"add": skylinks}), data=json.dumps({"add": skylinks}),
) )
if response.status_code == 204: print(json.dumps({"add": skylinks}))
print("Siad blocklist successfully updated with provided skylink")
else: if response.status_code != 204:
status_code = str(response.status_code) status_code = str(response.status_code)
response_text = response.text or "empty response" response_text = response.text or "empty response"
message = ( message = (
"Siad blocklist endpoint responded with code " "Airtable blocklist request responded with code "
+ status_code + status_code
+ ": " + ": "
+ response_text + response_text
) )
return await send_msg(message, force_notify=False) return await send_msg(message, force_notify=False)
# Remove from NGINX cache return await send_msg("Siad blocklist successfully updated with provided skylink")
# NOTE:
# If there are changes to how the NGINX cache is being cleared, the same
# changes need to be applied to the /scripts/blocklist-skylink.sh script.
print("Searching nginx cache for blocked files")
cached_files_count = 0
batch_size = 1000
for i in range(0, len(skylinks), batch_size):
cached_files_command = (
"find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ("
+ "|".join(skylinks[i : i + batch_size])
+ ")'"
)
cached_files_count += int(
exec(
'docker exec nginx bash -c "'
+ cached_files_command
+ ' | xargs -r rm -v | wc -l"'
)
)
if cached_files_count == 0:
return print("No nginx cached files matching blocked skylinks were found")
else:
print("Hot reloading nginx")
exec("docker exec nginx nginx -s reload")
message = (
"Purged " + str(cached_files_count) + " blocklisted files from nginx cache"
)
return await send_msg(message)
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
@ -186,6 +153,5 @@ loop.run_until_complete(run_checks())
# --- BASH EQUIVALENT # --- BASH EQUIVALENT
# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')") # skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")
# apipassword=$(docker exec sia cat /sia-data/apipassword) # ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx)
# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia) # curl --data "{\"add\" : ${skylinks}}" "${ipaddress}:8000/skynet/blocklist"
# curl -A "Sia-Agent" --user "":"${apipassword}" --data "{\"add\" : ${skylinks}}" "${ipaddress}:9980/skynet/blocklist"