Merge pull request #1497 from SkynetLabs/adjust-blocklist-airtable-script
do not prune nginx cache on airtable blocklist script
This commit is contained in:
Karol Wypchło
GitHub
commit
a7bad7411a
|
|
@ -1,6 +1,9 @@
|
|||
include /etc/nginx/conf.d/include/init-optional-variables;
|
||||
|
||||
location /skynet/blocklist {
|
||||
client_max_body_size 10m; # increase max body size to account for large lists
|
||||
client_body_buffer_size 10m; # force whole body to memory so we can read it
|
||||
|
||||
content_by_lua_block {
|
||||
local httpc = require("resty.http").new()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,18 +1,15 @@
|
|||
#! /usr/bin/env bash
|
||||
|
||||
# This script adds a skylink to the sia blocklist and removes the skylink from
|
||||
# nginx cache. The script should be run locally on each skynet webportal
|
||||
# server. The automatic script that is used to continuously sync an Airtable
|
||||
# sheet list with the blocklist on the web portals is
|
||||
# /setup-scripts/blocklist-airtable.py
|
||||
# This script is for manual skylink blocking. It accepts either a single
|
||||
# skylink or a file containing list of skylinks. The script is intented
|
||||
# for manual use and it should be run locally on each skynet webportal server.
|
||||
# The automatic script that is used to continuously sync an Airtable sheet
|
||||
# list with the blocklist on the web portals is /setup-scripts/blocklist-airtable.py
|
||||
|
||||
set -e # exit on first error
|
||||
|
||||
# Number of skylinks to block within one batch
|
||||
BATCH_SIZE=1000
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
echo "Please provide either a skylink or file with skylinks separated by new lines" && exit 1
|
||||
echo "Please provide either a skylink or a file with skylinks separated by new lines" && exit 1
|
||||
fi
|
||||
|
||||
#########################################################
|
||||
|
|
@ -37,45 +34,18 @@ else
|
|||
skylinks=("$1") # just single skylink passed as input argument
|
||||
fi
|
||||
|
||||
# Block skylinks in batches
|
||||
skylinks_len=${#skylinks[@]}
|
||||
for (( i = 0; i < $skylinks_len; i++ )); do
|
||||
# Add skylink to batch
|
||||
skylink="${skylinks[$i]}"
|
||||
echo ".. ⌁ Adding skylink ${skylink} to batch..."
|
||||
batch_skylinks+=("$skylink")
|
||||
# get local nginx ip adress
|
||||
nginx_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx)
|
||||
|
||||
# For performance reasons on each iteration we do not block a single
|
||||
# skylink, but we block skylinks in batches with BATCH_SIZE size mainly
|
||||
# because of nginx cache search.
|
||||
# If (batch len == batch size) or (we have last batch):
|
||||
if (( ${#batch_skylinks[@]} == $BATCH_SIZE || $i == $skylinks_len - 1 )); then
|
||||
echo "--------------------------------------------"
|
||||
# iterate over provided skylinks and block them one by one
|
||||
for skylink in "${skylinks[@]}"; do
|
||||
printf "Blocking ${skylink} ... "
|
||||
status_code=$(curl --write-out '%{http_code}' --silent --output /dev/null --data "{\"add\":[\"$skylink\"]}" "http://${nginx_ip}:8000/skynet/blocklist")
|
||||
|
||||
# Add to Sia blocklist
|
||||
echo "Blocking batch skylinks in skyd..."
|
||||
skylinks_space_separated="$(IFS=' '; echo "${batch_skylinks[*]}")"
|
||||
docker exec sia siac skynet blocklist add $skylinks_space_separated
|
||||
|
||||
# Remove from NGINX cache
|
||||
# NOTE:
|
||||
# If there are changes to how the NGINX cache is being cleared, the same
|
||||
# changes need to be applied to the /setup-scripts/blocklist-airtable.py
|
||||
# script.
|
||||
echo "Removing batch skylinks from Nginx cache..."
|
||||
skylinks_pipe_separated="$(IFS='|'; echo "${batch_skylinks[*]}")"
|
||||
cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ($skylinks_pipe_separated)'"
|
||||
docker exec -it nginx bash -c "${cached_files_command} | xargs -r rm"
|
||||
|
||||
# Clear batch
|
||||
batch_skylinks=()
|
||||
|
||||
echo "--------------------------------------------"
|
||||
# print blocklist response status code
|
||||
if [ $status_code = "204" ]; then
|
||||
echo "done"
|
||||
else
|
||||
echo "error $status_code"
|
||||
fi
|
||||
done
|
||||
|
||||
# Hot reload Nginx to get rid of deleted open files
|
||||
echo "Hot reloading nginx..."
|
||||
docker exec nginx nginx -s reload
|
||||
|
||||
echo "✓ All done !"
|
||||
|
|
|
|||
|
|
@ -122,63 +122,30 @@ async def block_skylinks_from_airtable():
|
|||
)
|
||||
await send_msg(message, file=("\n".join(invalid_skylinks)))
|
||||
|
||||
apipassword = exec("docker exec sia cat /sia-data/apipassword")
|
||||
ipaddress = exec(
|
||||
"docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia"
|
||||
"docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx"
|
||||
)
|
||||
|
||||
print("Sending blocklist request to siad")
|
||||
print("Sending blocklist request to siad through nginx")
|
||||
response = requests.post(
|
||||
"http://" + ipaddress + ":9980/skynet/blocklist",
|
||||
auth=("", apipassword),
|
||||
headers={"user-agent": "Sia-Agent"},
|
||||
"http://" + ipaddress + ":8000/skynet/blocklist",
|
||||
data=json.dumps({"add": skylinks}),
|
||||
)
|
||||
|
||||
if response.status_code == 204:
|
||||
print("Siad blocklist successfully updated with provided skylink")
|
||||
else:
|
||||
print(json.dumps({"add": skylinks}))
|
||||
|
||||
if response.status_code != 204:
|
||||
status_code = str(response.status_code)
|
||||
response_text = response.text or "empty response"
|
||||
message = (
|
||||
"Siad blocklist endpoint responded with code "
|
||||
"Airtable blocklist request responded with code "
|
||||
+ status_code
|
||||
+ ": "
|
||||
+ response_text
|
||||
)
|
||||
return await send_msg(message, force_notify=False)
|
||||
|
||||
# Remove from NGINX cache
|
||||
# NOTE:
|
||||
# If there are changes to how the NGINX cache is being cleared, the same
|
||||
# changes need to be applied to the /scripts/blocklist-skylink.sh script.
|
||||
print("Searching nginx cache for blocked files")
|
||||
cached_files_count = 0
|
||||
batch_size = 1000
|
||||
for i in range(0, len(skylinks), batch_size):
|
||||
cached_files_command = (
|
||||
"find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ("
|
||||
+ "|".join(skylinks[i : i + batch_size])
|
||||
+ ")'"
|
||||
)
|
||||
cached_files_count += int(
|
||||
exec(
|
||||
'docker exec nginx bash -c "'
|
||||
+ cached_files_command
|
||||
+ ' | xargs -r rm -v | wc -l"'
|
||||
)
|
||||
)
|
||||
|
||||
if cached_files_count == 0:
|
||||
return print("No nginx cached files matching blocked skylinks were found")
|
||||
else:
|
||||
print("Hot reloading nginx")
|
||||
exec("docker exec nginx nginx -s reload")
|
||||
|
||||
message = (
|
||||
"Purged " + str(cached_files_count) + " blocklisted files from nginx cache"
|
||||
)
|
||||
return await send_msg(message)
|
||||
return await send_msg("Siad blocklist successfully updated with provided skylink")
|
||||
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
|
|
@ -186,6 +153,5 @@ loop.run_until_complete(run_checks())
|
|||
|
||||
# --- BASH EQUIVALENT
|
||||
# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")
|
||||
# apipassword=$(docker exec sia cat /sia-data/apipassword)
|
||||
# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia)
|
||||
# curl -A "Sia-Agent" --user "":"${apipassword}" --data "{\"add\" : ${skylinks}}" "${ipaddress}:9980/skynet/blocklist"
|
||||
# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx)
|
||||
# curl --data "{\"add\" : ${skylinks}}" "${ipaddress}:8000/skynet/blocklist"
|
||||
|
|
|
|||
Reference in New Issue