Merge pull request #1321 from SkynetLabs/blocklist-refactor

Update blocklist skylink script
This commit is contained in:
Ivaylo Novakov 2021-10-26 18:31:36 +02:00 committed by GitHub
commit 5f6d984566
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 32 additions and 40 deletions

View File

@ -0,0 +1,2 @@
- Remove hardcoded server list from `blocklist-skylink.sh` so it removes server
list duplication and can also be called from Ansible.

View File

@ -1,9 +1,10 @@
#! /usr/bin/env bash #! /usr/bin/env bash
# This script is meant to be used when manually adding a skylink to the # This script adds a skylink to the sia blocklist and removes the skylink from
# blocklist on all the skynet web portals. The automatic script that is used to # nginx cache. The script should be run locally on each skynet webportal
# continuously sync a google sheets list with the blocklist on the web portals # server. The automatic script that is used to continuously sync an Airtable
# is /setup-scripts/blocklist-airtable.py # sheet list with the blocklist on the web portals is
# /setup-scripts/blocklist-airtable.py
set -e # exit on first error set -e # exit on first error
@ -17,54 +18,39 @@ fi
######################################################### #########################################################
skylinks=() skylinks=()
if test -f "$1"; then if test -f "$1"; then
OLDIFS=$IFS
IFS=','
line_number=1 line_number=1
while read line
# Read file including the last line even when it doesn't end with newline
while IFS="" read -r line || [ -n "$line" ];
do do
if [[ $line =~ ([a-zA-Z0-9_-]{46}) ]]; then if [[ $line =~ (^[a-zA-Z0-9_-]{46}$) ]]; then
skylinks+=("$BASH_REMATCH") skylinks+=("$line")
else else
echo "Incorrect skylink at line ${line_number}: $line" && exit 1 echo "Incorrect skylink at line ${line_number}: $line" && exit 1
fi fi
let line_number+=1 let line_number+=1
done < $1; done < $1;
IFS=$OLDIFS
else else
skylinks=("$1") # just single skylink passed as input argument skylinks=("$1") # just single skylink passed as input argument
fi fi
######################################################################### for skylink in "${skylinks[@]}";
# iterate through all servers, block the skylinks and purge it from cache
#########################################################################
declare -a servers=( "eu-ger-1.siasky.net" "eu-ger-2.siasky.net" "eu-ger-3.siasky.net" "eu-ger-4.siasky.net" "eu-ger-5.siasky.net" "eu-ger-6.siasky.net" "eu-ger-7.siasky.net" "eu-ger-8.siasky.net"
"eu-ger-9.siasky.net" "eu-ger-10.siasky.net" "eu-ger-11.siasky.net" "eu-ger-12.siasky.net"
"eu-fin-1.siasky.net" "eu-fin-2.siasky.net" "eu-fin-3.siasky.net" "eu-fin-4.siasky.net" "eu-fin-5.siasky.net" "eu-fin-6.siasky.net" "eu-fin-7-siasky.net" "eu-fin-8.siasky.net"
"eu-fin-9.siasky.net" "eu-fin-10.siasky.net" "eu-fin-11.siasky.net" "eu-fin-12.siasky.net" "eu-fin-13.siasky.net" "eu-fin-14.siasky.net" "eu-fin-15.siasky.net"
"eu-pol-1.siasky.net" "eu-pol-2.siasky.net" "eu-pol-3.siasky.net" "eu-pol-4.siasky.net" "eu-pol-5.siasky.net"
"us-ny-1.siasky.net" "us-ny-2.siasky.net"
"us-or-1.siasky.net" "us-or-2.siasky.net"
"us-la-1.siasky.net" "us-la-2.siasky.net" "us-la-3.siasky.net"
"us-pa-1.siasky.net" "us-pa-2.siasky.net"
"us-va-1.siasky.net" "us-va-2.siasky.net" "us-va-3.siasky.net" "us-va-4.siasky.net" "us-va-5.siasky.net" "us-va-6.siasky.net"
"as-hk-1.siasky.net" "as-sp-1.siasky.net" "as-sp-2.siasky.net"
"siasky.xyz" "dev1.siasky.dev" "dev2.siasky.dev" "dev3.siasky.dev")
for server in "${servers[@]}";
do do
for skylink in "${skylinks[@]}"; echo ".. ⌁ Blocking skylink ${skylink}"
do
echo ".. ⌁ Blocking skylink ${skylink} on ${server}" # Add to Sia blocklist
docker exec sia siac skynet blocklist add "${skylink}"
# Add to blocklist
ssh -q -t user@${server} "docker exec sia siac skynet blocklist add ${skylink}" # Remove from NGINX cache
# NOTE:
# Remove from NGINX cache # If there are changes to how the NGINX cache is being cleared, the same
cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'" # changes need to be applied to the /setup-scripts/blocklist-airtable.py
ssh -q -t user@${server} "docker exec -it nginx bash -c ${cached_files_command} | xargs -r rm" # script.
cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'"
echo ".. ⌁ Skylink ${skylink} Blocked on ${server}" docker exec -it nginx bash -c "${cached_files_command}" | xargs -r rm
echo "--------------------------------------------"
done echo ".. ⌁ Skylink ${skylink} Blocked"
echo "--------------------------------------------"
done done
echo "✓ All done !" echo "✓ All done !"

View File

@ -141,6 +141,10 @@ async def block_skylinks_from_airtable():
) )
return await send_msg(message, force_notify=False) return await send_msg(message, force_notify=False)
# Remove from NGINX cache
# NOTE:
# If there are changes to how the NGINX cache is being cleared, the same
# changes need to be applied to the /scripts/blocklist-skylink.sh script.
print("Searching nginx cache for blocked files") print("Searching nginx cache for blocked files")
cached_files_count = 0 cached_files_count = 0
batch_size = 1000 batch_size = 1000