diff --git a/changelog/items/other/refactor-blocklist.md b/changelog/items/other/refactor-blocklist.md new file mode 100644 index 00000000..28629dab --- /dev/null +++ b/changelog/items/other/refactor-blocklist.md @@ -0,0 +1,2 @@ +- Remove hardcoded server list from `blocklist-skylink.sh` so it removes server + list duplication and can also be called from Ansible. \ No newline at end of file diff --git a/scripts/blocklist-skylink.sh b/scripts/blocklist-skylink.sh index e9d7f778..8b81e09f 100755 --- a/scripts/blocklist-skylink.sh +++ b/scripts/blocklist-skylink.sh @@ -1,9 +1,10 @@ #! /usr/bin/env bash -# This script is meant to be used when manually adding a skylink to the -# blocklist on all the skynet web portals. The automatic script that is used to -# continuously sync a google sheets list with the blocklist on the web portals -# is /setup-scripts/blocklist-airtable.py +# This script adds a skylink to the sia blocklist and removes the skylink from +# nginx cache. The script should be run locally on each skynet webportal +# server. The automatic script that is used to continuously sync an Airtable +# sheet list with the blocklist on the web portals is +# /setup-scripts/blocklist-airtable.py set -e # exit on first error @@ -17,54 +18,39 @@ fi ######################################################### skylinks=() if test -f "$1"; then - OLDIFS=$IFS - IFS=',' line_number=1 - while read line + + # Read file including the last line even when it doesn't end with newline + while IFS="" read -r line || [ -n "$line" ]; do - if [[ $line =~ ([a-zA-Z0-9_-]{46}) ]]; then - skylinks+=("$BASH_REMATCH") + if [[ $line =~ (^[a-zA-Z0-9_-]{46}$) ]]; then + skylinks+=("$line") else echo "Incorrect skylink at line ${line_number}: $line" && exit 1 fi let line_number+=1 done < $1; - IFS=$OLDIFS else skylinks=("$1") # just single skylink passed as input argument fi -######################################################################### -# iterate through all servers, block the skylinks and purge it from cache -######################################################################### -declare -a servers=( "eu-ger-1.siasky.net" "eu-ger-2.siasky.net" "eu-ger-3.siasky.net" "eu-ger-4.siasky.net" "eu-ger-5.siasky.net" "eu-ger-6.siasky.net" "eu-ger-7.siasky.net" "eu-ger-8.siasky.net" - "eu-ger-9.siasky.net" "eu-ger-10.siasky.net" "eu-ger-11.siasky.net" "eu-ger-12.siasky.net" - "eu-fin-1.siasky.net" "eu-fin-2.siasky.net" "eu-fin-3.siasky.net" "eu-fin-4.siasky.net" "eu-fin-5.siasky.net" "eu-fin-6.siasky.net" "eu-fin-7-siasky.net" "eu-fin-8.siasky.net" - "eu-fin-9.siasky.net" "eu-fin-10.siasky.net" "eu-fin-11.siasky.net" "eu-fin-12.siasky.net" "eu-fin-13.siasky.net" "eu-fin-14.siasky.net" "eu-fin-15.siasky.net" - "eu-pol-1.siasky.net" "eu-pol-2.siasky.net" "eu-pol-3.siasky.net" "eu-pol-4.siasky.net" "eu-pol-5.siasky.net" - "us-ny-1.siasky.net" "us-ny-2.siasky.net" - "us-or-1.siasky.net" "us-or-2.siasky.net" - "us-la-1.siasky.net" "us-la-2.siasky.net" "us-la-3.siasky.net" - "us-pa-1.siasky.net" "us-pa-2.siasky.net" - "us-va-1.siasky.net" "us-va-2.siasky.net" "us-va-3.siasky.net" "us-va-4.siasky.net" "us-va-5.siasky.net" "us-va-6.siasky.net" - "as-hk-1.siasky.net" "as-sp-1.siasky.net" "as-sp-2.siasky.net" - "siasky.xyz" "dev1.siasky.dev" "dev2.siasky.dev" "dev3.siasky.dev") -for server in "${servers[@]}"; +for skylink in "${skylinks[@]}"; do - for skylink in "${skylinks[@]}"; - do - echo ".. ⌁ Blocking skylink ${skylink} on ${server}" - - # Add to blocklist - ssh -q -t user@${server} "docker exec sia siac skynet blocklist add ${skylink}" - - # Remove from NGINX cache - cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'" - ssh -q -t user@${server} "docker exec -it nginx bash -c ${cached_files_command} | xargs -r rm" - - echo ".. ⌁ Skylink ${skylink} Blocked on ${server}" - echo "--------------------------------------------" - done + echo ".. ⌁ Blocking skylink ${skylink}" + + # Add to Sia blocklist + docker exec sia siac skynet blocklist add "${skylink}" + + # Remove from NGINX cache + # NOTE: + # If there are changes to how the NGINX cache is being cleared, the same + # changes need to be applied to the /setup-scripts/blocklist-airtable.py + # script. + cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'" + docker exec -it nginx bash -c "${cached_files_command}" | xargs -r rm + + echo ".. ⌁ Skylink ${skylink} Blocked" + echo "--------------------------------------------" done echo "✓ All done !" diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py index 1d4c28ce..3dbce3ed 100755 --- a/setup-scripts/blocklist-airtable.py +++ b/setup-scripts/blocklist-airtable.py @@ -141,6 +141,10 @@ async def block_skylinks_from_airtable(): ) return await send_msg(message, force_notify=False) + # Remove from NGINX cache + # NOTE: + # If there are changes to how the NGINX cache is being cleared, the same + # changes need to be applied to the /scripts/blocklist-skylink.sh script. print("Searching nginx cache for blocked files") cached_files_count = 0 batch_size = 1000