Merge pull request #1321 from SkynetLabs/blocklist-refactor

Update blocklist skylink script

changelog/items/other/refactor-blocklist.md

@@ -0,0 +1,2 @@
+- Remove hardcoded server list from `blocklist-skylink.sh` so it removes server
+  list duplication and can also be called from Ansible.

scripts/blocklist-skylink.sh

@@ -1,9 +1,10 @@
 #! /usr/bin/env bash
 
-# This script is meant to be used when manually adding a skylink to the
+# This script adds a skylink to the sia blocklist and removes the skylink from
-# blocklist on all the skynet web portals. The automatic script that is used to
+# nginx cache. The script should be run locally on each skynet webportal
-# continuously sync a google sheets list with the blocklist on the web portals
+# server. The automatic script that is used to continuously sync an Airtable
-# is /setup-scripts/blocklist-airtable.py 
+# sheet list with the blocklist on the web portals is
+# /setup-scripts/blocklist-airtable.py 
 
 set -e # exit on first error
 
@@ -17,54 +18,39 @@ fi
 #########################################################
 skylinks=()
 if test -f "$1"; then
-    OLDIFS=$IFS
-    IFS=','
     line_number=1
-    while read line
+
+    # Read file including the last line even when it doesn't end with newline
+    while IFS="" read -r line || [ -n "$line" ];
     do
-        if [[ $line =~ ([a-zA-Z0-9_-]{46}) ]]; then
+        if [[ $line =~ (^[a-zA-Z0-9_-]{46}$) ]]; then
-            skylinks+=("$BASH_REMATCH")
+            skylinks+=("$line")
         else
             echo "Incorrect skylink at line ${line_number}: $line" && exit 1
         fi
         let line_number+=1
     done < $1;
-    IFS=$OLDIFS
 else
     skylinks=("$1") # just single skylink passed as input argument
 fi
 
-#########################################################################
+for skylink in "${skylinks[@]}";
-# iterate through all servers, block the skylinks and purge it from cache
-#########################################################################
-declare -a servers=(  "eu-ger-1.siasky.net" "eu-ger-2.siasky.net" "eu-ger-3.siasky.net" "eu-ger-4.siasky.net" "eu-ger-5.siasky.net" "eu-ger-6.siasky.net" "eu-ger-7.siasky.net" "eu-ger-8.siasky.net"
-                      "eu-ger-9.siasky.net" "eu-ger-10.siasky.net" "eu-ger-11.siasky.net" "eu-ger-12.siasky.net"
-                      "eu-fin-1.siasky.net" "eu-fin-2.siasky.net" "eu-fin-3.siasky.net" "eu-fin-4.siasky.net" "eu-fin-5.siasky.net" "eu-fin-6.siasky.net" "eu-fin-7-siasky.net" "eu-fin-8.siasky.net"
-                      "eu-fin-9.siasky.net" "eu-fin-10.siasky.net" "eu-fin-11.siasky.net" "eu-fin-12.siasky.net" "eu-fin-13.siasky.net" "eu-fin-14.siasky.net" "eu-fin-15.siasky.net"
-                      "eu-pol-1.siasky.net" "eu-pol-2.siasky.net" "eu-pol-3.siasky.net" "eu-pol-4.siasky.net" "eu-pol-5.siasky.net"
-                      "us-ny-1.siasky.net" "us-ny-2.siasky.net"
-                      "us-or-1.siasky.net" "us-or-2.siasky.net"
-                      "us-la-1.siasky.net" "us-la-2.siasky.net" "us-la-3.siasky.net"
-                      "us-pa-1.siasky.net" "us-pa-2.siasky.net"
-                      "us-va-1.siasky.net" "us-va-2.siasky.net" "us-va-3.siasky.net" "us-va-4.siasky.net" "us-va-5.siasky.net" "us-va-6.siasky.net"
-                      "as-hk-1.siasky.net" "as-sp-1.siasky.net" "as-sp-2.siasky.net"
-                      "siasky.xyz" "dev1.siasky.dev" "dev2.siasky.dev" "dev3.siasky.dev")
-for server in "${servers[@]}";
 do
-    for skylink in "${skylinks[@]}";
+    echo ".. ⌁ Blocking skylink ${skylink}"
-    do
+    
-        echo ".. ⌁ Blocking skylink ${skylink} on ${server}"
+    # Add to Sia blocklist
-        
+    docker exec sia siac skynet blocklist add "${skylink}"
-        # Add to blocklist
+    
-        ssh -q -t user@${server} "docker exec sia siac skynet blocklist add ${skylink}"
+    # Remove from NGINX cache
-        
+    # NOTE:
-        # Remove from NGINX cache
+    # If there are changes to how the NGINX cache is being cleared, the same
-        cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'"
+    # changes need to be applied to the /setup-scripts/blocklist-airtable.py
-        ssh -q -t user@${server} "docker exec -it nginx bash -c ${cached_files_command} | xargs -r rm"
+    # script.
-        
+    cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'"
-        echo ".. ⌁ Skylink ${skylink} Blocked on ${server}"
+    docker exec -it nginx bash -c "${cached_files_command}" | xargs -r rm
-        echo "--------------------------------------------"
+    
-    done
+    echo ".. ⌁ Skylink ${skylink} Blocked"
+    echo "--------------------------------------------"
 done
 
 echo "✓ All done !"

setup-scripts/blocklist-airtable.py

@@ -141,6 +141,10 @@ async def block_skylinks_from_airtable():
         )
         return await send_msg(message, force_notify=False)
 
+    # Remove from NGINX cache
+    # NOTE:
+    # If there are changes to how the NGINX cache is being cleared, the same
+    # changes need to be applied to the /scripts/blocklist-skylink.sh script.
     print("Searching nginx cache for blocked files")
     cached_files_count = 0
     batch_size = 1000