Block skylinks in batches

changelog/items/other/skylinks-block-batch.md

@@ -0,0 +1 @@
+- Block skylinks in batches to improve performance.

scripts/blocklist-skylink.sh

@@ -8,6 +8,9 @@
 
 set -e # exit on first error
 
+# Number of skylinks to block within one batch
+BATCH_SIZE=1000
+
 if [ -z "$1" ]; then
     echo "Please provide either a skylink or file with skylinks separated by new lines" && exit 1
 fi
@@ -34,23 +37,45 @@ else
     skylinks=("$1") # just single skylink passed as input argument
 fi
 
-for skylink in "${skylinks[@]}";
+# Block skylinks in batches
-do
+skylinks_len=${#skylinks[@]}
-    echo ".. ⌁ Blocking skylink ${skylink}"
+for (( i = 0; i < $skylinks_len; i++ )); do
-    
+    # Add skylink to batch
-    # Add to Sia blocklist
+    skylink="${skylinks[$i]}"
-    docker exec sia siac skynet blocklist add "${skylink}"
+    echo ".. ⌁ Adding skylink ${skylink} to batch..."
-    
+    batch_skylinks+=("$skylink")
-    # Remove from NGINX cache
+
-    # NOTE:
+    # For performance reasons on each iteration we do not block a single
-    # If there are changes to how the NGINX cache is being cleared, the same
+    # skylink, but we block skylinks in batches with BATCH_SIZE size mainly
-    # changes need to be applied to the /setup-scripts/blocklist-airtable.py
+    # because of nginx cache search.
-    # script.
+    # If (batch len == batch size) or (we have last batch):
-    cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'"
+    if (( ${#batch_skylinks[@]} == $BATCH_SIZE || $i == $skylinks_len - 1 )); then
-    docker exec -it nginx bash -c "${cached_files_command} | xargs -r rm"
+        echo "--------------------------------------------"
-    
+
-    echo ".. ⌁ Skylink ${skylink} Blocked"
+        # Add to Sia blocklist
-    echo "--------------------------------------------"
+        echo "Blocking batch skylinks in skyd..."
+        skylinks_space_separated="$(IFS=' '; echo "${batch_skylinks[*]}")"
+        docker exec sia siac skynet blocklist add $skylinks_space_separated
+
+        # Remove from NGINX cache
+        # NOTE:
+        # If there are changes to how the NGINX cache is being cleared, the same
+        # changes need to be applied to the /setup-scripts/blocklist-airtable.py
+        # script.
+        echo "Removing batch skylinks from Nginx cache..."
+        skylinks_pipe_separated="$(IFS='|'; echo "${batch_skylinks[*]}")"
+        cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ($skylinks_pipe_separated)'"
+        docker exec -it nginx bash -c "${cached_files_command} | xargs -r rm"
+
+        # Clear batch
+        batch_skylinks=()
+
+        echo "--------------------------------------------"
+    fi
 done
 
+# Hot reload Nginx to get rid of deleted open files
+echo "Hot reloading nginx..."
+docker exec nginx nginx -s reload
+
 echo "✓ All done !"

setup-scripts/blocklist-airtable.py

@@ -171,6 +171,9 @@ async def block_skylinks_from_airtable():
 
     if cached_files_count == 0:
         return print("No nginx cached files matching blocked skylinks were found")
+    else:
+        print("Hot reloading nginx")
+        exec('docker exec nginx nginx -s reload')
 
     message = (
         "Purged " + str(cached_files_count) + " blocklisted files from nginx cache"