From b4e2eec2fe78f5c66c9806d7f9d35f5fa862d5c8 Mon Sep 17 00:00:00 2001
From: Filip Rysavy <29089732+firyx@users.noreply.github.com>
Date: Mon, 13 Dec 2021 17:35:00 +0100
Subject: [PATCH] Block skylinks in batches

---
 changelog/items/other/skylinks-block-batch.md |  1 +
 scripts/blocklist-skylink.sh                  | 59 +++++++++++++------
 setup-scripts/blocklist-airtable.py           |  3 +
 3 files changed, 46 insertions(+), 17 deletions(-)
 create mode 100644 changelog/items/other/skylinks-block-batch.md

diff --git a/changelog/items/other/skylinks-block-batch.md b/changelog/items/other/skylinks-block-batch.md
new file mode 100644
index 00000000..e617082b
--- /dev/null
+++ b/changelog/items/other/skylinks-block-batch.md
@@ -0,0 +1 @@
+- Block skylinks in batches to improve performance.
\ No newline at end of file
diff --git a/scripts/blocklist-skylink.sh b/scripts/blocklist-skylink.sh
index cf14f138..93a36a61 100755
--- a/scripts/blocklist-skylink.sh
+++ b/scripts/blocklist-skylink.sh
@@ -8,6 +8,9 @@
 
 set -e # exit on first error
 
+# Number of skylinks to block within one batch
+BATCH_SIZE=1000
+
 if [ -z "$1" ]; then
     echo "Please provide either a skylink or file with skylinks separated by new lines" && exit 1
 fi
@@ -34,23 +37,45 @@ else
     skylinks=("$1") # just single skylink passed as input argument
 fi
 
-for skylink in "${skylinks[@]}";
-do
-    echo ".. ⌁ Blocking skylink ${skylink}"
-    
-    # Add to Sia blocklist
-    docker exec sia siac skynet blocklist add "${skylink}"
-    
-    # Remove from NGINX cache
-    # NOTE:
-    # If there are changes to how the NGINX cache is being cleared, the same
-    # changes need to be applied to the /setup-scripts/blocklist-airtable.py
-    # script.
-    cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'"
-    docker exec -it nginx bash -c "${cached_files_command} | xargs -r rm"
-    
-    echo ".. ⌁ Skylink ${skylink} Blocked"
-    echo "--------------------------------------------"
+# Block skylinks in batches
+skylinks_len=${#skylinks[@]}
+for (( i = 0; i < $skylinks_len; i++ )); do
+    # Add skylink to batch
+    skylink="${skylinks[$i]}"
+    echo ".. ⌁ Adding skylink ${skylink} to batch..."
+    batch_skylinks+=("$skylink")
+
+    # For performance reasons on each iteration we do not block a single
+    # skylink, but we block skylinks in batches with BATCH_SIZE size mainly
+    # because of nginx cache search.
+    # If (batch len == batch size) or (we have last batch):
+    if (( ${#batch_skylinks[@]} == $BATCH_SIZE || $i == $skylinks_len - 1 )); then
+        echo "--------------------------------------------"
+
+        # Add to Sia blocklist
+        echo "Blocking batch skylinks in skyd..."
+        skylinks_space_separated="$(IFS=' '; echo "${batch_skylinks[*]}")"
+        docker exec sia siac skynet blocklist add $skylinks_space_separated
+
+        # Remove from NGINX cache
+        # NOTE:
+        # If there are changes to how the NGINX cache is being cleared, the same
+        # changes need to be applied to the /setup-scripts/blocklist-airtable.py
+        # script.
+        echo "Removing batch skylinks from Nginx cache..."
+        skylinks_pipe_separated="$(IFS='|'; echo "${batch_skylinks[*]}")"
+        cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ($skylinks_pipe_separated)'"
+        docker exec -it nginx bash -c "${cached_files_command} | xargs -r rm"
+
+        # Clear batch
+        batch_skylinks=()
+
+        echo "--------------------------------------------"
+    fi
 done
 
+# Hot reload Nginx to get rid of deleted open files
+echo "Hot reloading nginx..."
+docker exec nginx nginx -s reload
+
 echo "✓ All done !"
diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index 31d8ee19..2443b488 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -171,6 +171,9 @@ async def block_skylinks_from_airtable():
 
     if cached_files_count == 0:
         return print("No nginx cached files matching blocked skylinks were found")
+    else:
+        print("Hot reloading nginx")
+        exec('docker exec nginx nginx -s reload')
 
     message = (
         "Purged " + str(cached_files_count) + " blocklisted files from nginx cache"