Add purge blocklist command

scripts/purge-nginx-blocklist.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+
+# TODO:
+#
+# 1. the purging should batch the skylinks to purge in a single command
+# 
+# python example:
+#
+# cached_files_command = (
+#     "find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ("
+#     + "|".join(skylinks[i : i + batch_size])
+#     + ")'"
+# )
+#
+# cached_files_count += int(
+#     exec(
+#         'docker exec nginx bash -c "'
+#         + cached_files_command
+#         + ' | xargs -r rm -v | wc -l"'
+#     )
+# )
+
+# This script reads skylinks from a file and purges them from the Nginx cache.
+# It uses the atomic mkdir operation to create a lock on the file, under which
+# it copies the file and truncates it.
+
+set -e # exit on first error
+
+# The following variables define the paths to the file containing the skylinks
+# that need to be purged, the file in which we store the queued skylinks and the
+# lock directory that ensures the blocker API and the crontab don't manipulate
+# the same files concurrently.
+NGINX_PURGE_SKYLINKS_FILE="/home/user/skynet-webportal/docker/data/nginx/blocker/skylinks.txt"
+NGINX_PURGE_SKYLINKS_QUEUED="/home/user/skynet-webportal/docker/data/nginx/blocker/queued.txt"
+NGINX_PURGE_SKYLINKS_LOCK="/home/user/skynet-webportal/docker/data/nginx/blocker/lock"
+
+purge_skylinks () {
+    # read all skylinks from the queued skylinks file
+    skylinks=()
+    line_number=1
+    while IFS="" read -r line || [ -n "$line" ];
+    do
+        if [[ $line =~ (^[a-zA-Z0-9_-]{46}$) ]]; then
+            skylinks+=("$line")
+        else
+            echo "Incorrect skylink at line ${line_number}: $line"
+        fi
+        let line_number+=1
+    done < $NGINX_PURGE_SKYLINKS_QUEUED;
+
+    for skylink in "${skylinks[@]}";
+    do
+        echo ".. ⌁ Purging skylink ${skylink}"
+        cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'"
+        docker exec -it nginx bash -c "${cached_files_command} | xargs -r rm"
+        
+        echo ".. ⌁ Skylink ${skylink} purged"
+        echo "--------------------------------------------"
+    done
+
+    # remove the queue file
+    rm $NGINX_PURGE_SKYLINKS_QUEUED
+}
+
+acquire_lock () {
+    attempts=0
+    locked=false
+    until [ "$attempts" -ge 10 ]
+    do 
+        if ! mkdir $NGINX_PURGE_SKYLINKS_LOCK 2>/dev/null
+        then
+            echo "skylinks file is locked, waiting..." + $attempts
+            ((attempts++))
+            sleep 1;
+        else
+            locked=true
+            break
+        fi
+    done
+
+    if ! $locked
+    then
+        echo "failed to acquire lock, warrants investigation"
+        exit 1
+    fi
+}
+
+release_lock () {
+    rmdir $NGINX_PURGE_SKYLINKS_LOCK
+}
+
+# if there is a queue file - purge all skylinks in that file from nginx cache
+if [ -f "$NGINX_PURGE_SKYLINKS_QUEUED" ]
+then
+    echo "found queue file, purging skylinks from file"
+    purge_skylinks
+    echo "✓ Done"
+    exit 1
+fi
+
+# if there is no skylinks file - escape early
+if [ ! -f "$NGINX_PURGE_SKYLINKS_FILE" ]
+then
+    echo "no skylinks found"
+    echo "✓ Done"
+    exit 1
+fi
+
+# move the skylinks file to the queue under lock
+acquire_lock
+mv $NGINX_PURGE_SKYLINKS_FILE $NGINX_PURGE_SKYLINKS_QUEUED
+release_lock
+
+# purge the skylinks from the queue file
+purge_skylinks
+echo "✓ Done"
+exit 1