Merge pull request #1441 from SkynetLabs/pj/nginx-purge

feat: Nginx Purge

docker-compose.blocker.yml

@@ -16,6 +16,8 @@ services:
     logging: *default-logging
     env_file:
       - .env
+    volumes:
+      - ./docker/data/nginx/blocker:/data/nginx/blocker
     expose:
       - 4000
     networks:

docker-compose.yml

@@ -67,6 +67,7 @@ services:
     volumes:
       - ./docker/nginx/nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf:ro
       - ./docker/data/nginx/cache:/data/nginx/cache
+      - ./docker/data/nginx/blocker:/data/nginx/blocker
       - ./docker/data/nginx/logs:/usr/local/openresty/nginx/logs
       - ./docker/data/nginx/skynet:/data/nginx/skynet:ro
       - ./docker/data/sia/apipassword:/data/sia/apipassword:ro

docker/nginx/Dockerfile

@@ -10,6 +10,7 @@ COPY mo ./
 COPY libs /etc/nginx/libs
 COPY conf.d /etc/nginx/conf.d
 COPY conf.d.templates /etc/nginx/conf.d.templates
+COPY scripts /etc/nginx/scripts
 
 CMD [ "bash", "-c", \
       "./mo < /etc/nginx/conf.d.templates/server.account.conf > /etc/nginx/conf.d/server.account.conf ; \

docker/nginx/scripts/purge-blocklist.sh

@@ -0,0 +1,118 @@
+#!/bin/bash
+
+# TODO:
+#
+# 1. the purging should batch the skylinks to purge in a single command
+# 
+# python example:
+#
+# cached_files_command = (
+#     "find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ("
+#     + "|".join(skylinks[i : i + batch_size])
+#     + ")'"
+# )
+#
+# cached_files_count += int(
+#     exec(
+#         'docker exec nginx bash -c "'
+#         + cached_files_command
+#         + ' | xargs -r rm -v | wc -l"'
+#     )
+# )
+
+# This script reads skylinks from a file and purges them from the Nginx cache.
+# It uses the atomic mkdir operation to create a lock on the file, under which
+# it copies the file and truncates it.
+
+set -e # exit on first error
+
+# The following variables define the paths to the file containing the skylinks
+# that need to be purged, the file in which we store the queued skylinks and the
+# lock directory that ensures the blocker API and the crontab don't manipulate
+# the same files concurrently.
+NGINX_PURGE_SKYLINKS_FILE="/data/nginx/blocker/skylinks.txt"
+NGINX_PURGE_SKYLINKS_QUEUED="/data/nginx/blocker/queued.txt"
+NGINX_PURGE_SKYLINKS_LOCK="/data/nginx/blocker/lock"
+NGINX_CACHE_DIR="/data/nginx/cache/"
+
+purge_skylinks () {
+    # read all skylinks from the queued skylinks file
+    skylinks=()
+    line_number=1
+    while IFS="" read -r line || [ -n "$line" ];
+    do
+        if [[ $line =~ (^[a-zA-Z0-9_-]{46}$) ]]; then
+            skylinks+=("$line")
+        else
+            echo "Incorrect skylink at line ${line_number}: $line"
+        fi
+        let line_number+=1
+    done < $NGINX_PURGE_SKYLINKS_QUEUED;
+
+    for skylink in "${skylinks[@]}";
+    do
+        echo ".. ⌁ Purging skylink ${skylink}"
+        cached_files_command="find ${NGINX_CACHE_DIR} -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'"
+        bash -c "${cached_files_command} | xargs -r rm"
+        
+        echo ".. ⌁ Skylink ${skylink} purged"
+        echo "--------------------------------------------"
+    done
+
+    # remove the queue file
+    rm $NGINX_PURGE_SKYLINKS_QUEUED
+}
+
+acquire_lock () {
+    attempts=0
+    locked=false
+    until [ "$attempts" -ge 10 ]
+    do 
+        if ! mkdir $NGINX_PURGE_SKYLINKS_LOCK 2>/dev/null
+        then
+            echo "skylinks file is locked, waiting..."
+            ((attempts++))
+            sleep 1;
+        else
+            locked=true
+            break
+        fi
+    done
+
+    if ! $locked
+    then
+        echo "failed to acquire lock, warrants investigation"
+        exit 1
+    fi
+}
+
+release_lock () {
+    rmdir $NGINX_PURGE_SKYLINKS_LOCK
+}
+
+# if there is a queue file - purge all skylinks in that file from nginx cache
+if [ -f "$NGINX_PURGE_SKYLINKS_QUEUED" ]
+then
+    echo "found queue file, purging skylinks from file"
+    purge_skylinks
+    echo "✓ Done"
+    exit 1
+fi
+
+# if there is no skylinks file - escape early
+if [ ! -f "$NGINX_PURGE_SKYLINKS_FILE" ]
+then
+    echo "no skylinks found"
+    echo "✓ Done"
+    exit 1
+fi
+
+# move the skylinks file to the queue under lock
+acquire_lock
+mv $NGINX_PURGE_SKYLINKS_FILE $NGINX_PURGE_SKYLINKS_QUEUED
+release_lock
+
+# purge the skylinks from the queue file
+purge_skylinks
+echo "✓ Done"
+exit 1

setup-scripts/support/crontab

@@ -5,3 +5,4 @@
 0 4 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1 >> /home/user/skynet-webportal/logs/db_backup_`date +"%Y-%m-%d-%H%M"`.log 2 > &1
 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200
 15 * * * * /home/user/skynet-webportal/scripts/nginx-prune.sh
+*/30 * * * * docker exec nginx /etc/nginx/scripts/purge-blocklist.sh