Merge pull request #1533 from SkynetLabs/get-rid-of-cache-purging-code

get rid of unused cache purgin code

docker/nginx/conf.d/include/location-skylink

@@ -3,12 +3,6 @@ include /etc/nginx/conf.d/include/proxy-buffer;
 include /etc/nginx/conf.d/include/proxy-cache-downloads;
 include /etc/nginx/conf.d/include/track-download;
 
-# redirect purge calls to separate location
-error_page 462 = @purge;
-if ($request_method = PURGE) {
-    return 462;
-}
-
 limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time
 
 # ensure that skylink that we pass around is base64 encoded (transform base32 encoded ones)

docker/nginx/conf.d/scripts/purge-multi.lua

@@ -1,68 +0,0 @@
--- Tit Petric, Monotek d.o.o., Tue 03 Jan 2017 06:54:56 PM CET
---
--- Delete nginx cached assets with a PURGE request against an endpoint
--- supports extended regular expression PURGE requests (/upload/.*)
---
--- https://scene-si.org/2017/01/08/improving-nginx-lua-cache-purge/
---
-
-function file_exists(name)
-    local f = io.open(name, "r")
-    if f~=nil then io.close(f) return true else return false end
-end
-
-function explode(d, p)
-    local t, ll
-    t={}
-    ll=0
-    if(#p == 1) then return {p} end
-            while true do
-                    l=string.find(p, d, ll, true) -- find the next d in the string
-                    if l~=nil then -- if "not not" found then..
-                            table.insert(t, string.sub(p, ll, l-1)) -- Save it in our array.
-                            ll=l+1 -- save just after where we found it for searching next time.
-                    else
-                            table.insert(t, string.sub(p, ll)) -- Save what's left in our array.
-                            break -- Break at end, as it should be, according to the lua manual.
-                    end
-            end
-    return t
-end
-
-function purge(filename)
-    if (file_exists(filename)) then
-            os.remove(filename)
-    end
-end
-
-function trim(s)
-    return (string.gsub(s, "^%s*(.-)%s*$", "%1"))
-end
-
-function exec(cmd)
-    local handle = io.popen(cmd)
-    local result = handle:read("*all")
-    handle:close()
-    return trim(result)
-end
-
-function list_files(cache_path, purge_pattern)
-    local result = exec("/usr/bin/find " .. cache_path .. " -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -El -m 1 '^KEY: " .. purge_pattern .. "' 2>&1")
-    if result == "" then
-            return {}
-    end
-    return explode("\n", result)
-end
-
-if ngx ~= nil then
-    -- list all cached items matching uri
-    local files = list_files(ngx.var.lua_purge_path, ngx.var.uri)
-
-    ngx.header["Content-type"] = "text/plain; charset=utf-8"
-    ngx.header["X-Purged-Count"] = table.getn(files)
-    for k, v in pairs(files) do
-            purge(v)
-    end
-    ngx.say("OK")
-    ngx.exit(ngx.OK)
-end

docker/nginx/conf.d/server/server.api

@@ -357,14 +357,6 @@ location ~ "^/file/(([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?)$" {
     include /etc/nginx/conf.d/include/location-skylink;
 }
 
-location @purge {
-    # secure traffic by limiting to only local networks
-    include /etc/nginx/conf.d/include/local-network-only;
-
-    set $lua_purge_path "/data/nginx/cache/";
-    content_by_lua_file /etc/nginx/conf.d/scripts/purge-multi.lua;
-}
-
 location /__internal/do/not/use/authenticated {
     include /etc/nginx/conf.d/include/cors;
 

docker/nginx/scripts/purge-blocklist.sh

@@ -1,118 +0,0 @@
-#!/bin/bash
-
-# TODO:
-#
-# 1. the purging should batch the skylinks to purge in a single command
-# 
-# python example:
-#
-# cached_files_command = (
-#     "find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ("
-#     + "|".join(skylinks[i : i + batch_size])
-#     + ")'"
-# )
-#
-# cached_files_count += int(
-#     exec(
-#         'docker exec nginx bash -c "'
-#         + cached_files_command
-#         + ' | xargs -r rm -v | wc -l"'
-#     )
-# )
-
-# This script reads skylinks from a file and purges them from the Nginx cache.
-# It uses the atomic mkdir operation to create a lock on the file, under which
-# it copies the file and truncates it.
-
-set -e # exit on first error
-
-# The following variables define the paths to the file containing the skylinks
-# that need to be purged, the file in which we store the queued skylinks and the
-# lock directory that ensures the blocker API and the crontab don't manipulate
-# the same files concurrently.
-NGINX_PURGE_SKYLINKS_FILE="/data/nginx/blocker/skylinks.txt"
-NGINX_PURGE_SKYLINKS_QUEUED="/data/nginx/blocker/queued.txt"
-NGINX_PURGE_SKYLINKS_LOCK="/data/nginx/blocker/lock"
-NGINX_CACHE_DIR="/data/nginx/cache/"
-
-purge_skylinks () {
-    # read all skylinks from the queued skylinks file
-    skylinks=()
-    line_number=1
-    while IFS="" read -r line || [ -n "$line" ];
-    do
-        if [[ $line =~ (^[a-zA-Z0-9_-]{46}$) ]]; then
-            skylinks+=("$line")
-        else
-            echo "Incorrect skylink at line ${line_number}: $line"
-        fi
-        let line_number+=1
-    done < $NGINX_PURGE_SKYLINKS_QUEUED;
-
-    for skylink in "${skylinks[@]}";
-    do
-        echo ".. ⌁ Purging skylink ${skylink}"
-        cached_files_command="find ${NGINX_CACHE_DIR} -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'"
-        bash -c "${cached_files_command} | xargs -r rm"
-        
-        echo ".. ⌁ Skylink ${skylink} purged"
-        echo "--------------------------------------------"
-    done
-
-    # remove the queue file
-    rm $NGINX_PURGE_SKYLINKS_QUEUED
-}
-
-acquire_lock () {
-    attempts=0
-    locked=false
-    until [ "$attempts" -ge 10 ]
-    do 
-        if ! mkdir $NGINX_PURGE_SKYLINKS_LOCK 2>/dev/null
-        then
-            echo "skylinks file is locked, waiting..."
-            ((attempts++))
-            sleep 1;
-        else
-            locked=true
-            break
-        fi
-    done
-
-    if ! $locked
-    then
-        echo "failed to acquire lock, warrants investigation"
-        exit 1
-    fi
-}
-
-release_lock () {
-    rmdir $NGINX_PURGE_SKYLINKS_LOCK
-}
-
-# if there is a queue file - purge all skylinks in that file from nginx cache
-if [ -f "$NGINX_PURGE_SKYLINKS_QUEUED" ]
-then
-    echo "found queue file, purging skylinks from file"
-    purge_skylinks
-    echo "✓ Done"
-    exit 1
-fi
-
-# if there is no skylinks file - escape early
-if [ ! -f "$NGINX_PURGE_SKYLINKS_FILE" ]
-then
-    echo "no skylinks found"
-    echo "✓ Done"
-    exit 1
-fi
-
-# move the skylinks file to the queue under lock
-acquire_lock
-mv $NGINX_PURGE_SKYLINKS_FILE $NGINX_PURGE_SKYLINKS_QUEUED
-release_lock
-
-# purge the skylinks from the queue file
-purge_skylinks
-echo "✓ Done"
-exit 1

setup-scripts/support/crontab

@@ -5,4 +5,3 @@
 0 4 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1 >> /home/user/skynet-webportal/logs/db_backup_`date +"%Y-%m-%d-%H%M"`.log 2 > &1
 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200
 15 * * * * /home/user/skynet-webportal/scripts/nginx-prune.sh
-*/30 * * * * docker exec nginx /etc/nginx/scripts/purge-blocklist.sh