diff --git a/docker/nginx/conf.d/include/location-skylink b/docker/nginx/conf.d/include/location-skylink index 895fd55f..1a80f41f 100644 --- a/docker/nginx/conf.d/include/location-skylink +++ b/docker/nginx/conf.d/include/location-skylink @@ -3,12 +3,6 @@ include /etc/nginx/conf.d/include/proxy-buffer; include /etc/nginx/conf.d/include/proxy-cache-downloads; include /etc/nginx/conf.d/include/track-download; -# redirect purge calls to separate location -error_page 462 = @purge; -if ($request_method = PURGE) { - return 462; -} - limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time # ensure that skylink that we pass around is base64 encoded (transform base32 encoded ones) diff --git a/docker/nginx/conf.d/scripts/purge-multi.lua b/docker/nginx/conf.d/scripts/purge-multi.lua deleted file mode 100644 index 95a5b90f..00000000 --- a/docker/nginx/conf.d/scripts/purge-multi.lua +++ /dev/null @@ -1,68 +0,0 @@ --- Tit Petric, Monotek d.o.o., Tue 03 Jan 2017 06:54:56 PM CET --- --- Delete nginx cached assets with a PURGE request against an endpoint --- supports extended regular expression PURGE requests (/upload/.*) --- --- https://scene-si.org/2017/01/08/improving-nginx-lua-cache-purge/ --- - -function file_exists(name) - local f = io.open(name, "r") - if f~=nil then io.close(f) return true else return false end -end - -function explode(d, p) - local t, ll - t={} - ll=0 - if(#p == 1) then return {p} end - while true do - l=string.find(p, d, ll, true) -- find the next d in the string - if l~=nil then -- if "not not" found then.. - table.insert(t, string.sub(p, ll, l-1)) -- Save it in our array. - ll=l+1 -- save just after where we found it for searching next time. - else - table.insert(t, string.sub(p, ll)) -- Save what's left in our array. - break -- Break at end, as it should be, according to the lua manual. - end - end - return t -end - -function purge(filename) - if (file_exists(filename)) then - os.remove(filename) - end -end - -function trim(s) - return (string.gsub(s, "^%s*(.-)%s*$", "%1")) -end - -function exec(cmd) - local handle = io.popen(cmd) - local result = handle:read("*all") - handle:close() - return trim(result) -end - -function list_files(cache_path, purge_pattern) - local result = exec("/usr/bin/find " .. cache_path .. " -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -El -m 1 '^KEY: " .. purge_pattern .. "' 2>&1") - if result == "" then - return {} - end - return explode("\n", result) -end - -if ngx ~= nil then - -- list all cached items matching uri - local files = list_files(ngx.var.lua_purge_path, ngx.var.uri) - - ngx.header["Content-type"] = "text/plain; charset=utf-8" - ngx.header["X-Purged-Count"] = table.getn(files) - for k, v in pairs(files) do - purge(v) - end - ngx.say("OK") - ngx.exit(ngx.OK) -end diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index 0057bad9..f217df05 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -321,14 +321,6 @@ location ~ "^/file/(([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?)$" { include /etc/nginx/conf.d/include/location-skylink; } -location @purge { - # secure traffic by limiting to only local networks - include /etc/nginx/conf.d/include/local-network-only; - - set $lua_purge_path "/data/nginx/cache/"; - content_by_lua_file /etc/nginx/conf.d/scripts/purge-multi.lua; -} - location /__internal/do/not/use/authenticated { include /etc/nginx/conf.d/include/cors; diff --git a/docker/nginx/scripts/purge-blocklist.sh b/docker/nginx/scripts/purge-blocklist.sh deleted file mode 100755 index 228dae1a..00000000 --- a/docker/nginx/scripts/purge-blocklist.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/bin/bash - -# TODO: -# -# 1. the purging should batch the skylinks to purge in a single command -# -# python example: -# -# cached_files_command = ( -# "find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: (" -# + "|".join(skylinks[i : i + batch_size]) -# + ")'" -# ) -# -# cached_files_count += int( -# exec( -# 'docker exec nginx bash -c "' -# + cached_files_command -# + ' | xargs -r rm -v | wc -l"' -# ) -# ) - -# This script reads skylinks from a file and purges them from the Nginx cache. -# It uses the atomic mkdir operation to create a lock on the file, under which -# it copies the file and truncates it. - -set -e # exit on first error - -# The following variables define the paths to the file containing the skylinks -# that need to be purged, the file in which we store the queued skylinks and the -# lock directory that ensures the blocker API and the crontab don't manipulate -# the same files concurrently. -NGINX_PURGE_SKYLINKS_FILE="/data/nginx/blocker/skylinks.txt" -NGINX_PURGE_SKYLINKS_QUEUED="/data/nginx/blocker/queued.txt" -NGINX_PURGE_SKYLINKS_LOCK="/data/nginx/blocker/lock" -NGINX_CACHE_DIR="/data/nginx/cache/" - -purge_skylinks () { - # read all skylinks from the queued skylinks file - skylinks=() - line_number=1 - while IFS="" read -r line || [ -n "$line" ]; - do - if [[ $line =~ (^[a-zA-Z0-9_-]{46}$) ]]; then - skylinks+=("$line") - else - echo "Incorrect skylink at line ${line_number}: $line" - fi - let line_number+=1 - done < $NGINX_PURGE_SKYLINKS_QUEUED; - - for skylink in "${skylinks[@]}"; - do - echo ".. ⌁ Purging skylink ${skylink}" - cached_files_command="find ${NGINX_CACHE_DIR} -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'" - bash -c "${cached_files_command} | xargs -r rm" - - echo ".. ⌁ Skylink ${skylink} purged" - echo "--------------------------------------------" - done - - # remove the queue file - rm $NGINX_PURGE_SKYLINKS_QUEUED -} - -acquire_lock () { - attempts=0 - locked=false - until [ "$attempts" -ge 10 ] - do - if ! mkdir $NGINX_PURGE_SKYLINKS_LOCK 2>/dev/null - then - echo "skylinks file is locked, waiting..." - ((attempts++)) - sleep 1; - else - locked=true - break - fi - done - - if ! $locked - then - echo "failed to acquire lock, warrants investigation" - exit 1 - fi -} - -release_lock () { - rmdir $NGINX_PURGE_SKYLINKS_LOCK -} - -# if there is a queue file - purge all skylinks in that file from nginx cache -if [ -f "$NGINX_PURGE_SKYLINKS_QUEUED" ] -then - echo "found queue file, purging skylinks from file" - purge_skylinks - echo "✓ Done" - exit 1 -fi - -# if there is no skylinks file - escape early -if [ ! -f "$NGINX_PURGE_SKYLINKS_FILE" ] -then - echo "no skylinks found" - echo "✓ Done" - exit 1 -fi - -# move the skylinks file to the queue under lock -acquire_lock -mv $NGINX_PURGE_SKYLINKS_FILE $NGINX_PURGE_SKYLINKS_QUEUED -release_lock - -# purge the skylinks from the queue file -purge_skylinks -echo "✓ Done" -exit 1 diff --git a/setup-scripts/support/crontab b/setup-scripts/support/crontab index 4467733b..29c8ec1a 100644 --- a/setup-scripts/support/crontab +++ b/setup-scripts/support/crontab @@ -5,4 +5,3 @@ 0 4 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1 >> /home/user/skynet-webportal/logs/db_backup_`date +"%Y-%m-%d-%H%M"`.log 2 > &1 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200 15 * * * * /home/user/skynet-webportal/scripts/nginx-prune.sh -*/30 * * * * docker exec nginx /etc/nginx/scripts/purge-blocklist.sh