Merge pull request #1533 from SkynetLabs/get-rid-of-cache-purging-code
get rid of unused cache purgin code
This commit is contained in:
Karol Wypchło
GitHub
commit
a86cb6e5b3
|
|
@ -3,12 +3,6 @@ include /etc/nginx/conf.d/include/proxy-buffer;
|
|||
include /etc/nginx/conf.d/include/proxy-cache-downloads;
|
||||
include /etc/nginx/conf.d/include/track-download;
|
||||
|
||||
# redirect purge calls to separate location
|
||||
error_page 462 = @purge;
|
||||
if ($request_method = PURGE) {
|
||||
return 462;
|
||||
}
|
||||
|
||||
limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time
|
||||
|
||||
# ensure that skylink that we pass around is base64 encoded (transform base32 encoded ones)
|
||||
|
|
|
|||
|
|
@ -1,68 +0,0 @@
|
|||
-- Tit Petric, Monotek d.o.o., Tue 03 Jan 2017 06:54:56 PM CET
|
||||
--
|
||||
-- Delete nginx cached assets with a PURGE request against an endpoint
|
||||
-- supports extended regular expression PURGE requests (/upload/.*)
|
||||
--
|
||||
-- https://scene-si.org/2017/01/08/improving-nginx-lua-cache-purge/
|
||||
--
|
||||
|
||||
function file_exists(name)
|
||||
local f = io.open(name, "r")
|
||||
if f~=nil then io.close(f) return true else return false end
|
||||
end
|
||||
|
||||
function explode(d, p)
|
||||
local t, ll
|
||||
t={}
|
||||
ll=0
|
||||
if(#p == 1) then return {p} end
|
||||
while true do
|
||||
l=string.find(p, d, ll, true) -- find the next d in the string
|
||||
if l~=nil then -- if "not not" found then..
|
||||
table.insert(t, string.sub(p, ll, l-1)) -- Save it in our array.
|
||||
ll=l+1 -- save just after where we found it for searching next time.
|
||||
else
|
||||
table.insert(t, string.sub(p, ll)) -- Save what's left in our array.
|
||||
break -- Break at end, as it should be, according to the lua manual.
|
||||
end
|
||||
end
|
||||
return t
|
||||
end
|
||||
|
||||
function purge(filename)
|
||||
if (file_exists(filename)) then
|
||||
os.remove(filename)
|
||||
end
|
||||
end
|
||||
|
||||
function trim(s)
|
||||
return (string.gsub(s, "^%s*(.-)%s*$", "%1"))
|
||||
end
|
||||
|
||||
function exec(cmd)
|
||||
local handle = io.popen(cmd)
|
||||
local result = handle:read("*all")
|
||||
handle:close()
|
||||
return trim(result)
|
||||
end
|
||||
|
||||
function list_files(cache_path, purge_pattern)
|
||||
local result = exec("/usr/bin/find " .. cache_path .. " -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -El -m 1 '^KEY: " .. purge_pattern .. "' 2>&1")
|
||||
if result == "" then
|
||||
return {}
|
||||
end
|
||||
return explode("\n", result)
|
||||
end
|
||||
|
||||
if ngx ~= nil then
|
||||
-- list all cached items matching uri
|
||||
local files = list_files(ngx.var.lua_purge_path, ngx.var.uri)
|
||||
|
||||
ngx.header["Content-type"] = "text/plain; charset=utf-8"
|
||||
ngx.header["X-Purged-Count"] = table.getn(files)
|
||||
for k, v in pairs(files) do
|
||||
purge(v)
|
||||
end
|
||||
ngx.say("OK")
|
||||
ngx.exit(ngx.OK)
|
||||
end
|
||||
|
|
@ -357,14 +357,6 @@ location ~ "^/file/(([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?)$" {
|
|||
include /etc/nginx/conf.d/include/location-skylink;
|
||||
}
|
||||
|
||||
location @purge {
|
||||
# secure traffic by limiting to only local networks
|
||||
include /etc/nginx/conf.d/include/local-network-only;
|
||||
|
||||
set $lua_purge_path "/data/nginx/cache/";
|
||||
content_by_lua_file /etc/nginx/conf.d/scripts/purge-multi.lua;
|
||||
}
|
||||
|
||||
location /__internal/do/not/use/authenticated {
|
||||
include /etc/nginx/conf.d/include/cors;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,118 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# TODO:
|
||||
#
|
||||
# 1. the purging should batch the skylinks to purge in a single command
|
||||
#
|
||||
# python example:
|
||||
#
|
||||
# cached_files_command = (
|
||||
# "find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ("
|
||||
# + "|".join(skylinks[i : i + batch_size])
|
||||
# + ")'"
|
||||
# )
|
||||
#
|
||||
# cached_files_count += int(
|
||||
# exec(
|
||||
# 'docker exec nginx bash -c "'
|
||||
# + cached_files_command
|
||||
# + ' | xargs -r rm -v | wc -l"'
|
||||
# )
|
||||
# )
|
||||
|
||||
# This script reads skylinks from a file and purges them from the Nginx cache.
|
||||
# It uses the atomic mkdir operation to create a lock on the file, under which
|
||||
# it copies the file and truncates it.
|
||||
|
||||
set -e # exit on first error
|
||||
|
||||
# The following variables define the paths to the file containing the skylinks
|
||||
# that need to be purged, the file in which we store the queued skylinks and the
|
||||
# lock directory that ensures the blocker API and the crontab don't manipulate
|
||||
# the same files concurrently.
|
||||
NGINX_PURGE_SKYLINKS_FILE="/data/nginx/blocker/skylinks.txt"
|
||||
NGINX_PURGE_SKYLINKS_QUEUED="/data/nginx/blocker/queued.txt"
|
||||
NGINX_PURGE_SKYLINKS_LOCK="/data/nginx/blocker/lock"
|
||||
NGINX_CACHE_DIR="/data/nginx/cache/"
|
||||
|
||||
purge_skylinks () {
|
||||
# read all skylinks from the queued skylinks file
|
||||
skylinks=()
|
||||
line_number=1
|
||||
while IFS="" read -r line || [ -n "$line" ];
|
||||
do
|
||||
if [[ $line =~ (^[a-zA-Z0-9_-]{46}$) ]]; then
|
||||
skylinks+=("$line")
|
||||
else
|
||||
echo "Incorrect skylink at line ${line_number}: $line"
|
||||
fi
|
||||
let line_number+=1
|
||||
done < $NGINX_PURGE_SKYLINKS_QUEUED;
|
||||
|
||||
for skylink in "${skylinks[@]}";
|
||||
do
|
||||
echo ".. ⌁ Purging skylink ${skylink}"
|
||||
cached_files_command="find ${NGINX_CACHE_DIR} -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'"
|
||||
bash -c "${cached_files_command} | xargs -r rm"
|
||||
|
||||
echo ".. ⌁ Skylink ${skylink} purged"
|
||||
echo "--------------------------------------------"
|
||||
done
|
||||
|
||||
# remove the queue file
|
||||
rm $NGINX_PURGE_SKYLINKS_QUEUED
|
||||
}
|
||||
|
||||
acquire_lock () {
|
||||
attempts=0
|
||||
locked=false
|
||||
until [ "$attempts" -ge 10 ]
|
||||
do
|
||||
if ! mkdir $NGINX_PURGE_SKYLINKS_LOCK 2>/dev/null
|
||||
then
|
||||
echo "skylinks file is locked, waiting..."
|
||||
((attempts++))
|
||||
sleep 1;
|
||||
else
|
||||
locked=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if ! $locked
|
||||
then
|
||||
echo "failed to acquire lock, warrants investigation"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
release_lock () {
|
||||
rmdir $NGINX_PURGE_SKYLINKS_LOCK
|
||||
}
|
||||
|
||||
# if there is a queue file - purge all skylinks in that file from nginx cache
|
||||
if [ -f "$NGINX_PURGE_SKYLINKS_QUEUED" ]
|
||||
then
|
||||
echo "found queue file, purging skylinks from file"
|
||||
purge_skylinks
|
||||
echo "✓ Done"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# if there is no skylinks file - escape early
|
||||
if [ ! -f "$NGINX_PURGE_SKYLINKS_FILE" ]
|
||||
then
|
||||
echo "no skylinks found"
|
||||
echo "✓ Done"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# move the skylinks file to the queue under lock
|
||||
acquire_lock
|
||||
mv $NGINX_PURGE_SKYLINKS_FILE $NGINX_PURGE_SKYLINKS_QUEUED
|
||||
release_lock
|
||||
|
||||
# purge the skylinks from the queue file
|
||||
purge_skylinks
|
||||
echo "✓ Done"
|
||||
exit 1
|
||||
|
|
@ -5,4 +5,3 @@
|
|||
0 4 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1 >> /home/user/skynet-webportal/logs/db_backup_`date +"%Y-%m-%d-%H%M"`.log 2 > &1
|
||||
0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200
|
||||
15 * * * * /home/user/skynet-webportal/scripts/nginx-prune.sh
|
||||
*/30 * * * * docker exec nginx /etc/nginx/scripts/purge-blocklist.sh
|
||||
|
|
|
|||
Reference in New Issue