117 lines
3.2 KiB
Bash
117 lines
3.2 KiB
Bash
|
#!/bin/bash
|
||
|
|
|
||
|
|
# TODO:
|
||
|
|
#
|
||
|
|
# 1. the purging should batch the skylinks to purge in a single command
|
||
|
|
#
|
||
|
|
# python example:
|
||
|
|
#
|
||
|
|
# cached_files_command = (
|
||
|
|
# "find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ("
|
||
|
|
# + "|".join(skylinks[i : i + batch_size])
|
||
|
|
# + ")'"
|
||
|
|
# )
|
||
|
|
#
|
||
|
|
# cached_files_count += int(
|
||
|
|
# exec(
|
||
|
|
# 'docker exec nginx bash -c "'
|
||
|
|
# + cached_files_command
|
||
|
|
# + ' | xargs -r rm -v | wc -l"'
|
||
|
|
# )
|
||
|
|
# )
|
||
|
|
|
||
|
|
# This script reads skylinks from a file and purges them from the Nginx cache.
|
||
|
|
# It uses the atomic mkdir operation to create a lock on the file, under which
|
||
|
|
# it copies the file and truncates it.
|
||
|
|
|
||
|
|
set -e # exit on first error
|
||
|
|
|
||
|
|
# The following variables define the paths to the file containing the skylinks
|
||
|
|
# that need to be purged, the file in which we store the queued skylinks and the
|
||
|
|
# lock directory that ensures the blocker API and the crontab don't manipulate
|
||
|
|
# the same files concurrently.
|
||
|
|
NGINX_PURGE_SKYLINKS_FILE="/home/user/skynet-webportal/docker/data/nginx/blocker/skylinks.txt"
|
||
|
|
NGINX_PURGE_SKYLINKS_QUEUED="/home/user/skynet-webportal/docker/data/nginx/blocker/queued.txt"
|
||
|
|
NGINX_PURGE_SKYLINKS_LOCK="/home/user/skynet-webportal/docker/data/nginx/blocker/lock"
|
||
|
|
|
||
|
|
purge_skylinks () {
|
||
|
|
# read all skylinks from the queued skylinks file
|
||
|
|
skylinks=()
|
||
|
|
line_number=1
|
||
|
|
while IFS="" read -r line || [ -n "$line" ];
|
||
|
|
do
|
||
|
|
if [[ $line =~ (^[a-zA-Z0-9_-]{46}$) ]]; then
|
||
|
|
skylinks+=("$line")
|
||
|
|
else
|
||
|
|
echo "Incorrect skylink at line ${line_number}: $line"
|
||
|
|
fi
|
||
|
|
let line_number+=1
|
||
|
|
done < $NGINX_PURGE_SKYLINKS_QUEUED;
|
||
|
|
|
||
|
|
for skylink in "${skylinks[@]}";
|
||
|
|
do
|
||
|
|
echo ".. ⌁ Purging skylink ${skylink}"
|
||
|
|
cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'"
|
||
|
|
docker exec -it nginx bash -c "${cached_files_command} | xargs -r rm"
|
||
|
|
|
||
|
|
echo ".. ⌁ Skylink ${skylink} purged"
|
||
|
|
echo "--------------------------------------------"
|
||
|
|
done
|
||
|
|
|
||
|
|
# remove the queue file
|
||
|
|
rm $NGINX_PURGE_SKYLINKS_QUEUED
|
||
|
|
}
|
||
|
|
|
||
|
|
acquire_lock () {
|
||
|
|
attempts=0
|
||
|
|
locked=false
|
||
|
|
until [ "$attempts" -ge 10 ]
|
||
|
|
do
|
||
|
|
if ! mkdir $NGINX_PURGE_SKYLINKS_LOCK 2>/dev/null
|
||
|
|
then
|
||
|
|
echo "skylinks file is locked, waiting..." + $attempts
|
||
|
|
((attempts++))
|
||
|
|
sleep 1;
|
||
|
|
else
|
||
|
|
locked=true
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
|
||
|
|
if ! $locked
|
||
|
|
then
|
||
|
|
echo "failed to acquire lock, warrants investigation"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
release_lock () {
|
||
|
|
rmdir $NGINX_PURGE_SKYLINKS_LOCK
|
||
|
|
}
|
||
|
|
|
||
|
|
# if there is a queue file - purge all skylinks in that file from nginx cache
|
||
|
|
if [ -f "$NGINX_PURGE_SKYLINKS_QUEUED" ]
|
||
|
|
then
|
||
|
|
echo "found queue file, purging skylinks from file"
|
||
|
|
purge_skylinks
|
||
|
|
echo "✓ Done"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
# if there is no skylinks file - escape early
|
||
|
|
if [ ! -f "$NGINX_PURGE_SKYLINKS_FILE" ]
|
||
|
|
then
|
||
|
|
echo "no skylinks found"
|
||
|
|
echo "✓ Done"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
# move the skylinks file to the queue under lock
|
||
|
|
acquire_lock
|
||
|
|
mv $NGINX_PURGE_SKYLINKS_FILE $NGINX_PURGE_SKYLINKS_QUEUED
|
||
|
|
release_lock
|
||
|
|
|
||
|
|
# purge the skylinks from the queue file
|
||
|
|
purge_skylinks
|
||
|
|
echo "✓ Done"
|
||
|
|
exit 1
|