diff --git a/docker-compose.blocker.yml b/docker-compose.blocker.yml index b76d2e43..56d23646 100644 --- a/docker-compose.blocker.yml +++ b/docker-compose.blocker.yml @@ -16,6 +16,8 @@ services: logging: *default-logging env_file: - .env + volumes: + - ./docker/data/nginx/blocker:/data/nginx/blocker expose: - 4000 networks: diff --git a/docker-compose.yml b/docker-compose.yml index ca4dc51a..e88b2e22 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -67,6 +67,7 @@ services: volumes: - ./docker/nginx/nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf:ro - ./docker/data/nginx/cache:/data/nginx/cache + - ./docker/data/nginx/blocker:/data/nginx/blocker - ./docker/data/nginx/logs:/usr/local/openresty/nginx/logs - ./docker/data/nginx/skynet:/data/nginx/skynet:ro - ./docker/data/sia/apipassword:/data/sia/apipassword:ro diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile index fd6e4f09..999df4b2 100644 --- a/docker/nginx/Dockerfile +++ b/docker/nginx/Dockerfile @@ -10,6 +10,7 @@ COPY mo ./ COPY libs /etc/nginx/libs COPY conf.d /etc/nginx/conf.d COPY conf.d.templates /etc/nginx/conf.d.templates +COPY scripts /etc/nginx/scripts CMD [ "bash", "-c", \ "./mo < /etc/nginx/conf.d.templates/server.account.conf > /etc/nginx/conf.d/server.account.conf ; \ diff --git a/docker/nginx/scripts/purge-blocklist.sh b/docker/nginx/scripts/purge-blocklist.sh new file mode 100755 index 00000000..228dae1a --- /dev/null +++ b/docker/nginx/scripts/purge-blocklist.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +# TODO: +# +# 1. the purging should batch the skylinks to purge in a single command +# +# python example: +# +# cached_files_command = ( +# "find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: (" +# + "|".join(skylinks[i : i + batch_size]) +# + ")'" +# ) +# +# cached_files_count += int( +# exec( +# 'docker exec nginx bash -c "' +# + cached_files_command +# + ' | xargs -r rm -v | wc -l"' +# ) +# ) + +# This script reads skylinks from a file and purges them from the Nginx cache. +# It uses the atomic mkdir operation to create a lock on the file, under which +# it copies the file and truncates it. + +set -e # exit on first error + +# The following variables define the paths to the file containing the skylinks +# that need to be purged, the file in which we store the queued skylinks and the +# lock directory that ensures the blocker API and the crontab don't manipulate +# the same files concurrently. +NGINX_PURGE_SKYLINKS_FILE="/data/nginx/blocker/skylinks.txt" +NGINX_PURGE_SKYLINKS_QUEUED="/data/nginx/blocker/queued.txt" +NGINX_PURGE_SKYLINKS_LOCK="/data/nginx/blocker/lock" +NGINX_CACHE_DIR="/data/nginx/cache/" + +purge_skylinks () { + # read all skylinks from the queued skylinks file + skylinks=() + line_number=1 + while IFS="" read -r line || [ -n "$line" ]; + do + if [[ $line =~ (^[a-zA-Z0-9_-]{46}$) ]]; then + skylinks+=("$line") + else + echo "Incorrect skylink at line ${line_number}: $line" + fi + let line_number+=1 + done < $NGINX_PURGE_SKYLINKS_QUEUED; + + for skylink in "${skylinks[@]}"; + do + echo ".. ⌁ Purging skylink ${skylink}" + cached_files_command="find ${NGINX_CACHE_DIR} -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'" + bash -c "${cached_files_command} | xargs -r rm" + + echo ".. ⌁ Skylink ${skylink} purged" + echo "--------------------------------------------" + done + + # remove the queue file + rm $NGINX_PURGE_SKYLINKS_QUEUED +} + +acquire_lock () { + attempts=0 + locked=false + until [ "$attempts" -ge 10 ] + do + if ! mkdir $NGINX_PURGE_SKYLINKS_LOCK 2>/dev/null + then + echo "skylinks file is locked, waiting..." + ((attempts++)) + sleep 1; + else + locked=true + break + fi + done + + if ! $locked + then + echo "failed to acquire lock, warrants investigation" + exit 1 + fi +} + +release_lock () { + rmdir $NGINX_PURGE_SKYLINKS_LOCK +} + +# if there is a queue file - purge all skylinks in that file from nginx cache +if [ -f "$NGINX_PURGE_SKYLINKS_QUEUED" ] +then + echo "found queue file, purging skylinks from file" + purge_skylinks + echo "✓ Done" + exit 1 +fi + +# if there is no skylinks file - escape early +if [ ! -f "$NGINX_PURGE_SKYLINKS_FILE" ] +then + echo "no skylinks found" + echo "✓ Done" + exit 1 +fi + +# move the skylinks file to the queue under lock +acquire_lock +mv $NGINX_PURGE_SKYLINKS_FILE $NGINX_PURGE_SKYLINKS_QUEUED +release_lock + +# purge the skylinks from the queue file +purge_skylinks +echo "✓ Done" +exit 1 diff --git a/setup-scripts/support/crontab b/setup-scripts/support/crontab index 29c8ec1a..4467733b 100644 --- a/setup-scripts/support/crontab +++ b/setup-scripts/support/crontab @@ -5,3 +5,4 @@ 0 4 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1 >> /home/user/skynet-webportal/logs/db_backup_`date +"%Y-%m-%d-%H%M"`.log 2 > &1 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200 15 * * * * /home/user/skynet-webportal/scripts/nginx-prune.sh +*/30 * * * * docker exec nginx /etc/nginx/scripts/purge-blocklist.sh