From 018844d1241a41c4f1f2a49ca6e97e53a285ddfd Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Thu, 9 Dec 2021 17:51:40 +0100 Subject: [PATCH 01/12] Add a new volume that allows blocker to write to nginx's local disk. --- docker-compose.blocker.yml | 2 ++ docker-compose.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/docker-compose.blocker.yml b/docker-compose.blocker.yml index b76d2e43..56d23646 100644 --- a/docker-compose.blocker.yml +++ b/docker-compose.blocker.yml @@ -16,6 +16,8 @@ services: logging: *default-logging env_file: - .env + volumes: + - ./docker/data/nginx/blocker:/data/nginx/blocker expose: - 4000 networks: diff --git a/docker-compose.yml b/docker-compose.yml index ca4dc51a..e88b2e22 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -67,6 +67,7 @@ services: volumes: - ./docker/nginx/nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf:ro - ./docker/data/nginx/cache:/data/nginx/cache + - ./docker/data/nginx/blocker:/data/nginx/blocker - ./docker/data/nginx/logs:/usr/local/openresty/nginx/logs - ./docker/data/nginx/skynet:/data/nginx/skynet:ro - ./docker/data/sia/apipassword:/data/sia/apipassword:ro From 0fe3f7cf8da08680180ffc6062c730af4d2c28da Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 10 Dec 2021 16:02:11 +0100 Subject: [PATCH 02/12] Add purge blocklist command --- scripts/purge-nginx-blocklist.sh | 117 +++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 scripts/purge-nginx-blocklist.sh diff --git a/scripts/purge-nginx-blocklist.sh b/scripts/purge-nginx-blocklist.sh new file mode 100644 index 00000000..ba55d4e4 --- /dev/null +++ b/scripts/purge-nginx-blocklist.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +# TODO: +# +# 1. the purging should batch the skylinks to purge in a single command +# +# python example: +# +# cached_files_command = ( +# "find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: (" +# + "|".join(skylinks[i : i + batch_size]) +# + ")'" +# ) +# +# cached_files_count += int( +# exec( +# 'docker exec nginx bash -c "' +# + cached_files_command +# + ' | xargs -r rm -v | wc -l"' +# ) +# ) + +# This script reads skylinks from a file and purges them from the Nginx cache. +# It uses the atomic mkdir operation to create a lock on the file, under which +# it copies the file and truncates it. + +set -e # exit on first error + +# The following variables define the paths to the file containing the skylinks +# that need to be purged, the file in which we store the queued skylinks and the +# lock directory that ensures the blocker API and the crontab don't manipulate +# the same files concurrently. +NGINX_PURGE_SKYLINKS_FILE="/home/user/skynet-webportal/docker/data/nginx/blocker/skylinks.txt" +NGINX_PURGE_SKYLINKS_QUEUED="/home/user/skynet-webportal/docker/data/nginx/blocker/queued.txt" +NGINX_PURGE_SKYLINKS_LOCK="/home/user/skynet-webportal/docker/data/nginx/blocker/lock" + +purge_skylinks () { + # read all skylinks from the queued skylinks file + skylinks=() + line_number=1 + while IFS="" read -r line || [ -n "$line" ]; + do + if [[ $line =~ (^[a-zA-Z0-9_-]{46}$) ]]; then + skylinks+=("$line") + else + echo "Incorrect skylink at line ${line_number}: $line" + fi + let line_number+=1 + done < $NGINX_PURGE_SKYLINKS_QUEUED; + + for skylink in "${skylinks[@]}"; + do + echo ".. ⌁ Purging skylink ${skylink}" + cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'" + docker exec -it nginx bash -c "${cached_files_command} | xargs -r rm" + + echo ".. ⌁ Skylink ${skylink} purged" + echo "--------------------------------------------" + done + + # remove the queue file + rm $NGINX_PURGE_SKYLINKS_QUEUED +} + +acquire_lock () { + attempts=0 + locked=false + until [ "$attempts" -ge 10 ] + do + if ! mkdir $NGINX_PURGE_SKYLINKS_LOCK 2>/dev/null + then + echo "skylinks file is locked, waiting..." + $attempts + ((attempts++)) + sleep 1; + else + locked=true + break + fi + done + + if ! $locked + then + echo "failed to acquire lock, warrants investigation" + exit 1 + fi +} + +release_lock () { + rmdir $NGINX_PURGE_SKYLINKS_LOCK +} + +# if there is a queue file - purge all skylinks in that file from nginx cache +if [ -f "$NGINX_PURGE_SKYLINKS_QUEUED" ] +then + echo "found queue file, purging skylinks from file" + purge_skylinks + echo "✓ Done" + exit 1 +fi + +# if there is no skylinks file - escape early +if [ ! -f "$NGINX_PURGE_SKYLINKS_FILE" ] +then + echo "no skylinks found" + echo "✓ Done" + exit 1 +fi + +# move the skylinks file to the queue under lock +acquire_lock +mv $NGINX_PURGE_SKYLINKS_FILE $NGINX_PURGE_SKYLINKS_QUEUED +release_lock + +# purge the skylinks from the queue file +purge_skylinks +echo "✓ Done" +exit 1 \ No newline at end of file From d3bc3533b0a6c7e3b903bfa9199dc2fb8559bc11 Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 10 Dec 2021 16:04:20 +0100 Subject: [PATCH 03/12] Add newline --- scripts/purge-nginx-blocklist.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/purge-nginx-blocklist.sh b/scripts/purge-nginx-blocklist.sh index ba55d4e4..3f88b160 100644 --- a/scripts/purge-nginx-blocklist.sh +++ b/scripts/purge-nginx-blocklist.sh @@ -114,4 +114,4 @@ release_lock # purge the skylinks from the queue file purge_skylinks echo "✓ Done" -exit 1 \ No newline at end of file +exit 1 From 1c3a3b26c5ed90a474056ea6533224cd8d918c3c Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 10 Dec 2021 16:46:17 +0100 Subject: [PATCH 04/12] Improve script --- scripts/purge-nginx-blocklist.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/purge-nginx-blocklist.sh b/scripts/purge-nginx-blocklist.sh index 3f88b160..13490c89 100644 --- a/scripts/purge-nginx-blocklist.sh +++ b/scripts/purge-nginx-blocklist.sh @@ -33,6 +33,7 @@ set -e # exit on first error NGINX_PURGE_SKYLINKS_FILE="/home/user/skynet-webportal/docker/data/nginx/blocker/skylinks.txt" NGINX_PURGE_SKYLINKS_QUEUED="/home/user/skynet-webportal/docker/data/nginx/blocker/queued.txt" NGINX_PURGE_SKYLINKS_LOCK="/home/user/skynet-webportal/docker/data/nginx/blocker/lock" +NGINX_CACHE_DIR="/home/user/skynet-webportal/docker/data/nginx/cache/" purge_skylinks () { # read all skylinks from the queued skylinks file @@ -51,7 +52,8 @@ purge_skylinks () { for skylink in "${skylinks[@]}"; do echo ".. ⌁ Purging skylink ${skylink}" - cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'" + cached_files_command="find ${NGINX_CACHE_DIR} -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'" + echo $cached_files_command docker exec -it nginx bash -c "${cached_files_command} | xargs -r rm" echo ".. ⌁ Skylink ${skylink} purged" @@ -69,7 +71,7 @@ acquire_lock () { do if ! mkdir $NGINX_PURGE_SKYLINKS_LOCK 2>/dev/null then - echo "skylinks file is locked, waiting..." + $attempts + echo "skylinks file is locked, waiting..." ((attempts++)) sleep 1; else From 76929f2d91b30ae8516f55c91917fcc3003b889c Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 10 Dec 2021 16:59:58 +0100 Subject: [PATCH 05/12] Make executable --- scripts/purge-nginx-blocklist.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/purge-nginx-blocklist.sh diff --git a/scripts/purge-nginx-blocklist.sh b/scripts/purge-nginx-blocklist.sh old mode 100644 new mode 100755 From 15c819eb964e594c71f423deeff311a433aa5fc7 Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 10 Dec 2021 17:02:50 +0100 Subject: [PATCH 06/12] Update paths --- scripts/purge-nginx-blocklist.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/purge-nginx-blocklist.sh b/scripts/purge-nginx-blocklist.sh index 13490c89..d095bfdb 100755 --- a/scripts/purge-nginx-blocklist.sh +++ b/scripts/purge-nginx-blocklist.sh @@ -30,10 +30,10 @@ set -e # exit on first error # that need to be purged, the file in which we store the queued skylinks and the # lock directory that ensures the blocker API and the crontab don't manipulate # the same files concurrently. -NGINX_PURGE_SKYLINKS_FILE="/home/user/skynet-webportal/docker/data/nginx/blocker/skylinks.txt" -NGINX_PURGE_SKYLINKS_QUEUED="/home/user/skynet-webportal/docker/data/nginx/blocker/queued.txt" -NGINX_PURGE_SKYLINKS_LOCK="/home/user/skynet-webportal/docker/data/nginx/blocker/lock" -NGINX_CACHE_DIR="/home/user/skynet-webportal/docker/data/nginx/cache/" +NGINX_PURGE_SKYLINKS_FILE="/data/nginx/blocker/skylinks.txt" +NGINX_PURGE_SKYLINKS_QUEUED="/data/nginx/blocker/queued.txt" +NGINX_PURGE_SKYLINKS_LOCK="/data/nginx/blocker/lock" +NGINX_CACHE_DIR="/data/nginx/cache/" purge_skylinks () { # read all skylinks from the queued skylinks file From 593a88c8ca5a1b1b89e14492a9b857e25ed25357 Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 10 Dec 2021 17:04:26 +0100 Subject: [PATCH 07/12] Update command --- scripts/purge-nginx-blocklist.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/purge-nginx-blocklist.sh b/scripts/purge-nginx-blocklist.sh index d095bfdb..b709fc99 100755 --- a/scripts/purge-nginx-blocklist.sh +++ b/scripts/purge-nginx-blocklist.sh @@ -54,7 +54,7 @@ purge_skylinks () { echo ".. ⌁ Purging skylink ${skylink}" cached_files_command="find ${NGINX_CACHE_DIR} -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'" echo $cached_files_command - docker exec -it nginx bash -c "${cached_files_command} | xargs -r rm" + bash -c "${cached_files_command} | xargs -r rm" echo ".. ⌁ Skylink ${skylink} purged" echo "--------------------------------------------" From 92323b907f98b4ae4d4143635ac71c3b6bc5cb61 Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 10 Dec 2021 17:07:09 +0100 Subject: [PATCH 08/12] Add logging --- scripts/purge-nginx-blocklist.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/purge-nginx-blocklist.sh b/scripts/purge-nginx-blocklist.sh index b709fc99..4f2b2ac8 100755 --- a/scripts/purge-nginx-blocklist.sh +++ b/scripts/purge-nginx-blocklist.sh @@ -110,6 +110,9 @@ fi # move the skylinks file to the queue under lock acquire_lock +echo "moving file from -> to" +echo $NGINX_PURGE_SKYLINKS_FILE +echo $NGINX_PURGE_SKYLINKS_QUEUED mv $NGINX_PURGE_SKYLINKS_FILE $NGINX_PURGE_SKYLINKS_QUEUED release_lock From c24017dc9d35576c8615c70ea9f948cff22f7c99 Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 10 Dec 2021 17:10:26 +0100 Subject: [PATCH 09/12] Remove logging --- scripts/purge-nginx-blocklist.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/purge-nginx-blocklist.sh b/scripts/purge-nginx-blocklist.sh index 4f2b2ac8..228dae1a 100755 --- a/scripts/purge-nginx-blocklist.sh +++ b/scripts/purge-nginx-blocklist.sh @@ -53,7 +53,6 @@ purge_skylinks () { do echo ".. ⌁ Purging skylink ${skylink}" cached_files_command="find ${NGINX_CACHE_DIR} -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'" - echo $cached_files_command bash -c "${cached_files_command} | xargs -r rm" echo ".. ⌁ Skylink ${skylink} purged" @@ -110,9 +109,6 @@ fi # move the skylinks file to the queue under lock acquire_lock -echo "moving file from -> to" -echo $NGINX_PURGE_SKYLINKS_FILE -echo $NGINX_PURGE_SKYLINKS_QUEUED mv $NGINX_PURGE_SKYLINKS_FILE $NGINX_PURGE_SKYLINKS_QUEUED release_lock From d0da240ab9c1ccd8f1efbdf8124a9aae16543700 Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 10 Dec 2021 17:15:49 +0100 Subject: [PATCH 10/12] Move script --- docker/nginx/Dockerfile | 1 + .../nginx/scripts/purge-blocklist.sh | 0 2 files changed, 1 insertion(+) rename scripts/purge-nginx-blocklist.sh => docker/nginx/scripts/purge-blocklist.sh (100%) diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile index fd6e4f09..999df4b2 100644 --- a/docker/nginx/Dockerfile +++ b/docker/nginx/Dockerfile @@ -10,6 +10,7 @@ COPY mo ./ COPY libs /etc/nginx/libs COPY conf.d /etc/nginx/conf.d COPY conf.d.templates /etc/nginx/conf.d.templates +COPY scripts /etc/nginx/scripts CMD [ "bash", "-c", \ "./mo < /etc/nginx/conf.d.templates/server.account.conf > /etc/nginx/conf.d/server.account.conf ; \ diff --git a/scripts/purge-nginx-blocklist.sh b/docker/nginx/scripts/purge-blocklist.sh similarity index 100% rename from scripts/purge-nginx-blocklist.sh rename to docker/nginx/scripts/purge-blocklist.sh From 644f7f3e84971e72262783cd532bdd80b914288a Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 10 Dec 2021 17:32:01 +0100 Subject: [PATCH 11/12] Add command to crontab --- setup-scripts/support/crontab | 1 + 1 file changed, 1 insertion(+) diff --git a/setup-scripts/support/crontab b/setup-scripts/support/crontab index 29c8ec1a..a3b9d47d 100644 --- a/setup-scripts/support/crontab +++ b/setup-scripts/support/crontab @@ -5,3 +5,4 @@ 0 4 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1 >> /home/user/skynet-webportal/logs/db_backup_`date +"%Y-%m-%d-%H%M"`.log 2 > &1 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200 15 * * * * /home/user/skynet-webportal/scripts/nginx-prune.sh +*/30 * * * * docker exec nginx /etc/nginx/scripts/purge-blocklist.sh \ No newline at end of file From 9be3e29aafcc8264b3a3dc6d4a8d14957c0b0eda Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 10 Dec 2021 17:33:23 +0100 Subject: [PATCH 12/12] Add newline --- setup-scripts/support/crontab | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-scripts/support/crontab b/setup-scripts/support/crontab index a3b9d47d..4467733b 100644 --- a/setup-scripts/support/crontab +++ b/setup-scripts/support/crontab @@ -5,4 +5,4 @@ 0 4 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1 >> /home/user/skynet-webportal/logs/db_backup_`date +"%Y-%m-%d-%H%M"`.log 2 > &1 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200 15 * * * * /home/user/skynet-webportal/scripts/nginx-prune.sh -*/30 * * * * docker exec nginx /etc/nginx/scripts/purge-blocklist.sh \ No newline at end of file +*/30 * * * * docker exec nginx /etc/nginx/scripts/purge-blocklist.sh