From 0af14d32db8a7bc7d35406fa6bb7d51d4c8e8259 Mon Sep 17 00:00:00 2001 From: Filip Rysavy <29089732+firyx@users.noreply.github.com> Date: Mon, 6 Dec 2021 13:35:17 +0100 Subject: [PATCH] Add pruning Nginx cache --- changelog/items/other/nginx-prune.md | 1 + scripts/README.md | 6 ++++++ scripts/lib/nginx-prune-cache-subscript.sh | 24 ++++++++++++++++++++++ scripts/nginx-prune.sh | 6 ++++++ setup-scripts/support/crontab | 1 + 5 files changed, 38 insertions(+) create mode 100644 changelog/items/other/nginx-prune.md create mode 100755 scripts/lib/nginx-prune-cache-subscript.sh create mode 100755 scripts/nginx-prune.sh diff --git a/changelog/items/other/nginx-prune.md b/changelog/items/other/nginx-prune.md new file mode 100644 index 00000000..42581090 --- /dev/null +++ b/changelog/items/other/nginx-prune.md @@ -0,0 +1 @@ +- Added script to prune nginx cache. \ No newline at end of file diff --git a/scripts/README.md b/scripts/README.md index e7b909b4..2085eff7 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -29,6 +29,12 @@ the health check. The `portal-upgrade.sh` script upgrades the docker images for a portal and clears and leftover images. +**nginx-prune.sh**\ +The `nginx-prune.sh` script deletes all entries from nginx cache larger than +the given size and smaller entries until nginx cache disk size is smaller than +the given cache size limit. Both values are configured in +`lib/nginx-prune-cache-subscript.sh`. The script doesn't require `sudo`. + ## Webportal Upgrade Procedures TODO... diff --git a/scripts/lib/nginx-prune-cache-subscript.sh b/scripts/lib/nginx-prune-cache-subscript.sh new file mode 100755 index 00000000..bf1e1e44 --- /dev/null +++ b/scripts/lib/nginx-prune-cache-subscript.sh @@ -0,0 +1,24 @@ +#!/usr/local/bin/bash + +# This subscript is expected to be run inside docker container using 'bash' +# image. The image is based on Alpine Linux. It's tools (find, stat, awk, sort) +# are non-standard versions from BusyBox. + +MAX_CACHE_DIR_SIZE=20000000000 +MAX_KEEP_FILE_SIZE=1000000000 + +total=0 + +find /home/user/skynet-webportal/docker/data/nginx/cache -type f -exec stat -c "%Y %n %s" {} + | sort -rgk1 | while read line +do + size=$(echo $line | awk '{print $3}') + new_total=$(($total + $size)) + if (("$size" <= "$MAX_KEEP_FILE_SIZE" && "$total" < "$new_total")) + then + total=$new_total + continue + fi + + filename=$(echo $line | awk '{print $2}') + rm $filename +done diff --git a/scripts/nginx-prune.sh b/scripts/nginx-prune.sh new file mode 100755 index 00000000..f67d29e7 --- /dev/null +++ b/scripts/nginx-prune.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# We execute the nginx cache pruning subscript from docker container so that we +# can run the pruning script in user crontab without sudo. + +docker run --rm -v /home/user:/home/user bash /home/user/skynet-webportal/scripts/lib/nginx-prune-cache-subscript.sh diff --git a/setup-scripts/support/crontab b/setup-scripts/support/crontab index ad766264..29c8ec1a 100644 --- a/setup-scripts/support/crontab +++ b/setup-scripts/support/crontab @@ -4,3 +4,4 @@ 30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env 0 4 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1 >> /home/user/skynet-webportal/logs/db_backup_`date +"%Y-%m-%d-%H%M"`.log 2 > &1 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200 +15 * * * * /home/user/skynet-webportal/scripts/nginx-prune.sh