diff --git a/changelog/items/other/nginx-prune.md b/changelog/items/other/nginx-prune.md new file mode 100644 index 00000000..42581090 --- /dev/null +++ b/changelog/items/other/nginx-prune.md @@ -0,0 +1 @@ +- Added script to prune nginx cache. \ No newline at end of file diff --git a/scripts/README.md b/scripts/README.md index e7b909b4..2085eff7 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -29,6 +29,12 @@ the health check. The `portal-upgrade.sh` script upgrades the docker images for a portal and clears and leftover images. +**nginx-prune.sh**\ +The `nginx-prune.sh` script deletes all entries from nginx cache larger than +the given size and smaller entries until nginx cache disk size is smaller than +the given cache size limit. Both values are configured in +`lib/nginx-prune-cache-subscript.sh`. The script doesn't require `sudo`. + ## Webportal Upgrade Procedures TODO... diff --git a/scripts/lib/nginx-prune-cache-subscript.sh b/scripts/lib/nginx-prune-cache-subscript.sh new file mode 100755 index 00000000..99edb899 --- /dev/null +++ b/scripts/lib/nginx-prune-cache-subscript.sh @@ -0,0 +1,30 @@ +#!/usr/local/bin/bash + +# This subscript is expected to be run inside docker container using 'bash' +# image. The image is based on Alpine Linux. It's tools (find, stat, awk, sort) +# are non-standard versions from BusyBox. + +MAX_CACHE_DIR_SIZE=20000000000 +MAX_KEEP_FILE_SIZE=1000000000 + +total=0 + +# We sort files by time, newest files are first. Format is: +# time (last modification as seconds since Epoch), filepath, size (bytes) +find /home/user/skynet-webportal/docker/data/nginx/cache -type f -exec stat -c "%Y %n %s" {} + | sort -rgk1 | while read line +do + size=$(echo $line | awk '{print $3}') + new_total=$(($total + $size)) + + # We always delete all files larger than MAX_KEEP_FILE_SIZE. + # We keep all files smaller than MAX_KEEP_FILE_SIZE when cache size is + # below MAX_CACHE_DIR_SIZE, then we delete also smaller files. + if (("$size" <= "$MAX_KEEP_FILE_SIZE" && "$new_total" < "$MAX_CACHE_DIR_SIZE")) + then + total=$new_total + continue + fi + + filename=$(echo $line | awk '{print $2}') + rm $filename +done diff --git a/scripts/nginx-prune.sh b/scripts/nginx-prune.sh new file mode 100755 index 00000000..f67d29e7 --- /dev/null +++ b/scripts/nginx-prune.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# We execute the nginx cache pruning subscript from docker container so that we +# can run the pruning script in user crontab without sudo. + +docker run --rm -v /home/user:/home/user bash /home/user/skynet-webportal/scripts/lib/nginx-prune-cache-subscript.sh diff --git a/setup-scripts/support/crontab b/setup-scripts/support/crontab index ad766264..29c8ec1a 100644 --- a/setup-scripts/support/crontab +++ b/setup-scripts/support/crontab @@ -4,3 +4,4 @@ 30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env 0 4 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1 >> /home/user/skynet-webportal/logs/db_backup_`date +"%Y-%m-%d-%H%M"`.log 2 > &1 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200 +15 * * * * /home/user/skynet-webportal/scripts/nginx-prune.sh