From 0af14d32db8a7bc7d35406fa6bb7d51d4c8e8259 Mon Sep 17 00:00:00 2001
From: Filip Rysavy <29089732+firyx@users.noreply.github.com>
Date: Mon, 6 Dec 2021 13:35:17 +0100
Subject: [PATCH 1/3] Add pruning Nginx cache

---
 changelog/items/other/nginx-prune.md       |  1 +
 scripts/README.md                          |  6 ++++++
 scripts/lib/nginx-prune-cache-subscript.sh | 24 ++++++++++++++++++++++
 scripts/nginx-prune.sh                     |  6 ++++++
 setup-scripts/support/crontab              |  1 +
 5 files changed, 38 insertions(+)
 create mode 100644 changelog/items/other/nginx-prune.md
 create mode 100755 scripts/lib/nginx-prune-cache-subscript.sh
 create mode 100755 scripts/nginx-prune.sh

diff --git a/changelog/items/other/nginx-prune.md b/changelog/items/other/nginx-prune.md
new file mode 100644
index 00000000..42581090
--- /dev/null
+++ b/changelog/items/other/nginx-prune.md
@@ -0,0 +1 @@
+- Added script to prune nginx cache.
\ No newline at end of file
diff --git a/scripts/README.md b/scripts/README.md
index e7b909b4..2085eff7 100644
--- a/scripts/README.md
+++ b/scripts/README.md
@@ -29,6 +29,12 @@ the health check.
 The `portal-upgrade.sh` script upgrades the docker images for a portal and
 clears and leftover images.
 
+**nginx-prune.sh**\
+The `nginx-prune.sh` script deletes all entries from nginx cache larger than
+the given size and smaller entries until nginx cache disk size is smaller than
+the given cache size limit. Both values are configured in
+`lib/nginx-prune-cache-subscript.sh`. The script doesn't require `sudo`.
+
 ## Webportal Upgrade Procedures
 
 TODO...
diff --git a/scripts/lib/nginx-prune-cache-subscript.sh b/scripts/lib/nginx-prune-cache-subscript.sh
new file mode 100755
index 00000000..bf1e1e44
--- /dev/null
+++ b/scripts/lib/nginx-prune-cache-subscript.sh
@@ -0,0 +1,24 @@
+#!/usr/local/bin/bash
+
+# This subscript is expected to be run inside docker container using 'bash'
+# image. The image is based on Alpine Linux. It's tools (find, stat, awk, sort)
+# are non-standard versions from BusyBox.
+
+MAX_CACHE_DIR_SIZE=20000000000
+MAX_KEEP_FILE_SIZE=1000000000
+
+total=0
+
+find /home/user/skynet-webportal/docker/data/nginx/cache -type f -exec stat -c "%Y %n %s" {} + | sort -rgk1 | while read line
+do
+    size=$(echo $line | awk '{print $3}')
+    new_total=$(($total + $size))
+    if (("$size" <= "$MAX_KEEP_FILE_SIZE" && "$total" < "$new_total"))
+    then
+        total=$new_total
+        continue
+    fi
+
+    filename=$(echo $line | awk '{print $2}')
+    rm $filename
+done
diff --git a/scripts/nginx-prune.sh b/scripts/nginx-prune.sh
new file mode 100755
index 00000000..f67d29e7
--- /dev/null
+++ b/scripts/nginx-prune.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# We execute the nginx cache pruning subscript from docker container so that we
+# can run the pruning script in user crontab without sudo.
+
+docker run --rm -v /home/user:/home/user bash /home/user/skynet-webportal/scripts/lib/nginx-prune-cache-subscript.sh
diff --git a/setup-scripts/support/crontab b/setup-scripts/support/crontab
index ad766264..29c8ec1a 100644
--- a/setup-scripts/support/crontab
+++ b/setup-scripts/support/crontab
@@ -4,3 +4,4 @@
 30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env
 0 4 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1 >> /home/user/skynet-webportal/logs/db_backup_`date +"%Y-%m-%d-%H%M"`.log 2 > &1
 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200
+15 * * * * /home/user/skynet-webportal/scripts/nginx-prune.sh

From aca71c245e662dc9dff57a125c8f0c3834809712 Mon Sep 17 00:00:00 2001
From: Filip Rysavy <29089732+firyx@users.noreply.github.com>
Date: Mon, 6 Dec 2021 14:13:56 +0100
Subject: [PATCH 2/3] Add comments to nginx pruning script

---
 scripts/lib/nginx-prune-cache-subscript.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scripts/lib/nginx-prune-cache-subscript.sh b/scripts/lib/nginx-prune-cache-subscript.sh
index bf1e1e44..5994f49f 100755
--- a/scripts/lib/nginx-prune-cache-subscript.sh
+++ b/scripts/lib/nginx-prune-cache-subscript.sh
@@ -9,10 +9,16 @@ MAX_KEEP_FILE_SIZE=1000000000
 
 total=0
 
+# We sort files by time, newest files are first. Format is:
+# time (last modification as seconds since Epoch), filepath, size (bytes)
 find /home/user/skynet-webportal/docker/data/nginx/cache -type f -exec stat -c "%Y %n %s" {} + | sort -rgk1 | while read line
 do
     size=$(echo $line | awk '{print $3}')
     new_total=$(($total + $size))
+
+    # We always delete all files larger than MAX_KEEP_FILE_SIZE.
+    # We keep all files smaller than MAX_KEEP_FILE_SIZE when cache size is
+    # below MAX_CACHE_DIR_SIZE, then we delete also smaller files.
     if (("$size" <= "$MAX_KEEP_FILE_SIZE" && "$total" < "$new_total"))
     then
         total=$new_total

From 1c1f3c6ec113de0abd1f706b5455e4f146a69a9d Mon Sep 17 00:00:00 2001
From: Filip Rysavy <29089732+firyx@users.noreply.github.com>
Date: Mon, 6 Dec 2021 16:13:00 +0100
Subject: [PATCH 3/3] Fix condition mistake

---
 scripts/lib/nginx-prune-cache-subscript.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/lib/nginx-prune-cache-subscript.sh b/scripts/lib/nginx-prune-cache-subscript.sh
index 5994f49f..99edb899 100755
--- a/scripts/lib/nginx-prune-cache-subscript.sh
+++ b/scripts/lib/nginx-prune-cache-subscript.sh
@@ -19,7 +19,7 @@ do
     # We always delete all files larger than MAX_KEEP_FILE_SIZE.
     # We keep all files smaller than MAX_KEEP_FILE_SIZE when cache size is
     # below MAX_CACHE_DIR_SIZE, then we delete also smaller files.
-    if (("$size" <= "$MAX_KEEP_FILE_SIZE" && "$total" < "$new_total"))
+    if (("$size" <= "$MAX_KEEP_FILE_SIZE" && "$new_total" < "$MAX_CACHE_DIR_SIZE"))
     then
         total=$new_total
         continue