From 3722969f0f8e0aab2f959bea3d0ab61affbb13f4 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 1 Dec 2021 10:57:28 +0100 Subject: [PATCH 01/23] Add blocker Dockerfile --- docker/blocker/Dockerfile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 docker/blocker/Dockerfile diff --git a/docker/blocker/Dockerfile b/docker/blocker/Dockerfile new file mode 100644 index 00000000..3dbc2f61 --- /dev/null +++ b/docker/blocker/Dockerfile @@ -0,0 +1,16 @@ +FROM golang:1.16.7 +LABEL maintainer="NebulousLabs " + +ENV GOOS linux +ENV GOARCH amd64 + +ARG branch=main + +WORKDIR /root + +RUN git clone --single-branch --branch ${branch} https://github.com/SkynetLabs/blocker.git && \ + cd blocker && \ + go mod download && \ + make release + +ENTRYPOINT ["blocker"] From 18e00431135eee71c14fc29899729d77cf243071 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 1 Dec 2021 11:02:15 +0100 Subject: [PATCH 02/23] Add docker-compose --- docker-compose.blocker.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 docker-compose.blocker.yml diff --git a/docker-compose.blocker.yml b/docker-compose.blocker.yml new file mode 100644 index 00000000..4e46893c --- /dev/null +++ b/docker-compose.blocker.yml @@ -0,0 +1,28 @@ +version: "3.7" + +x-logging: &default-logging + driver: json-file + options: + max-size: "10m" + max-file: "3" + +services: + blocker: + build: + context: ./docker/blocker + dockerfile: Dockerfile + args: + branch: main + container_name: blocker + restart: unless-stopped + logging: *default-logging + env_file: + - .env + expose: + - 4000 + networks: + shared: + ipv4_address: 10.10.10.102 + depends_on: + - mongo + - sia From fe87f19e68ec96b279e0c1da75b1bc00cea89e77 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 1 Dec 2021 11:02:55 +0100 Subject: [PATCH 03/23] Update dc to include blocker module --- dc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dc b/dc index 60418cb8..c041317a 100755 --- a/dc +++ b/dc @@ -13,6 +13,11 @@ for i in $(seq 1 ${#PORTAL_MODULES}); do COMPOSE_FILES+=" -f docker-compose.mongodb.yml -f docker-compose.accounts.yml" fi + # blocker module - alias "b" + if [[ ${PORTAL_MODULES:i-1:1} == "b" ]]; then + COMPOSE_FILES+=" -f docker-compose.blocker.yml" + fi + # jaeger module - alias "j" if [[ ${PORTAL_MODULES:i-1:1} == "j" ]]; then COMPOSE_FILES+=" -f docker-compose.jaeger.yml" From a03860419f459b2515b4a1f5f1306fb17b616246 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 1 Dec 2021 11:04:02 +0100 Subject: [PATCH 04/23] Update dockerfile --- docker/blocker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/blocker/Dockerfile b/docker/blocker/Dockerfile index 3dbc2f61..e0bbabaa 100644 --- a/docker/blocker/Dockerfile +++ b/docker/blocker/Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer="NebulousLabs " ENV GOOS linux ENV GOARCH amd64 -ARG branch=main +ARG branch=ivo/initial_impl WORKDIR /root From 1fe2c3d76f12913d306b814a97bc885b23097dc1 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 1 Dec 2021 11:18:25 +0100 Subject: [PATCH 05/23] Add abuse location --- docker/nginx/conf.d/server/server.api | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index 07be6fcb..3d6e7c2e 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -90,6 +90,10 @@ location /health-check { proxy_pass http://10.10.10.60:3100; # hardcoded ip because health-check waits for nginx } +location /abuse { + proxy_pass http://10.10.10.102:4000; +} + location /hns { # match the request_uri and extract the hns domain and anything that is passed in the uri after it # example: /hns/something/foo/bar matches: From 0f740419f3a02f092c4c5c1720877720510cd3f3 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 1 Dec 2021 11:19:06 +0100 Subject: [PATCH 06/23] Remove branch arg --- docker-compose.blocker.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.blocker.yml b/docker-compose.blocker.yml index 4e46893c..b76d2e43 100644 --- a/docker-compose.blocker.yml +++ b/docker-compose.blocker.yml @@ -11,8 +11,6 @@ services: build: context: ./docker/blocker dockerfile: Dockerfile - args: - branch: main container_name: blocker restart: unless-stopped logging: *default-logging From 63b283fc421f22d60333557f2dcf0a4d55c7dfe1 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 1 Dec 2021 12:56:25 +0100 Subject: [PATCH 07/23] Update block routes --- docker/nginx/conf.d/server/server.api | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index 3d6e7c2e..20fcff1e 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -90,8 +90,15 @@ location /health-check { proxy_pass http://10.10.10.60:3100; # hardcoded ip because health-check waits for nginx } +location /block/ { + proxy_method POST; + proxy_http_version 1.1; + proxy_pass http://10.10.10.102:4000/; + client_max_body_size 10M; +} + location /abuse { - proxy_pass http://10.10.10.102:4000; + proxy_pass https://vg2e911ounsg4d87jppvj5mkcg3gvnceh28aobjl2puonuu24696uno.siasky.dev/; } location /hns { From 25fd697d5fc533eae682db651bc81463518d6ef8 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 1 Dec 2021 16:23:56 +0100 Subject: [PATCH 08/23] Configure routes --- docker/nginx/conf.d/server/server.api | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index 20fcff1e..7a4b5895 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -90,15 +90,22 @@ location /health-check { proxy_pass http://10.10.10.60:3100; # hardcoded ip because health-check waits for nginx } -location /block/ { - proxy_method POST; - proxy_http_version 1.1; +location /abuse/ { proxy_pass http://10.10.10.102:4000/; - client_max_body_size 10M; } -location /abuse { - proxy_pass https://vg2e911ounsg4d87jppvj5mkcg3gvnceh28aobjl2puonuu24696uno.siasky.dev/; +location /report-abuse { + # include /etc/nginx/conf.d/include/cors; + + # set $skylink "vg2e911ounsg4d87jppvj5mkcg3gvnceh28aobjl2puonuu24696uno"; + # set $path $uri; + + # include /etc/nginx/conf.d/include/location-skylink; + + # proxy_intercept_errors on; + # error_page 400 404 490 500 502 503 504 =200 @fallback2; + # TODO: wanted to use proxy pass but ran into error here + return https://0404guluqu38oaqapku91ed11kbhkge55smh9lhjukmlrj37lfpm8no.siasky.net } location /hns { From 6c9980374916a798be2d75f3a6d9fed707815222 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 1 Dec 2021 16:51:40 +0100 Subject: [PATCH 09/23] Bypass CORS issue --- docker/nginx/conf.d/server/server.api | 34 +++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index 7a4b5895..afd20dc0 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -91,6 +91,40 @@ location /health-check { } location /abuse/ { + # TODO: this is probably temporary - it bypasses CORS issues on dev nodes + # although it's not that bad as it only allows the correct origin + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' 'https://0404guluqu38oaqapku91ed11kbhkge55smh9lhjukmlrj37lfpm8no.siasky.net'; + # + # Om nom nom cookies + # + add_header 'Access-Control-Allow-Credentials' 'true'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + # + # Custom headers and headers various browsers *should* be OK with but aren't + # + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'; + # + # Tell client that this pre-flight info is valid for 20 days + # + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + if ($request_method = 'POST') { + add_header 'Access-Control-Allow-Origin' 'https://0404guluqu38oaqapku91ed11kbhkge55smh9lhjukmlrj37lfpm8no.siasky.net'; + add_header 'Access-Control-Allow-Credentials' 'true'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'; + } + if ($request_method = 'GET') { + add_header 'Access-Control-Allow-Origin' 'https://0404guluqu38oaqapku91ed11kbhkge55smh9lhjukmlrj37lfpm8no.siasky.net'; + add_header 'Access-Control-Allow-Credentials' 'true'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'; + } + proxy_pass http://10.10.10.102:4000/; } From c89663b393615e79078f373f1af7172f763581a8 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 2 Dec 2021 14:05:41 +0100 Subject: [PATCH 10/23] Add missing semi colon --- docker/nginx/conf.d/server/server.api | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index afd20dc0..6d644f78 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -139,7 +139,7 @@ location /report-abuse { # proxy_intercept_errors on; # error_page 400 404 490 500 502 503 504 =200 @fallback2; # TODO: wanted to use proxy pass but ran into error here - return https://0404guluqu38oaqapku91ed11kbhkge55smh9lhjukmlrj37lfpm8no.siasky.net + return https://0404guluqu38oaqapku91ed11kbhkge55smh9lhjukmlrj37lfpm8no.siasky.net; } location /hns { From 31d73821efb54a665bfcf611531891a320f5d97b Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Thu, 2 Dec 2021 14:08:42 +0100 Subject: [PATCH 11/23] Add a volume to accounts, so it can find its JWKS.json --- docker-compose.accounts.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.accounts.yml b/docker-compose.accounts.yml index 0ce21695..c25557b9 100644 --- a/docker-compose.accounts.yml +++ b/docker-compose.accounts.yml @@ -41,6 +41,8 @@ services: - SKYNET_ACCOUNTS_LOG_LEVEL=${SKYNET_ACCOUNTS_LOG_LEVEL} - KRATOS_ADDR=${KRATOS_ADDR} - OATHKEEPER_ADDR=${OATHKEEPER_ADDR} + volumes: + - ./docker/accounts/conf:/accounts/conf expose: - 3000 networks: From 938c5905c54b26c4e4cd2edbcd56c798182786f7 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 2 Dec 2021 17:07:11 +0100 Subject: [PATCH 12/23] Remove CORS settings --- docker/nginx/conf.d/server/server.api | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index 6d644f78..cbcf10dc 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -91,8 +91,7 @@ location /health-check { } location /abuse/ { - # TODO: this is probably temporary - it bypasses CORS issues on dev nodes - # although it's not that bad as it only allows the correct origin + # set CORS headers if ($request_method = 'OPTIONS') { add_header 'Access-Control-Allow-Origin' 'https://0404guluqu38oaqapku91ed11kbhkge55smh9lhjukmlrj37lfpm8no.siasky.net'; # @@ -112,18 +111,6 @@ location /abuse/ { add_header 'Content-Length' 0; return 204; } - if ($request_method = 'POST') { - add_header 'Access-Control-Allow-Origin' 'https://0404guluqu38oaqapku91ed11kbhkge55smh9lhjukmlrj37lfpm8no.siasky.net'; - add_header 'Access-Control-Allow-Credentials' 'true'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'; - } - if ($request_method = 'GET') { - add_header 'Access-Control-Allow-Origin' 'https://0404guluqu38oaqapku91ed11kbhkge55smh9lhjukmlrj37lfpm8no.siasky.net'; - add_header 'Access-Control-Allow-Credentials' 'true'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'; - } proxy_pass http://10.10.10.102:4000/; } From 7f28d30a449f6f2cc8838e2782f047c96bc6f5d2 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 2 Dec 2021 17:20:14 +0100 Subject: [PATCH 13/23] Update branch arg --- docker/blocker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/blocker/Dockerfile b/docker/blocker/Dockerfile index e0bbabaa..3dbc2f61 100644 --- a/docker/blocker/Dockerfile +++ b/docker/blocker/Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer="NebulousLabs " ENV GOOS linux ENV GOARCH amd64 -ARG branch=ivo/initial_impl +ARG branch=main WORKDIR /root From 6e1af409c6689beceeb43cfd0bf39b615c06ea4f Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 2 Dec 2021 17:23:37 +0100 Subject: [PATCH 14/23] Remove commented out code --- docker/nginx/conf.d/server/server.api | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index cbcf10dc..a8df2b2d 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -116,16 +116,7 @@ location /abuse/ { } location /report-abuse { - # include /etc/nginx/conf.d/include/cors; - - # set $skylink "vg2e911ounsg4d87jppvj5mkcg3gvnceh28aobjl2puonuu24696uno"; - # set $path $uri; - - # include /etc/nginx/conf.d/include/location-skylink; - - # proxy_intercept_errors on; - # error_page 400 404 490 500 502 503 504 =200 @fallback2; - # TODO: wanted to use proxy pass but ran into error here + # TODO: do a proxy_pass return https://0404guluqu38oaqapku91ed11kbhkge55smh9lhjukmlrj37lfpm8no.siasky.net; } From 4d936a05c27e051c7d3259d4560713aa213415a9 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 2 Dec 2021 17:26:22 +0100 Subject: [PATCH 15/23] Add changelog --- changelog/items/other/add-abuse-config.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/items/other/add-abuse-config.md diff --git a/changelog/items/other/add-abuse-config.md b/changelog/items/other/add-abuse-config.md new file mode 100644 index 00000000..51a55918 --- /dev/null +++ b/changelog/items/other/add-abuse-config.md @@ -0,0 +1 @@ +- Add abuse report configuration From 341f313839e7ab68f5b91564f0b0f2a294a23be8 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 2 Dec 2021 17:28:40 +0100 Subject: [PATCH 16/23] Remove comments --- docker/nginx/conf.d/server/server.api | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index a8df2b2d..4402fc07 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -91,21 +91,14 @@ location /health-check { } location /abuse/ { - # set CORS headers if ($request_method = 'OPTIONS') { add_header 'Access-Control-Allow-Origin' 'https://0404guluqu38oaqapku91ed11kbhkge55smh9lhjukmlrj37lfpm8no.siasky.net'; - # - # Om nom nom cookies - # + add_header 'Access-Control-Allow-Credentials' 'true'; add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - # - # Custom headers and headers various browsers *should* be OK with but aren't - # add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'; - # - # Tell client that this pre-flight info is valid for 20 days - # + + # pre-flight info is valid for 20 days add_header 'Access-Control-Max-Age' 1728000; add_header 'Content-Type' 'text/plain charset=UTF-8'; add_header 'Content-Length' 0; From 1351fb79d26f0268b40a6f59a576da1acb062d4a Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Thu, 2 Dec 2021 17:34:22 +0100 Subject: [PATCH 17/23] New path to JWKS.json. --- docker-compose.accounts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.accounts.yml b/docker-compose.accounts.yml index c25557b9..8b0a53b3 100644 --- a/docker-compose.accounts.yml +++ b/docker-compose.accounts.yml @@ -42,7 +42,7 @@ services: - KRATOS_ADDR=${KRATOS_ADDR} - OATHKEEPER_ADDR=${OATHKEEPER_ADDR} volumes: - - ./docker/accounts/conf:/accounts/conf + - ./docker/accounts/conf:/conf expose: - 3000 networks: From 3bef37b789e780a35a507920a202cde255e70939 Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Thu, 2 Dec 2021 18:04:19 +0100 Subject: [PATCH 18/23] Revert the change of JWKS path --- docker-compose.accounts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.accounts.yml b/docker-compose.accounts.yml index 8b0a53b3..c25557b9 100644 --- a/docker-compose.accounts.yml +++ b/docker-compose.accounts.yml @@ -42,7 +42,7 @@ services: - KRATOS_ADDR=${KRATOS_ADDR} - OATHKEEPER_ADDR=${OATHKEEPER_ADDR} volumes: - - ./docker/accounts/conf:/conf + - ./docker/accounts/conf:/accounts/conf expose: - 3000 networks: From 7bf366db24cd8edc5e899bd4a33d4a05f38c5985 Mon Sep 17 00:00:00 2001 From: Matthew Sevey Date: Mon, 6 Dec 2021 06:49:58 -0500 Subject: [PATCH 19/23] Add serverload endpoint (#1410) * write usage script for cpu and free disk space tracking, testing new endpoint * Test alias nginx path * testing json * fix server load json file being served by nginx * Fix filenames * Add changelog * Add systemd file for serverload * Update setup-scripts/serverload.sh Co-authored-by: Peter-Jan Brone --- .gitignore | 4 ++ changelog/items/key-updates/serverload.md | 1 + docker/nginx/conf.d/server/server.api | 15 +++++++ setup-scripts/serverload.service | 8 ++++ setup-scripts/serverload.sh | 55 +++++++++++++++++++++++ 5 files changed, 83 insertions(+) create mode 100644 changelog/items/key-updates/serverload.md create mode 100644 setup-scripts/serverload.service create mode 100755 setup-scripts/serverload.sh diff --git a/.gitignore b/.gitignore index 0900fafd..fcf25b3b 100644 --- a/.gitignore +++ b/.gitignore @@ -96,3 +96,7 @@ docker/kratos/cr_certs/*.key # Oathkeeper JWKS signing token docker/kratos/oathkeeper/id_token.jwks.json /docker/kratos/config/kratos.yml + +# Setup-script log files +/setup-scripts/serverload.log +/setup-scripts/serverload.json \ No newline at end of file diff --git a/changelog/items/key-updates/serverload.md b/changelog/items/key-updates/serverload.md new file mode 100644 index 00000000..c626b753 --- /dev/null +++ b/changelog/items/key-updates/serverload.md @@ -0,0 +1 @@ +- Add `/serverload` endpoint for CPU usage and free disk space diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index 4402fc07..878569db 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -71,6 +71,21 @@ location /skynet/stats { proxy_pass http://sia:9980/skynet/stats; } +# Define path for server load endpoint +location /serverload { + # Define root directory in the nginx container to load file from + root /usr/local/share; + + # including this because of peer pressure from the other routes + include /etc/nginx/conf.d/include/cors; + + # tell nginx to expect json + default_type 'application/json'; + + # Allow for /serverload to load /serverload.json file + try_files $uri $uri.json =404; +} + location /skynet/health { include /etc/nginx/conf.d/include/cors; diff --git a/setup-scripts/serverload.service b/setup-scripts/serverload.service new file mode 100644 index 00000000..5d6a41d4 --- /dev/null +++ b/setup-scripts/serverload.service @@ -0,0 +1,8 @@ +[Unit] +Description=Ensure serverload script is running to provide serverload stats. + +[Service] +ExecStart=/bin/bash /home/user/skynet-webportal/serverload.sh + +[Install] +WantedBy=multi-user.target diff --git a/setup-scripts/serverload.sh b/setup-scripts/serverload.sh new file mode 100755 index 00000000..6945bcb0 --- /dev/null +++ b/setup-scripts/serverload.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +: ' +This script writes the CPU usage and the free disk space to a file in a loop. +The results are prepended to the file, so the most recent results are at the +top. This is so that the most recent information can easily be read from the +top of the file and the file can easily be truncated if needed. + +This script is run by the serverload.service systemd process. The +serverload.service file should be copied to +/etc/systemd/system/serverload.service. + +The systemd process can then be started with the following commands: +sudo systemctl start serverload.service + +The status of the process can be checked with: +sudo systemctl is-active serverload.service +' + +# Define Loop Interval +loop_interval=60 +webportal_repo_setup_scripts="/home/user/skynet-webportal/setup-scripts" +logfile_name="serverload.log" +logfile=$webportal_repo_setup_scripts/$logfile_name +jsonfile="serverload.json" +nginx_docker_path="/usr/local/share" + +# Create logfile if it doesn't exist +if [[ ! -e $logfile ]]; then + echo "init" > $logfile +fi + +# Write the output in an infinite loop. +while true; do + # CPU usage + cpu=$(echo $[100-$(vmstat 1 2|tail -1|awk '{print $15}')]) + sed -i "1iCPU: ${cpu}" $logfile + + # Disk Usage + disk=$(df -Ph . | tail -1 | awk '{print $4}') + sed -i "1iDISK: ${disk}" $logfile + + # Write the timestamp + timestamp=$(date) + sed -i "1iTIMESTAMP: ${timestamp}" $logfile + + # Write and copy a json file of the latest results to nginx docker container + # to serve + printf '{"cpu":"%s","disk":"%s","timestamp":"%s"}' "$cpu" "$disk" "$timestamp" > $webportal_repo_setup_scripts/$jsonfile + docker cp $webportal_repo_setup_scripts/$jsonfile nginx:$nginx_docker_path/$jsonfile + + # Sleep + sleep $loop_interval +done + From 0af14d32db8a7bc7d35406fa6bb7d51d4c8e8259 Mon Sep 17 00:00:00 2001 From: Filip Rysavy <29089732+firyx@users.noreply.github.com> Date: Mon, 6 Dec 2021 13:35:17 +0100 Subject: [PATCH 20/23] Add pruning Nginx cache --- changelog/items/other/nginx-prune.md | 1 + scripts/README.md | 6 ++++++ scripts/lib/nginx-prune-cache-subscript.sh | 24 ++++++++++++++++++++++ scripts/nginx-prune.sh | 6 ++++++ setup-scripts/support/crontab | 1 + 5 files changed, 38 insertions(+) create mode 100644 changelog/items/other/nginx-prune.md create mode 100755 scripts/lib/nginx-prune-cache-subscript.sh create mode 100755 scripts/nginx-prune.sh diff --git a/changelog/items/other/nginx-prune.md b/changelog/items/other/nginx-prune.md new file mode 100644 index 00000000..42581090 --- /dev/null +++ b/changelog/items/other/nginx-prune.md @@ -0,0 +1 @@ +- Added script to prune nginx cache. \ No newline at end of file diff --git a/scripts/README.md b/scripts/README.md index e7b909b4..2085eff7 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -29,6 +29,12 @@ the health check. The `portal-upgrade.sh` script upgrades the docker images for a portal and clears and leftover images. +**nginx-prune.sh**\ +The `nginx-prune.sh` script deletes all entries from nginx cache larger than +the given size and smaller entries until nginx cache disk size is smaller than +the given cache size limit. Both values are configured in +`lib/nginx-prune-cache-subscript.sh`. The script doesn't require `sudo`. + ## Webportal Upgrade Procedures TODO... diff --git a/scripts/lib/nginx-prune-cache-subscript.sh b/scripts/lib/nginx-prune-cache-subscript.sh new file mode 100755 index 00000000..bf1e1e44 --- /dev/null +++ b/scripts/lib/nginx-prune-cache-subscript.sh @@ -0,0 +1,24 @@ +#!/usr/local/bin/bash + +# This subscript is expected to be run inside docker container using 'bash' +# image. The image is based on Alpine Linux. It's tools (find, stat, awk, sort) +# are non-standard versions from BusyBox. + +MAX_CACHE_DIR_SIZE=20000000000 +MAX_KEEP_FILE_SIZE=1000000000 + +total=0 + +find /home/user/skynet-webportal/docker/data/nginx/cache -type f -exec stat -c "%Y %n %s" {} + | sort -rgk1 | while read line +do + size=$(echo $line | awk '{print $3}') + new_total=$(($total + $size)) + if (("$size" <= "$MAX_KEEP_FILE_SIZE" && "$total" < "$new_total")) + then + total=$new_total + continue + fi + + filename=$(echo $line | awk '{print $2}') + rm $filename +done diff --git a/scripts/nginx-prune.sh b/scripts/nginx-prune.sh new file mode 100755 index 00000000..f67d29e7 --- /dev/null +++ b/scripts/nginx-prune.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# We execute the nginx cache pruning subscript from docker container so that we +# can run the pruning script in user crontab without sudo. + +docker run --rm -v /home/user:/home/user bash /home/user/skynet-webportal/scripts/lib/nginx-prune-cache-subscript.sh diff --git a/setup-scripts/support/crontab b/setup-scripts/support/crontab index ad766264..29c8ec1a 100644 --- a/setup-scripts/support/crontab +++ b/setup-scripts/support/crontab @@ -4,3 +4,4 @@ 30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env 0 4 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1 >> /home/user/skynet-webportal/logs/db_backup_`date +"%Y-%m-%d-%H%M"`.log 2 > &1 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200 +15 * * * * /home/user/skynet-webportal/scripts/nginx-prune.sh From aca71c245e662dc9dff57a125c8f0c3834809712 Mon Sep 17 00:00:00 2001 From: Filip Rysavy <29089732+firyx@users.noreply.github.com> Date: Mon, 6 Dec 2021 14:13:56 +0100 Subject: [PATCH 21/23] Add comments to nginx pruning script --- scripts/lib/nginx-prune-cache-subscript.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/lib/nginx-prune-cache-subscript.sh b/scripts/lib/nginx-prune-cache-subscript.sh index bf1e1e44..5994f49f 100755 --- a/scripts/lib/nginx-prune-cache-subscript.sh +++ b/scripts/lib/nginx-prune-cache-subscript.sh @@ -9,10 +9,16 @@ MAX_KEEP_FILE_SIZE=1000000000 total=0 +# We sort files by time, newest files are first. Format is: +# time (last modification as seconds since Epoch), filepath, size (bytes) find /home/user/skynet-webportal/docker/data/nginx/cache -type f -exec stat -c "%Y %n %s" {} + | sort -rgk1 | while read line do size=$(echo $line | awk '{print $3}') new_total=$(($total + $size)) + + # We always delete all files larger than MAX_KEEP_FILE_SIZE. + # We keep all files smaller than MAX_KEEP_FILE_SIZE when cache size is + # below MAX_CACHE_DIR_SIZE, then we delete also smaller files. if (("$size" <= "$MAX_KEEP_FILE_SIZE" && "$total" < "$new_total")) then total=$new_total From 1c1f3c6ec113de0abd1f706b5455e4f146a69a9d Mon Sep 17 00:00:00 2001 From: Filip Rysavy <29089732+firyx@users.noreply.github.com> Date: Mon, 6 Dec 2021 16:13:00 +0100 Subject: [PATCH 22/23] Fix condition mistake --- scripts/lib/nginx-prune-cache-subscript.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lib/nginx-prune-cache-subscript.sh b/scripts/lib/nginx-prune-cache-subscript.sh index 5994f49f..99edb899 100755 --- a/scripts/lib/nginx-prune-cache-subscript.sh +++ b/scripts/lib/nginx-prune-cache-subscript.sh @@ -19,7 +19,7 @@ do # We always delete all files larger than MAX_KEEP_FILE_SIZE. # We keep all files smaller than MAX_KEEP_FILE_SIZE when cache size is # below MAX_CACHE_DIR_SIZE, then we delete also smaller files. - if (("$size" <= "$MAX_KEEP_FILE_SIZE" && "$total" < "$new_total")) + if (("$size" <= "$MAX_KEEP_FILE_SIZE" && "$new_total" < "$MAX_CACHE_DIR_SIZE")) then total=$new_total continue From 9fe0628fe72867f9890af926631271d5cc7cc30d Mon Sep 17 00:00:00 2001 From: Matthew Sevey Date: Mon, 6 Dec 2021 13:28:30 -0500 Subject: [PATCH 23/23] Mute check failures unless server is down --- setup-scripts/health-checker.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup-scripts/health-checker.py b/setup-scripts/health-checker.py index b66459a8..091ebe30 100755 --- a/setup-scripts/health-checker.py +++ b/setup-scripts/health-checker.py @@ -226,7 +226,8 @@ async def check_health(): message += "{}/{} CRITICAL checks failed over the last {} hours! ".format( critical_checks_failed, critical_checks_total, CHECK_HOURS ) - force_notify = True + # Disabling as it creates notification fatigue. + # force_notify = True else: message += "All {} critical checks passed. ".format(critical_checks_total) @@ -234,7 +235,8 @@ async def check_health(): message += "{}/{} extended checks failed over the last {} hours! ".format( extended_checks_failed, extended_checks_total, CHECK_HOURS ) - force_notify = True + # Disabling as it creates notification fatigue. + # force_notify = True else: message += "All {} extended checks passed. ".format(extended_checks_total)