Merge branch 'master' into add-shell-scripts-lint

.github/workflows/lint-shell-scripts.yml

@@ -0,0 +1,14 @@
+name: Lint Shell Scripts
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  shellcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ludeeus/action-shellcheck@1.1.0

docker-compose.jaeger.yml

@@ -21,7 +21,7 @@ services:
       - JAEGER_REPORTER_LOG_SPANS=false
 
   jaeger-agent:
-    image: jaegertracing/jaeger-agent:1.38.1
+    image: jaegertracing/jaeger-agent:1.38.0
     command:
       [
         "--reporter.grpc.host-port=jaeger-collector:14250",
@@ -43,7 +43,7 @@ services:
       - jaeger-collector
 
   jaeger-collector:
-    image: jaegertracing/jaeger-collector:1.38.1
+    image: jaegertracing/jaeger-collector:1.38.0
     entrypoint: /wait_to_start.sh
     container_name: jaeger-collector
     restart: on-failure
@@ -68,7 +68,7 @@ services:
       - elasticsearch
 
   jaeger-query:
-    image: jaegertracing/jaeger-query:1.38.1
+    image: jaegertracing/jaeger-query:1.38.0
     entrypoint: /wait_to_start.sh
     container_name: jaeger-query
     restart: on-failure

docker-compose.malware-scanner.yml

@@ -17,6 +17,10 @@ services:
       - ./docker/clamav/clamd.conf:/etc/clamav/clamd.conf:ro
     expose:
       - 3310 # NEVER expose this outside of the local network!
+    deploy:
+      resources:
+        limits:
+          cpus: "${CLAMAV_CPU:-0.50}"
     networks:
       shared:
         ipv4_address: 10.10.10.100

docker-compose.mongodb.yml

@@ -14,7 +14,7 @@ services:
       - MONGODB_PASSWORD=${SKYNET_DB_PASS}
 
   mongo:
-    image: mongo:4.4.17
+    image: mongo:4.4.16
     command: --keyFile=/data/mgkey --replSet=${SKYNET_DB_REPLICASET:-skynet} --setParameter ShardingTaskExecutorPoolMinSize=10
     container_name: mongo
     restart: unless-stopped

docker-compose.pinner.yml

@@ -10,7 +10,7 @@ services:
   pinner:
     # uncomment "build" and comment out "image" to build from sources
     # build: https://github.com/SkynetLabs/pinner.git#main
-    image: skynetlabs/pinner:0.7.8
+    image: skynetlabs/pinner:0.7.6
     container_name: pinner
     restart: unless-stopped
     logging: *default-logging

docker-compose.yml

@@ -60,7 +60,7 @@ services:
     #
     # make sure that the file has 0400 permissions with:
     # chmod 0400 ./docker/data/certbot/cloudflare.ini
-    image: certbot/dns-route53:v1.31.0
+    image: certbot/dns-route53:v1.30.0
     entrypoint: sh /entrypoint.sh
     container_name: certbot
     restart: unless-stopped
@@ -108,7 +108,7 @@ services:
     # build:
     #   context: https://github.com/SkynetLabs/webportal-website.git#main
     #   dockerfile: Dockerfile
-    image: skynetlabs/webportal-website:0.2.3
+    image: skynetlabs/webportal-website:0.2.2
     container_name: website
     restart: unless-stopped
     logging: *default-logging
@@ -142,7 +142,7 @@ services:
     # build:
     #   context: https://github.com/SkynetLabs/webportal-handshake-api.git#main
     #   dockerfile: Dockerfile
-    image: skynetlabs/webportal-handshake-api:0.1.3
+    image: skynetlabs/webportal-handshake-api:0.1.2
     container_name: handshake-api
     restart: unless-stopped
     logging: *default-logging
@@ -166,7 +166,7 @@ services:
     # build:
     #   context: https://github.com/SkynetLabs/webportal-dnslink-api.git#main
     #   dockerfile: Dockerfile
-    image: skynetlabs/webportal-dnslink-api:0.2.1
+    image: skynetlabs/webportal-dnslink-api:0.2.0
     container_name: dnslink-api
     restart: unless-stopped
     logging: *default-logging
@@ -181,7 +181,7 @@ services:
     # build:
     #   context: https://github.com/SkynetLabs/webportal-health-check.git#main
     #   dockerfile: Dockerfile
-    image: skynetlabs/webportal-health-check:1.0.0
+    image: skynetlabs/webportal-health-check:0.5.0
     container_name: health-check
     restart: unless-stopped
     logging: *default-logging

scripts/backup-aws-s3.sh

@@ -50,7 +50,6 @@ aws s3 sync --no-progress /home/user/skynet-webportal/docker/data/nginx/logs s3:
 # generate and sync skylinks dump
 SKYLINKS_PATH=logs/skylinks/$(date +"%Y-%m-%d").log
 mkdir -p /home/user/skynet-webportal/logs/skylinks # ensure path exists
-find /home/user/skynet-webportal/logs/skylinks -type f -mtime +7 -delete # delete skylink dumps older than 7 days
 docker exec sia siac skynet ls --recursive --alert-suppress > /home/user/skynet-webportal/${SKYLINKS_PATH}
 aws s3 cp --no-progress /home/user/skynet-webportal/${SKYLINKS_PATH} s3://${BUCKET_NAME}/${SERVER_PREFIX}/${SKYLINKS_PATH}
 

scripts/blocklist-skylink.sh

@@ -26,8 +26,8 @@ if test -f "$1"; then
         else
             echo "Incorrect skylink at line ${line_number}: $line" && exit 1
         fi
-        let line_number+=1
+        (( line_number++ ))
-    done < $1;
+    done < "$1";
 else
     skylinks=("$1") # just single skylink passed as input argument
 fi

scripts/portal-down.sh

@@ -20,7 +20,7 @@ fi
 
 countdown() {
     local secs=$1
-    while [ $secs -gt 0 ]; do
+    while [ "$secs" -gt 0 ]; do
         echo -ne "Waiting $secs\033[0K\r"
         sleep 1
         : $((secs--))
@@ -28,10 +28,10 @@ countdown() {
 }
 
 # delay disabling the portal
-countdown $delay
+countdown "$delay"
 
 # stop health-check so the server is taken our of load balancer
-docker exec health-check cli disable $reason
+docker exec health-check cli disable "$reason"
 
 # then wait 5 minutes for the load balancer to propagate the dns records
-countdown $timeout
+countdown "$timeout"

setup-scripts/health-checker.py

@@ -232,7 +232,8 @@ async def check_health():
         message += "{}/{} CRITICAL checks failed over the last {} hours! ".format(
             critical_checks_failed, critical_checks_total, CHECK_HOURS
         )
-        force_notify = True
+        # Disabling as it creates notification fatigue.
+        # force_notify = True
     else:
         message += "All {} critical checks passed. ".format(critical_checks_total)
 
@@ -240,7 +241,8 @@ async def check_health():
         message += "{}/{} extended checks failed over the last {} hours! ".format(
             extended_checks_failed, extended_checks_total, CHECK_HOURS
         )
-        force_notify = True
+        # Disabling as it creates notification fatigue.
+        # force_notify = True
     else:
         message += "All {} extended checks passed. ".format(extended_checks_total)
 

setup-scripts/logrotate.d/skynet-webportal-nginx

@@ -1,16 +0,0 @@
-/home/user/skynet-webportal/docker/data/nginx/logs/*.log {
-    daily
-    rotate 3650
-    minsize 500M
-    create 644 root root
-    notifempty
-    dateext
-    missingok
-    compress
-    compressoptions --best
-    delaycompress
-    sharedscripts
-    postrotate
-        docker exec nginx nginx -s reopen
-    endscript
-}

setup-scripts/logrotate.d/skynet-webportal-pinner

@@ -1,11 +0,0 @@
-/home/user/skynet-webportal/docker/data/pinner/*.log {
-    daily
-    rotate 10
-    minsize 100M
-    copytruncate
-    notifempty
-    dateext
-    missingok
-    compress
-    compressoptions --best
-}

setup-scripts/logrotate.d/skynet-webportal-skyd

@@ -1,12 +0,0 @@
-/home/user/skynet-webportal/docker/data/sia/*.log 
-/home/user/skynet-webportal/docker/data/sia/*/*.log {
-    daily
-    rotate 10
-    minsize 100M
-    copytruncate
-    notifempty
-    dateext
-    missingok
-    compress
-    compressoptions --best
-}

setup-scripts/serverload.sh

@@ -33,7 +33,7 @@ fi
 # Write the output in an infinite loop.
 while true; do
     # CPU usage
-    cpu=$(echo $[100-$(vmstat 1 2|tail -1|awk '{print $15}')]) 
+    cpu=$((100-$(vmstat 1 2|tail -1|awk '{print $15}')))
     sed -i "1iCPU: ${cpu}" $logfile
 
     # Disk Usage