Merge remote-tracking branch 'origin/master' into accounts-api-refactor

changelog/items/other/add-abuse-config.md

@@ -0,0 +1 @@
+- Add abuse report configuration

changelog/items/other/dump-disk-space-usage.md

@@ -0,0 +1,2 @@
+- Dump disk space usage when health-checker script disables portal due to
+  critical free disk space.

changelog/items/other/min-free-param.md

@@ -0,0 +1 @@
+- Set `min_free` parameter on the `proxy_cache_path` directive to `100g`

changelog/items/other/trim-airtable-skylinks.md

@@ -0,0 +1 @@
+- Add trimming Airtable skylinks from Takedown Request table.

dc

@@ -13,6 +13,11 @@ for i in $(seq 1 ${#PORTAL_MODULES}); do
     COMPOSE_FILES+=" -f docker-compose.mongodb.yml -f docker-compose.accounts.yml"
   fi
 
+  # blocker module - alias "b"
+  if [[ ${PORTAL_MODULES:i-1:1} == "b" ]]; then
+    COMPOSE_FILES+=" -f docker-compose.blocker.yml"
+  fi
+
   # jaeger module - alias "j"
   if [[ ${PORTAL_MODULES:i-1:1} == "j" ]]; then
     COMPOSE_FILES+=" -f docker-compose.jaeger.yml"

docker-compose.accounts.yml

@@ -30,7 +30,7 @@ services:
       - .env
     environment:
       - ACCOUNTS_EMAIL_URI=${ACCOUNTS_EMAIL_URI}
-      - ACCOUNTS_JWKS_FILE=/data/jwks.json
+      - ACCOUNTS_JWKS_FILE=/conf/jwks.json
       - COOKIE_DOMAIN=${COOKIE_DOMAIN}
       - COOKIE_HASH_KEY=${COOKIE_HASH_KEY}
       - COOKIE_ENC_KEY=${COOKIE_ENC_KEY}
@@ -45,6 +45,7 @@ services:
       - SKYNET_ACCOUNTS_LOG_LEVEL=${SKYNET_ACCOUNTS_LOG_LEVEL:-info}
     volumes:
       - ./docker/data/accounts:/data
+      - ./docker/accounts/conf:/accounts/conf
     expose:
       - 3000
     networks:

docker-compose.blocker.yml

@@ -0,0 +1,26 @@
+version: "3.7"
+
+x-logging: &default-logging
+  driver: json-file
+  options:
+    max-size: "10m"
+    max-file: "3"
+
+services:
+  blocker:
+    build:
+      context: ./docker/blocker
+      dockerfile: Dockerfile
+    container_name: blocker
+    restart: unless-stopped
+    logging: *default-logging
+    env_file:
+      - .env
+    expose:
+      - 4000
+    networks:
+      shared:
+        ipv4_address: 10.10.10.102
+    depends_on:
+      - mongo
+      - sia

docker/blocker/Dockerfile

@@ -0,0 +1,16 @@
+FROM golang:1.16.7
+LABEL maintainer="NebulousLabs <devs@nebulous.tech>"
+
+ENV GOOS linux
+ENV GOARCH amd64
+
+ARG branch=main
+
+WORKDIR /root
+
+RUN git clone --single-branch --branch ${branch} https://github.com/SkynetLabs/blocker.git && \
+    cd blocker && \
+    go mod download && \
+    make release
+
+ENTRYPOINT ["blocker"]

docker/nginx/conf.d/server/server.api

@@ -90,6 +90,29 @@ location /health-check {
     proxy_pass http://10.10.10.60:3100; # hardcoded ip because health-check waits for nginx
 }
 
+location /abuse/ {
+    if ($request_method = 'OPTIONS') {
+        add_header 'Access-Control-Allow-Origin' 'https://0404guluqu38oaqapku91ed11kbhkge55smh9lhjukmlrj37lfpm8no.siasky.net';
+
+        add_header 'Access-Control-Allow-Credentials' 'true';
+        add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+        add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+
+        # pre-flight info is valid for 20 days
+        add_header 'Access-Control-Max-Age' 1728000;
+        add_header 'Content-Type' 'text/plain charset=UTF-8';
+        add_header 'Content-Length' 0;
+        return 204;
+    }
+
+    proxy_pass http://10.10.10.102:4000/;
+}
+
+location /report-abuse {
+    # TODO: do a proxy_pass
+    return https://0404guluqu38oaqapku91ed11kbhkge55smh9lhjukmlrj37lfpm8no.siasky.net;
+}
+
 location /hns {
     # match the request_uri and extract the hns domain and anything that is passed in the uri after it
     # example: /hns/something/foo/bar matches:

docker/nginx/nginx.conf

@@ -70,7 +70,7 @@ http {
     proxy_http_version 1.1;
 
     # proxy cache definition
-    proxy_cache_path /data/nginx/cache levels=1:2 keys_zone=skynet:10m max_size=50g inactive=48h use_temp_path=off;
+    proxy_cache_path /data/nginx/cache levels=1:2 keys_zone=skynet:10m max_size=50g min_free=100g inactive=48h use_temp_path=off;
 
     # this runs before forking out nginx worker processes
     init_by_lua_block { 

setup-scripts/blocklist-airtable.py

@@ -100,8 +100,8 @@ async def block_skylinks_from_airtable():
             entry["fields"].get(AIRTABLE_FIELD, "") for entry in data["records"]
         ]
         skylinks = [
-            skylink for skylink in skylinks if skylink
-        ]  # filter empty skylinks, most likely empty rows
+            skylink.strip() for skylink in skylinks if skylink
+        ]  # filter empty skylinks, most likely empty rows, trim whitespace
 
         offset = data.get("offset")
 

setup-scripts/disk-usage-dump.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# Dumps disk usage to stdout or to the file
+#
+# Parameters:
+# - $1 (optional): Filename to append the output to.
+#
+# Usage:
+# - Dump disk usage to stdout:
+#   ./disk-usage-dump.sh
+#
+# - Dump disk usage appending to th file:
+#   ./disk-usage-dump.sh my-log-file.log
+#
+# Use docker container to get root (script can be run under regular user, no
+# need for sudo)
+
+dump () {
+    echo
+    echo "### Disk usage dump at $(date) ###"
+
+    # Free disk space
+    echo
+    df -h /home/user
+
+    # Home dirs
+    echo
+    echo "Home dirs:"
+    docker run -v /home/user:/home/user alpine:3.15.0 du -hs /home/user/*
+
+    # Docker data dirs
+    echo
+    echo "Docker data dirs:"
+    docker run -v /home/user:/home/user alpine:3.15.0 du -hs /home/user/skynet-webportal/docker/data/*
+
+    # Largest dirs/files
+    echo
+    echo "Dirs or files over 1GB (first 100):"
+    docker run -v /home/user:/home/user alpine:3.15.0 du -h /home/user | grep -E "^[0-9]+\.?[0-9]*G" | sort -r -n | head -100
+}
+
+# Check argument is present
+if [ -z "$1" ]; then
+    # Dump to stdout
+    dump
+else
+    # Handle log paths
+    filename=$(basename "$1")
+    dirname=$(dirname "$1")
+    abs_dirname=$(realpath "$dirname")
+
+    # Make sure log dir exists
+    mkdir -p "$abs_dirname"
+
+    # Append to file
+    {
+        dump
+    } >> "$abs_dirname/$filename" 2>&1
+fi

setup-scripts/health-checker.py

@@ -37,6 +37,9 @@ GB = 1 << 30  # 1 GiB in bytes
 FREE_DISK_SPACE_THRESHOLD = 100 * GB
 FREE_DISK_SPACE_THRESHOLD_CRITICAL = 60 * GB
 
+# Disk usage dump log file (relative to this .py script).
+DISK_USAGE_DUMP_LOG = "../../devops/disk-monitor/disk-usage-dump.log"
+
 setup()
 
 
@@ -69,7 +72,9 @@ async def check_load_average():
     load_av = re.match(pattern, uptime_string).group(1)
     if float(load_av) > 10:
         message = "High system load detected in uptime output: {}".format(uptime_string)
-        await send_msg(message, force_notify=True)
+        # Disabling pings until we have metrics solution and process to better
+        # address
+        await send_msg(message, force_notify=False)
 
 
 # check_disk checks the amount of free space on the /home partition and issues
@@ -103,11 +108,18 @@ async def check_disk():
         message = "CRITICAL! Very low disk space: {}GiB, **siad stopped**!".format(
             free_space_gb
         )
+
+        # dump disk usage
+        script_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
+        os.popen(
+            script_dir + "/disk-usage-dump.sh " + script_dir + "/" + DISK_USAGE_DUMP_LOG
+        )
+
         inspect = os.popen("docker inspect sia").read().strip()
         inspect_json = json.loads(inspect)
         if inspect_json[0]["State"]["Running"] is True:
             # mark portal as unhealthy
-            os.popen("docker exec health-check cli/disable")
+            os.popen("docker exec health-check cli disable 'critical free disk space'")
             time.sleep(300)  # wait 5 minutes to propagate dns changes
             os.popen("docker stop sia")  # stop sia container
         return await send_msg(message, force_notify=True)