diff --git a/changelog/items/other/add-abuse-config.md b/changelog/items/other/add-abuse-config.md new file mode 100644 index 00000000..51a55918 --- /dev/null +++ b/changelog/items/other/add-abuse-config.md @@ -0,0 +1 @@ +- Add abuse report configuration diff --git a/changelog/items/other/dump-disk-space-usage.md b/changelog/items/other/dump-disk-space-usage.md new file mode 100644 index 00000000..3c1c3147 --- /dev/null +++ b/changelog/items/other/dump-disk-space-usage.md @@ -0,0 +1,2 @@ +- Dump disk space usage when health-checker script disables portal due to + critical free disk space. \ No newline at end of file diff --git a/changelog/items/other/min-free-param.md b/changelog/items/other/min-free-param.md new file mode 100644 index 00000000..0a6239f3 --- /dev/null +++ b/changelog/items/other/min-free-param.md @@ -0,0 +1 @@ +- Set `min_free` parameter on the `proxy_cache_path` directive to `100g` diff --git a/changelog/items/other/trim-airtable-skylinks.md b/changelog/items/other/trim-airtable-skylinks.md new file mode 100644 index 00000000..a63a90d0 --- /dev/null +++ b/changelog/items/other/trim-airtable-skylinks.md @@ -0,0 +1 @@ +- Add trimming Airtable skylinks from Takedown Request table. \ No newline at end of file diff --git a/dc b/dc index 60418cb8..c041317a 100755 --- a/dc +++ b/dc @@ -13,6 +13,11 @@ for i in $(seq 1 ${#PORTAL_MODULES}); do COMPOSE_FILES+=" -f docker-compose.mongodb.yml -f docker-compose.accounts.yml" fi + # blocker module - alias "b" + if [[ ${PORTAL_MODULES:i-1:1} == "b" ]]; then + COMPOSE_FILES+=" -f docker-compose.blocker.yml" + fi + # jaeger module - alias "j" if [[ ${PORTAL_MODULES:i-1:1} == "j" ]]; then COMPOSE_FILES+=" -f docker-compose.jaeger.yml" diff --git a/docker-compose.accounts.yml b/docker-compose.accounts.yml index ac6e72cc..3d2b5c89 100644 --- a/docker-compose.accounts.yml +++ b/docker-compose.accounts.yml @@ -30,7 +30,7 @@ services: - .env environment: - ACCOUNTS_EMAIL_URI=${ACCOUNTS_EMAIL_URI} - - ACCOUNTS_JWKS_FILE=/data/jwks.json + - ACCOUNTS_JWKS_FILE=/conf/jwks.json - COOKIE_DOMAIN=${COOKIE_DOMAIN} - COOKIE_HASH_KEY=${COOKIE_HASH_KEY} - COOKIE_ENC_KEY=${COOKIE_ENC_KEY} @@ -45,6 +45,7 @@ services: - SKYNET_ACCOUNTS_LOG_LEVEL=${SKYNET_ACCOUNTS_LOG_LEVEL:-info} volumes: - ./docker/data/accounts:/data + - ./docker/accounts/conf:/accounts/conf expose: - 3000 networks: diff --git a/docker-compose.blocker.yml b/docker-compose.blocker.yml new file mode 100644 index 00000000..b76d2e43 --- /dev/null +++ b/docker-compose.blocker.yml @@ -0,0 +1,26 @@ +version: "3.7" + +x-logging: &default-logging + driver: json-file + options: + max-size: "10m" + max-file: "3" + +services: + blocker: + build: + context: ./docker/blocker + dockerfile: Dockerfile + container_name: blocker + restart: unless-stopped + logging: *default-logging + env_file: + - .env + expose: + - 4000 + networks: + shared: + ipv4_address: 10.10.10.102 + depends_on: + - mongo + - sia diff --git a/docker/blocker/Dockerfile b/docker/blocker/Dockerfile new file mode 100644 index 00000000..3dbc2f61 --- /dev/null +++ b/docker/blocker/Dockerfile @@ -0,0 +1,16 @@ +FROM golang:1.16.7 +LABEL maintainer="NebulousLabs " + +ENV GOOS linux +ENV GOARCH amd64 + +ARG branch=main + +WORKDIR /root + +RUN git clone --single-branch --branch ${branch} https://github.com/SkynetLabs/blocker.git && \ + cd blocker && \ + go mod download && \ + make release + +ENTRYPOINT ["blocker"] diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index 07be6fcb..4402fc07 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -90,6 +90,29 @@ location /health-check { proxy_pass http://10.10.10.60:3100; # hardcoded ip because health-check waits for nginx } +location /abuse/ { + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' 'https://0404guluqu38oaqapku91ed11kbhkge55smh9lhjukmlrj37lfpm8no.siasky.net'; + + add_header 'Access-Control-Allow-Credentials' 'true'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'; + + # pre-flight info is valid for 20 days + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + + proxy_pass http://10.10.10.102:4000/; +} + +location /report-abuse { + # TODO: do a proxy_pass + return https://0404guluqu38oaqapku91ed11kbhkge55smh9lhjukmlrj37lfpm8no.siasky.net; +} + location /hns { # match the request_uri and extract the hns domain and anything that is passed in the uri after it # example: /hns/something/foo/bar matches: diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf index a6883cb8..c8da6b0f 100644 --- a/docker/nginx/nginx.conf +++ b/docker/nginx/nginx.conf @@ -70,7 +70,7 @@ http { proxy_http_version 1.1; # proxy cache definition - proxy_cache_path /data/nginx/cache levels=1:2 keys_zone=skynet:10m max_size=50g inactive=48h use_temp_path=off; + proxy_cache_path /data/nginx/cache levels=1:2 keys_zone=skynet:10m max_size=50g min_free=100g inactive=48h use_temp_path=off; # this runs before forking out nginx worker processes init_by_lua_block { diff --git a/packages/website/static/terms.pdf b/packages/website/static/terms.pdf index 85b5983e..61561255 100644 Binary files a/packages/website/static/terms.pdf and b/packages/website/static/terms.pdf differ diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py index e3aecd05..31d8ee19 100755 --- a/setup-scripts/blocklist-airtable.py +++ b/setup-scripts/blocklist-airtable.py @@ -100,8 +100,8 @@ async def block_skylinks_from_airtable(): entry["fields"].get(AIRTABLE_FIELD, "") for entry in data["records"] ] skylinks = [ - skylink for skylink in skylinks if skylink - ] # filter empty skylinks, most likely empty rows + skylink.strip() for skylink in skylinks if skylink + ] # filter empty skylinks, most likely empty rows, trim whitespace offset = data.get("offset") diff --git a/setup-scripts/disk-usage-dump.sh b/setup-scripts/disk-usage-dump.sh new file mode 100755 index 00000000..9326d459 --- /dev/null +++ b/setup-scripts/disk-usage-dump.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# Dumps disk usage to stdout or to the file +# +# Parameters: +# - $1 (optional): Filename to append the output to. +# +# Usage: +# - Dump disk usage to stdout: +# ./disk-usage-dump.sh +# +# - Dump disk usage appending to th file: +# ./disk-usage-dump.sh my-log-file.log +# +# Use docker container to get root (script can be run under regular user, no +# need for sudo) + +dump () { + echo + echo "### Disk usage dump at $(date) ###" + + # Free disk space + echo + df -h /home/user + + # Home dirs + echo + echo "Home dirs:" + docker run -v /home/user:/home/user alpine:3.15.0 du -hs /home/user/* + + # Docker data dirs + echo + echo "Docker data dirs:" + docker run -v /home/user:/home/user alpine:3.15.0 du -hs /home/user/skynet-webportal/docker/data/* + + # Largest dirs/files + echo + echo "Dirs or files over 1GB (first 100):" + docker run -v /home/user:/home/user alpine:3.15.0 du -h /home/user | grep -E "^[0-9]+\.?[0-9]*G" | sort -r -n | head -100 +} + +# Check argument is present +if [ -z "$1" ]; then + # Dump to stdout + dump +else + # Handle log paths + filename=$(basename "$1") + dirname=$(dirname "$1") + abs_dirname=$(realpath "$dirname") + + # Make sure log dir exists + mkdir -p "$abs_dirname" + + # Append to file + { + dump + } >> "$abs_dirname/$filename" 2>&1 +fi diff --git a/setup-scripts/health-checker.py b/setup-scripts/health-checker.py index 7c5e9d94..b66459a8 100755 --- a/setup-scripts/health-checker.py +++ b/setup-scripts/health-checker.py @@ -37,6 +37,9 @@ GB = 1 << 30 # 1 GiB in bytes FREE_DISK_SPACE_THRESHOLD = 100 * GB FREE_DISK_SPACE_THRESHOLD_CRITICAL = 60 * GB +# Disk usage dump log file (relative to this .py script). +DISK_USAGE_DUMP_LOG = "../../devops/disk-monitor/disk-usage-dump.log" + setup() @@ -69,7 +72,9 @@ async def check_load_average(): load_av = re.match(pattern, uptime_string).group(1) if float(load_av) > 10: message = "High system load detected in uptime output: {}".format(uptime_string) - await send_msg(message, force_notify=True) + # Disabling pings until we have metrics solution and process to better + # address + await send_msg(message, force_notify=False) # check_disk checks the amount of free space on the /home partition and issues @@ -103,11 +108,18 @@ async def check_disk(): message = "CRITICAL! Very low disk space: {}GiB, **siad stopped**!".format( free_space_gb ) + + # dump disk usage + script_dir = os.path.dirname(os.path.realpath(sys.argv[0])) + os.popen( + script_dir + "/disk-usage-dump.sh " + script_dir + "/" + DISK_USAGE_DUMP_LOG + ) + inspect = os.popen("docker inspect sia").read().strip() inspect_json = json.loads(inspect) if inspect_json[0]["State"]["Running"] is True: # mark portal as unhealthy - os.popen("docker exec health-check cli/disable") + os.popen("docker exec health-check cli disable 'critical free disk space'") time.sleep(300) # wait 5 minutes to propagate dns changes os.popen("docker stop sia") # stop sia container return await send_msg(message, force_notify=True)