diff --git a/docker-compose.yml b/docker-compose.yml index 9c0d59d5..af9b53a2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,8 +64,6 @@ services: logging: *default-logging env_file: - .env - environment: - - SKYD_DISK_CACHE_ENABLED=${SKYD_DISK_CACHE_ENABLED:-true} volumes: - ./docker/data/nginx/cache:/data/nginx/cache - ./docker/data/nginx/blocker:/data/nginx/blocker diff --git a/docker/nginx/conf.d/include/location-skylink b/docker/nginx/conf.d/include/location-skylink index da4727c7..cfacf5c1 100644 --- a/docker/nginx/conf.d/include/location-skylink +++ b/docker/nginx/conf.d/include/location-skylink @@ -1,6 +1,5 @@ include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/proxy-buffer; -include /etc/nginx/conf.d/include/proxy-cache-downloads; include /etc/nginx/conf.d/include/track-download; limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time @@ -9,62 +8,10 @@ limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time # this is important because we want only one format in cache keys and logs set_by_lua_block $skylink { return require("skynet.skylink").parse(ngx.var.skylink) } -# $skylink_v1 and $skylink_v2 variables default to the same value but in case the requested skylink was: -# a) skylink v1 - it would not matter, no additional logic is executed -# b) skylink v2 - in a lua block below we will resolve the skylink v2 into skylink v1 and update -# $skylink_v1 variable so then the proxy request to skyd can be cached in nginx (proxy_cache_key -# in proxy-cache-downloads includes $skylink_v1 as a part of the cache key) -set $skylink_v1 $skylink; -set $skylink_v2 $skylink; - -# variable for Skynet-Proof header that we need to inject -# into a response if the request was for skylink v2 -set $skynet_proof ''; - # default download rate to unlimited set $limit_rate 0; access_by_lua_block { - -- the block below only makes sense if we are using nginx cache - if not ngx.var.skyd_disk_cache_enabled then - local httpc = require("resty.http").new() - - -- detect whether requested skylink is v2 - local isBase32v2 = string.len(ngx.var.skylink) == 55 and string.sub(ngx.var.skylink, 0, 2) == "04" - local isBase64v2 = string.len(ngx.var.skylink) == 46 and string.sub(ngx.var.skylink, 0, 2) == "AQ" - - if isBase32v2 or isBase64v2 then - -- 10.10.10.10 points to sia service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/resolve/" .. ngx.var.skylink_v2, { - headers = { ["User-Agent"] = "Sia-Agent" } - }) - - -- print error and exit with 500 or exit with response if status is not 200 - if err or (res and res.status ~= ngx.HTTP_OK) then - ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status - ngx.header["content-type"] = "text/plain" - ngx.say(err or res.body) - return ngx.exit(ngx.status) - end - - local json = require('cjson') - local resolve = json.decode(res.body) - ngx.var.skylink_v1 = resolve.skylink - ngx.var.skynet_proof = res.headers["Skynet-Proof"] - end - - -- check if skylink v1 is present on blocklist (compare hashes) - if require("skynet.blocklist").is_blocked(ngx.var.skylink_v1) then - return require("skynet.blocklist").exit_illegal() - end - - -- if skylink is found on nocache list then set internal nocache variable - -- to tell nginx that it should not try and cache this file (too large) - if ngx.shared.nocache:get(ngx.var.skylink_v1) then - ngx.var.nocache = "1" - end - end - if require("skynet.account").accounts_enabled() then -- check if portal is in authenticated only mode if require("skynet.account").is_access_unauthorized() then @@ -84,36 +31,10 @@ access_by_lua_block { end } -header_filter_by_lua_block { - ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. ngx.var.skynet_portal_domain - ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. ngx.var.skynet_server_domain - - -- the block below only makes sense if we are using nginx cache - if not ngx.var.skyd_disk_cache_enabled then - -- not empty skynet_proof means this is a skylink v2 request - -- so we should replace the Skynet-Proof header with the one - -- we got from /skynet/resolve/ endpoint, otherwise we would - -- be serving cached empty v1 skylink Skynet-Proof header - if ngx.var.skynet_proof and ngx.var.skynet_proof ~= "" then - ngx.header["Skynet-Proof"] = ngx.var.skynet_proof - end - - -- add skylink to nocache list if it exceeds 1GB (1e+9 bytes) threshold - -- (content length can be nil for already cached files - we can ignore them) - if ngx.header["Content-Length"] and tonumber(ngx.header["Content-Length"]) > 1e+9 then - ngx.shared.nocache:set(ngx.var.skylink_v1, ngx.header["Content-Length"]) - end - end -} - limit_rate_after 512k; limit_rate $limit_rate; proxy_read_timeout 600; proxy_set_header User-Agent: Sia-Agent; -# in case the requested skylink was v2 and we already resolved it to skylink v1, we are going to pass resolved -# skylink v1 to skyd to save that extra skylink v2 lookup in skyd but in turn, in case skyd returns a redirect -# we need to rewrite the skylink v1 to skylink v2 in the location header with proxy_redirect -proxy_redirect $skylink_v1 $skylink_v2; -proxy_pass http://sia:9980/skynet/skylink/$skylink_v1$path$is_args$args; +proxy_pass http://sia:9980/skynet/skylink/$skylink$path$is_args$args; diff --git a/docker/nginx/conf.d/include/proxy-cache-downloads b/docker/nginx/conf.d/include/proxy-cache-downloads deleted file mode 100644 index 85aeeb9e..00000000 --- a/docker/nginx/conf.d/include/proxy-cache-downloads +++ /dev/null @@ -1,21 +0,0 @@ -proxy_cache skynet; # cache name -proxy_cache_key $skylink_v1$path$arg_format$arg_attachment$arg_start$arg_end$http_range; # unique cache key -proxy_cache_min_uses 3; # cache after 3 uses -proxy_cache_valid 200 206 307 308 48h; # keep 200, 206, 307 and 308 responses valid for up to 2 days -add_header X-Proxy-Cache $upstream_cache_status; # add response header to indicate cache hits and misses - -# map skyd env variable value to "1" for true and "0" for false (expected by proxy_no_cache) -set_by_lua_block $skyd_disk_cache_enabled { - return os.getenv("SKYD_DISK_CACHE_ENABLED") == "true" and "1" or "0" -} - -# bypass - this will bypass cache hit on request (status BYPASS) -# but still stores file in cache if cache conditions are met -proxy_cache_bypass $cookie_nocache $arg_nocache $skyd_disk_cache_enabled; - -# no cache - this will ignore cache on request (status MISS) -# and does not store file in cache under no condition -set_if_empty $nocache "0"; - -# disable cache when nocache is set or skyd cache is enabled -proxy_no_cache $nocache $skyd_disk_cache_enabled; diff --git a/docker/nginx/conf.d/server.local.conf b/docker/nginx/conf.d/server.local.conf deleted file mode 100644 index 8a487a53..00000000 --- a/docker/nginx/conf.d/server.local.conf +++ /dev/null @@ -1,9 +0,0 @@ -server { - # local server - do not expose this port externally - listen 8000; - - # secure traffic by limiting to only local networks - include /etc/nginx/conf.d/include/local-network-only; - - include /etc/nginx/conf.d/server/server.local; -} diff --git a/docker/nginx/conf.d/server/server.dnslink b/docker/nginx/conf.d/server/server.dnslink index 32e454cc..cf385a1d 100644 --- a/docker/nginx/conf.d/server/server.dnslink +++ b/docker/nginx/conf.d/server/server.dnslink @@ -38,8 +38,6 @@ location / { end ngx.var.skylink = require("skynet.skylink").parse(ngx.var.skylink) - ngx.var.skylink_v1 = ngx.var.skylink - ngx.var.skylink_v2 = ngx.var.skylink } include /etc/nginx/conf.d/include/location-skylink; diff --git a/docker/nginx/conf.d/server/server.local b/docker/nginx/conf.d/server/server.local deleted file mode 100644 index 3a8ac118..00000000 --- a/docker/nginx/conf.d/server/server.local +++ /dev/null @@ -1,37 +0,0 @@ -include /etc/nginx/conf.d/include/init-optional-variables; - -location /skynet/blocklist { - client_max_body_size 10m; # increase max body size to account for large lists - client_body_buffer_size 10m; # force whole body to memory so we can read it - - content_by_lua_block { - local httpc = require("resty.http").new() - - ngx.req.read_body() -- ensure the post body data is read before using get_body_data - - -- proxy blocklist update request - -- 10.10.10.10 points to sia service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/blocklist", { - method = "POST", - body = ngx.req.get_body_data(), - headers = { - ["Content-Type"] = "application/x-www-form-urlencoded", - ["Authorization"] = require("skynet.utils").authorization_header(), - ["User-Agent"] = "Sia-Agent", - } - }) - - -- print error and exit with 500 or exit with response if status is not 204 - if err or (res and res.status ~= ngx.HTTP_NO_CONTENT) then - ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status - ngx.header["content-type"] = "text/plain" - ngx.say(err or res.body) - return ngx.exit(ngx.status) - end - - require("skynet.blocklist").reload() - - ngx.status = ngx.HTTP_NO_CONTENT - return ngx.exit(ngx.status) - } -} diff --git a/docker/nginx/libs/skynet/blocklist.lua b/docker/nginx/libs/skynet/blocklist.lua deleted file mode 100644 index 29f53032..00000000 --- a/docker/nginx/libs/skynet/blocklist.lua +++ /dev/null @@ -1,66 +0,0 @@ -local _M = {} - -function _M.reload() - local httpc = require("resty.http").new() - - -- fetch blocklist records (all blocked skylink hashes) - -- 10.10.10.10 points to sia service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/blocklist", { - headers = { - ["User-Agent"] = "Sia-Agent", - } - }) - - -- fail whole request in case this request failed, we want to make sure - -- the blocklist is pre cached before serving first skylink - if err or (res and res.status ~= ngx.HTTP_OK) then - ngx.log(ngx.ERR, "Failed skyd service request /skynet/blocklist: ", err or ("[HTTP " .. res.status .. "] " .. res.body)) - ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status - ngx.header["content-type"] = "text/plain" - ngx.say(err or res.body) - return ngx.exit(ngx.status) - elseif res and res.status == ngx.HTTP_OK then - local json = require('cjson') - local data = json.decode(res.body) - - -- mark all existing entries as expired - ngx.shared.blocklist:flush_all() - - -- check if blocklist is table (it is null when empty) - if type(data.blocklist) == "table" then - -- set all cache entries one by one (resets expiration) - for i, hash in ipairs(data.blocklist) do - ngx.shared.blocklist:set(hash, true) - end - end - - -- ensure that init flag is persisted - ngx.shared.blocklist:set("__init", true) - - -- remove all leftover expired entries - ngx.shared.blocklist:flush_expired() - end -end - -function _M.is_blocked(skylink) - -- make sure that blocklist has been preloaded - if not ngx.shared.blocklist:get("__init") then _M.reload() end - - -- hash skylink before comparing it with blocklist - local hash = require("skynet.skylink").hash(skylink) - - -- we need to use get_stale because we are expiring previous - -- entries when the blocklist is reloading and we still want - -- to block them until the reloading is finished - return ngx.shared.blocklist:get_stale(hash) == true -end - --- exit with 416 illegal content status code -function _M.exit_illegal() - ngx.status = ngx.HTTP_ILLEGAL - ngx.header["content-type"] = "text/plain" - ngx.say("Unavailable For Legal Reasons") - return ngx.exit(ngx.status) -end - -return _M diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf index 7f43d192..95bb61ac 100644 --- a/docker/nginx/nginx.conf +++ b/docker/nginx/nginx.conf @@ -31,7 +31,6 @@ env SERVER_DOMAIN; env PORTAL_MODULES; env ACCOUNTS_LIMIT_ACCESS; env SIA_API_PASSWORD; -env SKYD_DISK_CACHE_ENABLED; events { worker_connections 8192; @@ -75,20 +74,10 @@ http { # proxy cache definition proxy_cache_path /data/nginx/cache levels=1:2 keys_zone=skynet:10m max_size=50g min_free=100g inactive=48h use_temp_path=off; - # create a shared blocklist dictionary with size of 30 megabytes - # estimated capacity of 1 megabyte dictionary is 3500 blocklist entries - # that gives us capacity of around 100k entries in 30 megabyte dictionary - lua_shared_dict blocklist 30m; - - # create a shared dictionary to fill with skylinks that should not - # be cached due to the large size or some other reasons - lua_shared_dict nocache 10m; - # this runs before forking out nginx worker processes init_by_lua_block { require "cjson" require "resty.http" - require "skynet.blocklist" require "skynet.skylink" require "skynet.utils" } diff --git a/scripts/README.md b/scripts/README.md index 2085eff7..e7b909b4 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -29,12 +29,6 @@ the health check. The `portal-upgrade.sh` script upgrades the docker images for a portal and clears and leftover images. -**nginx-prune.sh**\ -The `nginx-prune.sh` script deletes all entries from nginx cache larger than -the given size and smaller entries until nginx cache disk size is smaller than -the given cache size limit. Both values are configured in -`lib/nginx-prune-cache-subscript.sh`. The script doesn't require `sudo`. - ## Webportal Upgrade Procedures TODO... diff --git a/scripts/blocklist-skylink.sh b/scripts/blocklist-skylink.sh index ee19c9c2..cb60bbdb 100755 --- a/scripts/blocklist-skylink.sh +++ b/scripts/blocklist-skylink.sh @@ -34,18 +34,16 @@ else skylinks=("$1") # just single skylink passed as input argument fi -# get local nginx ip adress -nginx_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx) +# get local skyd ip adress +ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia) + +# get sia api password either from env variable if exists or from apipassword file in sia-data directory +apipassword=$(docker exec sia sh -c '[ ! -z "${SIA_API_PASSWORD}" ] && echo ${SIA_API_PASSWORD} || $(cat /sia-data/apipassword | tr -d '\n')') # iterate over provided skylinks and block them one by one for skylink in "${skylinks[@]}"; do - printf "Blocking ${skylink} ... " - status_code=$(curl --write-out '%{http_code}' --silent --output /dev/null --data "{\"add\":[\"$skylink\"]}" "http://${nginx_ip}:8000/skynet/blocklist") + echo "> Blocking ${skylink} ... " - # print blocklist response status code - if [ $status_code = "204" ]; then - echo "done" - else - echo "error $status_code" - fi + # POST /skynet/blocklist always returns 200 and in case of failure print error message + curl -A Sia-Agent -u "":${apipassword} --data "{\"add\":[\"$skylink\"]}" "http://${ipaddress}:9980/skynet/blocklist" done diff --git a/scripts/lib/nginx-prune-cache-subscript.sh b/scripts/lib/nginx-prune-cache-subscript.sh deleted file mode 100755 index 99edb899..00000000 --- a/scripts/lib/nginx-prune-cache-subscript.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/local/bin/bash - -# This subscript is expected to be run inside docker container using 'bash' -# image. The image is based on Alpine Linux. It's tools (find, stat, awk, sort) -# are non-standard versions from BusyBox. - -MAX_CACHE_DIR_SIZE=20000000000 -MAX_KEEP_FILE_SIZE=1000000000 - -total=0 - -# We sort files by time, newest files are first. Format is: -# time (last modification as seconds since Epoch), filepath, size (bytes) -find /home/user/skynet-webportal/docker/data/nginx/cache -type f -exec stat -c "%Y %n %s" {} + | sort -rgk1 | while read line -do - size=$(echo $line | awk '{print $3}') - new_total=$(($total + $size)) - - # We always delete all files larger than MAX_KEEP_FILE_SIZE. - # We keep all files smaller than MAX_KEEP_FILE_SIZE when cache size is - # below MAX_CACHE_DIR_SIZE, then we delete also smaller files. - if (("$size" <= "$MAX_KEEP_FILE_SIZE" && "$new_total" < "$MAX_CACHE_DIR_SIZE")) - then - total=$new_total - continue - fi - - filename=$(echo $line | awk '{print $2}') - rm $filename -done diff --git a/scripts/nginx-prune.sh b/scripts/nginx-prune.sh deleted file mode 100755 index ea5a1384..00000000 --- a/scripts/nginx-prune.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# We execute the nginx cache pruning subscript from docker container so that we -# can run the pruning script in user crontab without sudo. -docker run --rm -v /home/user:/home/user bash /home/user/skynet-webportal/scripts/lib/nginx-prune-cache-subscript.sh - -# Some cache files are deleted, but are kept open, we hot reload nginx to get -# them closed and removed from filesystem. -docker exec nginx nginx -s reload diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py index 09711605..8bd9d2dc 100755 --- a/setup-scripts/blocklist-airtable.py +++ b/setup-scripts/blocklist-airtable.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -from bot_utils import setup, send_msg +from bot_utils import get_api_password, setup, send_msg from random import randint from time import sleep @@ -11,6 +11,8 @@ import asyncio import requests import json +from requests.auth import HTTPBasicAuth + setup() @@ -38,14 +40,14 @@ def exec(command): async def block_skylinks_from_airtable(): - # Get nginx's IP before doing anything else. If this step fails we don't + # Get sia IP before doing anything else. If this step fails we don't # need to continue with the execution of the script. ipaddress = exec( - "docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx" + "docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia" ) if ipaddress == "": - print("Nginx's IP could not be detected. Exiting.") + print("Skyd IP could not be detected. Exiting.") return print("Pulling blocked skylinks from Airtable via api integration") @@ -117,11 +119,13 @@ async def block_skylinks_from_airtable(): print( "Sending /skynet/blocklist request with " + str(len(skylinks)) - + " skylinks to siad through nginx" + + " skylinks to siad" ) response = requests.post( - "http://" + ipaddress + ":8000/skynet/blocklist", + "http://" + ipaddress + ":9980/skynet/blocklist", data=json.dumps({"add": skylinks}), + headers={"User-Agent": "Sia-Agent"}, + auth=HTTPBasicAuth("", get_api_password()), ) if response.status_code != 200: @@ -153,5 +157,5 @@ loop.run_until_complete(run_checks()) # --- BASH EQUIVALENT # skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')") -# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx) +# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia) # curl --data "{\"add\" : ${skylinks}}" "${ipaddress}:8000/skynet/blocklist" diff --git a/setup-scripts/support/crontab b/setup-scripts/support/crontab index 2181d09c..7fffdfb9 100644 --- a/setup-scripts/support/crontab +++ b/setup-scripts/support/crontab @@ -5,4 +5,3 @@ 44 5 * * * /home/user/skynet-webportal/scripts/backup-aws-s3.sh 1>>/home/user/skynet-webportal/logs/backup-aws-s3.log 2>>/home/user/skynet-webportal/logs/backup-aws-s3.log 6 13 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1>>/home/user/skynet-webportal/logs/db_backup.log 2>>/home/user/skynet-webportal/logs/db_backup.log 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200 -15 * * * * /home/user/skynet-webportal/scripts/nginx-prune.sh