Merge remote-tracking branch 'origin/master' into skyd-config-health-check
This commit is contained in:
Karol Wypchlo
commit
26cfaeec65
|
|
@ -0,0 +1 @@
|
|||
- expose generic skylink serving endpoint on domain aliases
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
- Remove hardcoded server list from `blocklist-skylink.sh` so it removes server
|
||||
list duplication and can also be called from Ansible.
|
||||
|
|
@ -75,7 +75,7 @@ access_by_lua_block {
|
|||
end
|
||||
}
|
||||
|
||||
# we proxy to another nginx location rather than directly to siad because we don't want to deal with caching here
|
||||
# we proxy to another nginx location rather than directly to siad because we do not want to deal with caching here
|
||||
proxy_pass https://127.0.0.1/$skylink$path$is_args$args;
|
||||
|
||||
# in case siad returns location header, we need to replace the skylink with the domain name
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ if ($request_method = PURGE) {
|
|||
limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time
|
||||
|
||||
# $skylink_v1 and $skylink_v2 variables default to the same value but in case the requested skylink was:
|
||||
# a) skylink v1 - it wouldn't matter, no additional logic is executed
|
||||
# a) skylink v1 - it would not matter, no additional logic is executed
|
||||
# b) skylink v2 - in a lua block below we will resolve the skylink v2 into skylink v1 and update
|
||||
# $skylink_v1 variable so then the proxy request to skyd can be cached in nginx (proxy_cache_key
|
||||
# in proxy-cache-downloads includes $skylink_v1 as a part of the cache key)
|
||||
|
|
@ -91,7 +91,7 @@ limit_rate $limit_rate;
|
|||
proxy_read_timeout 600;
|
||||
proxy_set_header User-Agent: Sia-Agent;
|
||||
|
||||
# in case the requested skylink was v2 and we already resolved it to skylink v1, we're going to pass resolved
|
||||
# in case the requested skylink was v2 and we already resolved it to skylink v1, we are going to pass resolved
|
||||
# skylink v1 to skyd to save that extra skylink v2 lookup in skyd but in turn, in case skyd returns a redirect
|
||||
# we need to rewrite the skylink v1 to skylink v2 in the location header with proxy_redirect
|
||||
proxy_redirect $skylink_v1 $skylink_v2;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
lua_shared_dict dnslink 10m;
|
||||
|
||||
server {
|
||||
listen 80 default_server;
|
||||
listen [::]:80 default_server;
|
||||
|
|
|
|||
|
|
@ -5,21 +5,40 @@ location / {
|
|||
set $path $uri;
|
||||
|
||||
rewrite_by_lua_block {
|
||||
local httpc = require("resty.http").new()
|
||||
local cache = ngx.shared.dnslink
|
||||
local cache_value = cache:get(ngx.var.host)
|
||||
|
||||
-- 10.10.10.55 points to dnslink-api service (alias not available when using resty-http)
|
||||
local res, err = httpc:request_uri("http://10.10.10.55:3100/dnslink/" .. ngx.var.host)
|
||||
if cache_value == nil then
|
||||
local httpc = require("resty.http").new()
|
||||
|
||||
if err or (res and res.status ~= ngx.HTTP_OK) then
|
||||
ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status
|
||||
ngx.header["content-type"] = "text/plain"
|
||||
ngx.say(err or res.body)
|
||||
ngx.exit(ngx.status)
|
||||
-- 10.10.10.55 points to dnslink-api service (alias not available when using resty-http)
|
||||
local res, err = httpc:request_uri("http://10.10.10.55:3100/dnslink/" .. ngx.var.host)
|
||||
|
||||
if err or (res and res.status ~= ngx.HTTP_OK) then
|
||||
-- check whether we can fallback to regular skylink request
|
||||
local match_skylink = ngx.re.match(ngx.var.uri, "^/([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?")
|
||||
|
||||
if match_skylink then
|
||||
ngx.var.skylink = match_skylink[1]
|
||||
ngx.var.path = match_skylink[2] or "/"
|
||||
else
|
||||
ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status
|
||||
ngx.header["content-type"] = "text/plain"
|
||||
ngx.say(err or res.body)
|
||||
ngx.exit(ngx.status)
|
||||
end
|
||||
else
|
||||
ngx.var.skylink = res.body
|
||||
|
||||
local cache_ttl = 300 -- 5 minutes cache expire time
|
||||
cache:set(ngx.var.host, ngx.var.skylink, cache_ttl)
|
||||
end
|
||||
else
|
||||
ngx.var.skylink = res.body
|
||||
ngx.var.skylink_v1 = ngx.var.skylink
|
||||
ngx.var.skylink_v2 = ngx.var.skylink
|
||||
ngx.var.skylink = cache_value
|
||||
end
|
||||
|
||||
ngx.var.skylink_v1 = ngx.var.skylink
|
||||
ngx.var.skylink_v2 = ngx.var.skylink
|
||||
}
|
||||
|
||||
include /etc/nginx/conf.d/include/location-skylink;
|
||||
|
|
|
|||
|
|
@ -5,8 +5,7 @@
|
|||
"license": "SEE LICENSE IN LICENSE.md",
|
||||
"dependencies": {
|
||||
"express": "^4.17.1",
|
||||
"is-valid-domain": "^0.1.2",
|
||||
"node-cache": "^5.1.2"
|
||||
"is-valid-domain": "^0.1.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"prettier": "^2.4.1"
|
||||
|
|
|
|||
|
|
@ -1,14 +1,11 @@
|
|||
const dns = require("dns");
|
||||
const express = require("express");
|
||||
const NodeCache = require("node-cache");
|
||||
const isValidDomain = require("is-valid-domain");
|
||||
|
||||
const host = process.env.DNSLINK_API_HOSTNAME || "0.0.0.0";
|
||||
const port = Number(process.env.DNSLINK_API_PORT) || 3100;
|
||||
const cacheTTL = Number(process.env.DNSLINK_API_CACHE_TTL) || 300; // default to 5 minutes
|
||||
|
||||
const server = express();
|
||||
const cache = new NodeCache({ stdTTL: cacheTTL });
|
||||
|
||||
const dnslinkNamespace = "skynet-ns";
|
||||
const dnslinkRegExp = new RegExp(`^dnslink=/${dnslinkNamespace}/.+$`);
|
||||
|
|
@ -23,10 +20,6 @@ server.get("/dnslink/:name", async (req, res) => {
|
|||
return failure(`"${req.params.name}" is not a valid domain`);
|
||||
}
|
||||
|
||||
if (cache.has(req.params.name)) {
|
||||
return success(cache.get(req.params.name));
|
||||
}
|
||||
|
||||
const lookup = `_dnslink.${req.params.name}`;
|
||||
|
||||
dns.resolveTxt(lookup, (error, records) => {
|
||||
|
|
@ -65,8 +58,6 @@ server.get("/dnslink/:name", async (req, res) => {
|
|||
|
||||
const skylink = matchSkylink[1];
|
||||
|
||||
cache.set(req.params.name, skylink);
|
||||
|
||||
console.log(`${req.params.name} => ${skylink}`);
|
||||
|
||||
return success(skylink);
|
||||
|
|
|
|||
|
|
@ -36,11 +36,6 @@ bytes@3.1.0:
|
|||
resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6"
|
||||
integrity sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==
|
||||
|
||||
clone@2.x:
|
||||
version "2.1.2"
|
||||
resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f"
|
||||
integrity sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18=
|
||||
|
||||
content-disposition@0.5.3:
|
||||
version "0.5.3"
|
||||
resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.3.tgz#e130caf7e7279087c5616c2007d0485698984fbd"
|
||||
|
|
@ -257,13 +252,6 @@ negotiator@0.6.2:
|
|||
resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb"
|
||||
integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==
|
||||
|
||||
node-cache@^5.1.2:
|
||||
version "5.1.2"
|
||||
resolved "https://registry.yarnpkg.com/node-cache/-/node-cache-5.1.2.tgz#f264dc2ccad0a780e76253a694e9fd0ed19c398d"
|
||||
integrity sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==
|
||||
dependencies:
|
||||
clone "2.x"
|
||||
|
||||
on-finished@~2.3.0:
|
||||
version "2.3.0"
|
||||
resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947"
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
#! /usr/bin/env bash
|
||||
|
||||
# This script is meant to be used when manually adding a skylink to the
|
||||
# blocklist on all the skynet web portals. The automatic script that is used to
|
||||
# continuously sync a google sheets list with the blocklist on the web portals
|
||||
# is /setup-scripts/blocklist-airtable.py
|
||||
# This script adds a skylink to the sia blocklist and removes the skylink from
|
||||
# nginx cache. The script should be run locally on each skynet webportal
|
||||
# server. The automatic script that is used to continuously sync an Airtable
|
||||
# sheet list with the blocklist on the web portals is
|
||||
# /setup-scripts/blocklist-airtable.py
|
||||
|
||||
set -e # exit on first error
|
||||
|
||||
|
|
@ -17,54 +18,39 @@ fi
|
|||
#########################################################
|
||||
skylinks=()
|
||||
if test -f "$1"; then
|
||||
OLDIFS=$IFS
|
||||
IFS=','
|
||||
line_number=1
|
||||
while read line
|
||||
|
||||
# Read file including the last line even when it doesn't end with newline
|
||||
while IFS="" read -r line || [ -n "$line" ];
|
||||
do
|
||||
if [[ $line =~ ([a-zA-Z0-9_-]{46}) ]]; then
|
||||
skylinks+=("$BASH_REMATCH")
|
||||
if [[ $line =~ (^[a-zA-Z0-9_-]{46}$) ]]; then
|
||||
skylinks+=("$line")
|
||||
else
|
||||
echo "Incorrect skylink at line ${line_number}: $line" && exit 1
|
||||
fi
|
||||
let line_number+=1
|
||||
done < $1;
|
||||
IFS=$OLDIFS
|
||||
else
|
||||
skylinks=("$1") # just single skylink passed as input argument
|
||||
fi
|
||||
|
||||
#########################################################################
|
||||
# iterate through all servers, block the skylinks and purge it from cache
|
||||
#########################################################################
|
||||
declare -a servers=( "eu-ger-1.siasky.net" "eu-ger-2.siasky.net" "eu-ger-3.siasky.net" "eu-ger-4.siasky.net" "eu-ger-5.siasky.net" "eu-ger-6.siasky.net" "eu-ger-7.siasky.net" "eu-ger-8.siasky.net"
|
||||
"eu-ger-9.siasky.net" "eu-ger-10.siasky.net" "eu-ger-11.siasky.net" "eu-ger-12.siasky.net"
|
||||
"eu-fin-1.siasky.net" "eu-fin-2.siasky.net" "eu-fin-3.siasky.net" "eu-fin-4.siasky.net" "eu-fin-5.siasky.net" "eu-fin-6.siasky.net" "eu-fin-7-siasky.net" "eu-fin-8.siasky.net"
|
||||
"eu-fin-9.siasky.net" "eu-fin-10.siasky.net" "eu-fin-11.siasky.net" "eu-fin-12.siasky.net" "eu-fin-13.siasky.net" "eu-fin-14.siasky.net" "eu-fin-15.siasky.net"
|
||||
"eu-pol-1.siasky.net" "eu-pol-2.siasky.net" "eu-pol-3.siasky.net" "eu-pol-4.siasky.net" "eu-pol-5.siasky.net"
|
||||
"us-ny-1.siasky.net" "us-ny-2.siasky.net"
|
||||
"us-or-1.siasky.net" "us-or-2.siasky.net"
|
||||
"us-la-1.siasky.net" "us-la-2.siasky.net" "us-la-3.siasky.net"
|
||||
"us-pa-1.siasky.net" "us-pa-2.siasky.net"
|
||||
"us-va-1.siasky.net" "us-va-2.siasky.net" "us-va-3.siasky.net" "us-va-4.siasky.net" "us-va-5.siasky.net" "us-va-6.siasky.net"
|
||||
"as-hk-1.siasky.net" "as-sp-1.siasky.net" "as-sp-2.siasky.net"
|
||||
"siasky.xyz" "dev1.siasky.dev" "dev2.siasky.dev" "dev3.siasky.dev")
|
||||
for server in "${servers[@]}";
|
||||
for skylink in "${skylinks[@]}";
|
||||
do
|
||||
for skylink in "${skylinks[@]}";
|
||||
do
|
||||
echo ".. ⌁ Blocking skylink ${skylink} on ${server}"
|
||||
|
||||
# Add to blocklist
|
||||
ssh -q -t user@${server} "docker exec sia siac skynet blocklist add ${skylink}"
|
||||
|
||||
# Remove from NGINX cache
|
||||
cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'"
|
||||
ssh -q -t user@${server} "docker exec -it nginx bash -c ${cached_files_command} | xargs -r rm"
|
||||
|
||||
echo ".. ⌁ Skylink ${skylink} Blocked on ${server}"
|
||||
echo "--------------------------------------------"
|
||||
done
|
||||
echo ".. ⌁ Blocking skylink ${skylink}"
|
||||
|
||||
# Add to Sia blocklist
|
||||
docker exec sia siac skynet blocklist add "${skylink}"
|
||||
|
||||
# Remove from NGINX cache
|
||||
# NOTE:
|
||||
# If there are changes to how the NGINX cache is being cleared, the same
|
||||
# changes need to be applied to the /setup-scripts/blocklist-airtable.py
|
||||
# script.
|
||||
cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'"
|
||||
docker exec -it nginx bash -c "${cached_files_command} | xargs -r rm"
|
||||
|
||||
echo ".. ⌁ Skylink ${skylink} Blocked"
|
||||
echo "--------------------------------------------"
|
||||
done
|
||||
|
||||
echo "✓ All done !"
|
||||
|
|
|
|||
|
|
@ -141,6 +141,10 @@ async def block_skylinks_from_airtable():
|
|||
)
|
||||
return await send_msg(message, force_notify=False)
|
||||
|
||||
# Remove from NGINX cache
|
||||
# NOTE:
|
||||
# If there are changes to how the NGINX cache is being cleared, the same
|
||||
# changes need to be applied to the /scripts/blocklist-skylink.sh script.
|
||||
print("Searching nginx cache for blocked files")
|
||||
cached_files_count = 0
|
||||
batch_size = 1000
|
||||
|
|
|
|||
Reference in New Issue