From 5cf5acb4e2789383382b35c6e0a936997cf08df8 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Thu, 23 Dec 2021 12:55:47 +0100 Subject: [PATCH 1/2] do not prune blocklist manually --- docker/nginx/conf.d/server/server.local | 3 ++ setup-scripts/blocklist-airtable.py | 54 +++++-------------------- 2 files changed, 13 insertions(+), 44 deletions(-) diff --git a/docker/nginx/conf.d/server/server.local b/docker/nginx/conf.d/server/server.local index 1de4ab72..3a8ac118 100644 --- a/docker/nginx/conf.d/server/server.local +++ b/docker/nginx/conf.d/server/server.local @@ -1,6 +1,9 @@ include /etc/nginx/conf.d/include/init-optional-variables; location /skynet/blocklist { + client_max_body_size 10m; # increase max body size to account for large lists + client_body_buffer_size 10m; # force whole body to memory so we can read it + content_by_lua_block { local httpc = require("resty.http").new() diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py index 9ec3314c..9ec81952 100755 --- a/setup-scripts/blocklist-airtable.py +++ b/setup-scripts/blocklist-airtable.py @@ -122,63 +122,30 @@ async def block_skylinks_from_airtable(): ) await send_msg(message, file=("\n".join(invalid_skylinks))) - apipassword = exec("docker exec sia cat /sia-data/apipassword") ipaddress = exec( - "docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia" + "docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx" ) - print("Sending blocklist request to siad") + print("Sending blocklist request to siad through nginx") response = requests.post( - "http://" + ipaddress + ":9980/skynet/blocklist", - auth=("", apipassword), - headers={"user-agent": "Sia-Agent"}, + "http://" + ipaddress + ":8000/skynet/blocklist", data=json.dumps({"add": skylinks}), ) - if response.status_code == 204: - print("Siad blocklist successfully updated with provided skylink") - else: + print(json.dumps({"add": skylinks})) + + if response.status_code != 204: status_code = str(response.status_code) response_text = response.text or "empty response" message = ( - "Siad blocklist endpoint responded with code " + "Airtable blocklist request responded with code " + status_code + ": " + response_text ) return await send_msg(message, force_notify=False) - # Remove from NGINX cache - # NOTE: - # If there are changes to how the NGINX cache is being cleared, the same - # changes need to be applied to the /scripts/blocklist-skylink.sh script. - print("Searching nginx cache for blocked files") - cached_files_count = 0 - batch_size = 1000 - for i in range(0, len(skylinks), batch_size): - cached_files_command = ( - "find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: (" - + "|".join(skylinks[i : i + batch_size]) - + ")'" - ) - cached_files_count += int( - exec( - 'docker exec nginx bash -c "' - + cached_files_command - + ' | xargs -r rm -v | wc -l"' - ) - ) - - if cached_files_count == 0: - return print("No nginx cached files matching blocked skylinks were found") - else: - print("Hot reloading nginx") - exec("docker exec nginx nginx -s reload") - - message = ( - "Purged " + str(cached_files_count) + " blocklisted files from nginx cache" - ) - return await send_msg(message) + return await send_msg("Siad blocklist successfully updated with provided skylink") loop = asyncio.get_event_loop() @@ -186,6 +153,5 @@ loop.run_until_complete(run_checks()) # --- BASH EQUIVALENT # skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')") -# apipassword=$(docker exec sia cat /sia-data/apipassword) -# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia) -# curl -A "Sia-Agent" --user "":"${apipassword}" --data "{\"add\" : ${skylinks}}" "${ipaddress}:9980/skynet/blocklist" +# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx) +# curl --data "{\"add\" : ${skylinks}}" "${ipaddress}:8000/skynet/blocklist" From dac4d57b1236a6250ce21ae782a3cd63a26dd643 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 10 Jan 2022 22:35:04 +0100 Subject: [PATCH 2/2] clean up manual blocklist script --- scripts/blocklist-skylink.sh | 64 ++++++++++-------------------------- 1 file changed, 17 insertions(+), 47 deletions(-) diff --git a/scripts/blocklist-skylink.sh b/scripts/blocklist-skylink.sh index 93a36a61..ee19c9c2 100755 --- a/scripts/blocklist-skylink.sh +++ b/scripts/blocklist-skylink.sh @@ -1,18 +1,15 @@ #! /usr/bin/env bash -# This script adds a skylink to the sia blocklist and removes the skylink from -# nginx cache. The script should be run locally on each skynet webportal -# server. The automatic script that is used to continuously sync an Airtable -# sheet list with the blocklist on the web portals is -# /setup-scripts/blocklist-airtable.py +# This script is for manual skylink blocking. It accepts either a single +# skylink or a file containing list of skylinks. The script is intented +# for manual use and it should be run locally on each skynet webportal server. +# The automatic script that is used to continuously sync an Airtable sheet +# list with the blocklist on the web portals is /setup-scripts/blocklist-airtable.py set -e # exit on first error -# Number of skylinks to block within one batch -BATCH_SIZE=1000 - if [ -z "$1" ]; then - echo "Please provide either a skylink or file with skylinks separated by new lines" && exit 1 + echo "Please provide either a skylink or a file with skylinks separated by new lines" && exit 1 fi ######################################################### @@ -37,45 +34,18 @@ else skylinks=("$1") # just single skylink passed as input argument fi -# Block skylinks in batches -skylinks_len=${#skylinks[@]} -for (( i = 0; i < $skylinks_len; i++ )); do - # Add skylink to batch - skylink="${skylinks[$i]}" - echo ".. ⌁ Adding skylink ${skylink} to batch..." - batch_skylinks+=("$skylink") +# get local nginx ip adress +nginx_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx) - # For performance reasons on each iteration we do not block a single - # skylink, but we block skylinks in batches with BATCH_SIZE size mainly - # because of nginx cache search. - # If (batch len == batch size) or (we have last batch): - if (( ${#batch_skylinks[@]} == $BATCH_SIZE || $i == $skylinks_len - 1 )); then - echo "--------------------------------------------" +# iterate over provided skylinks and block them one by one +for skylink in "${skylinks[@]}"; do + printf "Blocking ${skylink} ... " + status_code=$(curl --write-out '%{http_code}' --silent --output /dev/null --data "{\"add\":[\"$skylink\"]}" "http://${nginx_ip}:8000/skynet/blocklist") - # Add to Sia blocklist - echo "Blocking batch skylinks in skyd..." - skylinks_space_separated="$(IFS=' '; echo "${batch_skylinks[*]}")" - docker exec sia siac skynet blocklist add $skylinks_space_separated - - # Remove from NGINX cache - # NOTE: - # If there are changes to how the NGINX cache is being cleared, the same - # changes need to be applied to the /setup-scripts/blocklist-airtable.py - # script. - echo "Removing batch skylinks from Nginx cache..." - skylinks_pipe_separated="$(IFS='|'; echo "${batch_skylinks[*]}")" - cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ($skylinks_pipe_separated)'" - docker exec -it nginx bash -c "${cached_files_command} | xargs -r rm" - - # Clear batch - batch_skylinks=() - - echo "--------------------------------------------" + # print blocklist response status code + if [ $status_code = "204" ]; then + echo "done" + else + echo "error $status_code" fi done - -# Hot reload Nginx to get rid of deleted open files -echo "Hot reloading nginx..." -docker exec nginx nginx -s reload - -echo "✓ All done !"