LumeWeb
/
skynet-webportal
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

clean up manual blocklist script

This commit is contained in:
Karol Wypchlo 2022-01-10 22:35:04 +01:00
parent 5cf5acb4e2
commit dac4d57b12
No known key found for this signature in database
GPG Key ID: B515DE9EEBE241E1
1 changed files with 17 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
scripts/blocklist-skylink.sh
View File

@ -1,18 +1,15 @@
#! /usr/bin/env bash #! /usr/bin/env bash
# This script adds a skylink to the sia blocklist and removes the skylink from # This script is for manual skylink blocking. It accepts either a single
# nginx cache. The script should be run locally on each skynet webportal # skylink or a file containing list of skylinks. The script is intented
# server. The automatic script that is used to continuously sync an Airtable # for manual use and it should be run locally on each skynet webportal server.
# sheet list with the blocklist on the web portals is # The automatic script that is used to continuously sync an Airtable sheet
# /setup-scripts/blocklist-airtable.py # list with the blocklist on the web portals is /setup-scripts/blocklist-airtable.py
set -e # exit on first error set -e # exit on first error
# Number of skylinks to block within one batch
BATCH_SIZE=1000
if [ -z "$1" ]; then if [ -z "$1" ]; then
echo "Please provide either a skylink or file with skylinks separated by new lines" && exit 1 echo "Please provide either a skylink or a file with skylinks separated by new lines" && exit 1
fi fi
######################################################### #########################################################
@ -37,45 +34,18 @@ else
skylinks=("$1") # just single skylink passed as input argument skylinks=("$1") # just single skylink passed as input argument
fi fi
# Block skylinks in batches # get local nginx ip adress
skylinks_len=${#skylinks[@]} nginx_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx)
for (( i = 0; i < $skylinks_len; i++ )); do
# Add skylink to batch
skylink="${skylinks[$i]}"
echo ".. ⌁ Adding skylink ${skylink} to batch..."
batch_skylinks+=("$skylink")
# For performance reasons on each iteration we do not block a single # iterate over provided skylinks and block them one by one
# skylink, but we block skylinks in batches with BATCH_SIZE size mainly for skylink in "${skylinks[@]}"; do
# because of nginx cache search. printf "Blocking ${skylink} ... "
# If (batch len == batch size) or (we have last batch): status_code=$(curl --write-out '%{http_code}' --silent --output /dev/null --data "{\"add\":[\"$skylink\"]}" "http://${nginx_ip}:8000/skynet/blocklist")
if (( ${#batch_skylinks[@]} == $BATCH_SIZE || $i == $skylinks_len - 1 )); then
echo "--------------------------------------------"
# Add to Sia blocklist # print blocklist response status code
echo "Blocking batch skylinks in skyd..." if [ $status_code = "204" ]; then
skylinks_space_separated="$(IFS=' '; echo "${batch_skylinks[*]}")" echo "done"
docker exec sia siac skynet blocklist add $skylinks_space_separated else
echo "error $status_code"
# Remove from NGINX cache
# NOTE:
# If there are changes to how the NGINX cache is being cleared, the same
# changes need to be applied to the /setup-scripts/blocklist-airtable.py
# script.
echo "Removing batch skylinks from Nginx cache..."
skylinks_pipe_separated="$(IFS='|'; echo "${batch_skylinks[*]}")"
cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ($skylinks_pipe_separated)'"
docker exec -it nginx bash -c "${cached_files_command} | xargs -r rm"
# Clear batch
batch_skylinks=()
echo "--------------------------------------------"
fi fi
done done
# Hot reload Nginx to get rid of deleted open files
echo "Hot reloading nginx..."
docker exec nginx nginx -s reload
echo "✓ All done !"