diff --git a/changelog/items/other/skylinks-block-batch.md b/changelog/items/other/skylinks-block-batch.md new file mode 100644 index 00000000..e617082b --- /dev/null +++ b/changelog/items/other/skylinks-block-batch.md @@ -0,0 +1 @@ +- Block skylinks in batches to improve performance. \ No newline at end of file diff --git a/scripts/blocklist-skylink.sh b/scripts/blocklist-skylink.sh index cf14f138..93a36a61 100755 --- a/scripts/blocklist-skylink.sh +++ b/scripts/blocklist-skylink.sh @@ -8,6 +8,9 @@ set -e # exit on first error +# Number of skylinks to block within one batch +BATCH_SIZE=1000 + if [ -z "$1" ]; then echo "Please provide either a skylink or file with skylinks separated by new lines" && exit 1 fi @@ -34,23 +37,45 @@ else skylinks=("$1") # just single skylink passed as input argument fi -for skylink in "${skylinks[@]}"; -do - echo ".. ⌁ Blocking skylink ${skylink}" - - # Add to Sia blocklist - docker exec sia siac skynet blocklist add "${skylink}" - - # Remove from NGINX cache - # NOTE: - # If there are changes to how the NGINX cache is being cleared, the same - # changes need to be applied to the /setup-scripts/blocklist-airtable.py - # script. - cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'" - docker exec -it nginx bash -c "${cached_files_command} | xargs -r rm" - - echo ".. ⌁ Skylink ${skylink} Blocked" - echo "--------------------------------------------" +# Block skylinks in batches +skylinks_len=${#skylinks[@]} +for (( i = 0; i < $skylinks_len; i++ )); do + # Add skylink to batch + skylink="${skylinks[$i]}" + echo ".. ⌁ Adding skylink ${skylink} to batch..." + batch_skylinks+=("$skylink") + + # For performance reasons on each iteration we do not block a single + # skylink, but we block skylinks in batches with BATCH_SIZE size mainly + # because of nginx cache search. + # If (batch len == batch size) or (we have last batch): + if (( ${#batch_skylinks[@]} == $BATCH_SIZE || $i == $skylinks_len - 1 )); then + echo "--------------------------------------------" + + # Add to Sia blocklist + echo "Blocking batch skylinks in skyd..." + skylinks_space_separated="$(IFS=' '; echo "${batch_skylinks[*]}")" + docker exec sia siac skynet blocklist add $skylinks_space_separated + + # Remove from NGINX cache + # NOTE: + # If there are changes to how the NGINX cache is being cleared, the same + # changes need to be applied to the /setup-scripts/blocklist-airtable.py + # script. + echo "Removing batch skylinks from Nginx cache..." + skylinks_pipe_separated="$(IFS='|'; echo "${batch_skylinks[*]}")" + cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ($skylinks_pipe_separated)'" + docker exec -it nginx bash -c "${cached_files_command} | xargs -r rm" + + # Clear batch + batch_skylinks=() + + echo "--------------------------------------------" + fi done +# Hot reload Nginx to get rid of deleted open files +echo "Hot reloading nginx..." +docker exec nginx nginx -s reload + echo "✓ All done !" diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py index 31d8ee19..2443b488 100755 --- a/setup-scripts/blocklist-airtable.py +++ b/setup-scripts/blocklist-airtable.py @@ -171,6 +171,9 @@ async def block_skylinks_from_airtable(): if cached_files_count == 0: return print("No nginx cached files matching blocked skylinks were found") + else: + print("Hot reloading nginx") + exec('docker exec nginx nginx -s reload') message = ( "Purged " + str(cached_files_count) + " blocklisted files from nginx cache"