From ed19736e2296a5c670f9d840633fbd43e7450fc8 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Sun, 18 Jul 2021 09:24:45 +0200 Subject: [PATCH] smaller batch --- setup-scripts/blocklist-airtable.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py index 8b0d9705..379841eb 100755 --- a/setup-scripts/blocklist-airtable.py +++ b/setup-scripts/blocklist-airtable.py @@ -97,15 +97,14 @@ async def block_skylinks_from_airtable(): print("Searching nginx cache for blocked files") cached_files_count = 0 - for i in range(0, len(skylinks), 1000): + batch_size = 500 + for i in range(0, len(skylinks), batch_size): cached_files_command = ( - "find /data/nginx/cache/ -type f | xargs --no-run-if-empty -n1000 grep -Els '^Skynet-Skylink: (" - + "|".join(skylinks[i:i+1000]) + "find /data/nginx/cache/ -type f | xargs --no-run-if-empty -n" + batch_size + " grep -Els '^Skynet-Skylink: (" + + "|".join(skylinks[i:i+batch_size]) + ")'" ) - output = exec('docker exec -it nginx bash -c "' + cached_files_command + ' | xargs -r rm -v | wc -l"') - print(output) - cached_files_count+= int(output) + cached_files_count+= int(exec('docker exec -it nginx bash -c "' + cached_files_command + ' | xargs -r rm -v | wc -l"')) if cached_files_count == 0: return print("No nginx cached files matching blocked skylinks were found")