From eda08a22bffc6f4da345064ce8e5160e1ab72fa6 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 5 Mar 2021 23:43:18 +0100 Subject: [PATCH] batching --- setup-scripts/blocklist-airtable.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py index 8d5e2de3..52906712 100755 --- a/setup-scripts/blocklist-airtable.py +++ b/setup-scripts/blocklist-airtable.py @@ -75,13 +75,14 @@ async def block_skylinks_from_airtable(): return print(message) or await send_msg(client, message, force_notify=False) print("Searching nginx cache for blocked files") - - print("Searching nginx cache for blocked files") - exec('printf "^KEY: .*' + '\n^KEY: .*'.join(skylinks) + '" > /tmp/blocklist-aritable.txt') - cached_files_command = ( - "/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -Els --file /tmp/blocklist-aritable.txt" - ) - cached_files_count = int(exec('docker exec -it nginx bash -c "' + cached_files_command + ' | wc -l"') or 0) + cached_files_count = 0 + for i in xrange(0, len(skylinks), 1000): + cached_files_command = ( + "/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -Els '^KEY: .*(" + + "|".join(skylinks[i:i+1000]) + + ")'" + ) + cached_files_count += int(exec('docker exec -it nginx bash -c "' + cached_files_command + ' | wc -l"') or 0) if cached_files_count == 0: return print("No nginx cached files matching blocked skylinks were found")