From e816a33da118386ca4e38fb8c9aa712efebd7a35 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 5 Mar 2021 23:25:10 +0100 Subject: [PATCH 1/4] blocklist from file --- setup-scripts/blocklist-airtable.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py index 686e1702..14544b18 100755 --- a/setup-scripts/blocklist-airtable.py +++ b/setup-scripts/blocklist-airtable.py @@ -75,10 +75,12 @@ async def block_skylinks_from_airtable(): return print(message) or await send_msg(client, message, force_notify=False) print("Searching nginx cache for blocked files") + + f = open("/tmp/blocklist-aritable.txt", "w") + f.write("^KEY: .*" + "^KEY: .*\n".join(skylinks)) + f.close() cached_files_command = ( - "/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -Els '^KEY: .*(" - + "|".join(skylinks) - + ")'" + "/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -Els --file /tmp/blocklist-aritable.txt" ) cached_files_count = int(exec('docker exec -it nginx bash -c "' + cached_files_command + ' | wc -l"') or 0) From 1762c7f7b7b9bbf984746e71b6b612c10d4905f9 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 5 Mar 2021 23:37:27 +0100 Subject: [PATCH 2/4] fixed writing to file --- setup-scripts/blocklist-airtable.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py index 14544b18..8d5e2de3 100755 --- a/setup-scripts/blocklist-airtable.py +++ b/setup-scripts/blocklist-airtable.py @@ -76,9 +76,8 @@ async def block_skylinks_from_airtable(): print("Searching nginx cache for blocked files") - f = open("/tmp/blocklist-aritable.txt", "w") - f.write("^KEY: .*" + "^KEY: .*\n".join(skylinks)) - f.close() + print("Searching nginx cache for blocked files") + exec('printf "^KEY: .*' + '\n^KEY: .*'.join(skylinks) + '" > /tmp/blocklist-aritable.txt') cached_files_command = ( "/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -Els --file /tmp/blocklist-aritable.txt" ) From eda08a22bffc6f4da345064ce8e5160e1ab72fa6 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 5 Mar 2021 23:43:18 +0100 Subject: [PATCH 3/4] batching --- setup-scripts/blocklist-airtable.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py index 8d5e2de3..52906712 100755 --- a/setup-scripts/blocklist-airtable.py +++ b/setup-scripts/blocklist-airtable.py @@ -75,13 +75,14 @@ async def block_skylinks_from_airtable(): return print(message) or await send_msg(client, message, force_notify=False) print("Searching nginx cache for blocked files") - - print("Searching nginx cache for blocked files") - exec('printf "^KEY: .*' + '\n^KEY: .*'.join(skylinks) + '" > /tmp/blocklist-aritable.txt') - cached_files_command = ( - "/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -Els --file /tmp/blocklist-aritable.txt" - ) - cached_files_count = int(exec('docker exec -it nginx bash -c "' + cached_files_command + ' | wc -l"') or 0) + cached_files_count = 0 + for i in xrange(0, len(skylinks), 1000): + cached_files_command = ( + "/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -Els '^KEY: .*(" + + "|".join(skylinks[i:i+1000]) + + ")'" + ) + cached_files_count += int(exec('docker exec -it nginx bash -c "' + cached_files_command + ' | wc -l"') or 0) if cached_files_count == 0: return print("No nginx cached files matching blocked skylinks were found") From aee69767349da9ec32de86f26fe64d57347a43ff Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 5 Mar 2021 23:45:44 +0100 Subject: [PATCH 4/4] xrange --- setup-scripts/blocklist-airtable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py index 52906712..74f38d0d 100755 --- a/setup-scripts/blocklist-airtable.py +++ b/setup-scripts/blocklist-airtable.py @@ -76,7 +76,7 @@ async def block_skylinks_from_airtable(): print("Searching nginx cache for blocked files") cached_files_count = 0 - for i in xrange(0, len(skylinks), 1000): + for i in range(0, len(skylinks), 1000): cached_files_command = ( "/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -Els '^KEY: .*(" + "|".join(skylinks[i:i+1000])