LumeWeb
/
skynet-webportal
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #637 from NebulousLabs/blocklist-v2 

blocklist batching
This commit is contained in:
Karol Wypchło 2021-03-08 12:26:04 +01:00 committed by GitHub
parent 83b5e5fe77 aee6976734
commit 64ca0d1801
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
setup-scripts/blocklist-airtable.py
View File

@ -75,12 +75,14 @@ async def block_skylinks_from_airtable():
return print(message) or await send_msg(client, message, force_notify=False) return print(message) or await send_msg(client, message, force_notify=False)
print("Searching nginx cache for blocked files") print("Searching nginx cache for blocked files")
cached_files_count = 0
for i in range(0, len(skylinks), 1000):
cached_files_command = ( cached_files_command = (
"/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -Els '^KEY: .*(" "/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -Els '^KEY: .*("
+ "|".join(skylinks) + "|".join(skylinks[i:i+1000])
+ ")'" + ")'"
) )
cached_files_count = int(exec('docker exec -it nginx bash -c "' + cached_files_command + ' | wc -l"') or 0) cached_files_count += int(exec('docker exec -it nginx bash -c "' + cached_files_command + ' | wc -l"') or 0)
if cached_files_count == 0: if cached_files_count == 0:
return print("No nginx cached files matching blocked skylinks were found") return print("No nginx cached files matching blocked skylinks were found")