LumeWeb
/
skynet-webportal
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

smaller batch

This commit is contained in:
Karol Wypchlo 2021-07-18 09:24:45 +02:00
parent 6c1b96606e
commit ed19736e22
1 changed files with 5 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
setup-scripts/blocklist-airtable.py
View File

@ -97,15 +97,14 @@ async def block_skylinks_from_airtable():
print("Searching nginx cache for blocked files") print("Searching nginx cache for blocked files")
cached_files_count = 0 cached_files_count = 0
for i in range(0, len(skylinks), 1000): batch_size = 500
for i in range(0, len(skylinks), batch_size):
cached_files_command = ( cached_files_command = (
"find /data/nginx/cache/ -type f | xargs --no-run-if-empty -n1000 grep -Els '^Skynet-Skylink: (" "find /data/nginx/cache/ -type f | xargs --no-run-if-empty -n" + batch_size + " grep -Els '^Skynet-Skylink: ("
+ "|".join(skylinks[i:i+1000]) + "|".join(skylinks[i:i+batch_size])
+ ")'" + ")'"
) )
output = exec('docker exec -it nginx bash -c "' + cached_files_command + ' | xargs -r rm -v | wc -l"') cached_files_count+= int(exec('docker exec -it nginx bash -c "' + cached_files_command + ' | xargs -r rm -v | wc -l"'))
print(output)
cached_files_count+= int(output)
if cached_files_count == 0: if cached_files_count == 0:
return print("No nginx cached files matching blocked skylinks were found") return print("No nginx cached files matching blocked skylinks were found")