From 5cf5acb4e2789383382b35c6e0a936997cf08df8 Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Thu, 23 Dec 2021 12:55:47 +0100
Subject: [PATCH] do not prune blocklist manually

---
 docker/nginx/conf.d/server/server.local |  3 ++
 setup-scripts/blocklist-airtable.py     | 54 +++++--------------------
 2 files changed, 13 insertions(+), 44 deletions(-)

diff --git a/docker/nginx/conf.d/server/server.local b/docker/nginx/conf.d/server/server.local
index 1de4ab72..3a8ac118 100644
--- a/docker/nginx/conf.d/server/server.local
+++ b/docker/nginx/conf.d/server/server.local
@@ -1,6 +1,9 @@
 include /etc/nginx/conf.d/include/init-optional-variables;
 
 location /skynet/blocklist {
+    client_max_body_size 10m; # increase max body size to account for large lists
+    client_body_buffer_size 10m; # force whole body to memory so we can read it
+
     content_by_lua_block {        
         local httpc = require("resty.http").new()
 
diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index 9ec3314c..9ec81952 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -122,63 +122,30 @@ async def block_skylinks_from_airtable():
         )
         await send_msg(message, file=("\n".join(invalid_skylinks)))
 
-    apipassword = exec("docker exec sia cat /sia-data/apipassword")
     ipaddress = exec(
-        "docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia"
+        "docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx"
     )
 
-    print("Sending blocklist request to siad")
+    print("Sending blocklist request to siad through nginx")
     response = requests.post(
-        "http://" + ipaddress + ":9980/skynet/blocklist",
-        auth=("", apipassword),
-        headers={"user-agent": "Sia-Agent"},
+        "http://" + ipaddress + ":8000/skynet/blocklist",
         data=json.dumps({"add": skylinks}),
     )
 
-    if response.status_code == 204:
-        print("Siad blocklist successfully updated with provided skylink")
-    else:
+    print(json.dumps({"add": skylinks}))
+
+    if response.status_code != 204:
         status_code = str(response.status_code)
         response_text = response.text or "empty response"
         message = (
-            "Siad blocklist endpoint responded with code "
+            "Airtable blocklist request responded with code "
             + status_code
             + ": "
             + response_text
         )
         return await send_msg(message, force_notify=False)
 
-    # Remove from NGINX cache
-    # NOTE:
-    # If there are changes to how the NGINX cache is being cleared, the same
-    # changes need to be applied to the /scripts/blocklist-skylink.sh script.
-    print("Searching nginx cache for blocked files")
-    cached_files_count = 0
-    batch_size = 1000
-    for i in range(0, len(skylinks), batch_size):
-        cached_files_command = (
-            "find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ("
-            + "|".join(skylinks[i : i + batch_size])
-            + ")'"
-        )
-        cached_files_count += int(
-            exec(
-                'docker exec nginx bash -c "'
-                + cached_files_command
-                + ' | xargs -r rm -v | wc -l"'
-            )
-        )
-
-    if cached_files_count == 0:
-        return print("No nginx cached files matching blocked skylinks were found")
-    else:
-        print("Hot reloading nginx")
-        exec("docker exec nginx nginx -s reload")
-
-    message = (
-        "Purged " + str(cached_files_count) + " blocklisted files from nginx cache"
-    )
-    return await send_msg(message)
+    return await send_msg("Siad blocklist successfully updated with provided skylink")
 
 
 loop = asyncio.get_event_loop()
@@ -186,6 +153,5 @@ loop.run_until_complete(run_checks())
 
 # --- BASH EQUIVALENT
 # skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")
-# apipassword=$(docker exec sia cat /sia-data/apipassword)
-# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia)
-# curl -A "Sia-Agent" --user "":"${apipassword}" --data "{\"add\" : ${skylinks}}" "${ipaddress}:9980/skynet/blocklist"
+# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx)
+# curl --data "{\"add\" : ${skylinks}}" "${ipaddress}:8000/skynet/blocklist"