From 6a8b9e7ceb5a7d0fb4b6f4dc674704c62090971c Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 16:47:06 +0100
Subject: [PATCH 01/28] airtable blocklist integration

---
 setup-scripts/blocklist-airtable.py | 41 +++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100755 setup-scripts/blocklist-airtable.py

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
new file mode 100755
index 00000000..cf98ab00
--- /dev/null
+++ b/setup-scripts/blocklist-airtable.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+
+import os, asyncio, requests, json
+
+AIRTABLE_TABLE = "app89plJvA9EqTJEc"
+AIRTABLE_FIELD = "Link"
+
+async def block_skylinks_from_airtable():
+    headers = { "Authorization": "Bearer " + AIRTABLE_API_KEY }
+    airtable = requests.get(
+        "https://api.airtable.com/v0/" + AIRTABLE_TABLE + "/Table%201?fields%5B%5D=" + AIRTABLE_FIELD, headers=headers
+    ).json()
+    skylinks = [entry['fields'][AIRTABLE_FIELD] for entry in airtable['records']]
+    
+    apipassword = os.popen('docker exec sia cat /sia-data/apipassword').read().strip()
+    ipaddress = os.popen('docker inspect -f \'{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' sia').read().strip()
+    
+    headers = { 'user-agent': 'Sia-Agent' }
+    auth = ('', apipassword)
+    data = json.dumps({ 'add': skylinks })
+    response = requests.post('http://' + ipaddress + ':9980/skynet/blocklist', auth = auth, headers = headers, data = data)
+    
+    if response.status_code != 204:
+        message = "Blocklist responded with code " + str(response.status_code) + ": " + (response.text or "empty response")
+        print(message)
+
+async def exit_after(delay):
+    await asyncio.sleep(delay)
+    os._exit(0)
+
+async def on_ready():
+    await block_skylinks_from_airtable()
+    asyncio.create_task(exit_after(3))
+    
+asyncio.run(on_ready())
+
+# --- BASH EQUIVALENT
+# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_TABLE}/Table%201?fields%5B%5D=Link" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")
+# apipassword=$(docker exec sia cat /sia-data/apipassword)
+# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia)
+# curl -A "Sia-Agent" --user "":"${apipassword}" --data "{\"add\" : ${skylinks}}" "${ipaddress}:9980/skynet/blocklist"

From 98160a0fc8563f86f0448e5911d9b4b70d1d150f Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 16:49:08 +0100
Subject: [PATCH 02/28] crontab update

---
 setup-scripts/support/crontab | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup-scripts/support/crontab b/setup-scripts/support/crontab
index 30b947a8..75504aa8 100644
--- a/setup-scripts/support/crontab
+++ b/setup-scripts/support/crontab
@@ -1,3 +1,4 @@
 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/skynet-webportal/.env
 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/skynet-webportal/.env sia 8
 0 * * * * /home/user/skynet-webportal/setup-scripts/health-checker.py /home/user/skynet-webportal/.env sia 1
+30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env

From ac5dc3eca77514cded5bfc00e891e1a6913869ff Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 16:59:38 +0100
Subject: [PATCH 03/28] discord integration

---
 setup-scripts/blocklist-airtable.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index cf98ab00..e5b352fc 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -1,10 +1,14 @@
 #!/usr/bin/env python3
 
-import os, asyncio, requests, json
+import traceback, os, asyncio, requests, json, discord
+from bot_utils import setup, send_msg
 
 AIRTABLE_TABLE = "app89plJvA9EqTJEc"
 AIRTABLE_FIELD = "Link"
 
+bot_token = setup()
+client = discord.Client()
+
 async def block_skylinks_from_airtable():
     headers = { "Authorization": "Bearer " + AIRTABLE_API_KEY }
     airtable = requests.get(
@@ -22,17 +26,21 @@ async def block_skylinks_from_airtable():
     
     if response.status_code != 204:
         message = "Blocklist responded with code " + str(response.status_code) + ": " + (response.text or "empty response")
-        print(message)
+        await send_msg(client, message, force_notify=True)
 
 async def exit_after(delay):
     await asyncio.sleep(delay)
     os._exit(0)
 
+@client.event
 async def on_ready():
-    await block_skylinks_from_airtable()
+    try:
+        await block_skylinks_from_airtable()
+    except:  # catch all exceptions
+        await send_msg(client, "```\n{}\n```".format(traceback.format_exc()), force_notify=True)
     asyncio.create_task(exit_after(3))
     
-asyncio.run(on_ready())
+# asyncio.run(on_ready())
 
 # --- BASH EQUIVALENT
 # skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_TABLE}/Table%201?fields%5B%5D=Link" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")

From b0d49e2f2f10d23d51c66a12c1d345c3f117c846 Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 17:08:10 +0100
Subject: [PATCH 04/28] more logging

---
 setup-scripts/blocklist-airtable.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index e5b352fc..4657b22d 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -10,22 +10,27 @@ bot_token = setup()
 client = discord.Client()
 
 async def block_skylinks_from_airtable():
+    print("Pulling blocked skylinks from airtable via api integration")
     headers = { "Authorization": "Bearer " + AIRTABLE_API_KEY }
     airtable = requests.get(
         "https://api.airtable.com/v0/" + AIRTABLE_TABLE + "/Table%201?fields%5B%5D=" + AIRTABLE_FIELD, headers=headers
     ).json()
     skylinks = [entry['fields'][AIRTABLE_FIELD] for entry in airtable['records']]
+    print("Airtable returned " + str(len(skylinks)) + " skylinks to block")
     
     apipassword = os.popen('docker exec sia cat /sia-data/apipassword').read().strip()
     ipaddress = os.popen('docker inspect -f \'{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' sia').read().strip()
     
+    print("Sending blocklist request to siad")
     headers = { 'user-agent': 'Sia-Agent' }
     auth = ('', apipassword)
     data = json.dumps({ 'add': skylinks })
     response = requests.post('http://' + ipaddress + ':9980/skynet/blocklist', auth = auth, headers = headers, data = data)
     
-    if response.status_code != 204:
-        message = "Blocklist responded with code " + str(response.status_code) + ": " + (response.text or "empty response")
+    if response.status_code == 204:
+        print("Skylinks successfully added to siad blocklist")
+    else:
+        message = "Siad blocklist endpoint responded with code " + str(response.status_code) + ": " + (response.text or "empty response")
         await send_msg(client, message, force_notify=True)
 
 async def exit_after(delay):

From acf83ec621226c66d35f710a2902f82c1727dbbd Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 17:09:18 +0100
Subject: [PATCH 05/28] running param

---
 setup-scripts/blocklist-airtable.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index 4657b22d..a880fc9f 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -45,6 +45,8 @@ async def on_ready():
         await send_msg(client, "```\n{}\n```".format(traceback.format_exc()), force_notify=True)
     asyncio.create_task(exit_after(3))
     
+client.run(bot_token)
+
 # asyncio.run(on_ready())
 
 # --- BASH EQUIVALENT

From cfd74fc26e6cd489977673b2e1c38df0339d88fc Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 17:11:12 +0100
Subject: [PATCH 06/28] AIRTABLE_API_KEY

---
 setup-scripts/blocklist-airtable.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index a880fc9f..5c7b2b7f 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -5,6 +5,7 @@ from bot_utils import setup, send_msg
 
 AIRTABLE_TABLE = "app89plJvA9EqTJEc"
 AIRTABLE_FIELD = "Link"
+AIRTABLE_API_KEY = os.getenv('AIRTABLE_API_KEY')
 
 bot_token = setup()
 client = discord.Client()
@@ -31,7 +32,7 @@ async def block_skylinks_from_airtable():
         print("Skylinks successfully added to siad blocklist")
     else:
         message = "Siad blocklist endpoint responded with code " + str(response.status_code) + ": " + (response.text or "empty response")
-        await send_msg(client, message, force_notify=True)
+        await send_msg(client, message, force_notify=False)
 
 async def exit_after(delay):
     await asyncio.sleep(delay)
@@ -42,7 +43,7 @@ async def on_ready():
     try:
         await block_skylinks_from_airtable()
     except:  # catch all exceptions
-        await send_msg(client, "```\n{}\n```".format(traceback.format_exc()), force_notify=True)
+        await send_msg(client, "```\n{}\n```".format(traceback.format_exc()), force_notify=False)
     asyncio.create_task(exit_after(3))
     
 client.run(bot_token)

From 46fd77a2e238649de0765575262e6492b4a1fe1d Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 17:13:00 +0100
Subject: [PATCH 07/28] AIRTABLE_API_KEY

---
 setup-scripts/blocklist-airtable.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index 5c7b2b7f..cf1872d8 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -3,13 +3,13 @@
 import traceback, os, asyncio, requests, json, discord
 from bot_utils import setup, send_msg
 
+bot_token = setup()
+client = discord.Client()
+
 AIRTABLE_TABLE = "app89plJvA9EqTJEc"
 AIRTABLE_FIELD = "Link"
 AIRTABLE_API_KEY = os.getenv('AIRTABLE_API_KEY')
 
-bot_token = setup()
-client = discord.Client()
-
 async def block_skylinks_from_airtable():
     print("Pulling blocked skylinks from airtable via api integration")
     headers = { "Authorization": "Bearer " + AIRTABLE_API_KEY }

From 0d8b50e5a21a91b035e1a45e94f0a40d571b3b4e Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 17:19:13 +0100
Subject: [PATCH 08/28] better error handling

---
 setup-scripts/blocklist-airtable.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index cf1872d8..b91ab3a8 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -15,8 +15,13 @@ async def block_skylinks_from_airtable():
     headers = { "Authorization": "Bearer " + AIRTABLE_API_KEY }
     airtable = requests.get(
         "https://api.airtable.com/v0/" + AIRTABLE_TABLE + "/Table%201?fields%5B%5D=" + AIRTABLE_FIELD, headers=headers
-    ).json()
-    skylinks = [entry['fields'][AIRTABLE_FIELD] for entry in airtable['records']]
+    )
+
+    if airtable.status_code != 200:
+        message = "Airtable blocklist integration responded with code " + str(response.status_code) + ": " + (response.text or "empty response")
+        return print(message) and await send_msg(client, message, force_notify=False)
+    
+    skylinks = [entry['fields'][AIRTABLE_FIELD] for entry in airtable.json()['records']]
     print("Airtable returned " + str(len(skylinks)) + " skylinks to block")
     
     apipassword = os.popen('docker exec sia cat /sia-data/apipassword').read().strip()
@@ -32,7 +37,7 @@ async def block_skylinks_from_airtable():
         print("Skylinks successfully added to siad blocklist")
     else:
         message = "Siad blocklist endpoint responded with code " + str(response.status_code) + ": " + (response.text or "empty response")
-        await send_msg(client, message, force_notify=False)
+        await print(message) and send_msg(client, message, force_notify=False)
 
 async def exit_after(delay):
     await asyncio.sleep(delay)

From 937fe052b38aa9b92029f67e8fda4df6d0490b71 Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 17:20:42 +0100
Subject: [PATCH 09/28] better error handling

---
 setup-scripts/blocklist-airtable.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index b91ab3a8..99b40694 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -18,7 +18,7 @@ async def block_skylinks_from_airtable():
     )
 
     if airtable.status_code != 200:
-        message = "Airtable blocklist integration responded with code " + str(response.status_code) + ": " + (response.text or "empty response")
+        message = "Airtable blocklist integration responded with code " + str(airtable.status_code) + ": " + (airtable.text or "empty response")
         return print(message) and await send_msg(client, message, force_notify=False)
     
     skylinks = [entry['fields'][AIRTABLE_FIELD] for entry in airtable.json()['records']]

From fb0ab9bb502196cafa882f78df135e1a4cfa2c2a Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 17:21:53 +0100
Subject: [PATCH 10/28] better error handling

---
 setup-scripts/blocklist-airtable.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index 99b40694..b8b4246b 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -19,7 +19,7 @@ async def block_skylinks_from_airtable():
 
     if airtable.status_code != 200:
         message = "Airtable blocklist integration responded with code " + str(airtable.status_code) + ": " + (airtable.text or "empty response")
-        return print(message) and await send_msg(client, message, force_notify=False)
+        return print(message) or await send_msg(client, message, force_notify=False)
     
     skylinks = [entry['fields'][AIRTABLE_FIELD] for entry in airtable.json()['records']]
     print("Airtable returned " + str(len(skylinks)) + " skylinks to block")
@@ -34,10 +34,10 @@ async def block_skylinks_from_airtable():
     response = requests.post('http://' + ipaddress + ':9980/skynet/blocklist', auth = auth, headers = headers, data = data)
     
     if response.status_code == 204:
-        print("Skylinks successfully added to siad blocklist")
+        return print("Skylinks successfully added to siad blocklist")
     else:
         message = "Siad blocklist endpoint responded with code " + str(response.status_code) + ": " + (response.text or "empty response")
-        await print(message) and send_msg(client, message, force_notify=False)
+        return await print(message) or send_msg(client, message, force_notify=False)
 
 async def exit_after(delay):
     await asyncio.sleep(delay)

From 1c9bdd4dc667619a8e5bc1fccff2fec6d826f612 Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 17:22:43 +0100
Subject: [PATCH 11/28] notification

---
 setup-scripts/blocklist-airtable.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index b8b4246b..47b9757b 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -19,7 +19,7 @@ async def block_skylinks_from_airtable():
 
     if airtable.status_code != 200:
         message = "Airtable blocklist integration responded with code " + str(airtable.status_code) + ": " + (airtable.text or "empty response")
-        return print(message) or await send_msg(client, message, force_notify=False)
+        return print(message) or await send_msg(client, message, force_notify=True)
     
     skylinks = [entry['fields'][AIRTABLE_FIELD] for entry in airtable.json()['records']]
     print("Airtable returned " + str(len(skylinks)) + " skylinks to block")
@@ -37,7 +37,7 @@ async def block_skylinks_from_airtable():
         return print("Skylinks successfully added to siad blocklist")
     else:
         message = "Siad blocklist endpoint responded with code " + str(response.status_code) + ": " + (response.text or "empty response")
-        return await print(message) or send_msg(client, message, force_notify=False)
+        return await print(message) or send_msg(client, message, force_notify=True)
 
 async def exit_after(delay):
     await asyncio.sleep(delay)
@@ -48,7 +48,7 @@ async def on_ready():
     try:
         await block_skylinks_from_airtable()
     except:  # catch all exceptions
-        await send_msg(client, "```\n{}\n```".format(traceback.format_exc()), force_notify=False)
+        await send_msg(client, "```\n{}\n```".format(traceback.format_exc()), force_notify=True)
     asyncio.create_task(exit_after(3))
     
 client.run(bot_token)

From 0677e7c933572b435aec35302d785899d07d0ead Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 17:56:30 +0100
Subject: [PATCH 12/28] clear nginx cache

---
 setup-scripts/blocklist-airtable.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index 47b9757b..cd14e43b 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -22,7 +22,11 @@ async def block_skylinks_from_airtable():
         return print(message) or await send_msg(client, message, force_notify=True)
     
     skylinks = [entry['fields'][AIRTABLE_FIELD] for entry in airtable.json()['records']]
-    print("Airtable returned " + str(len(skylinks)) + " skylinks to block")
+
+    if len(skylinks) == 0:
+        return print("Airtable returned 0 skylinks to block - make sure your table configuration is correct")
+    else:
+        print("Airtable returned " + str(len(skylinks)) + " skylinks to block")
     
     apipassword = os.popen('docker exec sia cat /sia-data/apipassword').read().strip()
     ipaddress = os.popen('docker inspect -f \'{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' sia').read().strip()
@@ -34,11 +38,18 @@ async def block_skylinks_from_airtable():
     response = requests.post('http://' + ipaddress + ':9980/skynet/blocklist', auth = auth, headers = headers, data = data)
     
     if response.status_code == 204:
-        return print("Skylinks successfully added to siad blocklist")
+        print("Skylinks successfully added to siad blocklist")
     else:
         message = "Siad blocklist endpoint responded with code " + str(response.status_code) + ": " + (response.text or "empty response")
         return await print(message) or send_msg(client, message, force_notify=True)
 
+    print("Clearing nginx cache related to blocked skylinks")
+    find_all_cache_files = '/usr/bin/find /data/nginx/cache/ -type f'
+    grep_pattern = '^KEY: .*(' + '|'.join(skylinks) + ')'
+    filter_matching_files = '/usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -El ' + grep_pattern
+    print('docker exec -it nginx bash -c "' + find_all_cache_files + ' | ' + filter_matching_files + '"')
+    os.popen('docker exec -it nginx bash -c "' + find_all_cache_files + ' | ' + filter_matching_files + '"')
+
 async def exit_after(delay):
     await asyncio.sleep(delay)
     os._exit(0)

From ec77e1680638dc057e37fa1eb41b50d04b9a2506 Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 17:58:43 +0100
Subject: [PATCH 13/28] fix grep pattern

---
 setup-scripts/blocklist-airtable.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index cd14e43b..088b371a 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -45,7 +45,7 @@ async def block_skylinks_from_airtable():
 
     print("Clearing nginx cache related to blocked skylinks")
     find_all_cache_files = '/usr/bin/find /data/nginx/cache/ -type f'
-    grep_pattern = '^KEY: .*(' + '|'.join(skylinks) + ')'
+    grep_pattern = '\'^KEY: .*(' + '|'.join(skylinks) + ')\''
     filter_matching_files = '/usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -El ' + grep_pattern
     print('docker exec -it nginx bash -c "' + find_all_cache_files + ' | ' + filter_matching_files + '"')
     os.popen('docker exec -it nginx bash -c "' + find_all_cache_files + ' | ' + filter_matching_files + '"')

From ea8ccba3fa932ae4b1837357b76036f8a26d2dea Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 18:26:38 +0100
Subject: [PATCH 14/28] print cached files

---
 setup-scripts/blocklist-airtable.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index 088b371a..fa00ab86 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -43,12 +43,10 @@ async def block_skylinks_from_airtable():
         message = "Siad blocklist endpoint responded with code " + str(response.status_code) + ": " + (response.text or "empty response")
         return await print(message) or send_msg(client, message, force_notify=True)
 
-    print("Clearing nginx cache related to blocked skylinks")
-    find_all_cache_files = '/usr/bin/find /data/nginx/cache/ -type f'
-    grep_pattern = '\'^KEY: .*(' + '|'.join(skylinks) + ')\''
-    filter_matching_files = '/usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -El ' + grep_pattern
-    print('docker exec -it nginx bash -c "' + find_all_cache_files + ' | ' + filter_matching_files + '"')
-    os.popen('docker exec -it nginx bash -c "' + find_all_cache_files + ' | ' + filter_matching_files + '"')
+    print("Purging nginx cache containing blocked skylinks")
+    purge_command = '/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -El \'^KEY: .*(' + '|'.join(skylinks) + ')\''
+    cached_files = os.popen('docker exec -it nginx bash -c "' + purge_command + '"').read().strip()
+    print(cached_files)
 
 async def exit_after(delay):
     await asyncio.sleep(delay)

From 5473468dc864656f68d744d6d321296f318951e1 Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 18:39:43 +0100
Subject: [PATCH 15/28] purging script

---
 setup-scripts/blocklist-airtable.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index fa00ab86..938d7a31 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -41,12 +41,18 @@ async def block_skylinks_from_airtable():
         print("Skylinks successfully added to siad blocklist")
     else:
         message = "Siad blocklist endpoint responded with code " + str(response.status_code) + ": " + (response.text or "empty response")
-        return await print(message) or send_msg(client, message, force_notify=True)
+        return print(message) or await send_msg(client, message, force_notify=True)
 
     print("Purging nginx cache containing blocked skylinks")
-    purge_command = '/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -El \'^KEY: .*(' + '|'.join(skylinks) + ')\''
-    cached_files = os.popen('docker exec -it nginx bash -c "' + purge_command + '"').read().strip()
-    print(cached_files)
+    cached_files_command = '/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -El \'^KEY: .*(' + '|'.join(skylinks) + ')\''
+    cached_files_count = int(os.popen('docker exec -it nginx bash -c "' + cached_files_command + ' | wc -l"').read().strip())
+
+    if cached_files_count == 0:
+        return print("No nginx cached files matching blocked skylinks were found")
+
+    os.popen('docker exec -it nginx bash -c "' + cached_files_command + ' | xargs rm"')
+    message = "Purged " + str(cached_files_count) + " blocklisted files from nginx cache")
+    return print(message) or await send_msg(client, message)
 
 async def exit_after(delay):
     await asyncio.sleep(delay)

From b92bfe0150eb5b37c97a525ae35a1e086fba82da Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 18:40:39 +0100
Subject: [PATCH 16/28] typo

---
 setup-scripts/blocklist-airtable.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index 938d7a31..0157a680 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -51,7 +51,7 @@ async def block_skylinks_from_airtable():
         return print("No nginx cached files matching blocked skylinks were found")
 
     os.popen('docker exec -it nginx bash -c "' + cached_files_command + ' | xargs rm"')
-    message = "Purged " + str(cached_files_count) + " blocklisted files from nginx cache")
+    message = 'Purged ' + str(cached_files_count) + ' blocklisted files from nginx cache'
     return print(message) or await send_msg(client, message)
 
 async def exit_after(delay):

From 1d9215fdd1eb4876dd9683023feb7254b6d0f1f0 Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 18:41:41 +0100
Subject: [PATCH 17/28] typo

---
 setup-scripts/blocklist-airtable.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index 0157a680..60939993 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -43,7 +43,7 @@ async def block_skylinks_from_airtable():
         message = "Siad blocklist endpoint responded with code " + str(response.status_code) + ": " + (response.text or "empty response")
         return print(message) or await send_msg(client, message, force_notify=True)
 
-    print("Purging nginx cache containing blocked skylinks")
+    print("Searching nginx cache for blocked files")
     cached_files_command = '/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -El \'^KEY: .*(' + '|'.join(skylinks) + ')\''
     cached_files_count = int(os.popen('docker exec -it nginx bash -c "' + cached_files_command + ' | wc -l"').read().strip())
 

From d9ed1decf63207e9e65d7bcd511c9c61c6db2152 Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 18:42:33 +0100
Subject: [PATCH 18/28] typo

---
 setup-scripts/blocklist-airtable.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index 60939993..06644c84 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -38,7 +38,7 @@ async def block_skylinks_from_airtable():
     response = requests.post('http://' + ipaddress + ':9980/skynet/blocklist', auth = auth, headers = headers, data = data)
     
     if response.status_code == 204:
-        print("Skylinks successfully added to siad blocklist")
+        print("Siad blocklist succesfully updated with provided skylink")
     else:
         message = "Siad blocklist endpoint responded with code " + str(response.status_code) + ": " + (response.text or "empty response")
         return print(message) or await send_msg(client, message, force_notify=True)

From 6af13b47150e9a9b6790e9eb8d204805f1fca462 Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 18:50:01 +0100
Subject: [PATCH 19/28] typo

---
 setup-scripts/blocklist-airtable.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index 06644c84..93b4ea3c 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -38,7 +38,7 @@ async def block_skylinks_from_airtable():
     response = requests.post('http://' + ipaddress + ':9980/skynet/blocklist', auth = auth, headers = headers, data = data)
     
     if response.status_code == 204:
-        print("Siad blocklist succesfully updated with provided skylink")
+        print("Siad blocklist successfully updated with provided skylink")
     else:
         message = "Siad blocklist endpoint responded with code " + str(response.status_code) + ": " + (response.text or "empty response")
         return print(message) or await send_msg(client, message, force_notify=True)

From 002ae6c6d83eca27d1e809d0799c2ee655c49963 Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 23:08:38 +0100
Subject: [PATCH 20/28] pagination

---
 setup-scripts/blocklist-airtable.py | 37 ++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 14 deletions(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index 93b4ea3c..97e998cc 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -6,27 +6,36 @@ from bot_utils import setup, send_msg
 bot_token = setup()
 client = discord.Client()
 
-AIRTABLE_TABLE = "app89plJvA9EqTJEc"
-AIRTABLE_FIELD = "Link"
 AIRTABLE_API_KEY = os.getenv('AIRTABLE_API_KEY')
+AIRTABLE_TABLE = os.getenv('AIRTABLE_TABLE', 'app89plJvA9EqTJEc')
+AIRTABLE_FIELD = os.getenv('AIRTABLE_FIELD', 'Link')
 
 async def block_skylinks_from_airtable():
     print("Pulling blocked skylinks from airtable via api integration")
     headers = { "Authorization": "Bearer " + AIRTABLE_API_KEY }
-    airtable = requests.get(
-        "https://api.airtable.com/v0/" + AIRTABLE_TABLE + "/Table%201?fields%5B%5D=" + AIRTABLE_FIELD, headers=headers
-    )
+    skylinks = []
+    offset = ''
+    while len(skylinks) == 0 or offset:
+        query = '&'.join(['fields%5B%5D=' + AIRTABLE_FIELD, ('offset=' + offset) if offset else ''])
+        airtable = requests.get(
+            "https://api.airtable.com/v0/" + AIRTABLE_TABLE + "?" + query, headers=headers
+        )
 
-    if airtable.status_code != 200:
-        message = "Airtable blocklist integration responded with code " + str(airtable.status_code) + ": " + (airtable.text or "empty response")
-        return print(message) or await send_msg(client, message, force_notify=True)
+        if airtable.status_code != 200:
+            message = "Airtable blocklist integration responded with code " + str(airtable.status_code) + ": " + (airtable.text or "empty response")
+            return print(message) or await send_msg(client, message, force_notify=True)
+        
+        airtable_data = airtable.json()
+        skylinks = skylinks + [entry['fields'][AIRTABLE_FIELD] for entry in airtable_data['records']]
+
+        if len(skylinks) == 0:
+            return print("Airtable returned 0 skylinks - make sure your configuration is correct")
+        
+        print(airtable_data.offset)
+        offset = airtable_data.offset
+        print(airtable_data.offset)
     
-    skylinks = [entry['fields'][AIRTABLE_FIELD] for entry in airtable.json()['records']]
-
-    if len(skylinks) == 0:
-        return print("Airtable returned 0 skylinks to block - make sure your table configuration is correct")
-    else:
-        print("Airtable returned " + str(len(skylinks)) + " skylinks to block")
+    print("Airtable returned " + str(len(skylinks)) + " skylinks to block")
     
     apipassword = os.popen('docker exec sia cat /sia-data/apipassword').read().strip()
     ipaddress = os.popen('docker inspect -f \'{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' sia').read().strip()

From 3b5d09ec73d57672c9ee923f1aac61c16373b97f Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 23:13:28 +0100
Subject: [PATCH 21/28] pagination

---
 setup-scripts/blocklist-airtable.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index 97e998cc..8ca2b41a 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -7,7 +7,8 @@ bot_token = setup()
 client = discord.Client()
 
 AIRTABLE_API_KEY = os.getenv('AIRTABLE_API_KEY')
-AIRTABLE_TABLE = os.getenv('AIRTABLE_TABLE', 'app89plJvA9EqTJEc')
+AIRTABLE_BASE = os.getenv('AIRTABLE_BASE', 'app89plJvA9EqTJEc')
+AIRTABLE_TABLE = os.getenv('AIRTABLE_TABLE', 'Table%201')
 AIRTABLE_FIELD = os.getenv('AIRTABLE_FIELD', 'Link')
 
 async def block_skylinks_from_airtable():
@@ -18,12 +19,12 @@ async def block_skylinks_from_airtable():
     while len(skylinks) == 0 or offset:
         query = '&'.join(['fields%5B%5D=' + AIRTABLE_FIELD, ('offset=' + offset) if offset else ''])
         airtable = requests.get(
-            "https://api.airtable.com/v0/" + AIRTABLE_TABLE + "?" + query, headers=headers
+            "https://api.airtable.com/v0/" + AIRTABLE_BASE + "/" + AIRTABLE_TABLE + "?" + query, headers=headers
         )
 
         if airtable.status_code != 200:
             message = "Airtable blocklist integration responded with code " + str(airtable.status_code) + ": " + (airtable.text or "empty response")
-            return print(message) or await send_msg(client, message, force_notify=True)
+            return print(message) or await send_msg(client, message, force_notify=False)
         
         airtable_data = airtable.json()
         skylinks = skylinks + [entry['fields'][AIRTABLE_FIELD] for entry in airtable_data['records']]
@@ -50,7 +51,7 @@ async def block_skylinks_from_airtable():
         print("Siad blocklist successfully updated with provided skylink")
     else:
         message = "Siad blocklist endpoint responded with code " + str(response.status_code) + ": " + (response.text or "empty response")
-        return print(message) or await send_msg(client, message, force_notify=True)
+        return print(message) or await send_msg(client, message, force_notify=False)
 
     print("Searching nginx cache for blocked files")
     cached_files_command = '/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -El \'^KEY: .*(' + '|'.join(skylinks) + ')\''
@@ -72,7 +73,7 @@ async def on_ready():
     try:
         await block_skylinks_from_airtable()
     except:  # catch all exceptions
-        await send_msg(client, "```\n{}\n```".format(traceback.format_exc()), force_notify=True)
+        await send_msg(client, "```\n{}\n```".format(traceback.format_exc()), force_notify=False)
     asyncio.create_task(exit_after(3))
     
 client.run(bot_token)
@@ -80,7 +81,7 @@ client.run(bot_token)
 # asyncio.run(on_ready())
 
 # --- BASH EQUIVALENT
-# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_TABLE}/Table%201?fields%5B%5D=Link" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")
+# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/Table%201?fields%5B%5D=Link" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")
 # apipassword=$(docker exec sia cat /sia-data/apipassword)
 # ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia)
 # curl -A "Sia-Agent" --user "":"${apipassword}" --data "{\"add\" : ${skylinks}}" "${ipaddress}:9980/skynet/blocklist"

From e14a03c912d49c09c056e8ce1cf62a8e6ca2adfe Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 23:15:54 +0100
Subject: [PATCH 22/28] pagination

---
 setup-scripts/blocklist-airtable.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index 8ca2b41a..b684a466 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -15,7 +15,7 @@ async def block_skylinks_from_airtable():
     print("Pulling blocked skylinks from airtable via api integration")
     headers = { "Authorization": "Bearer " + AIRTABLE_API_KEY }
     skylinks = []
-    offset = ''
+    offset = None
     while len(skylinks) == 0 or offset:
         query = '&'.join(['fields%5B%5D=' + AIRTABLE_FIELD, ('offset=' + offset) if offset else ''])
         airtable = requests.get(
@@ -32,9 +32,9 @@ async def block_skylinks_from_airtable():
         if len(skylinks) == 0:
             return print("Airtable returned 0 skylinks - make sure your configuration is correct")
         
-        print(airtable_data.offset)
-        offset = airtable_data.offset
-        print(airtable_data.offset)
+        print(offset)
+        offset = airtable_data.get('offset')
+        print(offset)
     
     print("Airtable returned " + str(len(skylinks)) + " skylinks to block")
     

From 43ddd484c2ef6b6491d192a71579a48c48680af7 Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 23:22:53 +0100
Subject: [PATCH 23/28] pagination

---
 setup-scripts/blocklist-airtable.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index b684a466..0e00d09a 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -17,6 +17,7 @@ async def block_skylinks_from_airtable():
     skylinks = []
     offset = None
     while len(skylinks) == 0 or offset:
+        print("Requesting a batch of records from airtable with " + (offset if offset else "empty") + " offset")
         query = '&'.join(['fields%5B%5D=' + AIRTABLE_FIELD, ('offset=' + offset) if offset else ''])
         airtable = requests.get(
             "https://api.airtable.com/v0/" + AIRTABLE_BASE + "/" + AIRTABLE_TABLE + "?" + query, headers=headers
@@ -32,11 +33,9 @@ async def block_skylinks_from_airtable():
         if len(skylinks) == 0:
             return print("Airtable returned 0 skylinks - make sure your configuration is correct")
         
-        print(offset)
         offset = airtable_data.get('offset')
-        print(offset)
     
-    print("Airtable returned " + str(len(skylinks)) + " skylinks to block")
+    print("Airtable returned total " + str(len(skylinks)) + " skylinks to block")
     
     apipassword = os.popen('docker exec sia cat /sia-data/apipassword').read().strip()
     ipaddress = os.popen('docker inspect -f \'{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' sia').read().strip()
@@ -54,7 +53,7 @@ async def block_skylinks_from_airtable():
         return print(message) or await send_msg(client, message, force_notify=False)
 
     print("Searching nginx cache for blocked files")
-    cached_files_command = '/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -El \'^KEY: .*(' + '|'.join(skylinks) + ')\''
+    cached_files_command = '/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -Els \'^KEY: .*(' + '|'.join(skylinks) + ')\''
     cached_files_count = int(os.popen('docker exec -it nginx bash -c "' + cached_files_command + ' | wc -l"').read().strip())
 
     if cached_files_count == 0:

From 7e717021f4e7a28c0856a3d18ccd0029b2d17d09 Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Mon, 1 Mar 2021 23:39:28 +0100
Subject: [PATCH 24/28] black

---
 setup-scripts/blocklist-airtable.py | 88 ++++++++++++++++++-----------
 1 file changed, 54 insertions(+), 34 deletions(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index 0e00d09a..8a594290 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -6,75 +6,95 @@ from bot_utils import setup, send_msg
 bot_token = setup()
 client = discord.Client()
 
-AIRTABLE_API_KEY = os.getenv('AIRTABLE_API_KEY')
-AIRTABLE_BASE = os.getenv('AIRTABLE_BASE', 'app89plJvA9EqTJEc')
-AIRTABLE_TABLE = os.getenv('AIRTABLE_TABLE', 'Table%201')
-AIRTABLE_FIELD = os.getenv('AIRTABLE_FIELD', 'Link')
+AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY")
+AIRTABLE_BASE = os.getenv("AIRTABLE_BASE", "app89plJvA9EqTJEc")
+AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE", "Table%201")
+AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD", "Link")
+
+
+def exec(command):
+    return os.popen(command).read().strip()
+
 
 async def block_skylinks_from_airtable():
-    print("Pulling blocked skylinks from airtable via api integration")
-    headers = { "Authorization": "Bearer " + AIRTABLE_API_KEY }
+    print("Pulling blocked skylinks from Airtable via api integration")
+    headers = {"Authorization": "Bearer " + AIRTABLE_API_KEY}
     skylinks = []
     offset = None
     while len(skylinks) == 0 or offset:
-        print("Requesting a batch of records from airtable with " + (offset if offset else "empty") + " offset")
-        query = '&'.join(['fields%5B%5D=' + AIRTABLE_FIELD, ('offset=' + offset) if offset else ''])
-        airtable = requests.get(
-            "https://api.airtable.com/v0/" + AIRTABLE_BASE + "/" + AIRTABLE_TABLE + "?" + query, headers=headers
+        print("Requesting a batch of records from Airtable with " + (offset if offset else "empty") + " offset")
+        query = "&".join(["fields%5B%5D=" + AIRTABLE_FIELD, ("offset=" + offset) if offset else ""])
+        response = requests.get(
+            "https://api.airtable.com/v0/" + AIRTABLE_BASE + "/" + AIRTABLE_TABLE + "?" + query,
+            headers=headers,
         )
 
-        if airtable.status_code != 200:
-            message = "Airtable blocklist integration responded with code " + str(airtable.status_code) + ": " + (airtable.text or "empty response")
+        if response.status_code != 200:
+            status_code = str(response.status_code)
+            response_text = response.text or "empty response"
+            message = "Airtable blocklist integration responded with code " + status_code + ": " + response_text
             return print(message) or await send_msg(client, message, force_notify=False)
-        
-        airtable_data = airtable.json()
-        skylinks = skylinks + [entry['fields'][AIRTABLE_FIELD] for entry in airtable_data['records']]
+
+        data = response.json()
+        skylinks = skylinks + [entry["fields"][AIRTABLE_FIELD] for entry in data["records"]]
 
         if len(skylinks) == 0:
             return print("Airtable returned 0 skylinks - make sure your configuration is correct")
-        
-        offset = airtable_data.get('offset')
-    
+
+        offset = data.get("offset")
+
     print("Airtable returned total " + str(len(skylinks)) + " skylinks to block")
-    
-    apipassword = os.popen('docker exec sia cat /sia-data/apipassword').read().strip()
-    ipaddress = os.popen('docker inspect -f \'{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' sia').read().strip()
-    
+
+    apipassword = exec("docker exec sia cat /sia-data/apipassword")
+    ipaddress = exec("docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia")
+
     print("Sending blocklist request to siad")
-    headers = { 'user-agent': 'Sia-Agent' }
-    auth = ('', apipassword)
-    data = json.dumps({ 'add': skylinks })
-    response = requests.post('http://' + ipaddress + ':9980/skynet/blocklist', auth = auth, headers = headers, data = data)
-    
+    response = requests.post(
+        "http://" + ipaddress + ":9980/skynet/blocklist",
+        auth=("", apipassword),
+        headers={"user-agent": "Sia-Agent"},
+        data=json.dumps({"add": skylinks}),
+    )
+
     if response.status_code == 204:
         print("Siad blocklist successfully updated with provided skylink")
     else:
-        message = "Siad blocklist endpoint responded with code " + str(response.status_code) + ": " + (response.text or "empty response")
+        status_code = str(response.status_code)
+        response_text = response.text or "empty response"
+        message = "Siad blocklist endpoint responded with code " + status_code + ": " + response_text
         return print(message) or await send_msg(client, message, force_notify=False)
 
     print("Searching nginx cache for blocked files")
-    cached_files_command = '/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -Els \'^KEY: .*(' + '|'.join(skylinks) + ')\''
-    cached_files_count = int(os.popen('docker exec -it nginx bash -c "' + cached_files_command + ' | wc -l"').read().strip())
+    cached_files_command = (
+        "/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -Els '^KEY: .*("
+        + "|".join(skylinks)
+        + ")'"
+    )
+    cached_files_count = int(exec('docker exec -it nginx bash -c "' + cached_files_command + ' | wc -l"'))
 
     if cached_files_count == 0:
         return print("No nginx cached files matching blocked skylinks were found")
 
-    os.popen('docker exec -it nginx bash -c "' + cached_files_command + ' | xargs rm"')
-    message = 'Purged ' + str(cached_files_count) + ' blocklisted files from nginx cache'
+    exec('docker exec -it nginx bash -c "' + cached_files_command + ' | xargs rm"')
+    message = "Purged " + str(cached_files_count) + " blocklisted files from nginx cache"
     return print(message) or await send_msg(client, message)
 
+
 async def exit_after(delay):
     await asyncio.sleep(delay)
     os._exit(0)
 
+
 @client.event
 async def on_ready():
     try:
         await block_skylinks_from_airtable()
     except:  # catch all exceptions
-        await send_msg(client, "```\n{}\n```".format(traceback.format_exc()), force_notify=False)
+        message = "```\n{}\n```".format(traceback.format_exc())
+        await send_msg(client, message, force_notify=False)
     asyncio.create_task(exit_after(3))
-    
+
+
 client.run(bot_token)
 
 # asyncio.run(on_ready())

From f306a1f40810b6749039cb66cb1b84795f786584 Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Tue, 2 Mar 2021 12:43:23 +0100
Subject: [PATCH 25/28] remove leftover

---
 setup-scripts/blocklist-airtable.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index 8a594290..715eef69 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -97,10 +97,8 @@ async def on_ready():
 
 client.run(bot_token)
 
-# asyncio.run(on_ready())
-
 # --- BASH EQUIVALENT
-# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/Table%201?fields%5B%5D=Link" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")
+# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")
 # apipassword=$(docker exec sia cat /sia-data/apipassword)
 # ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia)
 # curl -A "Sia-Agent" --user "":"${apipassword}" --data "{\"add\" : ${skylinks}}" "${ipaddress}:9980/skynet/blocklist"

From bf0d8f21458bcace8aeb15c81954b24d9b89dc54 Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Tue, 2 Mar 2021 13:40:18 +0100
Subject: [PATCH 26/28] filter invalid skylinks

---
 setup-scripts/blocklist-airtable.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index 715eef69..d76e017c 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -45,6 +45,14 @@ async def block_skylinks_from_airtable():
 
     print("Airtable returned total " + str(len(skylinks)) + " skylinks to block")
 
+    skylinks_returned = skylinks
+    skylinks = [skylink for skylink in skylinks if re.search("^[a-zA-Z0-9_-]{46}$", skylink)]
+
+    if len(skylinks_returned) != len(skylinks):
+        message = (skylinks_returned - len(skylinks)) + " of the skylinks returned from Airtable are not valid"
+        invalid_skylinks = [str(skylink) for skylink in list(set(skylinks_returned) - set(skylinks))]
+        print(message) or await send_msg(client, message, file=("\n".join(invalid_skylinks)))
+
     apipassword = exec("docker exec sia cat /sia-data/apipassword")
     ipaddress = exec("docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia")
 

From 4f055528ca6d7d870470178234c0b7fb911680d1 Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Tue, 2 Mar 2021 13:41:23 +0100
Subject: [PATCH 27/28] filter invalid skylinks

---
 setup-scripts/blocklist-airtable.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index d76e017c..e61b6f9c 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-import traceback, os, asyncio, requests, json, discord
+import traceback, os, re, asyncio, requests, json, discord
 from bot_utils import setup, send_msg
 
 bot_token = setup()

From 6186d7439fb7b31c6f9833569a5b775821ae3847 Mon Sep 17 00:00:00 2001
From: Karol Wypchlo <kwypchlo@gmail.com>
Date: Tue, 2 Mar 2021 13:44:27 +0100
Subject: [PATCH 28/28] filter invalid skylinks

---
 setup-scripts/blocklist-airtable.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py
index e61b6f9c..7f5d4dd7 100755
--- a/setup-scripts/blocklist-airtable.py
+++ b/setup-scripts/blocklist-airtable.py
@@ -49,8 +49,8 @@ async def block_skylinks_from_airtable():
     skylinks = [skylink for skylink in skylinks if re.search("^[a-zA-Z0-9_-]{46}$", skylink)]
 
     if len(skylinks_returned) != len(skylinks):
-        message = (skylinks_returned - len(skylinks)) + " of the skylinks returned from Airtable are not valid"
         invalid_skylinks = [str(skylink) for skylink in list(set(skylinks_returned) - set(skylinks))]
+        message = str(len(invalid_skylinks)) + " of the skylinks returned from Airtable are not valid"
         print(message) or await send_msg(client, message, file=("\n".join(invalid_skylinks)))
 
     apipassword = exec("docker exec sia cat /sia-data/apipassword")