Merge remote-tracking branch 'origin/master' into account-only-portals

docker-compose.yml

@@ -25,6 +25,10 @@ services:
     logging: *default-logging
     environment:
       - SIA_MODULES=gctwra
+      - SKYD_DISK_CACHE_ENABLED=false
+      - SKYD_DISK_CACHE_SIZE=53690000000 # 50GB
+      - SKYD_DISK_CACHE_MIN_HITS=3
+      - SKYD_DISK_CACHE_HIT_PERIOD=3600 # 1h
 
     env_file:
       - .env

docker/nginx/conf.d/server/server.local

@@ -1,6 +1,9 @@
 include /etc/nginx/conf.d/include/init-optional-variables;
 
 location /skynet/blocklist {
+    client_max_body_size 10m; # increase max body size to account for large lists
+    client_body_buffer_size 10m; # force whole body to memory so we can read it
+
     content_by_lua_block {        
         local httpc = require("resty.http").new()
 

packages/website/data/news/skynets-approach-to-building-a-better-web3/index.md

@@ -0,0 +1,10 @@
+---
+title: "Skynet’s Approach to Building a Better Web3"
+date: "2022-01-11"
+description: An open response to the well-worded, thorough, and unexpectedly polite blog post by Moxie Marlinspike…
+thumbnail: ./thumbnail.png
+categories: ["blog"]
+author: David Vorick
+avatar: ../../team/david-vorick.png
+external: https://blog.sia.tech/skynets-approach-to-building-a-better-web3-c2b10623d6dd
+---

packages/website/package.json

@@ -68,7 +68,7 @@
     "cypress": "9.2.0",
     "cypress-file-upload": "5.0.8",
     "prettier": "2.5.1",
-    "tailwindcss": "3.0.12"
+    "tailwindcss": "3.0.11"
   },
   "keywords": [
     "gatsby"

packages/website/yarn.lock

@@ -10461,7 +10461,7 @@ postcss-reduce-transforms@^5.0.1:
     cssnano-utils "^2.0.1"
     postcss-value-parser "^4.1.0"
 
-postcss-selector-parser@^6.0.2, postcss-selector-parser@^6.0.4, postcss-selector-parser@^6.0.5, postcss-selector-parser@^6.0.6, postcss-selector-parser@^6.0.8:
+postcss-selector-parser@^6.0.2, postcss-selector-parser@^6.0.4, postcss-selector-parser@^6.0.5, postcss-selector-parser@^6.0.6, postcss-selector-parser@^6.0.7:
   version "6.0.8"
   resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.8.tgz#f023ed7a9ea736cd7ef70342996e8e78645a7914"
   integrity sha512-D5PG53d209Z1Uhcc0qAZ5U3t5HagH3cxu+WLZ22jt3gLUpXM4eXXfiO14jiDWST3NNooX/E8wISfOhZ9eIjGTQ==
@@ -12575,10 +12575,10 @@ table@^6.0.9:
     string-width "^4.2.3"
     strip-ansi "^6.0.1"
 
-tailwindcss@3.0.12:
-  version "3.0.12"
-  resolved "https://registry.yarnpkg.com/tailwindcss/-/tailwindcss-3.0.12.tgz#b43bf952dbfd62cec087319748eb69f8e1c7855d"
-  integrity sha512-VqhF86z2c34sJyS5ZS8Q2nYuN0KzqZw1GGsuQQO9kJ3mY1oG7Fsag0vICkxUVXk6P+1sUkTkjMjKWCjEF0hNHw==
+tailwindcss@3.0.11:
+  version "3.0.11"
+  resolved "https://registry.yarnpkg.com/tailwindcss/-/tailwindcss-3.0.11.tgz#c4e96cada1f693cba66269eef80c74d22331c767"
+  integrity sha512-JyMsQ2kPqpOvG8ow535XpauXj3wz3nQqcy2tVlXj4FQ0eNlsdzvlAqpRA3q5rPLboWirNG6r2DqKczwjW2uc8Q==
   dependencies:
     arg "^5.0.1"
     chalk "^4.1.2"
@@ -12596,7 +12596,7 @@ tailwindcss@3.0.12:
     postcss-js "^4.0.0"
     postcss-load-config "^3.1.0"
     postcss-nested "5.0.6"
-    postcss-selector-parser "^6.0.8"
+    postcss-selector-parser "^6.0.7"
     postcss-value-parser "^4.2.0"
     quick-lru "^5.1.1"
     resolve "^1.20.0"

scripts/blocklist-skylink.sh

@@ -1,18 +1,15 @@
 #! /usr/bin/env bash
 
-# This script adds a skylink to the sia blocklist and removes the skylink from
-# nginx cache. The script should be run locally on each skynet webportal
-# server. The automatic script that is used to continuously sync an Airtable
-# sheet list with the blocklist on the web portals is
-# /setup-scripts/blocklist-airtable.py 
+# This script is for manual skylink blocking. It accepts either a single 
+# skylink or a file containing list of skylinks. The script is intented
+# for manual use and it should be run locally on each skynet webportal server. 
+# The automatic script that is used to continuously sync an Airtable sheet 
+# list with the blocklist on the web portals is /setup-scripts/blocklist-airtable.py 
 
 set -e # exit on first error
 
-# Number of skylinks to block within one batch
-BATCH_SIZE=1000
-
 if [ -z "$1" ]; then
-    echo "Please provide either a skylink or file with skylinks separated by new lines" && exit 1
+    echo "Please provide either a skylink or a file with skylinks separated by new lines" && exit 1
 fi
 
 #########################################################
@@ -37,45 +34,18 @@ else
     skylinks=("$1") # just single skylink passed as input argument
 fi
 
-# Block skylinks in batches
-skylinks_len=${#skylinks[@]}
-for (( i = 0; i < $skylinks_len; i++ )); do
-    # Add skylink to batch
-    skylink="${skylinks[$i]}"
-    echo ".. ⌁ Adding skylink ${skylink} to batch..."
-    batch_skylinks+=("$skylink")
+# get local nginx ip adress
+nginx_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx)
 
-    # For performance reasons on each iteration we do not block a single
-    # skylink, but we block skylinks in batches with BATCH_SIZE size mainly
-    # because of nginx cache search.
-    # If (batch len == batch size) or (we have last batch):
-    if (( ${#batch_skylinks[@]} == $BATCH_SIZE || $i == $skylinks_len - 1 )); then
-        echo "--------------------------------------------"
+# iterate over provided skylinks and block them one by one
+for skylink in "${skylinks[@]}"; do
+    printf "Blocking ${skylink} ... "
+    status_code=$(curl --write-out '%{http_code}' --silent --output /dev/null --data "{\"add\":[\"$skylink\"]}" "http://${nginx_ip}:8000/skynet/blocklist")
 
-        # Add to Sia blocklist
-        echo "Blocking batch skylinks in skyd..."
-        skylinks_space_separated="$(IFS=' '; echo "${batch_skylinks[*]}")"
-        docker exec sia siac skynet blocklist add $skylinks_space_separated
-
-        # Remove from NGINX cache
-        # NOTE:
-        # If there are changes to how the NGINX cache is being cleared, the same
-        # changes need to be applied to the /setup-scripts/blocklist-airtable.py
-        # script.
-        echo "Removing batch skylinks from Nginx cache..."
-        skylinks_pipe_separated="$(IFS='|'; echo "${batch_skylinks[*]}")"
-        cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ($skylinks_pipe_separated)'"
-        docker exec -it nginx bash -c "${cached_files_command} | xargs -r rm"
-
-        # Clear batch
-        batch_skylinks=()
-
-        echo "--------------------------------------------"
+    # print blocklist response status code
+    if [ $status_code = "204" ]; then
+        echo "done"
+    else
+        echo "error $status_code"
     fi
 done
-
-# Hot reload Nginx to get rid of deleted open files
-echo "Hot reloading nginx..."
-docker exec nginx nginx -s reload
-
-echo "✓ All done !"

setup-scripts/blocklist-airtable.py

@@ -122,63 +122,30 @@ async def block_skylinks_from_airtable():
         )
         await send_msg(message, file=("\n".join(invalid_skylinks)))
 
-    apipassword = exec("docker exec sia cat /sia-data/apipassword")
     ipaddress = exec(
-        "docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia"
+        "docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx"
     )
 
-    print("Sending blocklist request to siad")
+    print("Sending blocklist request to siad through nginx")
     response = requests.post(
-        "http://" + ipaddress + ":9980/skynet/blocklist",
-        auth=("", apipassword),
-        headers={"user-agent": "Sia-Agent"},
+        "http://" + ipaddress + ":8000/skynet/blocklist",
         data=json.dumps({"add": skylinks}),
     )
 
-    if response.status_code == 204:
-        print("Siad blocklist successfully updated with provided skylink")
-    else:
+    print(json.dumps({"add": skylinks}))
+
+    if response.status_code != 204:
         status_code = str(response.status_code)
         response_text = response.text or "empty response"
         message = (
-            "Siad blocklist endpoint responded with code "
+            "Airtable blocklist request responded with code "
             + status_code
             + ": "
             + response_text
         )
         return await send_msg(message, force_notify=False)
 
-    # Remove from NGINX cache
-    # NOTE:
-    # If there are changes to how the NGINX cache is being cleared, the same
-    # changes need to be applied to the /scripts/blocklist-skylink.sh script.
-    print("Searching nginx cache for blocked files")
-    cached_files_count = 0
-    batch_size = 1000
-    for i in range(0, len(skylinks), batch_size):
-        cached_files_command = (
-            "find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ("
-            + "|".join(skylinks[i : i + batch_size])
-            + ")'"
-        )
-        cached_files_count += int(
-            exec(
-                'docker exec nginx bash -c "'
-                + cached_files_command
-                + ' | xargs -r rm -v | wc -l"'
-            )
-        )
-
-    if cached_files_count == 0:
-        return print("No nginx cached files matching blocked skylinks were found")
-    else:
-        print("Hot reloading nginx")
-        exec("docker exec nginx nginx -s reload")
-
-    message = (
-        "Purged " + str(cached_files_count) + " blocklisted files from nginx cache"
-    )
-    return await send_msg(message)
+    return await send_msg("Siad blocklist successfully updated with provided skylink")
 
 
 loop = asyncio.get_event_loop()
@@ -186,6 +153,5 @@ loop.run_until_complete(run_checks())
 
 # --- BASH EQUIVALENT
 # skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")
-# apipassword=$(docker exec sia cat /sia-data/apipassword)
-# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia)
-# curl -A "Sia-Agent" --user "":"${apipassword}" --data "{\"add\" : ${skylinks}}" "${ipaddress}:9980/skynet/blocklist"
+# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx)
+# curl --data "{\"add\" : ${skylinks}}" "${ipaddress}:8000/skynet/blocklist"