diff --git a/docker-compose.yml b/docker-compose.yml index 6f399cfe..20d38a4f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,6 +25,10 @@ services: logging: *default-logging environment: - SIA_MODULES=gctwra + - SKYD_DISK_CACHE_ENABLED=false + - SKYD_DISK_CACHE_SIZE=53690000000 # 50GB + - SKYD_DISK_CACHE_MIN_HITS=3 + - SKYD_DISK_CACHE_HIT_PERIOD=3600 # 1h env_file: - .env diff --git a/docker/nginx/conf.d/server/server.local b/docker/nginx/conf.d/server/server.local index 1de4ab72..3a8ac118 100644 --- a/docker/nginx/conf.d/server/server.local +++ b/docker/nginx/conf.d/server/server.local @@ -1,6 +1,9 @@ include /etc/nginx/conf.d/include/init-optional-variables; location /skynet/blocklist { + client_max_body_size 10m; # increase max body size to account for large lists + client_body_buffer_size 10m; # force whole body to memory so we can read it + content_by_lua_block { local httpc = require("resty.http").new() diff --git a/packages/website/data/news/skynets-approach-to-building-a-better-web3/index.md b/packages/website/data/news/skynets-approach-to-building-a-better-web3/index.md new file mode 100644 index 00000000..676c766e --- /dev/null +++ b/packages/website/data/news/skynets-approach-to-building-a-better-web3/index.md @@ -0,0 +1,10 @@ +--- +title: "Skynet’s Approach to Building a Better Web3" +date: "2022-01-11" +description: An open response to the well-worded, thorough, and unexpectedly polite blog post by Moxie Marlinspike… +thumbnail: ./thumbnail.png +categories: ["blog"] +author: David Vorick +avatar: ../../team/david-vorick.png +external: https://blog.sia.tech/skynets-approach-to-building-a-better-web3-c2b10623d6dd +--- diff --git a/packages/website/data/news/skynets-approach-to-building-a-better-web3/thumbnail.png b/packages/website/data/news/skynets-approach-to-building-a-better-web3/thumbnail.png new file mode 100644 index 00000000..2641e6db Binary files /dev/null and b/packages/website/data/news/skynets-approach-to-building-a-better-web3/thumbnail.png differ diff --git a/packages/website/package.json b/packages/website/package.json index 4ccd5fa4..a7aa7157 100644 --- a/packages/website/package.json +++ b/packages/website/package.json @@ -68,7 +68,7 @@ "cypress": "9.2.0", "cypress-file-upload": "5.0.8", "prettier": "2.5.1", - "tailwindcss": "3.0.12" + "tailwindcss": "3.0.11" }, "keywords": [ "gatsby" diff --git a/packages/website/yarn.lock b/packages/website/yarn.lock index 485e6d30..948b71b7 100644 --- a/packages/website/yarn.lock +++ b/packages/website/yarn.lock @@ -10461,7 +10461,7 @@ postcss-reduce-transforms@^5.0.1: cssnano-utils "^2.0.1" postcss-value-parser "^4.1.0" -postcss-selector-parser@^6.0.2, postcss-selector-parser@^6.0.4, postcss-selector-parser@^6.0.5, postcss-selector-parser@^6.0.6, postcss-selector-parser@^6.0.8: +postcss-selector-parser@^6.0.2, postcss-selector-parser@^6.0.4, postcss-selector-parser@^6.0.5, postcss-selector-parser@^6.0.6, postcss-selector-parser@^6.0.7: version "6.0.8" resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.8.tgz#f023ed7a9ea736cd7ef70342996e8e78645a7914" integrity sha512-D5PG53d209Z1Uhcc0qAZ5U3t5HagH3cxu+WLZ22jt3gLUpXM4eXXfiO14jiDWST3NNooX/E8wISfOhZ9eIjGTQ== @@ -12575,10 +12575,10 @@ table@^6.0.9: string-width "^4.2.3" strip-ansi "^6.0.1" -tailwindcss@3.0.12: - version "3.0.12" - resolved "https://registry.yarnpkg.com/tailwindcss/-/tailwindcss-3.0.12.tgz#b43bf952dbfd62cec087319748eb69f8e1c7855d" - integrity sha512-VqhF86z2c34sJyS5ZS8Q2nYuN0KzqZw1GGsuQQO9kJ3mY1oG7Fsag0vICkxUVXk6P+1sUkTkjMjKWCjEF0hNHw== +tailwindcss@3.0.11: + version "3.0.11" + resolved "https://registry.yarnpkg.com/tailwindcss/-/tailwindcss-3.0.11.tgz#c4e96cada1f693cba66269eef80c74d22331c767" + integrity sha512-JyMsQ2kPqpOvG8ow535XpauXj3wz3nQqcy2tVlXj4FQ0eNlsdzvlAqpRA3q5rPLboWirNG6r2DqKczwjW2uc8Q== dependencies: arg "^5.0.1" chalk "^4.1.2" @@ -12596,7 +12596,7 @@ tailwindcss@3.0.12: postcss-js "^4.0.0" postcss-load-config "^3.1.0" postcss-nested "5.0.6" - postcss-selector-parser "^6.0.8" + postcss-selector-parser "^6.0.7" postcss-value-parser "^4.2.0" quick-lru "^5.1.1" resolve "^1.20.0" diff --git a/scripts/blocklist-skylink.sh b/scripts/blocklist-skylink.sh index 93a36a61..ee19c9c2 100755 --- a/scripts/blocklist-skylink.sh +++ b/scripts/blocklist-skylink.sh @@ -1,18 +1,15 @@ #! /usr/bin/env bash -# This script adds a skylink to the sia blocklist and removes the skylink from -# nginx cache. The script should be run locally on each skynet webportal -# server. The automatic script that is used to continuously sync an Airtable -# sheet list with the blocklist on the web portals is -# /setup-scripts/blocklist-airtable.py +# This script is for manual skylink blocking. It accepts either a single +# skylink or a file containing list of skylinks. The script is intented +# for manual use and it should be run locally on each skynet webportal server. +# The automatic script that is used to continuously sync an Airtable sheet +# list with the blocklist on the web portals is /setup-scripts/blocklist-airtable.py set -e # exit on first error -# Number of skylinks to block within one batch -BATCH_SIZE=1000 - if [ -z "$1" ]; then - echo "Please provide either a skylink or file with skylinks separated by new lines" && exit 1 + echo "Please provide either a skylink or a file with skylinks separated by new lines" && exit 1 fi ######################################################### @@ -37,45 +34,18 @@ else skylinks=("$1") # just single skylink passed as input argument fi -# Block skylinks in batches -skylinks_len=${#skylinks[@]} -for (( i = 0; i < $skylinks_len; i++ )); do - # Add skylink to batch - skylink="${skylinks[$i]}" - echo ".. ⌁ Adding skylink ${skylink} to batch..." - batch_skylinks+=("$skylink") +# get local nginx ip adress +nginx_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx) - # For performance reasons on each iteration we do not block a single - # skylink, but we block skylinks in batches with BATCH_SIZE size mainly - # because of nginx cache search. - # If (batch len == batch size) or (we have last batch): - if (( ${#batch_skylinks[@]} == $BATCH_SIZE || $i == $skylinks_len - 1 )); then - echo "--------------------------------------------" +# iterate over provided skylinks and block them one by one +for skylink in "${skylinks[@]}"; do + printf "Blocking ${skylink} ... " + status_code=$(curl --write-out '%{http_code}' --silent --output /dev/null --data "{\"add\":[\"$skylink\"]}" "http://${nginx_ip}:8000/skynet/blocklist") - # Add to Sia blocklist - echo "Blocking batch skylinks in skyd..." - skylinks_space_separated="$(IFS=' '; echo "${batch_skylinks[*]}")" - docker exec sia siac skynet blocklist add $skylinks_space_separated - - # Remove from NGINX cache - # NOTE: - # If there are changes to how the NGINX cache is being cleared, the same - # changes need to be applied to the /setup-scripts/blocklist-airtable.py - # script. - echo "Removing batch skylinks from Nginx cache..." - skylinks_pipe_separated="$(IFS='|'; echo "${batch_skylinks[*]}")" - cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ($skylinks_pipe_separated)'" - docker exec -it nginx bash -c "${cached_files_command} | xargs -r rm" - - # Clear batch - batch_skylinks=() - - echo "--------------------------------------------" + # print blocklist response status code + if [ $status_code = "204" ]; then + echo "done" + else + echo "error $status_code" fi done - -# Hot reload Nginx to get rid of deleted open files -echo "Hot reloading nginx..." -docker exec nginx nginx -s reload - -echo "✓ All done !" diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py index 9ec3314c..9ec81952 100755 --- a/setup-scripts/blocklist-airtable.py +++ b/setup-scripts/blocklist-airtable.py @@ -122,63 +122,30 @@ async def block_skylinks_from_airtable(): ) await send_msg(message, file=("\n".join(invalid_skylinks))) - apipassword = exec("docker exec sia cat /sia-data/apipassword") ipaddress = exec( - "docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia" + "docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx" ) - print("Sending blocklist request to siad") + print("Sending blocklist request to siad through nginx") response = requests.post( - "http://" + ipaddress + ":9980/skynet/blocklist", - auth=("", apipassword), - headers={"user-agent": "Sia-Agent"}, + "http://" + ipaddress + ":8000/skynet/blocklist", data=json.dumps({"add": skylinks}), ) - if response.status_code == 204: - print("Siad blocklist successfully updated with provided skylink") - else: + print(json.dumps({"add": skylinks})) + + if response.status_code != 204: status_code = str(response.status_code) response_text = response.text or "empty response" message = ( - "Siad blocklist endpoint responded with code " + "Airtable blocklist request responded with code " + status_code + ": " + response_text ) return await send_msg(message, force_notify=False) - # Remove from NGINX cache - # NOTE: - # If there are changes to how the NGINX cache is being cleared, the same - # changes need to be applied to the /scripts/blocklist-skylink.sh script. - print("Searching nginx cache for blocked files") - cached_files_count = 0 - batch_size = 1000 - for i in range(0, len(skylinks), batch_size): - cached_files_command = ( - "find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: (" - + "|".join(skylinks[i : i + batch_size]) - + ")'" - ) - cached_files_count += int( - exec( - 'docker exec nginx bash -c "' - + cached_files_command - + ' | xargs -r rm -v | wc -l"' - ) - ) - - if cached_files_count == 0: - return print("No nginx cached files matching blocked skylinks were found") - else: - print("Hot reloading nginx") - exec("docker exec nginx nginx -s reload") - - message = ( - "Purged " + str(cached_files_count) + " blocklisted files from nginx cache" - ) - return await send_msg(message) + return await send_msg("Siad blocklist successfully updated with provided skylink") loop = asyncio.get_event_loop() @@ -186,6 +153,5 @@ loop.run_until_complete(run_checks()) # --- BASH EQUIVALENT # skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')") -# apipassword=$(docker exec sia cat /sia-data/apipassword) -# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia) -# curl -A "Sia-Agent" --user "":"${apipassword}" --data "{\"add\" : ${skylinks}}" "${ipaddress}:9980/skynet/blocklist" +# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nginx) +# curl --data "{\"add\" : ${skylinks}}" "${ipaddress}:8000/skynet/blocklist"