Merge remote-tracking branch 'origin/master' into accounts

packages/webapp/package.json

@@ -4,12 +4,12 @@
   "version": "1.0.0",
   "author": "Nebulous",
   "dependencies": {
+    "@fontsource/metropolis": "^4.2.1",
     "axios": "0.21.1",
     "boolean": "^3.0.2",
     "bytes": "3.1.0",
     "classnames": "2.2.6",
-    "fontsource-metropolis": "4.0.0",
+    "gatsby": "^2.32.4",
-    "gatsby": "2.32.3",
     "gatsby-image": "2.11.0",
     "gatsby-plugin-manifest": "2.12.0",
     "gatsby-plugin-matomo": "0.9.0",
@@ -39,7 +39,7 @@
     "cypress": "6.5.0",
     "cypress-file-upload": "5.0.2",
     "eslint": "7.20.0",
-    "eslint-config-prettier": "7.2.0",
+    "eslint-config-prettier": "8.0.0",
     "eslint-plugin-cypress": "2.11.2",
     "eslint-plugin-react": "7.22.0",
     "husky": "4.3.8",

packages/webapp/src/components/Footer/Footer.js

@@ -59,7 +59,7 @@ export default function Footer() {
               </a>
             </li>
             <li>
-              <a href="https://sia.tech/docs/#skynet" target="_blank" rel="noopener noreferrer">
+              <a href="https://siasky.net/docs" target="_blank" rel="noopener noreferrer">
                 API Docs
               </a>
             </li>
@@ -74,11 +74,11 @@ export default function Footer() {
             </li>
             <li>
               <a
-                href="https://support.siasky.net/article/vmmzyes1uy-skynet-sia-set-up"
+                href="https://support.siasky.net/key-concepts/skynet-portals/using-a-sia-node-as-a-portal"
                 target="_blank"
                 rel="noopener noreferrer"
               >
-                Sia Node Setup
+                Accessing Skynet Locally
               </a>
             </li>
           </ul>

packages/webapp/src/components/HomeNetwork/HomeNetwork.js

@@ -98,7 +98,7 @@ export default function HomeNetwork() {
               Anyone can access files that have been uploaded to Skynet as long as they possess the corresponding
               Skylinks. You can use any Webportal to download files!
               <a
-                href="https://skynet.helpdocs.io/article/3p9z5g9s0e-skynet-how-to"
+                href="https://support.siasky.net/using-skynet"
                 target="_blank"
                 rel="noopener noreferrer"
                 className="more read-more"
@@ -113,7 +113,7 @@ export default function HomeNetwork() {
               Applications can be deployed in under a minute and be immediately available globally. Skynet includes an
               API and SDKs which integrate seamlessly with existing applications.
               <a
-                href="https://skynet.helpdocs.io/article/hrshqsn9wz-integrating-skynet"
+                href="https://support.siasky.net/the-technology/developing-on-skynet"
                 target="_blank"
                 rel="noopener noreferrer"
                 className="more read-more"
@@ -128,7 +128,7 @@ export default function HomeNetwork() {
               Skynet&apos;s speeds rival centralized providers and surpass all decentralized offerings. A typical Skynet
               download starts in under 500 ms and can stream at rates as high as 1 Gbps!
               <a
-                href="https://skynet.helpdocs.io/article/430teoxgqc-skynet-speed"
+                href="https://support.siasky.net/key-concepts/faqs#how-fast-is-skynet"
                 target="_blank"
                 rel="noopener noreferrer"
                 className="more read-more"

packages/webapp/src/components/HomeSamples/HomeSamples.js

@@ -23,7 +23,7 @@ export default function HomeSamples() {
           SDKs
         </a>{" "}
         for popular programming languages and{" "}
-        <a href="https://sia.tech/docs/#skynet" target="_blank" rel="noopener noreferrer" className="link">
+        <a href="https://siasky.net/docs/" target="_blank" rel="noopener noreferrer" className="link">
           APIs
         </a>{" "}
         that integrate seamlessly with your existing apps. You can follow these guides to start using Skynet with{" "}

packages/webapp/src/pages/index.js

@@ -4,7 +4,7 @@ import SEO from "../components/seo";
 import { App } from "../components";
 import "../global.scss";
 import AppContext from "../AppContext";
-import "fontsource-metropolis/all.css"; // import Metropolis typeface
+import "@fontsource/metropolis/all.css"; // import Metropolis typeface
 
 export default function IndexPage({ location }) {
   const context = useMemo(

setup-scripts/blocklist-airtable.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+
+import traceback, os, re, asyncio, requests, json, discord
+from bot_utils import setup, send_msg
+
+bot_token = setup()
+client = discord.Client()
+
+AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY")
+AIRTABLE_BASE = os.getenv("AIRTABLE_BASE", "app89plJvA9EqTJEc")
+AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE", "Table%201")
+AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD", "Link")
+
+
+def exec(command):
+    return os.popen(command).read().strip()
+
+
+async def block_skylinks_from_airtable():
+    print("Pulling blocked skylinks from Airtable via api integration")
+    headers = {"Authorization": "Bearer " + AIRTABLE_API_KEY}
+    skylinks = []
+    offset = None
+    while len(skylinks) == 0 or offset:
+        print("Requesting a batch of records from Airtable with " + (offset if offset else "empty") + " offset")
+        query = "&".join(["fields%5B%5D=" + AIRTABLE_FIELD, ("offset=" + offset) if offset else ""])
+        response = requests.get(
+            "https://api.airtable.com/v0/" + AIRTABLE_BASE + "/" + AIRTABLE_TABLE + "?" + query,
+            headers=headers,
+        )
+
+        if response.status_code != 200:
+            status_code = str(response.status_code)
+            response_text = response.text or "empty response"
+            message = "Airtable blocklist integration responded with code " + status_code + ": " + response_text
+            return print(message) or await send_msg(client, message, force_notify=False)
+
+        data = response.json()
+        skylinks = skylinks + [entry["fields"][AIRTABLE_FIELD] for entry in data["records"]]
+
+        if len(skylinks) == 0:
+            return print("Airtable returned 0 skylinks - make sure your configuration is correct")
+
+        offset = data.get("offset")
+
+    print("Airtable returned total " + str(len(skylinks)) + " skylinks to block")
+
+    skylinks_returned = skylinks
+    skylinks = [skylink for skylink in skylinks if re.search("^[a-zA-Z0-9_-]{46}$", skylink)]
+
+    if len(skylinks_returned) != len(skylinks):
+        invalid_skylinks = [str(skylink) for skylink in list(set(skylinks_returned) - set(skylinks))]
+        message = str(len(invalid_skylinks)) + " of the skylinks returned from Airtable are not valid"
+        print(message) or await send_msg(client, message, file=("\n".join(invalid_skylinks)))
+
+    apipassword = exec("docker exec sia cat /sia-data/apipassword")
+    ipaddress = exec("docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia")
+
+    print("Sending blocklist request to siad")
+    response = requests.post(
+        "http://" + ipaddress + ":9980/skynet/blocklist",
+        auth=("", apipassword),
+        headers={"user-agent": "Sia-Agent"},
+        data=json.dumps({"add": skylinks}),
+    )
+
+    if response.status_code == 204:
+        print("Siad blocklist successfully updated with provided skylink")
+    else:
+        status_code = str(response.status_code)
+        response_text = response.text or "empty response"
+        message = "Siad blocklist endpoint responded with code " + status_code + ": " + response_text
+        return print(message) or await send_msg(client, message, force_notify=False)
+
+    print("Searching nginx cache for blocked files")
+    cached_files_command = (
+        "/usr/bin/find /data/nginx/cache/ -type f | /usr/bin/xargs --no-run-if-empty -n1000 /bin/grep -Els '^KEY: .*("
+        + "|".join(skylinks)
+        + ")'"
+    )
+    cached_files_count = int(exec('docker exec -it nginx bash -c "' + cached_files_command + ' | wc -l"'))
+
+    if cached_files_count == 0:
+        return print("No nginx cached files matching blocked skylinks were found")
+
+    exec('docker exec -it nginx bash -c "' + cached_files_command + ' | xargs rm"')
+    message = "Purged " + str(cached_files_count) + " blocklisted files from nginx cache"
+    return print(message) or await send_msg(client, message)
+
+
+async def exit_after(delay):
+    await asyncio.sleep(delay)
+    os._exit(0)
+
+
+@client.event
+async def on_ready():
+    try:
+        await block_skylinks_from_airtable()
+    except:  # catch all exceptions
+        message = "```\n{}\n```".format(traceback.format_exc())
+        await send_msg(client, message, force_notify=False)
+    asyncio.create_task(exit_after(3))
+
+
+client.run(bot_token)
+
+# --- BASH EQUIVALENT
+# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")
+# apipassword=$(docker exec sia cat /sia-data/apipassword)
+# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia)
+# curl -A "Sia-Agent" --user "":"${apipassword}" --data "{\"add\" : ${skylinks}}" "${ipaddress}:9980/skynet/blocklist"

setup-scripts/support/authorized_keys

@@ -2,7 +2,7 @@ ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCpBsw5mPBVIvVd5GX43VXWHWuLeR2h0lfw8vRyDFgm
 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCr3nrSQ+ag5gEm9LBoiw68UKALboot+Eemv0TbP6BPnvB6bnSdDstb7Eu1Dkla8uiyw3w2ZYi5Pg4dS5W8vnxwXvey8gBd3GYLpjtnSp9ukeYjHK0J2aX4PBC4GXvRSRjKxYfHauUqm8PaA4uQ4sBkblfwWDEH94um1yyqIamTabH6mfsYiaiiwTNu7ldZOAIlKR/G7cXlLmFz46An7Mn2wwbuv2Khin/f2bLtUF/smOolI7pjOH6ifhHR9LxotcY/xL+E5jRbU1XxldFvVXkL5CU8tEinE6oigwMH9zsPZr+Z70Q/wm20cylxNJu8qdMGQW+WhDg3S70KpCmjYlWJ6bF1HL3z9UkN0lS1EM21n13RIx1iEO7SEC3YPl8VqZiZS7P9Uf5D5z/vTG+fWouCsCBMSbq3HUcNXlm5MLGSdBWPKzZsUaCkHkQks/sxHVy21YAM/3xgST1a05PbIJU1RsqJ0wh0J2gg7/fBUE0ljFyKZ36mvfg6BNlwCUydAiVaQt1geqh+8/VRwjTw/jtHb8G7QhSNwDNo1BcQPU3LkdKePqgldyP5EYGl9bI4E4sYc2DooeJ22fXpWfuClLB+JcHGuCJf/Hg6si9IeeXKm8PwaBdxIVytRPEeJR+q5uOwzI4XWNgERdGU/UVbgfnrAPMuVPa9Jhyl96U9uUl+Cw== peterjan.brone@gmail.com
 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDMmZFdJsqig/zX7Ly7qJMMDomsdAKLgl7W7ET1w7xH9BBM48OCWmozuLCfCG8MUCEYSUD575hA028hNi6CAK40J3fF74IDDyc9DUb+le8Y8EuzHPKxYLE/gWsjr70XOcZcC4IxLcADQgpeLjrPZQs7A4EYfdxnTTLJVYrowZ9RR5ivcKBjyFOiQyCuFSIvtYMo11Xm2gU48SKYGJThhHUiE2kMOlH3notXJ+T81927IGJdza7J3DAyKtMGB2HEMA89ma3mvEvbPTDMggJFJ3VG7sukRLq6UmT7BT+f3BW+Nr87A1o4upkAuXdkL9cUrris7kQN61AcaCNFU/CuIJa4dUZ0nt+z5X7kWtc0zD75EPj3w6AjB+E1+MSPsqnxd5PnGtSCQqHoa5hg4hQMSweC2tQhSKoWDfx9W2fZiLpg1IL6QB5xCxjg+YKCXEJKxRwXDtbh1DHFdJ5N1kM7IDSeeblc80HNxYrJUPNH1ExWsPl11gmBEEWDAiRSet4bAnOmgDYcJ9Aw2KAndb01cNsw5RL0Dg/W63tb8S5Y9kz6spX6X91yz53JzrozZO7VFfKxa17nubPEeWPTqAQ3uRWPvpdbivVnOAoFCLacRvtTfvetuz/vGZ3JTpr6Ylb9Z76cIqpFe70+bnauZwmxjF+EEq2+u3gd2uewuV2//o+CYQ== kwypchlo@gmail.com
 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDgiq1etF0aD94rG/UVmYEt4ij5K8MvHZwb4wIUi6Ihr david@nebulouslabs.com
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDFO8IxPO3CjfBzm3rAI0sof0BuEBVj6g1UY4hEm9Wn3PXx/iHn96ZP/nSh37X5e5KABCq7ob18T16B4U9JVlARvpozvCCUso28C/Vm44Vt/Q4xoQAYX4eLlRGkPJHhEtA+GhTt4HSE06IZkegAlZ6HVSpSxNiFmSWytIQIa2uTVDel16U+N0PiwQ/9ZS6c/MeC6ZebVEeyEBHNTOL3vkrtFzD/Iupi4QKASK8ejCKEnzCjwoWNyZPUJJLwyUC1ttZOH0cKQid9rcwQDqwM6clnJ5OAAdMkD9GbHs1ItyeC5M1m/KwunmlGSc1eIpIYLvp/0cHrh6/0j8utO3hkqDD3pTWP8TEzw2f5TQVlFHNotcNZimJz8XU8X2k2fHTgyoYKL12HjhokObqBHBUAXol9vCkw0z05U8wVcBemzHrI+6GHnn2pLETshd8Ar8bJ0wQ08+3Agf+KmJuVoHOFdc314AkUX/5QHMrws1/GeS8urLR9FciEiUks8X790LF/sP0= cschinnerl@Christophers-MacBook-Pro.local
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAyIT2HqzDhQs6jS89ZsnY6+GJEklVMqF6fXe/i5s8d7 chris@nebulous.tech
 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFxLuZzjmFN9CgVOI5vaiVhQgMwG9dLQJ688wrsbpHH/ ivaylo@nebulous.tech
 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDB/sC5S9SdtU/XxTqAGHEakxgGaSLfg3yudR2xzffnW3GvPIRUjQjP3aAmfNcqa11MrwMkBn0tsMYG07+w1VLNjnETLzRwoFB7nfs34x0DQoZZj3EUhdPx/WRHMKEbu5tXX5l0a8A4tJoAhY6FAkenq4DOAI3YrXOInB3NGHj69ZsGi91Fb9UUUZPBRyhtU2ReQTivPs7ICam/FpP9e8h6uaReu8lSkf6HdwaTjPu+d+PXljw0lgQKKmQmKkc6SCUX+dZKK9yPHw6NWndeoocrnLaa7hgP1UhaPKhTWkF42lMAjrRUpef6iyvJVrP/Yg/7iyOV2UyIIoEy4ZLjmoMuCrv8tXTn3YuHttyNzeVkmhqu7rr5ZELhGqaqN+PJVZCE7iEmtP9x3ZdBip6L7BVB/2FCRz2bvjhdKdNvb5SaqxAjHeU2MGsWGoMM1z8ntq+JHOsrVrkz+OUEHjFXhjtQBLZS1UlrfCkuLvsV8lhSqCxwl0G+ZWkStH7RBzOVGBHRmvDnkPQ0txY45HQhXLkkgCmX+fvx1v2v9hL7DK+ZAvMcQYxvek8Z7HNEdFx/EKaqJkpbZoMWd7bA9bnh6kht1ciPgdFOvjU9NdGOS5bepLzJ1ovr47bucQShjFbmcjdHCDiw8H2ZZeQsvmo38TJZSRF0maDk6slXc6glwELxlw== mjsevey@gmail.com
 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINbAhwjJNAud7YIJvLth2bmeUg3kO20xl7ZfqBTvoXn8 Filip Rysavy

setup-scripts/support/crontab

@@ -1,3 +1,4 @@
 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/skynet-webportal/.env
 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/skynet-webportal/.env sia 8
 0 * * * * /home/user/skynet-webportal/setup-scripts/health-checker.py /home/user/skynet-webportal/.env sia 1
+30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env