Merge branch 'master' into add-shell-scripts-lint

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @kwypchlo @meeh0w

.github/workflows/codeql-analysis.yml

@@ -1,71 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ main ]
-  schedule:
-    - cron: '32 21 * * 0'
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'javascript', 'python' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
-        # Learn more:
-        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1

.github/workflows/lint-dockerfiles.yml

@@ -1,21 +0,0 @@
-name: Dockerfile Lint
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-
-jobs:
-  hadolint:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        dockerfile:
-          - docker/sia/Dockerfile
-    steps:
-      - uses: actions/checkout@v3
-      - uses: hadolint/hadolint-action@v2.0.0
-        with:
-          dockerfile: ${{ matrix.dockerfile }}

dc

@@ -5,56 +5,59 @@
 # would use docker-compose with the only difference being that you don't need to specify compose files. For more
 # information you can run `./dc` or `./dc help`.
 
-if [ -f .env ]; then
+# get current working directory of this script and prefix all files with it to
-  OLD_IFS=$IFS
+# be able to call this script from anywhere and not only root directory of
-  IFS=$'\n'
+# skynet-webportal project
-  for x in $(grep -v '^#.*' .env); do export $x; done
+cwd="$(dirname -- "$0";)";
-  IFS=$OLD_IFS
+
+# get portal modules configuration from .env file (if defined more than once, the last one is used)
+if [[ -f "${cwd}/.env" ]]; then
+  PORTAL_MODULES=$(grep -e "^PORTAL_MODULES=" ${cwd}/.env | tail -1 | sed "s/PORTAL_MODULES=//")
 fi
 
 # include base docker compose file
-COMPOSE_FILES="-f docker-compose.yml"
+COMPOSE_FILES="-f ${cwd}/docker-compose.yml"
 
 for i in $(seq 1 ${#PORTAL_MODULES}); do
   # accounts module - alias "a"
   if [[ ${PORTAL_MODULES:i-1:1} == "a" ]]; then
-    COMPOSE_FILES+=" -f docker-compose.mongodb.yml -f docker-compose.accounts.yml"
+    COMPOSE_FILES+=" -f ${cwd}/docker-compose.mongodb.yml -f ${cwd}/docker-compose.accounts.yml"
   fi
 
   # blocker module - alias "b"
   if [[ ${PORTAL_MODULES:i-1:1} == "b" ]]; then
-    COMPOSE_FILES+=" -f docker-compose.mongodb.yml -f docker-compose.blocker.yml"
+    COMPOSE_FILES+=" -f ${cwd}/docker-compose.mongodb.yml -f ${cwd}/docker-compose.blocker.yml"
   fi
 
   # jaeger module - alias "j"
   if [[ ${PORTAL_MODULES:i-1:1} == "j" ]]; then
-    COMPOSE_FILES+=" -f docker-compose.jaeger.yml"
+    COMPOSE_FILES+=" -f ${cwd}/docker-compose.jaeger.yml"
   fi
 
   # malware-scanner module - alias "s"
   if [[ ${PORTAL_MODULES:i-1:1} == "s" ]]; then
-    COMPOSE_FILES+=" -f docker-compose.blocker.yml -f docker-compose.mongodb.yml -f docker-compose.malware-scanner.yml"
+    COMPOSE_FILES+=" -f ${cwd}/docker-compose.blocker.yml -f ${cwd}/docker-compose.mongodb.yml -f ${cwd}/docker-compose.malware-scanner.yml"
   fi
 
   # mongodb module - alias "m"
   if [[ ${PORTAL_MODULES:i-1:1} == "m" ]]; then
-    COMPOSE_FILES+=" -f docker-compose.mongodb.yml"
+    COMPOSE_FILES+=" -f ${cwd}/docker-compose.mongodb.yml"
   fi
 
   # abuse-scanner module - alias "u"
   if [[ ${PORTAL_MODULES:i-1:1} == "u" ]]; then
-    COMPOSE_FILES+=" -f docker-compose.mongodb.yml -f docker-compose.blocker.yml -f docker-compose.abuse-scanner.yml"
+    COMPOSE_FILES+=" -f ${cwd}/docker-compose.mongodb.yml -f ${cwd}/docker-compose.blocker.yml -f ${cwd}/docker-compose.abuse-scanner.yml"
   fi
 
   # pinner module - alias "p"
   if [[ ${PORTAL_MODULES:i-1:1} == "p" ]]; then
-    COMPOSE_FILES+=" -f docker-compose.mongodb.yml -f docker-compose.pinner.yml"
+    COMPOSE_FILES+=" -f ${cwd}/docker-compose.mongodb.yml -f ${cwd}/docker-compose.pinner.yml"
   fi
 done
 
 # override file if exists
 if [[ -f docker-compose.override.yml ]]; then
-  COMPOSE_FILES+=" -f docker-compose.override.yml"
+  COMPOSE_FILES+=" -f ${cwd}/docker-compose.override.yml"
 fi
 
 docker-compose $COMPOSE_FILES $@

docker-compose.abuse-scanner.yml

@@ -1,4 +1,4 @@
-version: "3.7"
+version: "3.8"
 
 x-logging: &default-logging
   driver: json-file
@@ -10,7 +10,7 @@ services:
   abuse-scanner:
     # uncomment "build" and comment out "image" to build from sources
     # build: https://github.com/SkynetLabs/abuse-scanner.git#main
-    image: skynetlabs/abuse-scanner:0.1.1
+    image: skynetlabs/abuse-scanner:0.4.0
     container_name: abuse-scanner
     restart: unless-stopped
     logging: *default-logging
@@ -36,3 +36,6 @@ services:
     depends_on:
       - mongo
       - blocker
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /tmp:/tmp

docker-compose.accounts.yml

@@ -1,4 +1,4 @@
-version: "3.7"
+version: "3.8"
 
 x-logging: &default-logging
   driver: json-file
@@ -22,7 +22,7 @@ services:
   accounts:
     # uncomment "build" and comment out "image" to build from sources
     # build: https://github.com/SkynetLabs/skynet-accounts.git#main
-    image: skynetlabs/skynet-accounts:1.2.0
+    image: skynetlabs/skynet-accounts:1.3.0
     container_name: accounts
     restart: unless-stopped
     logging: *default-logging
@@ -59,15 +59,12 @@ services:
     # build:
     #   context: https://github.com/SkynetLabs/webportal-accounts-dashboard.git#main
     #   dockerfile: Dockerfile
-    image: skynetlabs/webportal-accounts-dashboard:1.1.1
+    image: skynetlabs/webportal-accounts-dashboard:2.1.0
     container_name: dashboard
     restart: unless-stopped
     logging: *default-logging
     env_file:
       - .env
-    environment:
-      - GATSBY_PORTAL_DOMAIN=${PORTAL_DOMAIN}
-      - GATSBY_STRIPE_PUBLISHABLE_KEY=${STRIPE_PUBLISHABLE_KEY}
     volumes:
       - ./docker/data/dashboard/.cache:/usr/app/.cache
       - ./docker/data/dashboard/public:/usr/app/public

docker-compose.blocker.yml

@@ -1,4 +1,4 @@
-version: "3.7"
+version: "3.8"
 
 x-logging: &default-logging
   driver: json-file
@@ -15,7 +15,7 @@ services:
   blocker:
     # uncomment "build" and comment out "image" to build from sources
     # build: https://github.com/SkynetLabs/blocker.git#main
-    image: skynetlabs/blocker:0.1.1
+    image: skynetlabs/blocker:0.1.2
     container_name: blocker
     restart: unless-stopped
     logging: *default-logging

docker-compose.jaeger.yml

@@ -1,4 +1,4 @@
-version: "3.7"
+version: "3.8"
 
 x-logging: &default-logging
   driver: json-file
@@ -21,7 +21,7 @@ services:
       - JAEGER_REPORTER_LOG_SPANS=false
 
   jaeger-agent:
-    image: jaegertracing/jaeger-agent:1.32.0
+    image: jaegertracing/jaeger-agent:1.37.0
     command:
       [
         "--reporter.grpc.host-port=jaeger-collector:14250",
@@ -43,7 +43,7 @@ services:
       - jaeger-collector
 
   jaeger-collector:
-    image: jaegertracing/jaeger-collector:1.32.0
+    image: jaegertracing/jaeger-collector:1.37.0
     entrypoint: /wait_to_start.sh
     container_name: jaeger-collector
     restart: on-failure
@@ -68,7 +68,7 @@ services:
       - elasticsearch
 
   jaeger-query:
-    image: jaegertracing/jaeger-query:1.32.0
+    image: jaegertracing/jaeger-query:1.37.0
     entrypoint: /wait_to_start.sh
     container_name: jaeger-query
     restart: on-failure
@@ -93,7 +93,7 @@ services:
       - elasticsearch
 
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.6
     container_name: elasticsearch
     restart: on-failure
     logging: *default-logging

docker-compose.malware-scanner.yml

@@ -1,4 +1,4 @@
-version: "3.7"
+version: "3.8"
 
 x-logging: &default-logging
   driver: json-file

docker-compose.mongodb.yml

@@ -1,4 +1,4 @@
-version: "3.7"
+version: "3.8"
 
 x-logging: &default-logging
   driver: json-file
@@ -14,7 +14,7 @@ services:
       - MONGODB_PASSWORD=${SKYNET_DB_PASS}
 
   mongo:
-    image: mongo:4.4.1
+    image: mongo:4.4.16
     command: --keyFile=/data/mgkey --replSet=${SKYNET_DB_REPLICASET:-skynet} --setParameter ShardingTaskExecutorPoolMinSize=10
     container_name: mongo
     restart: unless-stopped

docker-compose.pinner.yml

@@ -1,4 +1,4 @@
-version: "3.7"
+version: "3.8"
 
 x-logging: &default-logging
   driver: json-file
@@ -10,12 +10,14 @@ services:
   pinner:
     # uncomment "build" and comment out "image" to build from sources
     # build: https://github.com/SkynetLabs/pinner.git#main
-    image: skynetlabs/pinner:0.3.1
+    image: skynetlabs/pinner:0.7.5
     container_name: pinner
     restart: unless-stopped
     logging: *default-logging
     env_file:
       - .env
+    volumes:
+      - ./docker/data/pinner/logs:/logs
     environment:
       - PINNER_LOG_LEVEL=${PINNER_LOG_LEVEL:-info}
     expose:

docker-compose.yml

@@ -1,4 +1,4 @@
-version: "3.7"
+version: "3.8"
 
 x-logging: &default-logging
   driver: json-file
@@ -15,17 +15,19 @@ networks:
 
 services:
   sia:
-    build:
+    # uncomment "build" and comment out "image" to build from sources
-      context: ./docker/sia
+    # build:
-      dockerfile: Dockerfile
+    #   context: https://github.com/SkynetLabs/docker-skyd.git#main
-      args:
+    #   dockerfile: scratch/Dockerfile
-        branch: portal-latest
+    #   args:
+    #     branch: master
+    image: skynetlabs/skyd:1.6.8
+    command: --disable-api-security --api-addr :9980 --modules gctwra
     container_name: sia
     restart: unless-stopped
     stop_grace_period: 5m
     logging: *default-logging
     environment:
-      - SIA_MODULES=gctwra
       - SKYD_DISK_CACHE_ENABLED=${SKYD_DISK_CACHE_ENABLED:-true}
       - SKYD_DISK_CACHE_SIZE=${SKYD_DISK_CACHE_SIZE:-53690000000} # 50GB
       - SKYD_DISK_CACHE_MIN_HITS=${SKYD_DISK_CACHE_MIN_HITS:-3}
@@ -41,7 +43,24 @@ services:
       - 9980
 
   certbot:
-    image: certbot/dns-route53:v1.25.0
+    # replace this image with the image supporting your dns provider from
+    # https://hub.docker.com/r/certbot/certbot and adjust CERTBOT_ARGS env variable
+    # note: you will need to authenticate your dns request so consult the plugin docs
+    # configuration https://eff-certbot.readthedocs.io/en/stable/using.html#dns-plugins
+    #
+    # =================================================================================
+    # example docker-compose.yml changes required for Cloudflare dns provider:
+    #
+    # image: certbot/dns-cloudflare
+    # environment:
+    #   - CERTBOT_ARGS=--dns-cloudflare --dns-cloudflare-credentials /etc/letsencrypt/cloudflare.ini
+    #
+    # create ./docker/data/certbot/cloudflare.ini file with the following content:
+    # dns_cloudflare_api_token = <api key generated at https://dash.cloudflare.com/profile/api-tokens>
+    #
+    # make sure that the file has 0400 permissions with:
+    # chmod 0400 ./docker/data/certbot/cloudflare.ini
+    image: certbot/dns-route53:v1.30.0
     entrypoint: sh /entrypoint.sh
     container_name: certbot
     restart: unless-stopped
@@ -59,7 +78,7 @@ services:
     # build:
     #   context: https://github.com/SkynetLabs/webportal-nginx.git#main
     #   dockerfile: Dockerfile
-    image: skynetlabs/webportal-nginx:0.2.0
+    image: skynetlabs/webportal-nginx:0.5.2
     container_name: nginx
     restart: unless-stopped
     logging: *default-logging
@@ -89,7 +108,7 @@ services:
     # build:
     #   context: https://github.com/SkynetLabs/webportal-website.git#main
     #   dockerfile: Dockerfile
-    image: skynetlabs/webportal-website:0.1.0
+    image: skynetlabs/webportal-website:0.2.2
     container_name: website
     restart: unless-stopped
     logging: *default-logging
@@ -105,18 +124,11 @@ services:
       - 9000
 
   handshake:
-    image: skynetlabs/hsd:3.0.1
+    image: handshakeorg/hsd:4.0.2
-    command: --chain-migrate=2 --wallet-migrate=1
+    command: --chain-migrate=3 --no-wallet --no-auth --compact-tree-on-init --network=main --http-host=0.0.0.0
     container_name: handshake
     restart: unless-stopped
     logging: *default-logging
-    environment:
-      - HSD_LOG_CONSOLE=false
-      - HSD_HTTP_HOST=0.0.0.0
-      - HSD_NETWORK=main
-      - HSD_PORT=12037
-    env_file:
-      - .env
     volumes:
       - ./docker/data/handshake/.hsd:/root/.hsd
     networks:
@@ -130,7 +142,7 @@ services:
     # build:
     #   context: https://github.com/SkynetLabs/webportal-handshake-api.git#main
     #   dockerfile: Dockerfile
-    image: skynetlabs/webportal-handshake-api:0.1.1
+    image: skynetlabs/webportal-handshake-api:0.1.2
     container_name: handshake-api
     restart: unless-stopped
     logging: *default-logging
@@ -154,7 +166,7 @@ services:
     # build:
     #   context: https://github.com/SkynetLabs/webportal-dnslink-api.git#main
     #   dockerfile: Dockerfile
-    image: skynetlabs/webportal-dnslink-api:0.1.1
+    image: skynetlabs/webportal-dnslink-api:0.1.2
     container_name: dnslink-api
     restart: unless-stopped
     logging: *default-logging
@@ -169,7 +181,7 @@ services:
     # build:
     #   context: https://github.com/SkynetLabs/webportal-health-check.git#main
     #   dockerfile: Dockerfile
-    image: skynetlabs/webportal-health-check:0.1.3
+    image: skynetlabs/webportal-health-check:0.5.0
     container_name: health-check
     restart: unless-stopped
     logging: *default-logging

docker/sia/Dockerfile

@@ -1,16 +0,0 @@
-FROM golang:1.16.7 AS sia-builder
-
-ENV GOOS linux
-ENV GOARCH amd64
-
-ARG branch=portal-latest
-
-RUN git clone https://gitlab.com/SkynetLabs/skyd.git Sia --single-branch --branch ${branch} && \
-    make release --directory Sia
-
-FROM nebulouslabs/sia:1.5.6
-
-COPY --from=sia-builder /go/bin/ /usr/bin/
-
-RUN if [ -f "/usr/bin/skyd" ]; then mv /usr/bin/skyd /usr/bin/siad; fi && \
-    if [ -f "/usr/bin/skyc" ]; then mv /usr/bin/skyc /usr/bin/siac; fi

renovate.json

@@ -0,0 +1,7 @@
+{
+  "extends": ["config:base", ":prHourlyLimitNone"],
+  "enabledManagers": ["docker-compose"],
+  "packageRules": [
+    { "groupName": "jaegertracing", "matchPackagePatterns": ["jaegertracing"] }
+  ]
+}

scripts/blocklist-skylink.sh

@@ -3,8 +3,6 @@
 # This script is for manual skylink blocking. It accepts either a single 
 # skylink or a file containing list of skylinks. The script is intented
 # for manual use and it should be run locally on each skynet webportal server. 
-# The automatic script that is used to continuously sync an Airtable sheet 
-# list with the blocklist on the web portals is /setup-scripts/blocklist-airtable.py 
 
 set -e # exit on first error
 

scripts/es_cleaner.py

@@ -1,164 +0,0 @@
-#!/usr/bin/env python3
-
-import curator
-import elasticsearch
-import os
-import ssl
-import sys
-
-TIMEOUT = 120
-
-
-def main():
-    if len(sys.argv) != 3:
-        print(
-            'USAGE: [INDEX_PREFIX=(default "")] [ARCHIVE=(default false)] ... {} NUM_OF_DAYS http://HOSTNAME[:PORT]'.format(
-                sys.argv[0]
-            )
-        )
-        print(
-            "NUM_OF_DAYS ... delete indices that are older than the given number of days."
-        )
-        print(
-            "HOSTNAME ... specifies which Elasticsearch hosts URL to search and delete indices from."
-        )
-        print(
-            "TIMEOUT ...  number of seconds to wait for master node response, default: {}".format(
-                TIMEOUT
-            )
-        )
-        print("INDEX_PREFIX ... specifies index prefix.")
-        print("INDEX_DATE_SEPARATOR ... specifies index date separator.")
-        print(
-            "ARCHIVE ... specifies whether to remove archive indices (only works for rollover) (default false)."
-        )
-        print(
-            "ROLLOVER ... specifies whether to remove indices created by rollover (default false)."
-        )
-        print("ES_USERNAME ... The username required by Elasticsearch.")
-        print("ES_PASSWORD ... The password required by Elasticsearch.")
-        print("ES_TLS ... enable TLS (default false).")
-        print("ES_TLS_CA ... Path to TLS CA file.")
-        print("ES_TLS_CERT ... Path to TLS certificate file.")
-        print("ES_TLS_KEY ... Path to TLS key file.")
-        print(
-            "ES_TLS_SKIP_HOST_VERIFY ... (insecure) Skip server's certificate chain and host name verification."
-        )
-        sys.exit(1)
-
-    client = create_client(
-        os.getenv("ES_USERNAME"),
-        os.getenv("ES_PASSWORD"),
-        str2bool(os.getenv("ES_TLS", "false")),
-        os.getenv("ES_TLS_CA"),
-        os.getenv("ES_TLS_CERT"),
-        os.getenv("ES_TLS_KEY"),
-        str2bool(os.getenv("ES_TLS_SKIP_HOST_VERIFY", "false")),
-    )
-    ilo = curator.IndexList(client)
-    empty_list(ilo, "Elasticsearch has no indices")
-
-    prefix = os.getenv("INDEX_PREFIX", "")
-    if prefix != "":
-        prefix += "-"
-    separator = os.getenv("INDEX_DATE_SEPARATOR", "-")
-
-    if str2bool(os.getenv("ARCHIVE", "false")):
-        filter_archive_indices_rollover(ilo, prefix)
-    else:
-        if str2bool(os.getenv("ROLLOVER", "false")):
-            filter_main_indices_rollover(ilo, prefix)
-        else:
-            filter_main_indices(ilo, prefix, separator)
-
-    empty_list(ilo, "No indices to delete")
-
-    for index in ilo.working_list():
-        print("Removing", index)
-    timeout = int(os.getenv("TIMEOUT", TIMEOUT))
-    delete_indices = curator.DeleteIndices(ilo, master_timeout=timeout)
-    delete_indices.do_action()
-
-
-def filter_main_indices(ilo, prefix, separator):
-    date_regex = "\d{4}" + separator + "\d{2}" + separator + "\d{2}"
-    time_string = "%Y" + separator + "%m" + separator + "%d"
-
-    ilo.filter_by_regex(
-        kind="regex", value=prefix + "jaeger-(span|service|dependencies)-" + date_regex
-    )
-    empty_list(ilo, "No indices to delete")
-    # This excludes archive index as we use source='name'
-    # source `creation_date` would include archive index
-    ilo.filter_by_age(
-        source="name",
-        direction="older",
-        timestring=time_string,
-        unit="days",
-        unit_count=int(sys.argv[1]),
-    )
-
-
-def filter_main_indices_rollover(ilo, prefix):
-    ilo.filter_by_regex(kind="regex", value=prefix + "jaeger-(span|service)-\d{6}")
-    empty_list(ilo, "No indices to delete")
-    # do not remove active write indices
-    ilo.filter_by_alias(aliases=[prefix + "jaeger-span-write"], exclude=True)
-    empty_list(ilo, "No indices to delete")
-    ilo.filter_by_alias(aliases=[prefix + "jaeger-service-write"], exclude=True)
-    empty_list(ilo, "No indices to delete")
-    ilo.filter_by_age(
-        source="creation_date",
-        direction="older",
-        unit="days",
-        unit_count=int(sys.argv[1]),
-    )
-
-
-def filter_archive_indices_rollover(ilo, prefix):
-    # Remove only rollover archive indices
-    # Do not remove active write archive index
-    ilo.filter_by_regex(kind="regex", value=prefix + "jaeger-span-archive-\d{6}")
-    empty_list(ilo, "No indices to delete")
-    ilo.filter_by_alias(aliases=[prefix + "jaeger-span-archive-write"], exclude=True)
-    empty_list(ilo, "No indices to delete")
-    ilo.filter_by_age(
-        source="creation_date",
-        direction="older",
-        unit="days",
-        unit_count=int(sys.argv[1]),
-    )
-
-
-def empty_list(ilo, error_msg):
-    try:
-        ilo.empty_list_check()
-    except curator.NoIndices:
-        print(error_msg)
-        sys.exit(0)
-
-
-def str2bool(v):
-    return v.lower() in ("true", "1")
-
-
-def create_client(username, password, tls, ca, cert, key, skipHostVerify):
-    context = ssl.create_default_context()
-    if ca is not None:
-        context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=ca)
-    elif skipHostVerify:
-        context.check_hostname = False
-        context.verify_mode = ssl.CERT_NONE
-    if username is not None and password is not None:
-        return elasticsearch.Elasticsearch(
-            sys.argv[2:], http_auth=(username, password), ssl_context=context
-        )
-    elif tls:
-        context.load_cert_chain(certfile=cert, keyfile=key)
-        return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)
-    else:
-        return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)
-
-
-if __name__ == "__main__":
-    main()

setup-scripts/blocklist-airtable.py

@@ -1,161 +0,0 @@
-#!/usr/bin/env python3
-
-from bot_utils import get_api_password, setup, send_msg
-from random import randint
-from time import sleep
-
-import traceback
-import os
-import sys
-import asyncio
-import requests
-import json
-
-from requests.auth import HTTPBasicAuth
-
-setup()
-
-
-AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY")
-AIRTABLE_BASE = os.getenv("AIRTABLE_BASE")
-AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE")
-AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD")
-
-# Check environment variables are defined
-for value in [AIRTABLE_API_KEY, AIRTABLE_BASE, AIRTABLE_TABLE, AIRTABLE_FIELD]:
-    if not value:
-        sys.exit("Configuration error: Missing AirTable environment variable.")
-
-
-async def run_checks():
-    try:
-        await block_skylinks_from_airtable()
-    except:  # catch all exceptions
-        trace = traceback.format_exc()
-        await send_msg("```\n{}\n```".format(trace), force_notify=True)
-
-
-def exec(command):
-    return os.popen(command).read().strip()
-
-
-async def block_skylinks_from_airtable():
-    # Get sia IP before doing anything else. If this step fails we don't
-    # need to continue with the execution of the script.
-    ipaddress = exec(
-        "docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia"
-    )
-
-    if ipaddress == "":
-        print("Skyd IP could not be detected. Exiting.")
-        return
-
-    print("Pulling blocked skylinks from Airtable via api integration")
-    headers = {"Authorization": "Bearer " + AIRTABLE_API_KEY}
-    skylinks = []
-    offset = None
-    retry = 0
-    while len(skylinks) == 0 or offset:
-        print(
-            "Requesting a batch of records from Airtable with "
-            + (offset if offset else "empty")
-            + " offset"
-            + (" (retry " + str(retry) + ")" if retry else "")
-        )
-        query = "&".join(
-            ["fields%5B%5D=" + AIRTABLE_FIELD, ("offset=" + offset) if offset else ""]
-        )
-        response = requests.get(
-            "https://api.airtable.com/v0/"
-            + AIRTABLE_BASE
-            + "/"
-            + AIRTABLE_TABLE
-            + "?"
-            + query,
-            headers=headers,
-        )
-
-        # rate limited - sleep for 2-10 secs and retry (up to 100 times, ~10 minutes)
-        # https://support.airtable.com/hc/en-us/articles/203313985-Public-REST-API
-        # > 5 requests per second, per base
-        if response.status_code == 429:
-            if retry < 100:
-                retry = retry + 1
-                sleep(randint(1, 10))
-                continue
-            else:
-                return await send_msg(
-                    "Airtable: too many retries, aborting!", force_notify=True
-                )
-        retry = 0  # reset retry counter
-
-        if response.status_code != 200:
-            status_code = str(response.status_code)
-            response_text = response.text or "empty response"
-            message = (
-                "Airtable blocklist integration responded with code "
-                + status_code
-                + ": "
-                + response_text
-            )
-            return await send_msg(message, force_notify=False)
-
-        data = response.json()
-
-        if len(data["records"]) == 0:
-            return print(
-                "Airtable returned 0 records - make sure your configuration is correct"
-            )
-
-        skylinks = skylinks + [
-            entry["fields"].get(AIRTABLE_FIELD, "") for entry in data["records"]
-        ]
-        skylinks = [
-            skylink.strip() for skylink in skylinks if skylink
-        ]  # filter empty skylinks, most likely empty rows, trim whitespace
-
-        offset = data.get("offset")
-
-    print(
-        "Sending /skynet/blocklist request with "
-        + str(len(skylinks))
-        + " skylinks to siad"
-    )
-    response = requests.post(
-        "http://" + ipaddress + ":9980/skynet/blocklist",
-        data=json.dumps({"add": skylinks}),
-        headers={"User-Agent": "Sia-Agent"},
-        auth=HTTPBasicAuth("", get_api_password()),
-    )
-
-    if response.status_code != 200:
-        status_code = str(response.status_code)
-        response_text = response.text or "empty response"
-        message = (
-            "Airtable blocklist request responded with code "
-            + status_code
-            + ": "
-            + response_text
-        )
-        return await send_msg(message, force_notify=False)
-
-    response_json = json.loads(response.text)
-    invalid_skylinks = response_json["invalids"]
-
-    if invalid_skylinks is None:
-        return await send_msg("Blocklist successfully updated all skylinks")
-    return await send_msg(
-        "Blocklist responded ok but failed to update "
-        + str(len(invalid_skylinks))
-        + " skylinks: "
-        + json.dumps(invalid_skylinks)
-    )
-
-
-loop = asyncio.get_event_loop()
-loop.run_until_complete(run_checks())
-
-# --- BASH EQUIVALENT
-# skylinks=$(curl "https://api.airtable.com/v0/${AIRTABLE_BASE}/${AIRTABLE_TABLE}?fields%5B%5D=${AIRTABLE_FIELD}" -H "Authorization: Bearer ${AIRTABLE_KEY}" | python3 -c "import sys, json; print('[\"' + '\",\"'.join([entry['fields']['Link'] for entry in json.load(sys.stdin)['records']]) + '\"]')")
-# ipaddress=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sia)
-# curl --data "{\"add\" : ${skylinks}}" "${ipaddress}:8000/skynet/blocklist"

setup-scripts/support/authorized_keys

@@ -9,3 +9,4 @@ ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN6Kcx8yetova4/ALUQHigo/PBMJO33ZTKOsg2jxSO2a
 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDcenWnMQ6q/OEC4ZmQgjLDV2obWlR3fENV0zRGFvJF+ marcins@siasky.net
 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB7prtVOTwtcSN9HkXum107RwcW5H8Vggx6Qv7T57ItT daniel@siasky.net
 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH+4IrfEM9H16jqvPZncHkWWoHO4/BVq7d4pEyzK4e0W michal.leszczyk@skynetlabs.com
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHptEpqs57lhnHkfa+0SQgXQ4A63/YGV2cNTcGMQW+Jt david@skynetlabs.com

setup-scripts/support/crontab

@@ -1,7 +1,6 @@
 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/skynet-webportal/.env
 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/skynet-webportal/.env sia 8
 0 * * * * /home/user/skynet-webportal/setup-scripts/health-checker.py /home/user/skynet-webportal/.env sia 1
-30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env
 44 5 * * * /home/user/skynet-webportal/scripts/backup-aws-s3.sh 1>>/home/user/skynet-webportal/logs/backup-aws-s3.log 2>>/home/user/skynet-webportal/logs/backup-aws-s3.log
 6 13 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1>>/home/user/skynet-webportal/logs/db_backup.log 2>>/home/user/skynet-webportal/logs/db_backup.log
-0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200
+0 5 * * * /usr/bin/docker run --rm --net=host -e ROLLOVER=true jaegertracing/jaeger-es-index-cleaner:latest 1 http://localhost:9200