Merge pull request #296 from NebulousLabs/docker

Containerised sia setup

.github/dependabot.yml

@@ -1,7 +1,23 @@
 version: 2
 updates:
   - package-ecosystem: npm
-    directory: "/"
+    directory: "/packages/handshake-api"
+    schedule:
+      interval: weekly
+      time: "10:00"
+    open-pull-requests-limit: 10
+    assignees:
+      - kwypchlo
+  - package-ecosystem: npm
+    directory: "/packages/health-check"
+    schedule:
+      interval: weekly
+      time: "10:00"
+    open-pull-requests-limit: 10
+    assignees:
+      - kwypchlo
+  - package-ecosystem: npm
+    directory: "/packages/webapp"
     schedule:
       interval: weekly
       time: "10:00"

.github/workflows/build.yml

@@ -11,16 +11,22 @@ jobs:
       - name: Use Node.js
         uses: actions/setup-node@v1
         with:
-          node-version: 14.6
+          node-version: 14.x
 
       - name: Install dependencies
         run: yarn
 
-      - name: Static code analysis
+      - name: "Static code analysis: handshake-api"
-        run: yarn prettier --check .
+        run: yarn workspace handshake-api prettier --check .
 
-      - name: Build production bundle
+      - name: "Static code analysis: health-check"
-        run: yarn build
+        run: yarn workspace health-check prettier --check .
+
+      - name: "Static code analysis: webapp"
+        run: yarn workspace webapp prettier --check .
+
+      - name: "Build webapp"
+        run: yarn workspace webapp build
         env:
           GATSBY_API_URL: "https://siasky.net"
 
@@ -30,6 +36,7 @@ jobs:
           CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
+          working-directory: packages/webapp
           record: true
           start: npx http-server public --port 8000
           wait-on: "http://localhost:8000"

docker-compose.yml

@@ -4,16 +4,41 @@ networks:
   shared:
     driver: bridge
 
+volumes:
+  webapp:
+
 services:
-  docker-host:
+  sia:
-    image: qoomon/docker-host
+    build:
-    container_name: docker-host
+      context: ./docker/sia
+      dockerfile: Dockerfile
+      args:
+        branch: v1.5.0
+    container_name: sia
     restart: unless-stopped
-    cap_add:
+    environment:
-      - NET_ADMIN
+      - SIA_MODULES=gctwr
-      - NET_RAW
+    env_file:
+      - .env
+    volumes:
+      - ./docker/data/sia:/sia-data
     networks:
       - shared
+    expose:
+      - 9980
+
+  sia-upload:
+    image: nebulouslabs/sia:alpine-1.5.0
+    container_name: sia-upload
+    restart: unless-stopped
+    environment:
+      - SIA_MODULES=gctwr
+    volumes:
+      - ./docker/data/sia-upload:/sia-data
+    networks:
+      - shared
+    expose:
+      - 9980
 
   caddy:
     build:
@@ -33,13 +58,10 @@ services:
       - "80:80"
       - "443:443"
     depends_on:
-      - docker-host
       - nginx
 
   nginx:
-    build:
+    image: openresty/openresty:1.15.8.3-2-xenial
-      context: .
-      dockerfile: ./docker/nginx/Dockerfile
     container_name: nginx
     restart: unless-stopped
     env_file:
@@ -49,14 +71,26 @@ services:
       - ./docker/nginx/conf.d:/etc/nginx/conf.d:ro
       - ./docker/data/nginx/cache:/data/nginx/cache
       - ./docker/data/nginx/logs:/usr/local/openresty/nginx/logs
+      - ./docker/data/sia/apipassword:/data/sia/apipassword:ro
+      - webapp:/var/www/webportal:ro
     networks:
       - shared
     expose:
       - 80
     depends_on:
-      - docker-host
+      - sia
       - handshake-api
 
+  webapp:
+    build:
+      context: ./packages/webapp
+      dockerfile: Dockerfile
+    container_name: webapp
+    restart: unless-stopped
+    tty: true
+    volumes:
+      - webapp:/usr/app/public
+
   handshake:
     build:
       context: ./docker/handshake
@@ -79,8 +113,8 @@ services:
 
   handshake-api:
     build:
-      context: .
+      context: ./packages/handshake-api
-      dockerfile: ./docker/handshake-api/Dockerfile
+      dockerfile: Dockerfile
     container_name: handshake-api
     restart: unless-stopped
     environment:
@@ -99,12 +133,11 @@ services:
 
   health-check:
     build:
-      context: .
+      context: ./packages/health-check
-      dockerfile: ./docker/health-check/Dockerfile
+      dockerfile: Dockerfile
     container_name: health-check
     restart: unless-stopped
     volumes:
-      - ./health-check:/usr/app/health-check
       - ./docker/data/health-check/state:/usr/app/state
     networks:
       - shared

docker/handshake-api/Dockerfile

@@ -1,20 +0,0 @@
-FROM node:14.6.0
-
-WORKDIR /usr/app
-
-RUN yarn init -y && \
-    yarn add express express-http-proxy hs-client
-
-COPY handshake-api/index.js ./
-
-ENV HOST="localhost"
-ENV PORT=3100
-
-ENV HSD_NETWORK="main"
-ENV HSD_HOST="0.0.0.0"
-ENV HSD_PORT=12037
-ENV HSD_API_KEY="foo"
-
-EXPOSE $PORT
-
-ENTRYPOINT ["node", "--max-http-header-size=64000", "index.js"]

docker/handshake/Dockerfile

@@ -1,9 +1,10 @@
-FROM node:14.6.0
+# we can't use alpine image since it fails on node-gyp
+FROM node:14.8.0
 
 WORKDIR /opt/hsd
 
-RUN git clone https://github.com/handshake-org/hsd.git /opt/hsd && \
+RUN git clone https://github.com/handshake-org/hsd.git /opt/hsd
-    npm install --production
+RUN npm install --production
 
 ENV PATH="${PATH}:/opt/hsd/bin:/opt/hsd/node_modules/.bin"
 

docker/health-check/Dockerfile

@@ -1,11 +0,0 @@
-FROM node:14.6.0
-
-WORKDIR /usr/app
-
-RUN yarn init -y
-RUN yarn add express body-parser lowdb node-schedule superagent lodash http-status-codes
-
-EXPOSE 3100
-
-ENV NODE_ENV production
-CMD [ "node", "health-check/index.js" ]

docker/nginx/Dockerfile

@@ -1,15 +0,0 @@
-FROM node:14.6.0 AS webportal-builder
-
-COPY src ./src
-COPY static ./static
-COPY gatsby-config.js .
-COPY package.json .
-COPY yarn.lock .
-
-ENV CYPRESS_INSTALL_BINARY 0
-RUN yarn --frozen-lockfile
-RUN yarn build
-
-FROM openresty/openresty:1.15.8.3-2-xenial
-
-COPY --from=webportal-builder /public /var/www/webportal

docker/nginx/conf.d/client.conf

@@ -13,12 +13,8 @@ set_real_ip_from  172.16.0.0/12;
 set_real_ip_from  192.168.0.0/16;
 real_ip_header    X-Forwarded-For;
 
-upstream siad-upload {
-	server docker-host:9970;
-}
-
 upstream siad {
-	server docker-host:9980;
+	server sia:9980;
 }
 
 server {
@@ -96,6 +92,9 @@ server {
 		proxy_pass http://siad/skynet/stats;
 	}
 
+	# This endpoint is a backward compatilibilty workaround for reporting stats from legacy 
+	# upload siad node for portals that used double-siad setup. If your portal does not need
+	# to support additional siad node, you can remove it from this config.
 	location /statsdown {
 		include /etc/nginx/conf.d/include/cors;
 
@@ -103,7 +102,7 @@ server {
 		proxy_cache_valid any 10m; # cache stats for 10 minutes
 		proxy_set_header User-Agent: Sia-Agent;
 		proxy_read_timeout 5m; # extend the read timeout
-		proxy_pass http://siad-upload/skynet/stats; # serve upload node stats temporarily
+		proxy_pass http://sia-upload:9980/skynet/stats; # serve upload node stats temporarily
 	}
 
 	location /health-check {
@@ -177,6 +176,7 @@ server {
 		include /etc/nginx/conf.d/include/proxy-buffer;
 
 		limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time
+		add_header Cache-Control "public, max-age=86400"; # allow consumer to cache response
 
 		# we need to explicitly use set directive here because $1 will contain the skylink with
 		# decoded whitespaces and set will re-encode it for us before passing it to proxy_pass
@@ -193,6 +193,7 @@ server {
 		proxy_cache_min_uses 10;
 		proxy_cache_valid 200 1440m;
 		proxy_cache_bypass $cookie_nocache $arg_nocache; # add cache bypass option
+		add_header X-Proxy-Cache $upstream_cache_status;
 	}
 
 	location ~ "^/file/([a-zA-Z0-9-_]{46}(/.*)?)$" {
@@ -200,6 +201,7 @@ server {
 		include /etc/nginx/conf.d/include/proxy-buffer;
 
 		limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time
+		add_header Cache-Control "public, max-age=86400"; # allow consumer to cache response
 
 		# we need to explicitly use set directive here because $1 will contain the skylink with
 		# decoded whitespaces and set will re-encode it for us before passing it to proxy_pass
@@ -217,5 +219,6 @@ server {
 		proxy_cache_min_uses 10;
 		proxy_cache_valid 200 1440m;
 		proxy_cache_bypass $cookie_nocache $arg_nocache; # add cache bypass option
+		add_header X-Proxy-Cache $upstream_cache_status;
 	}
 }

docker/nginx/conf.d/include/sia-auth

@@ -1,10 +1,15 @@
 rewrite_by_lua_block {
-    -- local b64 = require("ngx.base64")
+    local b64 = require("ngx.base64")
-    -- pull apipassword from SIA_API_AUTHORIZATION environment variable
+    -- open apipassword file for reading (b flag is required for some reason)
-    -- local apipassword = os.getenv("SIA_API_AUTHORIZATION")
+    -- (file /etc/.sia/apipassword has to be mounted from the host system)
+    local apipassword_file = io.open("/data/sia/apipassword", "rb")
+    -- read apipassword file contents and trim newline (important)
+    local apipassword = apipassword_file:read("*all"):gsub("%s+", "")
+    -- make sure to close file after reading the password
+    apipassword_file.close()
     -- encode the user:password authorization string
     -- (in our case user is empty so it is just :password)
-    -- local content = b64.encode_base64url(":" .. apipassword)
+    local content = b64.encode_base64url(":" .. apipassword)
     -- set authorization header with proper base64 encoded string
-    ngx.req.set_header("Authorization", "Basic " .. os.getenv("SIA_API_AUTHORIZATION"))
+    ngx.req.set_header("Authorization", "Basic " .. content)
 }

docker/nginx/nginx.conf

@@ -16,10 +16,7 @@
 # See https://github.com/openresty/docker-openresty/blob/master/README.md#nginx-config-files
 #
 
-# expose environment variables
+user  root;
-env SIA_API_AUTHORIZATION;
-
-#user  nobody;
 worker_processes  1;
 
 #error_log  logs/error.log;

docker/sia/Dockerfile

@@ -0,0 +1,13 @@
+FROM golang AS sia-builder
+
+ENV GOOS linux
+ENV GOARCH amd64
+
+ARG branch=master
+
+RUN git clone https://gitlab.com/NebulousLabs/Sia.git --single-branch --branch ${branch}
+RUN make release --directory Sia
+
+FROM nebulouslabs/sia:alpine-1.5.0
+
+COPY --from=sia-builder /go/bin/siac /go/bin/siad /usr/bin/

package.json

@@ -1,86 +1,6 @@
 {
-  "name": "skynet-webportal",
+    "private": true,
-  "description": "Sia Skynet",
+    "workspaces": [
-  "version": "1.0.0",
+        "packages/*"
-  "author": "Nebulous",
-  "dependencies": {
-    "axios": "^0.19.2",
-    "bytes": "^3.1.0",
-    "classnames": "^2.2.6",
-    "gatsby": "^2.24.5",
-    "gatsby-image": "^2.4.13",
-    "gatsby-plugin-manifest": "^2.4.18",
-    "gatsby-plugin-matomo": "^0.8.3",
-    "gatsby-plugin-react-helmet": "^3.3.10",
-    "gatsby-plugin-remove-serviceworker": "^1.0.0",
-    "gatsby-plugin-robots-txt": "^1.5.1",
-    "gatsby-plugin-sass": "^2.3.12",
-    "gatsby-plugin-sharp": "^2.6.20",
-    "gatsby-source-filesystem": "^2.3.20",
-    "gatsby-transformer-sharp": "^2.5.11",
-    "http-status-codes": "^1.4.0",
-    "jsonp": "^0.2.1",
-    "node-sass": "^4.14.0",
-    "path-browserify": "^1.0.1",
-    "prop-types": "^15.7.2",
-    "react": "^16.13.1",
-    "react-countup": "^4.3.3",
-    "react-dom": "^16.13.1",
-    "react-dropzone": "^11.0.2",
-    "react-helmet": "^6.1.0",
-    "react-mailchimp-form": "^1.0.2",
-    "react-mailchimp-subscribe": "^2.1.0",
-    "react-reveal": "^1.2.2",
-    "react-syntax-highlighter": "^12.2.1",
-    "react-visibility-sensor": "^5.1.1",
-    "skynet-js": "0.0.8",
-    "typeface-metropolis": "^0.0.74"
-  },
-  "devDependencies": {
-    "cypress": "^4.10.0",
-    "cypress-file-upload": "^4.0.7",
-    "eslint": "^7.5.0",
-    "eslint-config-prettier": "^6.11.0",
-    "eslint-plugin-cypress": "^2.11.1",
-    "eslint-plugin-react": "^7.20.3",
-    "husky": "^4.2.5",
-    "lint-staged": "^10.2.11",
-    "prettier": "^2.0.5"
-  },
-  "keywords": [
-    "sia",
-    "skynet",
-    "nebulous",
-    "blockchain",
-    "decentralized",
-    "cloud storage"
-  ],
-  "license": "MIT",
-  "scripts": {
-    "build": "gatsby build",
-    "start": "gatsby develop",
-    "serve": "gatsby serve",
-    "clean": "gatsby clean"
-  },
-  "husky": {
-    "hooks": {
-      "pre-commit": "lint-staged"
-    }
-  },
-  "lint-staged": {
-    "*.{js,jsx,ts,tsx}": [
-      "eslint --fix",
-      "prettier --write"
-    ],
-    "*.{md,html,css,scss}": [
-      "prettier --write"
     ]
-  },
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/NebulousLabs/skynet-webportal"
-  },
-  "bugs": {
-    "url": "https://github.com/NebulousLabs/skynet-webportal/issues"
-  }
 }

packages/handshake-api/.prettierignore

@@ -0,0 +1 @@
+/package.json

packages/handshake-api/Dockerfile

@@ -0,0 +1,16 @@
+FROM node:14.8.0-alpine
+
+WORKDIR /usr/app
+
+COPY package.json .
+RUN yarn --no-lockfile
+COPY src/* src/
+
+ENV HSD_NETWORK="main"
+ENV HSD_HOST="0.0.0.0"
+ENV HSD_PORT=12037
+ENV HSD_API_KEY="foo"
+
+EXPOSE 3100
+ENV NODE_ENV production
+CMD [ "node", "--max-http-header-size=64000", "src/index.js" ]

packages/handshake-api/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "handshake-api",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "dependencies": {
+    "express": "^4.17.1",
+    "express-http-proxy": "^1.6.2",
+    "hs-client": "^0.0.8"
+  },
+  "devDependencies": {
+    "prettier": "^2.0.5"
+  }
+}

packages/health-check/.prettierignore

@@ -0,0 +1 @@
+/package.json

packages/health-check/.prettierrc

@@ -0,0 +1,3 @@
+{
+  "printWidth": 120
+}

packages/health-check/Dockerfile

@@ -0,0 +1,11 @@
+FROM node:14.8.0-alpine
+
+WORKDIR /usr/app
+
+COPY package.json .
+RUN yarn --no-lockfile
+COPY src/* src/
+
+EXPOSE 3100
+ENV NODE_ENV production
+CMD [ "node", "src/index.js" ]

packages/health-check/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "health-check",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "dependencies": {
+    "body-parser": "^1.19.0",
+    "express": "^4.17.1",
+    "http-status-codes": "^1.4.0",
+    "lodash": "^4.17.19",
+    "lowdb": "^1.0.0",
+    "node-schedule": "^1.3.2",
+    "superagent": "^5.3.1"
+  },
+  "devDependencies": {
+    "prettier": "^2.0.5"
+  }
+}

packages/webapp/.prettierrc

@@ -0,0 +1,3 @@
+{
+  "printWidth": 120
+}

packages/webapp/Dockerfile

@@ -0,0 +1,13 @@
+FROM node:14.8.0-alpine
+
+WORKDIR /usr/app
+
+COPY src ./src
+COPY static ./static
+COPY gatsby-config.js .
+COPY package.json .
+
+ENV CYPRESS_INSTALL_BINARY 0
+ENV GATSBY_TELEMETRY_DISABLED 1
+RUN yarn --no-lockfile
+RUN yarn build

packages/webapp/package.json

@@ -0,0 +1,86 @@
+{
+  "name": "webapp",
+  "description": "Sia Skynet",
+  "version": "1.0.0",
+  "author": "Nebulous",
+  "dependencies": {
+    "axios": "^0.19.2",
+    "bytes": "^3.1.0",
+    "classnames": "^2.2.6",
+    "gatsby": "^2.24.5",
+    "gatsby-image": "^2.4.13",
+    "gatsby-plugin-manifest": "^2.4.18",
+    "gatsby-plugin-matomo": "^0.8.3",
+    "gatsby-plugin-react-helmet": "^3.3.10",
+    "gatsby-plugin-remove-serviceworker": "^1.0.0",
+    "gatsby-plugin-robots-txt": "^1.5.1",
+    "gatsby-plugin-sass": "^2.3.12",
+    "gatsby-plugin-sharp": "^2.6.20",
+    "gatsby-source-filesystem": "^2.3.20",
+    "gatsby-transformer-sharp": "^2.5.11",
+    "http-status-codes": "^1.4.0",
+    "jsonp": "^0.2.1",
+    "node-sass": "^4.14.0",
+    "path-browserify": "^1.0.1",
+    "prop-types": "^15.7.2",
+    "react": "^16.13.1",
+    "react-countup": "^4.3.3",
+    "react-dom": "^16.13.1",
+    "react-dropzone": "^11.0.2",
+    "react-helmet": "^6.1.0",
+    "react-mailchimp-form": "^1.0.2",
+    "react-mailchimp-subscribe": "^2.1.0",
+    "react-reveal": "^1.2.2",
+    "react-syntax-highlighter": "^12.2.1",
+    "react-visibility-sensor": "^5.1.1",
+    "skynet-js": "0.0.8",
+    "typeface-metropolis": "^0.0.74"
+  },
+  "devDependencies": {
+    "cypress": "^4.10.0",
+    "cypress-file-upload": "^4.0.7",
+    "eslint": "^7.5.0",
+    "eslint-config-prettier": "^6.11.0",
+    "eslint-plugin-cypress": "^2.11.1",
+    "eslint-plugin-react": "^7.20.3",
+    "husky": "^4.2.5",
+    "lint-staged": "^10.2.11",
+    "prettier": "^2.0.5"
+  },
+  "keywords": [
+    "sia",
+    "skynet",
+    "nebulous",
+    "blockchain",
+    "decentralized",
+    "cloud storage"
+  ],
+  "license": "MIT",
+  "scripts": {
+    "build": "gatsby build",
+    "start": "gatsby develop",
+    "serve": "gatsby serve",
+    "clean": "gatsby clean"
+  },
+  "husky": {
+    "hooks": {
+      "pre-commit": "lint-staged"
+    }
+  },
+  "lint-staged": {
+    "*.{js,jsx,ts,tsx}": [
+      "eslint --fix",
+      "prettier --write"
+    ],
+    "*.{md,html,css,scss}": [
+      "prettier --write"
+    ]
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/NebulousLabs/skynet-webportal"
+  },
+  "bugs": {
+    "url": "https://github.com/NebulousLabs/skynet-webportal/issues"
+  }
+}