Merge branch 'master' into logging

.gitignore

@@ -1,2 +1,5 @@
 node_modules
 **/*/.next
+
+*.swo
+*.swp

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Nebulous
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,10 @@
+# Skynet Portal
+
+## Setup Guide
+A setup guide with scripts to install prerequisites can be found in the
+[setup-scripts](./setup-scripts) directory.
+
+Once the setup guide is complete you will be running:
+- `siad` configured as a ViewNode
+- a NodeJS app running a Skynet Portal
+- an nginx proxy

packages/siaviewnode-server/src/main.ts

@@ -4,12 +4,15 @@ import express, { Request, Response } from "express"
 import fileUpload, { UploadedFile } from "express-fileupload"
 import proxy from "express-http-proxy"
 import requestId from "express-request-id"
+import fs from "fs"
 import morgan from 'morgan'
+import { homedir } from "os"
 import R from "ramda"
 import shortid from "shortid"
 import { Logger } from "winston"
 import logger from "./logger"
 
+
 // import * as AxiosLogger from 'axios-logger'
 // AxiosLogger.setGlobalConfig({
 //   prefixText: 'your prefix',
@@ -40,7 +43,7 @@ const siad = axios.create({
   },
   auth: {
     username: "",
-    password: "d05bb024715aea0bb734ce057acbae27"
+    password: fs.readFileSync(homedir().concat("/.sia/apipassword"), "utf8").trim()
   }
 })
 

setup-scripts/README.md

@@ -0,0 +1,75 @@
+# Skynet Portal Setup Scripts
+
+This directory contains a setup guide and scripts that will install and
+configure some basic requirements for running a Skynet Portal. The assumption is
+that we are working with a Debian Buster Minimal system or similar.
+
+##  Initial Setup
+(Assumes we are logged in as root on a fresh installation of Debian)
+
+You may want to fork this repository and add your ssh pubkey to
+`authorized_keys` and optionally edit the `tmux` and `bash` configurations.
+
+0. SSH in a freshly installed Debian machine.
+1. `apt-get update && apt-get install sudo`
+2. `adduser user`
+3. `usermod -a -G sudo user`
+4. Quit the ssh session.
+
+You a can now ssh into your machine as the user `user`.
+
+5. On your local machine: `ssh-copy-id user@ip-addr`
+6. On your local machine: `ssh user@ip-addr`
+7. Now logged in as `user`: `sudo apt-get install git`
+8. `git clone https://github.com/NebulousLabs/skynet-webportal`
+9. `cd skynet-webportal/setup-scripts`
+11. `./setup.sh`
+12. Once DNS records are set you can run: `./letsencrypt-setup.sh`
+13. You should also change the nginx configuration to listen on port 443
+    instead.
+
+## Setting up siad
+
+NOTE: You must be running `siad` and `siac` by building from the `viewnode`
+branch.
+
+You still need to setup `siad` for the backend to be complete.
+1. `cd ~/; mkdir siad`
+2. `nohup siad &>/dev/null &`
+
+This will start syncing `siad` in the background.
+
+## ViewNode setup
+
+When `siad` is done syncing, create a new wallet and unlock the wallet.
+
+Then set an allowance (`siac renter setallowance`), with the suggested values
+below:
+- 10 KS (keep 25 KS in your wallet)
+- default period
+- default number of hosts
+- 8 week renewal time
+- 500 GB expected storage
+- 500 GB expected upload
+- 5 TB expected download
+- default redundancy
+
+Once your allowance is set you need to set your node to be a viewnode with the
+following command:
+`siac renter setallowance --payment-contract-initial-price 10SC`
+
+Now your node will begin making 10 contracts per block with many hosts so it can
+potentially view the whole network's files.
+
+## Running the Portal
+`cd` into the parent directory and run `yarn` to build dependencies.
+
+We recommend running the Portal through `pm2` (a nodejs process manager) in the background with the command:
+
+`pm2 --name skynet start npm -- start`
+
+`yarn start` will also work if not using `pm2`.
+
+The Protal which will automatically read your `siad` API password and startup a
+portal on `localhost:3000`. nginx will expose this to port 80 or 443 if you
+configured it for SSL.

setup-scripts/bashrc

@@ -0,0 +1,114 @@
+# ~/.bashrc: executed by bash(1) for non-login shells.
+# see /usr/share/doc/bash/examples/startup-files (in the package bash-doc)
+# for examples
+
+# If not running interactively, don't do anything
+case $- in
+    *i*) ;;
+      *) return;;
+esac
+
+# don't put duplicate lines or lines starting with space in the history.
+# See bash(1) for more options
+HISTCONTROL=ignoreboth
+
+# append to the history file, don't overwrite it
+shopt -s histappend
+
+# for setting history length see HISTSIZE and HISTFILESIZE in bash(1)
+HISTSIZE=1000
+HISTFILESIZE=2000
+
+# check the window size after each command and, if necessary,
+# update the values of LINES and COLUMNS.
+shopt -s checkwinsize
+
+# If set, the pattern "**" used in a pathname expansion context will
+# match all files and zero or more directories and subdirectories.
+#shopt -s globstar
+
+# make less more friendly for non-text input files, see lesspipe(1)
+#[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)"
+
+# set variable identifying the chroot you work in (used in the prompt below)
+if [ -z "${debian_chroot:-}" ] && [ -r /etc/debian_chroot ]; then
+    debian_chroot=$(cat /etc/debian_chroot)
+fi
+
+# set a fancy prompt (non-color, unless we know we "want" color)
+case "$TERM" in
+    alacritty|xterm-color|*-256color) color_prompt=yes;;
+esac
+
+# uncomment for a colored prompt, if the terminal has the capability; turned
+# off by default to not distract the user: the focus in a terminal window
+# should be on the output of commands, not on the prompt
+#force_color_prompt=yes
+
+if [ -n "$force_color_prompt" ]; then
+    if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then
+	# We have color support; assume it's compliant with Ecma-48
+	# (ISO/IEC-6429). (Lack of such support is extremely rare, and such
+	# a case would tend to support setf rather than setaf.)
+	color_prompt=yes
+    else
+	color_prompt=
+    fi
+fi
+
+if [ "$color_prompt" = yes ]; then
+    PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '
+else
+    PS1='${debian_chroot:+($debian_chroot)}\u@\h:\w\$ '
+fi
+unset color_prompt force_color_prompt
+
+# If this is an xterm set the title to user@host:dir
+case "$TERM" in
+xterm*|rxvt*)
+    PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1"
+    ;;
+*)
+    ;;
+esac
+
+# enable color support of ls and also add handy aliases
+if [ -x /usr/bin/dircolors ]; then
+    test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)"
+    alias ls='ls --color=auto'
+    #alias dir='dir --color=auto'
+    #alias vdir='vdir --color=auto'
+
+    #alias grep='grep --color=auto'
+    #alias fgrep='fgrep --color=auto'
+    #alias egrep='egrep --color=auto'
+fi
+
+# colored GCC warnings and errors
+#export GCC_COLORS='error=01;31:warning=01;35:note=01;36:caret=01;32:locus=01:quote=01'
+
+# some more ls aliases
+#alias ll='ls -l'
+#alias la='ls -A'
+#alias l='ls -CF'
+
+# Alias definitions.
+# You may want to put all your additions into a separate file like
+# ~/.bash_aliases, instead of adding them here directly.
+# See /usr/share/doc/bash-doc/examples in the bash-doc package.
+
+if [ -f ~/.bash_aliases ]; then
+    . ~/.bash_aliases
+fi
+
+# enable programmable completion features (you don't need to enable
+# this, if it's already enabled in /etc/bash.bashrc and /etc/profile
+# sources /etc/bash.bashrc).
+if ! shopt -oq posix; then
+  if [ -f /usr/share/bash-completion/bash_completion ]; then
+    . /usr/share/bash-completion/bash_completion
+  elif [ -f /etc/bash_completion ]; then
+    . /etc/bash_completion
+  fi
+fi
+export PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games:/usr/local/go/bin:/home/user/go/bin

setup-scripts/letsencrypt-setup.sh

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+set -e
+
+domain="$1"
+if [[ -z $domain ]]; then
+  echo "Usage $0 DOMAIN_NAME"
+  exit 1
+fi
+
+sudo certbot --nginx -d "$domain" -d www."$domain"
+sudo certbot renew --dry-run

setup-scripts/setup.sh

@@ -0,0 +1,68 @@
+#! /usr/bin/env bash
+set -e
+
+# Copy over basic configuration files.
+cp ./tmux.conf ~/.tmux.conf
+cp ./bashrc ~/.bashrc
+source ~/.bashrc
+
+# Add SSH keys and set SSH configs
+sudo cp ./ssh_config /etc/ssh/ssh_config
+mkdir -p ~/.ssh
+cat ./authorized_keys >>  ~/.ssh/authorized_keys
+
+# Nodejs install prerequisite. From official documentation.
+curl -sL https://deb.nodesource.com/setup_13.x | sudo -E bash -
+
+# Yarn install prerequisite.
+curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
+echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
+
+# Apt installations.
+sudo apt-get update
+sudo apt-get -y install ufw tmux ranger htop nload nginx certbot \
+  python-certbot-nginx nodejs gcc g++ make yarn git vim
+
+# Install pm2
+sudo npm i -g pm2
+
+# terminfo for alacritty terminal via ssh
+# If you don't use the alacritty terminal you can remove this step.
+wget -c https://raw.githubusercontent.com/alacritty/alacritty/master/extra/alacritty.info
+sudo tic -xe alacritty,alacritty-direct alacritty.info
+rm alacritty.info
+
+# Setup nginx config
+sudo cp ./skynet-nginx.conf /etc/nginx/sites-available/skynet
+sudo nginx -t
+sudo ln -s /etc/nginx/sites-available/skynet /etc/nginx/sites-enabled/skynet
+sudo rm /etc/nginx/sites-enabled/default
+sudo systemctl reload nginx
+
+# Setup firewall
+# TODO: disable plain HTTP eventually
+sudo ufw enable
+sudo ufw allow ssh
+sudo ufw allow 'Nginx Full'
+sudo ufw allow 'Nginx HTTP'
+
+# Install Go 1.13.7.
+wget -c https://dl.google.com/go/go1.13.7.linux-amd64.tar.gz
+sudo tar -C /usr/local -xzf go1.13.7.linux-amd64.tar.gz
+source ~/.bashrc
+rm go1.13.7.linux-amd64.tar.gz
+
+# Sanity check that will pass if go was installed correctly.
+go version
+
+cwd=$(pwd)
+
+# Install Sia
+cd ~/
+git clone https://gitlab.com/NebulousLabs/Sia
+cd Sia && git checkout viewnode && make
+
+# Setup skynet frontend.
+cd $cwd
+cd ../
+yarn

setup-scripts/skynet-nginx.conf

@@ -0,0 +1,69 @@
+server {
+        listen 80;
+
+        location / {
+		client_max_body_size 1000M;
+		#post_max_size 1000M;
+		proxy_read_timeout 600;
+                proxy_pass http://127.0.0.1:3000;
+                proxy_http_version 1.1;
+                proxy_set_header Upgrade $http_upgrade;
+                proxy_set_header Connection 'upgrade';
+                proxy_set_header Host $host;
+                proxy_cache_bypass $http_upgrade;
+        }
+
+        location ^~ /api/ {
+		client_max_body_size 1000M;
+		#post_max_size 1000M;
+		proxy_read_timeout 600;
+		rewrite ^/api/(.*)$ /$1 break;
+                proxy_pass http://127.0.0.1:4000/;
+        }
+
+	location ~ "^/[a-zA-Z0-9-_+/]{46}$" {
+		client_max_body_size 1000M;
+		#post_max_size 1000M;
+		proxy_read_timeout 600;
+		proxy_set_header User-Agent: "Sia-Agent";
+		proxy_set_header Host: localhost;
+		proxy_pass http://127.0.0.1:4000/skylink$request_uri;
+	}
+
+	location ~ "^/web/([a-zA-Z0-9-_+/]{46})$" {
+		client_max_body_size 1000M;
+		#post_max_size 1000M;
+		proxy_read_timeout 600;
+		proxy_set_header User-Agent: "Sia-Agent";
+		proxy_set_header Host: localhost;
+		proxy_pass http://127.0.0.1:4000/skylink/$1;
+
+		sub_filter 'sia://' 'https://$host/';
+    		sub_filter_once off;
+		sub_filter_types text/plain;
+	}
+
+	location /direct/ {
+		client_max_body_size 1000M;
+		#post_max_size 1000M;
+		proxy_read_timeout 600;
+		proxy_set_header User-Agent: "Sia-Agent";
+		proxy_pass http://127.0.0.1:9980/skynet/skylink/;
+	}
+
+	location /uploaddirect/ {
+		client_max_body_size 1000M;
+		#post_max_size 1000M;
+		proxy_read_timeout 600;
+		proxy_set_header User-Agent: "Sia-Agent";
+		proxy_pass http://127.0.0.1:9980/renter/linkfile/linkfiles/externaluploads/;
+	}
+
+	location /stream/ {
+		client_max_body_size 1000M;
+		#post_max_size 1000M;
+		proxy_read_timeout 600;
+		proxy_set_header User-Agent: "Sia-Agent";
+		proxy_pass http://127.0.0.1:9980/renter/stream/;
+	}
+}

setup-scripts/ssh_config

@@ -0,0 +1,51 @@
+
+# This is the ssh client system-wide configuration file.  See
+# ssh_config(5) for more information.  This file provides defaults for
+# users, and the values can be changed in per-user configuration files
+# or on the command line.
+
+# Configuration data is parsed as follows:
+#  1. command line options
+#  2. user-specific file
+#  3. system-wide file
+# Any configuration value is only changed the first time it is set.
+# Thus, host-specific definitions should be at the beginning of the
+# configuration file, and defaults at the end.
+
+# Site-wide defaults for some commonly used options.  For a comprehensive
+# list of available options, their meanings and defaults, please see the
+# ssh_config(5) man page.
+
+Host *
+#   ForwardAgent no
+#   ForwardX11 no
+#   ForwardX11Trusted yes
+    PasswordAuthentication no
+#   HostbasedAuthentication no
+#   GSSAPIAuthentication no
+#   GSSAPIDelegateCredentials no
+#   GSSAPIKeyExchange no
+#   GSSAPITrustDNS no
+#   BatchMode no
+#   CheckHostIP yes
+#   AddressFamily any
+#   ConnectTimeout 0
+#   StrictHostKeyChecking ask
+#   IdentityFile ~/.ssh/id_rsa
+#   IdentityFile ~/.ssh/id_dsa
+#   IdentityFile ~/.ssh/id_ecdsa
+#   IdentityFile ~/.ssh/id_ed25519
+#   Port 22
+#   Protocol 2
+#   Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-cbc,3des-cbc
+#   MACs hmac-md5,hmac-sha1,umac-64@openssh.com
+#   EscapeChar ~
+#   Tunnel no
+#   TunnelDevice any:any
+#   PermitLocalCommand no
+#   VisualHostKey no
+#   ProxyCommand ssh -q -W %h:%p gateway.example.com
+#   RekeyLimit 1G 1h
+    SendEnv LANG LC_*
+    HashKnownHosts no
+    GSSAPIAuthentication yes

setup-scripts/tmux.conf

@@ -0,0 +1,18 @@
+# remap prefix from 'C-b' to 'C-a'
+unbind C-b
+set-option -g prefix C-a
+bind-key C-a send-prefix
+
+# split panes using | and -
+bind | split-window -h
+bind - split-window -v
+unbind '"'
+unbind %
+
+# reload config file (change file location to your the tmux.conf you want to use)
+bind r source-file ~/.tmux.conf
+
+set -g visual-activity off
+set -g mouse on
+# This copies highlighted text.
+set -g mouse-select-window on