diff --git a/.gitignore b/.gitignore index 8257649e..cf2b977d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ node_modules -**/*/.next \ No newline at end of file +**/*/.next + +*.swo +*.swp diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..ae1aaeb7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Nebulous + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 00000000..98163dd0 --- /dev/null +++ b/README.md @@ -0,0 +1,10 @@ +# Skynet Portal + +## Setup Guide +A setup guide with scripts to install prerequisites can be found in the +[setup-scripts](./setup-scripts) directory. + +Once the setup guide is complete you will be running: +- `siad` configured as a ViewNode +- a NodeJS app running a Skynet Portal +- an nginx proxy diff --git a/packages/siaviewnode-server/src/main.ts b/packages/siaviewnode-server/src/main.ts index 03b7126a..86e2f75e 100644 --- a/packages/siaviewnode-server/src/main.ts +++ b/packages/siaviewnode-server/src/main.ts @@ -4,12 +4,15 @@ import express, { Request, Response } from "express" import fileUpload, { UploadedFile } from "express-fileupload" import proxy from "express-http-proxy" import requestId from "express-request-id" +import fs from "fs" import morgan from 'morgan' +import { homedir } from "os" import R from "ramda" import shortid from "shortid" import { Logger } from "winston" import logger from "./logger" + // import * as AxiosLogger from 'axios-logger' // AxiosLogger.setGlobalConfig({ // prefixText: 'your prefix', @@ -40,7 +43,7 @@ const siad = axios.create({ }, auth: { username: "", - password: "d05bb024715aea0bb734ce057acbae27" + password: fs.readFileSync(homedir().concat("/.sia/apipassword"), "utf8").trim() } }) diff --git a/setup-scripts/README.md b/setup-scripts/README.md new file mode 100644 index 00000000..f190d434 --- /dev/null +++ b/setup-scripts/README.md @@ -0,0 +1,75 @@ +# Skynet Portal Setup Scripts + +This directory contains a setup guide and scripts that will install and +configure some basic requirements for running a Skynet Portal. The assumption is +that we are working with a Debian Buster Minimal system or similar. + +## Initial Setup +(Assumes we are logged in as root on a fresh installation of Debian) + +You may want to fork this repository and add your ssh pubkey to +`authorized_keys` and optionally edit the `tmux` and `bash` configurations. + +0. SSH in a freshly installed Debian machine. +1. `apt-get update && apt-get install sudo` +2. `adduser user` +3. `usermod -a -G sudo user` +4. Quit the ssh session. + +You a can now ssh into your machine as the user `user`. + +5. On your local machine: `ssh-copy-id user@ip-addr` +6. On your local machine: `ssh user@ip-addr` +7. Now logged in as `user`: `sudo apt-get install git` +8. `git clone https://github.com/NebulousLabs/skynet-webportal` +9. `cd skynet-webportal/setup-scripts` +11. `./setup.sh` +12. Once DNS records are set you can run: `./letsencrypt-setup.sh` +13. You should also change the nginx configuration to listen on port 443 + instead. + +## Setting up siad + +NOTE: You must be running `siad` and `siac` by building from the `viewnode` +branch. + +You still need to setup `siad` for the backend to be complete. +1. `cd ~/; mkdir siad` +2. `nohup siad &>/dev/null &` + +This will start syncing `siad` in the background. + +## ViewNode setup + +When `siad` is done syncing, create a new wallet and unlock the wallet. + +Then set an allowance (`siac renter setallowance`), with the suggested values +below: +- 10 KS (keep 25 KS in your wallet) +- default period +- default number of hosts +- 8 week renewal time +- 500 GB expected storage +- 500 GB expected upload +- 5 TB expected download +- default redundancy + +Once your allowance is set you need to set your node to be a viewnode with the +following command: +`siac renter setallowance --payment-contract-initial-price 10SC` + +Now your node will begin making 10 contracts per block with many hosts so it can +potentially view the whole network's files. + +## Running the Portal +`cd` into the parent directory and run `yarn` to build dependencies. + +We recommend running the Portal through `pm2` (a nodejs process manager) in the background with the command: + +`pm2 --name skynet start npm -- start` + +`yarn start` will also work if not using `pm2`. + +The Protal which will automatically read your `siad` API password and startup a +portal on `localhost:3000`. nginx will expose this to port 80 or 443 if you +configured it for SSL. diff --git a/setup-scripts/authorized_keys b/setup-scripts/authorized_keys new file mode 100644 index 00000000..e69de29b diff --git a/setup-scripts/bashrc b/setup-scripts/bashrc new file mode 100644 index 00000000..3cc9f795 --- /dev/null +++ b/setup-scripts/bashrc @@ -0,0 +1,114 @@ +# ~/.bashrc: executed by bash(1) for non-login shells. +# see /usr/share/doc/bash/examples/startup-files (in the package bash-doc) +# for examples + +# If not running interactively, don't do anything +case $- in + *i*) ;; + *) return;; +esac + +# don't put duplicate lines or lines starting with space in the history. +# See bash(1) for more options +HISTCONTROL=ignoreboth + +# append to the history file, don't overwrite it +shopt -s histappend + +# for setting history length see HISTSIZE and HISTFILESIZE in bash(1) +HISTSIZE=1000 +HISTFILESIZE=2000 + +# check the window size after each command and, if necessary, +# update the values of LINES and COLUMNS. +shopt -s checkwinsize + +# If set, the pattern "**" used in a pathname expansion context will +# match all files and zero or more directories and subdirectories. +#shopt -s globstar + +# make less more friendly for non-text input files, see lesspipe(1) +#[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)" + +# set variable identifying the chroot you work in (used in the prompt below) +if [ -z "${debian_chroot:-}" ] && [ -r /etc/debian_chroot ]; then + debian_chroot=$(cat /etc/debian_chroot) +fi + +# set a fancy prompt (non-color, unless we know we "want" color) +case "$TERM" in + alacritty|xterm-color|*-256color) color_prompt=yes;; +esac + +# uncomment for a colored prompt, if the terminal has the capability; turned +# off by default to not distract the user: the focus in a terminal window +# should be on the output of commands, not on the prompt +#force_color_prompt=yes + +if [ -n "$force_color_prompt" ]; then + if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then + # We have color support; assume it's compliant with Ecma-48 + # (ISO/IEC-6429). (Lack of such support is extremely rare, and such + # a case would tend to support setf rather than setaf.) + color_prompt=yes + else + color_prompt= + fi +fi + +if [ "$color_prompt" = yes ]; then + PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' +else + PS1='${debian_chroot:+($debian_chroot)}\u@\h:\w\$ ' +fi +unset color_prompt force_color_prompt + +# If this is an xterm set the title to user@host:dir +case "$TERM" in +xterm*|rxvt*) + PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1" + ;; +*) + ;; +esac + +# enable color support of ls and also add handy aliases +if [ -x /usr/bin/dircolors ]; then + test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)" + alias ls='ls --color=auto' + #alias dir='dir --color=auto' + #alias vdir='vdir --color=auto' + + #alias grep='grep --color=auto' + #alias fgrep='fgrep --color=auto' + #alias egrep='egrep --color=auto' +fi + +# colored GCC warnings and errors +#export GCC_COLORS='error=01;31:warning=01;35:note=01;36:caret=01;32:locus=01:quote=01' + +# some more ls aliases +#alias ll='ls -l' +#alias la='ls -A' +#alias l='ls -CF' + +# Alias definitions. +# You may want to put all your additions into a separate file like +# ~/.bash_aliases, instead of adding them here directly. +# See /usr/share/doc/bash-doc/examples in the bash-doc package. + +if [ -f ~/.bash_aliases ]; then + . ~/.bash_aliases +fi + +# enable programmable completion features (you don't need to enable +# this, if it's already enabled in /etc/bash.bashrc and /etc/profile +# sources /etc/bash.bashrc). +if ! shopt -oq posix; then + if [ -f /usr/share/bash-completion/bash_completion ]; then + . /usr/share/bash-completion/bash_completion + elif [ -f /etc/bash_completion ]; then + . /etc/bash_completion + fi +fi +export PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games:/usr/local/go/bin:/home/user/go/bin diff --git a/setup-scripts/letsencrypt-setup.sh b/setup-scripts/letsencrypt-setup.sh new file mode 100755 index 00000000..a6b8f030 --- /dev/null +++ b/setup-scripts/letsencrypt-setup.sh @@ -0,0 +1,11 @@ +#! /usr/bin/env bash +set -e + +domain="$1" +if [[ -z $domain ]]; then + echo "Usage $0 DOMAIN_NAME" + exit 1 +fi + +sudo certbot --nginx -d "$domain" -d www."$domain" +sudo certbot renew --dry-run diff --git a/setup-scripts/setup.sh b/setup-scripts/setup.sh new file mode 100755 index 00000000..56d457da --- /dev/null +++ b/setup-scripts/setup.sh @@ -0,0 +1,68 @@ +#! /usr/bin/env bash +set -e + +# Copy over basic configuration files. +cp ./tmux.conf ~/.tmux.conf +cp ./bashrc ~/.bashrc +source ~/.bashrc + +# Add SSH keys and set SSH configs +sudo cp ./ssh_config /etc/ssh/ssh_config +mkdir -p ~/.ssh +cat ./authorized_keys >> ~/.ssh/authorized_keys + +# Nodejs install prerequisite. From official documentation. +curl -sL https://deb.nodesource.com/setup_13.x | sudo -E bash - + +# Yarn install prerequisite. +curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - +echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list + +# Apt installations. +sudo apt-get update +sudo apt-get -y install ufw tmux ranger htop nload nginx certbot \ + python-certbot-nginx nodejs gcc g++ make yarn git vim + +# Install pm2 +sudo npm i -g pm2 + +# terminfo for alacritty terminal via ssh +# If you don't use the alacritty terminal you can remove this step. +wget -c https://raw.githubusercontent.com/alacritty/alacritty/master/extra/alacritty.info +sudo tic -xe alacritty,alacritty-direct alacritty.info +rm alacritty.info + +# Setup nginx config +sudo cp ./skynet-nginx.conf /etc/nginx/sites-available/skynet +sudo nginx -t +sudo ln -s /etc/nginx/sites-available/skynet /etc/nginx/sites-enabled/skynet +sudo rm /etc/nginx/sites-enabled/default +sudo systemctl reload nginx + +# Setup firewall +# TODO: disable plain HTTP eventually +sudo ufw enable +sudo ufw allow ssh +sudo ufw allow 'Nginx Full' +sudo ufw allow 'Nginx HTTP' + +# Install Go 1.13.7. +wget -c https://dl.google.com/go/go1.13.7.linux-amd64.tar.gz +sudo tar -C /usr/local -xzf go1.13.7.linux-amd64.tar.gz +source ~/.bashrc +rm go1.13.7.linux-amd64.tar.gz + +# Sanity check that will pass if go was installed correctly. +go version + +cwd=$(pwd) + +# Install Sia +cd ~/ +git clone https://gitlab.com/NebulousLabs/Sia +cd Sia && git checkout viewnode && make + +# Setup skynet frontend. +cd $cwd +cd ../ +yarn diff --git a/setup-scripts/skynet-nginx.conf b/setup-scripts/skynet-nginx.conf new file mode 100644 index 00000000..8ecd4f66 --- /dev/null +++ b/setup-scripts/skynet-nginx.conf @@ -0,0 +1,69 @@ +server { + listen 80; + + location / { + client_max_body_size 1000M; + #post_max_size 1000M; + proxy_read_timeout 600; + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + } + + location ^~ /api/ { + client_max_body_size 1000M; + #post_max_size 1000M; + proxy_read_timeout 600; + rewrite ^/api/(.*)$ /$1 break; + proxy_pass http://127.0.0.1:4000/; + } + + location ~ "^/[a-zA-Z0-9-_+/]{46}$" { + client_max_body_size 1000M; + #post_max_size 1000M; + proxy_read_timeout 600; + proxy_set_header User-Agent: "Sia-Agent"; + proxy_set_header Host: localhost; + proxy_pass http://127.0.0.1:4000/skylink$request_uri; + } + + location ~ "^/web/([a-zA-Z0-9-_+/]{46})$" { + client_max_body_size 1000M; + #post_max_size 1000M; + proxy_read_timeout 600; + proxy_set_header User-Agent: "Sia-Agent"; + proxy_set_header Host: localhost; + proxy_pass http://127.0.0.1:4000/skylink/$1; + + sub_filter 'sia://' 'https://$host/'; + sub_filter_once off; + sub_filter_types text/plain; + } + + location /direct/ { + client_max_body_size 1000M; + #post_max_size 1000M; + proxy_read_timeout 600; + proxy_set_header User-Agent: "Sia-Agent"; + proxy_pass http://127.0.0.1:9980/skynet/skylink/; + } + + location /uploaddirect/ { + client_max_body_size 1000M; + #post_max_size 1000M; + proxy_read_timeout 600; + proxy_set_header User-Agent: "Sia-Agent"; + proxy_pass http://127.0.0.1:9980/renter/linkfile/linkfiles/externaluploads/; + } + + location /stream/ { + client_max_body_size 1000M; + #post_max_size 1000M; + proxy_read_timeout 600; + proxy_set_header User-Agent: "Sia-Agent"; + proxy_pass http://127.0.0.1:9980/renter/stream/; + } +} diff --git a/setup-scripts/ssh_config b/setup-scripts/ssh_config new file mode 100644 index 00000000..1b80182d --- /dev/null +++ b/setup-scripts/ssh_config @@ -0,0 +1,51 @@ + +# This is the ssh client system-wide configuration file. See +# ssh_config(5) for more information. This file provides defaults for +# users, and the values can be changed in per-user configuration files +# or on the command line. + +# Configuration data is parsed as follows: +# 1. command line options +# 2. user-specific file +# 3. system-wide file +# Any configuration value is only changed the first time it is set. +# Thus, host-specific definitions should be at the beginning of the +# configuration file, and defaults at the end. + +# Site-wide defaults for some commonly used options. For a comprehensive +# list of available options, their meanings and defaults, please see the +# ssh_config(5) man page. + +Host * +# ForwardAgent no +# ForwardX11 no +# ForwardX11Trusted yes + PasswordAuthentication no +# HostbasedAuthentication no +# GSSAPIAuthentication no +# GSSAPIDelegateCredentials no +# GSSAPIKeyExchange no +# GSSAPITrustDNS no +# BatchMode no +# CheckHostIP yes +# AddressFamily any +# ConnectTimeout 0 +# StrictHostKeyChecking ask +# IdentityFile ~/.ssh/id_rsa +# IdentityFile ~/.ssh/id_dsa +# IdentityFile ~/.ssh/id_ecdsa +# IdentityFile ~/.ssh/id_ed25519 +# Port 22 +# Protocol 2 +# Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-cbc,3des-cbc +# MACs hmac-md5,hmac-sha1,umac-64@openssh.com +# EscapeChar ~ +# Tunnel no +# TunnelDevice any:any +# PermitLocalCommand no +# VisualHostKey no +# ProxyCommand ssh -q -W %h:%p gateway.example.com +# RekeyLimit 1G 1h + SendEnv LANG LC_* + HashKnownHosts no + GSSAPIAuthentication yes diff --git a/setup-scripts/tmux.conf b/setup-scripts/tmux.conf new file mode 100644 index 00000000..043bab9f --- /dev/null +++ b/setup-scripts/tmux.conf @@ -0,0 +1,18 @@ +# remap prefix from 'C-b' to 'C-a' +unbind C-b +set-option -g prefix C-a +bind-key C-a send-prefix + +# split panes using | and - +bind | split-window -h +bind - split-window -v +unbind '"' +unbind % + +# reload config file (change file location to your the tmux.conf you want to use) +bind r source-file ~/.tmux.conf + +set -g visual-activity off +set -g mouse on +# This copies highlighted text. +set -g mouse-select-window on