From ecb73e7498f0e548e9dab62a56fffb84d88173c5 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 27 Aug 2021 14:15:22 +0200 Subject: [PATCH] drop caddy as proxy --- docker-compose.accounts.yml | 2 - docker-compose.yml | 14 +- docker/accounts/nginx.account.conf | 20 - docker/caddy/Caddyfile | 45 - docker/caddy/Dockerfile | 14 +- docker/caddy/caddy.json.template | 38 + docker/caddy/mo | 1106 +++++++++++++++++ docker/nginx/Dockerfile | 22 +- docker/nginx/Dockerfile.bionic | 185 --- .../conf.d.templates/server.account.conf | 39 + docker/nginx/conf.d.templates/server.api.conf | 37 + docker/nginx/conf.d.templates/server.hns.conf | 39 + .../conf.d.templates/server.skylink.conf | 37 + docker/nginx/conf.d/client.conf | 578 --------- docker/nginx/conf.d/dhparam.pem | 8 + .../conf.d/include/init-optional-variables | 9 + docker/nginx/conf.d/include/location-hns | 94 ++ docker/nginx/conf.d/include/location-skylink | 98 ++ .../conf.d/include/location-skynet-registry | 15 +- docker/nginx/conf.d/include/ssl-settings | 13 + docker/nginx/conf.d/include/track-download | 28 +- docker/nginx/conf.d/include/track-registry | 27 +- docker/nginx/conf.d/include/track-upload | 26 +- docker/nginx/conf.d/server.dnslink.conf | 16 + docker/nginx/conf.d/server/server.account | 10 + docker/nginx/conf.d/server/server.api | 294 +++++ docker/nginx/conf.d/server/server.dnslink | 26 + docker/nginx/conf.d/server/server.hns | 12 + docker/nginx/conf.d/server/server.http | 8 + docker/nginx/conf.d/server/server.skylink | 12 + docker/nginx/mo | 1106 +++++++++++++++++ docker/nginx/nginx.conf | 50 +- docker/nginx/nginx.vh.default.conf | 58 - packages/dnslink-api/src/index.js | 4 +- packages/health-check/Dockerfile | 5 +- setup-scripts/README.md | 76 +- setup-scripts/setup-docker-services.sh | 5 +- 37 files changed, 3157 insertions(+), 1019 deletions(-) delete mode 100644 docker/accounts/nginx.account.conf delete mode 100644 docker/caddy/Caddyfile create mode 100644 docker/caddy/caddy.json.template create mode 100755 docker/caddy/mo delete mode 100644 docker/nginx/Dockerfile.bionic create mode 100644 docker/nginx/conf.d.templates/server.account.conf create mode 100644 docker/nginx/conf.d.templates/server.api.conf create mode 100644 docker/nginx/conf.d.templates/server.hns.conf create mode 100644 docker/nginx/conf.d.templates/server.skylink.conf delete mode 100644 docker/nginx/conf.d/client.conf create mode 100644 docker/nginx/conf.d/dhparam.pem create mode 100644 docker/nginx/conf.d/include/init-optional-variables create mode 100644 docker/nginx/conf.d/include/location-hns create mode 100644 docker/nginx/conf.d/include/location-skylink create mode 100644 docker/nginx/conf.d/include/ssl-settings create mode 100644 docker/nginx/conf.d/server.dnslink.conf create mode 100644 docker/nginx/conf.d/server/server.account create mode 100644 docker/nginx/conf.d/server/server.api create mode 100644 docker/nginx/conf.d/server/server.dnslink create mode 100644 docker/nginx/conf.d/server/server.hns create mode 100644 docker/nginx/conf.d/server/server.http create mode 100644 docker/nginx/conf.d/server/server.skylink create mode 100755 docker/nginx/mo delete mode 100644 docker/nginx/nginx.vh.default.conf diff --git a/docker-compose.accounts.yml b/docker-compose.accounts.yml index b35b59ac..d0e1cc29 100644 --- a/docker-compose.accounts.yml +++ b/docker-compose.accounts.yml @@ -10,8 +10,6 @@ services: nginx: environment: - ACCOUNTS_ENABLED=true - volumes: - - ./docker/accounts/nginx.account.conf:/etc/nginx/conf.extra.d/nginx.account.conf:ro depends_on: - accounts diff --git a/docker-compose.yml b/docker-compose.yml index b61b44c3..b8af7933 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,15 +47,9 @@ services: volumes: - ./docker/data/caddy/data:/data - ./docker/data/caddy/config:/config - - ./docker/caddy/Caddyfile:/etc/caddy/Caddyfile networks: shared: ipv4_address: 10.10.10.20 - ports: - - "80:80" - - "443:443" - depends_on: - - nginx nginx: build: @@ -68,18 +62,20 @@ services: - .env volumes: - ./docker/nginx/nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf:ro - - ./docker/nginx/conf.d:/etc/nginx/conf.d:ro - ./docker/data/nginx/cache:/data/nginx/cache - ./docker/data/nginx/logs:/usr/local/openresty/nginx/logs - ./docker/data/nginx/skynet:/data/nginx/skynet:ro - ./docker/data/sia/apipassword:/data/sia/apipassword:ro + - ./docker/data/caddy/data:/data/caddy:ro networks: shared: ipv4_address: 10.10.10.30 - expose: - - 80 + ports: + - "443:443" + - "80:80" depends_on: - sia + - caddy - handshake-api - dnslink-api - website diff --git a/docker/accounts/nginx.account.conf b/docker/accounts/nginx.account.conf deleted file mode 100644 index f3705200..00000000 --- a/docker/accounts/nginx.account.conf +++ /dev/null @@ -1,20 +0,0 @@ -server { - listen 80; - listen [::]:80; - server_name account.*; - - location / { - proxy_redirect http://127.0.0.1/ https://$host/; - proxy_pass http://oathkeeper:4455; - } -} - -server { - listen 80; - listen [::]:80; - server_name secure.*; - - if ($host ~ secure.(.*)) { - return 301 $scheme://account.$1$request_uri; - } -} diff --git a/docker/caddy/Caddyfile b/docker/caddy/Caddyfile deleted file mode 100644 index a2f53797..00000000 --- a/docker/caddy/Caddyfile +++ /dev/null @@ -1,45 +0,0 @@ -# This block below is optional if you want to generate an internal certificate for the server ip address. -# It is useful in case you have services trying to reach the server through ip and not domain like health checks. -# It will generate an internal certificate so browsers will warn you when connecting but that not a problem. - -:443 { - tls internal { - on_demand - } - - reverse_proxy nginx:80 { - # add Dnslink-Lookup header so nginx knows that the request comes from a domain - # outside of our certificate string and should perform a dnslink lookup - header_up Dnslink-Lookup true - } -} - -# Make sure you have SSL_CERTIFICATE_STRING specified in .env file because you need it to fetch correct certificates. -# It needs to have at least 3 parts, the absolute part (ie. example.com), the wildcard part (ie. *.example.com) and -# the hns wildcard part (ie. *.hns.example.com). The resulting string should look like: -# example.com, *.example.com, *.hns.example.com -# In addition, if you are running multiple servers for the single portal like we do on siasky.net, you might want to -# add an aliased string that is going to help you access and distinguish between servers, the result would look like: -# example.com, *.example.com, *.hns.example.com, *.germany.example.com, *.hns.germany.example.com -# Note that you don't need to specify the absolute part for the alias since it's already covered in the wildcard part -# of the original certificate string (*.example.com). - -{$SSL_CERTIFICATE_STRING} { - # If you want to use basic http-01 (basic, good for one server setup) certificate challenge - # then uncomment the line below and make sure you have EMAIL_ADDRESS specified in .env file - # and comment the tls block that contains the dns challenge configuration. - - # tls {$EMAIL_ADDRESS} - - tls { - # We are using route53 as our dns provider and it requires additional AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - # environment variables in .env file. You can use other providers by using specific package from - # https://github.com/caddy-dns in the docker/caddy/Dockerfile instead of our route53 one. - - dns route53 { - max_retries 50 - } - } - - reverse_proxy nginx:80 -} diff --git a/docker/caddy/Dockerfile b/docker/caddy/Dockerfile index 02b9b777..9d1fdaea 100644 --- a/docker/caddy/Dockerfile +++ b/docker/caddy/Dockerfile @@ -1,8 +1,18 @@ -FROM caddy:2.4.1-builder AS caddy-builder +FROM caddy:2.4.3-builder AS caddy-builder # available dns resolvers: https://github.com/caddy-dns RUN xcaddy build --with github.com/caddy-dns/route53 -FROM caddy:2.4.1 +FROM caddy:2.4.3 COPY --from=caddy-builder /usr/bin/caddy /usr/bin/caddy + +# bash required for mo to work (mo is mustache templating engine - https://github.com/tests-always-included/mo) +RUN apk add --no-cache bash + +COPY caddy.json.template mo /etc/caddy/ + +CMD [ "sh", "-c", \ + "/etc/caddy/mo < /etc/caddy/caddy.json.template > /etc/caddy/caddy.json ; \ + caddy run --config /etc/caddy/caddy.json" \ + ] diff --git a/docker/caddy/caddy.json.template b/docker/caddy/caddy.json.template new file mode 100644 index 00000000..de5efcbf --- /dev/null +++ b/docker/caddy/caddy.json.template @@ -0,0 +1,38 @@ +{ + "apps": { + "tls": { + "certificates": { + "automate": [ + {{#PORTAL_DOMAIN}} + "{{PORTAL_DOMAIN}}", "*.{{PORTAL_DOMAIN}}", "*.hns.{{PORTAL_DOMAIN}}" + {{/PORTAL_DOMAIN}} + + {{#PORTAL_DOMAIN}}{{#SERVER_DOMAIN}},{{/SERVER_DOMAIN}}{{/PORTAL_DOMAIN}} + + {{#SERVER_DOMAIN}} + "{{SERVER_DOMAIN}}", "*.{{SERVER_DOMAIN}}", "*.hns.{{SERVER_DOMAIN}}" + {{/SERVER_DOMAIN}} + ] + }, + "automation": { + "policies": [ + { + "issuers": [ + { + "module": "acme", + "challenges": { + "dns": { + "provider": { + "name": "route53", + "max_retries": 100 + } + } + } + } + ] + } + ] + } + } + } +} diff --git a/docker/caddy/mo b/docker/caddy/mo new file mode 100755 index 00000000..ba8e48d1 --- /dev/null +++ b/docker/caddy/mo @@ -0,0 +1,1106 @@ +#!/usr/bin/env bash +# +#/ Mo is a mustache template rendering software written in bash. It inserts +#/ environment variables into templates. +#/ +#/ Simply put, mo will change {{VARIABLE}} into the value of that +#/ environment variable. You can use {{#VARIABLE}}content{{/VARIABLE}} to +#/ conditionally display content or iterate over the values of an array. +#/ +#/ Learn more about mustache templates at https://mustache.github.io/ +#/ +#/ Simple usage: +#/ +#/ mo [OPTIONS] filenames... +#/ +#/ Options: +#/ +#/ -u, --fail-not-set +#/ Fail upon expansion of an unset variable. +#/ -x, --fail-on-function +#/ Fail when a function returns a non-zero status code. +#/ -e, --false +#/ Treat the string "false" as empty for conditionals. +#/ -h, --help +#/ This message. +#/ -s=FILE, --source=FILE +#/ Load FILE into the environment before processing templates. +#/ Can be used multiple times. +# +# Mo is under a MIT style licence with an additional non-advertising clause. +# See LICENSE.md for the full text. +# +# This is open source! Please feel free to contribute. +# +# https://github.com/tests-always-included/mo + + +# Public: Template parser function. Writes templates to stdout. +# +# $0 - Name of the mo file, used for getting the help message. +# $@ - Filenames to parse. +# +# Options: +# +# --allow-function-arguments +# +# Permit functions in templates to be called with additional arguments. This +# puts template data directly in to the path of an eval statement. Use with +# caution. Not listed in the help because it only makes sense when mo is +# sourced. +# +# -u, --fail-not-set +# +# Fail upon expansion of an unset variable. Default behavior is to silently +# ignore and expand into empty string. +# +# -x, --fail-on-function +# +# Fail when a function used by a template returns an error status code. +# Alternately, ou may set the MO_FAIL_ON_FUNCTION environment variable to a +# non-empty value to enable this behavior. +# +# -e, --false +# +# Treat "false" as an empty value. You may set the MO_FALSE_IS_EMPTY +# environment variable instead to a non-empty value to enable this behavior. +# +# -h, --help +# +# Display a help message. +# +# -s=FILE, --source=FILE +# +# Source a file into the environment before processing template files. +# This can be used multiple times. +# +# -- +# +# Used to indicate the end of options. You may optionally use this when +# filenames may start with two hyphens. +# +# Mo uses the following environment variables: +# +# MO_ALLOW_FUNCTION_ARGUMENTS - When set to a non-empty value, this allows +# functions referenced in templates to receive additional +# options and arguments. This puts the content from the +# template directly into an eval statement. Use with extreme +# care. +# MO_FUNCTION_ARGS - Arguments passed to the function +# MO_FAIL_ON_FUNCTION - If a function returns a non-zero status code, abort +# with an error. +# MO_FAIL_ON_UNSET - When set to a non-empty value, expansion of an unset env +# variable will be aborted with an error. +# MO_FALSE_IS_EMPTY - When set to a non-empty value, the string "false" will be +# treated as an empty value for the purposes of conditionals. +# MO_ORIGINAL_COMMAND - Used to find the `mo` program in order to generate a +# help message. +# +# Returns nothing. +mo() ( + # This function executes in a subshell so IFS is reset. + # Namespace this variable so we don't conflict with desired values. + local moContent f2source files doubleHyphens + + IFS=$' \n\t' + files=() + doubleHyphens=false + + if [[ $# -gt 0 ]]; then + for arg in "$@"; do + if $doubleHyphens; then + #: After we encounter two hyphens together, all the rest + #: of the arguments are files. + files=("${files[@]}" "$arg") + else + case "$arg" in + -h|--h|--he|--hel|--help|-\?) + moUsage "$0" + exit 0 + ;; + + --allow-function-arguments) + # shellcheck disable=SC2030 + MO_ALLOW_FUNCTION_ARGUMENTS=true + ;; + + -u | --fail-not-set) + # shellcheck disable=SC2030 + MO_FAIL_ON_UNSET=true + ;; + + -x | --fail-on-function) + # shellcheck disable=SC2030 + MO_FAIL_ON_FUNCTION=true + ;; + + -e | --false) + # shellcheck disable=SC2030 + MO_FALSE_IS_EMPTY=true + ;; + + -s=* | --source=*) + if [[ "$arg" == --source=* ]]; then + f2source="${arg#--source=}" + else + f2source="${arg#-s=}" + fi + + if [[ -f "$f2source" ]]; then + # shellcheck disable=SC1090 + . "$f2source" + else + echo "No such file: $f2source" >&2 + exit 1 + fi + ;; + + --) + #: Set a flag indicating we've encountered double hyphens + doubleHyphens=true + ;; + + *) + #: Every arg that is not a flag or a option should be a file + files=(${files[@]+"${files[@]}"} "$arg") + ;; + esac + fi + done + fi + + moGetContent moContent "${files[@]}" || return 1 + moParse "$moContent" "" true +) + + +# Internal: Call a function. +# +# $1 - Variable for output +# $2 - Function to call +# $3 - Content to pass +# $4 - Additional arguments as a single string +# +# This can be dangerous, especially if you are using tags like +# {{someFunction ; rm -rf / }} +# +# Returns nothing. +moCallFunction() { + local moArgs moContent moFunctionArgs moFunctionResult + + moArgs=() + moTrimWhitespace moFunctionArgs "$4" + + # shellcheck disable=SC2031 + if [[ -n "${MO_ALLOW_FUNCTION_ARGUMENTS-}" ]]; then + # Intentionally bad behavior + # shellcheck disable=SC2206 + moArgs=($4) + fi + + moContent=$(echo -n "$3" | MO_FUNCTION_ARGS="$moFunctionArgs" eval "$2" "${moArgs[@]}") || { + moFunctionResult=$? + # shellcheck disable=SC2031 + if [[ -n "${MO_FAIL_ON_FUNCTION-}" && "$moFunctionResult" != 0 ]]; then + echo "Function '$2' with args (${moArgs[*]+"${moArgs[@]}"}) failed with status code $moFunctionResult" + exit "$moFunctionResult" + fi + } + + # shellcheck disable=SC2031 + local "$1" && moIndirect "$1" "$moContent" +} + + +# Internal: Scan content until the right end tag is found. Creates an array +# with the following members: +# +# [0] = Content before end tag +# [1] = End tag (complete tag) +# [2] = Content after end tag +# +# Everything using this function uses the "standalone tags" logic. +# +# $1 - Name of variable for the array +# $2 - Content +# $3 - Name of end tag +# $4 - If -z, do standalone tag processing before finishing +# +# Returns nothing. +moFindEndTag() { + local content remaining scanned standaloneBytes tag + + #: Find open tags + scanned="" + moSplit content "$2" '{{' '}}' + + while [[ "${#content[@]}" -gt 1 ]]; do + moTrimWhitespace tag "${content[1]}" + + #: Restore content[1] before we start using it + content[1]='{{'"${content[1]}"'}}' + + case $tag in + '#'* | '^'*) + #: Start another block + scanned="${scanned}${content[0]}${content[1]}" + moTrimWhitespace tag "${tag:1}" + moFindEndTag content "${content[2]}" "$tag" "loop" + scanned="${scanned}${content[0]}${content[1]}" + remaining=${content[2]} + ;; + + '/'*) + #: End a block - could be ours + moTrimWhitespace tag "${tag:1}" + scanned="$scanned${content[0]}" + + if [[ "$tag" == "$3" ]]; then + #: Found our end tag + if [[ -z "${4-}" ]] && moIsStandalone standaloneBytes "$scanned" "${content[2]}" true; then + #: This is also a standalone tag - clean up whitespace + #: and move those whitespace bytes to the "tag" element + # shellcheck disable=SC2206 + standaloneBytes=( $standaloneBytes ) + content[1]="${scanned:${standaloneBytes[0]}}${content[1]}${content[2]:0:${standaloneBytes[1]}}" + scanned="${scanned:0:${standaloneBytes[0]}}" + content[2]="${content[2]:${standaloneBytes[1]}}" + fi + + local "$1" && moIndirectArray "$1" "$scanned" "${content[1]}" "${content[2]}" + return 0 + fi + + scanned="$scanned${content[1]}" + remaining=${content[2]} + ;; + + *) + #: Ignore all other tags + scanned="${scanned}${content[0]}${content[1]}" + remaining=${content[2]} + ;; + esac + + moSplit content "$remaining" '{{' '}}' + done + + #: Did not find our closing tag + scanned="$scanned${content[0]}" + local "$1" && moIndirectArray "$1" "${scanned}" "" "" +} + + +# Internal: Find the first index of a substring. If not found, sets the +# index to -1. +# +# $1 - Destination variable for the index +# $2 - Haystack +# $3 - Needle +# +# Returns nothing. +moFindString() { + local pos string + + string=${2%%$3*} + [[ "$string" == "$2" ]] && pos=-1 || pos=${#string} + local "$1" && moIndirect "$1" "$pos" +} + + +# Internal: Generate a dotted name based on current context and target name. +# +# $1 - Target variable to store results +# $2 - Context name +# $3 - Desired variable name +# +# Returns nothing. +moFullTagName() { + if [[ -z "${2-}" ]] || [[ "$2" == *.* ]]; then + local "$1" && moIndirect "$1" "$3" + else + local "$1" && moIndirect "$1" "${2}.${3}" + fi +} + + +# Internal: Fetches the content to parse into a variable. Can be a list of +# partials for files or the content from stdin. +# +# $1 - Variable name to assign this content back as +# $2-@ - File names (optional) +# +# Returns nothing. +moGetContent() { + local moContent moFilename moTarget + + moTarget=$1 + shift + if [[ "${#@}" -gt 0 ]]; then + moContent="" + + for moFilename in "$@"; do + #: This is so relative paths work from inside template files + moContent="$moContent"'{{>'"$moFilename"'}}' + done + else + moLoadFile moContent || return 1 + fi + + local "$moTarget" && moIndirect "$moTarget" "$moContent" +} + + +# Internal: Indent a string, placing the indent at the beginning of every +# line that has any content. +# +# $1 - Name of destination variable to get an array of lines +# $2 - The indent string +# $3 - The string to reindent +# +# Returns nothing. +moIndentLines() { + local content fragment len posN posR result trimmed + + result="" + + #: Remove the period from the end of the string. + len=$((${#3} - 1)) + content=${3:0:$len} + + if [[ -z "${2-}" ]]; then + local "$1" && moIndirect "$1" "$content" + + return 0 + fi + + moFindString posN "$content" $'\n' + moFindString posR "$content" $'\r' + + while [[ "$posN" -gt -1 ]] || [[ "$posR" -gt -1 ]]; do + if [[ "$posN" -gt -1 ]]; then + fragment="${content:0:$posN + 1}" + content=${content:$posN + 1} + else + fragment="${content:0:$posR + 1}" + content=${content:$posR + 1} + fi + + moTrimChars trimmed "$fragment" false true " " $'\t' $'\n' $'\r' + + if [[ -n "$trimmed" ]]; then + fragment="$2$fragment" + fi + + result="$result$fragment" + + moFindString posN "$content" $'\n' + moFindString posR "$content" $'\r' + + # If the content ends in a newline, do not indent. + if [[ "$posN" -eq ${#content} ]]; then + # Special clause for \r\n + if [[ "$posR" -eq "$((posN - 1))" ]]; then + posR=-1 + fi + + posN=-1 + fi + + if [[ "$posR" -eq ${#content} ]]; then + posR=-1 + fi + done + + moTrimChars trimmed "$content" false true " " $'\t' + + if [[ -n "$trimmed" ]]; then + content="$2$content" + fi + + result="$result$content" + + local "$1" && moIndirect "$1" "$result" +} + + +# Internal: Send a variable up to the parent of the caller of this function. +# +# $1 - Variable name +# $2 - Value +# +# Examples +# +# callFunc () { +# local "$1" && moIndirect "$1" "the value" +# } +# callFunc dest +# echo "$dest" # writes "the value" +# +# Returns nothing. +moIndirect() { + unset -v "$1" + printf -v "$1" '%s' "$2" +} + + +# Internal: Send an array as a variable up to caller of a function +# +# $1 - Variable name +# $2-@ - Array elements +# +# Examples +# +# callFunc () { +# local myArray=(one two three) +# local "$1" && moIndirectArray "$1" "${myArray[@]}" +# } +# callFunc dest +# echo "${dest[@]}" # writes "one two three" +# +# Returns nothing. +moIndirectArray() { + unset -v "$1" + + # IFS must be set to a string containing space or unset in order for + # the array slicing to work regardless of the current IFS setting on + # bash 3. This is detailed further at + # https://github.com/fidian/gg-core/pull/7 + eval "$(printf "IFS= %s=(\"\${@:2}\") IFS=%q" "$1" "$IFS")" +} + + +# Internal: Determine if a given environment variable exists and if it is +# an array. +# +# $1 - Name of environment variable +# +# Be extremely careful. Even if strict mode is enabled, it is not honored +# in newer versions of Bash. Any errors that crop up here will not be +# caught automatically. +# +# Examples +# +# var=(abc) +# if moIsArray var; then +# echo "This is an array" +# echo "Make sure you don't accidentally use \$var" +# fi +# +# Returns 0 if the name is not empty, 1 otherwise. +moIsArray() { + # Namespace this variable so we don't conflict with what we're testing. + local moTestResult + + moTestResult=$(declare -p "$1" 2>/dev/null) || return 1 + [[ "${moTestResult:0:10}" == "declare -a" ]] && return 0 + [[ "${moTestResult:0:10}" == "declare -A" ]] && return 0 + + return 1 +} + + +# Internal: Determine if the given name is a defined function. +# +# $1 - Function name to check +# +# Be extremely careful. Even if strict mode is enabled, it is not honored +# in newer versions of Bash. Any errors that crop up here will not be +# caught automatically. +# +# Examples +# +# moo () { +# echo "This is a function" +# } +# if moIsFunction moo; then +# echo "moo is a defined function" +# fi +# +# Returns 0 if the name is a function, 1 otherwise. +moIsFunction() { + local functionList functionName + + functionList=$(declare -F) + # shellcheck disable=SC2206 + functionList=( ${functionList//declare -f /} ) + + for functionName in "${functionList[@]}"; do + if [[ "$functionName" == "$1" ]]; then + return 0 + fi + done + + return 1 +} + + +# Internal: Determine if the tag is a standalone tag based on whitespace +# before and after the tag. +# +# Passes back a string containing two numbers in the format "BEFORE AFTER" +# like "27 10". It indicates the number of bytes remaining in the "before" +# string (27) and the number of bytes to trim in the "after" string (10). +# Useful for string manipulation: +# +# $1 - Variable to set for passing data back +# $2 - Content before the tag +# $3 - Content after the tag +# $4 - true/false: is this the beginning of the content? +# +# Examples +# +# moIsStandalone RESULT "$before" "$after" false || return 0 +# RESULT_ARRAY=( $RESULT ) +# echo "${before:0:${RESULT_ARRAY[0]}}...${after:${RESULT_ARRAY[1]}}" +# +# Returns nothing. +moIsStandalone() { + local afterTrimmed beforeTrimmed char + + moTrimChars beforeTrimmed "$2" false true " " $'\t' + moTrimChars afterTrimmed "$3" true false " " $'\t' + char=$((${#beforeTrimmed} - 1)) + char=${beforeTrimmed:$char} + + # If the content before didn't end in a newline + if [[ "$char" != $'\n' ]] && [[ "$char" != $'\r' ]]; then + # and there was content or this didn't start the file + if [[ -n "$char" ]] || ! $4; then + # then this is not a standalone tag. + return 1 + fi + fi + + char=${afterTrimmed:0:1} + + # If the content after doesn't start with a newline and it is something + if [[ "$char" != $'\n' ]] && [[ "$char" != $'\r' ]] && [[ -n "$char" ]]; then + # then this is not a standalone tag. + return 2 + fi + + if [[ "$char" == $'\r' ]] && [[ "${afterTrimmed:1:1}" == $'\n' ]]; then + char="$char"$'\n' + fi + + local "$1" && moIndirect "$1" "$((${#beforeTrimmed})) $((${#3} + ${#char} - ${#afterTrimmed}))" +} + + +# Internal: Join / implode an array +# +# $1 - Variable name to receive the joined content +# $2 - Joiner +# $3-$* - Elements to join +# +# Returns nothing. +moJoin() { + local joiner part result target + + target=$1 + joiner=$2 + result=$3 + shift 3 + + for part in "$@"; do + result="$result$joiner$part" + done + + local "$target" && moIndirect "$target" "$result" +} + + +# Internal: Read a file into a variable. +# +# $1 - Variable name to receive the file's content +# $2 - Filename to load - if empty, defaults to /dev/stdin +# +# Returns nothing. +moLoadFile() { + local content len + + # The subshell removes any trailing newlines. We forcibly add + # a dot to the content to preserve all newlines. + # As a future optimization, it would be worth considering removing + # cat and replacing this with a read loop. + + content=$(cat -- "${2:-/dev/stdin}" && echo '.') || return 1 + len=$((${#content} - 1)) + content=${content:0:$len} # Remove last dot + + local "$1" && moIndirect "$1" "$content" +} + + +# Internal: Process a chunk of content some number of times. Writes output +# to stdout. +# +# $1 - Content to parse repeatedly +# $2 - Tag prefix (context name) +# $3-@ - Names to insert into the parsed content +# +# Returns nothing. +moLoop() { + local content context contextBase + + content=$1 + contextBase=$2 + shift 2 + + while [[ "${#@}" -gt 0 ]]; do + moFullTagName context "$contextBase" "$1" + moParse "$content" "$context" false + shift + done +} + + +# Internal: Parse a block of text, writing the result to stdout. +# +# $1 - Block of text to change +# $2 - Current name (the variable NAME for what {{.}} means) +# $3 - true when no content before this, false otherwise +# +# Returns nothing. +moParse() { + # Keep naming variables mo* here to not overwrite needed variables + # used in the string replacements + local moArgs moBlock moContent moCurrent moIsBeginning moNextIsBeginning moTag + + moCurrent=$2 + moIsBeginning=$3 + + # Find open tags + moSplit moContent "$1" '{{' '}}' + + while [[ "${#moContent[@]}" -gt 1 ]]; do + moTrimWhitespace moTag "${moContent[1]}" + moNextIsBeginning=false + + case $moTag in + '#'*) + # Loop, if/then, or pass content through function + # Sets context + moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" + moTrimWhitespace moTag "${moTag:1}" + + # Split arguments from the tag name. Arguments are passed to + # functions. + moArgs=$moTag + moTag=${moTag%% *} + moTag=${moTag%%$'\t'*} + moArgs=${moArgs:${#moTag}} + moFindEndTag moBlock "$moContent" "$moTag" + moFullTagName moTag "$moCurrent" "$moTag" + + if moTest "$moTag"; then + # Show / loop / pass through function + if moIsFunction "$moTag"; then + moCallFunction moContent "$moTag" "${moBlock[0]}" "$moArgs" + moParse "$moContent" "$moCurrent" false + moContent="${moBlock[2]}" + elif moIsArray "$moTag"; then + eval "moLoop \"\${moBlock[0]}\" \"$moTag\" \"\${!${moTag}[@]}\"" + else + moParse "${moBlock[0]}" "$moCurrent" true + fi + fi + + moContent="${moBlock[2]}" + ;; + + '>'*) + # Load partial - get name of file relative to cwd + moPartial moContent "${moContent[@]}" "$moIsBeginning" "$moCurrent" + moNextIsBeginning=${moContent[1]} + moContent=${moContent[0]} + ;; + + '/'*) + # Closing tag - If hit in this loop, we simply ignore + # Matching tags are found in moFindEndTag + moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" + ;; + + '^'*) + # Display section if named thing does not exist + moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" + moTrimWhitespace moTag "${moTag:1}" + moFindEndTag moBlock "$moContent" "$moTag" + moFullTagName moTag "$moCurrent" "$moTag" + + if ! moTest "$moTag"; then + moParse "${moBlock[0]}" "$moCurrent" false "$moCurrent" + fi + + moContent="${moBlock[2]}" + ;; + + '!'*) + # Comment - ignore the tag content entirely + # Trim spaces/tabs before the comment + moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" + ;; + + .) + # Current content (environment variable or function) + moStandaloneDenied moContent "${moContent[@]}" + moShow "$moCurrent" "$moCurrent" + ;; + + '=') + # Change delimiters + # Any two non-whitespace sequences separated by whitespace. + # This tag is ignored. + moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" + ;; + + '{'*) + # Unescaped - split on }}} not }} + moStandaloneDenied moContent "${moContent[@]}" + moContent="${moTag:1}"'}}'"$moContent" + moSplit moContent "$moContent" '}}}' + moTrimWhitespace moTag "${moContent[0]}" + moArgs=$moTag + moTag=${moTag%% *} + moTag=${moTag%%$'\t'*} + moArgs=${moArgs:${#moTag}} + moFullTagName moTag "$moCurrent" "$moTag" + moContent=${moContent[1]} + + # Now show the value + # Quote moArgs here, do not quote it later. + moShow "$moTag" "$moCurrent" "$moArgs" + ;; + + '&'*) + # Unescaped + moStandaloneDenied moContent "${moContent[@]}" + moTrimWhitespace moTag "${moTag:1}" + moFullTagName moTag "$moCurrent" "$moTag" + moShow "$moTag" "$moCurrent" + ;; + + *) + # Normal environment variable or function call + moStandaloneDenied moContent "${moContent[@]}" + moArgs=$moTag + moTag=${moTag%% *} + moTag=${moTag%%$'\t'*} + moArgs=${moArgs:${#moTag}} + moFullTagName moTag "$moCurrent" "$moTag" + + # Quote moArgs here, do not quote it later. + moShow "$moTag" "$moCurrent" "$moArgs" + ;; + esac + + moIsBeginning=$moNextIsBeginning + moSplit moContent "$moContent" '{{' '}}' + done + + echo -n "${moContent[0]}" +} + + +# Internal: Process a partial. +# +# Indentation should be applied to the entire partial. +# +# This sends back the "is beginning" flag because the newline after a +# standalone partial is consumed. That newline is very important in the middle +# of content. We send back this flag to reset the processing loop's +# `moIsBeginning` variable, so the software thinks we are back at the +# beginning of a file and standalone processing continues to work. +# +# Prefix all variables. +# +# $1 - Name of destination variable. Element [0] is the content, [1] is the +# true/false flag indicating if we are at the beginning of content. +# $2 - Content before the tag that was not yet written +# $3 - Tag content +# $4 - Content after the tag +# $5 - true/false: is this the beginning of the content? +# $6 - Current context name +# +# Returns nothing. +moPartial() { + # Namespace variables here to prevent conflicts. + local moContent moFilename moIndent moIsBeginning moPartial moStandalone moUnindented + + if moIsStandalone moStandalone "$2" "$4" "$5"; then + # shellcheck disable=SC2206 + moStandalone=( $moStandalone ) + echo -n "${2:0:${moStandalone[0]}}" + moIndent=${2:${moStandalone[0]}} + moContent=${4:${moStandalone[1]}} + moIsBeginning=true + else + moIndent="" + echo -n "$2" + moContent=$4 + moIsBeginning=$5 + fi + + moTrimWhitespace moFilename "${3:1}" + + # Execute in subshell to preserve current cwd and environment + ( + # It would be nice to remove `dirname` and use a function instead, + # but that's difficult when you're only given filenames. + cd "$(dirname -- "$moFilename")" || exit 1 + moUnindented="$( + moLoadFile moPartial "${moFilename##*/}" || exit 1 + moParse "${moPartial}" "$6" true + + # Fix bash handling of subshells and keep trailing whitespace. + # This is removed in moIndentLines. + echo -n "." + )" || exit 1 + moIndentLines moPartial "$moIndent" "$moUnindented" + echo -n "$moPartial" + ) || exit 1 + + # If this is a standalone tag, the trailing newline after the tag is + # removed and the contents of the partial are added, which typically + # contain a newline. We need to send a signal back to the processing + # loop that the moIsBeginning flag needs to be turned on again. + # + # [0] is the content, [1] is that flag. + local "$1" && moIndirectArray "$1" "$moContent" "$moIsBeginning" +} + + +# Internal: Show an environment variable or the output of a function to +# stdout. +# +# Limit/prefix any variables used. +# +# $1 - Name of environment variable or function +# $2 - Current context +# $3 - Arguments string if $1 is a function +# +# Returns nothing. +moShow() { + # Namespace these variables + local moJoined moNameParts moContent + + if moIsFunction "$1"; then + moCallFunction moContent "$1" "" "$3" + moParse "$moContent" "$2" false + return 0 + fi + + moSplit moNameParts "$1" "." + + if [[ -z "${moNameParts[1]-}" ]]; then + if moIsArray "$1"; then + eval moJoin moJoined "," "\${$1[@]}" + echo -n "$moJoined" + else + # shellcheck disable=SC2031 + if moTestVarSet "$1"; then + echo -n "${!1}" + elif [[ -n "${MO_FAIL_ON_UNSET-}" ]]; then + echo "Env variable not set: $1" >&2 + exit 1 + fi + fi + else + # Further subindexes are disallowed + eval "echo -n \"\${${moNameParts[0]}[${moNameParts[1]%%.*}]}\"" + fi +} + + +# Internal: Split a larger string into an array. +# +# $1 - Destination variable +# $2 - String to split +# $3 - Starting delimiter +# $4 - Ending delimiter (optional) +# +# Returns nothing. +moSplit() { + local pos result + + result=( "$2" ) + moFindString pos "${result[0]}" "$3" + + if [[ "$pos" -ne -1 ]]; then + # The first delimiter was found + result[1]=${result[0]:$pos + ${#3}} + result[0]=${result[0]:0:$pos} + + if [[ -n "${4-}" ]]; then + moFindString pos "${result[1]}" "$4" + + if [[ "$pos" -ne -1 ]]; then + # The second delimiter was found + result[2]="${result[1]:$pos + ${#4}}" + result[1]="${result[1]:0:$pos}" + fi + fi + fi + + local "$1" && moIndirectArray "$1" "${result[@]}" +} + + +# Internal: Handle the content for a standalone tag. This means removing +# whitespace (not newlines) before a tag and whitespace and a newline after +# a tag. That is, assuming, that the line is otherwise empty. +# +# $1 - Name of destination "content" variable. +# $2 - Content before the tag that was not yet written +# $3 - Tag content (not used) +# $4 - Content after the tag +# $5 - true/false: is this the beginning of the content? +# +# Returns nothing. +moStandaloneAllowed() { + local bytes + + if moIsStandalone bytes "$2" "$4" "$5"; then + # shellcheck disable=SC2206 + bytes=( $bytes ) + echo -n "${2:0:${bytes[0]}}" + local "$1" && moIndirect "$1" "${4:${bytes[1]}}" + else + echo -n "$2" + local "$1" && moIndirect "$1" "$4" + fi +} + + +# Internal: Handle the content for a tag that is never "standalone". No +# adjustments are made for newlines and whitespace. +# +# $1 - Name of destination "content" variable. +# $2 - Content before the tag that was not yet written +# $3 - Tag content (not used) +# $4 - Content after the tag +# +# Returns nothing. +moStandaloneDenied() { + echo -n "$2" + local "$1" && moIndirect "$1" "$4" +} + + +# Internal: Determines if the named thing is a function or if it is a +# non-empty environment variable. When MO_FALSE_IS_EMPTY is set to a +# non-empty value, then "false" is also treated is an empty value. +# +# Do not use variables without prefixes here if possible as this needs to +# check if any name exists in the environment +# +# $1 - Name of environment variable or function +# $2 - Current value (our context) +# MO_FALSE_IS_EMPTY - When set to a non-empty value, this will say the +# string value "false" is empty. +# +# Returns 0 if the name is not empty, 1 otherwise. When MO_FALSE_IS_EMPTY +# is set, this returns 1 if the name is "false". +moTest() { + # Test for functions + moIsFunction "$1" && return 0 + + if moIsArray "$1"; then + # Arrays must have at least 1 element + eval "[[ \"\${#${1}[@]}\" -gt 0 ]]" && return 0 + else + # If MO_FALSE_IS_EMPTY is set, then return 1 if the value of + # the variable is "false". + # shellcheck disable=SC2031 + [[ -n "${MO_FALSE_IS_EMPTY-}" ]] && [[ "${!1-}" == "false" ]] && return 1 + + # Environment variables must not be empty + [[ -n "${!1-}" ]] && return 0 + fi + + return 1 +} + +# Internal: Determine if a variable is assigned, even if it is assigned an empty +# value. +# +# $1 - Variable name to check. +# +# Returns true (0) if the variable is set, 1 if the variable is unset. +moTestVarSet() { + [[ "${!1-a}" == "${!1-b}" ]] +} + + +# Internal: Trim the leading whitespace only. +# +# $1 - Name of destination variable +# $2 - The string +# $3 - true/false - trim front? +# $4 - true/false - trim end? +# $5-@ - Characters to trim +# +# Returns nothing. +moTrimChars() { + local back current front last target varName + + target=$1 + current=$2 + front=$3 + back=$4 + last="" + shift 4 # Remove target, string, trim front flag, trim end flag + + while [[ "$current" != "$last" ]]; do + last=$current + + for varName in "$@"; do + $front && current="${current/#$varName}" + $back && current="${current/%$varName}" + done + done + + local "$target" && moIndirect "$target" "$current" +} + + +# Internal: Trim leading and trailing whitespace from a string. +# +# $1 - Name of variable to store trimmed string +# $2 - The string +# +# Returns nothing. +moTrimWhitespace() { + local result + + moTrimChars result "$2" true true $'\r' $'\n' $'\t' " " + local "$1" && moIndirect "$1" "$result" +} + + +# Internal: Displays the usage for mo. Pulls this from the file that +# contained the `mo` function. Can only work when the right filename +# comes is the one argument, and that only happens when `mo` is called +# with `$0` set to this file. +# +# $1 - Filename that has the help message +# +# Returns nothing. +moUsage() { + grep '^#/' "${MO_ORIGINAL_COMMAND}" | cut -c 4- + echo "" + echo "MO_VERSION=$MO_VERSION" +} + + +# Save the original command's path for usage later +MO_ORIGINAL_COMMAND="$(cd "${BASH_SOURCE[0]%/*}" || exit 1; pwd)/${BASH_SOURCE[0]##*/}" +MO_VERSION="2.2.0" + +# If sourced, load all functions. +# If executed, perform the actions as expected. +if [[ "$0" == "${BASH_SOURCE[0]}" ]] || [[ -z "${BASH_SOURCE[0]}" ]]; then + mo "$@" +fi diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile index a0abc542..a3c8260d 100644 --- a/docker/nginx/Dockerfile +++ b/docker/nginx/Dockerfile @@ -1,7 +1,19 @@ -FROM openresty/openresty:1.19.3.1-8-bionic +FROM openresty/openresty:1.19.3.2-3-bionic -# RUN apt-get update -qq && apt-get install cron logrotate -qq -RUN luarocks install luasocket +RUN luarocks install lua-resty-http && \ + openssl req -new -newkey rsa:2048 -days 3650 -nodes -x509 \ + -subj '/CN=local-certificate' \ + -keyout /etc/ssl/local-certificate.key \ + -out /etc/ssl/local-certificate.crt -# CMD ["sh", "-c", "service cron start;", "/usr/local/openresty/bin/openresty -g daemon off;"] -CMD ["/usr/local/openresty/bin/openresty", "-g", "daemon off;"] +COPY mo ./ +COPY conf.d /etc/nginx/conf.d +COPY conf.d.templates /etc/nginx/conf.d.templates + +CMD [ "bash", "-c", \ + "./mo < /etc/nginx/conf.d.templates/server.account.conf > /etc/nginx/conf.d/server.account.conf ; \ + ./mo < /etc/nginx/conf.d.templates/server.api.conf > /etc/nginx/conf.d/server.api.conf; \ + ./mo < /etc/nginx/conf.d.templates/server.hns.conf > /etc/nginx/conf.d/server.hns.conf; \ + ./mo < /etc/nginx/conf.d.templates/server.skylink.conf > /etc/nginx/conf.d/server.skylink.conf ; \ + /usr/local/openresty/bin/openresty '-g daemon off;'" \ + ] diff --git a/docker/nginx/Dockerfile.bionic b/docker/nginx/Dockerfile.bionic deleted file mode 100644 index d51e4c8a..00000000 --- a/docker/nginx/Dockerfile.bionic +++ /dev/null @@ -1,185 +0,0 @@ -# Dockerfile - Ubuntu Bionic -# https://github.com/openresty/docker-openresty - -ARG RESTY_IMAGE_BASE="ubuntu" -ARG RESTY_IMAGE_TAG="bionic" - -FROM ${RESTY_IMAGE_BASE}:${RESTY_IMAGE_TAG} - -LABEL maintainer="Evan Wies " - -# Docker Build Arguments -ARG RESTY_IMAGE_BASE="ubuntu" -ARG RESTY_IMAGE_TAG="bionic" -ARG RESTY_VERSION="1.19.3.1" -ARG RESTY_LUAROCKS_VERSION="3.5.0" -ARG RESTY_OPENSSL_VERSION="1.1.1i" -ARG RESTY_OPENSSL_PATCH_VERSION="1.1.1f" -ARG RESTY_OPENSSL_URL_BASE="https://www.openssl.org/source" -ARG RESTY_PCRE_VERSION="8.44" -ARG RESTY_J="1" -ARG RESTY_CONFIG_OPTIONS="\ - --with-compat \ - --with-file-aio \ - --with-http_addition_module \ - --with-http_auth_request_module \ - --with-http_dav_module \ - --with-http_flv_module \ - --with-http_geoip_module=dynamic \ - --with-http_gunzip_module \ - --with-http_gzip_static_module \ - --with-http_image_filter_module=dynamic \ - --with-http_mp4_module \ - --with-http_random_index_module \ - --with-http_realip_module \ - --with-http_secure_link_module \ - --with-http_slice_module \ - --with-http_ssl_module \ - --with-http_stub_status_module \ - --with-http_sub_module \ - --with-http_v2_module \ - --with-http_xslt_module=dynamic \ - --with-ipv6 \ - --with-mail \ - --with-mail_ssl_module \ - --with-md5-asm \ - --with-pcre-jit \ - --with-sha1-asm \ - --with-stream \ - --with-stream_ssl_module \ - --with-threads \ - " -ARG RESTY_CONFIG_OPTIONS_MORE="" -ARG RESTY_LUAJIT_OPTIONS="--with-luajit-xcflags='-DLUAJIT_NUMMODE=2 -DLUAJIT_ENABLE_LUA52COMPAT'" - -ARG RESTY_ADD_PACKAGE_BUILDDEPS="" -ARG RESTY_ADD_PACKAGE_RUNDEPS="" -ARG RESTY_EVAL_PRE_CONFIGURE="" -ARG RESTY_EVAL_POST_MAKE="" - -# These are not intended to be user-specified -ARG _RESTY_CONFIG_DEPS="--with-pcre \ - --with-cc-opt='-DNGX_LUA_ABORT_AT_PANIC -I/usr/local/openresty/pcre/include -I/usr/local/openresty/openssl/include' \ - --with-ld-opt='-L/usr/local/openresty/pcre/lib -L/usr/local/openresty/openssl/lib -Wl,-rpath,/usr/local/openresty/pcre/lib:/usr/local/openresty/openssl/lib' \ - " - -LABEL resty_image_base="${RESTY_IMAGE_BASE}" -LABEL resty_image_tag="${RESTY_IMAGE_TAG}" -LABEL resty_version="${RESTY_VERSION}" -LABEL resty_luarocks_version="${RESTY_LUAROCKS_VERSION}" -LABEL resty_openssl_version="${RESTY_OPENSSL_VERSION}" -LABEL resty_openssl_patch_version="${RESTY_OPENSSL_PATCH_VERSION}" -LABEL resty_openssl_url_base="${RESTY_OPENSSL_URL_BASE}" -LABEL resty_pcre_version="${RESTY_PCRE_VERSION}" -LABEL resty_config_options="${RESTY_CONFIG_OPTIONS}" -LABEL resty_config_options_more="${RESTY_CONFIG_OPTIONS_MORE}" -LABEL resty_config_deps="${_RESTY_CONFIG_DEPS}" -LABEL resty_add_package_builddeps="${RESTY_ADD_PACKAGE_BUILDDEPS}" -LABEL resty_add_package_rundeps="${RESTY_ADD_PACKAGE_RUNDEPS}" -LABEL resty_eval_pre_configure="${RESTY_EVAL_PRE_CONFIGURE}" -LABEL resty_eval_post_make="${RESTY_EVAL_POST_MAKE}" - - -RUN DEBIAN_FRONTEND=noninteractive apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - build-essential \ - ca-certificates \ - curl \ - gettext-base \ - libgd-dev \ - libgeoip-dev \ - libncurses5-dev \ - libperl-dev \ - libreadline-dev \ - libxslt1-dev \ - make \ - perl \ - unzip \ - zlib1g-dev \ - ${RESTY_ADD_PACKAGE_BUILDDEPS} \ - ${RESTY_ADD_PACKAGE_RUNDEPS} \ - && cd /tmp \ - && if [ -n "${RESTY_EVAL_PRE_CONFIGURE}" ]; then eval $(echo ${RESTY_EVAL_PRE_CONFIGURE}); fi \ - && curl -fSL "${RESTY_OPENSSL_URL_BASE}/openssl-${RESTY_OPENSSL_VERSION}.tar.gz" -o openssl-${RESTY_OPENSSL_VERSION}.tar.gz \ - && tar xzf openssl-${RESTY_OPENSSL_VERSION}.tar.gz \ - && cd openssl-${RESTY_OPENSSL_VERSION} \ - && if [ $(echo ${RESTY_OPENSSL_VERSION} | cut -c 1-5) = "1.1.1" ] ; then \ - echo 'patching OpenSSL 1.1.1 for OpenResty' \ - && curl -s https://raw.githubusercontent.com/openresty/openresty/master/patches/openssl-${RESTY_OPENSSL_PATCH_VERSION}-sess_set_get_cb_yield.patch | patch -p1 ; \ - fi \ - && if [ $(echo ${RESTY_OPENSSL_VERSION} | cut -c 1-5) = "1.1.0" ] ; then \ - echo 'patching OpenSSL 1.1.0 for OpenResty' \ - && curl -s https://raw.githubusercontent.com/openresty/openresty/ed328977028c3ec3033bc25873ee360056e247cd/patches/openssl-1.1.0j-parallel_build_fix.patch | patch -p1 \ - && curl -s https://raw.githubusercontent.com/openresty/openresty/master/patches/openssl-${RESTY_OPENSSL_PATCH_VERSION}-sess_set_get_cb_yield.patch | patch -p1 ; \ - fi \ - && ./config \ - no-threads shared zlib -g \ - enable-ssl3 enable-ssl3-method \ - --prefix=/usr/local/openresty/openssl \ - --libdir=lib \ - -Wl,-rpath,/usr/local/openresty/openssl/lib \ - && make -j${RESTY_J} \ - && make -j${RESTY_J} install_sw \ - && cd /tmp \ - && curl -fSL https://ftp.pcre.org/pub/pcre/pcre-${RESTY_PCRE_VERSION}.tar.gz -o pcre-${RESTY_PCRE_VERSION}.tar.gz \ - && tar xzf pcre-${RESTY_PCRE_VERSION}.tar.gz \ - && cd /tmp/pcre-${RESTY_PCRE_VERSION} \ - && ./configure \ - --prefix=/usr/local/openresty/pcre \ - --disable-cpp \ - --enable-jit \ - --enable-utf \ - --enable-unicode-properties \ - && make -j${RESTY_J} \ - && make -j${RESTY_J} install \ - && cd /tmp \ - && curl -fSL https://openresty.org/download/openresty-${RESTY_VERSION}.tar.gz -o openresty-${RESTY_VERSION}.tar.gz \ - && tar xzf openresty-${RESTY_VERSION}.tar.gz \ - && cd /tmp/openresty-${RESTY_VERSION} \ - && eval ./configure -j${RESTY_J} ${_RESTY_CONFIG_DEPS} ${RESTY_CONFIG_OPTIONS} ${RESTY_CONFIG_OPTIONS_MORE} ${RESTY_LUAJIT_OPTIONS} \ - && make -j${RESTY_J} \ - && make -j${RESTY_J} install \ - && cd /tmp \ - && rm -rf \ - openssl-${RESTY_OPENSSL_VERSION}.tar.gz openssl-${RESTY_OPENSSL_VERSION} \ - pcre-${RESTY_PCRE_VERSION}.tar.gz pcre-${RESTY_PCRE_VERSION} \ - openresty-${RESTY_VERSION}.tar.gz openresty-${RESTY_VERSION} \ - && curl -fSL https://luarocks.github.io/luarocks/releases/luarocks-${RESTY_LUAROCKS_VERSION}.tar.gz -o luarocks-${RESTY_LUAROCKS_VERSION}.tar.gz \ - && tar xzf luarocks-${RESTY_LUAROCKS_VERSION}.tar.gz \ - && cd luarocks-${RESTY_LUAROCKS_VERSION} \ - && ./configure \ - --prefix=/usr/local/openresty/luajit \ - --with-lua=/usr/local/openresty/luajit \ - --lua-suffix=jit-2.1.0-beta3 \ - --with-lua-include=/usr/local/openresty/luajit/include/luajit-2.1 \ - && make build \ - && make install \ - && cd /tmp \ - && if [ -n "${RESTY_EVAL_POST_MAKE}" ]; then eval $(echo ${RESTY_EVAL_POST_MAKE}); fi \ - && rm -rf luarocks-${RESTY_LUAROCKS_VERSION} luarocks-${RESTY_LUAROCKS_VERSION}.tar.gz \ - && if [ -n "${RESTY_ADD_PACKAGE_BUILDDEPS}" ]; then DEBIAN_FRONTEND=noninteractive apt-get remove -y --purge ${RESTY_ADD_PACKAGE_BUILDDEPS} ; fi \ - && DEBIAN_FRONTEND=noninteractive apt-get autoremove -y \ - && mkdir -p /var/run/openresty \ - && ln -sf /dev/stdout /usr/local/openresty/nginx/logs/access.log \ - && ln -sf /dev/stderr /usr/local/openresty/nginx/logs/error.log - -# Add additional binaries into PATH for convenience -ENV PATH=$PATH:/usr/local/openresty/luajit/bin:/usr/local/openresty/nginx/sbin:/usr/local/openresty/bin - -# Add LuaRocks paths -# If OpenResty changes, these may need updating: -# /usr/local/openresty/bin/resty -e 'print(package.path)' -# /usr/local/openresty/bin/resty -e 'print(package.cpath)' -ENV LUA_PATH="/usr/local/openresty/site/lualib/?.ljbc;/usr/local/openresty/site/lualib/?/init.ljbc;/usr/local/openresty/lualib/?.ljbc;/usr/local/openresty/lualib/?/init.ljbc;/usr/local/openresty/site/lualib/?.lua;/usr/local/openresty/site/lualib/?/init.lua;/usr/local/openresty/lualib/?.lua;/usr/local/openresty/lualib/?/init.lua;./?.lua;/usr/local/openresty/luajit/share/luajit-2.1.0-beta3/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/share/lua/5.1/?/init.lua;/usr/local/openresty/luajit/share/lua/5.1/?.lua;/usr/local/openresty/luajit/share/lua/5.1/?/init.lua" - -ENV LUA_CPATH="/usr/local/openresty/site/lualib/?.so;/usr/local/openresty/lualib/?.so;./?.so;/usr/local/lib/lua/5.1/?.so;/usr/local/openresty/luajit/lib/lua/5.1/?.so;/usr/local/lib/lua/5.1/loadall.so;/usr/local/openresty/luajit/lib/lua/5.1/?.so" - -# Copy nginx configuration files -COPY nginx.conf /usr/local/openresty/nginx/conf/nginx.conf -COPY nginx.vh.default.conf /etc/nginx/conf.d/default.conf - -CMD ["/usr/local/openresty/bin/openresty", "-g", "daemon off;"] - -# Use SIGQUIT instead of default SIGTERM to cleanly drain requests -# See https://github.com/openresty/docker-openresty/blob/master/README.md#tips--pitfalls -STOPSIGNAL SIGQUIT \ No newline at end of file diff --git a/docker/nginx/conf.d.templates/server.account.conf b/docker/nginx/conf.d.templates/server.account.conf new file mode 100644 index 00000000..0de88c03 --- /dev/null +++ b/docker/nginx/conf.d.templates/server.account.conf @@ -0,0 +1,39 @@ +{{#ACCOUNTS_ENABLED}} + {{#PORTAL_DOMAIN}} + server { + server_name account.{{PORTAL_DOMAIN}}; # example: account.siasky.net + + include /etc/nginx/conf.d/server/server.http; + } + + server { + server_name account.{{PORTAL_DOMAIN}}; # example: account.siasky.net + + ssl_certificate /data/caddy/caddy/certificates/acme-v02.api.letsencrypt.org-directory/wildcard_.{{PORTAL_DOMAIN}}/wildcard_.{{PORTAL_DOMAIN}}.crt; + ssl_certificate_key /data/caddy/caddy/certificates/acme-v02.api.letsencrypt.org-directory/wildcard_.{{PORTAL_DOMAIN}}/wildcard_.{{PORTAL_DOMAIN}}.key; + + include /etc/nginx/conf.d/server/server.account; + } + {{/PORTAL_DOMAIN}} + + {{#SERVER_DOMAIN}} + server { + server_name account.{{SERVER_DOMAIN}}; # example: account.eu-ger-1.siasky.net + + include /etc/nginx/conf.d/server/server.http; + + set_by_lua_block $server_alias { return string.match("{{SERVER_DOMAIN}}", "^([^.]+)") } + } + + server { + server_name account.{{SERVER_DOMAIN}}; # example: account.eu-ger-1.siasky.net + + ssl_certificate /data/caddy/caddy/certificates/acme-v02.api.letsencrypt.org-directory/wildcard_.{{SERVER_DOMAIN}}/wildcard_.{{SERVER_DOMAIN}}.crt; + ssl_certificate_key /data/caddy/caddy/certificates/acme-v02.api.letsencrypt.org-directory/wildcard_.{{SERVER_DOMAIN}}/wildcard_.{{SERVER_DOMAIN}}.key; + + include /etc/nginx/conf.d/server/server.account; + + set_by_lua_block $server_alias { return string.match("{{SERVER_DOMAIN}}", "^([^.]+)") } + } + {{/SERVER_DOMAIN}} +{{/ACCOUNTS_ENABLED}} diff --git a/docker/nginx/conf.d.templates/server.api.conf b/docker/nginx/conf.d.templates/server.api.conf new file mode 100644 index 00000000..1adb58c1 --- /dev/null +++ b/docker/nginx/conf.d.templates/server.api.conf @@ -0,0 +1,37 @@ +{{#PORTAL_DOMAIN}} +server { + server_name {{PORTAL_DOMAIN}}; # example: siasky.net + + include /etc/nginx/conf.d/server/server.http; +} + +server { + server_name {{PORTAL_DOMAIN}}; # example: siasky.net + + ssl_certificate /data/caddy/caddy/certificates/acme-v02.api.letsencrypt.org-directory/{{PORTAL_DOMAIN}}/{{PORTAL_DOMAIN}}.crt; + ssl_certificate_key /data/caddy/caddy/certificates/acme-v02.api.letsencrypt.org-directory/{{PORTAL_DOMAIN}}/{{PORTAL_DOMAIN}}.key; + + include /etc/nginx/conf.d/server/server.api; +} +{{/PORTAL_DOMAIN}} + +{{#SERVER_DOMAIN}} +server { + server_name {{SERVER_DOMAIN}}; # example: eu-ger-1.siasky.net + + include /etc/nginx/conf.d/server/server.http; + + set_by_lua_block $server_alias { return string.match("{{SERVER_DOMAIN}}", "^([^.]+)") } +} + +server { + server_name {{SERVER_DOMAIN}}; # example: eu-ger-1.siasky.net + + ssl_certificate /data/caddy/caddy/certificates/acme-v02.api.letsencrypt.org-directory/{{SERVER_DOMAIN}}/{{SERVER_DOMAIN}}.crt; + ssl_certificate_key /data/caddy/caddy/certificates/acme-v02.api.letsencrypt.org-directory/{{SERVER_DOMAIN}}/{{SERVER_DOMAIN}}.key; + + include /etc/nginx/conf.d/server/server.api; + + set_by_lua_block $server_alias { return string.match("{{SERVER_DOMAIN}}", "^([^.]+)") } +} +{{/SERVER_DOMAIN}} diff --git a/docker/nginx/conf.d.templates/server.hns.conf b/docker/nginx/conf.d.templates/server.hns.conf new file mode 100644 index 00000000..a40427f1 --- /dev/null +++ b/docker/nginx/conf.d.templates/server.hns.conf @@ -0,0 +1,39 @@ +{{#PORTAL_DOMAIN}} +server { + server_name *.hns.{{PORTAL_DOMAIN}}; # example: *.hns.siasky.net + + include /etc/nginx/conf.d/server/server.http; +} + +server { + server_name *.hns.{{PORTAL_DOMAIN}}; # example: *.hns.siasky.net + + ssl_certificate /data/caddy/caddy/certificates/acme-v02.api.letsencrypt.org-directory/wildcard_.hns.{{PORTAL_DOMAIN}}/wildcard_.hns.{{PORTAL_DOMAIN}}.crt; + ssl_certificate_key /data/caddy/caddy/certificates/acme-v02.api.letsencrypt.org-directory/wildcard_.hns.{{PORTAL_DOMAIN}}/wildcard_.hns.{{PORTAL_DOMAIN}}.key; + + proxy_set_header Host {{PORTAL_DOMAIN}}; + include /etc/nginx/conf.d/server/server.hns; +} +{{/PORTAL_DOMAIN}} + +{{#SERVER_DOMAIN}} +server { + server_name *.hns.{{SERVER_DOMAIN}}; # example: *.hns.eu-ger-1.siasky.net + + include /etc/nginx/conf.d/server/server.http; + + set_by_lua_block $server_alias { return string.match("{{SERVER_DOMAIN}}", "^([^.]+)") } +} + +server { + server_name *.hns.{{SERVER_DOMAIN}}; # example: *.hns.eu-ger-1.siasky.net + + ssl_certificate /data/caddy/caddy/certificates/acme-v02.api.letsencrypt.org-directory/wildcard_.hns.{{SERVER_DOMAIN}}/wildcard_.hns.{{SERVER_DOMAIN}}.crt; + ssl_certificate_key /data/caddy/caddy/certificates/acme-v02.api.letsencrypt.org-directory/wildcard_.hns.{{SERVER_DOMAIN}}/wildcard_.hns.{{SERVER_DOMAIN}}.key; + + proxy_set_header Host {{SERVER_DOMAIN}}; + include /etc/nginx/conf.d/server/server.hns; + + set_by_lua_block $server_alias { return string.match("{{SERVER_DOMAIN}}", "^([^.]+)") } +} +{{/SERVER_DOMAIN}} diff --git a/docker/nginx/conf.d.templates/server.skylink.conf b/docker/nginx/conf.d.templates/server.skylink.conf new file mode 100644 index 00000000..0cfc2027 --- /dev/null +++ b/docker/nginx/conf.d.templates/server.skylink.conf @@ -0,0 +1,37 @@ +{{#PORTAL_DOMAIN}} +server { + server_name *.{{PORTAL_DOMAIN}}; # example: *.siasky.net + + include /etc/nginx/conf.d/server/server.http; +} + +server { + server_name *.{{PORTAL_DOMAIN}}; # example: *.siasky.net + + ssl_certificate /data/caddy/caddy/certificates/acme-v02.api.letsencrypt.org-directory/wildcard_.{{PORTAL_DOMAIN}}/wildcard_.{{PORTAL_DOMAIN}}.crt; + ssl_certificate_key /data/caddy/caddy/certificates/acme-v02.api.letsencrypt.org-directory/wildcard_.{{PORTAL_DOMAIN}}/wildcard_.{{PORTAL_DOMAIN}}.key; + + include /etc/nginx/conf.d/server/server.skylink; +} +{{/PORTAL_DOMAIN}} + +{{#SERVER_DOMAIN}} +server { + server_name *.{{SERVER_DOMAIN}}; # example: *.eu-ger-1.siasky.net + + include /etc/nginx/conf.d/server/server.http; + + set_by_lua_block $server_alias { return string.match("{{SERVER_DOMAIN}}", "^([^.]+)") } +} + +server { + server_name *.{{SERVER_DOMAIN}}; # example: *.eu-ger-1.siasky.net + + ssl_certificate /data/caddy/caddy/certificates/acme-v02.api.letsencrypt.org-directory/wildcard_.{{SERVER_DOMAIN}}/wildcard_.{{SERVER_DOMAIN}}.crt; + ssl_certificate_key /data/caddy/caddy/certificates/acme-v02.api.letsencrypt.org-directory/wildcard_.{{SERVER_DOMAIN}}/wildcard_.{{SERVER_DOMAIN}}.key; + + include /etc/nginx/conf.d/server/server.skylink; + + set_by_lua_block $server_alias { return string.match("{{SERVER_DOMAIN}}", "^([^.]+)") } +} +{{/SERVER_DOMAIN}} diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf deleted file mode 100644 index ce0bd4e0..00000000 --- a/docker/nginx/conf.d/client.conf +++ /dev/null @@ -1,578 +0,0 @@ -proxy_cache_path /data/nginx/cache levels=1:2 keys_zone=skynet:10m max_size=50g inactive=48h use_temp_path=off; - -# this runs before forking out nginx worker processes -init_by_lua_block { - require "cjson" - require "socket.http" -} - -# ratelimit specified IPs -geo $limit { - default 0; - include /etc/nginx/conf.d/include/ratelimited; -} -map $limit $limit_key { - 0 ""; - 1 $binary_remote_addr; -} - -limit_req_zone $binary_remote_addr zone=uploads_by_ip:10m rate=10r/s; -limit_req_zone $limit_key zone=uploads_by_ip_throttled:10m rate=10r/m; - -limit_req_zone $binary_remote_addr zone=registry_access_by_ip:10m rate=60r/m; -limit_req_zone $limit_key zone=registry_access_by_ip_throttled:10m rate=20r/m; - -limit_conn_zone $binary_remote_addr zone=upload_conn:10m; -limit_conn_zone $limit_key zone=upload_conn_rl:10m; - -limit_conn_zone $binary_remote_addr zone=downloads_by_ip:10m; - -limit_req_status 429; -limit_conn_status 429; - -# since we are proxying request to nginx from caddy, access logs will contain caddy's ip address -# as the request address so we need to use real_ip_header module to use ip address from -# X-Forwarded-For header as a real ip address of the request -set_real_ip_from 10.0.0.0/8; -set_real_ip_from 127.0.0.1/32; -set_real_ip_from 172.16.0.0/12; -set_real_ip_from 192.168.0.0/16; -real_ip_header X-Forwarded-For; - -# skynet-jwt contains dash so we cannot use $cookie_skynet-jwt -# https://richardhart.me/2012/03/18/logging-nginx-cookies-with-dashes/ -map $http_cookie $skynet_jwt { - default ''; - ~skynet-jwt=(?[^\;]+) $match; -} - -upstream siad { - server sia:9980; -} - -server { - listen 80 default_server; - listen [::]:80 default_server; - - # understand the regex https://regex101.com/r/BGQvi6/6 - server_name "~^(((?([a-z0-9]{55}))|(?[^\.]+)\.hns)\.)?((?[^.]+)\.)?(?[^.]+)\.(?[^.]+)$"; - - # ddos protection: closing slow connections - client_body_timeout 5s; - client_header_timeout 5s; - - # Increase the body buffer size, to ensure the internal POSTs can always - # parse the full POST contents into memory. - client_body_buffer_size 128k; - client_max_body_size 128k; - - # legacy endpoint rewrite - rewrite ^/portals /skynet/portals permanent; - rewrite ^/stats /skynet/stats permanent; - rewrite ^/skynet/blacklist /skynet/blocklist permanent; - rewrite ^/account/(.*) https://account.$domain.$tld/$1 permanent; - - # This is only safe workaround to reroute based on some conditions - # See https://www.nginx.com/resources/wiki/start/topics/depth/ifisevil/ - recursive_error_pages on; - - # redirect links with base32 encoded skylink in subdomain - error_page 460 = @base32_subdomain; - if ($base32_subdomain != "") { - return 460; - } - - # redirect links with handshake domain on hns subdomain - error_page 461 = @hns_domain; - if ($hns_domain != "") { - return 461; - } - - # redirect to dnslookup endpoint - error_page 462 = @dnslink_lookup; - if ($http_dnslink_lookup) { - return 462; - } - - location / { - include /etc/nginx/conf.d/include/cors; - - proxy_pass http://website:9000; - } - - location /docs { - proxy_pass https://skynetlabs.github.io/skynet-docs; - } - - location /skynet/blocklist { - include /etc/nginx/conf.d/include/cors; - - proxy_cache skynet; - proxy_cache_valid any 1m; # cache blocklist for 1 minute - proxy_set_header User-Agent: Sia-Agent; - proxy_pass http://siad/skynet/blocklist; - } - - location /skynet/portals { - include /etc/nginx/conf.d/include/cors; - - proxy_cache skynet; - proxy_cache_valid any 1m; # cache portals for 1 minute - proxy_set_header User-Agent: Sia-Agent; - proxy_pass http://siad/skynet/portals; - } - - location /skynet/stats { - include /etc/nginx/conf.d/include/cors; - - proxy_cache skynet; - proxy_cache_valid any 1m; # cache stats for 1 minute - proxy_set_header User-Agent: Sia-Agent; - proxy_read_timeout 5m; # extend the read timeout - proxy_pass http://siad/skynet/stats; - } - - location /health-check { - include /etc/nginx/conf.d/include/cors; - - access_log off; # do not log traffic to health-check endpoint - - proxy_pass http://10.10.10.60:3100; # hardcoded ip because health-check waits for nginx - } - - location /hns { - include /etc/nginx/conf.d/include/proxy-buffer; - include /etc/nginx/conf.d/include/proxy-pass-internal; - - # variable definititions - we need to define a variable to be able to access it in lua by ngx.var.something - set $skylink ''; # placeholder for the raw 46 bit skylink - set $rest ''; # placeholder for the rest of the url that gets appended to skylink (path and args) - - # resolve handshake domain by requesting to /hnsres endpoint and assign correct values to $skylink and $rest - access_by_lua_block { - local json = require('cjson') - - -- match the request_uri and extract the hns domain and anything that is passed in the uri after it - -- example: /hns/something/foo/bar?baz=1 matches: - -- > hns_domain_name: something - -- > request_uri_rest: /foo/bar/?baz=1 - local hns_domain_name, request_uri_rest = string.match(ngx.var.request_uri, "/hns/([^/?]+)(.*)") - - -- make a get request to /hnsres endpoint with the domain name from request_uri - local hnsres_res = ngx.location.capture("/hnsres/" .. hns_domain_name) - - -- we want to fail with a generic 404 when /hnsres returns anything but 200 OK with a skylink - if hnsres_res.status ~= ngx.HTTP_OK then - ngx.exit(ngx.HTTP_NOT_FOUND) - end - - -- since /hnsres endpoint response is a json, we need to decode it before we access it - -- example response: '{"skylink":"sia://XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg"}' - local hnsres_json = json.decode(hnsres_res.body) - - -- define local variable containing rest of the skylink if provided - local skylink_rest - - if hnsres_json.skylink then - -- try to match the skylink with sia:// prefix - skylink, skylink_rest = string.match(hnsres_json.skylink, "sia://([^/?]+)(.*)") - - -- in case the skylink did not match, assume that there is no sia:// prefix and try to match again - if skylink == nil then - skylink, skylink_rest = string.match(hnsres_json.skylink, "/?([^/?]+)(.*)") - end - elseif hnsres_json.registry then - local publickey = hnsres_json.registry.publickey - local datakey = hnsres_json.registry.datakey - - -- make a get request to /skynet/registry endpoint with the credentials from text record - local registry_res = ngx.location.capture("/skynet/registry/cached?publickey=" .. publickey .. "&datakey=" .. datakey) - - -- we want to fail with a generic 404 when /skynet/registry returns anything but 200 OK - if registry_res.status ~= ngx.HTTP_OK then - ngx.exit(ngx.HTTP_NOT_FOUND) - end - - -- since /skynet/registry endpoint response is a json, we need to decode it before we access it - local registry_json = json.decode(registry_res.body) - -- response will contain a hex encoded skylink, we need to decode it - local data = (registry_json.data:gsub('..', function (cc) - return string.char(tonumber(cc, 16)) - end)) - - skylink = data - end - - -- fail with a generic 404 if skylink has not been extracted from a valid /hnsres response for some reason - if not skylink then - ngx.exit(ngx.HTTP_NOT_FOUND) - end - - ngx.var.skylink = skylink - if request_uri_rest == "/" and skylink_rest ~= nil and skylink_rest ~= "" and skylink_rest ~= "/" then - ngx.var.rest = skylink_rest - else - ngx.var.rest = request_uri_rest - end - } - - # we proxy to another nginx location rather than directly to siad because we don't want to deal with caching here - proxy_pass http://127.0.0.1/$skylink$rest; - - # in case siad returns location header, we need to replace the skylink with the domain name - header_filter_by_lua_block { - ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") - ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") - - if ngx.header.location then - -- match hns domain from the request_uri - local hns_domain_name = string.match(ngx.var.request_uri, "/hns/([^/?]+)") - - -- match location redirect part after the skylink - local location_rest = string.match(ngx.header.location, "[^/?]+(.*)"); - - -- because siad will set the location header to ie. XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg/index.html - -- we need to replace the skylink with the domain_name so we are not redirected to skylink - ngx.header.location = hns_domain_name .. location_rest - end - } - } - - location /hnsres { - include /etc/nginx/conf.d/include/cors; - - proxy_pass http://handshake-api:3100; - } - - # internal registry endpoint that caches calls for a certain period of time - # it is not suitable for every registry call but some requests might be cached - # and we are using it currently for caching registry resolutions from /hns calls - location /skynet/registry/cached { - include /etc/nginx/conf.d/include/location-skynet-registry; - - internal; # internal endpoint only - - proxy_cache skynet; - proxy_cache_key $args; # cache based on publickey and datakey args - proxy_cache_valid 200 30s; # cache only 200 responses and only for 30 seconds - proxy_cache_lock on; # queue cache requests for the same resource until it is fully cached - proxy_cache_bypass $cookie_nocache $arg_nocache; # add cache bypass option - } - - location /skynet/registry { - include /etc/nginx/conf.d/include/location-skynet-registry; - } - - location /skynet/skyfile { - include /etc/nginx/conf.d/include/cors; - include /etc/nginx/conf.d/include/sia-auth; - include /etc/nginx/conf.d/include/track-upload; - include /etc/nginx/conf.d/include/generate-siapath; - - limit_req zone=uploads_by_ip burst=100 nodelay; - limit_req zone=uploads_by_ip_throttled; - - limit_conn upload_conn 10; - limit_conn upload_conn_rl 1; - - client_max_body_size 1000M; # make sure to limit the size of upload to a sane value - - # increase request timeouts - proxy_read_timeout 600; - proxy_send_timeout 600; - - proxy_request_buffering off; # stream uploaded files through the proxy as it comes in - proxy_set_header Expect $http_expect; - proxy_set_header User-Agent: Sia-Agent; - - # access_by_lua_block { - # -- this block runs only when accounts are enabled - # if os.getenv("ACCOUNTS_ENABLED") ~= "true" then return end - - # ngx.var.upload_limit_rate = 5 * 1024 * 1024 - # local res = ngx.location.capture("/accounts/user", { copy_all_vars = true }) - # if res.status == ngx.HTTP_OK then - # local json = require('cjson') - # local user = json.decode(res.body) - # ngx.var.upload_limit_rate = ngx.var.upload_limit_rate * (user.tier + 1) - # end - # } - - # proxy this call to siad endpoint (make sure the ip is correct) - proxy_pass http://siad/skynet/skyfile/$dir1/$dir2/$dir3$is_args$args; - } - - # endpoing implementing resumable file uploads open protocol https://tus.io - location /skynet/tus { - include /etc/nginx/conf.d/include/cors; - include /etc/nginx/conf.d/include/track-upload; - - # TUS chunks size is 40M + leaving 10M of breathing room - client_max_body_size 50M; - - # Those timeouts need to be elevated since skyd can stall reading - # data for a while when overloaded which would terminate connection - client_body_timeout 1h; - proxy_send_timeout 1h; - - # Add X-Forwarded-* headers - proxy_set_header X-Forwarded-Host $host; - proxy_set_header X-Forwarded-Proto $scheme; - - # rewrite proxy request to use correct host uri from env variable (required to return correct location header) - set_by_lua $SKYNET_SERVER_API 'return os.getenv("SKYNET_SERVER_API")'; - proxy_redirect $scheme://$host $SKYNET_SERVER_API; - - # proxy /skynet/tus requests to siad endpoint with all arguments - proxy_pass http://siad; - - # set max upload size dynamically based on account limits - rewrite_by_lua_block { - -- set default limit value to 1 GB - ngx.req.set_header("SkynetMaxUploadSize", 1073741824) - - -- this block runs only when accounts are enabled - if os.getenv("ACCOUNTS_ENABLED") ~= "true" then return end - - -- fetch account limits and set max upload size accordingly - local res = ngx.location.capture("/accounts/user/limits", { copy_all_vars = true }) - if res.status == ngx.HTTP_OK then - local json = require('cjson') - local limits = json.decode(res.body) - ngx.req.set_header("SkynetMaxUploadSize", limits.maxUploadSize) - end - } - - # extract skylink from base64 encoded upload metadata and assign to a proper header - header_filter_by_lua_block { - ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") - ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") - - if ngx.header["Upload-Metadata"] then - local encodedSkylink = string.match(ngx.header["Upload-Metadata"], "Skylink ([^,?]+)") - - if encodedSkylink then - ngx.header["Skynet-Skylink"] = ngx.decode_base64(encodedSkylink) - end - end - } - } - - location /skynet/pin { - include /etc/nginx/conf.d/include/cors; - include /etc/nginx/conf.d/include/sia-auth; - include /etc/nginx/conf.d/include/track-upload; - include /etc/nginx/conf.d/include/generate-siapath; - - proxy_set_header User-Agent: Sia-Agent; - proxy_pass http://siad$uri?siapath=$dir1/$dir2/$dir3&$args; - } - - location /skynet/metadata { - include /etc/nginx/conf.d/include/cors; - - proxy_set_header User-Agent: Sia-Agent; - proxy_pass http://siad; - } - - location /skynet/health { - include /etc/nginx/conf.d/include/cors; - - proxy_cache skynet; - proxy_cache_key $request_uri; # use whole request uri (uri + args) as cache key - proxy_cache_valid any 1m; # cache any response for 1 minute - proxy_set_header User-Agent: Sia-Agent; - proxy_read_timeout 5m; # extend the read timeout to 5 minutes - proxy_pass http://siad; - } - - location /skynet/resolve { - include /etc/nginx/conf.d/include/cors; - - proxy_set_header User-Agent: Sia-Agent; - proxy_pass http://siad; - } - - location ~ "^/(([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?)$" { - include /etc/nginx/conf.d/include/cors; - include /etc/nginx/conf.d/include/proxy-buffer; - include /etc/nginx/conf.d/include/proxy-cache-downloads; - include /etc/nginx/conf.d/include/track-download; - - # redirect purge calls to separate location - error_page 462 = @purge; - if ($request_method = PURGE) { - return 462; - } - - limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time - - # we need to explicitly use set directive here because $2 and $3 will contain values with - # decoded whitespaces and set will re-encode it for us before passing it to proxy_pass - set $skylink $2; - set $path $3; - - # $skylink_v1 and $skylink_v2 variables default to the same value but in case the requested skylink was: - # a) skylink v1 - it wouldn't matter, no additional logic is executed - # b) skylink v2 - in a lua block below we will resolve the skylink v2 into skylink v1 and update - # $skylink_v1 variable so then the proxy request to skyd can be cached in nginx (proxy_cache_key - # in proxy-cache-downloads includes $skylink_v1 as a part of the cache key) - set $skylink_v1 $skylink; - set $skylink_v2 $skylink; - - # variable for Skynet-Proof header that we need to inject - # into a response if the request was for skylink v2 - set $skynet_proof ''; - - access_by_lua_block { - -- detect whether requested skylink is v2 - local isBase32v2 = string.len(ngx.var.skylink) == 55 and string.sub(ngx.var.skylink, 0, 2) == "04" - local isBase64v2 = string.len(ngx.var.skylink) == 46 and string.sub(ngx.var.skylink, 0, 2) == "AQ" - - if isBase32v2 or isBase64v2 then - local res = ngx.location.capture("/skynet/resolve/" .. ngx.var.skylink_v2) - if res.status == ngx.HTTP_OK then - local json = require('cjson') - local resolve = json.decode(res.body) - ngx.var.skylink_v1 = resolve.skylink - ngx.var.skynet_proof = res.header["Skynet-Proof"] - end - end - - -- this block runs only when accounts are enabled - if os.getenv("ACCOUNTS_ENABLED") ~= "true" then return end - - local res = ngx.location.capture("/accounts/user/limits", { copy_all_vars = true }) - if res.status == ngx.HTTP_OK then - local json = require('cjson') - local limits = json.decode(res.body) - ngx.var.limit_rate = limits.download - end - } - - header_filter_by_lua_block { - ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") - ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") - - -- not empty skynet_proof means this is a skylink v2 request - -- so we should replace the Skynet-Proof header with the one - -- we got from /skynet/resolve/ endpoint, otherwise we would - -- be serving cached empty v1 skylink Skynet-Proof header - if ngx.var.skynet_proof then - ngx.header["Skynet-Proof"] = ngx.var.skynet_proof - end - } - - proxy_read_timeout 600; - proxy_set_header User-Agent: Sia-Agent; - - # in case the requested skylink was v2 and we already resolved it to skylink v1, we're going to pass resolved - # skylink v1 to skyd to save that extra skylink v2 lookup in skyd but in turn, in case skyd returns a redirect - # we need to rewrite the skylink v1 to skylink v2 in the location header with proxy_redirect - proxy_redirect $skylink_v1 $skylink_v2; - proxy_pass http://siad/skynet/skylink/$skylink_v1$path$is_args$args; - } - - location @dnslink_lookup { - include /etc/nginx/conf.d/include/proxy-buffer; - include /etc/nginx/conf.d/include/proxy-pass-internal; - - set $dnslink ''; - - rewrite_by_lua_block { - local http = require("socket.http") - local ok, statusCode, headers, statusText = http.request { - url = "http://dnslink-api:3100/dnslink/" .. ngx.var.host - } - - if statusCode == ngx.HTTP_OK then - ngx.var.dnslink = headers["skynet-skylink"] - else - ngx.status = statusCode - ngx.header["content-type"] = "text/plain" - ngx.say(headers["dnslink-error"]) - ngx.exit(statusCode) - end - } - - proxy_set_header Dnslink-Lookup ""; - proxy_pass http://127.0.0.1/$dnslink$request_uri; - } - - location @base32_subdomain { - include /etc/nginx/conf.d/include/proxy-buffer; - include /etc/nginx/conf.d/include/proxy-pass-internal; - - proxy_pass http://127.0.0.1/$base32_subdomain$request_uri; - } - - location @hns_domain { - include /etc/nginx/conf.d/include/proxy-buffer; - include /etc/nginx/conf.d/include/proxy-pass-internal; - - proxy_pass http://127.0.0.1/hns/$hns_domain$request_uri; - } - - location @purge { - allow 10.0.0.0/8; - allow 127.0.0.1/32; - allow 172.16.0.0/12; - allow 192.168.0.0/16; - deny all; - - set $lua_purge_path "/data/nginx/cache/"; - content_by_lua_file /etc/nginx/conf.d/scripts/purge-multi.lua; - } - - location ~ "^/file/([a-zA-Z0-9-_]{46}(/.*)?)$" { - include /etc/nginx/conf.d/include/proxy-buffer; - include /etc/nginx/conf.d/include/proxy-pass-internal; - - rewrite /file/(.*) $1 break; # drop the /file/ prefix from uri - - proxy_pass http://127.0.0.1/$uri?attachment=true&$args; - } - - location /__internal/do/not/use/authenticated { - include /etc/nginx/conf.d/include/cors; - - charset utf-8; - charset_types application/json; - default_type application/json; - - content_by_lua_block { - local json = require('cjson') - -- this block runs only when accounts are enabled - if os.getenv("ACCOUNTS_ENABLED") ~= "true" then - ngx.say(json.encode{authenticated = false}) - return ngx.exit(ngx.HTTP_OK) - end - - local res = ngx.location.capture("/accounts/user", { copy_all_vars = true }) - if res.status == ngx.HTTP_OK then - local limits = json.decode(res.body) - ngx.say(json.encode{authenticated = limits.tier > 0}) - return ngx.exit(ngx.HTTP_OK) - end - - ngx.say(json.encode{authenticated = false}) - return ngx.exit(ngx.HTTP_OK) - } - } - - location /accounts { - internal; # internal endpoint only - access_log off; # do not log traffic - - proxy_cache skynet; # use general nginx cache - proxy_cache_key $uri+$skynet_jwt; # include skynet-jwt cookie (mapped to skynet_jwt) - proxy_cache_valid 200 401 1m; # cache success and unauthorized responses for 1 minute - proxy_buffer_size 8k; # increase size of the buffer to fit jwt in cache key - - rewrite /accounts(.*) $1 break; # drop the /accounts prefix from uri - proxy_pass http://10.10.10.70:3000; # hardcoded ip because accounts might not be available - } - - # include custom locations, specific to the server - include /etc/nginx/conf.d/server-override/*; -} diff --git a/docker/nginx/conf.d/dhparam.pem b/docker/nginx/conf.d/dhparam.pem new file mode 100644 index 00000000..9b182b72 --- /dev/null +++ b/docker/nginx/conf.d/dhparam.pem @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz ++8yTnc4kmz75fS/jY2MMddj2gbICrsRhetPfHtXV/WVhJDP1H18GbtCFY2VVPe0a +87VXE15/V8k1mE8McODmi3fipona8+/och3xWKE2rec1MKzKT0g6eXq8CrGCsyT7 +YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi +7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD +ssbzSibBsu/6iGtCOGEoXJf//////////wIBAg== +-----END DH PARAMETERS----- diff --git a/docker/nginx/conf.d/include/init-optional-variables b/docker/nginx/conf.d/include/init-optional-variables new file mode 100644 index 00000000..4f1be02b --- /dev/null +++ b/docker/nginx/conf.d/include/init-optional-variables @@ -0,0 +1,9 @@ +# optional variables initialisation - those variables are used in log_format +# but are not set on every route so we need to initialise them with empty value +# because otherwise logger with throw error + +# set only on hns routes +set $hns_domain ''; + +# set only if server has been access through SERVER_DOMAIN +set $server_alias ''; diff --git a/docker/nginx/conf.d/include/location-hns b/docker/nginx/conf.d/include/location-hns new file mode 100644 index 00000000..bd5644fd --- /dev/null +++ b/docker/nginx/conf.d/include/location-hns @@ -0,0 +1,94 @@ +include /etc/nginx/conf.d/include/proxy-buffer; +include /etc/nginx/conf.d/include/proxy-pass-internal; + +# variable definititions - we need to define a variable to be able to access it in lua by ngx.var.something +set $skylink ''; # placeholder for the raw 46 bit skylink + +# resolve handshake domain by requesting to /hnsres endpoint and assign correct values to $skylink and $rest +access_by_lua_block { + local json = require('cjson') + local httpc = require("resty.http").new() + + -- make a get request to /hnsres endpoint with the domain name from request_uri + -- 10.10.10.50 points to handshake-api service (alias not available when using resty-http) + local hnsres_res, hnsres_err = httpc:request_uri("http://10.10.10.50:3100/hnsres/" .. ngx.var.hns_domain) + + -- print error and exit with 500 or exit with response if status is not 200 + if hnsres_err or (hnsres_res and hnsres_res.status ~= ngx.HTTP_OK) then + ngx.status = (hnsres_err and ngx.HTTP_INTERNAL_SERVER_ERROR) or hnsres_res.status + ngx.header["content-type"] = "text/plain" + ngx.say(hnsres_err or hnsres_res.body) + return ngx.exit(ngx.status) + end + + -- since /hnsres endpoint response is a json, we need to decode it before we access it + -- example response: '{"skylink":"sia://XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg"}' + local hnsres_json = json.decode(hnsres_res.body) + + -- define local variable containing rest of the skylink if provided + local skylink_rest + + if hnsres_json.skylink then + -- try to match the skylink with sia:// prefix + skylink, skylink_rest = string.match(hnsres_json.skylink, "sia://([^/?]+)(.*)") + + -- in case the skylink did not match, assume that there is no sia:// prefix and try to match again + if skylink == nil then + skylink, skylink_rest = string.match(hnsres_json.skylink, "/?([^/?]+)(.*)") + end + elseif hnsres_json.registry then + local publickey = hnsres_json.registry.publickey + local datakey = hnsres_json.registry.datakey + + -- make a get request to /skynet/registry endpoint with the credentials from text record + -- 10.10.10.10 points to sia service (alias not available when using resty-http) + local registry_res, registry_err = httpc:request_uri("http://10.10.10.10:9980/skynet/registry?publickey=" .. publickey .. "&datakey=" .. datakey, { + headers = { ["User-Agent"] = "Sia-Agent" } + }) + + -- print error and exit with 500 or exit with response if status is not 200 + if registry_err or (registry_res and registry_res.status ~= ngx.HTTP_OK) then + ngx.status = (registry_err and ngx.HTTP_INTERNAL_SERVER_ERROR) or registry_res.status + ngx.header["content-type"] = "text/plain" + ngx.say(registry_err or registry_res.body) + return ngx.exit(ngx.status) + end + + -- since /skynet/registry endpoint response is a json, we need to decode it before we access it + local registry_json = json.decode(registry_res.body) + -- response will contain a hex encoded skylink, we need to decode it + local data = (registry_json.data:gsub('..', function (cc) + return string.char(tonumber(cc, 16)) + end)) + + skylink = data + end + + -- fail with a generic 404 if skylink has not been extracted from a valid /hnsres response for some reason + if not skylink then + return ngx.exit(ngx.HTTP_NOT_FOUND) + end + + ngx.var.skylink = skylink + if ngx.var.path == "/" and skylink_rest ~= nil and skylink_rest ~= "" and skylink_rest ~= "/" then + ngx.var.path = skylink_rest + end +} + +# we proxy to another nginx location rather than directly to siad because we don't want to deal with caching here +proxy_pass https://127.0.0.1/$skylink$path$is_args$args; + +# in case siad returns location header, we need to replace the skylink with the domain name +header_filter_by_lua_block { + ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") + ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") + + if ngx.header.location then + -- match location redirect part after the skylink + local path = string.match(ngx.header.location, "[^/?]+(.*)"); + + -- because siad will set the location header to ie. XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg/index.html + -- we need to replace the skylink with the domain_name so we are not redirected to skylink + ngx.header.location = ngx.var.hns_domain .. path + end +} diff --git a/docker/nginx/conf.d/include/location-skylink b/docker/nginx/conf.d/include/location-skylink new file mode 100644 index 00000000..a0963f87 --- /dev/null +++ b/docker/nginx/conf.d/include/location-skylink @@ -0,0 +1,98 @@ +include /etc/nginx/conf.d/include/cors; +include /etc/nginx/conf.d/include/proxy-buffer; +include /etc/nginx/conf.d/include/proxy-cache-downloads; +include /etc/nginx/conf.d/include/track-download; + +# redirect purge calls to separate location +error_page 462 = @purge; +if ($request_method = PURGE) { + return 462; +} + +limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time + +# $skylink_v1 and $skylink_v2 variables default to the same value but in case the requested skylink was: +# a) skylink v1 - it wouldn't matter, no additional logic is executed +# b) skylink v2 - in a lua block below we will resolve the skylink v2 into skylink v1 and update +# $skylink_v1 variable so then the proxy request to skyd can be cached in nginx (proxy_cache_key +# in proxy-cache-downloads includes $skylink_v1 as a part of the cache key) +set $skylink_v1 $skylink; +set $skylink_v2 $skylink; + +# variable for Skynet-Proof header that we need to inject +# into a response if the request was for skylink v2 +set $skynet_proof ''; + +# default download rate to unlimited +set $limit_rate 0; + +access_by_lua_block { + local httpc = require("resty.http").new() + + -- detect whether requested skylink is v2 + local isBase32v2 = string.len(ngx.var.skylink) == 55 and string.sub(ngx.var.skylink, 0, 2) == "04" + local isBase64v2 = string.len(ngx.var.skylink) == 46 and string.sub(ngx.var.skylink, 0, 2) == "AQ" + + if isBase32v2 or isBase64v2 then + -- 10.10.10.10 points to sia service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/resolve/" .. ngx.var.skylink_v2, { + headers = { ["User-Agent"] = "Sia-Agent" } + }) + + -- print error and exit with 500 or exit with response if status is not 200 + if err or (res and res.status ~= ngx.HTTP_OK) then + ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status + ngx.header["content-type"] = "text/plain" + ngx.say(err or res.body) + return ngx.exit(ngx.status) + end + + local json = require('cjson') + local resolve = json.decode(res.body) + ngx.var.skylink_v1 = resolve.skylink + ngx.var.skynet_proof = res.headers["Skynet-Proof"] + end + + -- this block runs only when accounts are enabled + if os.getenv("ACCOUNTS_ENABLED") ~= "true" then return end + + -- 10.10.10.70 points to accounts service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.70:3000/user/limits", { + headers = { ["User-Agent"] = "Sia-Agent", ["Cookie"] = "skynet-jwt=" .. ngx.var.skynet_jwt } + }) + + -- fail gracefully in case /user/limits failed + if err or (res and res.status ~= ngx.HTTP_OK) then + ngx.log(ngx.ERR, "Failed accounts service request /user/limits: ", err or ("[HTTP " .. res.status .. "] " .. res.body)) + ngx.var.limit_rate = 2621440 -- (20 * 1024 * 1024 / 8) conservative fallback to 20 mbps in case accounts failed to return limits + elseif res and res.status == ngx.HTTP_OK then + local json = require('cjson') + local limits = json.decode(res.body) + ngx.var.limit_rate = limits.download + end +} + +header_filter_by_lua_block { + ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") + ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") + + -- not empty skynet_proof means this is a skylink v2 request + -- so we should replace the Skynet-Proof header with the one + -- we got from /skynet/resolve/ endpoint, otherwise we would + -- be serving cached empty v1 skylink Skynet-Proof header + if ngx.var.skynet_proof and ngx.var.skynet_proof ~= "" then + ngx.header["Skynet-Proof"] = ngx.var.skynet_proof + end +} + +limit_rate_after 512k; +limit_rate $limit_rate; + +proxy_read_timeout 600; +proxy_set_header User-Agent: Sia-Agent; + +# in case the requested skylink was v2 and we already resolved it to skylink v1, we're going to pass resolved +# skylink v1 to skyd to save that extra skylink v2 lookup in skyd but in turn, in case skyd returns a redirect +# we need to rewrite the skylink v1 to skylink v2 in the location header with proxy_redirect +proxy_redirect $skylink_v1 $skylink_v2; +proxy_pass http://sia:9980/skynet/skylink/$skylink_v1$path$is_args$args; diff --git a/docker/nginx/conf.d/include/location-skynet-registry b/docker/nginx/conf.d/include/location-skynet-registry index 31f6efe5..288f53ac 100644 --- a/docker/nginx/conf.d/include/location-skynet-registry +++ b/docker/nginx/conf.d/include/location-skynet-registry @@ -7,14 +7,23 @@ limit_req zone=registry_access_by_ip_throttled burst=200 nodelay; proxy_set_header User-Agent: Sia-Agent; proxy_read_timeout 600; # siad should timeout with 404 after 5 minutes -proxy_pass http://siad/skynet/registry; +proxy_pass http://sia:9980/skynet/registry; access_by_lua_block { -- this block runs only when accounts are enabled if os.getenv("ACCOUNTS_ENABLED") ~= "true" then return end - local res = ngx.location.capture("/accounts/user/limits", { copy_all_vars = true }) - if res.status == ngx.HTTP_OK then + local httpc = require("resty.http").new() + + -- 10.10.10.70 points to accounts service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.70:3000/user/limits", { + headers = { ["User-Agent"] = "Sia-Agent", ["Cookie"] = "skynet-jwt=" .. ngx.var.skynet_jwt } + }) + + -- fail gracefully in case /user/limits failed + if err or (res and res.status ~= ngx.HTTP_OK) then + ngx.log(ngx.ERR, "Failed accounts service request /user/limits: ", err or ("[HTTP " .. res.status .. "] " .. res.body)) + elseif res and res.status == ngx.HTTP_OK then local json = require('cjson') local limits = json.decode(res.body) if limits.registry > 0 then diff --git a/docker/nginx/conf.d/include/ssl-settings b/docker/nginx/conf.d/include/ssl-settings new file mode 100644 index 00000000..e4a82394 --- /dev/null +++ b/docker/nginx/conf.d/include/ssl-settings @@ -0,0 +1,13 @@ +# https://ssl-config.mozilla.org/#server=nginx&version=1.17.7&config=intermediate&openssl=1.1.1d&hsts=false&ocsp=false&guideline=5.6 + +ssl_session_timeout 1d; +ssl_session_cache shared:MozSSL:10m; # about 40000 sessions +ssl_session_tickets off; + +# curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam +ssl_dhparam /etc/nginx/conf.d/dhparam.pem; + +# intermediate configuration +ssl_protocols TLSv1.2 TLSv1.3; +ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; +ssl_prefer_server_ciphers off; diff --git a/docker/nginx/conf.d/include/track-download b/docker/nginx/conf.d/include/track-download index 628e08db..9ed5eda9 100644 --- a/docker/nginx/conf.d/include/track-download +++ b/docker/nginx/conf.d/include/track-download @@ -3,17 +3,25 @@ log_by_lua_block { -- this block runs only when accounts are enabled if os.getenv("ACCOUNTS_ENABLED") ~= "true" then return end - local skylink = ngx.header["Skynet-Skylink"] - if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then - local http = require("socket.http") - local query = table.concat({ "status=" .. ngx.status, "bytes=" .. ngx.var.body_bytes_sent }, "&") - local ok, statusCode, headers, statusText = http.request { - url = "http://accounts:3000/track/download/" .. skylink .. "?" .. query, + local function track(premature, skylink, status, body_bytes_sent, jwt) + if premature then return end + + local httpc = require("resty.http").new() + local query = table.concat({ "status=" .. status, "bytes=" .. body_bytes_sent }, "&") + + -- 10.10.10.70 points to accounts service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.70:3000/track/download/" .. skylink .. "?" .. query, { method = "POST", - headers = ngx.req.get_headers() - } - if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then - ngx.log(ngx.ERR, "accounts endpoint /track/download/" .. skylink .. " failed with error " .. statusCode) + headers = { ["Cookie"] = "skynet-jwt=" .. jwt }, + }) + + if err or (res and res.status ~= ngx.HTTP_NO_CONTENT) then + ngx.log(ngx.ERR, "Failed accounts service request /track/download/" .. skylink .. ": ", err or ("[HTTP " .. res.status .. "] " .. res.body)) end end + + if ngx.header["Skynet-Skylink"] and ngx.var.skynet_jwt ~= "" and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then + local ok, err = ngx.timer.at(0, track, ngx.header["Skynet-Skylink"], ngx.status, ngx.var.body_bytes_sent, ngx.var.skynet_jwt) + if err then ngx.log(ngx.ERR, "Failed to create timer: ", err) end + end } diff --git a/docker/nginx/conf.d/include/track-registry b/docker/nginx/conf.d/include/track-registry index e0dc0ff2..ccab959b 100644 --- a/docker/nginx/conf.d/include/track-registry +++ b/docker/nginx/conf.d/include/track-registry @@ -3,16 +3,25 @@ log_by_lua_block { -- this block runs only when accounts are enabled if os.getenv("ACCOUNTS_ENABLED") ~= "true" then return end - if ngx.status == ngx.HTTP_OK or ngx.status == ngx.HTTP_NOT_FOUND then - local http = require("socket.http") - local method = ngx.req.get_method() == ngx.HTTP_GET and "read" or "write" - local ok, statusCode, headers, statusText = http.request { - url = "http://accounts:3000/track/registry/" .. method, + local function track(premature, request_method, jwt) + if premature then return end + + local httpc = require("resty.http").new() + local method = request_method == ngx.HTTP_GET and "read" or "write" + + -- 10.10.10.70 points to accounts service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.70:3000/track/registry/" .. method, { method = "POST", - headers = ngx.req.get_headers() - } - if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then - ngx.log(ngx.ERR, "accounts endpoint /track/registry/" .. method .. " failed with error " .. statusCode) + headers = { ["Cookie"] = "skynet-jwt=" .. jwt }, + }) + + if err or (res and res.status ~= ngx.HTTP_NO_CONTENT) then + ngx.log(ngx.ERR, "Failed accounts service request /track/registry/" .. method .. ": ", err or ("[HTTP " .. res.status .. "] " .. res.body)) end end + + if ngx.var.skynet_jwt ~= "" and (ngx.status == ngx.HTTP_OK or ngx.status == ngx.HTTP_NOT_FOUND) then + local ok, err = ngx.timer.at(0, track, ngx.req.get_method(), ngx.var.skynet_jwt) + if err then ngx.log(ngx.ERR, "Failed to create timer: ", err) end + end } diff --git a/docker/nginx/conf.d/include/track-upload b/docker/nginx/conf.d/include/track-upload index e4063f06..b9c54ad1 100644 --- a/docker/nginx/conf.d/include/track-upload +++ b/docker/nginx/conf.d/include/track-upload @@ -3,16 +3,24 @@ log_by_lua_block { -- this block runs only when accounts are enabled if os.getenv("ACCOUNTS_ENABLED") ~= "true" then return end - local skylink = ngx.header["Skynet-Skylink"] - if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then - local http = require("socket.http") - local ok, statusCode, headers, statusText = http.request { - url = "http://accounts:3000/track/upload/" .. skylink, + local function track(premature, skylink, jwt) + if premature then return end + + local httpc = require("resty.http").new() + + -- 10.10.10.70 points to accounts service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.70:3000/track/upload/" .. skylink, { method = "POST", - headers = ngx.req.get_headers() - } - if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then - ngx.log(ngx.ERR, "accounts endpoint /track/upload/" .. skylink .. " failed with error " .. statusCode) + headers = { ["Cookie"] = "skynet-jwt=" .. jwt }, + }) + + if err or (res and res.status ~= ngx.HTTP_NO_CONTENT) then + ngx.log(ngx.ERR, "Failed accounts service request /track/upload/" .. skylink .. ": ", err or ("[HTTP " .. res.status .. "] " .. res.body)) end end + + if ngx.header["Skynet-Skylink"] and ngx.var.skynet_jwt ~= "" then + local ok, err = ngx.timer.at(0, track, ngx.header["Skynet-Skylink"], ngx.var.skynet_jwt) + if err then ngx.log(ngx.ERR, "Failed to create timer: ", err) end + end } diff --git a/docker/nginx/conf.d/server.dnslink.conf b/docker/nginx/conf.d/server.dnslink.conf new file mode 100644 index 00000000..8a051d3f --- /dev/null +++ b/docker/nginx/conf.d/server.dnslink.conf @@ -0,0 +1,16 @@ +server { + listen 80 default_server; + listen [::]:80 default_server; + + include /etc/nginx/conf.d/server/server.dnslink; +} + +server { + listen 443 default_server; + listen [::]:443 default_server; + + ssl_certificate /etc/ssl/local-certificate.crt; + ssl_certificate_key /etc/ssl/local-certificate.key; + + include /etc/nginx/conf.d/server/server.dnslink; +} diff --git a/docker/nginx/conf.d/server/server.account b/docker/nginx/conf.d/server/server.account new file mode 100644 index 00000000..5c524a52 --- /dev/null +++ b/docker/nginx/conf.d/server/server.account @@ -0,0 +1,10 @@ +listen 443 ssl http2; +listen [::]:443 ssl http2; + +include /etc/nginx/conf.d/include/ssl-settings; +include /etc/nginx/conf.d/include/init-optional-variables; + +location / { + proxy_redirect http://127.0.0.1/ https://$host/; + proxy_pass http://oathkeeper:4455; +} diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api new file mode 100644 index 00000000..433cea50 --- /dev/null +++ b/docker/nginx/conf.d/server/server.api @@ -0,0 +1,294 @@ +listen 443 ssl http2; +listen [::]:443 ssl http2; + +include /etc/nginx/conf.d/include/ssl-settings; +include /etc/nginx/conf.d/include/init-optional-variables; + +# ddos protection: closing slow connections +client_body_timeout 1h; +client_header_timeout 1h; +send_timeout 1h; + +proxy_connect_timeout 1h; +proxy_read_timeout 1h; +proxy_send_timeout 1h; + +# Increase the body buffer size, to ensure the internal POSTs can always +# parse the full POST contents into memory. +client_body_buffer_size 128k; +client_max_body_size 128k; + +# legacy endpoint rewrite +rewrite ^/portals /skynet/portals permanent; +rewrite ^/stats /skynet/stats permanent; +rewrite ^/skynet/blacklist /skynet/blocklist permanent; + +location / { + include /etc/nginx/conf.d/include/cors; + + proxy_pass http://website:9000; +} + +location /docs { + proxy_pass https://skynetlabs.github.io/skynet-docs; +} + +location /skynet/blocklist { + include /etc/nginx/conf.d/include/cors; + + proxy_cache skynet; + proxy_cache_valid any 1m; # cache blocklist for 1 minute + proxy_set_header User-Agent: Sia-Agent; + proxy_pass http://sia:9980/skynet/blocklist; +} + +location /skynet/portals { + include /etc/nginx/conf.d/include/cors; + + proxy_cache skynet; + proxy_cache_valid any 1m; # cache portals for 1 minute + proxy_set_header User-Agent: Sia-Agent; + proxy_pass http://sia:9980/skynet/portals; +} + +location /skynet/stats { + include /etc/nginx/conf.d/include/cors; + + proxy_cache skynet; + proxy_cache_valid any 1m; # cache stats for 1 minute + proxy_set_header User-Agent: Sia-Agent; + proxy_read_timeout 5m; # extend the read timeout + proxy_pass http://sia:9980/skynet/stats; +} + +location /skynet/health { + include /etc/nginx/conf.d/include/cors; + + proxy_cache skynet; + proxy_cache_key $request_uri; # use whole request uri (uri + args) as cache key + proxy_cache_valid any 1m; # cache responses for 1 minute + proxy_set_header User-Agent: Sia-Agent; + proxy_read_timeout 5m; # extend the read timeout + proxy_pass http://sia:9980; +} + +location /health-check { + include /etc/nginx/conf.d/include/cors; + + access_log off; # do not log traffic to health-check endpoint + + proxy_pass http://10.10.10.60:3100; # hardcoded ip because health-check waits for nginx +} + +location /hns { + # match the request_uri and extract the hns domain and anything that is passed in the uri after it + # example: /hns/something/foo/bar matches: + # > hns_domain: something + # > path: /foo/bar/ + set_by_lua_block $hns_domain { return string.match(ngx.var.uri, "/hns/([^/?]+)") } + set_by_lua_block $path { return string.match(ngx.var.uri, "/hns/[^/?]+(.*)") } + + proxy_set_header Host $host; + include /etc/nginx/conf.d/include/location-hns; +} + +location /hnsres { + include /etc/nginx/conf.d/include/cors; + + proxy_pass http://handshake-api:3100; +} + +location /skynet/registry { + include /etc/nginx/conf.d/include/location-skynet-registry; +} + +location /skynet/skyfile { + include /etc/nginx/conf.d/include/cors; + include /etc/nginx/conf.d/include/sia-auth; + include /etc/nginx/conf.d/include/track-upload; + include /etc/nginx/conf.d/include/generate-siapath; + + limit_req zone=uploads_by_ip burst=100 nodelay; + limit_req zone=uploads_by_ip_throttled; + + limit_conn upload_conn 10; + limit_conn upload_conn_rl 1; + + client_max_body_size 1000M; # make sure to limit the size of upload to a sane value + + # increase request timeouts + proxy_read_timeout 600; + proxy_send_timeout 600; + + proxy_request_buffering off; # stream uploaded files through the proxy as it comes in + proxy_set_header Expect $http_expect; + proxy_set_header User-Agent: Sia-Agent; + + # access_by_lua_block { + # -- this block runs only when accounts are enabled + # if os.getenv("ACCOUNTS_ENABLED") ~= "true" then return end + + # ngx.var.upload_limit_rate = 5 * 1024 * 1024 + # local res = ngx.location.capture("/accounts/user", { copy_all_vars = true }) + # if res.status == ngx.HTTP_OK then + # local json = require('cjson') + # local user = json.decode(res.body) + # ngx.var.upload_limit_rate = ngx.var.upload_limit_rate * (user.tier + 1) + # end + # } + + # proxy this call to siad endpoint (make sure the ip is correct) + proxy_pass http://sia:9980/skynet/skyfile/$dir1/$dir2/$dir3$is_args$args; +} + +# endpoint implementing resumable file uploads open protocol https://tus.io +location /skynet/tus { + include /etc/nginx/conf.d/include/cors; + include /etc/nginx/conf.d/include/track-upload; + + # TUS chunks size is 40M + leaving 10M of breathing room + client_max_body_size 50M; + + # Those timeouts need to be elevated since skyd can stall reading + # data for a while when overloaded which would terminate connection + client_body_timeout 1h; + proxy_send_timeout 1h; + + # Add X-Forwarded-* headers + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + + # rewrite proxy request to use correct host uri from env variable (required to return correct location header) + set_by_lua $SKYNET_SERVER_API 'return os.getenv("SKYNET_SERVER_API")'; + proxy_redirect $scheme://$host $SKYNET_SERVER_API; + + # proxy /skynet/tus requests to siad endpoint with all arguments + proxy_pass http://sia:9980; + + # set max upload size dynamically based on account limits + rewrite_by_lua_block { + -- set default limit value to 1 GB + ngx.req.set_header("SkynetMaxUploadSize", 1073741824) + + -- this block runs only when accounts are enabled + if os.getenv("ACCOUNTS_ENABLED") ~= "true" then return end + + -- fetch account limits and set max upload size accordingly + local res, err = httpc:request_uri("http://10.10.10.70:3000/user/limits", { + headers = { ["User-Agent"] = "Sia-Agent", ["Cookie"] = "skynet-jwt=" .. ngx.var.skynet_jwt } + }) + + -- fail gracefully in case /user/limits failed + if err or (res and res.status ~= ngx.HTTP_OK) then + ngx.log(ngx.ERR, "Failed accounts service request /user/limits: ", err or ("[HTTP " .. res.status .. "] " .. res.body)) + elseif res and res.status == ngx.HTTP_OK then + local json = require('cjson') + local limits = json.decode(res.body) + ngx.req.set_header("SkynetMaxUploadSize", limits.maxUploadSize) + end + } + + # extract skylink from base64 encoded upload metadata and assign to a proper header + header_filter_by_lua_block { + ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") + ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") + + if ngx.header["Upload-Metadata"] then + local encodedSkylink = string.match(ngx.header["Upload-Metadata"], "Skylink ([^,?]+)") + + if encodedSkylink then + ngx.header["Skynet-Skylink"] = ngx.decode_base64(encodedSkylink) + end + end + } +} + +location /skynet/pin { + include /etc/nginx/conf.d/include/cors; + include /etc/nginx/conf.d/include/sia-auth; + include /etc/nginx/conf.d/include/track-upload; + include /etc/nginx/conf.d/include/generate-siapath; + + proxy_set_header User-Agent: Sia-Agent; + proxy_pass http://sia:9980$uri?siapath=$dir1/$dir2/$dir3&$args; +} + +location /skynet/metadata { + include /etc/nginx/conf.d/include/cors; + + proxy_set_header User-Agent: Sia-Agent; + proxy_pass http://sia:9980; +} + +location /skynet/resolve { + include /etc/nginx/conf.d/include/cors; + + proxy_set_header User-Agent: Sia-Agent; + proxy_pass http://sia:9980; +} + +location ~ "^/(([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?)$" { + set $skylink $2; + set $path $3; + + include /etc/nginx/conf.d/include/location-skylink; +} + +location ~ "^/file/(([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?)$" { + set $skylink $2; + set $path $3; + set $args attachment=true&$args; + #set $is_args ?; + + include /etc/nginx/conf.d/include/location-skylink; +} + +location @purge { + allow 10.0.0.0/8; + allow 127.0.0.1/32; + allow 172.16.0.0/12; + allow 192.168.0.0/16; + deny all; + + set $lua_purge_path "/data/nginx/cache/"; + content_by_lua_file /etc/nginx/conf.d/scripts/purge-multi.lua; +} + +location /__internal/do/not/use/authenticated { + include /etc/nginx/conf.d/include/cors; + + charset utf-8; + charset_types application/json; + default_type application/json; + + content_by_lua_block { + local json = require('cjson') + + -- this block runs only when accounts are enabled + if os.getenv("ACCOUNTS_ENABLED") ~= "true" then + ngx.say(json.encode{authenticated = false}) + return ngx.exit(ngx.HTTP_OK) + end + + local httpc = require("resty.http").new() + + -- 10.10.10.70 points to accounts service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.70:3000/user", { + headers = { ["User-Agent"] = "Sia-Agent", ["Cookie"] = "skynet-jwt=" .. ngx.var.skynet_jwt } + }) + + -- fail gracefully in case /user failed + if err or (res and res.status ~= ngx.HTTP_OK) then + ngx.log(ngx.ERR, "Failed accounts service request /user/limits: ", err or ("[HTTP " .. res.status .. "] " .. res.body)) + elseif res and res.status == ngx.HTTP_OK then + local limits = json.decode(res.body) + ngx.say(json.encode{authenticated = limits.tier > 0}) + return ngx.exit(ngx.HTTP_OK) + end + + ngx.say(json.encode{authenticated = false}) + return ngx.exit(ngx.HTTP_OK) + } +} + +include /etc/nginx/conf.d/server-override/*; diff --git a/docker/nginx/conf.d/server/server.dnslink b/docker/nginx/conf.d/server/server.dnslink new file mode 100644 index 00000000..db8948e3 --- /dev/null +++ b/docker/nginx/conf.d/server/server.dnslink @@ -0,0 +1,26 @@ +include /etc/nginx/conf.d/include/init-optional-variables; + +location / { + set $skylink ""; + set $path $uri; + + rewrite_by_lua_block { + local httpc = require("resty.http").new() + + -- 10.10.10.55 points to dnslink-api service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.55:3100/dnslink/" .. ngx.var.host) + + if err or (res and res.status ~= ngx.HTTP_OK) then + ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status + ngx.header["content-type"] = "text/plain" + ngx.say(err or res.body) + ngx.exit(ngx.status) + else + ngx.var.skylink = res.body + ngx.var.skylink_v1 = ngx.var.skylink + ngx.var.skylink_v2 = ngx.var.skylink + end + } + + include /etc/nginx/conf.d/include/location-skylink; +} diff --git a/docker/nginx/conf.d/server/server.hns b/docker/nginx/conf.d/server/server.hns new file mode 100644 index 00000000..3daa167f --- /dev/null +++ b/docker/nginx/conf.d/server/server.hns @@ -0,0 +1,12 @@ +listen 443 ssl http2; +listen [::]:443 ssl http2; + +include /etc/nginx/conf.d/include/ssl-settings; +include /etc/nginx/conf.d/include/init-optional-variables; + +location / { + set_by_lua_block $hns_domain { return string.match(ngx.var.host, "[^%.]+") } + set $path $uri; + + include /etc/nginx/conf.d/include/location-hns; +} diff --git a/docker/nginx/conf.d/server/server.http b/docker/nginx/conf.d/server/server.http new file mode 100644 index 00000000..77cce00a --- /dev/null +++ b/docker/nginx/conf.d/server/server.http @@ -0,0 +1,8 @@ +listen 80; +listen [::]:80; + +include /etc/nginx/conf.d/include/init-optional-variables; + +location / { + return 301 https://$host$request_uri; +} diff --git a/docker/nginx/conf.d/server/server.skylink b/docker/nginx/conf.d/server/server.skylink new file mode 100644 index 00000000..14c0870e --- /dev/null +++ b/docker/nginx/conf.d/server/server.skylink @@ -0,0 +1,12 @@ +listen 443 ssl http2; +listen [::]:443 ssl http2; + +include /etc/nginx/conf.d/include/ssl-settings; +include /etc/nginx/conf.d/include/init-optional-variables; + +location / { + set_by_lua_block $skylink { return string.match(ngx.var.host, "%w+") } + set $path $uri; + + include /etc/nginx/conf.d/include/location-skylink; +} diff --git a/docker/nginx/mo b/docker/nginx/mo new file mode 100755 index 00000000..ba8e48d1 --- /dev/null +++ b/docker/nginx/mo @@ -0,0 +1,1106 @@ +#!/usr/bin/env bash +# +#/ Mo is a mustache template rendering software written in bash. It inserts +#/ environment variables into templates. +#/ +#/ Simply put, mo will change {{VARIABLE}} into the value of that +#/ environment variable. You can use {{#VARIABLE}}content{{/VARIABLE}} to +#/ conditionally display content or iterate over the values of an array. +#/ +#/ Learn more about mustache templates at https://mustache.github.io/ +#/ +#/ Simple usage: +#/ +#/ mo [OPTIONS] filenames... +#/ +#/ Options: +#/ +#/ -u, --fail-not-set +#/ Fail upon expansion of an unset variable. +#/ -x, --fail-on-function +#/ Fail when a function returns a non-zero status code. +#/ -e, --false +#/ Treat the string "false" as empty for conditionals. +#/ -h, --help +#/ This message. +#/ -s=FILE, --source=FILE +#/ Load FILE into the environment before processing templates. +#/ Can be used multiple times. +# +# Mo is under a MIT style licence with an additional non-advertising clause. +# See LICENSE.md for the full text. +# +# This is open source! Please feel free to contribute. +# +# https://github.com/tests-always-included/mo + + +# Public: Template parser function. Writes templates to stdout. +# +# $0 - Name of the mo file, used for getting the help message. +# $@ - Filenames to parse. +# +# Options: +# +# --allow-function-arguments +# +# Permit functions in templates to be called with additional arguments. This +# puts template data directly in to the path of an eval statement. Use with +# caution. Not listed in the help because it only makes sense when mo is +# sourced. +# +# -u, --fail-not-set +# +# Fail upon expansion of an unset variable. Default behavior is to silently +# ignore and expand into empty string. +# +# -x, --fail-on-function +# +# Fail when a function used by a template returns an error status code. +# Alternately, ou may set the MO_FAIL_ON_FUNCTION environment variable to a +# non-empty value to enable this behavior. +# +# -e, --false +# +# Treat "false" as an empty value. You may set the MO_FALSE_IS_EMPTY +# environment variable instead to a non-empty value to enable this behavior. +# +# -h, --help +# +# Display a help message. +# +# -s=FILE, --source=FILE +# +# Source a file into the environment before processing template files. +# This can be used multiple times. +# +# -- +# +# Used to indicate the end of options. You may optionally use this when +# filenames may start with two hyphens. +# +# Mo uses the following environment variables: +# +# MO_ALLOW_FUNCTION_ARGUMENTS - When set to a non-empty value, this allows +# functions referenced in templates to receive additional +# options and arguments. This puts the content from the +# template directly into an eval statement. Use with extreme +# care. +# MO_FUNCTION_ARGS - Arguments passed to the function +# MO_FAIL_ON_FUNCTION - If a function returns a non-zero status code, abort +# with an error. +# MO_FAIL_ON_UNSET - When set to a non-empty value, expansion of an unset env +# variable will be aborted with an error. +# MO_FALSE_IS_EMPTY - When set to a non-empty value, the string "false" will be +# treated as an empty value for the purposes of conditionals. +# MO_ORIGINAL_COMMAND - Used to find the `mo` program in order to generate a +# help message. +# +# Returns nothing. +mo() ( + # This function executes in a subshell so IFS is reset. + # Namespace this variable so we don't conflict with desired values. + local moContent f2source files doubleHyphens + + IFS=$' \n\t' + files=() + doubleHyphens=false + + if [[ $# -gt 0 ]]; then + for arg in "$@"; do + if $doubleHyphens; then + #: After we encounter two hyphens together, all the rest + #: of the arguments are files. + files=("${files[@]}" "$arg") + else + case "$arg" in + -h|--h|--he|--hel|--help|-\?) + moUsage "$0" + exit 0 + ;; + + --allow-function-arguments) + # shellcheck disable=SC2030 + MO_ALLOW_FUNCTION_ARGUMENTS=true + ;; + + -u | --fail-not-set) + # shellcheck disable=SC2030 + MO_FAIL_ON_UNSET=true + ;; + + -x | --fail-on-function) + # shellcheck disable=SC2030 + MO_FAIL_ON_FUNCTION=true + ;; + + -e | --false) + # shellcheck disable=SC2030 + MO_FALSE_IS_EMPTY=true + ;; + + -s=* | --source=*) + if [[ "$arg" == --source=* ]]; then + f2source="${arg#--source=}" + else + f2source="${arg#-s=}" + fi + + if [[ -f "$f2source" ]]; then + # shellcheck disable=SC1090 + . "$f2source" + else + echo "No such file: $f2source" >&2 + exit 1 + fi + ;; + + --) + #: Set a flag indicating we've encountered double hyphens + doubleHyphens=true + ;; + + *) + #: Every arg that is not a flag or a option should be a file + files=(${files[@]+"${files[@]}"} "$arg") + ;; + esac + fi + done + fi + + moGetContent moContent "${files[@]}" || return 1 + moParse "$moContent" "" true +) + + +# Internal: Call a function. +# +# $1 - Variable for output +# $2 - Function to call +# $3 - Content to pass +# $4 - Additional arguments as a single string +# +# This can be dangerous, especially if you are using tags like +# {{someFunction ; rm -rf / }} +# +# Returns nothing. +moCallFunction() { + local moArgs moContent moFunctionArgs moFunctionResult + + moArgs=() + moTrimWhitespace moFunctionArgs "$4" + + # shellcheck disable=SC2031 + if [[ -n "${MO_ALLOW_FUNCTION_ARGUMENTS-}" ]]; then + # Intentionally bad behavior + # shellcheck disable=SC2206 + moArgs=($4) + fi + + moContent=$(echo -n "$3" | MO_FUNCTION_ARGS="$moFunctionArgs" eval "$2" "${moArgs[@]}") || { + moFunctionResult=$? + # shellcheck disable=SC2031 + if [[ -n "${MO_FAIL_ON_FUNCTION-}" && "$moFunctionResult" != 0 ]]; then + echo "Function '$2' with args (${moArgs[*]+"${moArgs[@]}"}) failed with status code $moFunctionResult" + exit "$moFunctionResult" + fi + } + + # shellcheck disable=SC2031 + local "$1" && moIndirect "$1" "$moContent" +} + + +# Internal: Scan content until the right end tag is found. Creates an array +# with the following members: +# +# [0] = Content before end tag +# [1] = End tag (complete tag) +# [2] = Content after end tag +# +# Everything using this function uses the "standalone tags" logic. +# +# $1 - Name of variable for the array +# $2 - Content +# $3 - Name of end tag +# $4 - If -z, do standalone tag processing before finishing +# +# Returns nothing. +moFindEndTag() { + local content remaining scanned standaloneBytes tag + + #: Find open tags + scanned="" + moSplit content "$2" '{{' '}}' + + while [[ "${#content[@]}" -gt 1 ]]; do + moTrimWhitespace tag "${content[1]}" + + #: Restore content[1] before we start using it + content[1]='{{'"${content[1]}"'}}' + + case $tag in + '#'* | '^'*) + #: Start another block + scanned="${scanned}${content[0]}${content[1]}" + moTrimWhitespace tag "${tag:1}" + moFindEndTag content "${content[2]}" "$tag" "loop" + scanned="${scanned}${content[0]}${content[1]}" + remaining=${content[2]} + ;; + + '/'*) + #: End a block - could be ours + moTrimWhitespace tag "${tag:1}" + scanned="$scanned${content[0]}" + + if [[ "$tag" == "$3" ]]; then + #: Found our end tag + if [[ -z "${4-}" ]] && moIsStandalone standaloneBytes "$scanned" "${content[2]}" true; then + #: This is also a standalone tag - clean up whitespace + #: and move those whitespace bytes to the "tag" element + # shellcheck disable=SC2206 + standaloneBytes=( $standaloneBytes ) + content[1]="${scanned:${standaloneBytes[0]}}${content[1]}${content[2]:0:${standaloneBytes[1]}}" + scanned="${scanned:0:${standaloneBytes[0]}}" + content[2]="${content[2]:${standaloneBytes[1]}}" + fi + + local "$1" && moIndirectArray "$1" "$scanned" "${content[1]}" "${content[2]}" + return 0 + fi + + scanned="$scanned${content[1]}" + remaining=${content[2]} + ;; + + *) + #: Ignore all other tags + scanned="${scanned}${content[0]}${content[1]}" + remaining=${content[2]} + ;; + esac + + moSplit content "$remaining" '{{' '}}' + done + + #: Did not find our closing tag + scanned="$scanned${content[0]}" + local "$1" && moIndirectArray "$1" "${scanned}" "" "" +} + + +# Internal: Find the first index of a substring. If not found, sets the +# index to -1. +# +# $1 - Destination variable for the index +# $2 - Haystack +# $3 - Needle +# +# Returns nothing. +moFindString() { + local pos string + + string=${2%%$3*} + [[ "$string" == "$2" ]] && pos=-1 || pos=${#string} + local "$1" && moIndirect "$1" "$pos" +} + + +# Internal: Generate a dotted name based on current context and target name. +# +# $1 - Target variable to store results +# $2 - Context name +# $3 - Desired variable name +# +# Returns nothing. +moFullTagName() { + if [[ -z "${2-}" ]] || [[ "$2" == *.* ]]; then + local "$1" && moIndirect "$1" "$3" + else + local "$1" && moIndirect "$1" "${2}.${3}" + fi +} + + +# Internal: Fetches the content to parse into a variable. Can be a list of +# partials for files or the content from stdin. +# +# $1 - Variable name to assign this content back as +# $2-@ - File names (optional) +# +# Returns nothing. +moGetContent() { + local moContent moFilename moTarget + + moTarget=$1 + shift + if [[ "${#@}" -gt 0 ]]; then + moContent="" + + for moFilename in "$@"; do + #: This is so relative paths work from inside template files + moContent="$moContent"'{{>'"$moFilename"'}}' + done + else + moLoadFile moContent || return 1 + fi + + local "$moTarget" && moIndirect "$moTarget" "$moContent" +} + + +# Internal: Indent a string, placing the indent at the beginning of every +# line that has any content. +# +# $1 - Name of destination variable to get an array of lines +# $2 - The indent string +# $3 - The string to reindent +# +# Returns nothing. +moIndentLines() { + local content fragment len posN posR result trimmed + + result="" + + #: Remove the period from the end of the string. + len=$((${#3} - 1)) + content=${3:0:$len} + + if [[ -z "${2-}" ]]; then + local "$1" && moIndirect "$1" "$content" + + return 0 + fi + + moFindString posN "$content" $'\n' + moFindString posR "$content" $'\r' + + while [[ "$posN" -gt -1 ]] || [[ "$posR" -gt -1 ]]; do + if [[ "$posN" -gt -1 ]]; then + fragment="${content:0:$posN + 1}" + content=${content:$posN + 1} + else + fragment="${content:0:$posR + 1}" + content=${content:$posR + 1} + fi + + moTrimChars trimmed "$fragment" false true " " $'\t' $'\n' $'\r' + + if [[ -n "$trimmed" ]]; then + fragment="$2$fragment" + fi + + result="$result$fragment" + + moFindString posN "$content" $'\n' + moFindString posR "$content" $'\r' + + # If the content ends in a newline, do not indent. + if [[ "$posN" -eq ${#content} ]]; then + # Special clause for \r\n + if [[ "$posR" -eq "$((posN - 1))" ]]; then + posR=-1 + fi + + posN=-1 + fi + + if [[ "$posR" -eq ${#content} ]]; then + posR=-1 + fi + done + + moTrimChars trimmed "$content" false true " " $'\t' + + if [[ -n "$trimmed" ]]; then + content="$2$content" + fi + + result="$result$content" + + local "$1" && moIndirect "$1" "$result" +} + + +# Internal: Send a variable up to the parent of the caller of this function. +# +# $1 - Variable name +# $2 - Value +# +# Examples +# +# callFunc () { +# local "$1" && moIndirect "$1" "the value" +# } +# callFunc dest +# echo "$dest" # writes "the value" +# +# Returns nothing. +moIndirect() { + unset -v "$1" + printf -v "$1" '%s' "$2" +} + + +# Internal: Send an array as a variable up to caller of a function +# +# $1 - Variable name +# $2-@ - Array elements +# +# Examples +# +# callFunc () { +# local myArray=(one two three) +# local "$1" && moIndirectArray "$1" "${myArray[@]}" +# } +# callFunc dest +# echo "${dest[@]}" # writes "one two three" +# +# Returns nothing. +moIndirectArray() { + unset -v "$1" + + # IFS must be set to a string containing space or unset in order for + # the array slicing to work regardless of the current IFS setting on + # bash 3. This is detailed further at + # https://github.com/fidian/gg-core/pull/7 + eval "$(printf "IFS= %s=(\"\${@:2}\") IFS=%q" "$1" "$IFS")" +} + + +# Internal: Determine if a given environment variable exists and if it is +# an array. +# +# $1 - Name of environment variable +# +# Be extremely careful. Even if strict mode is enabled, it is not honored +# in newer versions of Bash. Any errors that crop up here will not be +# caught automatically. +# +# Examples +# +# var=(abc) +# if moIsArray var; then +# echo "This is an array" +# echo "Make sure you don't accidentally use \$var" +# fi +# +# Returns 0 if the name is not empty, 1 otherwise. +moIsArray() { + # Namespace this variable so we don't conflict with what we're testing. + local moTestResult + + moTestResult=$(declare -p "$1" 2>/dev/null) || return 1 + [[ "${moTestResult:0:10}" == "declare -a" ]] && return 0 + [[ "${moTestResult:0:10}" == "declare -A" ]] && return 0 + + return 1 +} + + +# Internal: Determine if the given name is a defined function. +# +# $1 - Function name to check +# +# Be extremely careful. Even if strict mode is enabled, it is not honored +# in newer versions of Bash. Any errors that crop up here will not be +# caught automatically. +# +# Examples +# +# moo () { +# echo "This is a function" +# } +# if moIsFunction moo; then +# echo "moo is a defined function" +# fi +# +# Returns 0 if the name is a function, 1 otherwise. +moIsFunction() { + local functionList functionName + + functionList=$(declare -F) + # shellcheck disable=SC2206 + functionList=( ${functionList//declare -f /} ) + + for functionName in "${functionList[@]}"; do + if [[ "$functionName" == "$1" ]]; then + return 0 + fi + done + + return 1 +} + + +# Internal: Determine if the tag is a standalone tag based on whitespace +# before and after the tag. +# +# Passes back a string containing two numbers in the format "BEFORE AFTER" +# like "27 10". It indicates the number of bytes remaining in the "before" +# string (27) and the number of bytes to trim in the "after" string (10). +# Useful for string manipulation: +# +# $1 - Variable to set for passing data back +# $2 - Content before the tag +# $3 - Content after the tag +# $4 - true/false: is this the beginning of the content? +# +# Examples +# +# moIsStandalone RESULT "$before" "$after" false || return 0 +# RESULT_ARRAY=( $RESULT ) +# echo "${before:0:${RESULT_ARRAY[0]}}...${after:${RESULT_ARRAY[1]}}" +# +# Returns nothing. +moIsStandalone() { + local afterTrimmed beforeTrimmed char + + moTrimChars beforeTrimmed "$2" false true " " $'\t' + moTrimChars afterTrimmed "$3" true false " " $'\t' + char=$((${#beforeTrimmed} - 1)) + char=${beforeTrimmed:$char} + + # If the content before didn't end in a newline + if [[ "$char" != $'\n' ]] && [[ "$char" != $'\r' ]]; then + # and there was content or this didn't start the file + if [[ -n "$char" ]] || ! $4; then + # then this is not a standalone tag. + return 1 + fi + fi + + char=${afterTrimmed:0:1} + + # If the content after doesn't start with a newline and it is something + if [[ "$char" != $'\n' ]] && [[ "$char" != $'\r' ]] && [[ -n "$char" ]]; then + # then this is not a standalone tag. + return 2 + fi + + if [[ "$char" == $'\r' ]] && [[ "${afterTrimmed:1:1}" == $'\n' ]]; then + char="$char"$'\n' + fi + + local "$1" && moIndirect "$1" "$((${#beforeTrimmed})) $((${#3} + ${#char} - ${#afterTrimmed}))" +} + + +# Internal: Join / implode an array +# +# $1 - Variable name to receive the joined content +# $2 - Joiner +# $3-$* - Elements to join +# +# Returns nothing. +moJoin() { + local joiner part result target + + target=$1 + joiner=$2 + result=$3 + shift 3 + + for part in "$@"; do + result="$result$joiner$part" + done + + local "$target" && moIndirect "$target" "$result" +} + + +# Internal: Read a file into a variable. +# +# $1 - Variable name to receive the file's content +# $2 - Filename to load - if empty, defaults to /dev/stdin +# +# Returns nothing. +moLoadFile() { + local content len + + # The subshell removes any trailing newlines. We forcibly add + # a dot to the content to preserve all newlines. + # As a future optimization, it would be worth considering removing + # cat and replacing this with a read loop. + + content=$(cat -- "${2:-/dev/stdin}" && echo '.') || return 1 + len=$((${#content} - 1)) + content=${content:0:$len} # Remove last dot + + local "$1" && moIndirect "$1" "$content" +} + + +# Internal: Process a chunk of content some number of times. Writes output +# to stdout. +# +# $1 - Content to parse repeatedly +# $2 - Tag prefix (context name) +# $3-@ - Names to insert into the parsed content +# +# Returns nothing. +moLoop() { + local content context contextBase + + content=$1 + contextBase=$2 + shift 2 + + while [[ "${#@}" -gt 0 ]]; do + moFullTagName context "$contextBase" "$1" + moParse "$content" "$context" false + shift + done +} + + +# Internal: Parse a block of text, writing the result to stdout. +# +# $1 - Block of text to change +# $2 - Current name (the variable NAME for what {{.}} means) +# $3 - true when no content before this, false otherwise +# +# Returns nothing. +moParse() { + # Keep naming variables mo* here to not overwrite needed variables + # used in the string replacements + local moArgs moBlock moContent moCurrent moIsBeginning moNextIsBeginning moTag + + moCurrent=$2 + moIsBeginning=$3 + + # Find open tags + moSplit moContent "$1" '{{' '}}' + + while [[ "${#moContent[@]}" -gt 1 ]]; do + moTrimWhitespace moTag "${moContent[1]}" + moNextIsBeginning=false + + case $moTag in + '#'*) + # Loop, if/then, or pass content through function + # Sets context + moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" + moTrimWhitespace moTag "${moTag:1}" + + # Split arguments from the tag name. Arguments are passed to + # functions. + moArgs=$moTag + moTag=${moTag%% *} + moTag=${moTag%%$'\t'*} + moArgs=${moArgs:${#moTag}} + moFindEndTag moBlock "$moContent" "$moTag" + moFullTagName moTag "$moCurrent" "$moTag" + + if moTest "$moTag"; then + # Show / loop / pass through function + if moIsFunction "$moTag"; then + moCallFunction moContent "$moTag" "${moBlock[0]}" "$moArgs" + moParse "$moContent" "$moCurrent" false + moContent="${moBlock[2]}" + elif moIsArray "$moTag"; then + eval "moLoop \"\${moBlock[0]}\" \"$moTag\" \"\${!${moTag}[@]}\"" + else + moParse "${moBlock[0]}" "$moCurrent" true + fi + fi + + moContent="${moBlock[2]}" + ;; + + '>'*) + # Load partial - get name of file relative to cwd + moPartial moContent "${moContent[@]}" "$moIsBeginning" "$moCurrent" + moNextIsBeginning=${moContent[1]} + moContent=${moContent[0]} + ;; + + '/'*) + # Closing tag - If hit in this loop, we simply ignore + # Matching tags are found in moFindEndTag + moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" + ;; + + '^'*) + # Display section if named thing does not exist + moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" + moTrimWhitespace moTag "${moTag:1}" + moFindEndTag moBlock "$moContent" "$moTag" + moFullTagName moTag "$moCurrent" "$moTag" + + if ! moTest "$moTag"; then + moParse "${moBlock[0]}" "$moCurrent" false "$moCurrent" + fi + + moContent="${moBlock[2]}" + ;; + + '!'*) + # Comment - ignore the tag content entirely + # Trim spaces/tabs before the comment + moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" + ;; + + .) + # Current content (environment variable or function) + moStandaloneDenied moContent "${moContent[@]}" + moShow "$moCurrent" "$moCurrent" + ;; + + '=') + # Change delimiters + # Any two non-whitespace sequences separated by whitespace. + # This tag is ignored. + moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" + ;; + + '{'*) + # Unescaped - split on }}} not }} + moStandaloneDenied moContent "${moContent[@]}" + moContent="${moTag:1}"'}}'"$moContent" + moSplit moContent "$moContent" '}}}' + moTrimWhitespace moTag "${moContent[0]}" + moArgs=$moTag + moTag=${moTag%% *} + moTag=${moTag%%$'\t'*} + moArgs=${moArgs:${#moTag}} + moFullTagName moTag "$moCurrent" "$moTag" + moContent=${moContent[1]} + + # Now show the value + # Quote moArgs here, do not quote it later. + moShow "$moTag" "$moCurrent" "$moArgs" + ;; + + '&'*) + # Unescaped + moStandaloneDenied moContent "${moContent[@]}" + moTrimWhitespace moTag "${moTag:1}" + moFullTagName moTag "$moCurrent" "$moTag" + moShow "$moTag" "$moCurrent" + ;; + + *) + # Normal environment variable or function call + moStandaloneDenied moContent "${moContent[@]}" + moArgs=$moTag + moTag=${moTag%% *} + moTag=${moTag%%$'\t'*} + moArgs=${moArgs:${#moTag}} + moFullTagName moTag "$moCurrent" "$moTag" + + # Quote moArgs here, do not quote it later. + moShow "$moTag" "$moCurrent" "$moArgs" + ;; + esac + + moIsBeginning=$moNextIsBeginning + moSplit moContent "$moContent" '{{' '}}' + done + + echo -n "${moContent[0]}" +} + + +# Internal: Process a partial. +# +# Indentation should be applied to the entire partial. +# +# This sends back the "is beginning" flag because the newline after a +# standalone partial is consumed. That newline is very important in the middle +# of content. We send back this flag to reset the processing loop's +# `moIsBeginning` variable, so the software thinks we are back at the +# beginning of a file and standalone processing continues to work. +# +# Prefix all variables. +# +# $1 - Name of destination variable. Element [0] is the content, [1] is the +# true/false flag indicating if we are at the beginning of content. +# $2 - Content before the tag that was not yet written +# $3 - Tag content +# $4 - Content after the tag +# $5 - true/false: is this the beginning of the content? +# $6 - Current context name +# +# Returns nothing. +moPartial() { + # Namespace variables here to prevent conflicts. + local moContent moFilename moIndent moIsBeginning moPartial moStandalone moUnindented + + if moIsStandalone moStandalone "$2" "$4" "$5"; then + # shellcheck disable=SC2206 + moStandalone=( $moStandalone ) + echo -n "${2:0:${moStandalone[0]}}" + moIndent=${2:${moStandalone[0]}} + moContent=${4:${moStandalone[1]}} + moIsBeginning=true + else + moIndent="" + echo -n "$2" + moContent=$4 + moIsBeginning=$5 + fi + + moTrimWhitespace moFilename "${3:1}" + + # Execute in subshell to preserve current cwd and environment + ( + # It would be nice to remove `dirname` and use a function instead, + # but that's difficult when you're only given filenames. + cd "$(dirname -- "$moFilename")" || exit 1 + moUnindented="$( + moLoadFile moPartial "${moFilename##*/}" || exit 1 + moParse "${moPartial}" "$6" true + + # Fix bash handling of subshells and keep trailing whitespace. + # This is removed in moIndentLines. + echo -n "." + )" || exit 1 + moIndentLines moPartial "$moIndent" "$moUnindented" + echo -n "$moPartial" + ) || exit 1 + + # If this is a standalone tag, the trailing newline after the tag is + # removed and the contents of the partial are added, which typically + # contain a newline. We need to send a signal back to the processing + # loop that the moIsBeginning flag needs to be turned on again. + # + # [0] is the content, [1] is that flag. + local "$1" && moIndirectArray "$1" "$moContent" "$moIsBeginning" +} + + +# Internal: Show an environment variable or the output of a function to +# stdout. +# +# Limit/prefix any variables used. +# +# $1 - Name of environment variable or function +# $2 - Current context +# $3 - Arguments string if $1 is a function +# +# Returns nothing. +moShow() { + # Namespace these variables + local moJoined moNameParts moContent + + if moIsFunction "$1"; then + moCallFunction moContent "$1" "" "$3" + moParse "$moContent" "$2" false + return 0 + fi + + moSplit moNameParts "$1" "." + + if [[ -z "${moNameParts[1]-}" ]]; then + if moIsArray "$1"; then + eval moJoin moJoined "," "\${$1[@]}" + echo -n "$moJoined" + else + # shellcheck disable=SC2031 + if moTestVarSet "$1"; then + echo -n "${!1}" + elif [[ -n "${MO_FAIL_ON_UNSET-}" ]]; then + echo "Env variable not set: $1" >&2 + exit 1 + fi + fi + else + # Further subindexes are disallowed + eval "echo -n \"\${${moNameParts[0]}[${moNameParts[1]%%.*}]}\"" + fi +} + + +# Internal: Split a larger string into an array. +# +# $1 - Destination variable +# $2 - String to split +# $3 - Starting delimiter +# $4 - Ending delimiter (optional) +# +# Returns nothing. +moSplit() { + local pos result + + result=( "$2" ) + moFindString pos "${result[0]}" "$3" + + if [[ "$pos" -ne -1 ]]; then + # The first delimiter was found + result[1]=${result[0]:$pos + ${#3}} + result[0]=${result[0]:0:$pos} + + if [[ -n "${4-}" ]]; then + moFindString pos "${result[1]}" "$4" + + if [[ "$pos" -ne -1 ]]; then + # The second delimiter was found + result[2]="${result[1]:$pos + ${#4}}" + result[1]="${result[1]:0:$pos}" + fi + fi + fi + + local "$1" && moIndirectArray "$1" "${result[@]}" +} + + +# Internal: Handle the content for a standalone tag. This means removing +# whitespace (not newlines) before a tag and whitespace and a newline after +# a tag. That is, assuming, that the line is otherwise empty. +# +# $1 - Name of destination "content" variable. +# $2 - Content before the tag that was not yet written +# $3 - Tag content (not used) +# $4 - Content after the tag +# $5 - true/false: is this the beginning of the content? +# +# Returns nothing. +moStandaloneAllowed() { + local bytes + + if moIsStandalone bytes "$2" "$4" "$5"; then + # shellcheck disable=SC2206 + bytes=( $bytes ) + echo -n "${2:0:${bytes[0]}}" + local "$1" && moIndirect "$1" "${4:${bytes[1]}}" + else + echo -n "$2" + local "$1" && moIndirect "$1" "$4" + fi +} + + +# Internal: Handle the content for a tag that is never "standalone". No +# adjustments are made for newlines and whitespace. +# +# $1 - Name of destination "content" variable. +# $2 - Content before the tag that was not yet written +# $3 - Tag content (not used) +# $4 - Content after the tag +# +# Returns nothing. +moStandaloneDenied() { + echo -n "$2" + local "$1" && moIndirect "$1" "$4" +} + + +# Internal: Determines if the named thing is a function or if it is a +# non-empty environment variable. When MO_FALSE_IS_EMPTY is set to a +# non-empty value, then "false" is also treated is an empty value. +# +# Do not use variables without prefixes here if possible as this needs to +# check if any name exists in the environment +# +# $1 - Name of environment variable or function +# $2 - Current value (our context) +# MO_FALSE_IS_EMPTY - When set to a non-empty value, this will say the +# string value "false" is empty. +# +# Returns 0 if the name is not empty, 1 otherwise. When MO_FALSE_IS_EMPTY +# is set, this returns 1 if the name is "false". +moTest() { + # Test for functions + moIsFunction "$1" && return 0 + + if moIsArray "$1"; then + # Arrays must have at least 1 element + eval "[[ \"\${#${1}[@]}\" -gt 0 ]]" && return 0 + else + # If MO_FALSE_IS_EMPTY is set, then return 1 if the value of + # the variable is "false". + # shellcheck disable=SC2031 + [[ -n "${MO_FALSE_IS_EMPTY-}" ]] && [[ "${!1-}" == "false" ]] && return 1 + + # Environment variables must not be empty + [[ -n "${!1-}" ]] && return 0 + fi + + return 1 +} + +# Internal: Determine if a variable is assigned, even if it is assigned an empty +# value. +# +# $1 - Variable name to check. +# +# Returns true (0) if the variable is set, 1 if the variable is unset. +moTestVarSet() { + [[ "${!1-a}" == "${!1-b}" ]] +} + + +# Internal: Trim the leading whitespace only. +# +# $1 - Name of destination variable +# $2 - The string +# $3 - true/false - trim front? +# $4 - true/false - trim end? +# $5-@ - Characters to trim +# +# Returns nothing. +moTrimChars() { + local back current front last target varName + + target=$1 + current=$2 + front=$3 + back=$4 + last="" + shift 4 # Remove target, string, trim front flag, trim end flag + + while [[ "$current" != "$last" ]]; do + last=$current + + for varName in "$@"; do + $front && current="${current/#$varName}" + $back && current="${current/%$varName}" + done + done + + local "$target" && moIndirect "$target" "$current" +} + + +# Internal: Trim leading and trailing whitespace from a string. +# +# $1 - Name of variable to store trimmed string +# $2 - The string +# +# Returns nothing. +moTrimWhitespace() { + local result + + moTrimChars result "$2" true true $'\r' $'\n' $'\t' " " + local "$1" && moIndirect "$1" "$result" +} + + +# Internal: Displays the usage for mo. Pulls this from the file that +# contained the `mo` function. Can only work when the right filename +# comes is the one argument, and that only happens when `mo` is called +# with `$0` set to this file. +# +# $1 - Filename that has the help message +# +# Returns nothing. +moUsage() { + grep '^#/' "${MO_ORIGINAL_COMMAND}" | cut -c 4- + echo "" + echo "MO_VERSION=$MO_VERSION" +} + + +# Save the original command's path for usage later +MO_ORIGINAL_COMMAND="$(cd "${BASH_SOURCE[0]%/*}" || exit 1; pwd)/${BASH_SOURCE[0]##*/}" +MO_VERSION="2.2.0" + +# If sourced, load all functions. +# If executed, perform the actions as expected. +if [[ "$0" == "${BASH_SOURCE[0]}" ]] || [[ -z "${BASH_SOURCE[0]}" ]]; then + mo "$@" +fi diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf index 2ce92cef..c1f92d92 100644 --- a/docker/nginx/nginx.conf +++ b/docker/nginx/nginx.conf @@ -34,7 +34,6 @@ events { worker_connections 8192; } - http { include mime.types; default_type application/octet-stream; @@ -44,7 +43,7 @@ http { '"$http_user_agent" $upstream_response_time ' '$upstream_bytes_sent $upstream_bytes_received ' '"$upstream_http_content_type" "$upstream_cache_status" ' - '"$portal_domain" "$sent_http_skynet_skylink" ' + '"$server_alias" "$sent_http_skynet_skylink" ' '$upstream_connect_time $upstream_header_time ' '$request_time "$hns_domain"'; @@ -64,18 +63,61 @@ http { #keepalive_timeout 0; keepalive_timeout 65; - #gzip on; - # globally enable http 1.1 on all proxied requests # http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_http_version proxy_http_version 1.1; + # proxy cache definition + proxy_cache_path /data/nginx/cache levels=1:2 keys_zone=skynet:10m max_size=50g inactive=48h use_temp_path=off; + + # this runs before forking out nginx worker processes + init_by_lua_block { + require "cjson" + require "resty.http" + } + # include skynet-portal-api and skynet-server-api header on every request header_filter_by_lua_block { ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") } + # ratelimit specified IPs + geo $limit { + default 0; + include /etc/nginx/conf.d/include/ratelimited; + } + + map $limit $limit_key { + 0 ""; + 1 $binary_remote_addr; + } + + limit_req_zone $binary_remote_addr zone=uploads_by_ip:10m rate=10r/s; + limit_req_zone $limit_key zone=uploads_by_ip_throttled:10m rate=10r/m; + + limit_req_zone $binary_remote_addr zone=registry_access_by_ip:10m rate=60r/m; + limit_req_zone $limit_key zone=registry_access_by_ip_throttled:10m rate=20r/m; + + limit_conn_zone $binary_remote_addr zone=upload_conn:10m; + limit_conn_zone $limit_key zone=upload_conn_rl:10m; + + limit_conn_zone $binary_remote_addr zone=downloads_by_ip:10m; + + limit_req_status 429; + limit_conn_status 429; + + # Add X-Forwarded-* headers + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + + # skynet-jwt contains dash so we cannot use $cookie_skynet-jwt + # https://richardhart.me/2012/03/18/logging-nginx-cookies-with-dashes/ + map $http_cookie $skynet_jwt { + default ''; + ~skynet-jwt=(?[^\;]+) $match; + } + include /etc/nginx/conf.d/*.conf; include /etc/nginx/conf.extra.d/*.conf; } diff --git a/docker/nginx/nginx.vh.default.conf b/docker/nginx/nginx.vh.default.conf deleted file mode 100644 index d4aa8d5a..00000000 --- a/docker/nginx/nginx.vh.default.conf +++ /dev/null @@ -1,58 +0,0 @@ -# nginx.vh.default.conf -- docker-openresty -# -# This file is installed to: -# `/etc/nginx/conf.d/default.conf` -# -# It tracks the `server` section of the upstream OpenResty's `nginx.conf`. -# -# This config (and any other configs in `etc/nginx/conf.d/`) is loaded by -# default by the `include` directive in `/usr/local/openresty/nginx/conf/nginx.conf`. -# -# See https://github.com/openresty/docker-openresty/blob/master/README.md#nginx-config-files -# - - -server { - listen 80; - server_name localhost; - - #charset koi8-r; - #access_log /var/log/nginx/host.access.log main; - - location / { - root /usr/local/openresty/nginx/html; - index index.html index.htm; - } - - #error_page 404 /404.html; - - # redirect server error pages to the static page /50x.html - # - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/local/openresty/nginx/html; - } - - # proxy the PHP scripts to Apache listening on 127.0.0.1:80 - # - #location ~ \.php$ { - # proxy_pass http://127.0.0.1; - #} - - # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000 - # - #location ~ \.php$ { - # root /usr/local/openresty/nginx/html; - # fastcgi_pass 127.0.0.1:9000; - # fastcgi_index index.php; - # fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name; - # include fastcgi_params; - #} - - # deny access to .htaccess files, if Apache's document root - # concurs with nginx's one - # - #location ~ /\.ht { - # deny all; - #} -} \ No newline at end of file diff --git a/packages/dnslink-api/src/index.js b/packages/dnslink-api/src/index.js index 93e5cdeb..fdddf9aa 100644 --- a/packages/dnslink-api/src/index.js +++ b/packages/dnslink-api/src/index.js @@ -16,8 +16,8 @@ const dnslinkSkylinkRegExp = new RegExp(`^dnslink=/${dnslinkNamespace}/([a-zA-Z0 const hint = `valid example: dnslink=/${dnslinkNamespace}/3ACpC9Umme41zlWUgMQh1fw0sNwgWwyfDDhRQ9Sppz9hjQ`; server.get("/dnslink/:name", async (req, res) => { - const success = (skylink) => res.set("Skynet-Skylink", skylink).send(skylink); - const failure = (message) => res.status(400).set("Dnslink-Error", message).send(message); + const success = (skylink) => res.send(skylink); + const failure = (message) => res.status(400).send(message); if (!isValidDomain(req.params.name)) { return failure(`"${req.params.name}" is not a valid domain`); diff --git a/packages/health-check/Dockerfile b/packages/health-check/Dockerfile index 4f66027b..3bec6201 100644 --- a/packages/health-check/Dockerfile +++ b/packages/health-check/Dockerfile @@ -21,13 +21,14 @@ EXPOSE 3100 ENV NODE_ENV production # 1. start dnsmasq in the background with: -# - alias to siasky.net with current server ip so it overrides load balancer request +# - alias PORTAL_DOMAIN with current server ip so it overrides potential load balancer request # - default docker nameserver 127.0.0.11 for any other request # 2. replace docker nameserver with dnsmasq nameserver in /etc/resolv.conf # 3. start crond in the background to schedule periodic health checks # 4. start the health-check api service CMD [ "sh", "-c", \ - "dnsmasq --no-resolv --log-facility=/var/log/dnsmasq.log --address=/siasky.net/$(node src/whatismyip.js) --server=127.0.0.11 ; \ + "serverip=$(node src/whatismyip.js) ; \ + dnsmasq --no-resolv --log-facility=/var/log/dnsmasq.log --address=/$PORTAL_DOMAIN/$serverip --server=127.0.0.11 ; \ echo \"$(sed 's/127.0.0.11/127.0.0.1/' /etc/resolv.conf)\" > /etc/resolv.conf ; \ crond ; \ node src/index.js" \ diff --git a/setup-scripts/README.md b/setup-scripts/README.md index 05f64c28..e6ada579 100644 --- a/setup-scripts/README.md +++ b/setup-scripts/README.md @@ -84,7 +84,8 @@ At this point we have almost everything running, we just need to set up your wal 1. edit `/home/user/skynet-webportal/.env` and configure following environment variables - - `SSL_CERTIFICATE_STRING` is a list of comma separated paths that caddy will generate ssl certificates for + - `PORTAL_DOMAIN` (required) is a skynet portal domain (ex. siasky.net) + - `SERVER_DOMAIN` (optional) is an optional direct server domain (ex. eu-ger-1.siasky.net) - leave blank unless it is different than PORTAL_DOMAIN - `EMAIL_ADDRESS` is your email address used for communication regarding SSL certification (required if you're using http-01 challenge) - `SIA_WALLET_PASSWORD` is your wallet password (or seed if you did not set a password) - `HSD_API_KEY` this is a random security key for a handshake integration that gets generated automatically @@ -106,7 +107,6 @@ At this point we have almost everything running, we just need to set up your wal with path to the location in the bucket where we want to store the daily backups. 1. `docker-compose up -d` to restart the services so they pick up new env variables -1. `docker exec caddy caddy reload --config /etc/caddy/Caddyfile` to reload Caddyfile configuration 1. add your custom Kratos configuration to `/home/user/skynet-webportal/docker/kratos/config/kratos.yml` (in particular, the credentials for your mail server should be here, rather than in your source control). For a starting point you can take `docker/kratos/config/kratos.yml.sample`. ## Subdomains @@ -119,78 +119,6 @@ There is also an option to access handshake domain through the subdomain using ` To configure this on your portal, you have to make sure to configure the following: -### Wildcard SSL Certificate - -We need to ensure SSL encryption for skapps that are accessed through their -subdomain, therefore we need to have a wildcard certificate. This is very easily -achieved using wildcard certificates in Caddy. - -``` -{$SSL_CERTIFICATE_STRING} { - ... -} -``` - -Where `SSL_CERTIFICATE_STRING` environment variable should contain the wildcard for subdomains (ie. _.example.com) and -wildcard for hns subdomains (ie. _.hns.example.com). - -(see [docker/caddy/Caddyfile](../docker/Caddy/Caddyfile)) - -### Nginx configuration - -In Nginx two things need to happen: - -#### Match the specific parts of the uri - -``` -# understand the regex https://regex101.com/r/BGQvi6/6 -server_name "~^(((?([a-z0-9]{55}))|(?[^\.]+)\.hns)\.)?((?[^.]+)\.)?(?[^.]+)\.(?[^.]+)$"; -``` - -#### Redirect the requests to the appropriate location - -First you need to redirect the requests based on the regex above matching either `base32_subdomain` or `hns_domain`. - -``` -location / { - # This is the only safe workaround to reroute based on some conditions - # See https://www.nginx.com/resources/wiki/start/topics/depth/ifisevil/ - recursive_error_pages on; - - # redirect links with base32 encoded skylink in subdomain - error_page 418 = @base32_subdomain; - if ($base32_subdomain != "") { - return 418; - } - - # redirect links with handshake domain on hns subdomain - error_page 419 = @hns_domain; - if ($hns_domain != "") { - return 419; - } - - ... -} -``` - -Define locations for `@base32_subdomain` and `@hns_domain` redirects. - -``` -location @base32_subdomain { - include /etc/nginx/conf.d/include/proxy-buffer; - - proxy_pass http://127.0.0.1/$base32_subdomain/$request_uri; -} - -location @hns_domain { - include /etc/nginx/conf.d/include/proxy-buffer; - - proxy_pass http://127.0.0.1/hns/$hns_domain/$request_uri; -} -``` - -(see [docker/nginx/nginx.conf](../docker/nginx/nginx.conf)) - ## Useful Commands - Starting the whole stack diff --git a/setup-scripts/setup-docker-services.sh b/setup-scripts/setup-docker-services.sh index 135326ba..a616509d 100755 --- a/setup-scripts/setup-docker-services.sh +++ b/setup-scripts/setup-docker-services.sh @@ -21,7 +21,8 @@ sudo chmod +x /usr/local/bin/docker-compose docker-compose --version # sanity check # Create dummy .env file for docker-compose usage with variables -# * SSL_CERTIFICATE_STRING - certificate string that will be used to generate ssl certificates, read more in docker/caddy/Caddyfile +# * PORTAL_DOMAIN - (required) is a skynet portal domain (ex. siasky.net) +# * SERVER_DOMAIN - (optional) is an optional direct server domain (ex. eu-ger-1.siasky.net) - leave blank unless it is different than PORTAL_DOMAIN # * SKYNET_PORTAL_API - absolute url to the portal api ie. https://siasky.net (general portal address) # * SKYNET_SERVER_API - absolute url to the server api ie. https://eu-ger-1.siasky.net (direct server address, if this is single server portal use the same address as SKYNET_PORTAL_API) # * SKYNET_DASHBOARD_URL - (optional) absolute url to the portal dashboard ie. https://account.siasky.net @@ -46,7 +47,7 @@ docker-compose --version # sanity check # * CR_CLUSTER_NODES - (optional) if using `accounts` the list of servers (with ports) which make up your CockroachDB cluster, e.g. `helsinki.siasky.net:26257,germany.siasky.net:26257,us-east.siasky.net:26257` if ! [ -f /home/user/skynet-webportal/.env ]; then HSD_API_KEY=$(openssl rand -base64 32) # generate safe random key for handshake - printf "SSL_CERTIFICATE_STRING=siasky.net, *.siasky.net, *.hns.siasky.net\nSKYNET_PORTAL_API=https://siasky.net\nSKYNET_SERVER_API=https://eu-dc-1.siasky.net\nSKYNET_DASHBOARD_URL=https://account.example.com\nEMAIL_ADDRESS=email@example.com\nSIA_WALLET_PASSWORD=\nHSD_API_KEY=${HSD_API_KEY}\nCLOUDFLARE_AUTH_TOKEN=\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\nPORTAL_NAME=\DISCORD_WEBHOOK_URL=\nDISCORD_MENTION_USER_ID=\nDISCORD_MENTION_ROLE_ID=\n" > /home/user/skynet-webportal/.env + printf "PORTAL_DOMAIN=siasky.net\nSERVER_DOMAIN=\nSKYNET_PORTAL_API=https://siasky.net\nSKYNET_SERVER_API=https://eu-dc-1.siasky.net\nSKYNET_DASHBOARD_URL=https://account.example.com\nEMAIL_ADDRESS=email@example.com\nSIA_WALLET_PASSWORD=\nHSD_API_KEY=${HSD_API_KEY}\nCLOUDFLARE_AUTH_TOKEN=\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\nPORTAL_NAME=\DISCORD_WEBHOOK_URL=\nDISCORD_MENTION_USER_ID=\nDISCORD_MENTION_ROLE_ID=\n" > /home/user/skynet-webportal/.env fi # Start docker container with nginx and client