diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 05669038..beb0a9e2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,48 +3,48 @@ updates: - package-ecosystem: npm directory: "/packages/dashboard" schedule: - interval: weekly + interval: monthly - package-ecosystem: npm directory: "/packages/dnslink-api" schedule: - interval: weekly + interval: monthly - package-ecosystem: npm directory: "/packages/handshake-api" schedule: - interval: weekly + interval: monthly - package-ecosystem: npm directory: "/packages/health-check" schedule: - interval: weekly + interval: monthly - package-ecosystem: npm directory: "/packages/website" schedule: - interval: weekly + interval: monthly - package-ecosystem: docker directory: "/docker/nginx" schedule: - interval: weekly + interval: monthly - package-ecosystem: docker directory: "/docker/sia" schedule: - interval: weekly + interval: monthly - package-ecosystem: docker directory: "/packages/dashboard" schedule: - interval: weekly + interval: monthly - package-ecosystem: docker directory: "/packages/dnslink-api" schedule: - interval: weekly + interval: monthly - package-ecosystem: docker directory: "/packages/handshake-api" schedule: - interval: weekly + interval: monthly - package-ecosystem: docker directory: "/packages/health-check" schedule: - interval: weekly + interval: monthly - package-ecosystem: docker directory: "/packages/website" schedule: - interval: weekly + interval: monthly diff --git a/.github/workflows/lint-dockerfiles.yml b/.github/workflows/lint-dockerfiles.yml new file mode 100644 index 00000000..afdd6558 --- /dev/null +++ b/.github/workflows/lint-dockerfiles.yml @@ -0,0 +1,29 @@ +name: Dockerfile Lint + +on: + push: + branches: + - master + pull_request: + +jobs: + hadolint: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dockerfile: + - docker/nginx/Dockerfile + - docker/nginx/testing/Dockerfile + - docker/sia/Dockerfile + - packages/dashboard/Dockerfile + - packages/dashboard-v2/Dockerfile + - packages/dnslink-api/Dockerfile + - packages/handshake-api/Dockerfile + - packages/health-check/Dockerfile + - packages/website/Dockerfile + steps: + - uses: actions/checkout@v3 + - uses: hadolint/hadolint-action@v2.0.0 + with: + dockerfile: ${{ matrix.dockerfile }} diff --git a/.github/workflows/nginx-lua-unit-tests.yml b/.github/workflows/nginx-lua-unit-tests.yml index b86a4e04..5af9c101 100644 --- a/.github/workflows/nginx-lua-unit-tests.yml +++ b/.github/workflows/nginx-lua-unit-tests.yml @@ -6,48 +6,41 @@ name: Nginx Lua Unit Tests on: push: branches: - - "master" + - master paths: - - ".github/workflows/nginx-lua-unit-tests.yml" - - "docker/nginx/libs/**.lua" + - docker/nginx/libs/** pull_request: paths: - - ".github/workflows/nginx-lua-unit-tests.yml" - - "docker/nginx/libs/**.lua" + - docker/nginx/libs/** jobs: - build: + test: runs-on: ubuntu-latest - + container: openresty/openresty:1.19.9.1-focal steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: "3.x" - architecture: "x64" + - uses: actions/checkout@v3 - name: Install Dependencies run: | - pip install hererocks - hererocks env --lua=5.1 -rlatest - source env/bin/activate + luarocks install lua-resty-http + luarocks install hasher luarocks install busted luarocks install luacov - luarocks install hasher luarocks install luacheck - - name: Lint code - run: | - source env/bin/activate - luacheck docker/nginx/libs --std ngx_lua+busted + - name: Lint Code With Luacheck + run: luacheck docker/nginx/libs --std ngx_lua+busted - - name: Unit Tests - run: | - source env/bin/activate - busted --verbose --coverage --pattern=spec --directory=docker/nginx/libs . - cd docker/nginx/libs && luacov + - name: Run Tests With Busted + # ran from root repo directory; produces luacov.stats.out file + run: docker/nginx/testing/rbusted --lpath='docker/nginx/libs/?.lua;docker/nginx/libs/?/?.lua' --verbose --coverage --pattern=spec docker/nginx/libs - - uses: codecov/codecov-action@v2 + - name: Generate Code Coverage Report With Luacov + # requires config file in cwd; produces luacov.report.out file + run: cp docker/nginx/testing/.luacov . && luacov && rm .luacov + + - uses: codecov/codecov-action@v3 with: - directory: docker/nginx/libs + root_dir: ${GITHUB_WORKSPACE} + files: ./luacov.report.out flags: nginx-lua diff --git a/.gitignore b/.gitignore index 4b85194e..8a98ee28 100644 --- a/.gitignore +++ b/.gitignore @@ -86,6 +86,10 @@ __pycache__ /.idea/ /venv* +# Luacov file +luacov.stats.out +luacov.report.out + # Setup-script log files setup-scripts/serverload.log setup-scripts/serverload.json diff --git a/README.md b/README.md index 4c27d268..e2c1714f 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,10 @@ ## Latest Setup Documentation Latest Skynet Webportal setup documentation and the setup process Skynet Labs -supports is located at https://docs.siasky.net/webportal-management/overview. +supports is located at https://portal-docs.skynetlabs.com/. -Some of the scripts and setup documentation contained in this repository -(`skynet-webportal`) can be outdated and generally should not be used. +Some scripts and setup documentation contained in this repository +(`skynet-webportal`) may be outdated and generally should not be used. ## Web application @@ -35,7 +35,7 @@ For the purposes of complying with our code license, you can use the following S `fb6c9320bc7e01fbb9cd8d8c3caaa371386928793c736837832e634aaaa484650a3177d6714a` ## Running a Portal -For those interested in running a Webportal, head over to our developer docs [here](https://docs.siasky.net/webportal-management/overview.) to learn more. +For those interested in running a Webportal, head over to our developer docs [here](https://portal-docs.skynetlabs.com/) to learn more. ## Contributing diff --git a/changelog/items/bugs-fixed/fix-context.md b/changelog/items/bugs-fixed/fix-context.md new file mode 100644 index 00000000..e94c8e6e --- /dev/null +++ b/changelog/items/bugs-fixed/fix-context.md @@ -0,0 +1,2 @@ +- Fix `dashboard-v2` Dockerfile context in `docker-compose.accounts.yml` to + avoid Ansible deploy (docker compose build) `permission denied` issues. \ No newline at end of file diff --git a/docker-compose.abuse-scanner.yml b/docker-compose.abuse-scanner.yml index 4edb6556..2eae443b 100644 --- a/docker-compose.abuse-scanner.yml +++ b/docker-compose.abuse-scanner.yml @@ -10,7 +10,7 @@ services: abuse-scanner: # uncomment "build" and comment out "image" to build from sources # build: https://github.com/SkynetLabs/abuse-scanner.git#main - image: skynetlabs/abuse-scanner + image: skynetlabs/abuse-scanner:0.1.1 container_name: abuse-scanner restart: unless-stopped logging: *default-logging diff --git a/docker-compose.accounts.yml b/docker-compose.accounts.yml index 5d9d345c..99b10ff4 100644 --- a/docker-compose.accounts.yml +++ b/docker-compose.accounts.yml @@ -55,9 +55,11 @@ services: - mongo dashboard: - build: - context: ./packages/dashboard - dockerfile: Dockerfile + # uncomment "build" and comment out "image" to build from sources + # build: + # context: https://github.com/SkynetLabs/skynet-webportal.git#master + # dockerfile: ./packages/dashboard/Dockerfile + image: skynetlabs/dashboard container_name: dashboard restart: unless-stopped logging: *default-logging @@ -75,3 +77,26 @@ services: - 3000 depends_on: - mongo + + dashboard-v2: + build: + context: ./packages/dashboard-v2 + dockerfile: Dockerfile + container_name: dashboard-v2 + restart: unless-stopped + logging: *default-logging + env_file: + - .env + environment: + - GATSBY_PORTAL_DOMAIN=${PORTAL_DOMAIN} + - GATSBY_STRIPE_PUBLISHABLE_KEY=${STRIPE_PUBLISHABLE_KEY} + volumes: + - ./docker/data/dashboard-v2/.cache:/usr/app/.cache + - ./docker/data/dashboard-v2/public:/usr/app/public + networks: + shared: + ipv4_address: 10.10.10.86 + expose: + - 9000 + depends_on: + - mongo diff --git a/docker-compose.blocker.yml b/docker-compose.blocker.yml index edcb45c0..db398805 100644 --- a/docker-compose.blocker.yml +++ b/docker-compose.blocker.yml @@ -15,7 +15,7 @@ services: blocker: # uncomment "build" and comment out "image" to build from sources # build: https://github.com/SkynetLabs/blocker.git#main - image: skynetlabs/blocker + image: skynetlabs/blocker:0.1.1 container_name: blocker restart: unless-stopped logging: *default-logging diff --git a/docker-compose.malware-scanner.yml b/docker-compose.malware-scanner.yml index fba60f98..ee3b6700 100644 --- a/docker-compose.malware-scanner.yml +++ b/docker-compose.malware-scanner.yml @@ -28,7 +28,7 @@ services: malware-scanner: # uncomment "build" and comment out "image" to build from sources # build: https://github.com/SkynetLabs/malware-scanner.git#main - image: skynetlabs/malware-scanner + image: skynetlabs/malware-scanner:0.1.0 container_name: malware-scanner restart: unless-stopped logging: *default-logging diff --git a/docker-compose.mongodb.yml b/docker-compose.mongodb.yml index 610d5308..e8eb9aca 100644 --- a/docker-compose.mongodb.yml +++ b/docker-compose.mongodb.yml @@ -15,7 +15,7 @@ services: mongo: image: mongo:4.4.1 - command: --keyFile=/data/mgkey --replSet=${SKYNET_DB_REPLICASET:-skynet} + command: --keyFile=/data/mgkey --replSet=${SKYNET_DB_REPLICASET:-skynet} --setParameter ShardingTaskExecutorPoolMinSize=10 container_name: mongo restart: unless-stopped logging: *default-logging diff --git a/docker-compose.yml b/docker-compose.yml index 2879b8b6..6450a10f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,9 +54,11 @@ services: - ./docker/data/certbot:/etc/letsencrypt nginx: - build: - context: ./docker/nginx - dockerfile: Dockerfile + # uncomment "build" and comment out "image" to build from sources + # build: + # context: https://github.com/SkynetLabs/skynet-webportal.git#master + # dockerfile: ./docker/nginx/Dockerfile + image: skynetlabs/nginx container_name: nginx restart: unless-stopped logging: *default-logging @@ -69,6 +71,10 @@ services: - ./docker/data/nginx/skynet:/data/nginx/skynet:ro - ./docker/data/sia/apipassword:/data/sia/apipassword:ro - ./docker/data/certbot:/etc/letsencrypt + - ./docker/nginx/libs:/etc/nginx/libs + - ./docker/nginx/conf.d:/etc/nginx/conf.d + - ./docker/nginx/conf.d.templates:/etc/nginx/templates + - ./docker/nginx/nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf networks: shared: ipv4_address: 10.10.10.30 @@ -82,9 +88,11 @@ services: - website website: - build: - context: ./packages/website - dockerfile: Dockerfile + # uncomment "build" and comment out "image" to build from sources + # build: + # context: https://github.com/SkynetLabs/skynet-webportal.git#master + # dockerfile: ./packages/website/Dockerfile + image: skynetlabs/website container_name: website restart: unless-stopped logging: *default-logging @@ -118,9 +126,11 @@ services: - 12037 handshake-api: - build: - context: ./packages/handshake-api - dockerfile: Dockerfile + # uncomment "build" and comment out "image" to build from sources + # build: + # context: https://github.com/SkynetLabs/skynet-webportal.git#master + # dockerfile: ./packages/handshake-api/Dockerfile + image: skynetlabs/handshake-api container_name: handshake-api restart: unless-stopped logging: *default-logging @@ -140,9 +150,11 @@ services: - handshake dnslink-api: - build: - context: ./packages/dnslink-api - dockerfile: Dockerfile + # uncomment "build" and comment out "image" to build from sources + # build: + # context: https://github.com/SkynetLabs/skynet-webportal.git#master + # dockerfile: ./packages/dnslink-api/Dockerfile + image: skynetlabs/dnslink-api container_name: dnslink-api restart: unless-stopped logging: *default-logging @@ -153,9 +165,11 @@ services: - 3100 health-check: - build: - context: ./packages/health-check - dockerfile: Dockerfile + # uncomment "build" and comment out "image" to build from sources + # build: + # context: https://github.com/SkynetLabs/skynet-webportal.git#master + # dockerfile: ./packages/health-check/Dockerfile + image: skynetlabs/health-check container_name: health-check restart: unless-stopped logging: *default-logging diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile index 2093872c..f35c0aff 100644 --- a/docker/nginx/Dockerfile +++ b/docker/nginx/Dockerfile @@ -1,24 +1,21 @@ FROM openresty/openresty:1.19.9.1-focal -RUN luarocks install lua-resty-http && \ - luarocks install hasher && \ - openssl req -new -newkey rsa:2048 -days 3650 -nodes -x509 \ - -subj '/CN=local-certificate' \ - -keyout /etc/ssl/local-certificate.key \ - -out /etc/ssl/local-certificate.crt +WORKDIR / -COPY mo ./ -COPY libs /etc/nginx/libs -COPY conf.d /etc/nginx/conf.d -COPY conf.d.templates /etc/nginx/conf.d.templates -COPY nginx.conf /usr/local/openresty/nginx/conf/nginx.conf +RUN apt-get update && apt-get --no-install-recommends -y install bc=1.07.1-2build1 && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + luarocks install lua-resty-http && \ + luarocks install hasher -CMD [ "bash", "-c", \ - "./mo < /etc/nginx/conf.d.templates/server.account.conf > /etc/nginx/conf.d/server.account.conf ; \ - ./mo < /etc/nginx/conf.d.templates/server.api.conf > /etc/nginx/conf.d/server.api.conf; \ - ./mo < /etc/nginx/conf.d.templates/server.dnslink.conf > /etc/nginx/conf.d/server.dnslink.conf; \ - ./mo < /etc/nginx/conf.d.templates/server.hns.conf > /etc/nginx/conf.d/server.hns.conf; \ - ./mo < /etc/nginx/conf.d.templates/server.skylink.conf > /etc/nginx/conf.d/server.skylink.conf ; \ - while :; do sleep 6h & wait ${!}; /usr/local/openresty/bin/openresty -s reload; done & \ - /usr/local/openresty/bin/openresty '-g daemon off;'" \ - ] +# reload nginx every 6 hours (for reloading certificates) +ENV NGINX_ENTRYPOINT_RELOAD_EVERY_X_HOURS 6 + +# copy entrypoint and entrypoint scripts +COPY docker/nginx/docker-entrypoint.sh / +COPY docker/nginx/docker-entrypoint.d /docker-entrypoint.d + +ENTRYPOINT ["/docker-entrypoint.sh"] + +STOPSIGNAL SIGQUIT + +CMD ["nginx", "-g", "daemon off;"] diff --git a/docker/nginx/conf.d.templates/server.account.conf b/docker/nginx/conf.d.templates/server.account.conf deleted file mode 100644 index af3b7c4d..00000000 --- a/docker/nginx/conf.d.templates/server.account.conf +++ /dev/null @@ -1,45 +0,0 @@ -{{#ACCOUNTS_ENABLED}} - {{#PORTAL_DOMAIN}} - server { - server_name account.{{PORTAL_DOMAIN}}; # example: account.siasky.net - - include /etc/nginx/conf.d/server/server.http; - } - - server { - server_name account.{{PORTAL_DOMAIN}}; # example: account.siasky.net - - set_by_lua_block $skynet_portal_domain { return "{{PORTAL_DOMAIN}}" } - set_by_lua_block $skynet_server_domain { - -- fall back to portal domain if server domain is not defined - if "{{SERVER_DOMAIN}}" == "" then - return "{{PORTAL_DOMAIN}}" - end - return "{{SERVER_DOMAIN}}" - } - - include /etc/nginx/conf.d/server/server.account; - } - {{/PORTAL_DOMAIN}} - - {{#SERVER_DOMAIN}} - server { - server_name account.{{SERVER_DOMAIN}}; # example: account.eu-ger-1.siasky.net - - include /etc/nginx/conf.d/server/server.http; - - set_by_lua_block $server_alias { return string.match("{{SERVER_DOMAIN}}", "^([^.]+)") } - } - - server { - server_name account.{{SERVER_DOMAIN}}; # example: account.eu-ger-1.siasky.net - - set_by_lua_block $skynet_portal_domain { return "{{SERVER_DOMAIN}}" } - set_by_lua_block $skynet_server_domain { return "{{SERVER_DOMAIN}}" } - - include /etc/nginx/conf.d/server/server.account; - - set_by_lua_block $server_alias { return string.match("{{SERVER_DOMAIN}}", "^([^.]+)") } - } - {{/SERVER_DOMAIN}} -{{/ACCOUNTS_ENABLED}} diff --git a/docker/nginx/conf.d.templates/server.account.conf.template b/docker/nginx/conf.d.templates/server.account.conf.template new file mode 100644 index 00000000..8507c407 --- /dev/null +++ b/docker/nginx/conf.d.templates/server.account.conf.template @@ -0,0 +1,44 @@ +server { + server_name account.${PORTAL_DOMAIN}; # example: account.siasky.net + + include /etc/nginx/conf.d/server/server.http; +} + +server { + server_name account.${PORTAL_DOMAIN}; # example: account.siasky.net + + set_by_lua_block $skynet_portal_domain { return "${PORTAL_DOMAIN}" } + set_by_lua_block $skynet_server_domain { + -- fall back to portal domain if server domain is not defined + if "${SERVER_DOMAIN}" == "" then + return "${PORTAL_DOMAIN}" + end + return "${SERVER_DOMAIN}" + } + + include /etc/nginx/conf.d/server/server.account; +} + +server { + server_name account.${SERVER_DOMAIN}; # example: account.eu-ger-1.siasky.net + + include /etc/nginx/conf.d/server/server.http; + + set_by_lua_block $server_alias { return string.match("${SERVER_DOMAIN}", "^([^.]+)") } +} + +server { + server_name account.${SERVER_DOMAIN}; # example: account.eu-ger-1.siasky.net + + set_by_lua_block $skynet_portal_domain { + -- when accessing portal directly through server domain, portal domain should be set to server domain + -- motivation: skynet-js uses Skynet-Portal-Api header (that is set to $skynet_portal_domain) to detect current + -- portal address and it should be server domain when accessing specific server by its domain address + return "${SERVER_DOMAIN}" + } + set_by_lua_block $skynet_server_domain { return "${SERVER_DOMAIN}" } + + include /etc/nginx/conf.d/server/server.account; + + set_by_lua_block $server_alias { return string.match("${SERVER_DOMAIN}", "^([^.]+)") } +} diff --git a/docker/nginx/conf.d.templates/server.api.conf b/docker/nginx/conf.d.templates/server.api.conf deleted file mode 100644 index 591212ba..00000000 --- a/docker/nginx/conf.d.templates/server.api.conf +++ /dev/null @@ -1,43 +0,0 @@ -{{#PORTAL_DOMAIN}} -server { - server_name {{PORTAL_DOMAIN}}; # example: siasky.net - - include /etc/nginx/conf.d/server/server.http; -} - -server { - server_name {{PORTAL_DOMAIN}}; # example: siasky.net - - set_by_lua_block $skynet_portal_domain { return "{{PORTAL_DOMAIN}}" } - set_by_lua_block $skynet_server_domain { - -- fall back to portal domain if server domain is not defined - if "{{SERVER_DOMAIN}}" == "" then - return "{{PORTAL_DOMAIN}}" - end - return "{{SERVER_DOMAIN}}" - } - - include /etc/nginx/conf.d/server/server.api; -} -{{/PORTAL_DOMAIN}} - -{{#SERVER_DOMAIN}} -server { - server_name {{SERVER_DOMAIN}}; # example: eu-ger-1.siasky.net - - include /etc/nginx/conf.d/server/server.http; - - set_by_lua_block $server_alias { return string.match("{{SERVER_DOMAIN}}", "^([^.]+)") } -} - -server { - server_name {{SERVER_DOMAIN}}; # example: eu-ger-1.siasky.net - - set_by_lua_block $skynet_portal_domain { return "{{SERVER_DOMAIN}}" } - set_by_lua_block $skynet_server_domain { return "{{SERVER_DOMAIN}}" } - - include /etc/nginx/conf.d/server/server.api; - - set_by_lua_block $server_alias { return string.match("{{SERVER_DOMAIN}}", "^([^.]+)") } -} -{{/SERVER_DOMAIN}} diff --git a/docker/nginx/conf.d.templates/server.api.conf.template b/docker/nginx/conf.d.templates/server.api.conf.template new file mode 100644 index 00000000..5f742127 --- /dev/null +++ b/docker/nginx/conf.d.templates/server.api.conf.template @@ -0,0 +1,44 @@ +server { + server_name ${PORTAL_DOMAIN}; # example: siasky.net + + include /etc/nginx/conf.d/server/server.http; +} + +server { + server_name ${PORTAL_DOMAIN}; # example: siasky.net + + set_by_lua_block $skynet_portal_domain { return "${PORTAL_DOMAIN}" } + set_by_lua_block $skynet_server_domain { + -- fall back to portal domain if server domain is not defined + if "${SERVER_DOMAIN}" == "" then + return "${PORTAL_DOMAIN}" + end + return "${SERVER_DOMAIN}" + } + + include /etc/nginx/conf.d/server/server.api; +} + +server { + server_name ${SERVER_DOMAIN}; # example: eu-ger-1.siasky.net + + include /etc/nginx/conf.d/server/server.http; + + set_by_lua_block $server_alias { return string.match("${SERVER_DOMAIN}", "^([^.]+)") } +} + +server { + server_name ${SERVER_DOMAIN}; # example: eu-ger-1.siasky.net + + set_by_lua_block $skynet_portal_domain { + -- when accessing portal directly through server domain, portal domain should be set to server domain + -- motivation: skynet-js uses Skynet-Portal-Api header (that is set to $skynet_portal_domain) to detect current + -- portal address and it should be server domain when accessing specific server by its domain address + return "${SERVER_DOMAIN}" + } + set_by_lua_block $skynet_server_domain { return "${SERVER_DOMAIN}" } + + include /etc/nginx/conf.d/server/server.api; + + set_by_lua_block $server_alias { return string.match("${SERVER_DOMAIN}", "^([^.]+)") } +} diff --git a/docker/nginx/conf.d.templates/server.dnslink.conf b/docker/nginx/conf.d.templates/server.dnslink.conf.template similarity index 71% rename from docker/nginx/conf.d.templates/server.dnslink.conf rename to docker/nginx/conf.d.templates/server.dnslink.conf.template index d42ee245..95c623b6 100644 --- a/docker/nginx/conf.d.templates/server.dnslink.conf +++ b/docker/nginx/conf.d.templates/server.dnslink.conf.template @@ -12,13 +12,13 @@ server { ssl_certificate /etc/ssl/local-certificate.crt; ssl_certificate_key /etc/ssl/local-certificate.key; - set_by_lua_block $skynet_portal_domain { return "{{PORTAL_DOMAIN}}" } + set_by_lua_block $skynet_portal_domain { return "${PORTAL_DOMAIN}" } set_by_lua_block $skynet_server_domain { -- fall back to portal domain if server domain is not defined - if "{{SERVER_DOMAIN}}" == "" then - return "{{PORTAL_DOMAIN}}" + if "${SERVER_DOMAIN}" == "" then + return "${PORTAL_DOMAIN}" end - return "{{SERVER_DOMAIN}}" + return "${SERVER_DOMAIN}" } include /etc/nginx/conf.d/server/server.dnslink; diff --git a/docker/nginx/conf.d.templates/server.hns.conf b/docker/nginx/conf.d.templates/server.hns.conf deleted file mode 100644 index 0e4f21f3..00000000 --- a/docker/nginx/conf.d.templates/server.hns.conf +++ /dev/null @@ -1,45 +0,0 @@ -{{#PORTAL_DOMAIN}} -server { - server_name *.hns.{{PORTAL_DOMAIN}}; # example: *.hns.siasky.net - - include /etc/nginx/conf.d/server/server.http; -} - -server { - server_name *.hns.{{PORTAL_DOMAIN}}; # example: *.hns.siasky.net - - set_by_lua_block $skynet_portal_domain { return "{{PORTAL_DOMAIN}}" } - set_by_lua_block $skynet_server_domain { - -- fall back to portal domain if server domain is not defined - if "{{SERVER_DOMAIN}}" == "" then - return "{{PORTAL_DOMAIN}}" - end - return "{{SERVER_DOMAIN}}" - } - - proxy_set_header Host {{PORTAL_DOMAIN}}; - include /etc/nginx/conf.d/server/server.hns; -} -{{/PORTAL_DOMAIN}} - -{{#SERVER_DOMAIN}} -server { - server_name *.hns.{{SERVER_DOMAIN}}; # example: *.hns.eu-ger-1.siasky.net - - include /etc/nginx/conf.d/server/server.http; - - set_by_lua_block $server_alias { return string.match("{{SERVER_DOMAIN}}", "^([^.]+)") } -} - -server { - server_name *.hns.{{SERVER_DOMAIN}}; # example: *.hns.eu-ger-1.siasky.net - - set_by_lua_block $skynet_portal_domain { return "{{SERVER_DOMAIN}}" } - set_by_lua_block $skynet_server_domain { return "{{SERVER_DOMAIN}}" } - - proxy_set_header Host {{SERVER_DOMAIN}}; - include /etc/nginx/conf.d/server/server.hns; - - set_by_lua_block $server_alias { return string.match("{{SERVER_DOMAIN}}", "^([^.]+)") } -} -{{/SERVER_DOMAIN}} diff --git a/docker/nginx/conf.d.templates/server.hns.conf.template b/docker/nginx/conf.d.templates/server.hns.conf.template new file mode 100644 index 00000000..f7bb43fb --- /dev/null +++ b/docker/nginx/conf.d.templates/server.hns.conf.template @@ -0,0 +1,46 @@ +server { + server_name *.hns.${PORTAL_DOMAIN}; # example: *.hns.siasky.net + + include /etc/nginx/conf.d/server/server.http; +} + +server { + server_name *.hns.${PORTAL_DOMAIN}; # example: *.hns.siasky.net + + set_by_lua_block $skynet_portal_domain { return "${PORTAL_DOMAIN}" } + set_by_lua_block $skynet_server_domain { + -- fall back to portal domain if server domain is not defined + if "${SERVER_DOMAIN}" == "" then + return "${PORTAL_DOMAIN}" + end + return "${SERVER_DOMAIN}" + } + + proxy_set_header Host ${PORTAL_DOMAIN}; + include /etc/nginx/conf.d/server/server.hns; +} + +server { + server_name *.hns.${SERVER_DOMAIN}; # example: *.hns.eu-ger-1.siasky.net + + include /etc/nginx/conf.d/server/server.http; + + set_by_lua_block $server_alias { return string.match("${SERVER_DOMAIN}", "^([^.]+)") } +} + +server { + server_name *.hns.${SERVER_DOMAIN}; # example: *.hns.eu-ger-1.siasky.net + + set_by_lua_block $skynet_portal_domain { + -- when accessing portal directly through server domain, portal domain should be set to server domain + -- motivation: skynet-js uses Skynet-Portal-Api header (that is set to $skynet_portal_domain) to detect current + -- portal address and it should be server domain when accessing specific server by its domain address + return "${SERVER_DOMAIN}" + } + set_by_lua_block $skynet_server_domain { return "${SERVER_DOMAIN}" } + + proxy_set_header Host ${SERVER_DOMAIN}; + include /etc/nginx/conf.d/server/server.hns; + + set_by_lua_block $server_alias { return string.match("${SERVER_DOMAIN}", "^([^.]+)") } +} diff --git a/docker/nginx/conf.d.templates/server.skylink.conf b/docker/nginx/conf.d.templates/server.skylink.conf deleted file mode 100644 index a97e240c..00000000 --- a/docker/nginx/conf.d.templates/server.skylink.conf +++ /dev/null @@ -1,43 +0,0 @@ -{{#PORTAL_DOMAIN}} -server { - server_name *.{{PORTAL_DOMAIN}}; # example: *.siasky.net - - include /etc/nginx/conf.d/server/server.http; -} - -server { - server_name *.{{PORTAL_DOMAIN}}; # example: *.siasky.net - - set_by_lua_block $skynet_portal_domain { return "{{PORTAL_DOMAIN}}" } - set_by_lua_block $skynet_server_domain { - -- fall back to portal domain if server domain is not defined - if "{{SERVER_DOMAIN}}" == "" then - return "{{PORTAL_DOMAIN}}" - end - return "{{SERVER_DOMAIN}}" - } - - include /etc/nginx/conf.d/server/server.skylink; -} -{{/PORTAL_DOMAIN}} - -{{#SERVER_DOMAIN}} -server { - server_name *.{{SERVER_DOMAIN}}; # example: *.eu-ger-1.siasky.net - - include /etc/nginx/conf.d/server/server.http; - - set_by_lua_block $server_alias { return string.match("{{SERVER_DOMAIN}}", "^([^.]+)") } -} - -server { - server_name *.{{SERVER_DOMAIN}}; # example: *.eu-ger-1.siasky.net - - set_by_lua_block $skynet_portal_domain { return "{{SERVER_DOMAIN}}" } - set_by_lua_block $skynet_server_domain { return "{{SERVER_DOMAIN}}" } - - include /etc/nginx/conf.d/server/server.skylink; - - set_by_lua_block $server_alias { return string.match("{{SERVER_DOMAIN}}", "^([^.]+)") } -} -{{/SERVER_DOMAIN}} diff --git a/docker/nginx/conf.d.templates/server.skylink.conf.template b/docker/nginx/conf.d.templates/server.skylink.conf.template new file mode 100644 index 00000000..0d337abb --- /dev/null +++ b/docker/nginx/conf.d.templates/server.skylink.conf.template @@ -0,0 +1,44 @@ +server { + server_name *.${PORTAL_DOMAIN}; # example: *.siasky.net + + include /etc/nginx/conf.d/server/server.http; +} + +server { + server_name *.${PORTAL_DOMAIN}; # example: *.siasky.net + + set_by_lua_block $skynet_portal_domain { return "${PORTAL_DOMAIN}" } + set_by_lua_block $skynet_server_domain { + -- fall back to portal domain if server domain is not defined + if "${SERVER_DOMAIN}" == "" then + return "${PORTAL_DOMAIN}" + end + return "${SERVER_DOMAIN}" + } + + include /etc/nginx/conf.d/server/server.skylink; +} + +server { + server_name *.${SERVER_DOMAIN}; # example: *.eu-ger-1.siasky.net + + include /etc/nginx/conf.d/server/server.http; + + set_by_lua_block $server_alias { return string.match("${SERVER_DOMAIN}", "^([^.]+)") } +} + +server { + server_name *.${SERVER_DOMAIN}; # example: *.eu-ger-1.siasky.net + + set_by_lua_block $skynet_portal_domain { + -- when accessing portal directly through server domain, portal domain should be set to server domain + -- motivation: skynet-js uses Skynet-Portal-Api header (that is set to $skynet_portal_domain) to detect current + -- portal address and it should be server domain when accessing specific server by its domain address + return "${SERVER_DOMAIN}" + } + set_by_lua_block $skynet_server_domain { return "${SERVER_DOMAIN}" } + + include /etc/nginx/conf.d/server/server.skylink; + + set_by_lua_block $server_alias { return string.match("${SERVER_DOMAIN}", "^([^.]+)") } +} diff --git a/docker/nginx/conf.d/include/location-skylink b/docker/nginx/conf.d/include/location-skylink index 995a6e2d..b214e3a9 100644 --- a/docker/nginx/conf.d/include/location-skylink +++ b/docker/nginx/conf.d/include/location-skylink @@ -1,5 +1,4 @@ include /etc/nginx/conf.d/include/cors; -include /etc/nginx/conf.d/include/track-download; limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time @@ -37,3 +36,18 @@ proxy_read_timeout 600; proxy_set_header User-Agent: Sia-Agent; proxy_pass http://sia:9980/skynet/skylink/$skylink$path$is_args$args; + +log_by_lua_block { + local skynet_account = require("skynet.account") + local skynet_modules = require("skynet.modules") + local skynet_scanner = require("skynet.scanner") + local skynet_tracker = require("skynet.tracker") + + if skynet_modules.is_enabled("a") then + skynet_tracker.track_download(ngx.header["Skynet-Skylink"], ngx.status, skynet_account.get_auth_headers(), ngx.var.body_bytes_sent) + end + + if skynet_modules.is_enabled("s") then + skynet_scanner.scan_skylink(ngx.header["Skynet-Skylink"]) + end +} diff --git a/docker/nginx/conf.d/include/location-skynet-registry b/docker/nginx/conf.d/include/location-skynet-registry index 33838f70..cd450be9 100644 --- a/docker/nginx/conf.d/include/location-skynet-registry +++ b/docker/nginx/conf.d/include/location-skynet-registry @@ -1,6 +1,5 @@ include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/sia-auth; -include /etc/nginx/conf.d/include/track-registry; limit_req zone=registry_access_by_ip burst=600 nodelay; limit_req zone=registry_access_by_ip_throttled burst=200 nodelay; @@ -30,3 +29,10 @@ access_by_lua_block { end end } + +log_by_lua_block { + local skynet_account = require("skynet.account") + local skynet_tracker = require("skynet.tracker") + + skynet_tracker.track_registry(ngx.status, skynet_account.get_auth_headers(), ngx.req.get_method()) +} diff --git a/docker/nginx/conf.d/include/track-download b/docker/nginx/conf.d/include/track-download deleted file mode 100644 index 4e12fd41..00000000 --- a/docker/nginx/conf.d/include/track-download +++ /dev/null @@ -1,55 +0,0 @@ -log_by_lua_block { - local skynet_account = require("skynet.account") - - -- tracking runs only when request comes from authenticated user - if skynet_account.is_authenticated() then - local function track(premature, skylink, status, body_bytes_sent, auth_headers) - if premature then return end - - local httpc = require("resty.http").new() - local query = table.concat({ "status=" .. status, "bytes=" .. body_bytes_sent }, "&") - - -- 10.10.10.70 points to accounts service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.70:3000/track/download/" .. skylink .. "?" .. query, { - method = "POST", - headers = auth_headers, - }) - - if err or (res and res.status ~= ngx.HTTP_NO_CONTENT) then - local error_response = err or ("[HTTP " .. res.status .. "] " .. res.body) - ngx.log(ngx.ERR, "Failed accounts service request /track/download/" .. skylink .. ": ", error_response) - end - end - - if ngx.header["Skynet-Skylink"] and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then - local auth_headers = skynet_account.get_auth_headers() - local ok, err = ngx.timer.at(0, track, ngx.header["Skynet-Skylink"], ngx.status, ngx.var.body_bytes_sent, auth_headers) - if err then ngx.log(ngx.ERR, "Failed to create timer: ", err) end - end - end - - -- this block runs only when scanner module is enabled - if os.getenv("PORTAL_MODULES"):match("s") then - local function scan(premature, skylink) - if premature then return end - - local httpc = require("resty.http").new() - - -- 10.10.10.101 points to malware-scanner service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.101:4000/scan/" .. skylink, { - method = "POST", - }) - - if err or (res and res.status ~= ngx.HTTP_OK) then - local error_response = err or ("[HTTP " .. res.status .. "] " .. res.body) - ngx.log(ngx.ERR, "Failed malware-scanner request /scan/" .. skylink .. ": ", error_response) - end - end - - -- scan all skylinks but make sure to only run if skylink is present (empty if request failed) - if ngx.header["Skynet-Skylink"] then - local ok, err = ngx.timer.at(0, scan, ngx.header["Skynet-Skylink"]) - if err then ngx.log(ngx.ERR, "Failed to create timer: ", err) end - end - end -} diff --git a/docker/nginx/conf.d/include/track-registry b/docker/nginx/conf.d/include/track-registry deleted file mode 100644 index 2c840491..00000000 --- a/docker/nginx/conf.d/include/track-registry +++ /dev/null @@ -1,33 +0,0 @@ -log_by_lua_block { - local skynet_account = require("skynet.account") - - -- tracking runs only when request comes from authenticated user - if skynet_account.is_authenticated() then - local function track(premature, request_method, auth_headers) - if premature then return end - - local httpc = require("resty.http").new() - - -- based on request method we assign a registry action string used - -- in track endpoint namely "read" for GET and "write" for POST - local registry_action = request_method == "GET" and "read" or "write" - - -- 10.10.10.70 points to accounts service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.70:3000/track/registry/" .. registry_action, { - method = "POST", - headers = auth_headers, - }) - - if err or (res and res.status ~= ngx.HTTP_NO_CONTENT) then - local error_response = err or ("[HTTP " .. res.status .. "] " .. res.body) - ngx.log(ngx.ERR, "Failed accounts service request /track/registry/" .. registry_action .. ": ", error_response) - end - end - - if ngx.status == ngx.HTTP_OK or ngx.status == ngx.HTTP_NOT_FOUND then - local auth_headers = skynet_account.get_auth_headers() - local ok, err = ngx.timer.at(0, track, ngx.req.get_method(), auth_headers) - if err then ngx.log(ngx.ERR, "Failed to create timer: ", err) end - end - end -} diff --git a/docker/nginx/conf.d/include/track-upload b/docker/nginx/conf.d/include/track-upload deleted file mode 100644 index 36b12b9e..00000000 --- a/docker/nginx/conf.d/include/track-upload +++ /dev/null @@ -1,55 +0,0 @@ -log_by_lua_block { - local skynet_account = require("skynet.account") - - -- tracking runs only when request comes from authenticated user - if skynet_account.is_authenticated() then - local function track(premature, skylink, auth_headers) - if premature then return end - - local httpc = require("resty.http").new() - - -- 10.10.10.70 points to accounts service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.70:3000/track/upload/" .. skylink, { - method = "POST", - headers = auth_headers, - }) - - if err or (res and res.status ~= ngx.HTTP_NO_CONTENT) then - local error_response = err or ("[HTTP " .. res.status .. "] " .. res.body) - ngx.log(ngx.ERR, "Failed accounts service request /track/upload/" .. skylink .. ": ", error_response) - end - end - - -- report all skylinks (header empty if request failed) but only if jwt is preset (user is authenticated) - if ngx.header["Skynet-Skylink"] then - local auth_headers = skynet_account.get_auth_headers() - local ok, err = ngx.timer.at(0, track, ngx.header["Skynet-Skylink"], auth_headers) - if err then ngx.log(ngx.ERR, "Failed to create timer: ", err) end - end - end - - -- this block runs only when scanner module is enabled - if os.getenv("PORTAL_MODULES"):match("s") then - local function scan(premature, skylink) - if premature then return end - - local httpc = require("resty.http").new() - - -- 10.10.10.101 points to malware-scanner service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.101:4000/scan/" .. skylink, { - method = "POST", - }) - - if err or (res and res.status ~= ngx.HTTP_OK) then - local error_response = err or ("[HTTP " .. res.status .. "] " .. res.body) - ngx.log(ngx.ERR, "Failed malware-scanner request /scan/" .. skylink .. ": ", error_response) - end - end - - -- scan all skylinks but make sure to only run if skylink is present (empty if request failed) - if ngx.header["Skynet-Skylink"] then - local ok, err = ngx.timer.at(0, scan, ngx.header["Skynet-Skylink"]) - if err then ngx.log(ngx.ERR, "Failed to create timer: ", err) end - end - end -} diff --git a/docker/nginx/conf.d/server/server.account b/docker/nginx/conf.d/server/server.account index 127ba4bf..9d444296 100644 --- a/docker/nginx/conf.d/server/server.account +++ b/docker/nginx/conf.d/server/server.account @@ -3,6 +3,11 @@ listen 443 ssl http2; include /etc/nginx/conf.d/include/ssl-settings; include /etc/nginx/conf.d/include/init-optional-variables; +# Uncomment to launch new Dashboard under /v2 path +# location /v2 { +# proxy_pass http://dashboard-v2:9000; +# } + location / { proxy_pass http://dashboard:3000; } diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index f681cca8..48e7a638 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -206,7 +206,6 @@ location /skynet/registry/subscription { location /skynet/skyfile { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/sia-auth; - include /etc/nginx/conf.d/include/track-upload; include /etc/nginx/conf.d/include/generate-siapath; include /etc/nginx/conf.d/include/portal-access-check; @@ -228,12 +227,31 @@ location /skynet/skyfile { # proxy this call to siad endpoint (make sure the ip is correct) proxy_pass http://sia:9980/skynet/skyfile/$dir1/$dir2/$dir3$is_args$args; + + log_by_lua_block { + local skynet_account = require("skynet.account") + local skynet_modules = require("skynet.modules") + local skynet_scanner = require("skynet.scanner") + local skynet_tracker = require("skynet.tracker") + + if skynet_modules.is_enabled("a") then + skynet_tracker.track_upload( + ngx.header["Skynet-Skylink"], + ngx.status, + skynet_account.get_auth_headers(), + ngx.var.remote_addr + ) + end + + if skynet_modules.is_enabled("s") then + skynet_scanner.scan_skylink(ngx.header["Skynet-Skylink"]) + end + } } # endpoint implementing resumable file uploads open protocol https://tus.io location /skynet/tus { include /etc/nginx/conf.d/include/cors-headers; # include cors headers but do not overwrite OPTIONS response - include /etc/nginx/conf.d/include/track-upload; limit_req zone=uploads_by_ip burst=10 nodelay; limit_req zone=uploads_by_ip_throttled; @@ -241,8 +259,8 @@ location /skynet/tus { limit_conn upload_conn 5; limit_conn upload_conn_rl 1; - # TUS chunks size is 40M + leaving 10M of breathing room - client_max_body_size 50M; + # Do not limit body size in nginx, skyd will reject early on too large upload + client_max_body_size 0; # Those timeouts need to be elevated since skyd can stall reading # data for a while when overloaded which would terminate connection @@ -294,12 +312,31 @@ location /skynet/tus { end end } + + log_by_lua_block { + local skynet_account = require("skynet.account") + local skynet_modules = require("skynet.modules") + local skynet_scanner = require("skynet.scanner") + local skynet_tracker = require("skynet.tracker") + + if skynet_modules.is_enabled("a") then + skynet_tracker.track_upload( + ngx.header["Skynet-Skylink"], + ngx.status, + skynet_account.get_auth_headers(), + ngx.var.remote_addr + ) + end + + if skynet_modules.is_enabled("s") then + skynet_scanner.scan_skylink(ngx.header["Skynet-Skylink"]) + end + } } location /skynet/pin { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/sia-auth; - include /etc/nginx/conf.d/include/track-upload; include /etc/nginx/conf.d/include/generate-siapath; include /etc/nginx/conf.d/include/portal-access-check; @@ -311,6 +348,26 @@ location /skynet/pin { proxy_set_header User-Agent: Sia-Agent; proxy_pass http://sia:9980$uri?siapath=$dir1/$dir2/$dir3&$args; + + log_by_lua_block { + local skynet_account = require("skynet.account") + local skynet_modules = require("skynet.modules") + local skynet_scanner = require("skynet.scanner") + local skynet_tracker = require("skynet.tracker") + + if skynet_modules.is_enabled("a") then + skynet_tracker.track_upload( + ngx.header["Skynet-Skylink"], + ngx.status, + skynet_account.get_auth_headers(), + ngx.var.remote_addr + ) + end + + if skynet_modules.is_enabled("s") then + skynet_scanner.scan_skylink(ngx.header["Skynet-Skylink"]) + end + } } location /skynet/metadata { @@ -357,7 +414,6 @@ location ~ "^/file/(([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?)$" { location /skynet/trustless/basesector { include /etc/nginx/conf.d/include/cors; - include /etc/nginx/conf.d/include/track-download; limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time @@ -391,6 +447,21 @@ location /skynet/trustless/basesector { proxy_set_header User-Agent: Sia-Agent; proxy_pass http://sia:9980; + + log_by_lua_block { + local skynet_account = require("skynet.account") + local skynet_modules = require("skynet.modules") + local skynet_scanner = require("skynet.scanner") + local skynet_tracker = require("skynet.tracker") + + if skynet_modules.is_enabled("a") then + skynet_tracker.track_download(ngx.header["Skynet-Skylink"], ngx.status, skynet_account.get_auth_headers(), ngx.var.body_bytes_sent) + end + + if skynet_modules.is_enabled("s") then + skynet_scanner.scan_skylink(ngx.header["Skynet-Skylink"]) + end + } } location /__internal/do/not/use/accounts { diff --git a/docker/nginx/conf.d/server/server.dnslink b/docker/nginx/conf.d/server/server.dnslink index cf385a1d..139196ef 100644 --- a/docker/nginx/conf.d/server/server.dnslink +++ b/docker/nginx/conf.d/server/server.dnslink @@ -5,6 +5,7 @@ location / { set $path $uri; rewrite_by_lua_block { + local cjson = require("cjson") local cache = ngx.shared.dnslink local cache_value = cache:get(ngx.var.host) @@ -28,13 +29,23 @@ location / { ngx.exit(ngx.status) end else - ngx.var.skylink = res.body + local resolved = cjson.decode(res.body) + + ngx.var.skylink = resolved.skylink + if resolved.sponsor then + ngx.req.set_header("Skynet-Api-Key", resolved.sponsor) + end local cache_ttl = 300 -- 5 minutes cache expire time - cache:set(ngx.var.host, ngx.var.skylink, cache_ttl) + cache:set(ngx.var.host, res.body, cache_ttl) end else - ngx.var.skylink = cache_value + local resolved = cjson.decode(cache_value) + + ngx.var.skylink = resolved.skylink + if resolved.sponsor then + ngx.req.set_header("Skynet-Api-Key", resolved.sponsor) + end end ngx.var.skylink = require("skynet.skylink").parse(ngx.var.skylink) diff --git a/docker/nginx/docker-entrypoint.d/20-envsubst-on-templates.sh b/docker/nginx/docker-entrypoint.d/20-envsubst-on-templates.sh new file mode 100755 index 00000000..be9c5f8e --- /dev/null +++ b/docker/nginx/docker-entrypoint.d/20-envsubst-on-templates.sh @@ -0,0 +1,59 @@ +#!/bin/sh + +# https://github.com/nginxinc/docker-nginx/blob/master/entrypoint/20-envsubst-on-templates.sh +# https://github.com/nginxinc/docker-nginx/blob/master/LICENSE + +# Copyright (C) 2011-2016 Nginx, Inc. +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +set -e + +ME=$(basename $0) + +auto_envsubst() { + local template_dir="${NGINX_ENVSUBST_TEMPLATE_DIR:-/etc/nginx/templates}" + local suffix="${NGINX_ENVSUBST_TEMPLATE_SUFFIX:-.template}" + local output_dir="${NGINX_ENVSUBST_OUTPUT_DIR:-/etc/nginx/conf.d}" + + local template defined_envs relative_path output_path subdir + defined_envs=$(printf '${%s} ' $(env | cut -d= -f1)) + [ -d "$template_dir" ] || return 0 + if [ ! -w "$output_dir" ]; then + echo >&3 "$ME: ERROR: $template_dir exists, but $output_dir is not writable" + return 0 + fi + find "$template_dir" -follow -type f -name "*$suffix" -print | while read -r template; do + relative_path="${template#$template_dir/}" + output_path="$output_dir/${relative_path%$suffix}" + subdir=$(dirname "$relative_path") + # create a subdirectory where the template file exists + mkdir -p "$output_dir/$subdir" + echo >&3 "$ME: Running envsubst on $template to $output_path" + envsubst "$defined_envs" < "$template" > "$output_path" + done +} + +auto_envsubst + +exit 0 diff --git a/docker/nginx/docker-entrypoint.d/40-reload-every-x-hours.sh b/docker/nginx/docker-entrypoint.d/40-reload-every-x-hours.sh new file mode 100755 index 00000000..cdd7d17e --- /dev/null +++ b/docker/nginx/docker-entrypoint.d/40-reload-every-x-hours.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +# source: https://github.com/nginxinc/docker-nginx/pull/509 + +set -e + +ME=$(basename $0) + +[ "${NGINX_ENTRYPOINT_RELOAD_EVERY_X_HOURS:-}" ] || exit 0 +if [ $(echo "$NGINX_ENTRYPOINT_RELOAD_EVERY_X_HOURS > 0" | bc) = 0 ]; then + echo >&3 "$ME: Error. Provide integer or floating point number greater that 0. See 'man sleep'." + exit 1 +fi + +start_background_reload() { + echo >&3 "$ME: Reloading Nginx every $NGINX_ENTRYPOINT_RELOAD_EVERY_X_HOURS hour(s)" + while :; do sleep ${NGINX_ENTRYPOINT_RELOAD_EVERY_X_HOURS}h; echo >&3 "$ME: Reloading Nginx ..." && nginx -s reload; done & +} + +start_background_reload diff --git a/docker/nginx/docker-entrypoint.d/50-generate-local-certificate.sh b/docker/nginx/docker-entrypoint.d/50-generate-local-certificate.sh new file mode 100755 index 00000000..f5ecada1 --- /dev/null +++ b/docker/nginx/docker-entrypoint.d/50-generate-local-certificate.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +# Generate locally signed ssl certificate to be used on routes +# that do not require certificate issued by trusted CA + +set -e + +ME=$(basename $0) + +generate_local_certificate() { + echo >&3 "$ME: Generating locally signed ssl certificate" + openssl req -new -newkey rsa:2048 -days 3650 -nodes -x509 \ + -subj '/CN=local-certificate' \ + -keyout /etc/ssl/local-certificate.key \ + -out /etc/ssl/local-certificate.crt +} + +generate_local_certificate diff --git a/docker/nginx/docker-entrypoint.sh b/docker/nginx/docker-entrypoint.sh new file mode 100755 index 00000000..d45a6d9a --- /dev/null +++ b/docker/nginx/docker-entrypoint.sh @@ -0,0 +1,65 @@ +#!/bin/sh +# vim:sw=4:ts=4:et + +# https://github.com/nginxinc/docker-nginx/blob/master/entrypoint/docker-entrypoint.sh +# https://github.com/nginxinc/docker-nginx/blob/master/LICENSE + +# Copyright (C) 2011-2016 Nginx, Inc. +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +set -e + +if [ -z "${NGINX_ENTRYPOINT_QUIET_LOGS:-}" ]; then + exec 3>&1 +else + exec 3>/dev/null +fi + +if [ "$1" = "nginx" -o "$1" = "nginx-debug" ]; then + if /usr/bin/find "/docker-entrypoint.d/" -mindepth 1 -maxdepth 1 -type f -print -quit 2>/dev/null | read v; then + echo >&3 "$0: /docker-entrypoint.d/ is not empty, will attempt to perform configuration" + + echo >&3 "$0: Looking for shell scripts in /docker-entrypoint.d/" + find "/docker-entrypoint.d/" -follow -type f -print | sort -V | while read -r f; do + case "$f" in + *.sh) + if [ -x "$f" ]; then + echo >&3 "$0: Launching $f"; + "$f" + else + # warn on shell scripts without exec bit + echo >&3 "$0: Ignoring $f, not executable"; + fi + ;; + *) echo >&3 "$0: Ignoring $f";; + esac + done + + echo >&3 "$0: Configuration complete; ready for start up" + else + echo >&3 "$0: No files found in /docker-entrypoint.d/, skipping configuration" + fi +fi + +exec "$@" diff --git a/docker/nginx/libs/skynet/account.lua b/docker/nginx/libs/skynet/account.lua index 6fa2c4d2..709d8130 100644 --- a/docker/nginx/libs/skynet/account.lua +++ b/docker/nginx/libs/skynet/account.lua @@ -59,7 +59,9 @@ function _M.exit_access_forbidden(message) end function _M.accounts_enabled() - return os.getenv("PORTAL_MODULES"):match("a") ~= nil + local skynet_modules = require("skynet.modules") + + return skynet_modules.is_enabled("a") end function _M.get_account_limits() @@ -74,9 +76,16 @@ function _M.get_account_limits() if ngx.var.account_limits == "" then local httpc = require("resty.http").new() + local uri = "http://10.10.10.70:3000/user/limits" + + -- include skylink if it is available in the context of request + -- todo: this should not rely on skylink variable to be defined + if ngx.var.skylink ~= nil and ngx.var.skylink ~= "" then + uri = uri .. "/" .. ngx.var.skylink + end -- 10.10.10.70 points to accounts service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.70:3000/user/limits?unit=byte", { + local res, err = httpc:request_uri(uri .. "?unit=byte", { headers = auth_headers, }) diff --git a/docker/nginx/libs/skynet/modules.lua b/docker/nginx/libs/skynet/modules.lua new file mode 100644 index 00000000..607e6d8e --- /dev/null +++ b/docker/nginx/libs/skynet/modules.lua @@ -0,0 +1,23 @@ +local _M = {} + +local utils = require("utils") + +function _M.is_enabled(module_abbr) + if type(module_abbr) ~= "string" or module_abbr:len() ~= 1 then + error("Module abbreviation '" .. tostring(module_abbr) .. "' should be exactly one character long string") + end + + local enabled_modules = utils.getenv("PORTAL_MODULES") + + if not enabled_modules then + return false + end + + return enabled_modules:find(module_abbr) ~= nil +end + +function _M.is_disabled(module_abbr) + return not _M.is_enabled(module_abbr) +end + +return _M diff --git a/docker/nginx/libs/skynet/modules.spec.lua b/docker/nginx/libs/skynet/modules.spec.lua new file mode 100644 index 00000000..0eaaf081 --- /dev/null +++ b/docker/nginx/libs/skynet/modules.spec.lua @@ -0,0 +1,95 @@ +-- luacheck: ignore os + +local skynet_modules = require("skynet.modules") + +describe("is_enabled", function() + before_each(function() + stub(os, "getenv") + end) + + after_each(function() + os.getenv:revert() + end) + + it("should return false if PORTAL_MODULES are not defined", function() + os.getenv.on_call_with("PORTAL_MODULES").returns(nil) + + assert.is_false(skynet_modules.is_enabled("a")) + end) + + it("should return false if PORTAL_MODULES are empty", function() + os.getenv.on_call_with("PORTAL_MODULES").returns("") + + assert.is_false(skynet_modules.is_enabled("a")) + end) + + it("should return false if module is not enabled", function() + os.getenv.on_call_with("PORTAL_MODULES").returns("qwerty") + + assert.is_false(skynet_modules.is_enabled("a")) + end) + + it("should return true if module is enabled", function() + os.getenv.on_call_with("PORTAL_MODULES").returns("asdfg") + + assert.is_true(skynet_modules.is_enabled("a")) + end) + + it("should throw an error for empty module", function() + assert.has_error(function() + skynet_modules.is_enabled() + end, "Module abbreviation 'nil' should be exactly one character long string") + end) + + it("should throw an error for too long module", function() + assert.has_error(function() + skynet_modules.is_enabled("gandalf") + end, "Module abbreviation 'gandalf' should be exactly one character long string") + end) +end) + +describe("is_disabled", function() + before_each(function() + stub(os, "getenv") + end) + + after_each(function() + os.getenv:revert() + end) + + it("should return true if PORTAL_MODULES are not defined", function() + os.getenv.on_call_with("PORTAL_MODULES").returns(nil) + + assert.is_true(skynet_modules.is_disabled("a")) + end) + + it("should return true if PORTAL_MODULES are empty", function() + os.getenv.on_call_with("PORTAL_MODULES").returns("") + + assert.is_true(skynet_modules.is_disabled("a")) + end) + + it("should return true if module is not enabled", function() + os.getenv.on_call_with("PORTAL_MODULES").returns("qwerty") + + assert.is_true(skynet_modules.is_disabled("a")) + end) + + it("should return false if module is enabled", function() + os.getenv.on_call_with("PORTAL_MODULES").returns("asdfg") + + assert.is_false(skynet_modules.is_disabled("a")) + end) + + it("should throw an error for empty module", function() + assert.has_error(function() + skynet_modules.is_disabled() + end, "Module abbreviation 'nil' should be exactly one character long string") + end) + + it("should throw an error for too long module", function() + assert.has_error(function() + skynet_modules.is_disabled("gandalf") + end, "Module abbreviation 'gandalf' should be exactly one character long string") + end) +end) diff --git a/docker/nginx/libs/skynet/scanner.lua b/docker/nginx/libs/skynet/scanner.lua new file mode 100644 index 00000000..445f1ae9 --- /dev/null +++ b/docker/nginx/libs/skynet/scanner.lua @@ -0,0 +1,26 @@ +local _M = {} + +function _M.scan_skylink_timer(premature, skylink) + if premature then return end + + local httpc = require("resty.http").new() + + -- 10.10.10.101 points to malware-scanner service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.101:4000/scan/" .. skylink, { + method = "POST", + }) + + if err or (res and res.status ~= ngx.HTTP_OK) then + local error_response = err or ("[HTTP " .. res.status .. "] " .. res.body) + ngx.log(ngx.ERR, "Failed malware-scanner request /scan/" .. skylink .. ": ", error_response) + end +end + +function _M.scan_skylink(skylink) + if not skylink then return end + + local ok, err = ngx.timer.at(0, _M.scan_skylink_timer, skylink) + if not ok then ngx.log(ngx.ERR, "Failed to create timer: ", err) end +end + +return _M diff --git a/docker/nginx/libs/skynet/scanner.spec.lua b/docker/nginx/libs/skynet/scanner.spec.lua new file mode 100644 index 00000000..533ef44c --- /dev/null +++ b/docker/nginx/libs/skynet/scanner.spec.lua @@ -0,0 +1,119 @@ +-- luacheck: ignore ngx + +local skynet_scanner = require("skynet.scanner") +local skylink = "AQBG8n_sgEM_nlEp3G0w3vLjmdvSZ46ln8ZXHn-eObZNjA" + +describe("scan_skylink", function() + before_each(function() + stub(ngx.timer, "at") + end) + + after_each(function() + ngx.timer.at:revert() + end) + + it("should schedule a timer when skylink is provided", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_scanner.scan_skylink(skylink) + + assert.stub(ngx.timer.at).was_called_with(0, skynet_scanner.scan_skylink_timer, skylink) + end) + + it("should log an error if timer failed to create", function() + stub(ngx, "log") + + ngx.timer.at.invokes(function() return false, "such a failure" end) + + skynet_scanner.scan_skylink(skylink) + + assert.stub(ngx.timer.at).was_called_with(0, skynet_scanner.scan_skylink_timer, skylink) + assert.stub(ngx.log).was_called_with(ngx.ERR, "Failed to create timer: ", "such a failure") + + ngx.log:revert() + end) + + it("should not schedule a timer if skylink is not provided", function() + skynet_scanner.scan_skylink() + + assert.stub(ngx.timer.at).was_not_called() + end) +end) + +describe("scan_skylink_timer", function() + before_each(function() + stub(ngx, "log") + end) + + after_each(function() + local resty_http = require("resty.http") + + ngx.log:revert() + resty_http.new:revert() + end) + + it("should exit early on premature", function() + local resty_http = require("resty.http") + local request_uri = spy.new() + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_scanner.scan_skylink_timer(true, skylink) + + assert.stub(request_uri).was_not_called() + assert.stub(ngx.log).was_not_called() + end) + + it("should make a post request with skylink to scanner service", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 200 } -- return 200 success + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_scanner.scan_skylink_timer(false, skylink) + + local uri = "http://10.10.10.101:4000/scan/" .. skylink + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST" }) + assert.stub(ngx.log).was_not_called() + end) + + it("should log message on scanner request failure with response code", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 404, body = "baz" } -- return 404 failure + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_scanner.scan_skylink_timer(false, skylink) + + local uri = "http://10.10.10.101:4000/scan/" .. skylink + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST" }) + assert.stub(ngx.log).was_called_with( + ngx.ERR, + "Failed malware-scanner request /scan/AQBG8n_sgEM_nlEp3G0w3vLjmdvSZ46ln8ZXHn-eObZNjA: ", + "[HTTP 404] baz" + ) + end) + + it("should log message on scanner request error", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return nil, "foo != bar" -- return error + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_scanner.scan_skylink_timer(false, skylink) + + local uri = "http://10.10.10.101:4000/scan/" .. skylink + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST" }) + assert.stub(ngx.log).was_called_with( + ngx.ERR, + "Failed malware-scanner request /scan/AQBG8n_sgEM_nlEp3G0w3vLjmdvSZ46ln8ZXHn-eObZNjA: ", + "foo != bar" + ) + end) +end) diff --git a/docker/nginx/libs/skynet/tracker.lua b/docker/nginx/libs/skynet/tracker.lua new file mode 100644 index 00000000..37413215 --- /dev/null +++ b/docker/nginx/libs/skynet/tracker.lua @@ -0,0 +1,99 @@ +local _M = {} + +local utils = require("utils") + +function _M.track_download_timer(premature, skylink, status, auth_headers, body_bytes_sent) + if premature then return end + + local httpc = require("resty.http").new() + local query = table.concat({ "status=" .. status, "bytes=" .. body_bytes_sent }, "&") + + -- 10.10.10.70 points to accounts service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.70:3000/track/download/" .. skylink .. "?" .. query, { + method = "POST", + headers = auth_headers, + }) + + if err or (res and res.status ~= 204) then + local error_response = err or ("[HTTP " .. res.status .. "] " .. res.body) + ngx.log(ngx.ERR, "Failed accounts service request /track/download/" .. skylink .. ": ", error_response) + end +end + +function _M.track_download(skylink, status_code, auth_headers, body_bytes_sent) + local has_auth_headers = not utils.is_table_empty(auth_headers) + local status_success = status_code >= 200 and status_code <= 299 + + if skylink and status_success and has_auth_headers then + local ok, err = ngx.timer.at(0, _M.track_download_timer, skylink, status_code, auth_headers, body_bytes_sent) + if not ok then ngx.log(ngx.ERR, "Failed to create timer: ", err) end + end +end + +function _M.track_upload_timer(premature, skylink, auth_headers, uploader_ip) + if premature then return end + + local httpc = require("resty.http").new() + + -- set correct content type header and include auth headers + local headers = { + ["Content-Type"] = "application/x-www-form-urlencoded", + } + for key, value in pairs(auth_headers) do + headers[key] = value + end + + -- 10.10.10.70 points to accounts service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.70:3000/track/upload/" .. skylink, { + method = "POST", + headers = headers, + body = "ip=" .. uploader_ip, + }) + + if err or (res and res.status ~= 204) then + local error_response = err or ("[HTTP " .. res.status .. "] " .. res.body) + ngx.log(ngx.ERR, "Failed accounts service request /track/upload/" .. skylink .. ": ", error_response) + end +end + +function _M.track_upload(skylink, status_code, auth_headers, uploader_ip) + local status_success = status_code >= 200 and status_code <= 299 + + if skylink and status_success then + local ok, err = ngx.timer.at(0, _M.track_upload_timer, skylink, auth_headers, uploader_ip) + if not ok then ngx.log(ngx.ERR, "Failed to create timer: ", err) end + end +end + +function _M.track_registry_timer(premature, auth_headers, request_method) + if premature then return end + + local httpc = require("resty.http").new() + + -- based on request method we assign a registry action string used + -- in track endpoint namely "read" for GET and "write" for POST + local registry_action = request_method == "GET" and "read" or "write" + + -- 10.10.10.70 points to accounts service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.70:3000/track/registry/" .. registry_action, { + method = "POST", + headers = auth_headers, + }) + + if err or (res and res.status ~= 204) then + local error_response = err or ("[HTTP " .. res.status .. "] " .. res.body) + ngx.log(ngx.ERR, "Failed accounts service request /track/registry/" .. registry_action .. ": ", error_response) + end +end + +function _M.track_registry(status_code, auth_headers, request_method) + local has_auth_headers = not utils.is_table_empty(auth_headers) + local tracked_status = status_code == 200 or status_code == 404 + + if tracked_status and has_auth_headers then + local ok, err = ngx.timer.at(0, _M.track_registry_timer, auth_headers, request_method) + if not ok then ngx.log(ngx.ERR, "Failed to create timer: ", err) end + end +end + +return _M diff --git a/docker/nginx/libs/skynet/tracker.spec.lua b/docker/nginx/libs/skynet/tracker.spec.lua new file mode 100644 index 00000000..98d587d8 --- /dev/null +++ b/docker/nginx/libs/skynet/tracker.spec.lua @@ -0,0 +1,584 @@ +-- luacheck: ignore ngx + +local skynet_tracker = require("skynet.tracker") + +local valid_skylink = "AQBG8n_sgEM_nlEp3G0w3vLjmdvSZ46ln8ZXHn-eObZNjA" +local valid_status_code = 200 +local valid_auth_headers = { ["Skynet-Api-Key"] = "foo" } +local valid_ip = "12.34.56.78" + +describe("track_download", function() + local valid_body_bytes_sent = 12345 + + before_each(function() + stub(ngx.timer, "at") + end) + + after_each(function() + ngx.timer.at:revert() + end) + + it("should schedule a timer when conditions are met", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_tracker.track_download(valid_skylink, valid_status_code, valid_auth_headers, valid_body_bytes_sent) + + assert.stub(ngx.timer.at).was_called_with( + 0, + skynet_tracker.track_download_timer, + valid_skylink, + valid_status_code, + valid_auth_headers, + valid_body_bytes_sent + ) + end) + + it("should not schedule a timer if skylink is empty", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_tracker.track_download(nil, valid_status_code, valid_auth_headers, valid_body_bytes_sent) + + assert.stub(ngx.timer.at).was_not_called() + end) + + it("should not schedule a timer if status code is not in 2XX range", function() + ngx.timer.at.invokes(function() return true, nil end) + + -- couple of example of 4XX and 5XX codes + skynet_tracker.track_download(valid_skylink, 401, valid_auth_headers, valid_body_bytes_sent) + skynet_tracker.track_download(valid_skylink, 403, valid_auth_headers, valid_body_bytes_sent) + skynet_tracker.track_download(valid_skylink, 490, valid_auth_headers, valid_body_bytes_sent) + skynet_tracker.track_download(valid_skylink, 500, valid_auth_headers, valid_body_bytes_sent) + skynet_tracker.track_download(valid_skylink, 502, valid_auth_headers, valid_body_bytes_sent) + + assert.stub(ngx.timer.at).was_not_called() + end) + + it("should not schedule a timer if auth headers are empty", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_tracker.track_download(valid_skylink, valid_status_code, {}, valid_body_bytes_sent) + + assert.stub(ngx.timer.at).was_not_called() + end) + + it("should log an error if timer failed to create", function() + stub(ngx, "log") + ngx.timer.at.invokes(function() return false, "such a failure" end) + + skynet_tracker.track_download(valid_skylink, valid_status_code, valid_auth_headers, valid_body_bytes_sent) + + assert.stub(ngx.timer.at).was_called_with( + 0, + skynet_tracker.track_download_timer, + valid_skylink, + valid_status_code, + valid_auth_headers, + valid_body_bytes_sent + ) + assert.stub(ngx.log).was_called_with(ngx.ERR, "Failed to create timer: ", "such a failure") + + ngx.log:revert() + end) + + describe("track_download_timer", function() + before_each(function() + stub(ngx, "log") + end) + + after_each(function() + local resty_http = require("resty.http") + + ngx.log:revert() + resty_http.new:revert() + end) + + it("should exit early on premature", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 200 } -- return 200 success + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_download_timer( + true, + valid_skylink, + valid_status_code, + valid_auth_headers, + valid_body_bytes_sent + ) + + assert.stub(request_uri).was_not_called() + assert.stub(ngx.log).was_not_called() + end) + + it("should make a post request to tracker service", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 204 } -- return 204 success + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_download_timer( + false, + valid_skylink, + valid_status_code, + valid_auth_headers, + valid_body_bytes_sent + ) + + local uri_params = "status=" .. valid_status_code .. "&bytes=" .. valid_body_bytes_sent + local uri = "http://10.10.10.70:3000/track/download/" .. valid_skylink .. "?" .. uri_params + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST", headers = valid_auth_headers }) + assert.stub(ngx.log).was_not_called() + end) + + it("should log message on tracker request failure with response code", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 404, body = "baz" } -- return 404 failure + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_download_timer( + false, + valid_skylink, + valid_status_code, + valid_auth_headers, + valid_body_bytes_sent + ) + + local uri_params = "status=" .. valid_status_code .. "&bytes=" .. valid_body_bytes_sent + local uri = "http://10.10.10.70:3000/track/download/" .. valid_skylink .. "?" .. uri_params + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST", headers = valid_auth_headers }) + assert.stub(ngx.log).was_called_with( + ngx.ERR, + "Failed accounts service request /track/download/" .. valid_skylink .. ": ", + "[HTTP 404] baz" + ) + end) + + it("should log message on tracker request error", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return nil, "foo != bar" -- return error + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_download_timer( + false, + valid_skylink, + valid_status_code, + valid_auth_headers, + valid_body_bytes_sent + ) + + local uri_params = "status=" .. valid_status_code .. "&bytes=" .. valid_body_bytes_sent + local uri = "http://10.10.10.70:3000/track/download/" .. valid_skylink .. "?" .. uri_params + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST", headers = valid_auth_headers }) + assert.stub(ngx.log).was_called_with( + ngx.ERR, + "Failed accounts service request /track/download/" .. valid_skylink .. ": ", + "foo != bar" + ) + end) + end) +end) + +describe("track_upload", function() + before_each(function() + stub(ngx.timer, "at") + end) + + after_each(function() + ngx.timer.at:revert() + end) + + it("should schedule a timer when conditions are met", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_tracker.track_upload(valid_skylink, valid_status_code, valid_auth_headers, valid_ip) + + assert.stub(ngx.timer.at).was_called_with( + 0, + skynet_tracker.track_upload_timer, + valid_skylink, + valid_auth_headers, + valid_ip + ) + end) + + it("should not schedule a timer if skylink is empty", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_tracker.track_upload(nil, valid_status_code, valid_auth_headers, valid_ip) + + assert.stub(ngx.timer.at).was_not_called() + end) + + it("should not schedule a timer if status code is not in 2XX range", function() + ngx.timer.at.invokes(function() return true, nil end) + + -- couple of example of 4XX and 5XX codes + skynet_tracker.track_upload(valid_skylink, 401, valid_auth_headers, valid_ip) + skynet_tracker.track_upload(valid_skylink, 403, valid_auth_headers, valid_ip) + skynet_tracker.track_upload(valid_skylink, 490, valid_auth_headers, valid_ip) + skynet_tracker.track_upload(valid_skylink, 500, valid_auth_headers, valid_ip) + skynet_tracker.track_upload(valid_skylink, 502, valid_auth_headers, valid_ip) + + assert.stub(ngx.timer.at).was_not_called() + end) + + it("should schedule a timer if auth headers are empty", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_tracker.track_upload(valid_skylink, valid_status_code, {}, valid_ip) + + assert.stub(ngx.timer.at).was_called_with( + 0, + skynet_tracker.track_upload_timer, + valid_skylink, + {}, + valid_ip + ) + end) + + it("should log an error if timer failed to create", function() + stub(ngx, "log") + ngx.timer.at.invokes(function() return false, "such a failure" end) + + skynet_tracker.track_upload(valid_skylink, valid_status_code, valid_auth_headers, valid_ip) + + assert.stub(ngx.timer.at).was_called_with( + 0, + skynet_tracker.track_upload_timer, + valid_skylink, + valid_auth_headers, + valid_ip + ) + assert.stub(ngx.log).was_called_with(ngx.ERR, "Failed to create timer: ", "such a failure") + + ngx.log:revert() + end) + + describe("track_upload_timer", function() + before_each(function() + stub(ngx, "log") + end) + + after_each(function() + local resty_http = require("resty.http") + + ngx.log:revert() + resty_http.new:revert() + end) + + it("should exit early on premature", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 200 } -- return 200 success + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_upload_timer( + true, + valid_skylink, + valid_auth_headers, + valid_ip + ) + + assert.stub(request_uri).was_not_called() + assert.stub(ngx.log).was_not_called() + end) + + it("should make a post request to tracker service", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 204 } -- return 204 success + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_upload_timer( + false, + valid_skylink, + valid_auth_headers, + valid_ip + ) + + local uri = "http://10.10.10.70:3000/track/upload/" .. valid_skylink + assert.stub(request_uri).was_called_with(httpc, uri, { + method = "POST", + headers = { + ["Content-Type"] = "application/x-www-form-urlencoded", + ["Skynet-Api-Key"] = "foo", + }, + body = "ip=" .. valid_ip + }) + assert.stub(ngx.log).was_not_called() + end) + + it("should log message on tracker request failure with response code", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 404, body = "baz" } -- return 404 failure + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_upload_timer( + false, + valid_skylink, + valid_auth_headers, + valid_ip + ) + + local uri = "http://10.10.10.70:3000/track/upload/" .. valid_skylink + assert.stub(request_uri).was_called_with(httpc, uri, { + method = "POST", + headers = { + ["Content-Type"] = "application/x-www-form-urlencoded", + ["Skynet-Api-Key"] = "foo", + }, + body = "ip=" .. valid_ip + }) + assert.stub(ngx.log).was_called_with( + ngx.ERR, + "Failed accounts service request /track/upload/" .. valid_skylink .. ": ", + "[HTTP 404] baz" + ) + end) + + it("should log message on tracker request error", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return nil, "foo != bar" -- return error + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_upload_timer( + false, + valid_skylink, + valid_auth_headers, + valid_ip + ) + + local uri = "http://10.10.10.70:3000/track/upload/" .. valid_skylink + assert.stub(request_uri).was_called_with(httpc, uri, { + method = "POST", + headers = { + ["Content-Type"] = "application/x-www-form-urlencoded", + ["Skynet-Api-Key"] = "foo", + }, + body = "ip=" .. valid_ip + }) + assert.stub(ngx.log).was_called_with( + ngx.ERR, + "Failed accounts service request /track/upload/" .. valid_skylink .. ": ", + "foo != bar" + ) + end) + end) +end) + +describe("track_registry", function() + local status_code_ok = 200 + local status_code_not_found = 404 + local request_method_write = "POST" + local request_method_read = "GET" + + before_each(function() + stub(ngx.timer, "at") + end) + + after_each(function() + ngx.timer.at:revert() + end) + + it("should schedule a timer when status code was 200", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_tracker.track_registry(status_code_ok, valid_auth_headers, request_method_write) + + assert.stub(ngx.timer.at).was_called_with( + 0, + skynet_tracker.track_registry_timer, + valid_auth_headers, + request_method_write + ) + end) + + it("should schedule a timer when status code was 404", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_tracker.track_registry(status_code_not_found, valid_auth_headers, request_method_write) + + assert.stub(ngx.timer.at).was_called_with( + 0, + skynet_tracker.track_registry_timer, + valid_auth_headers, + request_method_write + ) + end) + + it("should not schedule a timer if status code is not in 200 or 404", function() + ngx.timer.at.invokes(function() return true, nil end) + + -- couple of example of invalid 2XX, 4XX and 5XX codes + skynet_tracker.track_registry(204, valid_auth_headers, request_method_write) + skynet_tracker.track_registry(206, valid_auth_headers, request_method_write) + skynet_tracker.track_registry(401, valid_auth_headers, request_method_write) + skynet_tracker.track_registry(403, valid_auth_headers, request_method_write) + skynet_tracker.track_registry(490, valid_auth_headers, request_method_write) + skynet_tracker.track_registry(500, valid_auth_headers, request_method_write) + skynet_tracker.track_registry(502, valid_auth_headers, request_method_write) + + assert.stub(ngx.timer.at).was_not_called() + end) + + it("should not schedule a timer if auth headers are empty", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_tracker.track_registry(status_code_ok, {}, request_method_write) + + assert.stub(ngx.timer.at).was_not_called() + end) + + it("should log an error if timer failed to create", function() + stub(ngx, "log") + ngx.timer.at.invokes(function() return false, "such a failure" end) + + skynet_tracker.track_registry(status_code_ok, valid_auth_headers, request_method_write) + + assert.stub(ngx.timer.at).was_called_with( + 0, + skynet_tracker.track_registry_timer, + valid_auth_headers, + request_method_write + ) + assert.stub(ngx.log).was_called_with(ngx.ERR, "Failed to create timer: ", "such a failure") + + ngx.log:revert() + end) + + describe("track_registry_timer", function() + before_each(function() + stub(ngx, "log") + end) + + after_each(function() + local resty_http = require("resty.http") + + ngx.log:revert() + resty_http.new:revert() + end) + + it("should exit early on premature", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 200 } -- return 200 success + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_registry_timer( + true, + valid_auth_headers, + request_method_write + ) + + assert.stub(request_uri).was_not_called() + assert.stub(ngx.log).was_not_called() + end) + + it("should make a post request to registry write tracker service", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 204 } -- return 204 success + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_registry_timer( + false, + valid_auth_headers, + request_method_write + ) + + local uri = "http://10.10.10.70:3000/track/registry/write" + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST", headers = valid_auth_headers }) + assert.stub(ngx.log).was_not_called() + end) + + it("should make a post request to registry read tracker service", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 204 } -- return 204 success + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_registry_timer( + false, + valid_auth_headers, + request_method_read + ) + + local uri = "http://10.10.10.70:3000/track/registry/read" + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST", headers = valid_auth_headers }) + assert.stub(ngx.log).was_not_called() + end) + + it("should log message on tracker request failure with response code", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 404, body = "baz" } -- return 404 failure + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_registry_timer( + false, + valid_auth_headers, + request_method_write + ) + + local uri = "http://10.10.10.70:3000/track/registry/write" + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST", headers = valid_auth_headers }) + assert.stub(ngx.log).was_called_with( + ngx.ERR, + "Failed accounts service request /track/registry/write: ", + "[HTTP 404] baz" + ) + end) + + it("should log message on tracker request error", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return nil, "foo != bar" -- return error + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_registry_timer( + false, + valid_auth_headers, + request_method_write + ) + + local uri = "http://10.10.10.70:3000/track/registry/write" + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST", headers = valid_auth_headers }) + assert.stub(ngx.log).was_called_with( + ngx.ERR, + "Failed accounts service request /track/registry/write: ", + "foo != bar" + ) + end) + end) +end) \ No newline at end of file diff --git a/docker/nginx/libs/skynet/utils.lua b/docker/nginx/libs/skynet/utils.lua index adee23b2..05755f7b 100644 --- a/docker/nginx/libs/skynet/utils.lua +++ b/docker/nginx/libs/skynet/utils.lua @@ -1,13 +1,20 @@ local _M = {} +local ngx_base64 = require("ngx.base64") +local utils = require("utils") + function _M.authorization_header() -- read api password from env variable - local apipassword = os.getenv("SIA_API_PASSWORD") + local apipassword = utils.getenv("SIA_API_PASSWORD") -- if api password is not available as env variable, read it from disk - if apipassword == nil or apipassword == "" then + if not apipassword then -- open apipassword file for reading (b flag is required for some reason) -- (file /etc/.sia/apipassword has to be mounted from the host system) local apipassword_file = io.open("/data/sia/apipassword", "rb") + -- make sure to throw a meaningful error if apipassword file does not exist + if not apipassword_file then + error("Error reading /data/sia/apipassword file") + end -- read apipassword file contents and trim newline (important) apipassword = apipassword_file:read("*all"):gsub("%s+", "") -- make sure to close file after reading the password @@ -15,7 +22,7 @@ function _M.authorization_header() end -- encode the user:password authorization string -- (in our case user is empty so it is just :password) - local content = require("ngx.base64").encode_base64url(":" .. apipassword) + local content = ngx_base64.encode_base64url(":" .. apipassword) -- set authorization header with proper base64 encoded string return "Basic " .. content end diff --git a/docker/nginx/libs/skynet/utils.spec.lua b/docker/nginx/libs/skynet/utils.spec.lua new file mode 100644 index 00000000..171be4fa --- /dev/null +++ b/docker/nginx/libs/skynet/utils.spec.lua @@ -0,0 +1,65 @@ +-- luacheck: ignore io + +local utils = require('utils') +local skynet_utils = require('skynet.utils') + +describe("authorization_header", function() + local apipassword = "ddd0c1430fbf2708" + local expected_header = "Basic OmRkZDBjMTQzMGZiZjI3MDg" + + it("reads SIA_API_PASSWORD from env variable and returns a header", function() + -- stub getenv on SIA_API_PASSWORD + stub(utils, "getenv") + utils.getenv.on_call_with("SIA_API_PASSWORD").returns(apipassword) + + local header = skynet_utils.authorization_header() + + assert.is_equal(header, expected_header) + + -- revert stub to original function + utils.getenv:revert() + end) + + it("uses /data/sia/apipassword file if SIA_API_PASSWORD env var is missing", function() + -- stub /data/sia/apipassword file + stub(io, "open") + io.open.on_call_with("/data/sia/apipassword", "rb").returns(mock({ + read = spy.new(function() return apipassword end), + close = spy.new() + })) + + local header = skynet_utils.authorization_header() + + assert.is_equal(header, expected_header) + + -- revert stub to original function + io.open:revert() + end) + + it("should choose env variable over file if both are available", function() + -- stub getenv on SIA_API_PASSWORD + stub(utils, "getenv") + utils.getenv.on_call_with("SIA_API_PASSWORD").returns(apipassword) + + -- stub /data/sia/apipassword file + stub(io, "open") + io.open.on_call_with("/data/sia/apipassword", "rb").returns(mock({ + read = spy.new(function() return "foooooooooooooo" end), + close = spy.new() + })) + + local header = skynet_utils.authorization_header() + + assert.is_equal(header, "Basic OmRkZDBjMTQzMGZiZjI3MDg") + + -- revert stubs to original function + utils.getenv:revert() + io.open:revert() + end) + + it("should error out if neither env variable is available nor file exists", function() + assert.has_error(function() + skynet_utils.authorization_header() + end, "Error reading /data/sia/apipassword file") + end) +end) diff --git a/docker/nginx/libs/utils.lua b/docker/nginx/libs/utils.lua index 4330c94c..8b77d802 100644 --- a/docker/nginx/libs/utils.lua +++ b/docker/nginx/libs/utils.lua @@ -42,4 +42,42 @@ function _M.extract_cookie_value(cookie_string, name_matcher) return string.sub(cookie, value_start) end +-- utility function that builds on os.getenv to get environment variable value +-- * will always return nil for both unset and empty env vars +-- * parse "boolean": "true" and "1" as true, "false" and "0" as false, throws for others +-- * parse "integer": any numerical string gets converted, otherwise returns nil +function _M.getenv(name, parse) + local value = os.getenv(name) + + -- treat empty string value as nil to simplify comparisons + if value == nil or value == "" then + return nil + end + + -- do not parse the value + if parse == nil then + return value + end + + -- try to parse as boolean + if parse == "boolean" then + if string.lower(value) == "true" or value == "1" then + return true + end + + if string.lower(value) == "false" or value == "0" then + return false + end + + error("utils.getenv: Parsing value '" .. tostring(value) .. "' to boolean is not supported") + end + + -- try to parse as integer + if parse == "integer" then + return tonumber(value) + end + + error("utils.getenv: Parsing to '" .. parse .. "' is not supported") +end + return _M diff --git a/docker/nginx/libs/utils.spec.lua b/docker/nginx/libs/utils.spec.lua index c853c8cd..71ef086c 100644 --- a/docker/nginx/libs/utils.spec.lua +++ b/docker/nginx/libs/utils.spec.lua @@ -1,3 +1,5 @@ +-- luacheck: ignore os + local utils = require('utils') describe("is_table_empty", function() @@ -77,3 +79,137 @@ describe("extract_cookie_value", function() assert.are.equals(value, "MTY0NzUyr8jD-ytiWtspm0tGabKfooxeIDuWcXhJ3lnY0eEw==") end) end) + +describe("getenv", function() + before_each(function() + stub(os, "getenv") + end) + + after_each(function() + os.getenv:revert() + end) + + it("should return nil for not existing env var", function() + os.getenv.on_call_with("foo").returns(nil) + + assert.is_nil(utils.getenv("foo")) + end) + + it("should return nil for env var that is an empty string", function() + os.getenv.on_call_with("foo").returns("") + + assert.is_nil(utils.getenv("foo")) + end) + + it("should return the value as is when it is non empty string", function() + os.getenv.on_call_with("foo").returns("bar") + + assert.is_equal(utils.getenv("foo"), "bar") + end) + + describe("parse", function() + it("should throw on not supported parser", function() + os.getenv.on_call_with("foo").returns("test") + + assert.has_error(function() + utils.getenv("foo", "starwars") + end, "utils.getenv: Parsing to 'starwars' is not supported") + end) + + describe("as boolean", function() + it("should return nil for not existing env var", function() + os.getenv.on_call_with("foo").returns(nil) + + assert.is_nil(utils.getenv("foo", "boolean")) + end) + + it("should return nil for env var that is an empty string", function() + os.getenv.on_call_with("foo").returns("") + + assert.is_nil(utils.getenv("foo", "boolean")) + end) + + it("should parse 'true' string as true", function() + os.getenv.on_call_with("foo").returns("true") + + assert.is_true(utils.getenv("foo", "boolean")) + end) + + it("should parse 'True' string as true", function() + os.getenv.on_call_with("foo").returns("True") + + assert.is_true(utils.getenv("foo", "boolean")) + end) + + it("should parse '1' string as true", function() + os.getenv.on_call_with("foo").returns("1") + + assert.is_true(utils.getenv("foo", "boolean")) + end) + + it("should parse 'false' string as false", function() + os.getenv.on_call_with("foo").returns("false") + + assert.is_false(utils.getenv("foo", "boolean")) + end) + + it("should parse 'False' string as false", function() + os.getenv.on_call_with("foo").returns("False") + + assert.is_false(utils.getenv("foo", "boolean")) + end) + + it("should parse '0' string as false", function() + os.getenv.on_call_with("foo").returns("0") + + assert.is_false(utils.getenv("foo", "boolean")) + end) + + it("should throw an error for not supported string", function() + os.getenv.on_call_with("foo").returns("mandalorian") + + assert.has_error(function() + utils.getenv("foo", "boolean") + end, "utils.getenv: Parsing value 'mandalorian' to boolean is not supported") + end) + end) + + describe("as integer", function() + it("should return nil for not existing env var", function() + os.getenv.on_call_with("foo").returns(nil) + + assert.is_nil(utils.getenv("foo", "integer")) + end) + + it("should return nil for env var that is an empty string", function() + os.getenv.on_call_with("foo").returns("") + + assert.is_nil(utils.getenv("foo", "integer")) + end) + + it("should parse '0' string as 0", function() + os.getenv.on_call_with("foo").returns("0") + + assert.equals(utils.getenv("foo", "integer"), 0) + end) + + it("should parse '1' string as 1", function() + os.getenv.on_call_with("foo").returns("1") + + assert.equals(utils.getenv("foo", "integer"), 1) + end) + + it("should parse '-1' string as -1", function() + os.getenv.on_call_with("foo").returns("-1") + + assert.equals(utils.getenv("foo", "integer"), -1) + end) + + it("should return nil for non numerical string", function() + os.getenv.on_call_with("foo").returns("test") + + assert.is_nil(utils.getenv("foo", "integer")) + end) + end) + end) +end) diff --git a/docker/nginx/mo b/docker/nginx/mo deleted file mode 100755 index ba8e48d1..00000000 --- a/docker/nginx/mo +++ /dev/null @@ -1,1106 +0,0 @@ -#!/usr/bin/env bash -# -#/ Mo is a mustache template rendering software written in bash. It inserts -#/ environment variables into templates. -#/ -#/ Simply put, mo will change {{VARIABLE}} into the value of that -#/ environment variable. You can use {{#VARIABLE}}content{{/VARIABLE}} to -#/ conditionally display content or iterate over the values of an array. -#/ -#/ Learn more about mustache templates at https://mustache.github.io/ -#/ -#/ Simple usage: -#/ -#/ mo [OPTIONS] filenames... -#/ -#/ Options: -#/ -#/ -u, --fail-not-set -#/ Fail upon expansion of an unset variable. -#/ -x, --fail-on-function -#/ Fail when a function returns a non-zero status code. -#/ -e, --false -#/ Treat the string "false" as empty for conditionals. -#/ -h, --help -#/ This message. -#/ -s=FILE, --source=FILE -#/ Load FILE into the environment before processing templates. -#/ Can be used multiple times. -# -# Mo is under a MIT style licence with an additional non-advertising clause. -# See LICENSE.md for the full text. -# -# This is open source! Please feel free to contribute. -# -# https://github.com/tests-always-included/mo - - -# Public: Template parser function. Writes templates to stdout. -# -# $0 - Name of the mo file, used for getting the help message. -# $@ - Filenames to parse. -# -# Options: -# -# --allow-function-arguments -# -# Permit functions in templates to be called with additional arguments. This -# puts template data directly in to the path of an eval statement. Use with -# caution. Not listed in the help because it only makes sense when mo is -# sourced. -# -# -u, --fail-not-set -# -# Fail upon expansion of an unset variable. Default behavior is to silently -# ignore and expand into empty string. -# -# -x, --fail-on-function -# -# Fail when a function used by a template returns an error status code. -# Alternately, ou may set the MO_FAIL_ON_FUNCTION environment variable to a -# non-empty value to enable this behavior. -# -# -e, --false -# -# Treat "false" as an empty value. You may set the MO_FALSE_IS_EMPTY -# environment variable instead to a non-empty value to enable this behavior. -# -# -h, --help -# -# Display a help message. -# -# -s=FILE, --source=FILE -# -# Source a file into the environment before processing template files. -# This can be used multiple times. -# -# -- -# -# Used to indicate the end of options. You may optionally use this when -# filenames may start with two hyphens. -# -# Mo uses the following environment variables: -# -# MO_ALLOW_FUNCTION_ARGUMENTS - When set to a non-empty value, this allows -# functions referenced in templates to receive additional -# options and arguments. This puts the content from the -# template directly into an eval statement. Use with extreme -# care. -# MO_FUNCTION_ARGS - Arguments passed to the function -# MO_FAIL_ON_FUNCTION - If a function returns a non-zero status code, abort -# with an error. -# MO_FAIL_ON_UNSET - When set to a non-empty value, expansion of an unset env -# variable will be aborted with an error. -# MO_FALSE_IS_EMPTY - When set to a non-empty value, the string "false" will be -# treated as an empty value for the purposes of conditionals. -# MO_ORIGINAL_COMMAND - Used to find the `mo` program in order to generate a -# help message. -# -# Returns nothing. -mo() ( - # This function executes in a subshell so IFS is reset. - # Namespace this variable so we don't conflict with desired values. - local moContent f2source files doubleHyphens - - IFS=$' \n\t' - files=() - doubleHyphens=false - - if [[ $# -gt 0 ]]; then - for arg in "$@"; do - if $doubleHyphens; then - #: After we encounter two hyphens together, all the rest - #: of the arguments are files. - files=("${files[@]}" "$arg") - else - case "$arg" in - -h|--h|--he|--hel|--help|-\?) - moUsage "$0" - exit 0 - ;; - - --allow-function-arguments) - # shellcheck disable=SC2030 - MO_ALLOW_FUNCTION_ARGUMENTS=true - ;; - - -u | --fail-not-set) - # shellcheck disable=SC2030 - MO_FAIL_ON_UNSET=true - ;; - - -x | --fail-on-function) - # shellcheck disable=SC2030 - MO_FAIL_ON_FUNCTION=true - ;; - - -e | --false) - # shellcheck disable=SC2030 - MO_FALSE_IS_EMPTY=true - ;; - - -s=* | --source=*) - if [[ "$arg" == --source=* ]]; then - f2source="${arg#--source=}" - else - f2source="${arg#-s=}" - fi - - if [[ -f "$f2source" ]]; then - # shellcheck disable=SC1090 - . "$f2source" - else - echo "No such file: $f2source" >&2 - exit 1 - fi - ;; - - --) - #: Set a flag indicating we've encountered double hyphens - doubleHyphens=true - ;; - - *) - #: Every arg that is not a flag or a option should be a file - files=(${files[@]+"${files[@]}"} "$arg") - ;; - esac - fi - done - fi - - moGetContent moContent "${files[@]}" || return 1 - moParse "$moContent" "" true -) - - -# Internal: Call a function. -# -# $1 - Variable for output -# $2 - Function to call -# $3 - Content to pass -# $4 - Additional arguments as a single string -# -# This can be dangerous, especially if you are using tags like -# {{someFunction ; rm -rf / }} -# -# Returns nothing. -moCallFunction() { - local moArgs moContent moFunctionArgs moFunctionResult - - moArgs=() - moTrimWhitespace moFunctionArgs "$4" - - # shellcheck disable=SC2031 - if [[ -n "${MO_ALLOW_FUNCTION_ARGUMENTS-}" ]]; then - # Intentionally bad behavior - # shellcheck disable=SC2206 - moArgs=($4) - fi - - moContent=$(echo -n "$3" | MO_FUNCTION_ARGS="$moFunctionArgs" eval "$2" "${moArgs[@]}") || { - moFunctionResult=$? - # shellcheck disable=SC2031 - if [[ -n "${MO_FAIL_ON_FUNCTION-}" && "$moFunctionResult" != 0 ]]; then - echo "Function '$2' with args (${moArgs[*]+"${moArgs[@]}"}) failed with status code $moFunctionResult" - exit "$moFunctionResult" - fi - } - - # shellcheck disable=SC2031 - local "$1" && moIndirect "$1" "$moContent" -} - - -# Internal: Scan content until the right end tag is found. Creates an array -# with the following members: -# -# [0] = Content before end tag -# [1] = End tag (complete tag) -# [2] = Content after end tag -# -# Everything using this function uses the "standalone tags" logic. -# -# $1 - Name of variable for the array -# $2 - Content -# $3 - Name of end tag -# $4 - If -z, do standalone tag processing before finishing -# -# Returns nothing. -moFindEndTag() { - local content remaining scanned standaloneBytes tag - - #: Find open tags - scanned="" - moSplit content "$2" '{{' '}}' - - while [[ "${#content[@]}" -gt 1 ]]; do - moTrimWhitespace tag "${content[1]}" - - #: Restore content[1] before we start using it - content[1]='{{'"${content[1]}"'}}' - - case $tag in - '#'* | '^'*) - #: Start another block - scanned="${scanned}${content[0]}${content[1]}" - moTrimWhitespace tag "${tag:1}" - moFindEndTag content "${content[2]}" "$tag" "loop" - scanned="${scanned}${content[0]}${content[1]}" - remaining=${content[2]} - ;; - - '/'*) - #: End a block - could be ours - moTrimWhitespace tag "${tag:1}" - scanned="$scanned${content[0]}" - - if [[ "$tag" == "$3" ]]; then - #: Found our end tag - if [[ -z "${4-}" ]] && moIsStandalone standaloneBytes "$scanned" "${content[2]}" true; then - #: This is also a standalone tag - clean up whitespace - #: and move those whitespace bytes to the "tag" element - # shellcheck disable=SC2206 - standaloneBytes=( $standaloneBytes ) - content[1]="${scanned:${standaloneBytes[0]}}${content[1]}${content[2]:0:${standaloneBytes[1]}}" - scanned="${scanned:0:${standaloneBytes[0]}}" - content[2]="${content[2]:${standaloneBytes[1]}}" - fi - - local "$1" && moIndirectArray "$1" "$scanned" "${content[1]}" "${content[2]}" - return 0 - fi - - scanned="$scanned${content[1]}" - remaining=${content[2]} - ;; - - *) - #: Ignore all other tags - scanned="${scanned}${content[0]}${content[1]}" - remaining=${content[2]} - ;; - esac - - moSplit content "$remaining" '{{' '}}' - done - - #: Did not find our closing tag - scanned="$scanned${content[0]}" - local "$1" && moIndirectArray "$1" "${scanned}" "" "" -} - - -# Internal: Find the first index of a substring. If not found, sets the -# index to -1. -# -# $1 - Destination variable for the index -# $2 - Haystack -# $3 - Needle -# -# Returns nothing. -moFindString() { - local pos string - - string=${2%%$3*} - [[ "$string" == "$2" ]] && pos=-1 || pos=${#string} - local "$1" && moIndirect "$1" "$pos" -} - - -# Internal: Generate a dotted name based on current context and target name. -# -# $1 - Target variable to store results -# $2 - Context name -# $3 - Desired variable name -# -# Returns nothing. -moFullTagName() { - if [[ -z "${2-}" ]] || [[ "$2" == *.* ]]; then - local "$1" && moIndirect "$1" "$3" - else - local "$1" && moIndirect "$1" "${2}.${3}" - fi -} - - -# Internal: Fetches the content to parse into a variable. Can be a list of -# partials for files or the content from stdin. -# -# $1 - Variable name to assign this content back as -# $2-@ - File names (optional) -# -# Returns nothing. -moGetContent() { - local moContent moFilename moTarget - - moTarget=$1 - shift - if [[ "${#@}" -gt 0 ]]; then - moContent="" - - for moFilename in "$@"; do - #: This is so relative paths work from inside template files - moContent="$moContent"'{{>'"$moFilename"'}}' - done - else - moLoadFile moContent || return 1 - fi - - local "$moTarget" && moIndirect "$moTarget" "$moContent" -} - - -# Internal: Indent a string, placing the indent at the beginning of every -# line that has any content. -# -# $1 - Name of destination variable to get an array of lines -# $2 - The indent string -# $3 - The string to reindent -# -# Returns nothing. -moIndentLines() { - local content fragment len posN posR result trimmed - - result="" - - #: Remove the period from the end of the string. - len=$((${#3} - 1)) - content=${3:0:$len} - - if [[ -z "${2-}" ]]; then - local "$1" && moIndirect "$1" "$content" - - return 0 - fi - - moFindString posN "$content" $'\n' - moFindString posR "$content" $'\r' - - while [[ "$posN" -gt -1 ]] || [[ "$posR" -gt -1 ]]; do - if [[ "$posN" -gt -1 ]]; then - fragment="${content:0:$posN + 1}" - content=${content:$posN + 1} - else - fragment="${content:0:$posR + 1}" - content=${content:$posR + 1} - fi - - moTrimChars trimmed "$fragment" false true " " $'\t' $'\n' $'\r' - - if [[ -n "$trimmed" ]]; then - fragment="$2$fragment" - fi - - result="$result$fragment" - - moFindString posN "$content" $'\n' - moFindString posR "$content" $'\r' - - # If the content ends in a newline, do not indent. - if [[ "$posN" -eq ${#content} ]]; then - # Special clause for \r\n - if [[ "$posR" -eq "$((posN - 1))" ]]; then - posR=-1 - fi - - posN=-1 - fi - - if [[ "$posR" -eq ${#content} ]]; then - posR=-1 - fi - done - - moTrimChars trimmed "$content" false true " " $'\t' - - if [[ -n "$trimmed" ]]; then - content="$2$content" - fi - - result="$result$content" - - local "$1" && moIndirect "$1" "$result" -} - - -# Internal: Send a variable up to the parent of the caller of this function. -# -# $1 - Variable name -# $2 - Value -# -# Examples -# -# callFunc () { -# local "$1" && moIndirect "$1" "the value" -# } -# callFunc dest -# echo "$dest" # writes "the value" -# -# Returns nothing. -moIndirect() { - unset -v "$1" - printf -v "$1" '%s' "$2" -} - - -# Internal: Send an array as a variable up to caller of a function -# -# $1 - Variable name -# $2-@ - Array elements -# -# Examples -# -# callFunc () { -# local myArray=(one two three) -# local "$1" && moIndirectArray "$1" "${myArray[@]}" -# } -# callFunc dest -# echo "${dest[@]}" # writes "one two three" -# -# Returns nothing. -moIndirectArray() { - unset -v "$1" - - # IFS must be set to a string containing space or unset in order for - # the array slicing to work regardless of the current IFS setting on - # bash 3. This is detailed further at - # https://github.com/fidian/gg-core/pull/7 - eval "$(printf "IFS= %s=(\"\${@:2}\") IFS=%q" "$1" "$IFS")" -} - - -# Internal: Determine if a given environment variable exists and if it is -# an array. -# -# $1 - Name of environment variable -# -# Be extremely careful. Even if strict mode is enabled, it is not honored -# in newer versions of Bash. Any errors that crop up here will not be -# caught automatically. -# -# Examples -# -# var=(abc) -# if moIsArray var; then -# echo "This is an array" -# echo "Make sure you don't accidentally use \$var" -# fi -# -# Returns 0 if the name is not empty, 1 otherwise. -moIsArray() { - # Namespace this variable so we don't conflict with what we're testing. - local moTestResult - - moTestResult=$(declare -p "$1" 2>/dev/null) || return 1 - [[ "${moTestResult:0:10}" == "declare -a" ]] && return 0 - [[ "${moTestResult:0:10}" == "declare -A" ]] && return 0 - - return 1 -} - - -# Internal: Determine if the given name is a defined function. -# -# $1 - Function name to check -# -# Be extremely careful. Even if strict mode is enabled, it is not honored -# in newer versions of Bash. Any errors that crop up here will not be -# caught automatically. -# -# Examples -# -# moo () { -# echo "This is a function" -# } -# if moIsFunction moo; then -# echo "moo is a defined function" -# fi -# -# Returns 0 if the name is a function, 1 otherwise. -moIsFunction() { - local functionList functionName - - functionList=$(declare -F) - # shellcheck disable=SC2206 - functionList=( ${functionList//declare -f /} ) - - for functionName in "${functionList[@]}"; do - if [[ "$functionName" == "$1" ]]; then - return 0 - fi - done - - return 1 -} - - -# Internal: Determine if the tag is a standalone tag based on whitespace -# before and after the tag. -# -# Passes back a string containing two numbers in the format "BEFORE AFTER" -# like "27 10". It indicates the number of bytes remaining in the "before" -# string (27) and the number of bytes to trim in the "after" string (10). -# Useful for string manipulation: -# -# $1 - Variable to set for passing data back -# $2 - Content before the tag -# $3 - Content after the tag -# $4 - true/false: is this the beginning of the content? -# -# Examples -# -# moIsStandalone RESULT "$before" "$after" false || return 0 -# RESULT_ARRAY=( $RESULT ) -# echo "${before:0:${RESULT_ARRAY[0]}}...${after:${RESULT_ARRAY[1]}}" -# -# Returns nothing. -moIsStandalone() { - local afterTrimmed beforeTrimmed char - - moTrimChars beforeTrimmed "$2" false true " " $'\t' - moTrimChars afterTrimmed "$3" true false " " $'\t' - char=$((${#beforeTrimmed} - 1)) - char=${beforeTrimmed:$char} - - # If the content before didn't end in a newline - if [[ "$char" != $'\n' ]] && [[ "$char" != $'\r' ]]; then - # and there was content or this didn't start the file - if [[ -n "$char" ]] || ! $4; then - # then this is not a standalone tag. - return 1 - fi - fi - - char=${afterTrimmed:0:1} - - # If the content after doesn't start with a newline and it is something - if [[ "$char" != $'\n' ]] && [[ "$char" != $'\r' ]] && [[ -n "$char" ]]; then - # then this is not a standalone tag. - return 2 - fi - - if [[ "$char" == $'\r' ]] && [[ "${afterTrimmed:1:1}" == $'\n' ]]; then - char="$char"$'\n' - fi - - local "$1" && moIndirect "$1" "$((${#beforeTrimmed})) $((${#3} + ${#char} - ${#afterTrimmed}))" -} - - -# Internal: Join / implode an array -# -# $1 - Variable name to receive the joined content -# $2 - Joiner -# $3-$* - Elements to join -# -# Returns nothing. -moJoin() { - local joiner part result target - - target=$1 - joiner=$2 - result=$3 - shift 3 - - for part in "$@"; do - result="$result$joiner$part" - done - - local "$target" && moIndirect "$target" "$result" -} - - -# Internal: Read a file into a variable. -# -# $1 - Variable name to receive the file's content -# $2 - Filename to load - if empty, defaults to /dev/stdin -# -# Returns nothing. -moLoadFile() { - local content len - - # The subshell removes any trailing newlines. We forcibly add - # a dot to the content to preserve all newlines. - # As a future optimization, it would be worth considering removing - # cat and replacing this with a read loop. - - content=$(cat -- "${2:-/dev/stdin}" && echo '.') || return 1 - len=$((${#content} - 1)) - content=${content:0:$len} # Remove last dot - - local "$1" && moIndirect "$1" "$content" -} - - -# Internal: Process a chunk of content some number of times. Writes output -# to stdout. -# -# $1 - Content to parse repeatedly -# $2 - Tag prefix (context name) -# $3-@ - Names to insert into the parsed content -# -# Returns nothing. -moLoop() { - local content context contextBase - - content=$1 - contextBase=$2 - shift 2 - - while [[ "${#@}" -gt 0 ]]; do - moFullTagName context "$contextBase" "$1" - moParse "$content" "$context" false - shift - done -} - - -# Internal: Parse a block of text, writing the result to stdout. -# -# $1 - Block of text to change -# $2 - Current name (the variable NAME for what {{.}} means) -# $3 - true when no content before this, false otherwise -# -# Returns nothing. -moParse() { - # Keep naming variables mo* here to not overwrite needed variables - # used in the string replacements - local moArgs moBlock moContent moCurrent moIsBeginning moNextIsBeginning moTag - - moCurrent=$2 - moIsBeginning=$3 - - # Find open tags - moSplit moContent "$1" '{{' '}}' - - while [[ "${#moContent[@]}" -gt 1 ]]; do - moTrimWhitespace moTag "${moContent[1]}" - moNextIsBeginning=false - - case $moTag in - '#'*) - # Loop, if/then, or pass content through function - # Sets context - moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" - moTrimWhitespace moTag "${moTag:1}" - - # Split arguments from the tag name. Arguments are passed to - # functions. - moArgs=$moTag - moTag=${moTag%% *} - moTag=${moTag%%$'\t'*} - moArgs=${moArgs:${#moTag}} - moFindEndTag moBlock "$moContent" "$moTag" - moFullTagName moTag "$moCurrent" "$moTag" - - if moTest "$moTag"; then - # Show / loop / pass through function - if moIsFunction "$moTag"; then - moCallFunction moContent "$moTag" "${moBlock[0]}" "$moArgs" - moParse "$moContent" "$moCurrent" false - moContent="${moBlock[2]}" - elif moIsArray "$moTag"; then - eval "moLoop \"\${moBlock[0]}\" \"$moTag\" \"\${!${moTag}[@]}\"" - else - moParse "${moBlock[0]}" "$moCurrent" true - fi - fi - - moContent="${moBlock[2]}" - ;; - - '>'*) - # Load partial - get name of file relative to cwd - moPartial moContent "${moContent[@]}" "$moIsBeginning" "$moCurrent" - moNextIsBeginning=${moContent[1]} - moContent=${moContent[0]} - ;; - - '/'*) - # Closing tag - If hit in this loop, we simply ignore - # Matching tags are found in moFindEndTag - moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" - ;; - - '^'*) - # Display section if named thing does not exist - moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" - moTrimWhitespace moTag "${moTag:1}" - moFindEndTag moBlock "$moContent" "$moTag" - moFullTagName moTag "$moCurrent" "$moTag" - - if ! moTest "$moTag"; then - moParse "${moBlock[0]}" "$moCurrent" false "$moCurrent" - fi - - moContent="${moBlock[2]}" - ;; - - '!'*) - # Comment - ignore the tag content entirely - # Trim spaces/tabs before the comment - moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" - ;; - - .) - # Current content (environment variable or function) - moStandaloneDenied moContent "${moContent[@]}" - moShow "$moCurrent" "$moCurrent" - ;; - - '=') - # Change delimiters - # Any two non-whitespace sequences separated by whitespace. - # This tag is ignored. - moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" - ;; - - '{'*) - # Unescaped - split on }}} not }} - moStandaloneDenied moContent "${moContent[@]}" - moContent="${moTag:1}"'}}'"$moContent" - moSplit moContent "$moContent" '}}}' - moTrimWhitespace moTag "${moContent[0]}" - moArgs=$moTag - moTag=${moTag%% *} - moTag=${moTag%%$'\t'*} - moArgs=${moArgs:${#moTag}} - moFullTagName moTag "$moCurrent" "$moTag" - moContent=${moContent[1]} - - # Now show the value - # Quote moArgs here, do not quote it later. - moShow "$moTag" "$moCurrent" "$moArgs" - ;; - - '&'*) - # Unescaped - moStandaloneDenied moContent "${moContent[@]}" - moTrimWhitespace moTag "${moTag:1}" - moFullTagName moTag "$moCurrent" "$moTag" - moShow "$moTag" "$moCurrent" - ;; - - *) - # Normal environment variable or function call - moStandaloneDenied moContent "${moContent[@]}" - moArgs=$moTag - moTag=${moTag%% *} - moTag=${moTag%%$'\t'*} - moArgs=${moArgs:${#moTag}} - moFullTagName moTag "$moCurrent" "$moTag" - - # Quote moArgs here, do not quote it later. - moShow "$moTag" "$moCurrent" "$moArgs" - ;; - esac - - moIsBeginning=$moNextIsBeginning - moSplit moContent "$moContent" '{{' '}}' - done - - echo -n "${moContent[0]}" -} - - -# Internal: Process a partial. -# -# Indentation should be applied to the entire partial. -# -# This sends back the "is beginning" flag because the newline after a -# standalone partial is consumed. That newline is very important in the middle -# of content. We send back this flag to reset the processing loop's -# `moIsBeginning` variable, so the software thinks we are back at the -# beginning of a file and standalone processing continues to work. -# -# Prefix all variables. -# -# $1 - Name of destination variable. Element [0] is the content, [1] is the -# true/false flag indicating if we are at the beginning of content. -# $2 - Content before the tag that was not yet written -# $3 - Tag content -# $4 - Content after the tag -# $5 - true/false: is this the beginning of the content? -# $6 - Current context name -# -# Returns nothing. -moPartial() { - # Namespace variables here to prevent conflicts. - local moContent moFilename moIndent moIsBeginning moPartial moStandalone moUnindented - - if moIsStandalone moStandalone "$2" "$4" "$5"; then - # shellcheck disable=SC2206 - moStandalone=( $moStandalone ) - echo -n "${2:0:${moStandalone[0]}}" - moIndent=${2:${moStandalone[0]}} - moContent=${4:${moStandalone[1]}} - moIsBeginning=true - else - moIndent="" - echo -n "$2" - moContent=$4 - moIsBeginning=$5 - fi - - moTrimWhitespace moFilename "${3:1}" - - # Execute in subshell to preserve current cwd and environment - ( - # It would be nice to remove `dirname` and use a function instead, - # but that's difficult when you're only given filenames. - cd "$(dirname -- "$moFilename")" || exit 1 - moUnindented="$( - moLoadFile moPartial "${moFilename##*/}" || exit 1 - moParse "${moPartial}" "$6" true - - # Fix bash handling of subshells and keep trailing whitespace. - # This is removed in moIndentLines. - echo -n "." - )" || exit 1 - moIndentLines moPartial "$moIndent" "$moUnindented" - echo -n "$moPartial" - ) || exit 1 - - # If this is a standalone tag, the trailing newline after the tag is - # removed and the contents of the partial are added, which typically - # contain a newline. We need to send a signal back to the processing - # loop that the moIsBeginning flag needs to be turned on again. - # - # [0] is the content, [1] is that flag. - local "$1" && moIndirectArray "$1" "$moContent" "$moIsBeginning" -} - - -# Internal: Show an environment variable or the output of a function to -# stdout. -# -# Limit/prefix any variables used. -# -# $1 - Name of environment variable or function -# $2 - Current context -# $3 - Arguments string if $1 is a function -# -# Returns nothing. -moShow() { - # Namespace these variables - local moJoined moNameParts moContent - - if moIsFunction "$1"; then - moCallFunction moContent "$1" "" "$3" - moParse "$moContent" "$2" false - return 0 - fi - - moSplit moNameParts "$1" "." - - if [[ -z "${moNameParts[1]-}" ]]; then - if moIsArray "$1"; then - eval moJoin moJoined "," "\${$1[@]}" - echo -n "$moJoined" - else - # shellcheck disable=SC2031 - if moTestVarSet "$1"; then - echo -n "${!1}" - elif [[ -n "${MO_FAIL_ON_UNSET-}" ]]; then - echo "Env variable not set: $1" >&2 - exit 1 - fi - fi - else - # Further subindexes are disallowed - eval "echo -n \"\${${moNameParts[0]}[${moNameParts[1]%%.*}]}\"" - fi -} - - -# Internal: Split a larger string into an array. -# -# $1 - Destination variable -# $2 - String to split -# $3 - Starting delimiter -# $4 - Ending delimiter (optional) -# -# Returns nothing. -moSplit() { - local pos result - - result=( "$2" ) - moFindString pos "${result[0]}" "$3" - - if [[ "$pos" -ne -1 ]]; then - # The first delimiter was found - result[1]=${result[0]:$pos + ${#3}} - result[0]=${result[0]:0:$pos} - - if [[ -n "${4-}" ]]; then - moFindString pos "${result[1]}" "$4" - - if [[ "$pos" -ne -1 ]]; then - # The second delimiter was found - result[2]="${result[1]:$pos + ${#4}}" - result[1]="${result[1]:0:$pos}" - fi - fi - fi - - local "$1" && moIndirectArray "$1" "${result[@]}" -} - - -# Internal: Handle the content for a standalone tag. This means removing -# whitespace (not newlines) before a tag and whitespace and a newline after -# a tag. That is, assuming, that the line is otherwise empty. -# -# $1 - Name of destination "content" variable. -# $2 - Content before the tag that was not yet written -# $3 - Tag content (not used) -# $4 - Content after the tag -# $5 - true/false: is this the beginning of the content? -# -# Returns nothing. -moStandaloneAllowed() { - local bytes - - if moIsStandalone bytes "$2" "$4" "$5"; then - # shellcheck disable=SC2206 - bytes=( $bytes ) - echo -n "${2:0:${bytes[0]}}" - local "$1" && moIndirect "$1" "${4:${bytes[1]}}" - else - echo -n "$2" - local "$1" && moIndirect "$1" "$4" - fi -} - - -# Internal: Handle the content for a tag that is never "standalone". No -# adjustments are made for newlines and whitespace. -# -# $1 - Name of destination "content" variable. -# $2 - Content before the tag that was not yet written -# $3 - Tag content (not used) -# $4 - Content after the tag -# -# Returns nothing. -moStandaloneDenied() { - echo -n "$2" - local "$1" && moIndirect "$1" "$4" -} - - -# Internal: Determines if the named thing is a function or if it is a -# non-empty environment variable. When MO_FALSE_IS_EMPTY is set to a -# non-empty value, then "false" is also treated is an empty value. -# -# Do not use variables without prefixes here if possible as this needs to -# check if any name exists in the environment -# -# $1 - Name of environment variable or function -# $2 - Current value (our context) -# MO_FALSE_IS_EMPTY - When set to a non-empty value, this will say the -# string value "false" is empty. -# -# Returns 0 if the name is not empty, 1 otherwise. When MO_FALSE_IS_EMPTY -# is set, this returns 1 if the name is "false". -moTest() { - # Test for functions - moIsFunction "$1" && return 0 - - if moIsArray "$1"; then - # Arrays must have at least 1 element - eval "[[ \"\${#${1}[@]}\" -gt 0 ]]" && return 0 - else - # If MO_FALSE_IS_EMPTY is set, then return 1 if the value of - # the variable is "false". - # shellcheck disable=SC2031 - [[ -n "${MO_FALSE_IS_EMPTY-}" ]] && [[ "${!1-}" == "false" ]] && return 1 - - # Environment variables must not be empty - [[ -n "${!1-}" ]] && return 0 - fi - - return 1 -} - -# Internal: Determine if a variable is assigned, even if it is assigned an empty -# value. -# -# $1 - Variable name to check. -# -# Returns true (0) if the variable is set, 1 if the variable is unset. -moTestVarSet() { - [[ "${!1-a}" == "${!1-b}" ]] -} - - -# Internal: Trim the leading whitespace only. -# -# $1 - Name of destination variable -# $2 - The string -# $3 - true/false - trim front? -# $4 - true/false - trim end? -# $5-@ - Characters to trim -# -# Returns nothing. -moTrimChars() { - local back current front last target varName - - target=$1 - current=$2 - front=$3 - back=$4 - last="" - shift 4 # Remove target, string, trim front flag, trim end flag - - while [[ "$current" != "$last" ]]; do - last=$current - - for varName in "$@"; do - $front && current="${current/#$varName}" - $back && current="${current/%$varName}" - done - done - - local "$target" && moIndirect "$target" "$current" -} - - -# Internal: Trim leading and trailing whitespace from a string. -# -# $1 - Name of variable to store trimmed string -# $2 - The string -# -# Returns nothing. -moTrimWhitespace() { - local result - - moTrimChars result "$2" true true $'\r' $'\n' $'\t' " " - local "$1" && moIndirect "$1" "$result" -} - - -# Internal: Displays the usage for mo. Pulls this from the file that -# contained the `mo` function. Can only work when the right filename -# comes is the one argument, and that only happens when `mo` is called -# with `$0` set to this file. -# -# $1 - Filename that has the help message -# -# Returns nothing. -moUsage() { - grep '^#/' "${MO_ORIGINAL_COMMAND}" | cut -c 4- - echo "" - echo "MO_VERSION=$MO_VERSION" -} - - -# Save the original command's path for usage later -MO_ORIGINAL_COMMAND="$(cd "${BASH_SOURCE[0]%/*}" || exit 1; pwd)/${BASH_SOURCE[0]##*/}" -MO_VERSION="2.2.0" - -# If sourced, load all functions. -# If executed, perform the actions as expected. -if [[ "$0" == "${BASH_SOURCE[0]}" ]] || [[ -z "${BASH_SOURCE[0]}" ]]; then - mo "$@" -fi diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf index 3517a6bc..de55d276 100644 --- a/docker/nginx/nginx.conf +++ b/docker/nginx/nginx.conf @@ -19,6 +19,9 @@ user root; worker_processes auto; +# Enables the use of JIT for regular expressions to speed-up their processing. +pcre_jit on; + #error_log logs/error.log; #error_log logs/error.log notice; #error_log logs/error.log info; diff --git a/docker/nginx/testing/.luacov b/docker/nginx/testing/.luacov new file mode 100644 index 00000000..2c55f3da --- /dev/null +++ b/docker/nginx/testing/.luacov @@ -0,0 +1,6 @@ +exclude = { + "/usr/local/openresty", -- internal openresty libraries + "rbusted", -- busted executable + "basexx", -- external library https://github.com/aiq/basexx +} +includeuntestedfiles = true diff --git a/docker/nginx/testing/Dockerfile b/docker/nginx/testing/Dockerfile new file mode 100644 index 00000000..ed7740b6 --- /dev/null +++ b/docker/nginx/testing/Dockerfile @@ -0,0 +1,11 @@ +FROM openresty/openresty:1.19.9.1-focal + +WORKDIR /etc/nginx + +RUN luarocks install lua-resty-http && \ + luarocks install hasher && \ + luarocks install busted + +COPY rbusted /etc/nginx/ + +CMD ["/etc/nginx/rbusted", "--verbose", "--pattern=spec", "/usr/local/openresty/site/lualib"] diff --git a/docker/nginx/testing/README.md b/docker/nginx/testing/README.md new file mode 100644 index 00000000..f40e8d95 --- /dev/null +++ b/docker/nginx/testing/README.md @@ -0,0 +1,3 @@ +# Running tests locally + +`docker run -v $(pwd)/docker/nginx/libs:/usr/local/openresty/site/lualib --rm -it $(docker build -q docker/nginx/testing)` diff --git a/docker/nginx/testing/rbusted b/docker/nginx/testing/rbusted new file mode 100755 index 00000000..94149350 --- /dev/null +++ b/docker/nginx/testing/rbusted @@ -0,0 +1,8 @@ +#!/usr/bin/env resty + +setmetatable(_G, nil) + +pcall(require, "luarocks.loader") + +-- Busted command-line runner +require "busted.runner"({ standalone = false }) diff --git a/docker/sia/Dockerfile b/docker/sia/Dockerfile index 887b92e9..953dd27c 100644 --- a/docker/sia/Dockerfile +++ b/docker/sia/Dockerfile @@ -5,12 +5,12 @@ ENV GOARCH amd64 ARG branch=portal-latest -RUN git clone https://gitlab.com/SkynetLabs/skyd.git Sia --single-branch --branch ${branch} -RUN make release --directory Sia +RUN git clone https://gitlab.com/SkynetLabs/skyd.git Sia --single-branch --branch ${branch} && \ + make release --directory Sia -FROM nebulouslabs/sia:latest +FROM nebulouslabs/sia:1.5.6 COPY --from=sia-builder /go/bin/ /usr/bin/ -RUN mv /usr/bin/skyd /usr/bin/siad || true && \ - mv /usr/bin/skyc /usr/bin/siac || true +RUN if [ -f "/usr/bin/skyd" ]; then mv /usr/bin/skyd /usr/bin/siad; fi && \ + if [ -f "/usr/bin/skyc" ]; then mv /usr/bin/skyc /usr/bin/siac; fi diff --git a/packages/dashboard-v2/Dockerfile b/packages/dashboard-v2/Dockerfile new file mode 100644 index 00000000..86fe89bc --- /dev/null +++ b/packages/dashboard-v2/Dockerfile @@ -0,0 +1,18 @@ +FROM node:16.14.2-alpine + +WORKDIR /usr/app + +COPY package.json \ + yarn.lock \ + ./ + +RUN yarn --frozen-lockfile + +COPY static ./static +COPY src ./src +COPY gatsby*.js \ + postcss.config.js \ + tailwind.config.js \ + ./ + +CMD ["sh", "-c", "yarn build && yarn serve --host 0.0.0.0 -p 9000"] diff --git a/packages/dashboard-v2/README.md b/packages/dashboard-v2/README.md index ab0421f8..e8924bc3 100644 --- a/packages/dashboard-v2/README.md +++ b/packages/dashboard-v2/README.md @@ -11,15 +11,20 @@ This is a Gatsby application. To run it locally, all you need is: ## Accessing remote APIs -To be able to log in on a local environment with your production credentials, you'll need to make the browser believe you're actually on the same domain, otherwise the browser will block the session cookie. +To have a fully functioning local environment, you'll need to make the browser believe you're actually on the same domain as a working API (i.e. a remote dev or production server) -- otherwise the browser will block the session cookie. +To do the trick, configure proper environment variables in the `.env.development` file. +This file allows to easily control which domain name you want to use locally and which API you'd like to access. -To do the trick, edit your `/etc/hosts` file and add a record like this: +Example: -``` -127.0.0.1 local.skynetpro.net +```env +GATSBY_PORTAL_DOMAIN=skynetfree.net # Use skynetfree.net APIs +GATSBY_HOST=local.skynetfree.net # Address of your local build ``` -then run `yarn develop:secure` -- it will run `gatsby develop` with `--https --host=local.skynetpro.net -p=443` options. -If you're on macOS, you may need to `sudo` the command to successfully bind to port `443`. +> It's recommended to keep the 2LD the same, so any cookies dispatched by the API work without issues. -> **NOTE:** This should become easier once we have a docker image for the new dashboard. +With the file configured, run `yarn develop:secure` -- it will run `gatsby develop` with `--https -p=443` options. +If you're on macOS, you may need to `sudo` the command to successfully bind to port `443` (https). + +Gatsby will automatically add a proper entry to your `/etc/hosts` file and clean it up when process exits. diff --git a/packages/dashboard-v2/gatsby-browser.js b/packages/dashboard-v2/gatsby-browser.js index 79b58e24..927fd206 100644 --- a/packages/dashboard-v2/gatsby-browser.js +++ b/packages/dashboard-v2/gatsby-browser.js @@ -1,4 +1,7 @@ import * as React from "react"; +import { SWRConfig } from "swr"; +import { Elements } from "@stripe/react-stripe-js"; +import { loadStripe } from "@stripe/stripe-js"; import "@fontsource/sora/300.css"; // light import "@fontsource/sora/400.css"; // normal import "@fontsource/sora/500.css"; // medium @@ -6,17 +9,24 @@ import "@fontsource/sora/600.css"; // semibold import "@fontsource/source-sans-pro/400.css"; // normal import "@fontsource/source-sans-pro/600.css"; // semibold import "./src/styles/global.css"; +import swrConfig from "./src/lib/swrConfig"; import { MODAL_ROOT_ID } from "./src/components/Modal"; import { PortalSettingsProvider } from "./src/contexts/portal-settings"; +const stripePromise = loadStripe(process.env.GATSBY_STRIPE_PUBLISHABLE_KEY); + export function wrapPageElement({ element, props }) { const Layout = element.type.Layout ?? React.Fragment; return ( - - {element} -
- + + + + {element} +
+ + + ); } diff --git a/packages/dashboard-v2/gatsby-config.js b/packages/dashboard-v2/gatsby-config.js index ce35de3a..d087fa65 100644 --- a/packages/dashboard-v2/gatsby-config.js +++ b/packages/dashboard-v2/gatsby-config.js @@ -1,10 +1,17 @@ +require("dotenv").config({ + path: `.env.${process.env.NODE_ENV}`, +}); + const { createProxyMiddleware } = require("http-proxy-middleware"); +const { GATSBY_PORTAL_DOMAIN } = process.env; + module.exports = { siteMetadata: { - title: `Accounts Dashboard`, - siteUrl: `https://www.yourdomain.tld`, + title: `Account Dashboard`, + siteUrl: `https://account.${GATSBY_PORTAL_DOMAIN}`, }, + pathPrefix: "/v2", trailingSlash: "never", plugins: [ "gatsby-plugin-image", @@ -18,19 +25,33 @@ module.exports = { resolve: "gatsby-source-filesystem", options: { name: "images", - path: "./src/images/", + path: "./static/images/", }, __key: "images", }, ], developMiddleware: (app) => { + // Proxy Accounts service API requests: app.use( "/api/", createProxyMiddleware({ - target: "https://account.skynetpro.net", + target: `https://account.${GATSBY_PORTAL_DOMAIN}`, secure: false, // Do not reject self-signed certificates. changeOrigin: true, }) ); + + // Proxy /skynet requests (e.g. uploads) + app.use( + ["/skynet", "/__internal/"], + createProxyMiddleware({ + target: `https://${GATSBY_PORTAL_DOMAIN}`, + secure: false, // Do not reject self-signed certificates. + changeOrigin: true, + pathRewrite: { + "^/skynet": "", + }, + }) + ); }, }; diff --git a/packages/dashboard-v2/gatsby-ssr.js b/packages/dashboard-v2/gatsby-ssr.js index 79b58e24..927fd206 100644 --- a/packages/dashboard-v2/gatsby-ssr.js +++ b/packages/dashboard-v2/gatsby-ssr.js @@ -1,4 +1,7 @@ import * as React from "react"; +import { SWRConfig } from "swr"; +import { Elements } from "@stripe/react-stripe-js"; +import { loadStripe } from "@stripe/stripe-js"; import "@fontsource/sora/300.css"; // light import "@fontsource/sora/400.css"; // normal import "@fontsource/sora/500.css"; // medium @@ -6,17 +9,24 @@ import "@fontsource/sora/600.css"; // semibold import "@fontsource/source-sans-pro/400.css"; // normal import "@fontsource/source-sans-pro/600.css"; // semibold import "./src/styles/global.css"; +import swrConfig from "./src/lib/swrConfig"; import { MODAL_ROOT_ID } from "./src/components/Modal"; import { PortalSettingsProvider } from "./src/contexts/portal-settings"; +const stripePromise = loadStripe(process.env.GATSBY_STRIPE_PUBLISHABLE_KEY); + export function wrapPageElement({ element, props }) { const Layout = element.type.Layout ?? React.Fragment; return ( - - {element} -
- + + + + {element} +
+ + + ); } diff --git a/packages/dashboard-v2/package.json b/packages/dashboard-v2/package.json index b760bf48..53edfa12 100644 --- a/packages/dashboard-v2/package.json +++ b/packages/dashboard-v2/package.json @@ -9,10 +9,10 @@ ], "scripts": { "develop": "gatsby develop", - "develop:secure": "gatsby develop --https --host=local.skynetpro.net -p=443", + "develop:secure": "dotenv -e .env.development -- gatsby develop --https -p=443", "start": "gatsby develop", - "build": "gatsby build", - "serve": "gatsby serve", + "build": "gatsby build --prefix-paths", + "serve": "gatsby serve --prefix-paths", "clean": "gatsby clean", "lint": "eslint .", "prettier": "prettier .", @@ -22,6 +22,8 @@ "dependencies": { "@fontsource/sora": "^4.5.3", "@fontsource/source-sans-pro": "^4.5.3", + "@stripe/react-stripe-js": "^1.7.1", + "@stripe/stripe-js": "^1.27.0", "classnames": "^2.3.1", "copy-text-to-clipboard": "^3.0.1", "dayjs": "^1.10.8", @@ -33,7 +35,6 @@ "nanoid": "^3.3.1", "path-browserify": "^1.0.1", "postcss": "^8.4.6", - "pretty-bytes": "^6.0.0", "react": "^17.0.1", "react-dom": "^17.0.1", "react-dropzone": "^12.0.4", @@ -60,6 +61,8 @@ "babel-loader": "^8.2.3", "babel-plugin-preval": "^5.1.0", "babel-plugin-styled-components": "^2.0.2", + "dotenv": "^16.0.0", + "dotenv-cli": "^5.1.0", "eslint": "^8.9.0", "eslint-config-react-app": "^7.0.0", "eslint-plugin-storybook": "^0.5.6", diff --git a/packages/dashboard-v2/src/components/APIKeyList/APIKey.js b/packages/dashboard-v2/src/components/APIKeyList/APIKey.js index 5cb6680a..b90a539e 100644 --- a/packages/dashboard-v2/src/components/APIKeyList/APIKey.js +++ b/packages/dashboard-v2/src/components/APIKeyList/APIKey.js @@ -4,7 +4,7 @@ import { useCallback, useState } from "react"; import { Alert } from "../Alert"; import { Button } from "../Button"; -import { AddSkylinkToAPIKeyForm } from "../forms/AddSkylinkToAPIKeyForm"; +import { AddSkylinkToSponsorKeyForm } from "../forms/AddSkylinkToSponsorKeyForm"; import { CogIcon, TrashIcon } from "../Icons"; import { Modal } from "../Modal"; @@ -13,7 +13,7 @@ import { useAPIKeyRemoval } from "./useAPIKeyRemoval"; export const APIKey = ({ apiKey, onRemoved, onEdited, onRemovalError }) => { const { id, name, createdAt, skylinks } = apiKey; - const isPublic = apiKey.public === "true"; + const isSponsorKey = apiKey.public === "true"; const [error, setError] = useState(null); const onSkylinkListEdited = useCallback(() => { @@ -53,9 +53,9 @@ export const APIKey = ({ apiKey, onRemoved, onEdited, onRemovalError }) => { }, [abortEdit]); const skylinksNumber = skylinks?.length ?? 0; - const isNotConfigured = isPublic && skylinksNumber === 0; + const isNotConfigured = isSponsorKey && skylinksNumber === 0; const skylinksPhrasePrefix = skylinksNumber === 0 ? "No" : skylinksNumber; - const skylinksPhrase = `${skylinksPhrasePrefix} ${skylinksNumber === 1 ? "skylink" : "skylinks"} configured`; + const skylinksPhrase = `${skylinksPhrasePrefix} ${skylinksNumber === 1 ? "skylink" : "skylinks"} sponsored`; return (
  • { {name || "unnamed key"} - + {isSponsorKey && ( + + )} {dayjs(createdAt).format("MMM DD, YYYY")} - {isPublic && ( + {isSponsorKey && ( )} - @@ -113,7 +121,7 @@ export const APIKey = ({ apiKey, onRemoved, onEdited, onRemovalError }) => { )} {editInitiated && ( -

    Covered skylinks

    +

    Sponsored skylinks

    {skylinks?.length > 0 ? (
      {skylinks.map((skylink) => ( @@ -121,7 +129,11 @@ export const APIKey = ({ apiKey, onRemoved, onEdited, onRemovalError }) => { {skylink} - @@ -133,7 +145,7 @@ export const APIKey = ({ apiKey, onRemoved, onEdited, onRemovalError }) => {
      {error && {error}} - +
      diff --git a/packages/dashboard-v2/src/components/AvatarUploader/AvatarUploader.js b/packages/dashboard-v2/src/components/AvatarUploader/AvatarUploader.js index 9f5bbc82..f97ca2d5 100644 --- a/packages/dashboard-v2/src/components/AvatarUploader/AvatarUploader.js +++ b/packages/dashboard-v2/src/components/AvatarUploader/AvatarUploader.js @@ -1,16 +1,16 @@ import { useEffect, useState } from "react"; import { useUser } from "../../contexts/user"; -import { SimpleUploadIcon } from "../Icons"; +// import { SimpleUploadIcon } from "../Icons"; -const AVATAR_PLACEHOLDER = "/images/avatar-placeholder.svg"; +import avatarPlaceholder from "../../../static/images/avatar-placeholder.svg"; export const AvatarUploader = (props) => { const { user } = useUser(); - const [imageUrl, setImageUrl] = useState(AVATAR_PLACEHOLDER); + const [imageUrl, setImageUrl] = useState(avatarPlaceholder); useEffect(() => { - setImageUrl(user.avatarUrl ?? AVATAR_PLACEHOLDER); + setImageUrl(user.avatarUrl ?? avatarPlaceholder); }, [user]); return ( @@ -20,6 +20,7 @@ export const AvatarUploader = (props) => { >
      + {/* TODO: uncomment when avatar uploads work
      - {/* TODO: actual uploading */}
      + */}
  • ); }; diff --git a/packages/dashboard-v2/src/components/Button/Button.js b/packages/dashboard-v2/src/components/Button/Button.js index 328d52cd..2a49244e 100644 --- a/packages/dashboard-v2/src/components/Button/Button.js +++ b/packages/dashboard-v2/src/components/Button/Button.js @@ -5,15 +5,25 @@ import styled from "styled-components"; /** * Primary UI component for user interaction */ -export const Button = styled.button.attrs(({ disabled, $primary, type }) => ({ - type, - className: cn("px-6 py-2.5 rounded-full font-sans uppercase text-xs tracking-wide transition-[opacity_filter]", { - "bg-primary text-palette-600": $primary, - "bg-white border-2 border-black text-palette-600": !$primary, - "cursor-not-allowed opacity-60": disabled, - "hover:brightness-90": !disabled, - }), -}))``; +export const Button = styled.button.attrs(({ as: polymorphicAs, disabled, $primary, type }) => { + // We want to default to type=button in most cases, but sometimes we use this component + // as a polymorphic one (i.e. for links), and then we should avoid setting `type` property, + // as it breaks styling in Safari. + const typeAttr = polymorphicAs && polymorphicAs !== "button" ? undefined : type; + + return { + type: typeAttr, + className: cn( + "px-6 py-2.5 inline-block rounded-full font-sans uppercase text-xs tracking-wide transition-[opacity_filter]", + { + "bg-primary text-palette-600": $primary, + "bg-white border-2 border-black text-palette-600": !$primary, + "cursor-not-allowed opacity-60": disabled, + "hover:brightness-90": !disabled, + } + ), + }; +})``; Button.propTypes = { /** diff --git a/packages/dashboard-v2/src/components/CopyButton.js b/packages/dashboard-v2/src/components/CopyButton.js index 479352d2..9cbf43d2 100644 --- a/packages/dashboard-v2/src/components/CopyButton.js +++ b/packages/dashboard-v2/src/components/CopyButton.js @@ -22,7 +22,7 @@ const TooltipContent = styled.div.attrs({ className: "bg-primary-light/10 text-palette-600 py-2 px-4 ", })``; -export const CopyButton = ({ value, className }) => { +export const CopyButton = ({ value, className, ariaLabel = "Copy" }) => { const containerRef = useRef(); const [copied, setCopied] = useState(false); const [timer, setTimer] = useState(null); @@ -39,7 +39,7 @@ export const CopyButton = ({ value, className }) => { return (
    - diff --git a/packages/dashboard-v2/src/components/CurrentPlan/CurrentPlan.js b/packages/dashboard-v2/src/components/CurrentPlan/CurrentPlan.js index f8a5cf9e..f9bc101a 100644 --- a/packages/dashboard-v2/src/components/CurrentPlan/CurrentPlan.js +++ b/packages/dashboard-v2/src/components/CurrentPlan/CurrentPlan.js @@ -3,6 +3,7 @@ import relativeTime from "dayjs/plugin/relativeTime"; import { useUser } from "../../contexts/user"; import useActivePlan from "../../hooks/useActivePlan"; +import humanBytes from "../../lib/humanBytes"; import { ContainerLoadingIndicator } from "../LoadingIndicator"; import LatestPayment from "./LatestPayment"; @@ -28,17 +29,20 @@ const CurrentPlan = () => { } return ( -
    +

    {activePlan.name}

    -
    - {activePlan.price === 0 &&

    100GB without paying a dime! 🎉

    } +
    + {activePlan.price === 0 && activePlan.limits && ( +

    {humanBytes(activePlan.limits.storageLimit)} without paying a dime! 🎉

    + )} {activePlan.price !== 0 && (user.subscriptionCancelAtPeriodEnd ? (

    Your subscription expires {dayjs(user.subscribedUntil).fromNow()}

    ) : (

    {dayjs(user.subscribedUntil).fromNow(true)} until the next payment

    ))} - + + {user.subscriptionStatus && }
    diff --git a/packages/dashboard-v2/src/components/CurrentPlan/SuggestedPlan.js b/packages/dashboard-v2/src/components/CurrentPlan/SuggestedPlan.js index 21aa9b48..ed59c382 100644 --- a/packages/dashboard-v2/src/components/CurrentPlan/SuggestedPlan.js +++ b/packages/dashboard-v2/src/components/CurrentPlan/SuggestedPlan.js @@ -14,7 +14,7 @@ const SuggestedPlan = ({ plans, activePlan }) => {

    Discover {nextPlan.name}

    {nextPlan.description}

    -
    diff --git a/packages/dashboard-v2/src/components/CurrentUsage/CurrentUsage.js b/packages/dashboard-v2/src/components/CurrentUsage/CurrentUsage.js index 44be79ed..cac40771 100644 --- a/packages/dashboard-v2/src/components/CurrentUsage/CurrentUsage.js +++ b/packages/dashboard-v2/src/components/CurrentUsage/CurrentUsage.js @@ -1,6 +1,6 @@ import { useEffect, useMemo, useState } from "react"; -import fileSize from "pretty-bytes"; import { Link } from "gatsby"; +import cn from "classnames"; import useSWR from "swr"; import { useUser } from "../../contexts/user"; @@ -9,6 +9,7 @@ import { ContainerLoadingIndicator } from "../LoadingIndicator"; import { GraphBar } from "./GraphBar"; import { UsageGraph } from "./UsageGraph"; +import humanBytes from "../../lib/humanBytes"; const useUsageData = () => { const { user } = useUser(); @@ -44,7 +45,7 @@ const useUsageData = () => { }; const size = (bytes) => { - const text = fileSize(bytes ?? 0, { maximumFractionDigits: 0 }); + const text = humanBytes(bytes ?? 0, { precision: 0 }); const [value, unit] = text.split(" "); return { @@ -62,7 +63,9 @@ const ErrorMessage = () => ( ); export default function CurrentUsage() { + const { activePlan, plans } = useActivePlan(); const { usage, error, loading } = useUsageData(); + const nextPlan = useMemo(() => plans.find(({ tier }) => tier > activePlan?.tier), [plans, activePlan]); const storageUsage = size(usage.storageUsed); const storageLimit = size(usage.storageLimit); const filesUsedLabel = useMemo(() => ({ value: usage.filesUsed, unit: "files" }), [usage.filesUsed]); @@ -89,19 +92,21 @@ export default function CurrentUsage() { {storageLimit.text}
    - +
    Files UPGRADE {" "} - {/* TODO: proper URL */} {usage.filesLimit}
    diff --git a/packages/dashboard-v2/src/components/CurrentUsage/GraphBar.js b/packages/dashboard-v2/src/components/CurrentUsage/GraphBar.js index 96421f6e..fd9a015e 100644 --- a/packages/dashboard-v2/src/components/CurrentUsage/GraphBar.js +++ b/packages/dashboard-v2/src/components/CurrentUsage/GraphBar.js @@ -12,18 +12,20 @@ const BarTip = styled.span.attrs({ })``; const BarLabel = styled.span.attrs({ - className: "bg-white rounded border-2 border-palette-200 px-3 whitespace-nowrap absolute shadow", + className: "usage-label bg-white rounded border-2 border-palette-200 px-3 whitespace-nowrap absolute shadow", })` - right: max(0%, ${({ $percentage }) => 100 - $percentage}%); - top: -0.5rem; - transform: translateX(50%); + ${({ $percentage }) => ` + left: max(0%, ${$percentage}%); + top: -0.5rem; + transform: translateX(-${$percentage}%); + `} `; -export const GraphBar = ({ value, limit, label }) => { +export const GraphBar = ({ value, limit, label, className }) => { const percentage = typeof limit !== "number" || limit === 0 ? 0 : (value / limit) * 100; return ( -
    +
    diff --git a/packages/dashboard-v2/src/components/CurrentUsage/UsageGraph.js b/packages/dashboard-v2/src/components/CurrentUsage/UsageGraph.js index 3f6f23c2..de4e7e46 100644 --- a/packages/dashboard-v2/src/components/CurrentUsage/UsageGraph.js +++ b/packages/dashboard-v2/src/components/CurrentUsage/UsageGraph.js @@ -1,9 +1,11 @@ import styled from "styled-components"; +import usageGraphBg from "../../../static/images/usage-graph-bg.svg"; + export const UsageGraph = styled.div.attrs({ className: "w-full my-3 grid grid-flow-row grid-rows-2", })` height: 146px; - background: url(/images/usage-graph-bg.svg) no-repeat; + background: url(${usageGraphBg}) no-repeat; background-size: cover; `; diff --git a/packages/dashboard-v2/src/components/DropdownMenu/DropdownMenu.js b/packages/dashboard-v2/src/components/DropdownMenu/DropdownMenu.js index 86cbad5f..1bf6f001 100644 --- a/packages/dashboard-v2/src/components/DropdownMenu/DropdownMenu.js +++ b/packages/dashboard-v2/src/components/DropdownMenu/DropdownMenu.js @@ -7,13 +7,10 @@ import { ChevronDownIcon } from "../Icons"; const dropDown = keyframes` 0% { - transform: scaleY(0); - } - 80% { - transform: scaleY(1.1); + transform: rotateX(-90deg); } 100% { - transform: scaleY(1); + transform: rotateX(0deg); } `; @@ -35,10 +32,11 @@ const Flyout = styled.div.attrs(({ open }) => ({ bg-white shadow-md shadow-palette-200/50 ${open ? "visible" : "invisible"}`, }))` + transform-origin: top center; animation: ${({ open }) => open ? css` - ${dropDown} 0.1s ease-in-out + ${dropDown} .15s ease-in-out forwards; ` : "none"}; `; diff --git a/packages/dashboard-v2/src/components/FileList/FileList.js b/packages/dashboard-v2/src/components/FileList/FileList.js index 6342b970..ad6087ed 100644 --- a/packages/dashboard-v2/src/components/FileList/FileList.js +++ b/packages/dashboard-v2/src/components/FileList/FileList.js @@ -1,29 +1,40 @@ -import * as React from "react"; +import { useState } from "react"; import useSWR from "swr"; import { useMedia } from "react-use"; import theme from "../../lib/theme"; import { ContainerLoadingIndicator } from "../LoadingIndicator"; -import { Select, SelectOption } from "../Select"; -import { Switch } from "../Switch"; -import { TextInputIcon } from "../TextInputIcon/TextInputIcon"; -import { SearchIcon } from "../Icons"; import FileTable from "./FileTable"; import useFormattedFilesData from "./useFormattedFilesData"; +import { MobileFileList } from "./MobileFileList"; +import { Pagination } from "./Pagination"; + +const PAGE_SIZE = 10; const FileList = ({ type }) => { const isMediumScreenOrLarger = useMedia(`(min-width: ${theme.screens.md})`); - const { data, error } = useSWR(`user/${type}?pageSize=10`); + const [offset, setOffset] = useState(0); + const baseUrl = `user/${type}?pageSize=${PAGE_SIZE}`; + const { + data, + error, + mutate: refreshList, + } = useSWR(`${baseUrl}&offset=${offset}`, { + revalidateOnMount: true, + }); const items = useFormattedFilesData(data?.items || []); + const count = data?.count || 0; - const setFilter = (name, value) => console.log("filter", name, "set to", value); + // Next page preloading + const hasMoreRecords = data ? data.offset + data.pageSize < data.count : false; + const nextPageOffset = hasMoreRecords ? data.offset + data.pageSize : offset; + useSWR(`${baseUrl}&offset=${nextPageOffset}`); if (!items.length) { return (
    - {/* TODO: proper error message */} {!data && !error && } {!data && error &&

    An error occurred while loading this data.

    } {data &&

    No {type} found.

    } @@ -32,42 +43,14 @@ const FileList = ({ type }) => { } return ( -
    -
    - } - onChange={console.log.bind(console)} - /> -
    - setFilter("showSmallFiles", value)} className="mr-8"> - - Show small files - - -
    - File type: - -
    -
    - Sort: - -
    -
    -
    - {/* TODO: mobile view (it's not tabular) */} - {isMediumScreenOrLarger ? : "Mobile view"} -
    + <> + {isMediumScreenOrLarger ? ( + + ) : ( + + )} + + ); }; diff --git a/packages/dashboard-v2/src/components/FileList/FileTable.js b/packages/dashboard-v2/src/components/FileList/FileTable.js index 90c9600f..88477648 100644 --- a/packages/dashboard-v2/src/components/FileList/FileTable.js +++ b/packages/dashboard-v2/src/components/FileList/FileTable.js @@ -2,110 +2,78 @@ import { CogIcon, ShareIcon } from "../Icons"; import { PopoverMenu } from "../PopoverMenu/PopoverMenu"; import { Table, TableBody, TableCell, TableHead, TableHeadCell, TableRow } from "../Table"; import { CopyButton } from "../CopyButton"; +import { useSkylinkOptions } from "./useSkylinkOptions"; +import { useSkylinkSharing } from "./useSkylinkSharing"; -const buildShareMenu = (item) => { - return [ - { - label: "Facebook", - callback: () => { - console.info("share to Facebook", item); - }, - }, - { - label: "Twitter", - callback: () => { - console.info("share to Twitter", item); - }, - }, - { - label: "Discord", - callback: () => { - console.info("share to Discord", item); - }, - }, - ]; -}; +const SkylinkOptionsMenu = ({ skylink, onUpdated }) => { + const { inProgress, options } = useSkylinkOptions({ skylink, onUpdated }); -const buildOptionsMenu = (item) => { - return [ - { - label: "Preview", - callback: () => { - console.info("preview", item); - }, - }, - { - label: "Download", - callback: () => { - console.info("download", item); - }, - }, - { - label: "Unpin", - callback: () => { - console.info("unpin", item); - }, - }, - { - label: "Report", - callback: () => { - console.info("report", item); - }, - }, - ]; -}; - -export default function FileTable({ items }) { return ( - - - - Name - Type - - Size - - Uploaded - Skylink - Activity - - - - {items.map((item) => { - const { id, name, type, size, date, skylink } = item; + + + + ); +}; - return ( - - {name} - {type} - - {size} - - {date} - -
    - - {skylink} -
    -
    - -
    - - - - - - -
    -
    -
    - ); - })} -
    -
    +const SkylinkSharingMenu = ({ skylink }) => { + const { options } = useSkylinkSharing(skylink); + + return ( + + + + ); +}; + +export default function FileTable({ items, onUpdated }) { + return ( +
    + + + + Name + Type + + Size + + Uploaded + Skylink + Activity + + + + {items.map((item) => { + const { id, name, type, size, date, skylink } = item; + + return ( + + {name} + {type} + + {size} + + {date} + +
    + + {skylink} +
    +
    + +
    + + +
    +
    +
    + ); + })} +
    +
    +
    ); } diff --git a/packages/dashboard-v2/src/components/FileList/MobileFileList.js b/packages/dashboard-v2/src/components/FileList/MobileFileList.js new file mode 100644 index 00000000..bd11aa10 --- /dev/null +++ b/packages/dashboard-v2/src/components/FileList/MobileFileList.js @@ -0,0 +1,84 @@ +import { useState } from "react"; +import cn from "classnames"; + +import { ChevronDownIcon } from "../Icons"; +import { useSkylinkSharing } from "./useSkylinkSharing"; +import { ContainerLoadingIndicator } from "../LoadingIndicator"; +import { useSkylinkOptions } from "./useSkylinkOptions"; + +const SharingMenu = ({ skylink }) => { + const { options } = useSkylinkSharing(skylink); + + return ( +
    + {options.map(({ label, callback }, index) => ( + + ))} +
    + ); +}; + +const OptionsMenu = ({ skylink, onUpdated }) => { + const { inProgress, options } = useSkylinkOptions({ skylink, onUpdated }); + + return ( +
    +
    + {options.map(({ label, callback }, index) => ( + + ))} +
    + {inProgress && ( + + )} +
    + ); +}; + +const ListItem = ({ item, onUpdated }) => { + const { name, type, size, date, skylink } = item; + const [open, setOpen] = useState(false); + + const toggle = () => setOpen((open) => !open); + + return ( +
    +
    +
    +
    {name}
    +
    + {type} + {size} + {date} +
    +
    + +
    +
    + + +
    +
    + ); +}; + +export const MobileFileList = ({ items, onUpdated }) => { + return ( +
    + {items.map((item) => ( + + ))} +
    + ); +}; diff --git a/packages/dashboard-v2/src/components/FileList/Pagination.js b/packages/dashboard-v2/src/components/FileList/Pagination.js new file mode 100644 index 00000000..248c03a3 --- /dev/null +++ b/packages/dashboard-v2/src/components/FileList/Pagination.js @@ -0,0 +1,32 @@ +import { Button } from "../Button"; + +export const Pagination = ({ count, offset, setOffset, pageSize }) => { + const start = count ? offset + 1 : 0; + const end = offset + pageSize > count ? count : offset + pageSize; + + const showPaginationButtons = offset > 0 || count > end; + + return ( + + ); +}; diff --git a/packages/dashboard-v2/src/components/FileList/useFormattedFilesData.js b/packages/dashboard-v2/src/components/FileList/useFormattedFilesData.js index 82d95090..10639458 100644 --- a/packages/dashboard-v2/src/components/FileList/useFormattedFilesData.js +++ b/packages/dashboard-v2/src/components/FileList/useFormattedFilesData.js @@ -1,6 +1,7 @@ import { useMemo } from "react"; -import prettyBytes from "pretty-bytes"; import dayjs from "dayjs"; +import { DATE_FORMAT } from "../../lib/config"; +import humanBytes from "../../lib/humanBytes"; const parseFileName = (fileName) => { const lastDotIndex = Math.max(0, fileName.lastIndexOf(".")) || Infinity; @@ -10,12 +11,12 @@ const parseFileName = (fileName) => { const formatItem = ({ size, name: rawFileName, uploadedOn, downloadedOn, ...rest }) => { const [name, type] = parseFileName(rawFileName); - const date = dayjs(uploadedOn || downloadedOn).format("MM/DD/YYYY; HH:MM"); + const date = dayjs(uploadedOn || downloadedOn).format(DATE_FORMAT); return { ...rest, date, - size: prettyBytes(size), + size: humanBytes(size, { precision: 2 }), type, name, }; diff --git a/packages/dashboard-v2/src/components/FileList/useSkylinkOptions.js b/packages/dashboard-v2/src/components/FileList/useSkylinkOptions.js new file mode 100644 index 00000000..ad116cd4 --- /dev/null +++ b/packages/dashboard-v2/src/components/FileList/useSkylinkOptions.js @@ -0,0 +1,35 @@ +import { useMemo, useState } from "react"; + +import accountsService from "../../services/accountsService"; +import skynetClient from "../../services/skynetClient"; + +export const useSkylinkOptions = ({ skylink, onUpdated }) => { + const [inProgress, setInProgress] = useState(false); + + const options = useMemo( + () => [ + { + label: "Preview", + callback: async () => window.open(await skynetClient.getSkylinkUrl(skylink)), + }, + { + label: "Download", + callback: () => skynetClient.downloadFile(skylink), + }, + { + label: "Unpin", + callback: async () => { + setInProgress(true); + await accountsService.delete(`user/uploads/${skylink}`); + await onUpdated(); // No need to setInProgress(false), since at this point this hook should already be unmounted + }, + }, + ], + [skylink, onUpdated] + ); + + return { + inProgress, + options, + }; +}; diff --git a/packages/dashboard-v2/src/components/FileList/useSkylinkSharing.js b/packages/dashboard-v2/src/components/FileList/useSkylinkSharing.js new file mode 100644 index 00000000..0b09302b --- /dev/null +++ b/packages/dashboard-v2/src/components/FileList/useSkylinkSharing.js @@ -0,0 +1,40 @@ +import { useEffect, useMemo, useState } from "react"; +import copy from "copy-text-to-clipboard"; + +import skynetClient from "../../services/skynetClient"; + +const COPY_LINK_LABEL = "Copy link"; + +export const useSkylinkSharing = (skylink) => { + const [copied, setCopied] = useState(false); + const [copyLabel, setCopyLabel] = useState(COPY_LINK_LABEL); + + useEffect(() => { + if (copied) { + setCopyLabel("Copied!"); + + const timeout = setTimeout(() => setCopied(false), 1500); + + return () => clearTimeout(timeout); + } else { + setCopyLabel(COPY_LINK_LABEL); + } + }, [copied]); + + const options = useMemo( + () => [ + { + label: copyLabel, + callback: async () => { + setCopied(true); + copy(await skynetClient.getSkylinkUrl(skylink)); + }, + }, + ], + [skylink, copyLabel] + ); + + return { + options, + }; +}; diff --git a/packages/dashboard-v2/src/components/Footer/Footer.js b/packages/dashboard-v2/src/components/Footer/Footer.js index 501d502d..8096337a 100644 --- a/packages/dashboard-v2/src/components/Footer/Footer.js +++ b/packages/dashboard-v2/src/components/Footer/Footer.js @@ -1,8 +1,19 @@ import * as React from "react"; +import styled from "styled-components"; + import { PageContainer } from "../PageContainer"; +const FooterLink = styled.a.attrs({ + className: "text-palette-400 underline decoration-dotted decoration-offset-4 decoration-1", + rel: "noreferrer", + target: "_blank", +})``; + export const Footer = () => ( -

    © Skynet Labs Inc. All rights reserved.

    +

    + Made by Skynet Labs. Open-sourced{" "} + on Github. +

    ); diff --git a/packages/dashboard-v2/src/components/HighlightedLink.js b/packages/dashboard-v2/src/components/HighlightedLink.js index 5fa6b079..f189b98b 100644 --- a/packages/dashboard-v2/src/components/HighlightedLink.js +++ b/packages/dashboard-v2/src/components/HighlightedLink.js @@ -2,5 +2,5 @@ import { Link } from "gatsby"; import styled from "styled-components"; export default styled(Link).attrs({ - className: "text-primary underline-offset-3 decoration-dotted hover:text-primary-light hover:underline", + className: "text-primary underline-offset-2 decoration-1 decoration-dotted hover:text-primary-light hover:underline", })``; diff --git a/packages/dashboard-v2/src/components/LatestActivity/ActivityTable.js b/packages/dashboard-v2/src/components/LatestActivity/ActivityTable.js index 345a2daa..3bea206f 100644 --- a/packages/dashboard-v2/src/components/LatestActivity/ActivityTable.js +++ b/packages/dashboard-v2/src/components/LatestActivity/ActivityTable.js @@ -3,12 +3,13 @@ import useSWR from "swr"; import { Table, TableBody, TableCell, TableRow } from "../Table"; import { ContainerLoadingIndicator } from "../LoadingIndicator"; +import useFormattedFilesData from "../FileList/useFormattedFilesData"; -import useFormattedActivityData from "./useFormattedActivityData"; +import { ViewAllLink } from "./ViewAllLink"; export default function ActivityTable({ type }) { const { data, error } = useSWR(`user/${type}?pageSize=3`); - const items = useFormattedActivityData(data?.items || []); + const items = useFormattedFilesData(data?.items || []); if (!items.length) { return ( @@ -22,20 +23,23 @@ export default function ActivityTable({ type }) { } return ( - - - {items.map(({ id, name, type, size, date, skylink }) => ( - - {name} - {type} - - {size} - - {date} - {skylink} - - ))} - -
    + <> + + + {items.map(({ id, name, type, size, date, skylink }) => ( + + {name} + {type} + + {size} + + {date} + {skylink} + + ))} + +
    + + ); } diff --git a/packages/dashboard-v2/src/components/LatestActivity/LatestActivity.js b/packages/dashboard-v2/src/components/LatestActivity/LatestActivity.js index 87825661..627314b7 100644 --- a/packages/dashboard-v2/src/components/LatestActivity/LatestActivity.js +++ b/packages/dashboard-v2/src/components/LatestActivity/LatestActivity.js @@ -1,36 +1,13 @@ import * as React from "react"; -import { Link } from "gatsby"; import { Panel } from "../Panel"; -import { Tab, TabPanel, Tabs } from "../Tabs"; -import { ArrowRightIcon } from "../Icons"; import ActivityTable from "./ActivityTable"; -const ViewAllLink = (props) => ( - - - - - View all - -); - export default function LatestActivity() { return ( - - - - - - - - - - - - - + + ); } diff --git a/packages/dashboard-v2/src/components/LatestActivity/ViewAllLink.js b/packages/dashboard-v2/src/components/LatestActivity/ViewAllLink.js new file mode 100644 index 00000000..2cee28ea --- /dev/null +++ b/packages/dashboard-v2/src/components/LatestActivity/ViewAllLink.js @@ -0,0 +1,12 @@ +import { Link } from "gatsby"; + +import { ArrowRightIcon } from "../Icons"; + +export const ViewAllLink = (props) => ( + + + + + View all + +); diff --git a/packages/dashboard-v2/src/components/LatestActivity/useFormattedActivityData.js b/packages/dashboard-v2/src/components/LatestActivity/useFormattedActivityData.js deleted file mode 100644 index cf55703c..00000000 --- a/packages/dashboard-v2/src/components/LatestActivity/useFormattedActivityData.js +++ /dev/null @@ -1,26 +0,0 @@ -import { useMemo } from "react"; -import prettyBytes from "pretty-bytes"; -import dayjs from "dayjs"; - -const parseFileName = (fileName) => { - const lastDotIndex = Math.max(0, fileName.lastIndexOf(".")) || Infinity; - - return [fileName.substr(0, lastDotIndex), fileName.substr(lastDotIndex)]; -}; - -const formatItem = ({ size, name: rawFileName, uploadedOn, downloadedOn, ...rest }) => { - const [name, type] = parseFileName(rawFileName); - const date = dayjs(uploadedOn || downloadedOn).format("MM/DD/YYYY; HH:MM"); - - return { - ...rest, - date, - size: prettyBytes(size), - type, - name, - }; -}; - -const useFormattedActivityData = (items) => useMemo(() => items.map(formatItem), [items]); - -export default useFormattedActivityData; diff --git a/packages/dashboard-v2/src/components/Metadata/Metadata.js b/packages/dashboard-v2/src/components/Metadata/Metadata.js new file mode 100644 index 00000000..0e1dd2b6 --- /dev/null +++ b/packages/dashboard-v2/src/components/Metadata/Metadata.js @@ -0,0 +1,42 @@ +import { Helmet } from "react-helmet"; +import { graphql, useStaticQuery } from "gatsby"; + +import favicon from "../../../static/favicon.ico"; +import favicon16 from "../../../static/favicon-16x16.png"; +import favicon32 from "../../../static/favicon-32x32.png"; +import appleIcon144 from "../../../static/apple-touch-icon-144x144.png"; +import appleIcon152 from "../../../static/apple-touch-icon-152x152.png"; +import msTileIcon from "../../../static/mstile-144x144.png"; + +export const Metadata = ({ children }) => { + const { site } = useStaticQuery( + graphql` + query Q { + site { + siteMetadata { + title + } + } + } + ` + ); + + const { title } = site.siteMetadata; + + return ( + + + + + + + + + + + + + {children} + + ); +}; diff --git a/packages/dashboard-v2/src/components/Metadata/index.js b/packages/dashboard-v2/src/components/Metadata/index.js new file mode 100644 index 00000000..8abb6696 --- /dev/null +++ b/packages/dashboard-v2/src/components/Metadata/index.js @@ -0,0 +1 @@ +export * from "./Metadata"; diff --git a/packages/dashboard-v2/src/components/Modal/Modal.js b/packages/dashboard-v2/src/components/Modal/Modal.js index c183e190..ac7bd98e 100644 --- a/packages/dashboard-v2/src/components/Modal/Modal.js +++ b/packages/dashboard-v2/src/components/Modal/Modal.js @@ -11,7 +11,7 @@ export const Modal = ({ children, className, onClose }) => (
    - {children} diff --git a/packages/dashboard-v2/src/components/NavBar/NavBar.js b/packages/dashboard-v2/src/components/NavBar/NavBar.js index f75030bb..9ad8e91c 100644 --- a/packages/dashboard-v2/src/components/NavBar/NavBar.js +++ b/packages/dashboard-v2/src/components/NavBar/NavBar.js @@ -94,9 +94,10 @@ export const NavBar = () => { partiallyActive /> diff --git a/packages/dashboard-v2/src/components/Panel/Panel.js b/packages/dashboard-v2/src/components/Panel/Panel.js index 27551ecd..fe50cc89 100644 --- a/packages/dashboard-v2/src/components/Panel/Panel.js +++ b/packages/dashboard-v2/src/components/Panel/Panel.js @@ -14,8 +14,8 @@ const PanelTitle = styled.h6.attrs({ * * These additional props will be rendered onto the panel's body element. */ -export const Panel = ({ title, ...props }) => ( -
    +export const Panel = ({ title, wrapperClassName, ...props }) => ( +
    {title && {title}}
    @@ -26,8 +26,13 @@ Panel.propTypes = { * Label of the panel */ title: PropTypes.oneOfType([PropTypes.string, PropTypes.node]), + /** + * CSS class to be applied to the external container + */ + wrapperClassName: PropTypes.string, }; Panel.defaultProps = { title: "", + wrapperClassName: "", }; diff --git a/packages/dashboard-v2/src/components/PopoverMenu/PopoverMenu.js b/packages/dashboard-v2/src/components/PopoverMenu/PopoverMenu.js index 1826cd92..dd5a8597 100644 --- a/packages/dashboard-v2/src/components/PopoverMenu/PopoverMenu.js +++ b/packages/dashboard-v2/src/components/PopoverMenu/PopoverMenu.js @@ -2,6 +2,9 @@ import { Children, cloneElement, useRef, useState } from "react"; import PropTypes from "prop-types"; import { useClickAway } from "react-use"; import styled, { css, keyframes } from "styled-components"; +import cn from "classnames"; + +import { ContainerLoadingIndicator } from "../LoadingIndicator"; const dropDown = keyframes` 0% { @@ -41,15 +44,15 @@ const Option = styled.li.attrs({ hover:before:content-['']`, })``; -export const PopoverMenu = ({ options, children, openClassName, ...props }) => { +export const PopoverMenu = ({ options, children, openClassName, inProgress, ...props }) => { const containerRef = useRef(); const [open, setOpen] = useState(false); useClickAway(containerRef, () => setOpen(false)); - const handleChoice = (callback) => () => { + const handleChoice = (callback) => async () => { + await callback(); setOpen(false); - callback(); }; return ( @@ -62,11 +65,16 @@ export const PopoverMenu = ({ options, children, openClassName, ...props }) => { )} {open && ( - {options.map(({ label, callback }) => ( - - ))} +
    + {options.map(({ label, callback }) => ( + + ))} + {inProgress && ( + + )} +
    )} @@ -87,4 +95,9 @@ PopoverMenu.propTypes = { callback: PropTypes.func.isRequired, }) ).isRequired, + + /** + * If true, a loading icon will be displayed to signal an async action is taking place. + */ + inProgress: PropTypes.bool, }; diff --git a/packages/dashboard-v2/src/components/Slider/Bullets.js b/packages/dashboard-v2/src/components/Slider/Bullets.js index c090cef4..10b0a0ed 100644 --- a/packages/dashboard-v2/src/components/Slider/Bullets.js +++ b/packages/dashboard-v2/src/components/Slider/Bullets.js @@ -12,6 +12,7 @@ export default function Bullets({ visibleSlides, activeIndex, allSlides, changeS .map((_, index) => ( Add, or drop your files here @@ -114,7 +118,7 @@ const Uploader = ({ mode }) => {
    {uploads.length > 0 && ( -
    +
    {uploads.map((upload) => ( ))} diff --git a/packages/dashboard-v2/src/components/Uploader/UploaderItem.js b/packages/dashboard-v2/src/components/Uploader/UploaderItem.js index 8ea279d6..7e19051b 100644 --- a/packages/dashboard-v2/src/components/Uploader/UploaderItem.js +++ b/packages/dashboard-v2/src/components/Uploader/UploaderItem.js @@ -1,6 +1,5 @@ import * as React from "react"; import cn from "classnames"; -import bytes from "pretty-bytes"; import { StatusCodes } from "http-status-codes"; import copy from "copy-text-to-clipboard"; import path from "path-browserify"; @@ -9,6 +8,7 @@ import { ProgressBar } from "./ProgressBar"; import UploaderItemIcon from "./UploaderItemIcon"; import buildUploadErrorMessage from "./buildUploadErrorMessage"; import skynetClient from "../../services/skynetClient"; +import humanBytes from "../../lib/humanBytes"; const getFilePath = (file) => file.webkitRelativePath || file.path || file.name; @@ -88,7 +88,7 @@ export default function UploaderItem({ onUploadStateChange, upload }) {
    {upload.status === "uploading" && ( - Uploading {bytes(upload.file.size * upload.progress)} of {bytes(upload.file.size)} + Uploading {humanBytes(upload.file.size * upload.progress)} of {humanBytes(upload.file.size)} )} {upload.status === "enqueued" && Upload in queue, please wait} @@ -109,7 +109,6 @@ export default function UploaderItem({ onUploadStateChange, upload }) { {upload.status === "uploading" && ( {Math.floor(upload.progress * 100)}% )} - {upload.status === "processing" && Wait} {upload.status === "complete" && ( - )} + + +
    +
    )} @@ -110,5 +104,5 @@ AddAPIKeyForm.displayName = "AddAPIKeyForm"; AddAPIKeyForm.propTypes = { onSuccess: PropTypes.func.isRequired, - type: PropTypes.oneOf([APIKeyType.Public, APIKeyType.General]).isRequired, + type: PropTypes.oneOf([APIKeyType.Sponsor, APIKeyType.General]).isRequired, }; diff --git a/packages/dashboard-v2/src/components/forms/AddSkylinkToAPIKeyForm.js b/packages/dashboard-v2/src/components/forms/AddSkylinkToSponsorKeyForm.js similarity index 89% rename from packages/dashboard-v2/src/components/forms/AddSkylinkToAPIKeyForm.js rename to packages/dashboard-v2/src/components/forms/AddSkylinkToSponsorKeyForm.js index 55f27a01..71742096 100644 --- a/packages/dashboard-v2/src/components/forms/AddSkylinkToAPIKeyForm.js +++ b/packages/dashboard-v2/src/components/forms/AddSkylinkToSponsorKeyForm.js @@ -19,7 +19,7 @@ const newSkylinkSchema = Yup.object().shape({ }), }); -export const AddSkylinkToAPIKeyForm = ({ addSkylink }) => ( +export const AddSkylinkToSponsorKeyForm = ({ addSkylink }) => ( ( {isSubmitting ? ( ) : ( - )} @@ -58,6 +58,6 @@ export const AddSkylinkToAPIKeyForm = ({ addSkylink }) => ( ); -AddSkylinkToAPIKeyForm.propTypes = { +AddSkylinkToSponsorKeyForm.propTypes = { addSkylink: PropTypes.func.isRequired, }; diff --git a/packages/dashboard-v2/src/components/forms/AddPublicAPIKeyForm.js b/packages/dashboard-v2/src/components/forms/AddSponsorKeyForm.js similarity index 89% rename from packages/dashboard-v2/src/components/forms/AddPublicAPIKeyForm.js rename to packages/dashboard-v2/src/components/forms/AddSponsorKeyForm.js index 2184c513..236cdc9b 100644 --- a/packages/dashboard-v2/src/components/forms/AddPublicAPIKeyForm.js +++ b/packages/dashboard-v2/src/components/forms/AddSponsorKeyForm.js @@ -25,7 +25,7 @@ const skylinkValidator = (optional) => (value) => { } }; -const newPublicAPIKeySchema = Yup.object().shape({ +const newSponsorKeySchema = Yup.object().shape({ name: Yup.string(), skylinks: Yup.array().of(Yup.string().test("skylink", "Provide a valid Skylink", skylinkValidator(false))), nextSkylink: Yup.string().when("skylinks", { @@ -41,7 +41,7 @@ const State = { Failure: "FAILURE", }; -export const AddPublicAPIKeyForm = forwardRef(({ onSuccess }, ref) => { +export const AddSponsorKeyForm = forwardRef(({ onSuccess }, ref) => { const [state, setState] = useState(State.Pure); const [generatedKey, setGeneratedKey] = useState(null); @@ -52,14 +52,14 @@ export const AddPublicAPIKeyForm = forwardRef(({ onSuccess }, ref) => { return (
    {state === State.Success && ( - + Success!

    Please copy your new API key below. We'll never show it again!

    -
    +
    {generatedKey} - +
    )} @@ -72,7 +72,7 @@ export const AddPublicAPIKeyForm = forwardRef(({ onSuccess }, ref) => { skylinks: [], nextSkylink: "", }} - validationSchema={newPublicAPIKeySchema} + validationSchema={newSponsorKeySchema} onSubmit={async ({ name, skylinks, nextSkylink }, { resetForm }) => { try { const { key } = await accountsService @@ -80,7 +80,7 @@ export const AddPublicAPIKeyForm = forwardRef(({ onSuccess }, ref) => { json: { name, public: "true", - skylinks: [...skylinks, nextSkylink].filter(Boolean).map(parseSkylink), + skylinks: [...skylinks, nextSkylink].filter(Boolean).map((skylink) => parseSkylink(skylink)), }, }) .json(); @@ -101,14 +101,14 @@ export const AddPublicAPIKeyForm = forwardRef(({ onSuccess }, ref) => { type="text" id="name" name="name" - label="Public API Key Name" + label="Sponsor API Key Name" placeholder="my_applications_statistics" error={errors.name} touched={touched.name} />
    -
    Skylinks accessible with the new key
    +
    Skylinks sponsored by the new key
    { @@ -137,7 +137,7 @@ export const AddPublicAPIKeyForm = forwardRef(({ onSuccess }, ref) => { touched={skylinksTouched[index]} /> - @@ -160,6 +160,7 @@ export const AddPublicAPIKeyForm = forwardRef(({ onSuccess }, ref) => { />
    @@ -191,8 +192,8 @@ export const AddPublicAPIKeyForm = forwardRef(({ onSuccess }, ref) => { ); }); -AddPublicAPIKeyForm.displayName = "AddAPIKeyForm"; +AddSponsorKeyForm.displayName = "AddSponsorKeyForm"; -AddPublicAPIKeyForm.propTypes = { +AddSponsorKeyForm.propTypes = { onSuccess: PropTypes.func.isRequired, }; diff --git a/packages/dashboard-v2/src/components/forms/LoginForm.js b/packages/dashboard-v2/src/components/forms/LoginForm.js index a5a1deeb..eeafc092 100644 --- a/packages/dashboard-v2/src/components/forms/LoginForm.js +++ b/packages/dashboard-v2/src/components/forms/LoginForm.js @@ -82,7 +82,7 @@ export const LoginForm = ({ onSuccess }) => {

    - Don't have an account? Sign up + Don't have an account? Sign up

    )} diff --git a/packages/dashboard-v2/src/components/forms/SignUpForm.js b/packages/dashboard-v2/src/components/forms/SignUpForm.js index 6725c673..c6ebed3e 100644 --- a/packages/dashboard-v2/src/components/forms/SignUpForm.js +++ b/packages/dashboard-v2/src/components/forms/SignUpForm.js @@ -32,14 +32,16 @@ export const SignUpForm = ({ onSuccess, onFailure }) => ( validationSchema={registrationSchema} onSubmit={async ({ email, password }, { setErrors }) => { try { - await accountsService.post("user", { - json: { - email, - password, - }, - }); + const user = await accountsService + .post("user", { + json: { + email, + password, + }, + }) + .json(); - onSuccess(); + onSuccess(user); } catch (err) { let isFormErrorSet = false; diff --git a/packages/dashboard-v2/src/contexts/plans/PlansProvider.js b/packages/dashboard-v2/src/contexts/plans/PlansProvider.js index 135c9bcb..7c6579ad 100644 --- a/packages/dashboard-v2/src/contexts/plans/PlansProvider.js +++ b/packages/dashboard-v2/src/contexts/plans/PlansProvider.js @@ -19,7 +19,14 @@ const aggregatePlansAndLimits = (plans, limits, { includeFreePlan }) => { // Decorate each plan with its corresponding limits data, if available. if (limits?.length) { - return sortedPlans.map((plan) => ({ ...plan, limits: limits[plan.tier] || null })); + return limits.map((limitsDescriptor, index) => { + const asssociatedPlan = sortedPlans.find((plan) => plan.tier === index) || {}; + + return { + ...asssociatedPlan, + limits: limitsDescriptor || null, + }; + }); } // If we don't have the limits data yet, set just return the plans. @@ -40,10 +47,12 @@ export const PlansProvider = ({ children }) => { if (plansError || limitsError) { setLoading(false); setError(plansError || limitsError); - } else if (rawPlans) { + } else if (rawPlans || limits) { setLoading(false); setPlans( - aggregatePlansAndLimits(rawPlans, limits?.userLimits, { includeFreePlan: !settings.isSubscriptionRequired }) + aggregatePlansAndLimits(rawPlans || [], limits?.userLimits, { + includeFreePlan: !settings.isSubscriptionRequired, + }) ); } }, [rawPlans, limits, plansError, limitsError, settings.isSubscriptionRequired]); diff --git a/packages/dashboard-v2/src/contexts/portal-settings/PortalSettingsProvider.js b/packages/dashboard-v2/src/contexts/portal-settings/PortalSettingsProvider.js index a5a033c8..5d8de1ab 100644 --- a/packages/dashboard-v2/src/contexts/portal-settings/PortalSettingsProvider.js +++ b/packages/dashboard-v2/src/contexts/portal-settings/PortalSettingsProvider.js @@ -16,7 +16,7 @@ const fetcher = async (path) => { }; export const PortalSettingsProvider = ({ children }) => { - const { data, error } = useSWRImmutable("/__internal/do/not/use/accounts", fetcher); + const { data, error } = useSWRImmutable("__internal/do/not/use/accounts", fetcher); const [loading, setLoading] = useState(true); const [settings, setSettings] = useState(defaultSettings); diff --git a/packages/dashboard-v2/src/contexts/user/UserProvider.js b/packages/dashboard-v2/src/contexts/user/UserProvider.js index bb10ffe4..014d5b7f 100644 --- a/packages/dashboard-v2/src/contexts/user/UserProvider.js +++ b/packages/dashboard-v2/src/contexts/user/UserProvider.js @@ -1,17 +1,41 @@ +import { navigate } from "gatsby"; import { useEffect, useState } from "react"; import useSWRImmutable from "swr/immutable"; +import { UnauthorizedError } from "../../lib/swrConfig"; +import { FullScreenLoadingIndicator } from "../../components/LoadingIndicator"; import { UserContext } from "./UserContext"; -export const UserProvider = ({ children }) => { +export const UserProvider = ({ children, allowGuests = false, allowAuthenticated = true }) => { const { data: user, error, mutate } = useSWRImmutable("user"); const [loading, setLoading] = useState(true); useEffect(() => { - if (user || error) { - setLoading(false); - } - }, [user, error]); + const guard = async () => { + if (user) { + if (!allowAuthenticated) { + navigate("/"); + } else { + setLoading(false); + } + } else if (error) { + if (error instanceof UnauthorizedError && !allowGuests) { + await navigate(`/auth/login?return_to=${encodeURIComponent(window.location.href)}`); + } else { + setLoading(false); + } + } else if (user === null) { + setLoading(false); + } + }; - return {children}; + guard(); + }, [user, error, allowGuests, allowAuthenticated]); + + return ( + + {loading && } + {!loading && children} + + ); }; diff --git a/packages/dashboard-v2/src/hooks/useUpgradeRedirect.js b/packages/dashboard-v2/src/hooks/useUpgradeRedirect.js index 5529c152..037db65f 100644 --- a/packages/dashboard-v2/src/hooks/useUpgradeRedirect.js +++ b/packages/dashboard-v2/src/hooks/useUpgradeRedirect.js @@ -17,8 +17,7 @@ export default function useUpgradeRedirect() { if (isDataLoaded) { if (settings.isSubscriptionRequired && !hasPaidSubscription) { - console.log("redirecting", user, settings); - navigate("/upgrade"); + navigate("/payments"); } else { setVerifyingSubscription(false); } diff --git a/packages/dashboard-v2/src/layouts/AuthLayout.js b/packages/dashboard-v2/src/layouts/AuthLayout.js index 85604321..323e4319 100644 --- a/packages/dashboard-v2/src/layouts/AuthLayout.js +++ b/packages/dashboard-v2/src/layouts/AuthLayout.js @@ -1,14 +1,15 @@ import * as React from "react"; import styled from "styled-components"; -import { SWRConfig } from "swr"; import { UserProvider } from "../contexts/user"; -import { guestsOnly, allUsers } from "../lib/swrConfig"; + +import skynetLogo from "../../static/images/logo-black-text.svg"; +import authBg from "../../static/images/auth-bg.svg"; const Layout = styled.div.attrs({ className: "min-h-screen w-screen bg-black flex", })` - background-image: url(/images/auth-bg.svg); + background-image: url(${authBg}); background-repeat: no-repeat; background-position: center center; `; @@ -22,29 +23,39 @@ const Content = styled.div.attrs({ })``; const AuthLayout = - (swrConfig) => - ({ children }) => { - return ( + (userProviderProps) => + ({ children }) => + ( <> - - - - -
    -

    - The decentralized revolution starts with decentralized storage -

    + + + +
    +

    + The decentralized revolution starts with decentralized storage +

    +
    +
    + +
    +
    + Skynet
    - - {children} - - - + {children} +
    +
    +
    +
    ); - }; // Some pages (e.g. email confirmation) need to be accessible to both logged-in and guest users. -export const AllUsersAuthLayout = AuthLayout(allUsers); +export const AllUsersAuthLayout = AuthLayout({ + allowGuests: true, + allowAuthenticated: true, +}); -export default AuthLayout(guestsOnly); +export default AuthLayout({ + allowGuests: true, + allowAuthenticated: false, +}); diff --git a/packages/dashboard-v2/src/layouts/DashboardLayout.js b/packages/dashboard-v2/src/layouts/DashboardLayout.js index 76af9218..8ac7c393 100644 --- a/packages/dashboard-v2/src/layouts/DashboardLayout.js +++ b/packages/dashboard-v2/src/layouts/DashboardLayout.js @@ -1,19 +1,18 @@ import * as React from "react"; import styled from "styled-components"; -import { SWRConfig } from "swr"; - -import { authenticatedOnly } from "../lib/swrConfig"; import { PageContainer } from "../components/PageContainer"; -import { NavBar } from "../components/Navbar"; +import { NavBar } from "../components/NavBar"; import { Footer } from "../components/Footer"; import { UserProvider, useUser } from "../contexts/user"; import { FullScreenLoadingIndicator } from "../components/LoadingIndicator"; +import dashboardBg from "../../static/images/dashboard-bg.svg"; + const Wrapper = styled.div.attrs({ className: "min-h-screen overflow-hidden", })` - background-image: url(/images/dashboard-bg.svg); + background-image: url(${dashboardBg}); background-position: center -280px; background-repeat: no-repeat; `; @@ -30,22 +29,16 @@ const Layout = ({ children }) => { ); }; -const DashboardLayout = ({ children }) => { - return ( - <> - - - - - -
    {children}
    -
    -