chore: remove usage of buf and rust in docker image

This reverts commit f6b28b0ee0.

.github/workflows/ci.yml

@@ -0,0 +1,35 @@
+name: Build/Publish
+
+on:
+    workflow_dispatch:
+        inputs:
+            debug_enabled:
+                description: Debug
+                type: boolean
+                default: false
+    push:
+        branches:
+            - master
+            - develop
+            - develop-*
+
+jobs:
+    build:
+        runs-on: ubuntu-latest
+        steps:
+            -   uses: actions/checkout@v3
+            -   name: Install SSH key
+                uses: shimataro/ssh-key-action@v2
+                with:
+                    key: ${{ secrets.GITEA_SSH_KEY }}
+                    known_hosts: ${{ secrets.GITEA_KNOWN_HOST }}
+            -   name: Publish
+                uses: go-semantic-release/action@v1
+                with:
+                    github-token: ${{ secrets.GITHUB_TOKEN }}
+                    prerelease: true
+            -   name: Setup tmate session
+                uses: mxschmitt/action-tmate@v3
+                if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled && failure() }}
+                with:
+                    limit-access-to-actor: true

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "api/account/app"]
+	path = api/account/app
+	url = https://git.lumeweb.com/LumeWeb/portal-dashboard.git
+	branch = develop

.semrelrc

@@ -0,0 +1,27 @@
+{
+    "plugins": {
+        "commit-analyzer": {
+            "name": "default@^1.0.0"
+        },
+        "ci-condition": {
+            "name": "github",
+            "options": {
+                "defaultBranch": "*"
+            }
+        },
+        "changelog-generator": {
+            "name": "default",
+            "options": {
+                "emojis": "true"
+            }
+        },
+        "provider": {
+            "name": "git",
+            "options": {
+                "default_branch": "develop",
+                "tagger_email": "gitea@git.lumeweb.com",
+                "auth": "ssh"
+            }
+        }
+    }
+}

CHANGELOG.md

@@ -0,0 +1,115 @@
+# [0.1.0-develop.3](https://git.lumeweb.com/LumeWeb/portal/compare/v0.1.0-develop.2...v0.1.0-develop.3) (2023-09-09)
+
+
+### Bug Fixes
+
+* handle failure on verifying token ([a06b79a](https://git.lumeweb.com/LumeWeb/portal/commit/a06b79a537f08d741faeb8319d558c9e64977c4b))
+
+# [0.1.0-develop.2](https://git.lumeweb.com/LumeWeb/portal/compare/v0.1.0-develop.1...v0.1.0-develop.2) (2023-08-15)
+
+
+### Bug Fixes
+
+* need to change dnslink route registration to use a path param based route ([ae071a3](https://git.lumeweb.com/LumeWeb/portal/commit/ae071a30ecaa62ff431878c71a54059e3d3ce8b7))
+* need to string off forward slash at beginning to match manifest file paths ([2f64f18](https://git.lumeweb.com/LumeWeb/portal/commit/2f64f18e24fa1e4ddd74ed6a8d2d44e483fff1dc))
+
+# [0.1.0-develop.1](https://git.lumeweb.com/LumeWeb/portal/compare/v0.0.1...v0.1.0-develop.1) (2023-08-15)
+
+
+### Bug Fixes
+
+* abort if we don't have a password for the account, assume its pubkey only ([c20dec0](https://git.lumeweb.com/LumeWeb/portal/commit/c20dec020437d91cf2728852b8bed5c4a0c481e9))
+* add a check for a 500 error ([df08fc9](https://git.lumeweb.com/LumeWeb/portal/commit/df08fc980ac3f710a67bd692b8126eb978699d5b))
+* add missing request connection close ([dff3ca4](https://git.lumeweb.com/LumeWeb/portal/commit/dff3ca45895095b82ba2e76b2e61487e28151b7d))
+* add shutdown signal and flag for renterd ([fb65690](https://git.lumeweb.com/LumeWeb/portal/commit/fb65690abd5c190dce30d3cfe0d079b27040a309))
+* **auth:** eager load the account relation to return it ([a23d165](https://git.lumeweb.com/LumeWeb/portal/commit/a23d165caa3ba4832c9d37a0b833b9b58df60732))
+* change jwtKey to ed25519.PrivateKey ([bf576df](https://git.lumeweb.com/LumeWeb/portal/commit/bf576dfaeef51078d7bdae885550fc235d49c1eb))
+* close db on shutdown ([78ee15c](https://git.lumeweb.com/LumeWeb/portal/commit/78ee15cf4b5d3a55209a9c7559700a2c5b227f87))
+* Ctx must be public ([a0d747f](https://git.lumeweb.com/LumeWeb/portal/commit/a0d747fdf4e6ee3fa6a3b4dca180e4f14af30ed9))
+* ctx needs to be public in AuthService ([a3cfeba](https://git.lumeweb.com/LumeWeb/portal/commit/a3cfebab307a87bc895d7b1c1f0e6632a708562c))
+* **db:** need to set charset, parseTime and loc in connection for mysql ([5d15ca3](https://git.lumeweb.com/LumeWeb/portal/commit/5d15ca330abd26576ef9865c110975aeb27c3ab3))
+* disable client warnings ([9b8cb38](https://git.lumeweb.com/LumeWeb/portal/commit/9b8cb38496541b0ab50d28eef63658f9723c5802))
+* dont try to stream if we have an error ([b21a425](https://git.lumeweb.com/LumeWeb/portal/commit/b21a425e24f5543802e7267369f37967d4805697))
+* encode size as uint64 to the end of the cid ([5aca66d](https://git.lumeweb.com/LumeWeb/portal/commit/5aca66d91981d8fae88194df6b03c239dbd179a8))
+* ensure all models auto increment the id field ([934f8e6](https://git.lumeweb.com/LumeWeb/portal/commit/934f8e6236ef1eef8db1d06a1d7a7fded8afe694))
+* ensure we store the pubkey in lowercase ([def1b50](https://git.lumeweb.com/LumeWeb/portal/commit/def1b50cfcba8c68f3b95209790418638374fad9))
+* handle duplicate tus uploads by hash ([f3172b0](https://git.lumeweb.com/LumeWeb/portal/commit/f3172b0d31f844b95a0e64b3a5d821f71b0fbe07))
+* hasher needs the size set to 32 ([294370d](https://git.lumeweb.com/LumeWeb/portal/commit/294370d88dd159ae173a6a955a417a1547de60ed))
+* if upload status code isn't 200, make it an err based on the body ([039a4a3](https://git.lumeweb.com/LumeWeb/portal/commit/039a4a33547a59b4f3ec86199664b5bb94d258a6))
+* if uploading returns a 500 and its a slab error, treat as a 404 ([6ddef03](https://git.lumeweb.com/LumeWeb/portal/commit/6ddef03790971e346fa0a7d33a462f39348bc6cc))
+* if we have an existing upload, just return it as if successful ([90170e5](https://git.lumeweb.com/LumeWeb/portal/commit/90170e5b81831f3d768291fd37c7c13e32d522fe))
+* iris context.User needs to be embedded in our User struct for type checking to properly work ([1cfc222](https://git.lumeweb.com/LumeWeb/portal/commit/1cfc2223a6df614f26fd0337ced68d92e774589f))
+* just use the any route ([e100429](https://git.lumeweb.com/LumeWeb/portal/commit/e100429b60e783f6c7c3ddecab7bb9b4dd599726))
+* load awsConfig before db ([58165e0](https://git.lumeweb.com/LumeWeb/portal/commit/58165e01af9f2b183d654d3d8809cbd1eda0a9bb))
+* make an attempt to look for the token before adding to db ([f11b285](https://git.lumeweb.com/LumeWeb/portal/commit/f11b285d4e255c1c4c95f6ac15aa904d7a5730e4))
+* missing setting SetTusComposer ([80561f8](https://git.lumeweb.com/LumeWeb/portal/commit/80561f89e92dfa86887ada8361e0046ee6288234))
+* newer gorm version causes db rebuilds every boot ([72255eb](https://git.lumeweb.com/LumeWeb/portal/commit/72255eb3c50892aa5f2cfdc4cb1daa5883f0affc))
+* only panic if the error is other than a missing awsConfig file ([6e0ec8a](https://git.lumeweb.com/LumeWeb/portal/commit/6e0ec8aaf90e86bcb7cb6c8c53f6569e6885e0aa))
+* output error info ([cfa7ceb](https://git.lumeweb.com/LumeWeb/portal/commit/cfa7ceb2f422a6e594a424315c8eaeffc6572926))
+* PostPubkeyChallenge should be lowercasing the pubkey for consistency ([d680f06](https://git.lumeweb.com/LumeWeb/portal/commit/d680f0660f910e323356a1169ee13ef2e647a015))
+* PostPubkeyChallenge should be using ChallengeRequest ([36745bb](https://git.lumeweb.com/LumeWeb/portal/commit/36745bb55b1d7cd464b085e410333089504591c1))
+* PostPubkeyChallenge should not be checking email, but pubkey ([db3ba1f](https://git.lumeweb.com/LumeWeb/portal/commit/db3ba1f0148b6abc34b4606f9b8103963a3c6850))
+* PostPubkeyLogin should be lowercasing the pubkey and signature ([09d53ff](https://git.lumeweb.com/LumeWeb/portal/commit/09d53ffa7645b64aed4170e698b8eb62d2c3590e))
+* PostPubkeyLogin should not preload any model ([27e7ea7](https://git.lumeweb.com/LumeWeb/portal/commit/27e7ea7d7a0bbf6c147ff625591acf6376c6c62d))
+* properly handle missing size bytes ([c0df04d](https://git.lumeweb.com/LumeWeb/portal/commit/c0df04d7d5309e32348ceecc68eecd64c5e5cba4))
+* public_key should be pubkey ([09b9f19](https://git.lumeweb.com/LumeWeb/portal/commit/09b9f195f47ea9ae47069a517a77609c74ea3ca5))
+* register LoginSession model ([48164ec](https://git.lumeweb.com/LumeWeb/portal/commit/48164ec320c693937ead352246ec1e94bede3684))
+* register request validation ([c197b14](https://git.lumeweb.com/LumeWeb/portal/commit/c197b1425bbd689e8f662846de0478aff8d38f35))
+* remove PrivateKey, rename PublicKey in Key model ([00f2b96](https://git.lumeweb.com/LumeWeb/portal/commit/00f2b962a0da956f971dc94d75726c1bab693232))
+* rewrite gorm query logic for tus uploads ([f8aaeff](https://git.lumeweb.com/LumeWeb/portal/commit/f8aaeff6de2dc5e5321840460d55d79ad1b5ab1a))
+* rewrite sql logic ([ce1b5e3](https://git.lumeweb.com/LumeWeb/portal/commit/ce1b5e31d5d6a69dc91d88a6fd2f1317e07dc1ea))
+* rewrite streaming logic and centralize in a helper function ([bb26cfc](https://git.lumeweb.com/LumeWeb/portal/commit/bb26cfca5b4017bbbbf5aeee9bd3577c724f83ca))
+* save upload info after every chunk ([038d2c4](https://git.lumeweb.com/LumeWeb/portal/commit/038d2c440b24b7c0f1ea72e0bfeda369f766c691))
+* temp workaround on race condition ([e2db880](https://git.lumeweb.com/LumeWeb/portal/commit/e2db880038f51e0e16ce270fe29fce7785cce878))
+* **tus:** switch to normal clone package, not generic ([faaec64](https://git.lumeweb.com/LumeWeb/portal/commit/faaec649ead00567ced56edfa9db11eb34655178))
+* update default flag values ([241db4d](https://git.lumeweb.com/LumeWeb/portal/commit/241db4deb6808d950d55efa38e11d60469cc6778))
+* update model relationships ([628f1b4](https://git.lumeweb.com/LumeWeb/portal/commit/628f1b4acaac1d2bf373b7008f2e0c070fd64ae5))
+* **upload:** add account to upload record ([e018a4b](https://git.lumeweb.com/LumeWeb/portal/commit/e018a4b7430bc375ff3b72537e71295cdf67ef93))
+* uploading of main file ([7aea462](https://git.lumeweb.com/LumeWeb/portal/commit/7aea462ab752e999030837d13733508369524cf3))
+* upstream renterd updates ([5ad91ad](https://git.lumeweb.com/LumeWeb/portal/commit/5ad91ad263f01830623958141a7e7c8523bee85f))
+* use AccountID not Account ([f5e4377](https://git.lumeweb.com/LumeWeb/portal/commit/f5e437777a52e2a9bbf55903cea17ec073fbb406))
+* use bufio reader ([90e4ce6](https://git.lumeweb.com/LumeWeb/portal/commit/90e4ce6408391dc270ca4405a7c5282c2d4766b2))
+* use challengeObj ([9b82fa7](https://git.lumeweb.com/LumeWeb/portal/commit/9b82fa7828946803289add03fc84be1dc4f86d8b))
+* use database.path over database.name ([25c7d6d](https://git.lumeweb.com/LumeWeb/portal/commit/25c7d6d4fb48b69239eba131232a78e90a576e2f))
+* use getWorkerObjectUrl ([4ff1334](https://git.lumeweb.com/LumeWeb/portal/commit/4ff1334d8afd9379db687fc6b764f5b0f1bcc08c))
+* Use gorm save, and return nil if successful ([26042b6](https://git.lumeweb.com/LumeWeb/portal/commit/26042b62acd7f7346f1a99a0ac37b3f2f99e3f75))
+* we can't use AddHandler inside BeginRequest ([f941ee4](https://git.lumeweb.com/LumeWeb/portal/commit/f941ee46d469a3f0a6302b188f566029fdec4e70))
+* wrap Register api in an atomic transaction to avoid dead locks ([e09e51b](https://git.lumeweb.com/LumeWeb/portal/commit/e09e51bb52d513abcbbf53352a5d8ff68eb5364a))
+* wrong algo ([86380c7](https://git.lumeweb.com/LumeWeb/portal/commit/86380c7b3a97e785b99af456305c01d18f776ddf))
+
+
+### Features
+
+* add a status endpoint and move cid validation to a utility method ([38b7615](https://git.lumeweb.com/LumeWeb/portal/commit/38b76155af954dc3602a5035cb7b53a7f625fbfd))
+* add a Status method for uploads ([1f195cf](https://git.lumeweb.com/LumeWeb/portal/commit/1f195cf328ee176be9283ab0cc40e65bb6c40948))
+* add auth status endpoint ([1dd4fa2](https://git.lumeweb.com/LumeWeb/portal/commit/1dd4fa22cdfc749c5474f94108bca0aec34aea81))
+* add bao package and rust bao wasm library ([4c649bf](https://git.lumeweb.com/LumeWeb/portal/commit/4c649bfcb92e8632e45cf10b27fa062ff1680c32))
+* add cid package ([706f7a0](https://git.lumeweb.com/LumeWeb/portal/commit/706f7a05b9a4ed464f693941235aa7e9ca14145a))
+* add ComputeFile bao RPC method ([687f26c](https://git.lumeweb.com/LumeWeb/portal/commit/687f26cc779f4f50166108d6e78fe1456cfa128d))
+* add debug mode logging support ([99d7b83](https://git.lumeweb.com/LumeWeb/portal/commit/99d7b8347af25fe65a1f1aecc9960424a101c279))
+* add download endpoint ([79fd550](https://git.lumeweb.com/LumeWeb/portal/commit/79fd550c54bf74e84d012805f60c036c19fbbef2))
+* add EncodeString function ([488f873](https://git.lumeweb.com/LumeWeb/portal/commit/488f8737c09b7757c5649b3d8a3568e3c1d5fe45))
+* add files service with upload endpoint ([b16beeb](https://git.lumeweb.com/LumeWeb/portal/commit/b16beebabb254488897edde870e9588b7be5293e))
+* add files/upload/limit endpoint ([b77bebe](https://git.lumeweb.com/LumeWeb/portal/commit/b77bebe3b1a03cecdd7e80f575452d5ce91ccfac))
+* add getCurrentUserId helper function ([29d6db2](https://git.lumeweb.com/LumeWeb/portal/commit/29d6db20096e61efa9a792ef837ef93ca14107ae))
+* add global cors ([1f5a3d1](https://git.lumeweb.com/LumeWeb/portal/commit/1f5a3d19e44f1db2f8587623e868fa48b23d1a74))
+* add jwt package ([ea99108](https://git.lumeweb.com/LumeWeb/portal/commit/ea991083276a576003eb3633bd1bde98e13dfe84))
+* add more validation, and put account creation, with optional pubkey in a transaction ([699e424](https://git.lumeweb.com/LumeWeb/portal/commit/699e4244e0d877d8d9df9d3d4894351785fe7f4d))
+* add new user service object that implements iris context User interface ([a14dad4](https://git.lumeweb.com/LumeWeb/portal/commit/a14dad43ed3140f73d817ef2438aacbc0939de69))
+* add newrelic support ([06b3ab8](https://git.lumeweb.com/LumeWeb/portal/commit/06b3ab87f7e1b982d3fb42a3e06897a2fd1387ed))
+* add pin model ([aaa2c17](https://git.lumeweb.com/LumeWeb/portal/commit/aaa2c17212bd5e646036252a0e1f8d8bdb68f5a7))
+* add pin service method ([8692a02](https://git.lumeweb.com/LumeWeb/portal/commit/8692a0225ebb71502811cba063e32dd11cdd10c9))
+* add PostPinBy controller endpoint for pinning a file ([be03a6c](https://git.lumeweb.com/LumeWeb/portal/commit/be03a6c6867f305529af90e6206a0597bb84f015))
+* add pprof support ([ee17409](https://git.lumeweb.com/LumeWeb/portal/commit/ee17409e1252e9cbae0b17ccbb1949c9a81dff82))
+* add proof download ([3b1e860](https://git.lumeweb.com/LumeWeb/portal/commit/3b1e860256297d3515f0fcd58dd28292c316d79f))
+* add StringHash ([118c679](https://git.lumeweb.com/LumeWeb/portal/commit/118c679f769bec2971e4e4b00ec41841a02b8a1c))
+* add swagger support ([49c3844](https://git.lumeweb.com/LumeWeb/portal/commit/49c38444066c89d7258fd85d114d9d74babb8d55))
+* add upload model ([f73a04b](https://git.lumeweb.com/LumeWeb/portal/commit/f73a04bb2e48b78e22b531a9121fe4baa011deaf))
+* add Valid, and Decode methods, and create CID struct ([4e6c29f](https://git.lumeweb.com/LumeWeb/portal/commit/4e6c29f1fd7c33ce442fe741e08b32c8e3e9f393))
+* add validation to account register ([7257b5d](https://git.lumeweb.com/LumeWeb/portal/commit/7257b5d597a28069c87437cabd71f51c187eb80c))
+* generate and/or load an ed25519 private key for jwt token generation ([85a0295](https://git.lumeweb.com/LumeWeb/portal/commit/85a02952dffb1873c557f30483606d678e46749d))
+* initial dnslink support ([cd2f63e](https://git.lumeweb.com/LumeWeb/portal/commit/cd2f63eb72c2bfc404d8d1b5a6fdb53f61a31d1b))
+* pin file after basic upload ([892f093](https://git.lumeweb.com/LumeWeb/portal/commit/892f093d93348459d113041104d773fdd5124a8d))
+* pin file after tus upload ([5579ab8](https://git.lumeweb.com/LumeWeb/portal/commit/5579ab85a374be457163d06caf1ac6e260082cca))
+* tus support ([3005be6](https://git.lumeweb.com/LumeWeb/portal/commit/3005be6fec8136214c1e9480c788f62564a2c5f9))
+* wip version ([9a4c3d5](https://git.lumeweb.com/LumeWeb/portal/commit/9a4c3d5d13a3e76fe91eb5d78a6f2f0f8e238f80))

Dockerfile

@@ -0,0 +1,55 @@
+# Use the official Node.js image as the base image for building the api/account/portal
+FROM node:20-alpine as nodejs-builder
+
+# Set the working directory
+WORKDIR /portal
+
+# Clone the repository with submodules
+RUN apk add --no-cache git \
+    && git clone --recurse-submodules https://git.lumeweb.com/LumeWeb/portal.git -b develop .
+
+# Set the working directory
+WORKDIR /portal/api/account/app
+
+# Build the dashboard
+RUN npm ci && npm run build
+
+# Use the official Go image as the base image for the final Go build
+FROM golang:1.21.6-alpine as go-builder
+
+# Set the working directory
+WORKDIR /portal
+
+# Build the Go application with configurable tags
+ARG BUILD_TAGS
+RUN apk add --no-cache git && git clone --recurse-submodules https://git.lumeweb.com/LumeWeb/portal.git -b develop .
+
+# Copy the built dashboard from the nodejs-builder stage
+COPY --from=nodejs-builder /portal/api/account/app/build/client /portal/api/account/app/build/client
+
+# Install the necessary dependencies
+RUN apk add bash gcc curl musl-dev
+
+## Build the Go application
+RUN go mod download
+
+## Build the Go application
+RUN go generate ./...
+
+## Build the Go application
+RUN go build -tags "${BUILD_TAGS}" -gcflags="all=-N -l" -o portal ./cmd/portal
+
+# Use a lightweight base image for the final stage
+FROM alpine:latest
+
+# Set the working directory
+WORKDIR /portal
+
+# Copy the built binary from the go-builder stage
+COPY --from=go-builder /portal/portal .
+
+# Expose the necessary port(s)
+EXPOSE 8080
+
+# Run the application
+CMD ["./portal"]

account/account.go

@@ -0,0 +1,731 @@
+package account
+
+import (
+	"context"
+	"crypto/ed25519"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/go-sql-driver/mysql"
+
+	"git.lumeweb.com/LumeWeb/portal/metadata"
+
+	"git.lumeweb.com/LumeWeb/portal/mailer"
+
+	"gorm.io/gorm/clause"
+
+	"git.lumeweb.com/LumeWeb/portal/config"
+
+	"git.lumeweb.com/LumeWeb/portal/db/models"
+	"go.uber.org/fx"
+	"golang.org/x/crypto/bcrypt"
+	"gorm.io/gorm"
+)
+
+var (
+	ErrInvalidOTPCode = errors.New("Invalid OTP code")
+)
+
+const ACCOUNT_SUBDOMAIN = "account"
+
+type AccountServiceParams struct {
+	fx.In
+	Db       *gorm.DB
+	Config   *config.Manager
+	Identity ed25519.PrivateKey
+	Mailer   *mailer.Mailer
+	Metadata metadata.MetadataService
+}
+
+var Module = fx.Module("account",
+	fx.Options(
+		fx.Provide(NewAccountService),
+	),
+)
+
+type AccountServiceDefault struct {
+	db       *gorm.DB
+	config   *config.Manager
+	identity ed25519.PrivateKey
+	mailer   *mailer.Mailer
+	metadata metadata.MetadataService
+}
+
+func NewAccountService(params AccountServiceParams) *AccountServiceDefault {
+	return &AccountServiceDefault{db: params.Db, config: params.Config, identity: params.Identity, mailer: params.Mailer, metadata: params.Metadata}
+}
+
+func (s *AccountServiceDefault) EmailExists(email string) (bool, *models.User, error) {
+	user := &models.User{}
+	exists, model, err := s.exists(user, map[string]interface{}{"email": email})
+	if !exists || err != nil {
+		return false, nil, err
+	}
+	return true, model.(*models.User), nil // Type assertion since `exists` returns interface{}
+}
+
+func (s *AccountServiceDefault) PubkeyExists(pubkey string) (bool, *models.PublicKey, error) {
+	publicKey := &models.PublicKey{}
+	exists, model, err := s.exists(publicKey, map[string]interface{}{"key": pubkey})
+	if !exists || err != nil {
+		return false, nil, err
+	}
+	return true, model.(*models.PublicKey), nil // Type assertion is necessary
+}
+
+func (s *AccountServiceDefault) AccountExists(id uint) (bool, *models.User, error) {
+	user := &models.User{}
+	exists, model, err := s.exists(user, map[string]interface{}{"id": id})
+	if !exists || err != nil {
+		return false, nil, err
+	}
+	return true, model.(*models.User), nil // Ensure to assert the type correctly
+}
+
+func (s *AccountServiceDefault) HashPassword(password string) (string, error) {
+	bytes, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
+	if err != nil {
+		return "", NewAccountError(ErrKeyHashingFailed, err)
+	}
+	return string(bytes), nil
+}
+
+func (s *AccountServiceDefault) CreateAccount(email string, password string, verifyEmail bool) (*models.User, error) {
+	passwordHash, err := s.HashPassword(password)
+	if err != nil {
+		return nil, err
+	}
+
+	user := models.User{
+		Email:        email,
+		PasswordHash: passwordHash,
+	}
+
+	result := s.db.Create(&user)
+	if result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrDuplicatedKey) {
+			return nil, NewAccountError(ErrKeyEmailAlreadyExists, nil)
+		}
+
+		if err, ok := result.Error.(*mysql.MySQLError); ok {
+			if err.Number == 1062 {
+				return nil, NewAccountError(ErrKeyEmailAlreadyExists, nil)
+			}
+		}
+
+		return nil, NewAccountError(ErrKeyAccountCreationFailed, result.Error)
+	}
+
+	if verifyEmail {
+		err = s.SendEmailVerification(user.ID)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &user, nil
+}
+
+func (s AccountServiceDefault) SendEmailVerification(userId uint) error {
+	exists, user, err := s.AccountExists(userId)
+	if !exists || err != nil {
+		return err
+	}
+
+	if user.Verified {
+		return NewAccountError(ErrKeyAccountAlreadyVerified, nil)
+	}
+
+	token := GenerateSecurityToken()
+
+	var verification models.EmailVerification
+
+	verification.UserID = user.ID
+	verification.Token = token
+	verification.ExpiresAt = time.Now().Add(time.Hour)
+
+	err = s.db.Create(&verification).Error
+	if err != nil {
+		return NewAccountError(ErrKeyDatabaseOperationFailed, err)
+	}
+
+	verifyUrl := fmt.Sprintf("%s/account/verify?token=%s", fmt.Sprintf("https://%s.%s", ACCOUNT_SUBDOMAIN, s.config.Config().Core.Domain), token)
+
+	vars := map[string]interface{}{
+		"FirstName":        user.FirstName,
+		"Email":            user.Email,
+		"VerificationLink": verifyUrl,
+		"ExpireTime":       verification.ExpiresAt.Sub(time.Now()).Round(time.Second * 2),
+		"PortalName":       s.config.Config().Core.PortalName,
+	}
+
+	return s.mailer.TemplateSend(mailer.TPL_VERIFY_EMAIL, vars, vars, user.Email)
+}
+
+func (s AccountServiceDefault) SendPasswordReset(user *models.User) error {
+	token := GenerateSecurityToken()
+
+	var reset models.PasswordReset
+
+	reset.UserID = user.ID
+	reset.Token = token
+	reset.ExpiresAt = time.Now().Add(time.Hour)
+
+	err := s.db.Create(&reset).Error
+	if err != nil {
+		return NewAccountError(ErrKeyDatabaseOperationFailed, err)
+	}
+
+	vars := map[string]interface{}{
+		"FirstName":    user.FirstName,
+		"Email":        user.Email,
+		"ResetCode":    token,
+		"ExpireTime":   reset.ExpiresAt,
+		"PortalName":   s.config.Config().Core.PortalName,
+		"PortalDomain": s.config.Config().Core.Domain,
+	}
+
+	return s.mailer.TemplateSend(mailer.TPL_PASSWORD_RESET, vars, vars, user.Email)
+}
+
+func (s AccountServiceDefault) VerifyEmail(email string, token string) error {
+	var verification models.EmailVerification
+
+	verification.Token = token
+
+	result := s.db.Model(&verification).
+		Preload("User").
+		Where(&verification).
+		First(&verification)
+
+	if result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrRecordNotFound) {
+			return NewAccountError(ErrKeySecurityInvalidToken, nil)
+		}
+
+		return NewAccountError(ErrKeyDatabaseOperationFailed, nil)
+	}
+
+	if verification.ExpiresAt.Before(time.Now()) {
+		return NewAccountError(ErrKeySecurityTokenExpired, nil)
+	}
+
+	if len(verification.NewEmail) > 0 && verification.NewEmail != email {
+		return NewAccountError(ErrKeySecurityInvalidToken, nil)
+	} else if verification.User.Email != email {
+		return NewAccountError(ErrKeySecurityInvalidToken, nil)
+	}
+
+	var update models.User
+
+	doUpdate := false
+
+	if !verification.User.Verified {
+		update.Verified = true
+		doUpdate = true
+	}
+
+	if len(verification.NewEmail) > 0 {
+		update.Email = verification.NewEmail
+		doUpdate = true
+	}
+
+	if doUpdate {
+		err := s.updateAccountInfo(verification.UserID, update)
+		if err != nil {
+			return err
+		}
+	}
+
+	verification = models.EmailVerification{
+		UserID: verification.UserID,
+	}
+
+	if result := s.db.Where(&verification).Delete(&verification); result.Error != nil {
+		return NewAccountError(ErrKeyDatabaseOperationFailed, result.Error)
+	}
+
+	return nil
+}
+
+func (s AccountServiceDefault) ResetPassword(email string, token string, password string) error {
+	var reset models.PasswordReset
+
+	reset.Token = token
+
+	result := s.db.Model(&reset).
+		Preload("User").
+		Where(&reset).
+		First(&reset)
+
+	if result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrRecordNotFound) {
+			return NewAccountError(ErrKeyUserNotFound, result.Error)
+		}
+
+		return NewAccountError(ErrKeyDatabaseOperationFailed, result.Error)
+	}
+
+	if reset.ExpiresAt.Before(time.Now()) {
+		return NewAccountError(ErrKeySecurityTokenExpired, nil)
+	}
+
+	if reset.User.Email != email {
+		return NewAccountError(ErrKeySecurityInvalidToken, nil)
+	}
+
+	passwordHash, err := s.HashPassword(password)
+	if err != nil {
+		return err
+	}
+
+	err = s.updateAccountInfo(reset.UserID, models.User{PasswordHash: passwordHash})
+	if err != nil {
+		return err
+	}
+
+	reset = models.PasswordReset{
+		UserID: reset.UserID,
+	}
+
+	if result := s.db.Where(&reset).Delete(&reset); result.Error != nil {
+		return NewAccountError(ErrKeyDatabaseOperationFailed, result.Error)
+	}
+
+	return nil
+}
+
+func (s AccountServiceDefault) UpdateAccountName(userId uint, firstName string, lastName string) error {
+	return s.updateAccountInfo(userId, models.User{FirstName: firstName, LastName: lastName})
+}
+
+func (s AccountServiceDefault) UpdateAccountEmail(userId uint, email string, password string) error {
+	exists, euser, err := s.EmailExists(email)
+	if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) || (exists && euser.ID != userId) {
+		return NewAccountError(ErrKeyEmailAlreadyExists, nil)
+	}
+
+	valid, user, err := s.ValidLoginByUserID(userId, password)
+	if err != nil {
+		return err
+	}
+
+	if !valid {
+		return NewAccountError(ErrKeyInvalidLogin, nil)
+	}
+
+	if user.Email == email {
+		return NewAccountError(ErrKeyUpdatingSameEmail, nil)
+	}
+
+	var update models.User
+
+	update.Email = email
+
+	return s.updateAccountInfo(userId, update)
+}
+
+func (s AccountServiceDefault) UpdateAccountPassword(userId uint, password string, newPassword string) error {
+	valid, _, err := s.ValidLoginByUserID(userId, password)
+	if err != nil {
+		return err
+	}
+
+	if !valid {
+		return NewAccountError(ErrKeyInvalidPassword, nil)
+	}
+
+	passwordHash, err := s.HashPassword(newPassword)
+	if err != nil {
+		return err
+	}
+
+	return s.updateAccountInfo(userId, models.User{PasswordHash: passwordHash})
+}
+
+func (s AccountServiceDefault) AddPubkeyToAccount(user models.User, pubkey string) error {
+	var model models.PublicKey
+
+	model.Key = pubkey
+	model.UserID = user.ID
+
+	result := s.db.Create(&model)
+
+	if result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrDuplicatedKey) {
+			return NewAccountError(ErrKeyPublicKeyExists, result.Error)
+		}
+
+		return NewAccountError(ErrKeyDatabaseOperationFailed, result.Error)
+	}
+
+	return nil
+}
+func (s AccountServiceDefault) LoginPassword(email string, password string, ip string) (string, *models.User, error) {
+	valid, user, err := s.ValidLoginByEmail(email, password)
+
+	if err != nil {
+		return "", nil, err
+	}
+
+	if !valid {
+		return "", nil, nil
+	}
+
+	token, err := s.doLogin(user, ip, false)
+
+	if err != nil {
+		return "", nil, err
+	}
+
+	return token, user, nil
+}
+
+func (s AccountServiceDefault) LoginOTP(userId uint, code string) (string, error) {
+	valid, err := s.OTPVerify(userId, code)
+
+	if err != nil {
+		return "", err
+	}
+
+	if !valid {
+		return "", NewAccountError(ErrKeyInvalidOTPCode, nil)
+	}
+
+	var user models.User
+	user.ID = userId
+
+	token, tokenErr := JWTGenerateToken(s.config.Config().Core.Domain, s.identity, user.ID, JWTPurposeLogin)
+	if tokenErr != nil {
+		return "", err
+	}
+
+	return token, nil
+}
+
+func (s AccountServiceDefault) ValidLoginByUserObj(user *models.User, password string) bool {
+	return s.validPassword(user, password)
+}
+
+func (s AccountServiceDefault) ValidLoginByEmail(email string, password string) (bool, *models.User, error) {
+	var user models.User
+
+	result := s.db.Model(&models.User{}).Where(&models.User{Email: email}).First(&user)
+
+	if result.RowsAffected == 0 || result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrRecordNotFound) {
+			return false, nil, NewAccountError(ErrKeyInvalidLogin, result.Error)
+		}
+
+		return false, nil, NewAccountError(ErrKeyDatabaseOperationFailed, result.Error)
+	}
+
+	valid := s.ValidLoginByUserObj(&user, password)
+
+	if !valid {
+		return false, nil, nil
+	}
+
+	return true, &user, nil
+}
+
+func (s AccountServiceDefault) ValidLoginByUserID(id uint, password string) (bool, *models.User, error) {
+	var user models.User
+
+	user.ID = id
+
+	result := s.db.Model(&user).Where(&user).First(&user)
+
+	if result.RowsAffected == 0 || result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrRecordNotFound) {
+			return false, nil, NewAccountError(ErrKeyInvalidLogin, result.Error)
+		}
+
+		return false, nil, NewAccountError(ErrKeyDatabaseOperationFailed, result.Error)
+	}
+
+	valid := s.ValidLoginByUserObj(&user, password)
+
+	if !valid {
+		return false, nil, nil
+	}
+
+	return true, &user, nil
+}
+
+func (s AccountServiceDefault) LoginPubkey(pubkey string, ip string) (string, error) {
+	var model models.PublicKey
+
+	result := s.db.Model(&models.PublicKey{}).Preload("User").Where(&models.PublicKey{Key: pubkey}).First(&model)
+
+	if result.RowsAffected == 0 || result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrRecordNotFound) {
+			return "", NewAccountError(ErrKeyInvalidLogin, result.Error)
+		}
+
+		return "", NewAccountError(ErrKeyDatabaseOperationFailed, result.Error)
+	}
+
+	user := model.User
+
+	token, err := s.doLogin(&user, ip, true)
+
+	if err != nil {
+		return "", err
+	}
+
+	return token, nil
+}
+
+func (s AccountServiceDefault) AccountPins(id uint, createdAfter uint64) ([]models.Pin, error) {
+	var pins []models.Pin
+
+	result := s.db.Model(&models.Pin{}).
+		Preload("Upload"). // Preload the related Upload for each Pin
+		Where(&models.Pin{UserID: id}).
+		Where("created_at > ?", createdAfter).
+		Order("created_at desc").
+		Find(&pins)
+
+	if result.Error != nil {
+		return nil, NewAccountError(ErrKeyPinsRetrievalFailed, result.Error)
+	}
+
+	return pins, nil
+}
+
+func (s AccountServiceDefault) DeletePinByHash(hash []byte, userId uint) error {
+	// Define a struct for the query condition
+	uploadQuery := models.Upload{Hash: hash}
+
+	// Retrieve the upload ID for the given hash
+	var uploadID uint
+	result := s.db.
+		Model(&models.Upload{}).
+		Where(&uploadQuery).
+		Select("id").
+		First(&uploadID)
+
+	if result.Error != nil {
+		if result.Error == gorm.ErrRecordNotFound {
+			// No record found, nothing to delete
+			return nil
+		}
+		return result.Error
+	}
+
+	// Delete pins with the retrieved upload ID and matching account ID
+	pinQuery := models.Pin{UploadID: uploadID, UserID: userId}
+	result = s.db.
+		Where(&pinQuery).
+		Delete(&models.Pin{})
+
+	if result.Error != nil {
+		return result.Error
+	}
+
+	return nil
+}
+func (s AccountServiceDefault) PinByHash(hash []byte, userId uint) error {
+	// Define a struct for the query condition
+	uploadQuery := models.Upload{Hash: hash}
+
+	result := s.db.
+		Model(&uploadQuery).
+		Where(&uploadQuery).
+		First(&uploadQuery)
+
+	if result.Error != nil {
+		return result.Error
+	}
+
+	return s.PinByID(uploadQuery.ID, userId)
+}
+
+func (s AccountServiceDefault) PinByID(uploadId uint, userId uint) error {
+	result := s.db.Model(&models.Pin{}).Where(&models.Pin{UploadID: uploadId, UserID: userId}).First(&models.Pin{})
+
+	if result.Error != nil && !errors.Is(result.Error, gorm.ErrRecordNotFound) {
+		return result.Error
+	}
+
+	if result.RowsAffected > 0 {
+		return nil
+	}
+
+	// Create a pin with the retrieved upload ID and matching account ID
+	pinQuery := models.Pin{UploadID: uploadId, UserID: userId}
+	result = s.db.Create(&pinQuery)
+
+	if result.Error != nil {
+		return result.Error
+	}
+
+	return nil
+}
+
+func (s AccountServiceDefault) OTPGenerate(userId uint) (string, error) {
+	exists, user, err := s.AccountExists(userId)
+
+	if !exists || err != nil {
+		return "", err
+	}
+
+	otp, otpErr := TOTPGenerate(user.Email, s.config.Config().Core.Domain)
+	if otpErr != nil {
+		return "", NewAccountError(ErrKeyOTPGenerationFailed, otpErr)
+	}
+
+	err = s.updateAccountInfo(user.ID, models.User{OTPSecret: otp})
+	return otp, nil
+}
+
+func (s AccountServiceDefault) OTPVerify(userId uint, code string) (bool, error) {
+	exists, user, err := s.AccountExists(userId)
+
+	if !exists || err != nil {
+		return false, err
+	}
+
+	valid := TOTPValidate(user.OTPSecret, code)
+	if !valid {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func (s AccountServiceDefault) OTPEnable(userId uint, code string) error {
+	verify, err := s.OTPVerify(userId, code)
+	if err != nil {
+		return err
+	}
+
+	if !verify {
+		return ErrInvalidOTPCode
+	}
+
+	return s.updateAccountInfo(userId, models.User{OTPEnabled: true})
+}
+
+func (s AccountServiceDefault) OTPDisable(userId uint) error {
+	return s.updateAccountInfo(userId, models.User{OTPEnabled: false, OTPSecret: ""})
+}
+
+func (s AccountServiceDefault) DNSLinkExists(hash []byte) (bool, *models.DNSLink, error) {
+	upload, err := s.metadata.GetUpload(context.Background(), hash)
+	if err != nil {
+		return false, nil, err
+	}
+
+	exists, model, err := s.exists(&models.DNSLink{}, map[string]interface{}{"upload_id": upload.ID})
+	if !exists || err != nil {
+		return false, nil, err
+	}
+
+	pinned, err := s.UploadPinned(hash)
+	if err != nil {
+		return false, nil, err
+	}
+
+	if !pinned {
+		return false, nil, nil
+	}
+
+	return true, model.(*models.DNSLink), nil
+}
+
+func (s AccountServiceDefault) UploadPinned(hash []byte) (bool, error) {
+	upload, err := s.metadata.GetUpload(context.Background(), hash)
+	if err != nil {
+		return false, err
+	}
+
+	var pin models.Pin
+	result := s.db.Model(&models.Pin{}).Where(&models.Pin{UploadID: upload.ID}).First(&pin)
+
+	if result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrRecordNotFound) {
+			return false, nil
+		}
+
+		return false, result.Error
+	}
+
+	return true, nil
+}
+
+func GenerateSecurityToken() string {
+	const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
+	b := make([]byte, 6)
+	_, err := rand.Read(b)
+	if err != nil {
+		panic(err)
+	}
+	for i := 0; i < 6; i++ {
+		b[i] = charset[b[i]%byte(len(charset))]
+	}
+	return string(b)
+}
+
+func (s AccountServiceDefault) doLogin(user *models.User, ip string, bypassSecurity bool) (string, error) {
+	purpose := JWTPurposeLogin
+
+	if user.OTPEnabled && !bypassSecurity {
+		purpose = JWTPurpose2FA
+	}
+
+	token, jwtErr := JWTGenerateToken(s.config.Config().Core.Domain, s.identity, user.ID, purpose)
+	if jwtErr != nil {
+		return "", NewAccountError(ErrKeyJWTGenerationFailed, jwtErr)
+	}
+
+	now := time.Now()
+
+	err := s.updateAccountInfo(user.ID, models.User{LastLoginIP: ip, LastLogin: &now})
+	if err != nil {
+		return "", err
+	}
+
+	return token, nil
+}
+
+func (s AccountServiceDefault) updateAccountInfo(userId uint, info models.User) error {
+	var user models.User
+
+	user.ID = userId
+
+	result := s.db.Model(&models.User{}).Where(&user).Updates(info)
+
+	if result.Error != nil {
+		return NewAccountError(ErrKeyDatabaseOperationFailed, result.Error)
+	}
+
+	return nil
+}
+
+func (s AccountServiceDefault) exists(model interface{}, conditions map[string]interface{}) (bool, interface{}, error) {
+	// Conduct a query with the provided model and conditions
+	result := s.db.Preload(clause.Associations).Model(model).Where(conditions).First(model)
+
+	// Check if any rows were found
+	exists := result.RowsAffected > 0
+
+	if errors.Is(result.Error, gorm.ErrRecordNotFound) {
+		return false, nil, nil
+	}
+
+	if exists {
+		return true, model, nil
+	}
+
+	return false, model, NewAccountError(ErrKeyDatabaseOperationFailed, result.Error)
+}
+
+func (s AccountServiceDefault) validPassword(user *models.User, password string) bool {
+	err := bcrypt.CompareHashAndPassword([]byte(user.PasswordHash), []byte(password))
+
+	return err == nil
+}

account/errors.go

@@ -0,0 +1,179 @@
+package account
+
+import (
+	"fmt"
+	"net/http"
+)
+
+const (
+	// Account creation errors
+	ErrKeyAccountCreationFailed = "ErrAccountCreationFailed"
+	ErrKeyEmailAlreadyExists    = "ErrEmailAlreadyExists"
+	ErrKeyUpdatingSameEmail     = "ErrUpdatingSameEmail"
+	ErrKeyPasswordHashingFailed = "ErrPasswordHashingFailed"
+
+	// Account lookup and existence verification errors
+	ErrKeyUserNotFound      = "ErrUserNotFound"
+	ErrKeyPublicKeyNotFound = "ErrPublicKeyNotFound"
+
+	// Authentication and login errors
+	ErrKeyInvalidLogin          = "ErrInvalidLogin"
+	ErrKeyInvalidPassword       = "ErrInvalidPassword"
+	ErrKeyInvalidOTPCode        = "ErrInvalidOTPCode"
+	ErrKeyOTPVerificationFailed = "ErrOTPVerificationFailed"
+	ErrKeyLoginFailed           = "ErrLoginFailed"
+	ErrKeyHashingFailed         = "ErrHashingFailed"
+
+	// Account update errors
+	ErrKeyAccountUpdateFailed    = "ErrAccountUpdateFailed"
+	ErrKeyAccountAlreadyVerified = "ErrAccountAlreadyVerified"
+
+	// JWT generation errors
+	ErrKeyJWTGenerationFailed = "ErrJWTGenerationFailed"
+
+	// OTP management errors
+	ErrKeyOTPGenerationFailed = "ErrOTPGenerationFailed"
+	ErrKeyOTPEnableFailed     = "ErrOTPEnableFailed"
+	ErrKeyOTPDisableFailed    = "ErrOTPDisableFailed"
+
+	// Public key management errors
+	ErrKeyAddPublicKeyFailed = "ErrAddPublicKeyFailed"
+	ErrKeyPublicKeyExists    = "ErrPublicKeyExists"
+
+	// Pin management errors
+	ErrKeyPinAddFailed        = "ErrPinAddFailed"
+	ErrKeyPinDeleteFailed     = "ErrPinDeleteFailed"
+	ErrKeyPinsRetrievalFailed = "ErrPinsRetrievalFailed"
+
+	// General errors
+	ErrKeyDatabaseOperationFailed = "ErrDatabaseOperationFailed"
+
+	// Security token errors
+	ErrKeySecurityTokenExpired = "ErrSecurityTokenExpired"
+	ErrKeySecurityInvalidToken = "ErrSecurityInvalidToken"
+)
+
+var defaultErrorMessages = map[string]string{
+	// Account creation errors
+	ErrKeyAccountCreationFailed: "Account creation failed due to an internal error.",
+	ErrKeyEmailAlreadyExists:    "The email address provided is already in use.",
+	ErrKeyPasswordHashingFailed: "Failed to secure the password, please try again later.",
+	ErrKeyUpdatingSameEmail:     "The email address provided is the same as your current one.",
+
+	// Account lookup and existence verification errors
+	ErrKeyUserNotFound:      "The requested user was not found.",
+	ErrKeyPublicKeyNotFound: "The specified public key was not found.",
+	ErrKeyHashingFailed:     "Failed to hash the password.",
+
+	// Authentication and login errors
+	ErrKeyInvalidLogin:          "The login credentials provided are invalid.",
+	ErrKeyInvalidPassword:       "The password provided is incorrect.",
+	ErrKeyInvalidOTPCode:        "The OTP code provided is invalid or expired.",
+	ErrKeyOTPVerificationFailed: "OTP verification failed, please try again.",
+	ErrKeyLoginFailed:           "Login failed due to an internal error.",
+
+	// Account update errors
+	ErrKeyAccountUpdateFailed:    "Failed to update account information.",
+	ErrKeyAccountAlreadyVerified: "Account is already verified.",
+
+	// JWT generation errors
+	ErrKeyJWTGenerationFailed: "Failed to generate a new JWT token.",
+
+	// OTP management errors
+	ErrKeyOTPGenerationFailed: "Failed to generate a new OTP secret.",
+	ErrKeyOTPEnableFailed:     "Enabling OTP authentication failed.",
+	ErrKeyOTPDisableFailed:    "Disabling OTP authentication failed.",
+
+	// Public key management errors
+	ErrKeyAddPublicKeyFailed: "Adding the public key to the account failed.",
+	ErrKeyPublicKeyExists:    "The public key already exists for this account.",
+
+	// Pin management errors
+	ErrKeyPinAddFailed:        "Failed to add the pin.",
+	ErrKeyPinDeleteFailed:     "Failed to delete the pin.",
+	ErrKeyPinsRetrievalFailed: "Failed to retrieve pins.",
+
+	// General errors
+	ErrKeyDatabaseOperationFailed: "A database operation failed.",
+
+	// Security token errors
+	ErrKeySecurityTokenExpired: "The security token has expired.",
+	ErrKeySecurityInvalidToken: "The security token is invalid.",
+}
+
+var (
+	ErrorCodeToHttpStatus = map[string]int{
+		// Account creation errors
+		ErrKeyAccountCreationFailed: http.StatusInternalServerError,
+		ErrKeyEmailAlreadyExists:    http.StatusConflict,
+		ErrKeyPasswordHashingFailed: http.StatusInternalServerError,
+
+		// Account lookup and existence verification errors
+		ErrKeyUserNotFound:      http.StatusNotFound,
+		ErrKeyPublicKeyNotFound: http.StatusNotFound,
+
+		// Authentication and login errors
+		ErrKeyInvalidLogin:          http.StatusUnauthorized,
+		ErrKeyInvalidPassword:       http.StatusUnauthorized,
+		ErrKeyInvalidOTPCode:        http.StatusBadRequest,
+		ErrKeyOTPVerificationFailed: http.StatusBadRequest,
+		ErrKeyLoginFailed:           http.StatusInternalServerError,
+
+		// Account update errors
+		ErrKeyAccountUpdateFailed:    http.StatusInternalServerError,
+		ErrKeyAccountAlreadyVerified: http.StatusConflict,
+
+		// JWT generation errors
+		ErrKeyJWTGenerationFailed: http.StatusInternalServerError,
+
+		// OTP management errors
+		ErrKeyOTPGenerationFailed: http.StatusInternalServerError,
+		ErrKeyOTPEnableFailed:     http.StatusInternalServerError,
+		ErrKeyOTPDisableFailed:    http.StatusInternalServerError,
+
+		// Public key management errors
+		ErrKeyAddPublicKeyFailed: http.StatusInternalServerError,
+		ErrKeyPublicKeyExists:    http.StatusConflict,
+
+		// Pin management errors
+		ErrKeyPinAddFailed:        http.StatusInternalServerError,
+		ErrKeyPinDeleteFailed:     http.StatusInternalServerError,
+		ErrKeyPinsRetrievalFailed: http.StatusInternalServerError,
+
+		// General errors
+		ErrKeyDatabaseOperationFailed: http.StatusInternalServerError,
+		ErrKeyHashingFailed:           http.StatusInternalServerError,
+
+		// Security token errors
+		ErrKeySecurityTokenExpired: http.StatusUnauthorized,
+		ErrKeySecurityInvalidToken: http.StatusUnauthorized,
+	}
+)
+
+type AccountError struct {
+	Key     string // A unique identifier for the error type
+	Message string // Human-readable error message
+	Err     error  // Underlying error, if any
+}
+
+func (e *AccountError) Error() string {
+	if e.Err != nil {
+		return fmt.Sprintf("%s: %v", e.Message, e.Err)
+	}
+	return e.Message
+}
+
+func NewAccountError(key string, err error, customMessage ...string) *AccountError {
+	message, exists := defaultErrorMessages[key]
+	if !exists {
+		message = "An unknown error occurred"
+	}
+	if len(customMessage) > 0 {
+		message = customMessage[0]
+	}
+	return &AccountError{
+		Key:     key,
+		Message: message,
+		Err:     err,
+	}
+}

account/jwt.go

@@ -0,0 +1,187 @@
+package account
+
+import (
+	"crypto/ed25519"
+	"errors"
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+
+	"git.lumeweb.com/LumeWeb/portal/config"
+
+	"github.com/samber/lo"
+
+	"go.sia.tech/jape"
+
+	"git.lumeweb.com/LumeWeb/portal/api/router"
+
+	apiRegistry "git.lumeweb.com/LumeWeb/portal/api/registry"
+
+	"github.com/golang-jwt/jwt/v5"
+)
+
+const AUTH_COOKIE_NAME = "auth_token"
+
+type JWTPurpose string
+type VerifyTokenFunc func(claim *jwt.RegisteredClaims) error
+
+var (
+	nopVerifyFunc VerifyTokenFunc = func(claim *jwt.RegisteredClaims) error {
+		return nil
+	}
+
+	ErrJWTUnexpectedClaimsType = errors.New("unexpected claims type")
+	ErrJWTUnexpectedIssuer     = errors.New("unexpected issuer")
+	ErrJWTInvalid              = errors.New("invalid JWT")
+)
+
+const (
+	JWTPurposeLogin JWTPurpose = "login"
+	JWTPurpose2FA   JWTPurpose = "2fa"
+	JWTPurposeNone  JWTPurpose = ""
+)
+
+func JWTGenerateToken(domain string, privateKey ed25519.PrivateKey, userID uint, purpose JWTPurpose) (string, error) {
+	return JWTGenerateTokenWithDuration(domain, privateKey, userID, time.Hour*24, purpose)
+}
+
+func JWTGenerateTokenWithDuration(domain string, privateKey ed25519.PrivateKey, userID uint, duration time.Duration, purpose JWTPurpose) (string, error) {
+
+	// Define the claims
+	claims := jwt.RegisteredClaims{
+		Issuer:    domain,
+		Subject:   strconv.Itoa(int(userID)),
+		ExpiresAt: jwt.NewNumericDate(time.Now().Add(duration)),
+		IssuedAt:  jwt.NewNumericDate(time.Now()),
+		Audience:  []string{string(purpose)},
+	}
+
+	// Create the token
+	token := jwt.NewWithClaims(jwt.SigningMethodEdDSA, claims)
+
+	// Sign the token with the Ed25519 private key
+	tokenString, err := token.SignedString(privateKey)
+	if err != nil {
+		return "", err
+	}
+
+	return tokenString, nil
+}
+
+func JWTVerifyToken(token string, domain string, privateKey ed25519.PrivateKey, verifyFunc VerifyTokenFunc) (*jwt.RegisteredClaims, error) {
+	validatedToken, err := jwt.ParseWithClaims(token, &jwt.RegisteredClaims{}, func(token *jwt.Token) (interface{}, error) {
+		if _, ok := token.Method.(*jwt.SigningMethodEd25519); !ok {
+			return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"])
+		}
+
+		publicKey := privateKey.Public()
+
+		return publicKey, nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	if verifyFunc == nil {
+		verifyFunc = nopVerifyFunc
+	}
+
+	claim, ok := validatedToken.Claims.(*jwt.RegisteredClaims)
+
+	if !ok {
+		return nil, fmt.Errorf("%w: %s", ErrJWTUnexpectedClaimsType, validatedToken.Claims)
+	}
+
+	if domain != claim.Issuer {
+		return nil, fmt.Errorf("%w: %s", ErrJWTUnexpectedIssuer, claim.Issuer)
+	}
+
+	err = verifyFunc(claim)
+
+	return claim, err
+}
+
+func SetAuthCookie(jc jape.Context, c *config.Manager, jwt string) {
+
+	for _, api := range apiRegistry.GetAllAPIs() {
+		routeableApi, ok := api.(router.RoutableAPI)
+		if !ok {
+			continue
+		}
+
+		http.SetCookie(jc.ResponseWriter, &http.Cookie{
+			Name:     routeableApi.AuthTokenName(),
+			Value:    jwt,
+			MaxAge:   int((24 * time.Hour).Seconds()),
+			Secure:   true,
+			HttpOnly: true,
+			Path:     "/",
+			Domain:   c.Config().Core.Domain,
+		})
+	}
+}
+
+func EchoAuthCookie(jc jape.Context, config *config.Manager) {
+	for _, api := range apiRegistry.GetAllAPIs() {
+		routeableApi, ok := api.(router.RoutableAPI)
+		if !ok {
+			continue
+		}
+
+		cookies := lo.Filter(jc.Request.Cookies(), func(item *http.Cookie, _ int) bool {
+			return item.Name == routeableApi.AuthTokenName()
+		})
+
+		if len(cookies) == 0 {
+			continue
+		}
+
+		unverified, _, err := jwt.NewParser().ParseUnverified(cookies[0].Value, &jwt.RegisteredClaims{})
+		if err != nil {
+			http.Error(jc.ResponseWriter, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		exp, err := unverified.Claims.GetExpirationTime()
+		if err != nil {
+			http.Error(jc.ResponseWriter, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		http.SetCookie(jc.ResponseWriter, &http.Cookie{
+			Name:     cookies[0].Name,
+			Value:    cookies[0].Value,
+			MaxAge:   int(exp.Time.Sub(time.Now()).Seconds()),
+			Secure:   true,
+			HttpOnly: true,
+			Path:     "/",
+			Domain:   config.Config().Core.Domain,
+		})
+	}
+}
+
+func ClearAuthCookie(jc jape.Context, config *config.Manager) {
+	for _, api := range apiRegistry.GetAllAPIs() {
+		routeableApi, ok := api.(router.RoutableAPI)
+		if !ok {
+			continue
+		}
+
+		jc.ResponseWriter.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
+		jc.ResponseWriter.Header().Set("Pragma", "no-cache")
+		jc.ResponseWriter.Header().Set("Expires", "0")
+
+		http.SetCookie(jc.ResponseWriter, &http.Cookie{
+			Name:     routeableApi.AuthTokenName(),
+			Value:    "",
+			Expires:  time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+			MaxAge:   -1,
+			Secure:   true,
+			HttpOnly: true,
+			Path:     "/",
+			Domain:   config.Config().Core.Domain,
+		})
+	}
+}

account/totp.go

@@ -0,0 +1,19 @@
+package account
+
+import "github.com/pquerna/otp/totp"
+
+func TOTPGenerate(domain string, email string) (string, error) {
+	key, err := totp.Generate(totp.GenerateOpts{
+		Issuer:      domain,
+		AccountName: email,
+	})
+	if err != nil {
+		return "", err
+	}
+
+	return key.Secret(), nil
+}
+
+func TOTPValidate(secret string, code string) bool {
+	return totp.Validate(code, secret)
+}

account/web.go

@@ -0,0 +1,7 @@
+package account
+
+import "go.sia.tech/jape"
+
+func SendJWT(jc jape.Context, jwt string) {
+	jc.ResponseWriter.Header().Set("Authorization", "Bearer "+jwt)
+}

api/account.go

@@ -0,0 +1,13 @@
+package api
+
+import (
+	"git.lumeweb.com/LumeWeb/portal/api/account"
+	"git.lumeweb.com/LumeWeb/portal/api/registry"
+)
+
+func init() {
+	registry.RegisterEntry(registry.APIEntry{
+		Key:    "account",
+		Module: account.Module,
+	})
+}

api/account/account.go

@@ -0,0 +1,509 @@
+package account
+
+import (
+	"context"
+	"crypto/ed25519"
+	"embed"
+	_ "embed"
+	"errors"
+	"io/fs"
+	"net/http"
+	"strings"
+
+	"github.com/rs/cors"
+
+	"git.lumeweb.com/LumeWeb/portal/api/swagger"
+
+	"git.lumeweb.com/LumeWeb/portal/api/router"
+
+	"git.lumeweb.com/LumeWeb/portal/config"
+
+	"go.uber.org/zap"
+
+	"github.com/julienschmidt/httprouter"
+
+	"git.lumeweb.com/LumeWeb/portal/account"
+	"git.lumeweb.com/LumeWeb/portal/api/middleware"
+	"git.lumeweb.com/LumeWeb/portal/api/registry"
+	"go.sia.tech/jape"
+	"go.uber.org/fx"
+)
+
+//go:embed swagger.yaml
+var swagSpec []byte
+
+//go:embed all:app/build/client
+var appFs embed.FS
+
+var (
+	_ registry.API       = (*AccountAPI)(nil)
+	_ router.RoutableAPI = (*AccountAPI)(nil)
+)
+
+type AccountAPI struct {
+	config   *config.Manager
+	accounts *account.AccountServiceDefault
+	identity ed25519.PrivateKey
+	logger   *zap.Logger
+}
+
+type AccountAPIParams struct {
+	fx.In
+	Config   *config.Manager
+	Accounts *account.AccountServiceDefault
+	Identity ed25519.PrivateKey
+	Logger   *zap.Logger
+}
+
+func NewS5(params AccountAPIParams) AccountApiResult {
+	api := &AccountAPI{
+		config:   params.Config,
+		accounts: params.Accounts,
+		identity: params.Identity,
+		logger:   params.Logger,
+	}
+
+	return AccountApiResult{
+		API:        api,
+		AccountAPI: api,
+	}
+}
+
+var Module = fx.Module("s5_api",
+	fx.Provide(NewS5),
+)
+
+type AccountApiResult struct {
+	fx.Out
+	API        registry.API `group:"api"`
+	AccountAPI *AccountAPI
+}
+
+func (a AccountAPI) Name() string {
+	return "account"
+}
+
+func (a *AccountAPI) Init() error {
+	return nil
+}
+
+func (a AccountAPI) Start(ctx context.Context) error {
+	return nil
+}
+
+func (a AccountAPI) Stop(ctx context.Context) error {
+	return nil
+}
+
+func (a AccountAPI) login(jc jape.Context) {
+	var request LoginRequest
+
+	if jc.Decode(&request) != nil {
+		return
+	}
+
+	exists, _, err := a.accounts.EmailExists(request.Email)
+
+	if !exists {
+		_ = jc.Error(account.NewAccountError(account.ErrKeyInvalidLogin, nil), http.StatusUnauthorized)
+		if err != nil {
+			a.logger.Error("failed to check if email exists", zap.Error(err))
+		}
+		return
+	}
+
+	jwt, user, err := a.accounts.LoginPassword(request.Email, request.Password, jc.Request.RemoteAddr)
+	if err != nil || user == nil {
+		_ = jc.Error(account.NewAccountError(account.ErrKeyInvalidLogin, err), http.StatusUnauthorized)
+		if err != nil {
+			a.logger.Error("failed to login", zap.Error(err))
+		}
+		return
+	}
+
+	account.SetAuthCookie(jc, a.config, jwt)
+	account.SendJWT(jc, jwt)
+
+	jc.Encode(&LoginResponse{
+		Token: jwt,
+		Otp:   user.OTPEnabled && user.OTPVerified,
+	})
+}
+
+func (a AccountAPI) register(jc jape.Context) {
+	var request RegisterRequest
+
+	if jc.Decode(&request) != nil {
+		return
+	}
+
+	if len(request.FirstName) == 0 || len(request.LastName) == 0 {
+		_ = jc.Error(account.NewAccountError(account.ErrKeyAccountCreationFailed, nil), http.StatusBadRequest)
+		return
+	}
+
+	user, err := a.accounts.CreateAccount(request.Email, request.Password, true)
+	if err != nil {
+		_ = jc.Error(err, http.StatusUnauthorized)
+		a.logger.Error("failed to update account name", zap.Error(err))
+		return
+	}
+
+	err = a.accounts.UpdateAccountName(user.ID, request.FirstName, request.LastName)
+
+	if err != nil {
+		_ = jc.Error(account.NewAccountError(account.ErrKeyAccountCreationFailed, err), http.StatusBadRequest)
+		a.logger.Error("failed to update account name", zap.Error(err))
+		return
+	}
+}
+
+func (a AccountAPI) verifyEmail(jc jape.Context) {
+	var request VerifyEmailRequest
+
+	if jc.Decode(&request) != nil {
+		return
+	}
+
+	if request.Email == "" || request.Token == "" {
+		_ = jc.Error(errors.New("invalid request"), http.StatusBadRequest)
+		return
+	}
+
+	err := a.accounts.VerifyEmail(request.Email, request.Token)
+
+	if jc.Check("Failed to verify email", err) != nil {
+		return
+	}
+}
+
+func (a AccountAPI) resendVerifyEmail(jc jape.Context) {
+	user := middleware.GetUserFromContext(jc.Request.Context())
+
+	err := a.accounts.SendEmailVerification(user)
+
+	if jc.Check("failed to resend email verification", err) != nil {
+		return
+	}
+}
+
+func (a AccountAPI) otpGenerate(jc jape.Context) {
+	user := middleware.GetUserFromContext(jc.Request.Context())
+
+	otp, err := a.accounts.OTPGenerate(user)
+	if jc.Check("failed to generate otp", err) != nil {
+		return
+	}
+
+	jc.Encode(&OTPGenerateResponse{
+		OTP: otp,
+	})
+}
+
+func (a AccountAPI) otpVerify(jc jape.Context) {
+	user := middleware.GetUserFromContext(jc.Request.Context())
+
+	var request OTPVerifyRequest
+
+	if jc.Decode(&request) != nil {
+		return
+	}
+
+	err := a.accounts.OTPEnable(user, request.OTP)
+
+	if jc.Check("failed to verify otp", err) != nil {
+		return
+	}
+}
+
+func (a AccountAPI) otpValidate(jc jape.Context) {
+	user := middleware.GetUserFromContext(jc.Request.Context())
+
+	var request OTPValidateRequest
+
+	if jc.Decode(&request) != nil {
+		return
+	}
+
+	jwt, err := a.accounts.LoginOTP(user, request.OTP)
+	if jc.Check("failed to validate otp", err) != nil {
+		return
+	}
+
+	account.SetAuthCookie(jc, a.config, jwt)
+	account.SendJWT(jc, jwt)
+
+	jc.Encode(&LoginResponse{
+		Token: jwt,
+		Otp:   false,
+	})
+}
+
+func (a AccountAPI) otpDisable(jc jape.Context) {
+	user := middleware.GetUserFromContext(jc.Request.Context())
+
+	var request OTPDisableRequest
+
+	if jc.Decode(&request) != nil {
+		return
+	}
+
+	valid, _, err := a.accounts.ValidLoginByUserID(user, request.Password)
+
+	if !valid {
+		_ = jc.Error(account.NewAccountError(account.ErrKeyInvalidLogin, nil), http.StatusUnauthorized)
+		return
+	}
+
+	err = a.accounts.OTPDisable(user)
+	if jc.Check("failed to disable otp", err) != nil {
+		return
+	}
+}
+
+func (a AccountAPI) passwordResetRequest(jc jape.Context) {
+	var request PasswordResetRequest
+
+	if jc.Decode(&request) != nil {
+		return
+	}
+
+	exists, user, err := a.accounts.EmailExists(request.Email)
+	if jc.Check("invalid request", err) != nil || !exists {
+		return
+	}
+
+	err = a.accounts.SendPasswordReset(user)
+	if jc.Check("failed to request password reset", err) != nil {
+		return
+	}
+
+	jc.ResponseWriter.WriteHeader(http.StatusOK)
+}
+
+func (a AccountAPI) passwordResetConfirm(jc jape.Context) {
+	var request PasswordResetVerifyRequest
+
+	if jc.Decode(&request) != nil {
+		return
+	}
+
+	exists, _, err := a.accounts.EmailExists(request.Email)
+	if jc.Check("invalid request", err) != nil || !exists {
+		return
+	}
+
+	err = a.accounts.ResetPassword(request.Email, request.Password, request.Token)
+	if jc.Check("failed to reset password", err) != nil {
+		return
+	}
+
+	jc.ResponseWriter.WriteHeader(http.StatusOK)
+}
+
+func (a AccountAPI) ping(jc jape.Context) {
+	token := middleware.GetAuthTokenFromContext(jc.Request.Context())
+	account.EchoAuthCookie(jc, a.config)
+	jc.Encode(&PongResponse{
+		Ping:  "pong",
+		Token: token,
+	})
+}
+
+func (a AccountAPI) accountInfo(jc jape.Context) {
+	user := middleware.GetUserFromContext(jc.Request.Context())
+
+	_, acct, _ := a.accounts.AccountExists(user)
+
+	jc.Encode(&AccountInfoResponse{
+		ID:        acct.ID,
+		Email:     acct.Email,
+		FirstName: acct.FirstName,
+		LastName:  acct.LastName,
+		Verified:  acct.Verified,
+	})
+
+}
+
+func (a AccountAPI) logout(c jape.Context) {
+	account.ClearAuthCookie(c, a.config)
+}
+
+func (a AccountAPI) uploadLimit(c jape.Context) {
+	c.Encode(&UploadLimitResponse{
+		Limit: a.config.Config().Core.PostUploadLimit,
+	})
+}
+
+func (a AccountAPI) updateEmail(c jape.Context) {
+	user := middleware.GetUserFromContext(c.Request.Context())
+
+	var request UpdateEmailRequest
+
+	if c.Decode(&request) != nil {
+		return
+	}
+
+	err := a.accounts.UpdateAccountEmail(user, request.Email, request.Password)
+	if c.Check("failed to update email", err) != nil {
+		return
+	}
+}
+
+func (a AccountAPI) updatePassword(c jape.Context) {
+	user := middleware.GetUserFromContext(c.Request.Context())
+
+	var request UpdatePasswordRequest
+
+	if c.Decode(&request) != nil {
+		return
+	}
+
+	err := a.accounts.UpdateAccountPassword(user, request.CurrentPassword, request.NewPassword)
+	if c.Check("failed to update password", err) != nil {
+		return
+	}
+
+}
+
+func (a AccountAPI) meta(c jape.Context) {
+	c.Encode(&MetaResponse{
+		Domain: a.config.Config().Core.Domain,
+	})
+
+}
+
+func (a *AccountAPI) Routes() (*httprouter.Router, error) {
+	loginAuthMw2fa := authMiddleware(middleware.AuthMiddlewareOptions{
+		Identity:       a.identity,
+		Accounts:       a.accounts,
+		Config:         a.config,
+		Purpose:        account.JWTPurpose2FA,
+		EmptyAllowed:   true,
+		ExpiredAllowed: true,
+	})
+
+	authMw := authMiddleware(middleware.AuthMiddlewareOptions{
+		Identity: a.identity,
+		Accounts: a.accounts,
+		Config:   a.config,
+		Purpose:  account.JWTPurposeNone,
+	})
+
+	pingAuthMw := authMiddleware(middleware.AuthMiddlewareOptions{
+		Identity: a.identity,
+		Accounts: a.accounts,
+		Config:   a.config,
+		Purpose:  account.JWTPurposeLogin,
+	})
+
+	appFiles, _ := fs.Sub(appFs, "app/build/client")
+	appServ := http.FileServer(http.FS(appFiles))
+
+	appHandler := func(c jape.Context) {
+		appServ.ServeHTTP(c.ResponseWriter, c.Request)
+	}
+
+	appServer := middleware.ApplyMiddlewares(appHandler, middleware.ProxyMiddleware)
+
+	swaggerRoutes, err := swagger.Swagger(swagSpec, map[string]jape.Handler{})
+	if err != nil {
+		return nil, err
+	}
+
+	swaggerJape := jape.Mux(swaggerRoutes)
+
+	getApiJape := jape.Mux(map[string]jape.Handler{
+		"GET /api/auth/otp/generate": middleware.ApplyMiddlewares(a.otpGenerate, authMw, middleware.ProxyMiddleware),
+		"GET /api/account":           middleware.ApplyMiddlewares(a.accountInfo, authMw, middleware.ProxyMiddleware),
+		"GET /api/upload-limit":      middleware.ApplyMiddlewares(a.uploadLimit, middleware.ProxyMiddleware),
+		"GET /api/meta":              middleware.ApplyMiddlewares(a.meta, middleware.ProxyMiddleware),
+	})
+
+	getHandler := func(c jape.Context) {
+		if strings.HasPrefix(c.Request.URL.Path, "/api") {
+			getApiJape.ServeHTTP(c.ResponseWriter, c.Request)
+			return
+		}
+
+		if strings.HasPrefix(c.Request.URL.Path, "/swagger") {
+			swaggerJape.ServeHTTP(c.ResponseWriter, c.Request)
+			return
+		}
+
+		if !strings.HasPrefix(c.Request.URL.Path, "/assets") && c.Request.URL.Path != "favicon.ico" && c.Request.URL.Path != "/" && !strings.HasSuffix(c.Request.URL.Path, ".html") {
+			c.Request.URL.Path = "/"
+		}
+		appServer(c)
+	}
+
+	corsMw := cors.New(cors.Options{
+		AllowOriginFunc: func(origin string) bool {
+			return true
+		},
+		AllowedMethods: []string{"GET", "POST", "DELETE"},
+		AllowedHeaders: []string{"Authorization", "Content-Type"},
+	})
+
+	corsOptionsHandler := func(c jape.Context) {
+		c.ResponseWriter.WriteHeader(http.StatusOK)
+	}
+
+	routes := map[string]jape.Handler{
+		// Auth
+		"POST /api/auth/ping":         middleware.ApplyMiddlewares(a.ping, corsMw.Handler, pingAuthMw, middleware.ProxyMiddleware),
+		"POST /api/auth/login":        middleware.ApplyMiddlewares(a.login, corsMw.Handler, loginAuthMw2fa, middleware.ProxyMiddleware),
+		"POST /api/auth/register":     middleware.ApplyMiddlewares(a.register, corsMw.Handler, middleware.ProxyMiddleware),
+		"POST /api/auth/otp/validate": middleware.ApplyMiddlewares(a.otpValidate, corsMw.Handler, authMw, middleware.ProxyMiddleware),
+		"POST /api/auth/logout":       middleware.ApplyMiddlewares(a.logout, corsMw.Handler, authMw, middleware.ProxyMiddleware),
+
+		// Account
+		"POST /api/account/verify-email":           middleware.ApplyMiddlewares(a.verifyEmail, corsMw.Handler, authMw, middleware.ProxyMiddleware),
+		"POST /api/account/verify-email/resend":    middleware.ApplyMiddlewares(a.resendVerifyEmail, corsMw.Handler, authMw, middleware.ProxyMiddleware),
+		"POST /api/account/otp/verify":             middleware.ApplyMiddlewares(a.otpVerify, corsMw.Handler, authMw, middleware.ProxyMiddleware),
+		"POST /api/account/otp/disable":            middleware.ApplyMiddlewares(a.otpDisable, corsMw.Handler, authMw, middleware.ProxyMiddleware),
+		"POST /api/account/password-reset/request": middleware.ApplyMiddlewares(a.passwordResetRequest, corsMw.Handler, middleware.ProxyMiddleware),
+		"POST /api/account/password-reset/confirm": middleware.ApplyMiddlewares(a.passwordResetConfirm, corsMw.Handler, middleware.ProxyMiddleware),
+		"POST /api/account/update-email":           middleware.ApplyMiddlewares(a.updateEmail, corsMw.Handler, authMw, middleware.ProxyMiddleware),
+		"POST /api/account/update-password":        middleware.ApplyMiddlewares(a.updatePassword, corsMw.Handler, authMw, middleware.ProxyMiddleware),
+
+		// CORS
+		"OPTIONS /api/auth/ping":         middleware.ApplyMiddlewares(corsOptionsHandler, corsMw.Handler, authMw, middleware.ProxyMiddleware),
+		"OPTIONS /api/auth/login":        middleware.ApplyMiddlewares(corsOptionsHandler, corsMw.Handler, loginAuthMw2fa, middleware.ProxyMiddleware),
+		"OPTIONS /api/auth/register":     middleware.ApplyMiddlewares(corsOptionsHandler, corsMw.Handler, middleware.ProxyMiddleware),
+		"OPTIONS /api/auth/otp/validate": middleware.ApplyMiddlewares(corsOptionsHandler, corsMw.Handler, middleware.ProxyMiddleware),
+		"OPTIONS /api/auth/logout":       middleware.ApplyMiddlewares(corsOptionsHandler, corsMw.Handler, authMw, middleware.ProxyMiddleware),
+
+		"OPTIONS /api/account/verify-email":           middleware.ApplyMiddlewares(corsOptionsHandler, corsMw.Handler, middleware.ProxyMiddleware),
+		"OPTIONS /api/account/verify-email/resend":    middleware.ApplyMiddlewares(corsOptionsHandler, corsMw.Handler, authMw, middleware.ProxyMiddleware),
+		"OPTIONS /api/account/otp/verify":             middleware.ApplyMiddlewares(corsOptionsHandler, corsMw.Handler, authMw, middleware.ProxyMiddleware),
+		"OPTIONS /api/account/otp/disable":            middleware.ApplyMiddlewares(corsOptionsHandler, corsMw.Handler, authMw, middleware.ProxyMiddleware),
+		"OPTIONS /api/account/password-reset/request": middleware.ApplyMiddlewares(corsOptionsHandler, corsMw.Handler, middleware.ProxyMiddleware),
+		"OPTIONS /api/account/password-reset/confirm": middleware.ApplyMiddlewares(corsOptionsHandler, corsMw.Handler, middleware.ProxyMiddleware),
+		"OPTIONS /api/account/update-email":           middleware.ApplyMiddlewares(corsOptionsHandler, corsMw.Handler, authMw, middleware.ProxyMiddleware),
+		"OPTIONS /api/account/update-password":        middleware.ApplyMiddlewares(corsOptionsHandler, corsMw.Handler, authMw, middleware.ProxyMiddleware),
+
+		// Get Routes
+		"OPTIONS /api/upload-limit":      middleware.ApplyMiddlewares(corsOptionsHandler, corsMw.Handler, middleware.ProxyMiddleware),
+		"OPTIONS /api/account":           middleware.ApplyMiddlewares(corsOptionsHandler, corsMw.Handler, authMw, middleware.ProxyMiddleware),
+		"OPTIONS /api/auth/otp/generate": middleware.ApplyMiddlewares(corsOptionsHandler, corsMw.Handler, authMw, middleware.ProxyMiddleware),
+
+		"GET /*path": middleware.ApplyMiddlewares(getHandler, corsMw.Handler),
+	}
+
+	return jape.Mux(routes), nil
+}
+func (a AccountAPI) Can(w http.ResponseWriter, r *http.Request) bool {
+	return false
+}
+
+func (a AccountAPI) Handle(w http.ResponseWriter, r *http.Request) {
+}
+
+func (a *AccountAPI) Domain() string {
+	return router.BuildSubdomain(a, a.config)
+}
+
+func (a AccountAPI) AuthTokenName() string {
+	return account.AUTH_COOKIE_NAME
+}

api/account/app

@@ -0,0 +1 @@
+Subproject commit d7f0154fb89dc9dfafc2bc7fd22e24c6ebaaafc7

api/account/messages.go

@@ -0,0 +1,73 @@
+package account
+
+type LoginRequest struct {
+	Email    string `json:"email"`
+	Password string `json:"password"`
+}
+
+type LoginResponse struct {
+	Token string `json:"token"`
+	Otp   bool   `json:"otp"`
+}
+
+type RegisterRequest struct {
+	FirstName string `json:"first_name"`
+	LastName  string `json:"last_name"`
+	Email     string `json:"email"`
+	Password  string `json:"password"`
+}
+
+type OTPGenerateResponse struct {
+	OTP string `json:"otp"`
+}
+
+type OTPVerifyRequest struct {
+	OTP string `json:"otp"`
+}
+
+type OTPValidateRequest struct {
+	OTP string `json:"otp"`
+}
+type OTPDisableRequest struct {
+	Password string `json:"password"`
+}
+type VerifyEmailRequest struct {
+	Email string `json:"email"`
+	Token string `json:"token"`
+}
+type PasswordResetRequest struct {
+	Email string `json:"email"`
+}
+type PasswordResetVerifyRequest struct {
+	Email    string `json:"email"`
+	Token    string `json:"token"`
+	Password string `json:"password"`
+}
+
+type PongResponse struct {
+	Ping  string `json:"ping"`
+	Token string `json:"token"`
+}
+type AccountInfoResponse struct {
+	ID        uint   `json:"id"`
+	Email     string `json:"email"`
+	FirstName string `json:"first_name"`
+	LastName  string `json:"last_name"`
+	Verified  bool   `json:"verified"`
+}
+
+type UploadLimitResponse struct {
+	Limit uint64 `json:"limit"`
+}
+type UpdateEmailRequest struct {
+	Email    string `json:"email"`
+	Password string `json:"password"`
+}
+type UpdatePasswordRequest struct {
+	CurrentPassword string `json:"current_password"`
+	NewPassword     string `json:"new_password"`
+}
+
+type MetaResponse struct {
+	Domain string `json:"domain"`
+}

api/account/middleware.go

@@ -0,0 +1,23 @@
+package account
+
+import (
+	"net/http"
+
+	"git.lumeweb.com/LumeWeb/portal/account"
+
+	"git.lumeweb.com/LumeWeb/portal/api/middleware"
+)
+
+const (
+	authCookieName = account.AUTH_COOKIE_NAME
+	authQueryParam = "auth_token"
+)
+
+func findToken(r *http.Request) string {
+	return middleware.FindAuthToken(r, authCookieName, authQueryParam)
+}
+
+func authMiddleware(options middleware.AuthMiddlewareOptions) middleware.HttpMiddlewareFunc {
+	options.FindToken = findToken
+	return middleware.AuthMiddleware(options)
+}

api/account/swagger.yaml

@@ -0,0 +1,351 @@
+openapi: 3.0.0
+info:
+    title: Account Management API
+    version: "1.0"
+    description: API for managing user accounts, including login, registration, OTP operations, and password resets.
+paths:
+    /api/auth/login:
+        post:
+            summary: Login to the system
+            requestBody:
+                required: true
+                content:
+                    application/json:
+                        schema:
+                            $ref: '#/components/schemas/LoginRequest'
+            responses:
+                '200':
+                    description: Successfully logged in
+                    content:
+                        application/json:
+                            schema:
+                                $ref: '#/components/schemas/LoginResponse'
+                '401':
+                    description: Unauthorized
+    /api/auth/logout:
+        post:
+            summary: Logout of account service
+            responses:
+                '200':
+                    description: Successfully logged out
+    /api/auth/register:
+        post:
+            summary: Register a new account
+            requestBody:
+                required: true
+                content:
+                    application/json:
+                        schema:
+                            $ref: '#/components/schemas/RegisterRequest'
+            responses:
+                '200':
+                    description: Successfully registered
+                '400':
+                    description: Bad Request
+    /api/account/verify-email:
+        post:
+            summary: Verify email address
+            requestBody:
+                required: true
+                content:
+                    application/json:
+                        schema:
+                            $ref: '#/components/schemas/VerifyEmailRequest'
+            responses:
+                '200':
+                    description: Email verified successfully
+    /api/account/verify-email/resend:
+        post:
+            summary: Resend email verification
+            responses:
+                '200':
+                    description: Email verification resent successfully
+    /api/auth/otp/generate:
+        get:
+            summary: Generate OTP for two-factor authentication
+            responses:
+                '200':
+                    description: OTP generated successfully
+                    content:
+                        application/json:
+                            schema:
+                                $ref: '#/components/schemas/OTPGenerateResponse'
+    /api/account/otp/verify:
+        post:
+            summary: Verify OTP for enabling two-factor authentication
+            requestBody:
+                required: true
+                content:
+                    application/json:
+                        schema:
+                            $ref: '#/components/schemas/OTPVerifyRequest'
+            responses:
+                '200':
+                    description: OTP verified successfully
+    /api/account/otp/validate:
+        post:
+            summary: Validate OTP for two-factor authentication login
+            requestBody:
+                required: true
+                content:
+                    application/json:
+                        schema:
+                            $ref: '#/components/schemas/OTPValidateRequest'
+            responses:
+                '200':
+                    description: OTP validated successfully
+    /api/auth/otp/disable:
+        post:
+            summary: Disable OTP for two-factor authentication
+            requestBody:
+                required: true
+                content:
+                    application/json:
+                        schema:
+                            $ref: '#/components/schemas/OTPDisableRequest'
+            responses:
+                '200':
+                    description: OTP disabled successfully
+    /api/account/password-reset/request:
+        post:
+            summary: Request a password reset
+            requestBody:
+                required: true
+                content:
+                    application/json:
+                        schema:
+                            $ref: '#/components/schemas/PasswordResetRequest'
+            responses:
+                '200':
+                    description: Password reset requested successfully
+    /api/account/password-reset/confirm:
+        post:
+            summary: Confirm a password reset
+            requestBody:
+                required: true
+                content:
+                    application/json:
+                        schema:
+                            $ref: '#/components/schemas/PasswordResetVerifyRequest'
+            responses:
+                '200':
+                    description: Password reset successfully
+    /api/auth/ping:
+        post:
+            summary: Auth check endpoint
+            responses:
+                '200':
+                    description: Pong
+                    content:
+                        application/json:
+                            schema:
+                                $ref: '#/components/schemas/PingResponse'
+                '401':
+                    description: Unauthorized
+    /api/account:
+        get:
+            summary: Get account information
+            responses:
+                '200':
+                    description: Account information retrieved successfully
+                    content:
+                        application/json:
+                            schema:
+                                $ref: '#/components/schemas/AccountInfoResponse'
+                '401':
+                    description: Unauthorized
+    /api/account/update-email:
+        post:
+            summary: Update email address
+            requestBody:
+                required: true
+                content:
+                    application/json:
+                        schema:
+                            $ref: '#/components/schemas/UpdateEmailRequest'
+            responses:
+                '200':
+                    description: Email updated successfully
+    /api/account/update-password:
+        post:
+            summary: Update password
+            requestBody:
+                required: true
+                content:
+                    application/json:
+                        schema:
+                            $ref: '#/components/schemas/UpdatePasswordRequest'
+            responses:
+                '200':
+                    description: Password updated successfully
+    /api/upload-limit:
+        get:
+            summary: Get the basic file upload (POST) upload limit set by the portal
+            responses:
+                '200':
+                    description: Upload limit retrieved successfully
+                    content:
+                        application/json:
+                            schema:
+                                $ref: '#/components/schemas/UploadLimitResponse'
+    /api/meta:
+        get:
+            summary: Get metadata about the portal
+            responses:
+                '200':
+                    description: Metadata retrieved successfully
+                    content:
+                        application/json:
+                            schema:
+                                $ref: '#/components/schemas/MetaResponse'
+
+
+components:
+    schemas:
+        LoginRequest:
+            type: object
+            required:
+                - email
+                - password
+            properties:
+                email:
+                    type: string
+                password:
+                    type: string
+        LoginResponse:
+            type: object
+            properties:
+                token:
+                    type: string
+        RegisterRequest:
+            type: object
+            required:
+                - first_name
+                - last_name
+                - email
+                - password
+            properties:
+                first_name:
+                    type: string
+                last_name:
+                    type: string
+                email:
+                    type: string
+                password:
+                    type: string
+        VerifyEmailRequest:
+            type: object
+            required:
+                - email
+                - token
+            properties:
+                email:
+                    type: string
+                token:
+                    type: string
+        OTPGenerateResponse:
+            type: object
+            properties:
+                OTP:
+                    type: string
+        OTPVerifyRequest:
+            type: object
+            required:
+                - OTP
+            properties:
+                OTP:
+                    type: string
+        OTPValidateRequest:
+            type: object
+            required:
+                - OTP
+            properties:
+                OTP:
+                    type: string
+        OTPDisableRequest:
+            type: object
+            required:
+                - password
+            properties:
+                password:
+                    type: string
+        PasswordResetRequest:
+            type: object
+            required:
+                - email
+            properties:
+                email:
+                    type: string
+        PasswordResetVerifyRequest:
+            type: object
+            required:
+                - email
+                - token
+                - password
+            properties:
+                email:
+                    type: string
+                token:
+                    type: string
+                password:
+                    type: string
+        UpdateEmailRequest:
+            type: object
+            required:
+                - email
+                - password
+            properties:
+                email:
+                    type: string
+                password:
+                    type: string
+        UpdatePasswordRequest:
+            type: object
+            required:
+                - current_password
+                - new_password
+            properties:
+                current_password:
+                    type: string
+                new_password:
+                    type: string
+        PingResponse:
+            type: object
+            properties:
+                ping:
+                    type: string
+                token:
+                    type: string
+        AccountInfoResponse:
+            type: object
+            required:
+                - id
+                - first_name
+                - last_name
+                - email
+                - verified
+            properties:
+                id:
+                    type: number
+                first_name:
+                    type: string
+                last_name:
+                    type: string
+                email:
+                    type: string
+                verified:
+                    type: boolean
+        UploadLimitResponse:
+            type: object
+            properties:
+                limit:
+                    type: number
+            required:
+                - limit
+        MetaResponse:
+            type: object
+            required:
+                - domain
+            properties:
+                domain:
+                    type: string

api/api.go

@@ -0,0 +1,68 @@
+package api
+
+import (
+	"context"
+	"slices"
+
+	"git.lumeweb.com/LumeWeb/portal/config"
+
+	"git.lumeweb.com/LumeWeb/portal/api/registry"
+	"go.uber.org/fx"
+)
+
+var alwaysEnabled = []string{"account"}
+
+func BuildApis(cm *config.Manager) fx.Option {
+	var options []fx.Option
+	enabledProtocols := cm.Viper().GetStringSlice("core.protocols")
+	for _, entry := range registry.GetEntryRegistry() {
+		if slices.Contains(enabledProtocols, entry.Key) || slices.Contains(alwaysEnabled, entry.Key) {
+			options = append(options, entry.Module)
+		}
+	}
+
+	type initParams struct {
+		fx.In
+		Apis []registry.API `group:"api"`
+	}
+
+	options = append(options, fx.Invoke(func(params initParams) error {
+		for _, protocol := range params.Apis {
+			err := protocol.Init()
+			if err != nil {
+				return err
+			}
+
+			registry.RegisterAPI(protocol)
+		}
+
+		return nil
+	}))
+
+	return fx.Module("api", fx.Options(options...))
+}
+
+type LifecyclesParams struct {
+	fx.In
+
+	Protocols []registry.API `group:"protocol"`
+}
+
+func SetupLifecycles(lifecycle fx.Lifecycle, params LifecyclesParams) error {
+	for _, entry := range registry.GetEntryRegistry() {
+		for _, protocol := range params.Protocols {
+			if protocol.Name() == entry.Key {
+				lifecycle.Append(fx.Hook{
+					OnStart: func(ctx context.Context) error {
+						return protocol.Start(ctx)
+					},
+					OnStop: func(ctx context.Context) error {
+						return protocol.Stop(ctx)
+					},
+				})
+			}
+		}
+	}
+
+	return nil
+}

api/casbin.go

@@ -0,0 +1,100 @@
+package api
+
+import (
+	"github.com/casbin/casbin/v2"
+	"github.com/casbin/casbin/v2/model"
+	"github.com/casbin/casbin/v2/persist"
+	"go.uber.org/zap"
+	"strings"
+	"sync"
+)
+
+func NewCasbin(logger *zap.Logger) *casbin.Enforcer {
+	m := model.NewModel()
+	m.AddDef("r", "r", "sub, obj, act")
+	m.AddDef("p", "p", "sub, obj, act")
+	m.AddDef("g", "g", "_, _")
+	m.AddDef("e", "e", "some(where (p.eft == allow))")
+	m.AddDef("m", "m", "g(r.sub, p.sub) && r.obj == p.obj && r.act == p.act")
+
+	a := NewPolicyAdapter(logger)
+
+	e, err := casbin.NewEnforcer(m, a)
+	if err != nil {
+		logger.Fatal("Failed to create casbin enforcer", zap.Error(err))
+	}
+
+	// Add policies after creating the enforcer
+	_ = a.AddPolicy("p", "p", []string{"admin", "/admin*"})
+
+	err = e.LoadPolicy()
+	if err != nil {
+		logger.Fatal("Failed to load policies into Casbin model", zap.Error(err))
+	}
+
+	return e
+}
+
+type PolicyAdapter struct {
+	policy []string
+	lock   sync.RWMutex
+	logger *zap.Logger
+}
+
+// NewPolicyAdapter creates a new PolicyAdapter instance.
+func NewPolicyAdapter(logger *zap.Logger) *PolicyAdapter {
+	return &PolicyAdapter{
+		policy: make([]string, 0),
+		logger: logger,
+	}
+}
+
+// LoadPolicy loads all policy rules from the storage.
+func (a *PolicyAdapter) LoadPolicy(model model.Model) error {
+	a.lock.RLock()
+	defer a.lock.RUnlock()
+
+	for _, line := range a.policy {
+		err := persist.LoadPolicyLine(line, model)
+		if err != nil {
+			a.logger.Fatal("Failed to load policy line", zap.Error(err))
+		}
+	}
+	return nil
+}
+
+// SavePolicy saves all policy rules to the storage.
+func (a *PolicyAdapter) SavePolicy(model model.Model) error {
+	return nil
+}
+
+// AddPolicy adds a policy rule to the storage.
+// AddPolicy adds a policy rule to the storage.
+func (a *PolicyAdapter) AddPolicy(sec string, ptype string, rule []string) error {
+	a.lock.Lock()
+	defer a.lock.Unlock()
+
+	// Create a line representing the policy rule with the section
+	line := sec + ", " + ptype + ", " + strings.Join(rule, ", ")
+
+	// Check if the policy rule already exists
+	for _, existingLine := range a.policy {
+		if line == existingLine {
+			return nil // Policy rule already exists, no need to add it again
+		}
+	}
+
+	// Add the policy rule to the storage
+	a.policy = append(a.policy, line)
+	return nil
+}
+
+// RemovePolicy removes a policy rule from the storage.
+func (a *PolicyAdapter) RemovePolicy(sec string, ptype string, rule []string) error {
+	return nil
+}
+
+// RemoveFilteredPolicy removes policy rules that match the filter from the storage.
+func (a *PolicyAdapter) RemoveFilteredPolicy(sec string, ptype string, fieldIndex int, fieldValues ...string) error {
+	return nil
+}

api/middleware/middleware.go

@@ -0,0 +1,260 @@
+package middleware
+
+import (
+	"context"
+	"crypto/ed25519"
+	"errors"
+	"net/http"
+	"slices"
+	"strconv"
+	"strings"
+
+	"git.lumeweb.com/LumeWeb/portal/config"
+
+	"git.lumeweb.com/LumeWeb/portal/account"
+	"github.com/golang-jwt/jwt/v5"
+	"go.sia.tech/jape"
+)
+
+const DEFAULT_AUTH_CONTEXT_KEY = "user_id"
+const AUTH_TOKEN_CONTEXT_KEY = "auth_token"
+
+type JapeMiddlewareFunc func(jape.Handler) jape.Handler
+type HttpMiddlewareFunc func(http.Handler) http.Handler
+
+type FindAuthTokenFunc func(r *http.Request) string
+
+func AdaptMiddleware(mid func(http.Handler) http.Handler) JapeMiddlewareFunc {
+	return jape.Adapt(func(h http.Handler) http.Handler {
+		handler := mid(h)
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			handler.ServeHTTP(w, r)
+		})
+	})
+}
+
+// ProxyMiddleware creates a new HTTP middleware for handling X-Forwarded-For headers.
+func ProxyMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
+			ips := strings.Split(xff, ", ")
+			if len(ips) > 0 {
+				r.RemoteAddr = ips[0]
+			}
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
+func ApplyMiddlewares(handler jape.Handler, middlewares ...interface{}) jape.Handler {
+	for i := len(middlewares) - 1; i >= 0; i-- {
+		switch middlewares[i].(type) {
+		case JapeMiddlewareFunc:
+			mid := middlewares[i].(JapeMiddlewareFunc)
+			handler = mid(handler)
+		case func(http.Handler) http.Handler:
+			mid := middlewares[i].(func(http.Handler) http.Handler)
+			handler = AdaptMiddleware(mid)(handler)
+		case HttpMiddlewareFunc:
+			mid := middlewares[i].(HttpMiddlewareFunc)
+			handler = AdaptMiddleware(mid)(handler)
+
+		default:
+			panic("Invalid middleware type")
+		}
+	}
+	return handler
+}
+
+func FindAuthToken(r *http.Request, cookieName string, queryParam string) string {
+	authHeader := ParseAuthTokenHeader(r.Header)
+
+	if authHeader != "" {
+		return authHeader
+	}
+
+	if cookie, err := r.Cookie(cookieName); cookie != nil && err == nil {
+		return cookie.Value
+	}
+
+	if cookie, err := r.Cookie(account.AUTH_COOKIE_NAME); cookie != nil && err == nil {
+		return cookie.Value
+	}
+
+	return r.FormValue(queryParam)
+}
+
+func ParseAuthTokenHeader(headers http.Header) string {
+	authHeader := headers.Get("Authorization")
+	if authHeader == "" {
+		return ""
+	}
+
+	authHeader = strings.TrimPrefix(authHeader, "Bearer ")
+	authHeader = strings.TrimPrefix(authHeader, "bearer ")
+
+	return authHeader
+}
+
+type AuthMiddlewareOptions struct {
+	Identity       ed25519.PrivateKey
+	Accounts       *account.AccountServiceDefault
+	FindToken      FindAuthTokenFunc
+	Purpose        account.JWTPurpose
+	AuthContextKey string
+	Config         *config.Manager
+	EmptyAllowed   bool
+	ExpiredAllowed bool
+}
+
+func AuthMiddleware(options AuthMiddlewareOptions) func(http.Handler) http.Handler {
+	if options.AuthContextKey == "" {
+		options.AuthContextKey = DEFAULT_AUTH_CONTEXT_KEY
+	}
+
+	domain := options.Config.Config().Core.Domain
+
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			authToken := options.FindToken(r)
+
+			if authToken == "" {
+				if !options.EmptyAllowed {
+					http.Error(w, "Invalid JWT", http.StatusUnauthorized)
+					return
+				}
+				next.ServeHTTP(w, r)
+				return
+			}
+
+			var audList *jwt.ClaimStrings
+
+			claim, err := account.JWTVerifyToken(authToken, domain, options.Identity, func(claim *jwt.RegisteredClaims) error {
+				aud, _ := claim.GetAudience()
+
+				audList = &aud
+
+				if options.Purpose != account.JWTPurposeNone && jwtPurposeEqual(aud, options.Purpose) == false {
+					return account.ErrJWTInvalid
+				}
+
+				return nil
+			})
+
+			if err != nil {
+				unauthorized := true
+				if errors.Is(err, jwt.ErrTokenExpired) && options.ExpiredAllowed {
+					unauthorized = false
+				}
+
+				if !unauthorized && audList == nil {
+					if audList == nil {
+						var claim jwt.RegisteredClaims
+
+						unverified, _, err := jwt.NewParser().ParseUnverified(authToken, &claim)
+						if err != nil {
+							http.Error(w, err.Error(), http.StatusInternalServerError)
+							return
+						}
+
+						audList, err := unverified.Claims.GetAudience()
+						if err != nil {
+							http.Error(w, err.Error(), http.StatusInternalServerError)
+							return
+						}
+
+						if jwtPurposeEqual(audList, options.Purpose) == true {
+							unauthorized = true
+						}
+
+					}
+				}
+
+				if unauthorized {
+					http.Error(w, err.Error(), http.StatusUnauthorized)
+					return
+				}
+			}
+
+			if claim == nil && options.ExpiredAllowed {
+				next.ServeHTTP(w, r)
+				return
+			}
+
+			userId, err := strconv.ParseUint(claim.Subject, 10, 64)
+
+			if err != nil {
+				http.Error(w, account.ErrJWTInvalid.Error(), http.StatusBadRequest)
+				return
+			}
+
+			exists, _, err := options.Accounts.AccountExists(uint(userId))
+
+			if !exists || err != nil {
+				http.Error(w, account.ErrJWTInvalid.Error(), http.StatusBadRequest)
+				return
+			}
+
+			ctx := context.WithValue(r.Context(), options.AuthContextKey, uint(userId))
+			ctx = context.WithValue(ctx, AUTH_TOKEN_CONTEXT_KEY, authToken)
+			r = r.WithContext(ctx)
+
+			next.ServeHTTP(w, r)
+		})
+	}
+}
+
+func MergeRoutes(routes ...map[string]jape.Handler) map[string]jape.Handler {
+	merged := make(map[string]jape.Handler)
+
+	for _, route := range routes {
+		for k, v := range route {
+			merged[k] = v
+		}
+	}
+
+	return merged
+}
+
+func GetUserFromContext(ctx context.Context, key ...string) uint {
+	realKey := ""
+
+	if len(key) > 0 {
+		realKey = key[0]
+	}
+
+	if realKey == "" {
+		realKey = DEFAULT_AUTH_CONTEXT_KEY
+	}
+
+	userId, ok := ctx.Value(realKey).(uint)
+
+	if !ok {
+		panic("user id stored in context is not of type uint")
+	}
+
+	return userId
+}
+
+func GetAuthTokenFromContext(ctx context.Context) string {
+	authToken, ok := ctx.Value(AUTH_TOKEN_CONTEXT_KEY).(string)
+
+	if !ok {
+		panic("auth token stored in context is not of type string")
+	}
+
+	return authToken
+}
+
+func CtxAborted(ctx context.Context) bool {
+	select {
+	case <-ctx.Done():
+		return true
+	default:
+		return false
+	}
+}
+
+func jwtPurposeEqual(aud jwt.ClaimStrings, purpose account.JWTPurpose) bool {
+	return slices.Contains[jwt.ClaimStrings, string](aud, string(purpose))
+}

api/registry/registry.go

@@ -0,0 +1,57 @@
+package registry
+
+import (
+	"context"
+
+	router2 "git.lumeweb.com/LumeWeb/portal/api/router"
+	"go.uber.org/fx"
+)
+
+type API interface {
+	Name() string
+	Init() error
+	Start(ctx context.Context) error
+	Stop(ctx context.Context) error
+}
+
+type APIEntry struct {
+	Key    string
+	Module fx.Option
+}
+
+var apiEntryRegistry []APIEntry
+var apiRegistry map[string]API
+var router *router2.APIRouter
+
+func init() {
+	router = router2.NewAPIRouter()
+	apiRegistry = make(map[string]API)
+}
+
+func RegisterEntry(entry APIEntry) {
+	apiEntryRegistry = append(apiEntryRegistry, entry)
+}
+
+func RegisterAPI(api API) {
+	apiRegistry[api.Name()] = api
+}
+
+func GetEntryRegistry() []APIEntry {
+	return apiEntryRegistry
+}
+
+func GetAPI(name string) API {
+	if _, ok := apiRegistry[name]; !ok {
+		panic("API not found: " + name)
+	}
+
+	return apiRegistry[name]
+}
+
+func GetAllAPIs() map[string]API {
+	return apiRegistry
+}
+
+func GetRouter() *router2.APIRouter {
+	return router
+}

api/router/router.go

@@ -0,0 +1,115 @@
+package router
+
+import (
+	"net/http"
+	"sync"
+
+	"git.lumeweb.com/LumeWeb/portal/config"
+
+	"go.uber.org/zap"
+
+	"github.com/julienschmidt/httprouter"
+)
+
+type RoutableAPI interface {
+	Name() string
+	Domain() string
+	AuthTokenName() string
+	Can(w http.ResponseWriter, r *http.Request) bool
+	Handle(w http.ResponseWriter, r *http.Request)
+	Routes() (*httprouter.Router, error)
+}
+
+type APIRouter struct {
+	apis        map[string]RoutableAPI
+	apiDomain   map[string]string
+	apiHandlers map[string]http.Handler
+	logger      *zap.Logger
+	config      *config.Manager
+	mutex       *sync.RWMutex
+}
+
+// Implement the ServeHTTP method on our new type
+func (hs APIRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if handler := hs.getHandlerByDomain(r.Host); handler != nil {
+		handler.ServeHTTP(w, r)
+		return
+	}
+
+	for _, api := range hs.apis {
+		if api.Can(w, r) {
+			api.Handle(w, r)
+			return
+		}
+	}
+
+	http.NotFound(w, r)
+}
+
+func (hs *APIRouter) RegisterAPI(impl RoutableAPI) {
+	name := impl.Name()
+	hs.apis[name] = impl
+	hs.apiDomain[name+"."+hs.config.Config().Core.Domain] = name
+}
+
+func (hs *APIRouter) getHandlerByDomain(domain string) http.Handler {
+	if apiName := hs.apiDomain[domain]; apiName != "" {
+		return hs.getHandler(apiName)
+	}
+
+	return nil
+}
+
+func (hs *APIRouter) getHandler(protocol string) http.Handler {
+	hs.mutex.RLock()
+	handler, ok := hs.apiHandlers[protocol]
+	hs.mutex.RUnlock()
+
+	if ok {
+		return handler
+	}
+
+	hs.mutex.Lock()
+	defer hs.mutex.Unlock()
+
+	// Double-check if the handler was created while acquiring the write lock
+	if handler, ok := hs.apiHandlers[protocol]; ok {
+		return handler
+	}
+
+	proto, ok := hs.apis[protocol]
+	if !ok {
+		hs.logger.Fatal("Protocol not found", zap.String("protocol", protocol))
+		return nil
+	}
+
+	routes, err := proto.Routes()
+	if err != nil {
+		hs.logger.Fatal("Error getting routes", zap.Error(err))
+		return nil
+	}
+
+	hs.apiHandlers[protocol] = routes
+	return routes
+}
+
+func NewAPIRouter() *APIRouter {
+	return &APIRouter{
+		apis:        make(map[string]RoutableAPI),
+		apiHandlers: make(map[string]http.Handler),
+		apiDomain:   make(map[string]string),
+		mutex:       &sync.RWMutex{},
+	}
+}
+
+func (hs *APIRouter) SetLogger(logger *zap.Logger) {
+	hs.logger = logger
+}
+
+func (hs *APIRouter) SetConfig(config *config.Manager) {
+	hs.config = config
+}
+
+func BuildSubdomain(api RoutableAPI, cfg *config.Manager) string {
+	return api.Name() + "." + cfg.Config().Core.Domain
+}

api/s5.go

@@ -0,0 +1,15 @@
+//go:build s5
+
+package api
+
+import (
+	"git.lumeweb.com/LumeWeb/portal/api/registry"
+	"git.lumeweb.com/LumeWeb/portal/api/s5"
+)
+
+func init() {
+	registry.RegisterEntry(registry.APIEntry{
+		Key:    "s5",
+		Module: s5.Module,
+	})
+}

api/s5/errors.go

@@ -0,0 +1,122 @@
+package s5
+
+import (
+	"fmt"
+	"net/http"
+)
+
+// S5-specific error keys
+const (
+	// File-related errors
+	ErrKeyFileUploadFailed     = "ErrFileUploadFailed"
+	ErrKeyFileDownloadFailed   = "ErrFileDownloadFailed"
+	ErrKeyMetadataFetchFailed  = "ErrMetadataFetchFailed"
+	ErrKeyInvalidFileFormat    = "ErrInvalidFileFormat"
+	ErrKeyUnsupportedFileType  = "ErrUnsupportedFileType"
+	ErrKeyFileProcessingFailed = "ErrFileProcessingFailed"
+
+	// Storage and data handling errors
+	ErrKeyStorageOperationFailed = "ErrStorageOperationFailed"
+	ErrKeyResourceNotFound       = "ErrResourceNotFound"
+	ErrKeyResourceLimitExceeded  = "ErrResourceLimitExceeded"
+	ErrKeyDataIntegrityError     = "ErrDataIntegrityError"
+
+	// User and permission errors
+	ErrKeyPermissionDenied     = "ErrPermissionDenied"
+	ErrKeyInvalidOperation     = "ErrInvalidOperation"
+	ErrKeyAuthenticationFailed = "ErrAuthenticationFailed"
+	ErrKeyAuthorizationFailed  = "ErrAuthorizationFailed"
+
+	// Network and communication errors
+	ErrKeyNetworkError       = "ErrNetworkError"
+	ErrKeyServiceUnavailable = "ErrServiceUnavailable"
+
+	// General errors
+	ErrKeyInternalError      = "ErrInternalError"
+	ErrKeyConfigurationError = "ErrConfigurationError"
+	ErrKeyOperationTimeout   = "ErrOperationTimeout"
+)
+
+// Default error messages for S5-specific errors
+var defaultErrorMessages = map[string]string{
+	ErrKeyFileUploadFailed:       "File upload failed due to an internal error.",
+	ErrKeyFileDownloadFailed:     "File download failed.",
+	ErrKeyMetadataFetchFailed:    "Failed to fetch metadata for the resource.",
+	ErrKeyInvalidFileFormat:      "Invalid file format provided.",
+	ErrKeyUnsupportedFileType:    "Unsupported file type.",
+	ErrKeyFileProcessingFailed:   "Failed to process the file.",
+	ErrKeyStorageOperationFailed: "Storage operation failed unexpectedly.",
+	ErrKeyResourceNotFound:       "The specified resource was not found.",
+	ErrKeyResourceLimitExceeded:  "The operation exceeded the resource limit.",
+	ErrKeyDataIntegrityError:     "Data integrity check failed.",
+	ErrKeyPermissionDenied:       "Permission denied for the requested operation.",
+	ErrKeyInvalidOperation:       "Invalid or unsupported operation requested.",
+	ErrKeyAuthenticationFailed:   "Authentication failed.",
+	ErrKeyAuthorizationFailed:    "Authorization failed or insufficient permissions.",
+	ErrKeyNetworkError:           "Network error or connectivity issue.",
+	ErrKeyServiceUnavailable:     "The requested service is temporarily unavailable.",
+	ErrKeyInternalError:          "An internal server error occurred.",
+	ErrKeyConfigurationError:     "Configuration error or misconfiguration detected.",
+	ErrKeyOperationTimeout:       "The operation timed out.",
+}
+
+// Mapping of S5-specific error keys to HTTP status codes
+var errorCodeToHttpStatus = map[string]int{
+	ErrKeyFileUploadFailed:       http.StatusInternalServerError,
+	ErrKeyFileDownloadFailed:     http.StatusInternalServerError,
+	ErrKeyMetadataFetchFailed:    http.StatusInternalServerError,
+	ErrKeyInvalidFileFormat:      http.StatusBadRequest,
+	ErrKeyUnsupportedFileType:    http.StatusBadRequest,
+	ErrKeyFileProcessingFailed:   http.StatusInternalServerError,
+	ErrKeyStorageOperationFailed: http.StatusInternalServerError,
+	ErrKeyResourceNotFound:       http.StatusNotFound,
+	ErrKeyResourceLimitExceeded:  http.StatusForbidden,
+	ErrKeyDataIntegrityError:     http.StatusInternalServerError,
+	ErrKeyPermissionDenied:       http.StatusForbidden,
+	ErrKeyInvalidOperation:       http.StatusBadRequest,
+	ErrKeyAuthenticationFailed:   http.StatusUnauthorized,
+	ErrKeyAuthorizationFailed:    http.StatusUnauthorized,
+	ErrKeyNetworkError:           http.StatusBadGateway,
+	ErrKeyServiceUnavailable:     http.StatusServiceUnavailable,
+	ErrKeyInternalError:          http.StatusInternalServerError,
+	ErrKeyConfigurationError:     http.StatusInternalServerError,
+	ErrKeyOperationTimeout:       http.StatusRequestTimeout,
+}
+
+// S5Error struct for representing S5-specific errors
+type S5Error struct {
+	Key     string
+	Message string
+	Err     error
+}
+
+// Error method to implement the error interface
+func (e *S5Error) Error() string {
+	if e.Err != nil {
+		return fmt.Sprintf("%s: %v", e.Message, e.Err)
+	}
+	return e.Message
+}
+
+func (e *S5Error) HttpStatus() int {
+	if code, exists := errorCodeToHttpStatus[e.Key]; exists {
+		return code
+	}
+	return http.StatusInternalServerError
+}
+
+func NewS5Error(key string, err error, customMessage ...string) *S5Error {
+	message, exists := defaultErrorMessages[key]
+	if !exists {
+		message = "An unknown error occurred"
+	}
+	if len(customMessage) > 0 {
+		message = customMessage[0]
+	}
+
+	return &S5Error{
+		Key:     key,
+		Message: message,
+		Err:     err,
+	}
+}

api/s5/file.go

@@ -0,0 +1,547 @@
+package s5
+
+import (
+	"context"
+	"encoding/hex"
+	"errors"
+	"io"
+	"io/fs"
+	"path"
+	"slices"
+	"sort"
+	"strings"
+	"time"
+
+	s5libmetadata "git.lumeweb.com/LumeWeb/libs5-go/metadata"
+
+	"git.lumeweb.com/LumeWeb/portal/protocols/s5"
+
+	"git.lumeweb.com/LumeWeb/portal/metadata"
+
+	"git.lumeweb.com/LumeWeb/portal/storage"
+
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+)
+
+var _ io.ReadSeekCloser = (*S5File)(nil)
+var _ fs.File = (*S5File)(nil)
+var _ fs.ReadDirFile = (*S5File)(nil)
+var _ fs.DirEntry = (*S5File)(nil)
+var _ fs.FileInfo = (*S5FileInfo)(nil)
+
+type S5File struct {
+	reader   io.ReadCloser
+	hash     []byte
+	storage  storage.StorageService
+	metadata metadata.MetadataService
+	record   *metadata.UploadMetadata
+	protocol *s5.S5Protocol
+	cid      *encoding.CID
+	typ      types.CIDType
+	read     bool
+	tus      *s5.TusHandler
+	ctx      context.Context
+	name     string
+	root     []byte
+	rootType types.CIDType
+	rootCid  *encoding.CID
+}
+
+func (f *S5File) IsDir() bool {
+	return f.typ == types.CIDTypeDirectory
+}
+
+func (f *S5File) Type() fs.FileMode {
+	if f.typ == types.CIDTypeDirectory {
+		return fs.ModeDir
+	}
+
+	return 0
+}
+
+func (f *S5File) Info() (fs.FileInfo, error) {
+	return f.Stat()
+}
+
+type FileParams struct {
+	Storage  storage.StorageService
+	Metadata metadata.MetadataService
+	Hash     []byte
+	Type     types.CIDType
+	Protocol *s5.S5Protocol
+	Tus      *s5.TusHandler
+	Name     string
+	Root     []byte
+	RootType types.CIDType
+}
+
+func NewFile(params FileParams) *S5File {
+	return &S5File{
+		storage:  params.Storage,
+		metadata: params.Metadata,
+		hash:     params.Hash,
+		typ:      params.Type,
+		protocol: params.Protocol,
+		tus:      params.Tus,
+		ctx:      context.Background(),
+		name:     params.Name,
+		root:     params.Root,
+		rootType: params.RootType,
+	}
+}
+
+func (f *S5File) Exists() bool {
+	ctx := context.Background()
+	exists, _ := f.tus.UploadExists(ctx, f.hash)
+
+	if exists {
+		return true
+	}
+
+	_, err := f.metadata.GetUpload(context.Background(), f.hash)
+
+	if err != nil {
+		return false
+	}
+
+	return true
+}
+
+func (f *S5File) Read(p []byte) (n int, err error) {
+	err = f.init(0)
+	if err != nil {
+		return 0, err
+	}
+	f.read = true
+
+	return f.reader.Read(p)
+}
+
+func (f *S5File) Seek(offset int64, whence int) (int64, error) {
+	switch whence {
+	case io.SeekStart:
+		if !f.read && offset == 0 {
+			return 0, nil
+		}
+
+		if f.reader != nil {
+			err := f.reader.Close()
+			if err != nil {
+				return 0, err
+			}
+			f.reader = nil
+		}
+		err := f.init(offset)
+		if err != nil {
+			return 0, err
+		}
+	case io.SeekCurrent:
+		return 0, errors.New("not supported")
+	case io.SeekEnd:
+		return int64(f.Size()), nil
+	default:
+		return 0, errors.New("invalid whence")
+	}
+
+	return 0, nil
+}
+
+func (f *S5File) Close() error {
+	if f.reader != nil {
+		r := f.reader
+		f.reader = nil
+		return r.Close()
+	}
+
+	return nil
+}
+
+func (f *S5File) init(offset int64) error {
+	if f.reader == nil {
+		reader, err := f.tus.GetUploadReader(f.ctx, f.hash, offset)
+
+		if err == nil {
+			f.reader = reader
+			f.read = false
+			return nil
+		}
+
+		reader, err = f.storage.DownloadObject(context.Background(), f.StorageProtocol(), f.hash, offset)
+		if err != nil {
+			return err
+		}
+
+		f.reader = reader
+		f.read = false
+	}
+
+	return nil
+}
+
+func (f *S5File) Record() (*metadata.UploadMetadata, error) {
+	if f.record == nil {
+		exists, tusRecord := f.tus.UploadExists(context.Background(), f.hash)
+
+		if exists {
+			size, err := f.tus.GetUploadSize(context.Background(), f.hash)
+			if err != nil {
+				return nil, err
+			}
+			return &metadata.UploadMetadata{
+				Hash:       f.hash,
+				Size:       uint64(size),
+				MimeType:   tusRecord.MimeType,
+				Created:    tusRecord.CreatedAt,
+				Protocol:   f.protocol.Name(),
+				UploaderIP: tusRecord.UploaderIP,
+				UserID:     tusRecord.UploaderID,
+			}, nil
+		}
+
+		record, err := f.metadata.GetUpload(context.Background(), f.hash)
+
+		if err != nil {
+			return nil, errors.New("file does not exist")
+		}
+
+		f.record = &record
+	}
+
+	return f.record, nil
+}
+
+func (f *S5File) Hash() []byte {
+	hashStr := f.HashString()
+
+	if hashStr == "" {
+		return nil
+	}
+
+	str, err := hex.DecodeString(hashStr)
+	if err != nil {
+		return nil
+	}
+
+	return str
+}
+
+func (f *S5File) HashString() string {
+	record, err := f.Record()
+	if err != nil {
+		return ""
+	}
+
+	return hex.EncodeToString(record.Hash)
+}
+
+func (f *S5File) Name() string {
+	if f.name != "" {
+		return f.name
+	}
+
+	cid, _ := f.CID().ToString()
+
+	return cid
+}
+
+func (f *S5File) Modtime() time.Time {
+	record, err := f.Record()
+	if err != nil {
+		return time.Unix(0, 0)
+	}
+
+	return record.Created
+}
+func (f *S5File) Size() uint64 {
+	record, err := f.Record()
+	if err != nil {
+		return 0
+	}
+
+	return record.Size
+}
+func (f *S5File) CID() *encoding.CID {
+	if f.cid == nil {
+		multihash := encoding.MultihashFromBytes(f.Hash(), types.HashTypeBlake3)
+
+		typ := f.typ
+		if typ == 0 {
+			typ = types.CIDTypeRaw
+		}
+
+		cid := encoding.NewCID(typ, *multihash, f.Size())
+		f.cid = cid
+	}
+	return f.cid
+}
+
+func (f *S5File) RootCID() *encoding.CID {
+	if f.rootCid == nil {
+		if f.root == nil {
+			return nil
+		}
+		multihash := encoding.MultihashFromBytes(f.root, types.HashTypeBlake3)
+		typ := f.rootType
+		if typ == 0 {
+			typ = types.CIDTypeRaw
+		}
+
+		cid := encoding.NewCID(typ, *multihash, f.Size())
+		f.rootCid = cid
+
+	}
+
+	return f.rootCid
+}
+
+func (f *S5File) Mime() string {
+	record, err := f.Record()
+	if err != nil {
+		return ""
+	}
+
+	return record.MimeType
+}
+
+func (f *S5File) StorageProtocol() storage.StorageProtocol {
+	return s5.GetStorageProtocol(f.protocol)
+}
+
+func (f *S5File) Proof() ([]byte, error) {
+	object, err := f.storage.DownloadObjectProof(context.Background(), f.StorageProtocol(), f.hash)
+
+	if err != nil {
+		return nil, err
+	}
+
+	proof, err := io.ReadAll(object)
+	if err != nil {
+		return nil, err
+	}
+
+	err = object.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	return proof, nil
+}
+func (f *S5File) Manifest() (s5libmetadata.Metadata, error) {
+	cid := f.RootCID()
+
+	if cid == nil {
+		cid = f.CID()
+	}
+
+	if f.Exists() {
+		data, err := io.ReadAll(f)
+		if err != nil {
+			return nil, err
+		}
+
+		_, err = f.Seek(0, io.SeekStart)
+		if err != nil {
+			return nil, err
+		}
+
+		md, err := f.protocol.Node().Services().Storage().ParseMetadata(data, cid)
+		if err != nil {
+			return nil, err
+		}
+
+		return md, nil
+	}
+
+	meta, err := f.protocol.Node().Services().Storage().GetMetadataByCID(cid)
+	if err != nil {
+		return nil, err
+	}
+
+	return meta, nil
+}
+
+func (f *S5File) Stat() (fs.FileInfo, error) {
+	return newS5FileInfo(f), nil
+}
+
+type S5FileInfo struct {
+	file *S5File
+}
+
+func (s S5FileInfo) Name() string {
+	return s.file.Name()
+}
+
+func (s S5FileInfo) Size() int64 {
+	return int64(s.file.Size())
+}
+
+func (s S5FileInfo) Mode() fs.FileMode {
+	return 0
+}
+
+func (s S5FileInfo) ModTime() time.Time {
+	return s.file.Modtime()
+}
+
+func (s S5FileInfo) IsDir() bool {
+	if s.file.name == "." {
+		return true
+	}
+
+	manifest, err := s.file.Manifest()
+	if err == nil && s.file.root != nil {
+		webApp, ok := manifest.(*s5libmetadata.WebAppMetadata)
+		if ok {
+			if slices.Contains(webApp.TryFiles, path.Base(s.file.name)) {
+				return true
+			}
+		}
+	}
+
+	return s.file.typ == types.CIDTypeDirectory
+}
+
+func (s S5FileInfo) Sys() any {
+	return nil
+}
+
+func (f *S5File) ReadDir(n int) ([]fs.DirEntry, error) {
+	manifest, err := f.Manifest()
+	if err != nil {
+		return nil, err
+	}
+
+	switch f.CID().Type {
+	case types.CIDTypeDirectory:
+		dir, ok := manifest.(*s5libmetadata.DirectoryMetadata)
+		if !ok {
+			return nil, errors.New("manifest is not a directory")
+		}
+
+		var entries []fs.DirEntry
+
+		for _, file := range dir.Files.Items() {
+			entries = append(entries, NewFile(FileParams{
+				Storage:  f.storage,
+				Metadata: f.metadata,
+				Hash:     file.File.CID().Hash.HashBytes(),
+				Type:     file.File.CID().Type,
+				Tus:      f.tus,
+				Name:     file.Name,
+			}))
+		}
+
+		for _, subDir := range dir.Directories.Items() {
+			cid, err := resolveDirCid(subDir, f.protocol.Node())
+			if err != nil {
+				return nil, err
+			}
+			entries = append(entries, NewFile(FileParams{
+				Storage:  f.storage,
+				Metadata: f.metadata,
+				Hash:     cid.Hash.HashBytes(),
+				Type:     cid.Type,
+				Name:     subDir.Name,
+			}))
+		}
+
+		return entries, nil
+
+	case types.CIDTypeMetadataWebapp:
+		webApp, ok := manifest.(*s5libmetadata.WebAppMetadata)
+		if !ok {
+			return nil, errors.New("manifest is not a web app")
+		}
+
+		var entries []fs.DirEntry
+		dirMap := make(map[string]bool)
+
+		webApp.Paths.Keys()
+
+		for _, path := range webApp.Paths.Keys() {
+			pathSegments := strings.Split(path, "/")
+
+			// Check if the path is an immediate child (either a file or a direct subdirectory)
+			if len(pathSegments) == 1 {
+				// It's a file directly within `dirPath`
+				entries = append(entries, newWebAppEntry(pathSegments[0], false))
+			} else if len(pathSegments) > 1 {
+				// It's a subdirectory, but ensure to add each unique subdirectory only once
+				subDirName := pathSegments[0] // The immediate subdirectory name
+				if _, exists := dirMap[subDirName]; !exists {
+					entries = append(entries, newWebAppEntry(subDirName, true))
+					dirMap[subDirName] = true
+				}
+			}
+		}
+
+		sort.Slice(entries, func(i, j int) bool {
+			return entries[i].Name() < entries[j].Name()
+		})
+
+		return entries, nil
+	}
+
+	return nil, errors.New("unsupported CID type")
+}
+
+func newS5FileInfo(file *S5File) *S5FileInfo {
+	return &S5FileInfo{
+		file: file,
+	}
+}
+
+type webAppEntry struct {
+	name  string
+	isDir bool
+}
+
+func newWebAppEntry(name string, isDir bool) *webAppEntry {
+	return &webAppEntry{name: name, isDir: isDir}
+}
+
+func (d *webAppEntry) Name() string {
+	return d.name
+}
+
+func (d *webAppEntry) IsDir() bool {
+	return d.isDir
+}
+
+func (d *webAppEntry) Type() fs.FileMode {
+	if d.isDir {
+		return fs.ModeDir
+	}
+
+	return 0
+}
+
+func (d *webAppEntry) Info() (fs.FileInfo, error) {
+	return &webAppFileInfo{name: d.name, isDir: true}, nil
+}
+
+type webAppFileInfo struct {
+	name  string
+	isDir bool
+}
+
+func (fi *webAppFileInfo) Name() string { return fi.name }
+func (fi *webAppFileInfo) Size() int64  { return 0 }
+func (fi *webAppFileInfo) Mode() fs.FileMode {
+	if fi.isDir {
+		return fs.ModeDir
+	}
+
+	return 0
+}
+func (fi *webAppFileInfo) ModTime() time.Time {
+	return time.Time{}
+}
+func (fi *webAppFileInfo) IsDir() bool {
+	return fi.isDir
+}
+func (fi *webAppFileInfo) Sys() interface{} {
+	return nil
+}

api/s5/fs_dir.go

@@ -0,0 +1,132 @@
+package s5
+
+import (
+	"errors"
+	"io/fs"
+	"strings"
+
+	"git.lumeweb.com/LumeWeb/libs5-go/node"
+
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/metadata"
+)
+
+var _ fs.FS = (*dirFs)(nil)
+
+type dirFs struct {
+	root *encoding.CID
+	s5   *S5API
+}
+
+func (w *dirFs) Open(name string) (fs.File, error) {
+	file := w.s5.newFile(FileParams{
+		Hash: w.root.Hash.HashBytes(),
+		Type: w.root.Type,
+	})
+
+	manifest, err := file.Manifest()
+	if err != nil {
+		return nil, err
+	}
+
+	dir, ok := manifest.(*metadata.DirectoryMetadata)
+
+	if !ok {
+		return nil, errors.New("manifest is not a directory")
+	}
+
+	segments := strings.Split(name, "/")
+
+	if len(segments) == 1 {
+		return w.openDirectly(name, dir)
+	}
+
+	nextDirName := segments[0]
+	remainingPath := strings.Join(segments[1:], "/")
+
+	return w.openNestedDir(nextDirName, remainingPath, dir)
+}
+
+func (w *dirFs) openDirectly(name string, dir *metadata.DirectoryMetadata) (fs.File, error) {
+	file := dir.Files.Get(name)
+	subDir := dir.Directories.Get(name)
+
+	if file != nil {
+		return w.s5.newFile(FileParams{
+			Hash: file.File.CID().Hash.HashBytes(),
+			Type: file.File.CID().Type,
+			Name: file.Name,
+		}), nil
+	}
+
+	if subDir != nil {
+		cid, err := w.resolveDirCid(subDir)
+		if err != nil {
+			return nil, err
+		}
+
+		return w.s5.newFile(FileParams{
+			Hash: cid.Hash.HashBytes(),
+			Type: cid.Type,
+			Name: name,
+		}), nil
+	}
+
+	if name == "." {
+		return w.s5.newFile(FileParams{
+			Hash: w.root.Hash.HashBytes(),
+			Type: w.root.Type,
+			Name: name,
+		}), nil
+
+	}
+
+	return nil, fs.ErrNotExist
+}
+
+func (w dirFs) openNestedDir(name string, remainingPath string, dir *metadata.DirectoryMetadata) (fs.File, error) {
+	subDir := dir.Directories.Get(name)
+
+	if subDir == nil {
+		return nil, fs.ErrNotExist
+	}
+
+	cid, err := w.resolveDirCid(subDir)
+	if err != nil {
+		return nil, err
+	}
+
+	nestedFs := newDirFs(cid, w.s5)
+
+	return nestedFs.Open(remainingPath)
+
+}
+
+func (w *dirFs) resolveDirCid(dir *metadata.DirectoryReference) (*encoding.CID, error) {
+	return resolveDirCid(dir, w.s5.getNode())
+}
+
+func newDirFs(root *encoding.CID, s5 *S5API) *dirFs {
+	return &dirFs{
+		root: root,
+		s5:   s5,
+	}
+}
+func resolveDirCid(dir *metadata.DirectoryReference, node *node.Node) (*encoding.CID, error) {
+	if len(dir.PublicKey) == 0 {
+		return nil, errors.New("missing public key")
+	}
+
+	entry, err := node.Services().Registry().Get(dir.PublicKey)
+	if err != nil {
+		return nil, err
+	}
+
+	cid, err := encoding.CIDFromRegistry(entry.Data())
+
+	if err != nil {
+		return nil, err
+	}
+
+	return cid, nil
+}

api/s5/fs_webapp.go

@@ -0,0 +1,73 @@
+package s5
+
+import (
+	"errors"
+	"io/fs"
+	"path"
+
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/metadata"
+)
+
+var _ fs.FS = (*webAppFs)(nil)
+
+type webAppFs struct {
+	root *encoding.CID
+	s5   *S5API
+}
+
+func (w webAppFs) Open(name string) (fs.File, error) {
+	file := w.s5.newFile(FileParams{
+		Hash: w.root.Hash.HashBytes(),
+		Type: w.root.Type,
+	})
+
+	manifest, err := file.Manifest()
+	if err != nil {
+		return nil, err
+	}
+
+	webApp, ok := manifest.(*metadata.WebAppMetadata)
+
+	if !ok {
+		return nil, errors.New("manifest is not a web app")
+	}
+
+	if name == "." {
+		return w.s5.newFile(FileParams{
+			Hash: w.root.Hash.HashBytes(),
+			Type: w.root.Type,
+			Name: name,
+		}), nil
+	}
+
+	item, ok := webApp.Paths.Get(name)
+
+	if !ok {
+		name = path.Join(name, "index.html")
+		item, ok = webApp.Paths.Get(name)
+		if !ok {
+			return nil, fs.ErrNotExist
+		}
+
+		return w.s5.newFile(FileParams{
+			Hash:     item.Cid.Hash.HashBytes(),
+			Type:     item.Cid.Type,
+			Name:     name,
+			Root:     w.root.Hash.HashBytes(),
+			RootType: w.root.Type,
+		}), nil
+	}
+	return w.s5.newFile(FileParams{
+		Hash: item.Cid.Hash.HashBytes(),
+		Type: item.Cid.Type,
+		Name: name,
+	}), nil
+}
+
+func newWebAppFs(root *encoding.CID, s5 *S5API) *webAppFs {
+	return &webAppFs{
+		root: root,
+		s5:   s5,
+	}
+}

api/s5/messages.go

@@ -0,0 +1,140 @@
+package s5
+
+import (
+	"time"
+
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"git.lumeweb.com/LumeWeb/portal/db/models"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+var (
+	_ msgpack.CustomEncoder = (*AccountPinBinaryResponse)(nil)
+)
+
+type AccountRegisterRequest struct {
+	Pubkey    string `json:"pubkey"`
+	Response  string `json:"response"`
+	Signature string `json:"signature"`
+	Email     string `json:"email,omitempty"`
+}
+
+type SmallUploadResponse struct {
+	CID string `json:"cid"`
+}
+type AccountRegisterChallengeResponse struct {
+	Challenge string `json:"challenge"`
+}
+
+type AccountLoginRequest struct {
+	Pubkey    string `json:"pubkey"`
+	Response  string `json:"response"`
+	Signature string `json:"signature"`
+}
+type AccountLoginChallengeResponse struct {
+	Challenge string `json:"challenge"`
+}
+type AccountInfoResponse struct {
+	Email          string      `json:"email"`
+	QuotaExceeded  bool        `json:"quotaExceeded"`
+	EmailConfirmed bool        `json:"emailConfirmed"`
+	IsRestricted   bool        `json:"isRestricted"`
+	Tier           AccountTier `json:"tier"`
+}
+
+type AccountStatsResponse struct {
+	AccountInfoResponse
+	Stats AccountStats `json:"stats"`
+}
+
+type AccountTier struct {
+	Id              uint64        `json:"id"`
+	Name            string        `json:"name"`
+	UploadBandwidth uint64        `json:"uploadBandwidth"`
+	StorageLimit    uint64        `json:"storageLimit"`
+	Scopes          []interface{} `json:"scopes"`
+}
+
+type AccountStats struct {
+	Total AccountStatsTotal `json:"total"`
+}
+
+type AccountStatsTotal struct {
+	UsedStorage uint64 `json:"usedStorage"`
+}
+type AppUploadResponse struct {
+	CID string `json:"cid"`
+}
+type RegistryQueryResponse struct {
+	Pk        string `json:"pk"`
+	Revision  uint64 `json:"revision"`
+	Data      string `json:"data"`
+	Signature string `json:"signature"`
+}
+
+type RegistrySetRequest struct {
+	Pk        string `json:"pk"`
+	Revision  uint64 `json:"revision"`
+	Data      string `json:"data"`
+	Signature string `json:"signature"`
+}
+
+type DebugStorageLocation struct {
+	Type   int      `json:"type"`
+	Parts  []string `json:"parts"`
+	Expiry int64    `json:"expiry"`
+	NodeId string   `json:"nodeId"`
+	Score  float64  `json:"score"`
+}
+
+type DebugStorageLocationsResponse struct {
+	Locations []DebugStorageLocation `json:"locations"`
+}
+
+type AccountPinBinaryResponse struct {
+	Pins   []models.Pin
+	Cursor uint64
+}
+
+func (a AccountPinBinaryResponse) EncodeMsgpack(enc *msgpack.Encoder) error {
+	err := enc.EncodeInt(0)
+	if err != nil {
+		return err
+	}
+
+	err = enc.EncodeInt(int64(a.Cursor))
+	if err != nil {
+		return err
+	}
+
+	err = enc.EncodeArrayLen(len(a.Pins))
+	if err != nil {
+		return err
+	}
+
+	for _, pin := range a.Pins {
+		err = enc.EncodeBytes(encoding.MultihashFromBytes(pin.Upload.Hash, types.HashTypeBlake3).FullBytes())
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+type AccountPinResponse struct {
+	Pins []AccountPin `json:"pins"`
+}
+
+type AccountPin struct {
+	Hash     string    `json:"hash"`
+	Size     uint64    `json:"size"`
+	PinnedAt time.Time `json:"pinned_at"`
+	MimeType string    `json:"mime_type"`
+}
+
+type AccountPinStatusResponse struct {
+	Status   models.ImportStatus `json:"status"`
+	Progress float64             `json:"progress"`
+}

api/s5/middleware.go

@@ -0,0 +1,20 @@
+package s5
+
+import (
+	"git.lumeweb.com/LumeWeb/portal/api/middleware"
+	"net/http"
+)
+
+const (
+	authCookieName = "s5-auth-token"
+	authQueryParam = "auth_token"
+)
+
+func findToken(r *http.Request) string {
+	return middleware.FindAuthToken(r, authCookieName, authQueryParam)
+}
+
+func authMiddleware(options middleware.AuthMiddlewareOptions) middleware.HttpMiddlewareFunc {
+	options.FindToken = findToken
+	return middleware.AuthMiddleware(options)
+}

api/swagger/generate.go

@@ -0,0 +1,111 @@
+// This program downloads the dist assets for the current swagger-ui version and places them into the embed directory
+// TODO: Compress?
+
+//go:build ignore
+// +build ignore
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+type releaseResp []struct {
+	// TagName is a release tag name
+	TagName string `json:"tag_name"`
+}
+
+func main() {
+	log.SetFlags(0)
+	releases := releaseResp{}
+	// get the releases so we can download the latest one
+	req, _ := http.NewRequest("GET", "https://api.github.com/repos/swagger-api/swagger-ui/releases", nil)
+	req.Header.Set("Accept", "application/vnd.github.v3+json")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		log.Fatalf("error getting release list: %v", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		log.Fatalf("got status [%s] on release list download", resp.Status)
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil {
+		log.Fatalf("error decoding response: %v", err)
+	}
+	resp.Body.Close()
+	if len(releases) == 0 {
+		log.Fatal("somehow got no releases, nothing to do")
+	}
+	tag := releases[0].TagName
+
+	log.Printf("downloading release %s...", tag)
+
+	resp, err = http.Get(fmt.Sprintf("https://github.com/swagger-api/swagger-ui/archive/%s.tar.gz", tag))
+	if err != nil {
+		log.Fatalf("error downloading release archive: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		log.Fatalf("got status [%s] on release archive download", resp.Status)
+	}
+	zr, err := gzip.NewReader(resp.Body)
+	if err != nil {
+		log.Fatalf("error opening file as gzip: %v", err)
+	}
+	if err := os.RemoveAll("embed"); err != nil {
+		log.Fatalf("error removing old embed directory")
+	}
+	if err := os.Mkdir("embed", 0o700); err != nil {
+		log.Fatalf("error recreating embed directory")
+	}
+	tr := tar.NewReader(zr)
+	for {
+		header, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Fatalf("tar parsing error: %v", err)
+		}
+		if header.Typeflag == tar.TypeReg {
+			// got a file, remove version directory
+			fname := header.Name[strings.Index(header.Name, `/`):]
+			if strings.HasPrefix(fname, `/dist`) {
+				fname = strings.TrimPrefix(fname, `/dist`)
+				out, err := os.Create(filepath.Join("embed", fname))
+				if err != nil {
+					log.Fatalf("error create output file: %v", err)
+				}
+				if _, err := io.Copy(out, tr); err != nil {
+					log.Fatalf("error writing output file: %v", err)
+				}
+			}
+		}
+	}
+	// replace the hard-coded JSON file with a generic file and disable the topbar
+	initFile, err := os.ReadFile(filepath.Join("embed", "swagger-initializer.js"))
+	if err != nil {
+		log.Fatalf("error opening swagger-initializer.js for templating :%v", err)
+	}
+	newInit := regexp.MustCompile(`url:\s+"[^"]*"`).ReplaceAllLiteral(initFile, []byte(`url: "/swagger.json"`))
+	newInit = regexp.MustCompile(`,?\s+SwaggerUIStandalonePreset.*\n`).ReplaceAllLiteral(newInit, []byte("\n"))
+	newInit = regexp.MustCompile(`(?s),\s+plugins: \[.*],\n`).ReplaceAllLiteral(newInit, []byte("\n"))
+	newInit = regexp.MustCompile(`\n\s*layout:.*\n`).ReplaceAllLiteral(newInit, []byte("\n"))
+	newinitFile, err := os.Create(filepath.Join("embed", "swagger-initializer.js"))
+	if err != nil {
+		log.Fatalf("error re-creating swagger-initializer.js file: %v", err)
+	}
+	defer newinitFile.Close()
+	if _, err := newinitFile.Write(newInit); err != nil {
+		log.Fatalf("unable to write to swagger-initializer.js: %v", err)
+	}
+}

api/swagger/swagger.go

@@ -0,0 +1,65 @@
+package swagger
+
+import (
+	"embed"
+	"io/fs"
+	"net/http"
+
+	"git.lumeweb.com/LumeWeb/portal/api/middleware"
+
+	"go.sia.tech/jape"
+
+	"github.com/getkin/kin-openapi/openapi3"
+)
+
+//go:generate go run generate.go
+
+//go:embed embed
+var swagfs embed.FS
+
+func byteHandler(b []byte) jape.Handler {
+	return func(c jape.Context) {
+		c.ResponseWriter.Header().Set("Content-Type", "application/json")
+		c.ResponseWriter.Write(b)
+	}
+}
+
+func Swagger(spec []byte, routes map[string]jape.Handler) (map[string]jape.Handler, error) {
+	loader := openapi3.NewLoader()
+	doc, err := loader.LoadFromData(spec)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = doc.Validate(loader.Context); err != nil {
+		return nil, err
+	}
+
+	jsonDoc, err := doc.MarshalJSON()
+
+	if err != nil {
+		return nil, err
+	}
+
+	swaggerFiles, _ := fs.Sub(swagfs, "embed")
+	swaggerServ := http.FileServer(http.FS(swaggerFiles))
+	handler := func(c jape.Context) {
+		swaggerServ.ServeHTTP(c.ResponseWriter, c.Request)
+	}
+
+	strip := func(next http.Handler) http.Handler {
+		return http.StripPrefix("/swagger", next)
+	}
+
+	redirect := func(jc jape.Context) {
+		http.Redirect(jc.ResponseWriter, jc.Request, "/swagger/", http.StatusMovedPermanently)
+	}
+
+	swagRoutes := map[string]jape.Handler{
+		"GET /swagger.json":  byteHandler(jsonDoc),
+		"GET /swagger":       redirect,
+		"GET /swagger/*path": middleware.ApplyMiddlewares(handler, strip),
+	}
+
+	return middleware.MergeRoutes(routes, swagRoutes), nil
+}

bao/bao.go

@@ -1,31 +1,161 @@
 package bao
 
 import (
-	"bufio"
+	"bytes"
 	_ "embed"
+	"errors"
 	"io"
-	"lukechampine.com/blake3"
+	"time"
+
+	"github.com/samber/lo"
+
+	"go.uber.org/zap"
+
+	"lukechampine.com/blake3/bao"
 )
 
-func ComputeTree(reader io.Reader, size int64) ([]byte, [32]byte, error) {
-	bufSize := blake3.BaoEncodedSize(int(size), true)
-	buf := bufferAt{buf: make([]byte, bufSize)}
+var _ io.ReadCloser = (*Verifier)(nil)
+var _ io.WriterAt = (*proofWriter)(nil)
 
-	hash, err := blake3.BaoEncode(&buf, bufio.NewReader(reader), size, true)
+var ErrVerifyFailed = errors.New("verification failed")
+
+const groupLog = 8
+const groupChunks = 1 << groupLog
+
+type Verifier struct {
+	r          io.ReadCloser
+	proof      Result
+	read       uint64
+	buffer     *bytes.Buffer
+	logger     *zap.Logger
+	readTime   []time.Duration
+	verifyTime time.Duration
+}
+
+type Result struct {
+	Hash   []byte
+	Proof  []byte
+	Length uint
+}
+
+func (v *Verifier) Read(p []byte) (int, error) {
+	// Initial attempt to read from the buffer
+	n, err := v.buffer.Read(p)
+	if n == len(p) {
+		// If the buffer already had enough data to fulfill the request, return immediately
+		return n, nil
+	} else if err != nil && err != io.EOF {
+		// For errors other than EOF, return the error immediately
+		return n, err
+	}
+
+	buf := make([]byte, groupChunks)
+	// Continue reading from the source and verifying until we have enough data or hit an error
+	for v.buffer.Len() < len(p)-n {
+		readStart := time.Now()
+		bytesRead, err := io.ReadFull(v.r, buf)
+		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
+			return n, err // Return any read error immediately
+		}
+
+		readEnd := time.Now()
+
+		v.readTime = append(v.readTime, readEnd.Sub(readStart))
+
+		timeStart := time.Now()
+
+		if bytesRead > 0 {
+			if status := bao.VerifyChunk(buf[:bytesRead], v.proof.Proof, groupChunks, v.read, [32]byte(v.proof.Hash)); !status {
+				return n, errors.Join(ErrVerifyFailed, err)
+			}
+			v.read += uint64(bytesRead)
+			v.buffer.Write(buf[:bytesRead]) // Append new data to the buffer
+		}
+
+		timeEnd := time.Now()
+		v.verifyTime += timeEnd.Sub(timeStart)
+
+		if err == io.EOF {
+			// If EOF, break the loop as no more data can be read
+			break
+		}
+	}
+
+	if len(v.readTime) > 0 {
+		averageReadTime := lo.Reduce(v.readTime, func(acc time.Duration, cur time.Duration, _ int) time.Duration {
+			return acc + cur
+		}, time.Duration(0)) / time.Duration(len(v.readTime))
+
+		v.logger.Debug("Read time", zap.Duration("average", averageReadTime))
+	}
+
+	averageVerifyTime := v.verifyTime / time.Duration(v.read/groupChunks)
+	v.logger.Debug("Verification time", zap.Duration("average", averageVerifyTime))
+
+	// Attempt to read the remainder of the data from the buffer
+	additionalBytes, _ := v.buffer.Read(p[n:])
+	return n + additionalBytes, nil
+}
+
+func (v *Verifier) Close() error {
+	return v.r.Close()
+}
+func Hash(r io.Reader, size uint64) (*Result, error) {
+	reader := newSizeReader(r)
+	writer := newProofWriter(int(size))
+
+	hash, err := bao.Encode(writer, reader, int64(size), groupLog, true)
 	if err != nil {
-		return nil, [32]byte{}, err
+		return nil, err
 	}
 
-	return buf.buf, hash, nil
+	return &Result{
+		Hash:   hash[:],
+		Proof:  writer.buf,
+		Length: uint(size),
+	}, nil
 }
 
-type bufferAt struct {
+func NewVerifier(r io.ReadCloser, proof Result, logger *zap.Logger) *Verifier {
+	return &Verifier{
+		r:      r,
+		proof:  proof,
+		buffer: new(bytes.Buffer),
+		logger: logger,
+	}
+}
+
+type proofWriter struct {
 	buf []byte
 }
 
-func (b *bufferAt) WriteAt(p []byte, off int64) (int, error) {
-	if copy(b.buf[off:], p) != len(p) {
+func (p proofWriter) WriteAt(b []byte, off int64) (n int, err error) {
+	if copy(p.buf[off:], b) != len(b) {
 		panic("bad buffer size")
 	}
-	return len(p), nil
+	return len(b), nil
+}
+
+func newProofWriter(size int) *proofWriter {
+	return &proofWriter{
+		buf: make([]byte, bao.EncodedSize(size, groupLog, true)),
+	}
+}
+
+type sizeReader struct {
+	reader io.Reader
+	read   int64
+}
+
+func (s sizeReader) Read(p []byte) (int, error) {
+	n, err := s.reader.Read(p)
+	s.read += int64(n)
+	return n, err
+}
+
+func newSizeReader(r io.Reader) *sizeReader {
+	return &sizeReader{
+		reader: r,
+		read:   0,
+	}
 }

cid/cid.go

@@ -1,100 +0,0 @@
-package cid
-
-import (
-	"bytes"
-	"encoding/binary"
-	"encoding/hex"
-	"errors"
-	"github.com/multiformats/go-multibase"
-)
-
-var MAGIC_BYTES = []byte{0x26, 0x1f}
-
-var (
-	ErrMissingEmptySize = errors.New("Missing or empty size")
-	ErrInvalidCIDMagic  = errors.New("CID magic bytes missing or invalid")
-)
-
-type CID struct {
-	Hash [32]byte
-	Size uint64
-}
-
-func (c CID) StringHash() string {
-	return hex.EncodeToString(c.Hash[:])
-}
-
-func Encode(hash []byte, size uint64) (string, error) {
-	var hashBytes [32]byte
-	copy(hashBytes[:], hash)
-
-	return EncodeFixed(hashBytes, size)
-}
-
-func EncodeFixed(hash [32]byte, size uint64) (string, error) {
-	sizeBytes := make([]byte, 8)
-	binary.LittleEndian.PutUint64(sizeBytes, size)
-
-	prefixedHash := append(MAGIC_BYTES, hash[:]...)
-	prefixedHash = append(prefixedHash, sizeBytes...)
-
-	return multibase.Encode(multibase.Base58BTC, prefixedHash)
-}
-
-func EncodeString(hash string, size uint64) (string, error) {
-	hashBytes, err := hex.DecodeString(hash)
-	if err != nil {
-		return "", err
-	}
-
-	return Encode(hashBytes, size)
-}
-
-func Valid(cid string) (bool, error) {
-	_, err := maybeDecode(cid)
-	if err != nil {
-		return false, err
-	}
-
-	return true, nil
-
-}
-
-func Decode(cid string) (*CID, error) {
-	data, err := maybeDecode(cid)
-	if err != nil {
-		return &CID{}, err
-	}
-
-	data = data[len(MAGIC_BYTES):]
-	var hash [32]byte
-	copy(hash[:], data[:])
-	size := binary.LittleEndian.Uint64(data[32:])
-
-	return &CID{Hash: hash, Size: size}, nil
-}
-
-func maybeDecode(cid string) ([]byte, error) {
-	_, data, err := multibase.Decode(cid)
-	if err != nil {
-		return nil, err
-	}
-
-	if bytes.Compare(data[0:len(MAGIC_BYTES)], MAGIC_BYTES) != 0 {
-		return nil, ErrInvalidCIDMagic
-	}
-
-	sizeBytes := data[len(MAGIC_BYTES)+32:]
-
-	if len(sizeBytes) == 0 {
-		return nil, ErrMissingEmptySize
-	}
-
-	size := binary.LittleEndian.Uint64(sizeBytes)
-
-	if size == 0 {
-		return nil, ErrMissingEmptySize
-	}
-
-	return data, nil
-}

cmd/portal/init.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"context"
+	"crypto/ed25519"
+	"net"
+	"net/http"
+	"strconv"
+
+	"git.lumeweb.com/LumeWeb/portal/api/router"
+
+	"git.lumeweb.com/LumeWeb/portal/config"
+
+	"git.lumeweb.com/LumeWeb/portal/api/registry"
+	"go.sia.tech/core/wallet"
+	"go.uber.org/fx"
+	"go.uber.org/zap"
+)
+
+func NewIdentity(config *config.Manager, logger *zap.Logger) (ed25519.PrivateKey, error) {
+	var seed [32]byte
+	identitySeed := config.Config().Core.Identity
+
+	if identitySeed == "" {
+		logger.Info("Generating new identity seed")
+		identitySeed = wallet.NewSeedPhrase()
+		config.Viper().Set("core.identity", identitySeed)
+		err := config.Save()
+		if err != nil {
+			return nil, err
+		}
+	}
+	err := wallet.SeedFromPhrase(&seed, identitySeed)
+	if err != nil {
+		return nil, err
+	}
+
+	return ed25519.PrivateKey(wallet.KeyFromSeed(&seed, 0)), nil
+}
+
+type NewServerParams struct {
+	fx.In
+	Config *config.Manager
+	Logger *zap.Logger
+	APIs   []registry.API `group:"api"`
+}
+
+func NewServer(lc fx.Lifecycle, params NewServerParams) (*http.Server, error) {
+	r := registry.GetRouter()
+
+	r.SetConfig(params.Config)
+	r.SetLogger(params.Logger)
+
+	for _, api := range params.APIs {
+		routableAPI, ok := interface{}(api).(router.RoutableAPI)
+
+		if !ok {
+			params.Logger.Fatal("API does not implement RoutableAPI", zap.String("api", api.Name()))
+		}
+
+		r.RegisterAPI(routableAPI)
+	}
+
+	srv := &http.Server{
+		Addr:    ":" + strconv.FormatUint(uint64(params.Config.Config().Core.Port), 10),
+		Handler: r,
+	}
+
+	lc.Append(fx.Hook{
+		OnStart: func(ctx context.Context) error {
+			ln, err := net.Listen("tcp", srv.Addr)
+			if err != nil {
+				return err
+			}
+
+			go func() {
+				err := srv.Serve(ln)
+				if err != nil {
+					params.Logger.Fatal("Failed to serve", zap.Error(err))
+				}
+			}()
+
+			return nil
+		},
+		OnStop: func(ctx context.Context) error {
+			return srv.Shutdown(ctx)
+		},
+	})
+	return srv, nil
+}

cmd/portal/main.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"flag"
+	"net/http"
+
+	_import "git.lumeweb.com/LumeWeb/portal/import"
+
+	"git.lumeweb.com/LumeWeb/portal/mailer"
+
+	"git.lumeweb.com/LumeWeb/portal/config"
+
+	"git.lumeweb.com/LumeWeb/portal/account"
+	"git.lumeweb.com/LumeWeb/portal/api"
+	"git.lumeweb.com/LumeWeb/portal/cron"
+	"git.lumeweb.com/LumeWeb/portal/db"
+	_logger "git.lumeweb.com/LumeWeb/portal/logger"
+	"git.lumeweb.com/LumeWeb/portal/metadata"
+	"git.lumeweb.com/LumeWeb/portal/protocols"
+	"git.lumeweb.com/LumeWeb/portal/renter"
+	"git.lumeweb.com/LumeWeb/portal/storage"
+	"go.uber.org/fx"
+	"go.uber.org/fx/fxevent"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+)
+
+func main() {
+	cfg, err := config.NewManager()
+
+	logger, logLevel := _logger.NewLogger(cfg)
+
+	if err != nil {
+		logger.Fatal("Failed to load config", zap.Error(err))
+	}
+
+	var fxDebug bool
+
+	flag.BoolVar(&fxDebug, "fx-debug", false, "Enable fx framework debug logging")
+	flag.Parse()
+
+	var fxLogger fx.Option
+
+	fxLogger = fx.WithLogger(func(logger *zap.Logger) fxevent.Logger {
+		log := &fxevent.ZapLogger{Logger: logger}
+		log.UseLogLevel(zapcore.InfoLevel)
+		log.UseErrorLevel(zapcore.ErrorLevel)
+		return log
+	})
+
+	if fxDebug {
+		fxLogger = fx.Options()
+	}
+
+	fx.New(
+		fx.Supply(cfg),
+		fx.Supply(logger, logLevel),
+		fxLogger,
+		fx.Provide(NewIdentity),
+		db.Module,
+		renter.Module,
+		storage.Module,
+		cron.Module,
+		account.Module,
+		metadata.Module,
+		_import.Module,
+		mailer.Module,
+		protocols.BuildProtocols(cfg),
+		api.BuildApis(cfg),
+		fx.Provide(api.NewCasbin),
+		fx.Invoke(protocols.SetupLifecycles),
+		fx.Invoke(api.SetupLifecycles),
+		fx.Provide(NewServer),
+		fx.Invoke(func(*http.Server) {}),
+	).Run()
+}

config/cluster.go

@@ -0,0 +1,56 @@
+package config
+
+import (
+	"reflect"
+
+	"github.com/mitchellh/mapstructure"
+)
+
+type ClusterConfig struct {
+	Enabled bool         `mapstructure:"enabled"`
+	Redis   *RedisConfig `mapstructure:"redis"`
+	Etcd    *EtcdConfig  `mapstructure:"etcd"`
+}
+
+func clusterConfigHook() mapstructure.DecodeHookFuncType {
+	return func(f reflect.Type, t reflect.Type, data interface{}) (interface{}, error) {
+		if f.Kind() != reflect.Map || t != reflect.TypeOf(&ClusterConfig{}) {
+			return data, nil
+		}
+
+		var clusterConfig ClusterConfig
+		if err := mapstructure.Decode(data, &clusterConfig); err != nil {
+			return nil, err
+		}
+
+		// Check if the input data map includes "redis" configuration
+		if opts, ok := data.(map[string]interface{})["redis"].(map[string]interface{}); ok && opts != nil {
+			var redisOptions RedisConfig
+			if err := mapstructure.Decode(opts, &redisOptions); err != nil {
+				return nil, err
+			}
+
+			if err := redisOptions.Validate(); err != nil {
+				return nil, err
+			}
+
+			clusterConfig.Redis = &redisOptions
+		}
+
+		// Check if the input data map includes "etcd" configuration
+		if opts, ok := data.(map[string]interface{})["etcd"].(map[string]interface{}); ok && opts != nil {
+			var etcdOptions EtcdConfig
+			if err := mapstructure.Decode(opts, &etcdOptions); err != nil {
+				return nil, err
+			}
+
+			if err := etcdOptions.Validate(); err != nil {
+				return nil, err
+			}
+
+			clusterConfig.Etcd = &etcdOptions
+		}
+
+		return &clusterConfig, nil
+	}
+}

config/config.go

@@ -3,9 +3,10 @@ package config
 import (
 	"errors"
 	"fmt"
-	"github.com/spf13/pflag"
+	"reflect"
+
 	"github.com/spf13/viper"
-	"log"
+	"go.uber.org/zap"
 )
 
 var (
@@ -16,9 +17,252 @@ var (
 	}
 )
 
-func Init() {
+type Defaults interface {
+	Defaults() map[string]interface{}
+}
+
+type Validator interface {
+	Validate() error
+}
+
+type Config struct {
+	Core     CoreConfig             `mapstructure:"core"`
+	Protocol map[string]interface{} `mapstructure:"protocol"`
+}
+
+type Manager struct {
+	viper   *viper.Viper
+	root    *Config
+	changes bool
+}
+
+func NewManager() (*Manager, error) {
+	v, err := newConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	var config Config
+
+	m := &Manager{
+		viper: v,
+		root:  &config,
+	}
+
+	m.setDefaultsForObject(m.root.Core, "core")
+	err = m.maybeSave()
+	if err != nil {
+		return nil, err
+	}
+
+	err = v.Unmarshal(&config, viper.DecodeHook(clusterConfigHook()), viper.DecodeHook(cacheConfigHook()))
+	if err != nil {
+		return nil, err
+	}
+
+	err = m.validateObject(m.root)
+	if err != nil {
+		return nil, err
+	}
+
+	err = m.maybeConfigureCluster()
+	if err != nil {
+		return m, err
+	}
+
+	return m, nil
+}
+
+func (m *Manager) ConfigureProtocol(name string, cfg ProtocolConfig) error {
+	protocolPrefix := fmt.Sprintf("protocol.%s", name)
+
+	m.setDefaultsForObject(cfg, protocolPrefix)
+	err := m.maybeSave()
+	if err != nil {
+		return err
+	}
+
+	err = m.viper.Sub(protocolPrefix).Unmarshal(cfg)
+	if err != nil {
+		return err
+	}
+
+	err = m.validateObject(cfg)
+	if err != nil {
+		return err
+	}
+
+	if m.root.Protocol == nil {
+		m.root.Protocol = make(map[string]interface{})
+	}
+
+	m.root.Protocol[name] = cfg
+
+	return nil
+}
+
+func (m *Manager) setDefaultsForObject(obj interface{}, prefix string) {
+	// Reflect on the object to traverse its fields
+	objValue := reflect.ValueOf(obj)
+	objType := reflect.TypeOf(obj)
+
+	// If the object is a pointer, we need to work with its element
+	if objValue.Kind() == reflect.Ptr {
+		objValue = objValue.Elem()
+		objType = objType.Elem()
+	}
+
+	// Check if the object itself implements Defaults
+	if setter, ok := obj.(Defaults); ok {
+		m.applyDefaults(setter, prefix)
+	}
+
+	// Recursively handle struct fields
+	for i := 0; i < objValue.NumField(); i++ {
+		field := objValue.Field(i)
+		fieldType := objType.Field(i)
+
+		// Check if the field is exported and can be interfaced
+		if !field.CanInterface() {
+			continue
+		}
+
+		mapstructureTag := fieldType.Tag.Get("mapstructure")
+
+		// Construct new prefix based on the mapstructure tag, if available
+		newPrefix := prefix
+		if mapstructureTag != "" && mapstructureTag != "-" {
+			if newPrefix != "" {
+				newPrefix += "."
+			}
+			newPrefix += mapstructureTag
+		}
+
+		// If field is a struct or pointer to a struct, recurse
+		if field.Kind() == reflect.Struct || (field.Kind() == reflect.Ptr && field.Elem().Kind() == reflect.Struct) {
+			if field.Kind() == reflect.Ptr && field.IsNil() {
+				// Initialize nil pointer to struct
+				field.Set(reflect.New(fieldType.Type.Elem()))
+			}
+			m.setDefaultsForObject(field.Interface(), newPrefix)
+		}
+	}
+}
+
+func (m *Manager) validateObject(obj interface{}) error {
+	// Reflect on the object to traverse its fields
+	objValue := reflect.ValueOf(obj)
+	objType := reflect.TypeOf(obj)
+
+	// If the object is a pointer, we need to work with its element
+	if objValue.Kind() == reflect.Ptr {
+		objValue = objValue.Elem()
+		objType = objType.Elem()
+	}
+
+	// Check if the object itself implements Defaults
+	if validator, ok := obj.(Validator); ok {
+		err := validator.Validate()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Recursively handle struct fields
+	for i := 0; i < objValue.NumField(); i++ {
+		field := objValue.Field(i)
+		fieldType := objType.Field(i)
+
+		if !field.CanInterface() {
+			continue
+		}
+
+		// If field is a struct or pointer to a struct, recurse
+		if field.Kind() == reflect.Struct || (field.Kind() == reflect.Ptr && field.Elem().Kind() == reflect.Struct) {
+			if field.Kind() == reflect.Ptr && field.IsNil() {
+				// Initialize nil pointer to struct
+				field.Set(reflect.New(fieldType.Type.Elem()))
+			}
+			err := m.validateObject(field.Interface())
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (m *Manager) applyDefaults(setter Defaults, prefix string) {
+	defaults := setter.Defaults()
+	for key, value := range defaults {
+		fullKey := key
+		if prefix != "" {
+			fullKey = fmt.Sprintf("%s.%s", prefix, key)
+		}
+		if m.setDefault(fullKey, value) {
+			m.changes = true
+		}
+	}
+}
+
+func (m *Manager) setDefault(key string, value interface{}) bool {
+	if !m.viper.IsSet(key) {
+		m.viper.SetDefault(key, value)
+		return true
+	}
+
+	return false
+}
+
+func (m *Manager) maybeSave() error {
+	if m.changes {
+		ret := m.viper.WriteConfig()
+		if ret != nil {
+			return ret
+		}
+		m.changes = false
+	}
+
+	return nil
+}
+
+func (m *Manager) maybeConfigureCluster() error {
+	if m.root.Core.Clustered != nil && m.root.Core.Clustered.Enabled {
+		m.root.Core.DB.Cache.Mode = "redis"
+		m.root.Core.DB.Cache.Options = m.root.Core.Clustered.Redis
+	}
+
+	return nil
+}
+
+func (m *Manager) Config() *Config {
+	return m.root
+}
+
+func (m *Manager) Viper() *viper.Viper {
+	return m.viper
+}
+
+func (m *Manager) Save() error {
+	err := m.viper.WriteConfig()
+	if err != nil {
+		return err
+	}
+
+	err = m.viper.Unmarshal(&m.root)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func newConfig() (*viper.Viper, error) {
+	logger := newFallbackLogger()
+
 	viper.SetConfigName("config")
-	viper.SetConfigType("json")
+	viper.SetConfigType("yaml")
 
 	for _, path := range ConfigFilePaths {
 		viper.AddConfigPath(path)
@@ -27,33 +271,26 @@ func Init() {
 	viper.SetEnvPrefix("LUME_WEB_PORTAL")
 	viper.AutomaticEnv()
 
-	pflag.String("database.type", "sqlite", "Database type")
-	pflag.String("database.host", "localhost", "Database host")
-	pflag.Int("database.port", 3306, "Database port")
-	pflag.String("database.user", "root", "Database user")
-	pflag.String("database.password", "", "Database password")
-	pflag.String("database.name", "lumeweb_portal", "Database name")
-	pflag.String("database.path", "./db.sqlite", "Database path for SQLite")
-	pflag.String("renterd-api-password", ".", "admin password for renterd")
-	pflag.Bool("debug", false, "enable debug mode")
-	pflag.Parse()
-
-	err := viper.BindPFlags(pflag.CommandLine)
-
+	err := viper.ReadInConfig()
 	if err != nil {
-		log.Fatalf("Fatal error arguments: %s \n", err)
-		return
+		if !errors.As(err, &viper.ConfigFileNotFoundError{}) {
+			return nil, err
 		}
 
-	err = viper.ReadInConfig()
+		logger.Info("Config file not found, using default settings.")
+		err := viper.SafeWriteConfig()
 		if err != nil {
-		if errors.As(err, &viper.ConfigFileNotFoundError{}) {
-			// Config file not found, this is not an error.
-			fmt.Println("Config file not found, using default settings.")
-		} else {
-			// Other error, panic.
-			panic(fmt.Errorf("Fatal error config file: %s \n", err))
-		}
+			return nil, err
 		}
 
+		return viper.GetViper(), nil
+
+	}
+
+	return viper.GetViper(), nil
+}
+func newFallbackLogger() *zap.Logger {
+	l, _ := zap.NewDevelopment()
+
+	return l
 }

config/core.go

@@ -0,0 +1,45 @@
+package config
+
+import (
+	"errors"
+
+	"github.com/docker/go-units"
+)
+
+var _ Defaults = (*CoreConfig)(nil)
+var _ Validator = (*CoreConfig)(nil)
+
+type CoreConfig struct {
+	DB              DatabaseConfig `mapstructure:"db"`
+	Domain          string         `mapstructure:"domain"`
+	PortalName      string         `mapstructure:"portal_name"`
+	ExternalPort    uint           `mapstructure:"external_port"`
+	Identity        string         `mapstructure:"identity"`
+	Log             LogConfig      `mapstructure:"log"`
+	Port            uint           `mapstructure:"port"`
+	PostUploadLimit uint64         `mapstructure:"post_upload_limit"`
+	Storage         StorageConfig  `mapstructure:"storage"`
+	Protocols       []string       `mapstructure:"protocols"`
+	Mail            MailConfig     `mapstructure:"mail"`
+	Clustered       *ClusterConfig `mapstructure:"clustered"`
+}
+
+func (c CoreConfig) Validate() error {
+	if c.Domain == "" {
+		return errors.New("core.domain is required")
+	}
+	if c.PortalName == "" {
+		return errors.New("core.portal_name is required")
+	}
+	if c.Port == 0 {
+		return errors.New("core.port is required")
+	}
+
+	return nil
+}
+
+func (c CoreConfig) Defaults() map[string]interface{} {
+	return map[string]interface{}{
+		"post_upload_limit": units.MiB * 100,
+	}
+}

config/database.go

@@ -0,0 +1,95 @@
+package config
+
+import (
+	"errors"
+	"reflect"
+
+	"github.com/mitchellh/mapstructure"
+)
+
+var _ Defaults = (*DatabaseConfig)(nil)
+var _ Validator = (*DatabaseConfig)(nil)
+
+type DatabaseConfig struct {
+	Charset  string       `mapstructure:"charset"`
+	Host     string       `mapstructure:"host"`
+	Name     string       `mapstructure:"name"`
+	Password string       `mapstructure:"password"`
+	Port     int          `mapstructure:"port"`
+	Username string       `mapstructure:"username"`
+	Cache    *CacheConfig `mapstructure:"cache"`
+}
+
+func (d DatabaseConfig) Validate() error {
+	if d.Host == "" {
+		return errors.New("core.db.host is required")
+	}
+	if d.Port == 0 {
+		return errors.New("core.db.port is required")
+	}
+	if d.Username == "" {
+		return errors.New("core.db.username is required")
+	}
+	if d.Password == "" {
+		return errors.New("core.db.password is required")
+	}
+	if d.Name == "" {
+		return errors.New("core.db.name is required")
+	}
+
+	return nil
+}
+
+func (d DatabaseConfig) Defaults() map[string]interface{} {
+	return map[string]interface{}{
+		"host":    "localhost",
+		"charset": "utf8mb4",
+		"port":    3306,
+		"name":    "portal",
+	}
+}
+
+type CacheConfig struct {
+	Mode    string      `mapstructure:"mode"`
+	Options interface{} `mapstructure:"options"`
+}
+
+type MemoryConfig struct {
+}
+
+func cacheConfigHook() mapstructure.DecodeHookFuncType {
+	return func(f reflect.Type, t reflect.Type, data interface{}) (interface{}, error) {
+		// This hook is designed to operate on the options field within the CacheConfig
+		if f.Kind() != reflect.Map || t != reflect.TypeOf(&CacheConfig{}) {
+			return data, nil
+		}
+
+		var cacheConfig CacheConfig
+		if err := mapstructure.Decode(data, &cacheConfig); err != nil {
+			return nil, err
+		}
+
+		// Assuming the input data map includes "mode" and "options"
+		switch cacheConfig.Mode {
+		case "redis":
+			var redisOptions RedisConfig
+			if opts, ok := cacheConfig.Options.(map[string]interface{}); ok && opts != nil {
+				if err := mapstructure.Decode(opts, &redisOptions); err != nil {
+					return nil, err
+				}
+				cacheConfig.Options = redisOptions
+			}
+		case "memory":
+			// For "memory", you might simply use an empty MemoryConfig,
+			// or decode options similarly if there are any specific to memory caching.
+			cacheConfig.Options = MemoryConfig{}
+		case "false":
+			// If "false", ensure no options are set, or set to a nil or similar neutral value.
+			cacheConfig.Options = nil
+		default:
+			cacheConfig.Options = nil
+		}
+
+		return cacheConfig, nil
+	}
+}

config/etcd.go

@@ -0,0 +1,23 @@
+package config
+
+import "errors"
+
+var _ Defaults = (*EtcdConfig)(nil)
+
+type EtcdConfig struct {
+	Endpoints   []string `mapstructure:"endpoints"`
+	DialTimeout int      `mapstructure:"dial_timeout"`
+}
+
+func (r *EtcdConfig) Validate() error {
+	if len(r.Endpoints) == 0 {
+		return errors.New("endpoints is required")
+	}
+	return nil
+}
+
+func (r *EtcdConfig) Defaults() map[string]interface{} {
+	return map[string]interface{}{
+		"dial_timeout": 5,
+	}
+}

config/log.go

@@ -0,0 +1,13 @@
+package config
+
+var _ Defaults = (*LogConfig)(nil)
+
+type LogConfig struct {
+	Level string `mapstructure:"level"`
+}
+
+func (l LogConfig) Defaults() map[string]interface{} {
+	return map[string]interface{}{
+		"level": "info",
+	}
+}

config/mail.go

@@ -0,0 +1,39 @@
+package config
+
+import (
+	"errors"
+)
+
+var _ Validator = (*MailConfig)(nil)
+var _ Defaults = (*MailConfig)(nil)
+
+type MailConfig struct {
+	Host     string `mapstructure:"host"`
+	Port     int    `mapstructure:"port"`
+	SSL      bool   `mapstructure:"ssl"`
+	AuthType string `mapstructure:"auth_type"`
+	Username string `mapstructure:"username"`
+	Password string `mapstructure:"password"`
+	From     string `mapstructure:"from"`
+}
+
+func (m MailConfig) Validate() error {
+	if m.Host == "" {
+		return errors.New("core.mail.host is required")
+	}
+	if m.Username == "" {
+		return errors.New("core.mail.username is required")
+	}
+	if m.Password == "" {
+		return errors.New("core.mail.password is required")
+	}
+	if m.From == "" {
+		return errors.New("core.mail.from is required")
+	}
+	return nil
+}
+func (c MailConfig) Defaults() map[string]interface{} {
+	return map[string]interface{}{
+		"auth_type": "plain",
+	}
+}

config/protocol.go

@@ -0,0 +1,5 @@
+package config
+
+type ProtocolConfig interface {
+	Defaults
+}

config/redis.go

@@ -0,0 +1,26 @@
+package config
+
+import "errors"
+
+var _ Validator = (*RedisConfig)(nil)
+var _ Defaults = (*RedisConfig)(nil)
+
+type RedisConfig struct {
+	Address  string `mapstructure:"address"`
+	Password string `mapstructure:"password"`
+	DB       int    `mapstructure:"db"`
+}
+
+func (r *RedisConfig) Defaults() map[string]interface{} {
+	return map[string]interface{}{
+		"address": "localhost:6379",
+		"db":      0,
+	}
+}
+
+func (r *RedisConfig) Validate() error {
+	if r.Address == "" {
+		return errors.New("address is required")
+	}
+	return nil
+}

config/s3.go

@@ -0,0 +1,32 @@
+package config
+
+import "errors"
+
+var _ Validator = (*DatabaseConfig)(nil)
+
+type S3Config struct {
+	BufferBucket string `mapstructure:"buffer_bucket"`
+	Endpoint     string `mapstructure:"endpoint"`
+	Region       string `mapstructure:"region"`
+	AccessKey    string `mapstructure:"access_key"`
+	SecretKey    string `mapstructure:"secret_key"`
+}
+
+func (s S3Config) Validate() error {
+	if s.BufferBucket == "" {
+		return errors.New("core.storage.s3.buffer_bucket is required")
+	}
+	if s.Endpoint == "" {
+		return errors.New("core.storage.s3.endpoint is required")
+	}
+	if s.Region == "" {
+		return errors.New("core.storage.s3.region is required")
+	}
+	if s.AccessKey == "" {
+		return errors.New("core.storage.s3.access_key is required")
+	}
+	if s.SecretKey == "" {
+		return errors.New("core.storage.s3.secret_key is required")
+	}
+	return nil
+}

config/sia.go

@@ -0,0 +1,75 @@
+package config
+
+import (
+	"errors"
+	"math/big"
+)
+
+var _ Validator = (*SiaConfig)(nil)
+var _ Defaults = (*SiaConfig)(nil)
+
+type SiaConfig struct {
+	Key                string `mapstructure:"key"`
+	URL                string `mapstructure:"url"`
+	PriceHistoryDays   uint64 `mapstructure:"price_history_days"`
+	MaxUploadPrice     string `mapstructure:"max_upload_price"`
+	MaxDownloadPrice   string `mapstructure:"max_download_price"`
+	MaxStoragePrice    string `mapstructure:"max_storage_price"`
+	MaxContractSCPrice string `mapstructure:"max_contract_sc_price"`
+	MaxRPCSCPrice      string `mapstructure:"max_rpc_sc_price"`
+}
+
+func (s SiaConfig) Defaults() map[string]interface{} {
+	return map[string]interface{}{
+		"max_rpc_sc_price":      0.1,
+		"max_contract_sc_price": 1,
+		"price_history_days":    90,
+	}
+}
+
+func (s SiaConfig) Validate() error {
+	if s.Key == "" {
+		return errors.New("core.storage.sia.key is required")
+	}
+	if s.URL == "" {
+		return errors.New("core.storage.sia.url is required")
+	}
+
+	if err := validateStringNumber(s.MaxUploadPrice, "core.storage.sia.max_upload_price"); err != nil {
+		return err
+	}
+
+	if err := validateStringNumber(s.MaxDownloadPrice, "core.storage.sia.max_download_price"); err != nil {
+		return err
+	}
+
+	if err := validateStringNumber(s.MaxStoragePrice, "core.storage.sia.max_storage_price"); err != nil {
+		return err
+	}
+
+	if err := validateStringNumber(s.MaxContractSCPrice, "core.storage.sia.max_contract_sc_price"); err != nil {
+		return err
+	}
+
+	if err := validateStringNumber(s.MaxRPCSCPrice, "core.storage.sia.max_rpc_sc_price"); err != nil {
+		return err
+	}
+	return nil
+}
+
+func validateStringNumber(s string, name string) error {
+	if s == "" {
+		return errors.New(name + " is required")
+	}
+
+	rat, ok := new(big.Rat).SetString(s)
+	if !ok {
+		return errors.New("failed to parse " + name)
+	}
+
+	if rat.Cmp(new(big.Rat).SetUint64(0)) <= 0 {
+		return errors.New(name + " must be greater than 0")
+	}
+
+	return nil
+}

config/storage.go

@@ -0,0 +1,6 @@
+package config
+
+type StorageConfig struct {
+	S3  S3Config  `mapstructure:"s3"`
+	Sia SiaConfig `mapstructure:"sia"`
+}

controller/account.go

@@ -1,35 +0,0 @@
-package controller
-
-import (
-	"git.lumeweb.com/LumeWeb/portal/controller/request"
-	"git.lumeweb.com/LumeWeb/portal/service/account"
-	"github.com/kataras/iris/v12"
-)
-
-type AccountController struct {
-	Controller
-}
-
-func (a *AccountController) PostRegister() {
-	ri, success := tryParseRequest(request.RegisterRequest{}, a.Ctx)
-	if !success {
-		return
-	}
-
-	r, _ := ri.(*request.RegisterRequest)
-
-	err := account.Register(r.Email, r.Password, r.Pubkey)
-
-	if err != nil {
-		if err == account.ErrQueryingAcct || err == account.ErrFailedCreateAccount {
-			a.Ctx.StopWithError(iris.StatusInternalServerError, err)
-		} else {
-			a.Ctx.StopWithError(iris.StatusBadRequest, err)
-		}
-
-		return
-	}
-
-	// Return a success response to the client.
-	a.Ctx.StatusCode(iris.StatusCreated)
-}

controller/auth.go

@@ -1,112 +0,0 @@
-package controller
-
-import (
-	"git.lumeweb.com/LumeWeb/portal/controller/request"
-	"git.lumeweb.com/LumeWeb/portal/controller/response"
-	"git.lumeweb.com/LumeWeb/portal/middleware"
-	"git.lumeweb.com/LumeWeb/portal/service/auth"
-	"github.com/kataras/iris/v12"
-)
-
-type AuthController struct {
-	Controller
-}
-
-// PostLogin handles the POST /api/auth/login request to authenticate a user and return a JWT token.
-func (a *AuthController) PostLogin() {
-	ri, success := tryParseRequest(request.LoginRequest{}, a.Ctx)
-	if !success {
-		return
-	}
-
-	r, _ := ri.(*request.LoginRequest)
-
-	token, err := auth.LoginWithPassword(r.Email, r.Password)
-
-	if err != nil {
-		if err == auth.ErrFailedGenerateToken {
-			a.Ctx.StopWithError(iris.StatusInternalServerError, err)
-		} else {
-			a.Ctx.StopWithError(iris.StatusUnauthorized, err)
-		}
-		return
-	}
-
-	a.respondJSON(&response.LoginResponse{Token: token})
-}
-
-// PostChallenge handles the POST /api/auth/pubkey/challenge request to generate a challenge for a user's public key.
-func (a *AuthController) PostPubkeyChallenge() {
-	ri, success := tryParseRequest(request.PubkeyChallengeRequest{}, a.Ctx)
-	if !success {
-		return
-	}
-
-	r, _ := (ri).(*request.PubkeyChallengeRequest)
-
-	challenge, err := auth.GeneratePubkeyChallenge(r.Pubkey)
-	if err != nil {
-		if err == auth.ErrFailedGenerateKeyChallenge {
-			a.Ctx.StopWithError(iris.StatusInternalServerError, err)
-		} else {
-			a.Ctx.StopWithError(iris.StatusUnauthorized, err)
-		}
-		return
-	}
-
-	a.respondJSON(&response.ChallengeResponse{Challenge: challenge})
-}
-
-// PostKeyLogin handles the POST /api/auth/pubkey/login request to authenticate a user using a public key challenge and return a JWT token.
-func (a *AuthController) PostPubkeyLogin() {
-	ri, success := tryParseRequest(request.PubkeyLoginRequest{}, a.Ctx)
-	if !success {
-		return
-	}
-
-	r, _ := ri.(*request.PubkeyLoginRequest)
-
-	token, err := auth.LoginWithPubkey(r.Pubkey, r.Challenge, r.Signature)
-
-	if err != nil {
-		if err == auth.ErrFailedGenerateKeyChallenge || err == auth.ErrFailedGenerateToken || err == auth.ErrFailedSaveToken {
-			a.Ctx.StopWithError(iris.StatusInternalServerError, err)
-		} else {
-			a.Ctx.StopWithError(iris.StatusUnauthorized, err)
-		}
-		return
-	}
-
-	a.respondJSON(&response.LoginResponse{Token: token})
-
-}
-
-// PostLogout handles the POST /api/auth/logout request to invalidate a JWT token.
-func (a *AuthController) PostLogout() {
-	ri, success := tryParseRequest(request.LogoutRequest{}, a.Ctx)
-	if !success {
-		return
-	}
-
-	r, _ := ri.(*request.LogoutRequest)
-
-	err := auth.Logout(r.Token)
-
-	if err != nil {
-		a.Ctx.StopWithError(iris.StatusBadRequest, err)
-		return
-	}
-
-	// Return a success response to the client.
-	a.Ctx.StatusCode(iris.StatusNoContent)
-}
-
-func (a *AuthController) GetStatus() {
-	middleware.VerifyJwt(a.Ctx)
-
-	if a.Ctx.IsStopped() {
-		return
-	}
-
-	a.respondJSON(&response.AuthStatusResponse{Status: true})
-}

controller/controller.go

@@ -1,86 +0,0 @@
-package controller
-
-import (
-	"git.lumeweb.com/LumeWeb/portal/controller/validators"
-	"git.lumeweb.com/LumeWeb/portal/logger"
-	"github.com/kataras/iris/v12"
-	"go.uber.org/zap"
-	"strconv"
-)
-
-func tryParseRequest(r interface{}, ctx iris.Context) (interface{}, bool) {
-	v, ok := r.(validators.Validatable)
-
-	if !ok {
-		return r, true
-	}
-
-	var d map[string]interface{}
-
-	// Read the logout request from the client.
-	if err := ctx.ReadJSON(&d); err != nil {
-		logger.Get().Debug("failed to parse request", zap.Error(err))
-		ctx.StopWithError(iris.StatusBadRequest, err)
-		return nil, false
-	}
-
-	data, err := v.Import(d)
-	if err != nil {
-		logger.Get().Debug("failed to parse request", zap.Error(err))
-		ctx.StopWithError(iris.StatusBadRequest, err)
-		return nil, false
-	}
-
-	if err := data.Validate(); err != nil {
-		logger.Get().Debug("failed to parse request", zap.Error(err))
-		ctx.StopWithError(iris.StatusBadRequest, err)
-		return nil, false
-	}
-
-	return data, true
-}
-
-func sendErrorCustom(ctx iris.Context, err error, customError error, irisError int) bool {
-	if err != nil {
-		if customError != nil {
-			err = customError
-		}
-		ctx.StopWithError(irisError, err)
-		return true
-	}
-
-	return false
-}
-func internalError(ctx iris.Context, err error) bool {
-	return sendErrorCustom(ctx, err, nil, iris.StatusInternalServerError)
-}
-func internalErrorCustom(ctx iris.Context, err error, customError error) bool {
-	return sendErrorCustom(ctx, err, customError, iris.StatusInternalServerError)
-}
-func sendError(ctx iris.Context, err error, irisError int) bool {
-	return sendErrorCustom(ctx, err, nil, irisError)
-}
-
-type Controller struct {
-	Ctx iris.Context
-}
-
-func (c Controller) respondJSON(data interface{}) {
-	err := c.Ctx.JSON(data)
-	if err != nil {
-		logger.Get().Error("failed to generate response", zap.Error(err))
-	}
-}
-
-func getCurrentUserId(ctx iris.Context) uint {
-	usr := ctx.User()
-
-	if usr == nil {
-		return 0
-	}
-
-	sid, _ := usr.GetID()
-	userID, _ := strconv.Atoi(sid)
-
-	return uint(userID)
-}

controller/files.go

@@ -1,212 +0,0 @@
-package controller
-
-import (
-	"errors"
-	"git.lumeweb.com/LumeWeb/portal/cid"
-	"git.lumeweb.com/LumeWeb/portal/controller/response"
-	"git.lumeweb.com/LumeWeb/portal/logger"
-	"git.lumeweb.com/LumeWeb/portal/middleware"
-	"git.lumeweb.com/LumeWeb/portal/service/files"
-	"github.com/kataras/iris/v12"
-	"go.uber.org/zap"
-	"io"
-)
-
-var errStreamDone = errors.New("done")
-
-type FilesController struct {
-	Controller
-}
-
-func (f *FilesController) BeginRequest(ctx iris.Context) {
-	middleware.VerifyJwt(ctx)
-}
-func (f *FilesController) EndRequest(ctx iris.Context) {
-}
-
-func (f *FilesController) PostUpload() {
-	ctx := f.Ctx
-
-	file, meta, err := f.Ctx.FormFile("file")
-	if internalErrorCustom(ctx, err, errors.New("invalid file data")) {
-		logger.Get().Debug("invalid file data", zap.Error(err))
-		return
-	}
-
-	upload, err := files.Upload(file, meta.Size, nil)
-
-	if internalError(ctx, err) {
-		logger.Get().Debug("failed uploading file", zap.Error(err))
-		return
-	}
-
-	err = files.Pin(upload.Hash, upload.AccountID)
-
-	if internalError(ctx, err) {
-		logger.Get().Debug("failed pinning file", zap.Error(err))
-		return
-	}
-
-	cidString, err := cid.EncodeString(upload.Hash, uint64(meta.Size))
-
-	if internalError(ctx, err) {
-		logger.Get().Debug("failed creating cid", zap.Error(err))
-		return
-	}
-
-	err = ctx.JSON(&response.UploadResponse{Cid: cidString})
-
-	if err != nil {
-		logger.Get().Error("failed to create response", zap.Error(err))
-	}
-}
-
-func (f *FilesController) GetDownloadBy(cidString string) {
-	ctx := f.Ctx
-
-	hashHex, valid := validateCid(cidString, true, ctx)
-
-	if !valid {
-		return
-	}
-
-	download, err := files.Download(hashHex)
-	if internalError(ctx, err) {
-		logger.Get().Debug("failed fetching file", zap.Error(err))
-		return
-	}
-
-	err = passThroughStream(download, ctx)
-	if err != errStreamDone && internalError(ctx, err) {
-		logger.Get().Debug("failed streaming file", zap.Error(err))
-	}
-}
-
-func (f *FilesController) GetProofBy(cidString string) {
-	ctx := f.Ctx
-
-	hashHex, valid := validateCid(cidString, true, ctx)
-
-	if !valid {
-		return
-	}
-
-	proof, err := files.DownloadProof(hashHex)
-	if internalError(ctx, err) {
-		logger.Get().Debug("failed fetching file proof", zap.Error(err))
-		return
-	}
-
-	err = passThroughStream(proof, ctx)
-	if internalError(ctx, err) {
-		logger.Get().Debug("failed streaming file proof", zap.Error(err))
-	}
-}
-
-func (f *FilesController) GetStatusBy(cidString string) {
-	ctx := f.Ctx
-
-	hashHex, valid := validateCid(cidString, false, ctx)
-
-	if !valid {
-		return
-	}
-
-	status := files.Status(hashHex)
-
-	var statusCode string
-
-	switch status {
-	case files.STATUS_UPLOADED:
-		statusCode = "uploaded"
-		break
-	case files.STATUS_UPLOADING:
-		statusCode = "uploading"
-		break
-	case files.STATUS_NOT_FOUND:
-		statusCode = "not_found"
-		break
-	}
-
-	f.respondJSON(&response.FileStatusResponse{Status: statusCode})
-
-}
-
-func (f *FilesController) PostPinBy(cidString string) {
-	ctx := f.Ctx
-
-	hashHex, valid := validateCid(cidString, true, ctx)
-
-	if !valid {
-		return
-	}
-
-	err := files.Pin(hashHex, getCurrentUserId(ctx))
-	if internalError(ctx, err) {
-		logger.Get().Error(err.Error())
-		return
-	}
-
-	f.Ctx.StatusCode(iris.StatusCreated)
-}
-
-func (f *FilesController) GetUploadLimit() {
-	f.respondJSON(&response.UploadLimit{Limit: f.Ctx.Application().ConfigurationReadOnly().GetPostMaxMemory()})
-}
-
-func validateCid(cidString string, validateStatus bool, ctx iris.Context) (string, bool) {
-	_, err := cid.Valid(cidString)
-	if sendError(ctx, err, iris.StatusBadRequest) {
-		logger.Get().Debug("invalid cid", zap.Error(err))
-		return "", false
-	}
-
-	cidObject, _ := cid.Decode(cidString)
-	hashHex := cidObject.StringHash()
-
-	if validateStatus {
-		status := files.Status(hashHex)
-
-		if status == files.STATUS_NOT_FOUND {
-			err := errors.New("cid not found")
-			sendError(ctx, errors.New("cid not found"), iris.StatusNotFound)
-			logger.Get().Debug("cid not found", zap.Error(err))
-			return "", false
-		}
-	}
-
-	return hashHex, true
-}
-
-func passThroughStream(stream io.Reader, ctx iris.Context) error {
-	closed := false
-
-	err := ctx.StreamWriter(func(w io.Writer) error {
-		if closed {
-			return errStreamDone
-		}
-
-		count, err := io.CopyN(w, stream, 1024)
-		if count == 0 || err == io.EOF {
-			err = stream.(io.Closer).Close()
-			if err != nil {
-				logger.Get().Error("failed closing stream", zap.Error(err))
-				return err
-			}
-			closed = true
-			return nil
-		}
-
-		if err != nil {
-			return err
-		}
-
-		return nil
-	})
-
-	if err == errStreamDone {
-		err = nil
-	}
-
-	return err
-}

controller/request/login.go

@@ -1,23 +0,0 @@
-package request
-
-import (
-	"git.lumeweb.com/LumeWeb/portal/controller/validators"
-	validation "github.com/go-ozzo/ozzo-validation/v4"
-	"github.com/go-ozzo/ozzo-validation/v4/is"
-)
-
-type LoginRequest struct {
-	validatable validators.ValidatableImpl
-	Email       string `json:"email"`
-	Password    string `json:"password"`
-}
-
-func (r LoginRequest) Validate() error {
-	return validation.ValidateStruct(&r,
-		validation.Field(&r.Email, is.EmailFormat, validation.Required),
-		validation.Field(&r.Password, validation.Required),
-	)
-}
-func (r LoginRequest) Import(d map[string]interface{}) (validators.Validatable, error) {
-	return r.validatable.Import(d, r)
-}

controller/request/logout.go

@@ -1,19 +0,0 @@
-package request
-
-import (
-	"git.lumeweb.com/LumeWeb/portal/controller/validators"
-	validation "github.com/go-ozzo/ozzo-validation/v4"
-)
-
-type LogoutRequest struct {
-	validatable validators.ValidatableImpl
-	Token       string `json:"token"`
-}
-
-func (r LogoutRequest) Validate() error {
-	return validation.ValidateStruct(&r, validation.Field(&r.Token, validation.Required))
-}
-
-func (r LogoutRequest) Import(d map[string]interface{}) (validators.Validatable, error) {
-	return r.validatable.Import(d, r)
-}

controller/request/pubkey_challenge.go

@@ -1,21 +0,0 @@
-package request
-
-import (
-	"git.lumeweb.com/LumeWeb/portal/controller/validators"
-	validation "github.com/go-ozzo/ozzo-validation/v4"
-)
-
-type PubkeyChallengeRequest struct {
-	validatable validators.ValidatableImpl
-	Pubkey      string `json:"pubkey"`
-}
-
-func (r PubkeyChallengeRequest) Validate() error {
-	return validation.ValidateStruct(&r,
-		validation.Field(&r.Pubkey, validation.Required, validation.By(validators.CheckPubkeyValidator)),
-	)
-}
-
-func (r PubkeyChallengeRequest) Import(d map[string]interface{}) (validators.Validatable, error) {
-	return r.validatable.Import(d, r)
-}

controller/request/pubkey_login.go

@@ -1,25 +0,0 @@
-package request
-
-import (
-	"git.lumeweb.com/LumeWeb/portal/controller/validators"
-	validation "github.com/go-ozzo/ozzo-validation/v4"
-)
-
-type PubkeyLoginRequest struct {
-	validatable validators.ValidatableImpl
-	Pubkey      string `json:"pubkey"`
-	Challenge   string `json:"challenge"`
-	Signature   string `json:"signature"`
-}
-
-func (r PubkeyLoginRequest) Validate() error {
-	return validation.ValidateStruct(&r,
-		validation.Field(&r.Pubkey, validation.Required, validation.By(validators.CheckPubkeyValidator)),
-		validation.Field(&r.Challenge, validation.Required),
-		validation.Field(&r.Signature, validation.Required, validation.Length(128, 128)),
-	)
-}
-
-func (r PubkeyLoginRequest) Import(d map[string]interface{}) (validators.Validatable, error) {
-	return r.validatable.Import(d, r)
-}

controller/request/register.go

@@ -1,25 +0,0 @@
-package request
-
-import (
-	"git.lumeweb.com/LumeWeb/portal/controller/validators"
-	validation "github.com/go-ozzo/ozzo-validation/v4"
-	"github.com/go-ozzo/ozzo-validation/v4/is"
-)
-
-type RegisterRequest struct {
-	validatable validators.ValidatableImpl
-	Email       string `json:"email"`
-	Password    string `json:"password"`
-	Pubkey      string `json:"pubkey"`
-}
-
-func (r RegisterRequest) Validate() error {
-	return validation.ValidateStruct(&r,
-		validation.Field(&r.Email, validation.Required, is.EmailFormat),
-		validation.Field(&r.Pubkey, validation.When(len(r.Password) == 0, validation.Required, validation.By(validators.CheckPubkeyValidator))),
-		validation.Field(&r.Password, validation.When(len(r.Pubkey) == 0, validation.Required)),
-	)
-}
-func (r RegisterRequest) Import(d map[string]interface{}) (validators.Validatable, error) {
-	return r.validatable.Import(d, r)
-}

controller/response/auth_status.go

@@ -1,5 +0,0 @@
-package response
-
-type AuthStatusResponse struct {
-	Status bool `json:"status"`
-}

controller/response/challenge.go

@@ -1,5 +0,0 @@
-package response
-
-type ChallengeResponse struct {
-	Challenge string `json:"challenge"`
-}

controller/response/file_status.go

@@ -1,5 +0,0 @@
-package response
-
-type FileStatusResponse struct {
-	Status string `json:"status"`
-}

controller/response/login.go

@@ -1,5 +0,0 @@
-package response
-
-type LoginResponse struct {
-	Token string `json:"token"`
-}

controller/response/upload.go

@@ -1,5 +0,0 @@
-package response
-
-type UploadResponse struct {
-	Cid string `json:"cid"`
-}

controller/response/upload_limit.go

@@ -1,5 +0,0 @@
-package response
-
-type UploadLimit struct {
-	Limit int64 `json:"limit"`
-}

controller/validators/validators.go

@@ -1,43 +0,0 @@
-package validators
-
-import (
-	"crypto/ed25519"
-	"encoding/hex"
-	"errors"
-	"fmt"
-	validation "github.com/go-ozzo/ozzo-validation/v4"
-	"github.com/imdario/mergo"
-	"reflect"
-)
-
-func CheckPubkeyValidator(value interface{}) error {
-	p, _ := value.(string)
-	pubkeyBytes, err := hex.DecodeString(p)
-	if err != nil {
-		return err
-	}
-
-	if len(pubkeyBytes) != ed25519.PublicKeySize {
-		return errors.New(fmt.Sprintf("pubkey must be %d bytes in hexadecimal format", ed25519.PublicKeySize))
-	}
-
-	return nil
-}
-
-type Validatable interface {
-	validation.Validatable
-	Import(d map[string]interface{}) (Validatable, error)
-}
-
-type ValidatableImpl struct {
-}
-
-func (v ValidatableImpl) Import(d map[string]interface{}, destType Validatable) (Validatable, error) {
-	instance := reflect.New(reflect.TypeOf(destType)).Interface().(Validatable)
-	// Perform the import logic
-	if err := mergo.Map(instance, d, mergo.WithOverride); err != nil {
-		return nil, err
-	}
-
-	return instance, nil
-}

cron/cron.go

@@ -0,0 +1,208 @@
+package cron
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"time"
+
+	"github.com/google/uuid"
+	"go.uber.org/fx"
+	"go.uber.org/zap"
+
+	"github.com/go-co-op/gocron/v2"
+)
+
+var (
+	ErrRetryLimitReached = errors.New("Retry limit reached")
+)
+
+type CronService interface {
+	Scheduler() gocron.Scheduler
+	RegisterService(service CronableService)
+}
+
+type CronableService interface {
+	LoadInitialTasks(cron CronService) error
+}
+
+type CronServiceParams struct {
+	fx.In
+	Logger    *zap.Logger
+	Scheduler gocron.Scheduler
+}
+
+var Module = fx.Module("cron",
+	fx.Options(
+		fx.Provide(NewCronService),
+		fx.Provide(gocron.NewScheduler),
+	),
+)
+
+type CronServiceDefault struct {
+	scheduler gocron.Scheduler
+	services  []CronableService
+	logger    *zap.Logger
+}
+
+type RetryableJobParams struct {
+	Name     string
+	Tags     []string
+	Function any
+	Args     []any
+	Attempt  uint
+	Limit    uint
+	After    func(jobID uuid.UUID, jobName string)
+	Error    func(jobID uuid.UUID, jobName string, err error)
+}
+
+type CronJob struct {
+	JobId   uuid.UUID
+	Job     gocron.JobDefinition
+	Task    gocron.Task
+	Options []gocron.JobOption
+}
+
+func (c *CronServiceDefault) Scheduler() gocron.Scheduler {
+	return c.scheduler
+}
+
+func NewCronService(lc fx.Lifecycle, params CronServiceParams) *CronServiceDefault {
+	sc := &CronServiceDefault{
+		logger:    params.Logger,
+		scheduler: params.Scheduler,
+	}
+
+	lc.Append(fx.Hook{
+		OnStart: func(ctx context.Context) error {
+			return sc.start()
+		},
+	})
+
+	return sc
+}
+
+func (c *CronServiceDefault) start() error {
+	for _, service := range c.services {
+		err := service.LoadInitialTasks(c)
+		if err != nil {
+			c.logger.Fatal("Failed to load initial tasks for service", zap.Error(err))
+		}
+	}
+
+	go c.scheduler.Start()
+
+	return nil
+}
+
+func (c *CronServiceDefault) RegisterService(service CronableService) {
+	c.services = append(c.services, service)
+}
+
+func (c *CronServiceDefault) RetryableJob(params RetryableJobParams) CronJob {
+	job := gocron.OneTimeJob(gocron.OneTimeJobStartImmediately())
+
+	if params.Attempt > 0 {
+		job = gocron.OneTimeJob(gocron.OneTimeJobStartDateTime(time.Now().Add(time.Duration(params.Attempt) * time.Minute)))
+	}
+
+	task := gocron.NewTask(params.Function, params.Args...)
+
+	if params.After == nil {
+		params.After = func(jobID uuid.UUID, jobName string) {}
+	}
+
+	if params.Error == nil {
+		params.Error = func(jobID uuid.UUID, jobName string, err error) {}
+	}
+
+	listeners := gocron.WithEventListeners(gocron.AfterJobRunsWithError(func(jobID uuid.UUID, jobName string, err error) {
+		params.Error(jobID, jobName, err)
+
+		if params.Attempt >= params.Limit && params.Limit > 0 {
+			c.logger.Error("Retryable task limit reached", zap.String("jobName", jobName), zap.String("jobID", jobID.String()))
+			params.Error(jobID, jobName, ErrRetryLimitReached)
+			return
+		}
+
+		taskRetry := params
+		taskRetry.Attempt++
+
+		retryTask := c.RetryableJob(taskRetry)
+		retryTask.JobId = jobID
+
+		_, err = c.RerunJob(retryTask)
+		if err != nil {
+			c.logger.Error("Failed to create retry job", zap.Error(err))
+		}
+	}), gocron.AfterJobRuns(params.After))
+
+	name := gocron.WithName(params.Name)
+	options := []gocron.JobOption{listeners, name}
+
+	if len(params.Tags) > 0 {
+		options = append(options, gocron.WithTags(params.Tags...))
+	}
+
+	return CronJob{
+		Job:     job,
+		Task:    task,
+		Options: options,
+	}
+}
+
+func (c *CronServiceDefault) CreateJob(job CronJob) (gocron.Job, error) {
+	ret, err := c.Scheduler().NewJob(job.Job, job.Task, job.Options...)
+	if err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+func (c *CronServiceDefault) RerunJob(job CronJob) (gocron.Job, error) {
+	ret, err := c.Scheduler().Update(job.JobId, job.Job, job.Task, job.Options...)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+func (c *CronServiceDefault) GetJobsByPrefix(prefix string) []gocron.Job {
+	jobs := c.Scheduler().Jobs()
+
+	var ret []gocron.Job
+
+	for _, job := range jobs {
+		if strings.HasPrefix(job.Name(), prefix) {
+			ret = append(ret, job)
+		}
+	}
+
+	return ret
+}
+
+func (c *CronServiceDefault) GetJobByName(name string) gocron.Job {
+	jobs := c.Scheduler().Jobs()
+
+	for _, job := range jobs {
+		if job.Name() == name {
+			return job
+		}
+	}
+
+	return nil
+}
+
+func (c *CronServiceDefault) GetJobByID(id uuid.UUID) gocron.Job {
+	jobs := c.Scheduler().Jobs()
+
+	for _, job := range jobs {
+		if job.ID() == id {
+			return job
+		}
+	}
+
+	return nil
+}

db/cache_memory.go

@@ -0,0 +1,48 @@
+package db
+
+import (
+	"context"
+	"sync"
+
+	"github.com/go-gorm/caches/v4"
+)
+
+type memoryCacher struct {
+	store *sync.Map
+}
+
+func (c *memoryCacher) init() {
+	if c.store == nil {
+		c.store = &sync.Map{}
+	}
+}
+
+func (c *memoryCacher) Get(ctx context.Context, key string, q *caches.Query[any]) (*caches.Query[any], error) {
+	c.init()
+	val, ok := c.store.Load(key)
+	if !ok {
+		return nil, nil
+	}
+
+	if err := q.Unmarshal(val.([]byte)); err != nil {
+		return nil, err
+	}
+
+	return q, nil
+}
+
+func (c *memoryCacher) Store(ctx context.Context, key string, val *caches.Query[any]) error {
+	c.init()
+	res, err := val.Marshal()
+	if err != nil {
+		return err
+	}
+
+	c.store.Store(key, res)
+	return nil
+}
+
+func (c *memoryCacher) Invalidate(ctx context.Context) error {
+	c.store = &sync.Map{}
+	return nil
+}

db/cache_redis.go

@@ -0,0 +1,70 @@
+package db
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/go-gorm/caches/v4"
+	"github.com/redis/go-redis/v9"
+)
+
+type redisCacher struct {
+	rdb *redis.Client
+}
+
+func (c *redisCacher) Get(ctx context.Context, key string, q *caches.Query[any]) (*caches.Query[any], error) {
+	res, err := c.rdb.Get(ctx, key).Result()
+	if errors.Is(err, redis.Nil) {
+		return nil, nil
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	if err := q.Unmarshal([]byte(res)); err != nil {
+		return nil, err
+	}
+
+	return q, nil
+}
+
+func (c *redisCacher) Store(ctx context.Context, key string, val *caches.Query[any]) error {
+	res, err := val.Marshal()
+	if err != nil {
+		return err
+	}
+
+	c.rdb.Set(ctx, key, res, 300*time.Second) // Set proper cache time
+	return nil
+}
+
+func (c *redisCacher) Invalidate(ctx context.Context) error {
+	var (
+		cursor uint64
+		keys   []string
+	)
+	for {
+		var (
+			k   []string
+			err error
+		)
+		k, cursor, err = c.rdb.Scan(ctx, cursor, fmt.Sprintf("%s*", caches.IdentifierPrefix), 0).Result()
+		if err != nil {
+			return err
+		}
+		keys = append(keys, k...)
+		if cursor == 0 {
+			break
+		}
+	}
+
+	if len(keys) > 0 {
+		if _, err := c.rdb.Del(ctx, keys...).Result(); err != nil {
+			return err
+		}
+	}
+	return nil
+}

db/db.go

@@ -1,65 +1,116 @@
 package db
 
 import (
+	"context"
 	"fmt"
-	"git.lumeweb.com/LumeWeb/portal/model"
-	"github.com/spf13/viper"
+
+	"git.lumeweb.com/LumeWeb/portal/db/models"
+
+	"github.com/redis/go-redis/v9"
+
+	"go.uber.org/zap"
+
+	"git.lumeweb.com/LumeWeb/portal/config"
+
+	"github.com/go-gorm/caches/v4"
+	"go.uber.org/fx"
 	"gorm.io/driver/mysql"
-	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 )
 
-// Declare a global variable to hold the database connection.
-var db *gorm.DB
-
-// Init initializes the database connection based on the app's configuration settings.
-func Init() {
-	// If the database connection has already been initialized, panic.
-	if db != nil {
-		panic("DB already initialized")
+type DatabaseParams struct {
+	fx.In
+	Config      *config.Manager
+	Logger      *zap.Logger
+	LoggerLevel *zap.AtomicLevel
 }
 
-	// Retrieve database connection settings from the app's configuration using the viper library.
-	dbType := viper.GetString("database.type")
-	dbHost := viper.GetString("database.host")
-	dbPort := viper.GetInt("database.port")
-	dbSocket := viper.GetString("database.socket")
-	dbUser := viper.GetString("database.user")
-	dbPassword := viper.GetString("database.password")
-	dbName := viper.GetString("database.name")
-	dbPath := viper.GetString("database.path")
+var Module = fx.Module("db",
+	fx.Options(
+		fx.Provide(NewDatabase),
+	),
+)
 
-	var err error
-	var dsn string
-	switch dbType {
-	// Connect to a MySQL database.
-	case "mysql":
-		if dbSocket != "" {
-			dsn = fmt.Sprintf("%s:%s@unix(%s)/%s", dbUser, dbPassword, dbSocket, dbName)
-		} else {
-			dsn = fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", dbUser, dbPassword, dbHost, dbPort, dbName)
-		}
-		db, err = gorm.Open(mysql.Open(dsn), &gorm.Config{})
-	// Connect to a SQLite database.
-	case "sqlite":
-		db, err = gorm.Open(sqlite.Open(dbPath), &gorm.Config{})
-	// If the database type is unsupported, panic.
-	default:
-		panic(fmt.Errorf("Unsupported database type: %s \n", dbType))
-	}
-	// If there was an error connecting to the database, panic.
+func NewDatabase(lc fx.Lifecycle, params DatabaseParams) *gorm.DB {
+	username := params.Config.Config().Core.DB.Username
+	password := params.Config.Config().Core.DB.Password
+	host := params.Config.Config().Core.DB.Host
+	port := params.Config.Config().Core.DB.Port
+	dbname := params.Config.Config().Core.DB.Name
+	charset := params.Config.Config().Core.DB.Charset
+
+	dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=%s&parseTime=True&loc=Local", username, password, host, port, dbname, charset)
+
+	db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{
+		Logger: newLogger(params.Logger, params.LoggerLevel),
+	})
 	if err != nil {
-		panic(fmt.Errorf("Failed to connect to database: %s \n", err))
+		panic(err)
 	}
 
-	// Automatically migrate the database schema based on the model definitions.
-	err = db.Migrator().AutoMigrate(&model.Account{}, &model.Key{}, &model.KeyChallenge{}, &model.LoginSession{}, &model.Upload{}, &model.Pin{}, &model.Tus{})
+	cacher := getCacher(params.Config, params.Logger)
+	if cacher != nil {
+		cache := &caches.Caches{Conf: &caches.Config{
+			Cacher: cacher,
+		}}
+		err := db.Use(cache)
 		if err != nil {
-		panic(fmt.Errorf("Database setup failed database type: %s \n", err))
+			return nil
 		}
 	}
 
-// Get returns the database connection instance.
-func Get() *gorm.DB {
+	lc.Append(fx.Hook{
+		OnStart: func(ctx context.Context) error {
+			return db.AutoMigrate(models.GetModels()...)
+		},
+	})
+
 	return db
 }
+
+func getCacheMode(cm *config.Manager, logger *zap.Logger) string {
+
+	if cm.Config().Core.DB.Cache == nil {
+		return "none"
+	}
+
+	switch cm.Config().Core.DB.Cache.Mode {
+	case "", "none":
+		return "none"
+	case "memory":
+		return "memory"
+	case "redis":
+		return "redis"
+	default:
+		logger.Fatal("invalid cache mode", zap.String("mode", cm.Config().Core.DB.Cache.Mode))
+	}
+
+	return "none"
+}
+
+func getCacher(cm *config.Manager, logger *zap.Logger) caches.Cacher {
+	mode := getCacheMode(cm, logger)
+
+	switch mode {
+	case "none":
+		return nil
+
+	case "memory":
+		return &memoryCacher{}
+	case "redis":
+		rcfg, ok := cm.Config().Core.DB.Cache.Options.(config.RedisConfig)
+		if !ok {
+			logger.Fatal("invalid redis config")
+			return nil
+		}
+		return &redisCacher{
+			redis.NewClient(&redis.Options{
+				Addr:     rcfg.Address,
+				Password: rcfg.Password,
+				DB:       rcfg.DB,
+			}),
+		}
+	}
+
+	return nil
+}

db/logger.go

@@ -0,0 +1,82 @@
+package db
+
+import (
+	"context"
+	"errors"
+	"strconv"
+	"time"
+
+	"gorm.io/gorm"
+
+	"go.uber.org/zap"
+	dbLogger "gorm.io/gorm/logger"
+)
+
+var _ dbLogger.Interface = (*logger)(nil)
+
+var (
+	levels = map[dbLogger.LogLevel]zap.AtomicLevel{
+		dbLogger.Silent: zap.NewAtomicLevelAt(zap.InfoLevel),
+		dbLogger.Error:  zap.NewAtomicLevelAt(zap.ErrorLevel),
+		dbLogger.Warn:   zap.NewAtomicLevelAt(zap.WarnLevel),
+		dbLogger.Info:   zap.NewAtomicLevelAt(zap.InfoLevel),
+	}
+)
+
+type logger struct {
+	logger *zap.Logger
+	level  *zap.AtomicLevel
+}
+
+func (l logger) LogMode(level dbLogger.LogLevel) dbLogger.Interface {
+	if atomicLevel, ok := levels[level]; ok {
+		l.level.SetLevel(atomicLevel.Level())
+		return l
+	}
+
+	l.logger.Fatal("invalid log level", zap.Int("level", int(level)))
+	return nil
+}
+
+func (l logger) Info(ctx context.Context, s string, i ...interface{}) {
+	l.logger.Info(s, interfacesToFields(i...)...)
+}
+
+func (l logger) Warn(ctx context.Context, s string, i ...interface{}) {
+	l.logger.Warn(s, interfacesToFields(i...)...)
+}
+
+func (l logger) Error(ctx context.Context, s string, i ...interface{}) {
+	l.logger.Error(s, interfacesToFields(i...)...)
+}
+
+func (l logger) Trace(ctx context.Context, begin time.Time, fc func() (sql string, rowsAffected int64), err error) {
+	if l.level.Level() <= zap.DebugLevel {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			return
+		}
+
+		sql, rowsAffected := fc()
+		fields := []zap.Field{
+			zap.String("sql", sql),
+			zap.Int64("rows_affected", rowsAffected),
+			zap.Duration("elapsed", time.Since(begin)),
+		}
+		if err != nil {
+			fields = append(fields, zap.Error(err))
+		}
+		l.logger.Debug("trace", fields...)
+	}
+}
+
+func newLogger(zlog *zap.Logger, zlogLevel *zap.AtomicLevel) *logger {
+	return &logger{logger: zlog, level: zlogLevel}
+}
+
+func interfacesToFields(i ...interface{}) []zap.Field {
+	fields := make([]zap.Field, 0)
+	for idx, v := range i {
+		fields = append(fields, zap.Any(strconv.Itoa(idx), v))
+	}
+	return fields
+}

db/models/api_key.go

@@ -0,0 +1,14 @@
+package models
+
+import "gorm.io/gorm"
+
+func init() {
+	registerModel(&APIKey{})
+}
+
+type APIKey struct {
+	gorm.Model
+	UserID uint
+	Key    string
+	User   User
+}

db/models/blocklist.go

@@ -0,0 +1,22 @@
+package models
+
+import (
+	"time"
+
+	"gorm.io/gorm"
+)
+
+func init() {
+	registerModel(&Blocklist{})
+}
+
+type Blocklist struct {
+	gorm.Model
+	IP        string
+	Reason    string
+	BlockedAt time.Time
+}
+
+func (Blocklist) TableName() string {
+	return "blocklist"
+}

db/models/dnslink.go

@@ -0,0 +1,15 @@
+package models
+
+import "gorm.io/gorm"
+
+func init() {
+	registerModel(&DNSLink{})
+}
+
+type DNSLink struct {
+	gorm.Model
+	UserID   uint `gorm:"uniqueIndex:idx_user_id_upload"`
+	User     User
+	UploadID uint `gorm:"uniqueIndex:idx_user_id_upload"`
+	Upload   Upload
+}

db/models/download.go

@@ -0,0 +1,21 @@
+package models
+
+import (
+	"time"
+
+	"gorm.io/gorm"
+)
+
+func init() {
+	registerModel(&Download{})
+}
+
+type Download struct {
+	gorm.Model
+	UserID       uint
+	User         User
+	UploadID     uint
+	Upload       Upload
+	DownloadedAt time.Time
+	IP           string
+}

db/models/email_verification.go

@@ -0,0 +1,21 @@
+package models
+
+import (
+	"time"
+
+	"gorm.io/gorm"
+)
+
+func init() {
+	registerModel(&EmailVerification{})
+}
+
+type EmailVerification struct {
+	gorm.Model
+
+	UserID    uint
+	User      User
+	NewEmail  string
+	Token     string
+	ExpiresAt time.Time
+}

db/models/import.go

@@ -0,0 +1,26 @@
+package models
+
+import "gorm.io/gorm"
+
+type ImportStatus string
+
+const (
+	ImportStatusQueued     ImportStatus = "queued"
+	ImportStatusProcessing ImportStatus = "processing"
+	ImportStatusCompleted  ImportStatus = "completed"
+)
+
+func init() {
+	registerModel(&Import{})
+}
+
+type Import struct {
+	gorm.Model
+	UserID     uint
+	Hash       []byte `gorm:"type:binary(32);"`
+	Protocol   string
+	User       User
+	ImporterIP string
+	Status     ImportStatus
+	Progress   float64
+}

db/models/models.go

@@ -0,0 +1,11 @@
+package models
+
+var registered []interface{}
+
+func registerModel(model interface{}) {
+	registered = append(registered, model)
+}
+
+func GetModels() []interface{} {
+	return registered
+}

db/models/password_reset.go

@@ -0,0 +1,20 @@
+package models
+
+import (
+	"time"
+
+	"gorm.io/gorm"
+)
+
+func init() {
+	registerModel(&PasswordReset{})
+}
+
+type PasswordReset struct {
+	gorm.Model
+
+	UserID    uint
+	User      User
+	Token     string
+	ExpiresAt time.Time
+}

db/models/pin.go

@@ -0,0 +1,15 @@
+package models
+
+import "gorm.io/gorm"
+
+func init() {
+	registerModel(&Pin{})
+}
+
+type Pin struct {
+	gorm.Model
+	UploadID uint
+	Upload   Upload
+	UserID   uint
+	User     User
+}

db/models/public_key.go

@@ -0,0 +1,14 @@
+package models
+
+import "gorm.io/gorm"
+
+func init() {
+	registerModel(&PublicKey{})
+}
+
+type PublicKey struct {
+	gorm.Model
+	UserID uint
+	Key    string `gorm:"unique;not null"`
+	User   User
+}

db/models/s3_upload.go

@@ -0,0 +1,14 @@
+package models
+
+import "gorm.io/gorm"
+
+func init() {
+	registerModel(&S3Upload{})
+}
+
+type S3Upload struct {
+	gorm.Model
+	UploadID string `gorm:"unique;not null"`
+	Bucket   string `gorm:"not null;index:idx_bucket_key"`
+	Key      string `gorm:"not null;index:idx_bucket_key"`
+}

db/models/s5_challenge.go

@@ -0,0 +1,16 @@
+//go:build s5
+
+package models
+
+import "gorm.io/gorm"
+
+func init() {
+	registerModel(&S5Challenge{})
+}
+
+type S5Challenge struct {
+	gorm.Model
+	Challenge string
+	Pubkey    string
+	Type      string
+}

db/models/sc_price_history.go

@@ -0,0 +1,26 @@
+package models
+
+import (
+	"time"
+
+	"github.com/shopspring/decimal"
+
+	"gorm.io/gorm"
+	"gorm.io/gorm/schema"
+)
+
+var _ schema.Tabler = (*SCPriceHistory)(nil)
+
+func init() {
+	registerModel(&SCPriceHistory{})
+}
+
+type SCPriceHistory struct {
+	gorm.Model
+	CreatedAt time.Time       `gorm:"index:idx_rate"`
+	Rate      decimal.Decimal `gorm:"type:DECIMAL(30,20);index:idx_rate"`
+}
+
+func (SCPriceHistory) TableName() string {
+	return "sc_price_history"
+}

db/models/sia_upload.go

@@ -0,0 +1,14 @@
+package models
+
+import "gorm.io/gorm"
+
+func init() {
+	registerModel(&SiaUpload{})
+}
+
+type SiaUpload struct {
+	gorm.Model
+	UploadID string `gorm:"unique;not null"`
+	Bucket   string `gorm:"not null;index:idx_bucket_key"`
+	Key      string `gorm:"not null;index:idx_bucket_key"`
+}

db/models/tus_lock.go

@@ -0,0 +1,94 @@
+package models
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+func init() {
+	registerModel(&TusLock{})
+
+}
+
+var (
+	ErrTusLockBusy = errors.New("lock is currently held by another process")
+)
+
+type TusLock struct {
+	gorm.Model
+	LockId           string `gorm:"index:idx_lock_id,unique"`
+	HolderPID        int    `gorm:"index"`
+	AcquiredAt       time.Time
+	ExpiresAt        time.Time
+	ReleaseRequested bool
+	DeletedAt        gorm.DeletedAt `gorm:"index:idx_lock_id,unique"`
+}
+
+func (t *TusLock) TryLock(db *gorm.DB, ctx context.Context) error {
+	return db.Transaction(func(tx *gorm.DB) error {
+		var existingLock TusLock
+
+		if err := tx.WithContext(ctx).Clauses(clause.Locking{Strength: "UPDATE"}).Where("lock_id = ?", t.LockId).First(&existingLock).Error; err != nil {
+			if errors.Is(err, gorm.ErrRecordNotFound) {
+				// Insert new lock record
+				err := tx.WithContext(ctx).Create(t).Error
+				if err != nil {
+					return err
+				}
+
+				t = &existingLock
+				return nil
+			}
+
+			return err
+		}
+
+		// Check if existing lock is expired
+		if existingLock.ExpiresAt.Before(time.Now()) || existingLock.ReleaseRequested {
+
+			err := tx.Model(&existingLock).Updates(t).Error
+			if err != nil {
+				return err
+			}
+
+			t = &existingLock
+
+			return nil
+		}
+
+		// Lock is currently held by another process
+		return ErrTusLockBusy
+	})
+}
+func (t *TusLock) RequestRelease(db *gorm.DB) error {
+	return db.Transaction(func(tx *gorm.DB) error {
+		// Update the ReleaseRequested flag in the database for the specific lock
+		return tx.Model(t).Where("lock_id = ?", t.LockId).Update("release_requested", true).Error
+	})
+}
+func (t *TusLock) Released(db *gorm.DB) error {
+	return db.Transaction(func(tx *gorm.DB) error {
+		// Update the ReleaseRequested flag in the database for the specific lock
+		return tx.Model(t).Where("lock_id = ?", t.LockId).Update("release_requested", false).Error
+	})
+}
+
+func (t *TusLock) IsReleaseRequested(db *gorm.DB) (bool, error) {
+	var count int64
+	err := db.Model(&TusLock{}).Where(&TusLock{LockId: t.LockId, ReleaseRequested: true}).Count(&count).Error
+	if err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			return true, nil
+		}
+		return false, err
+	}
+
+	return count > 0, nil
+}
+func (t *TusLock) Delete(db *gorm.DB) error {
+	return db.Where("lock_id = ?", t.LockId).Delete(&TusLock{}).Error
+}

db/models/tus_upload.go

@@ -0,0 +1,21 @@
+package models
+
+import "gorm.io/gorm"
+
+func init() {
+	registerModel(&TusUpload{})
+
+}
+
+type TusUpload struct {
+	gorm.Model
+	Hash       []byte `gorm:"type:binary(32);uniqueIndex:idx_hash_deleted"`
+	MimeType   string
+	UploadID   string `gorm:"uniqueIndex"`
+	UploaderID uint
+	UploaderIP string
+	Uploader   User `gorm:"foreignKey:UploaderID"`
+	Protocol   string
+	Completed  bool
+	DeletedAt  gorm.DeletedAt `gorm:"uniqueIndex:idx_hash_deleted"`
+}

db/models/upload.go

@@ -0,0 +1,18 @@
+package models
+
+import "gorm.io/gorm"
+
+func init() {
+	registerModel(&Upload{})
+}
+
+type Upload struct {
+	gorm.Model
+	UserID     uint
+	Hash       []byte `gorm:"type:binary(32);uniqueIndex"`
+	MimeType   string
+	Protocol   string
+	User       User
+	UploaderIP string
+	Size       uint64
+}

db/models/user.go

@@ -0,0 +1,61 @@
+package models
+
+import (
+	"errors"
+	"time"
+
+	emailverifier "github.com/AfterShip/email-verifier"
+	"gorm.io/gorm"
+)
+
+func init() {
+	registerModel(&User{})
+}
+
+type User struct {
+	gorm.Model
+	FirstName          string
+	LastName           string
+	Email              string `gorm:"unique"`
+	PasswordHash       string
+	Role               string
+	PublicKeys         []PublicKey
+	APIKeys            []APIKey
+	Uploads            []Upload
+	LastLogin          *time.Time
+	LastLoginIP        string
+	OTPEnabled         bool `gorm:"default:false;"`
+	OTPVerified        bool `gorm:"default:false;"`
+	OTPSecret          string
+	OTPAuthUrl         string
+	Verified           bool `gorm:"default:false;"`
+	EmailVerifications []EmailVerification
+	PasswordResets     []PasswordReset
+}
+
+func (u *User) BeforeUpdate(tx *gorm.DB) error {
+	dest := tx.Statement.Dest.(User)
+
+	if tx.Statement.Changed("Email") {
+		verify, err := getEmailVerfier().Verify(dest.Email)
+		if err != nil {
+			return err
+		}
+		if !verify.Syntax.Valid {
+			return errors.New("email is invalid")
+		}
+	}
+
+	return nil
+}
+
+func getEmailVerfier() *emailverifier.Verifier {
+	verifier := emailverifier.NewVerifier()
+
+	verifier.DisableSMTPCheck()
+	verifier.DisableGravatarCheck()
+	verifier.DisableDomainSuggest()
+	verifier.DisableAutoUpdateDisposable()
+
+	return verifier
+}

go.mod

@@ -1,127 +1,152 @@
 module git.lumeweb.com/LumeWeb/portal
 
-go 1.18
+go 1.21.6
 
 require (
-	github.com/go-ozzo/ozzo-validation v3.6.0+incompatible
-	github.com/go-ozzo/ozzo-validation/v4 v4.3.0
-	github.com/go-resty/resty/v2 v2.7.0
-	github.com/golang-queue/queue v0.1.3
-	github.com/iris-contrib/swagger v0.0.0-20230311205341-32127a753a68
-	github.com/joomcode/errorx v1.1.0
-	github.com/kataras/iris/v12 v12.2.0
-	github.com/kataras/jwt v0.1.8
-	github.com/multiformats/go-multibase v0.2.0
-	github.com/spf13/pflag v1.0.5
-	github.com/spf13/viper v1.15.0
-	github.com/swaggo/swag v1.16.1
-	github.com/tus/tusd v1.11.0
-	go.uber.org/zap v1.24.0
-	golang.org/x/crypto v0.8.0
-	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
-	gorm.io/driver/mysql v1.4.6
-	gorm.io/driver/sqlite v1.4.3
-	gorm.io/gorm v1.24.3
-	lukechampine.com/blake3 v1.2.1
+	git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240314105331-6510beddf2cf
+	github.com/AfterShip/email-verifier v1.4.0
+	github.com/LumeWeb/siacentral-api v0.0.0-20240311114304-4ff40c07bce5
+	github.com/aws/aws-sdk-go-v2 v1.25.1
+	github.com/aws/aws-sdk-go-v2/config v1.27.2
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.50.3
+	github.com/casbin/casbin/v2 v2.82.0
+	github.com/ddo/rq v0.0.0-20190828174524-b3daa55fcaba
+	github.com/dnslink-std/go v0.6.0
+	github.com/docker/go-units v0.5.0
+	github.com/gabriel-vasile/mimetype v1.4.3
+	github.com/getkin/kin-openapi v0.118.0
+	github.com/go-co-op/gocron/v2 v2.2.4
+	github.com/go-gorm/caches/v4 v4.0.0
+	github.com/go-resty/resty/v2 v2.11.0
+	github.com/go-sql-driver/mysql v1.7.1
+	github.com/golang-jwt/jwt/v5 v5.2.0
+	github.com/golang-queue/queue v0.2.0
+	github.com/google/uuid v1.6.0
+	github.com/hashicorp/go-plugin v1.6.0
+	github.com/julienschmidt/httprouter v1.3.0
+	github.com/mitchellh/mapstructure v1.5.0
+	github.com/pquerna/otp v1.4.0
+	github.com/redis/go-redis/v9 v9.5.1
+	github.com/rs/cors v1.10.1
+	github.com/samber/lo v1.39.0
+	github.com/shopspring/decimal v1.3.1
+	github.com/spf13/viper v1.18.2
+	github.com/tus/tusd/v2 v2.2.3-0.20240125123123-9080d351525d
+	github.com/vmihailenco/msgpack/v5 v5.4.1
+	github.com/wneessen/go-mail v0.4.1
+	go.sia.tech/core v0.1.12
+	go.sia.tech/jape v0.11.2-0.20240228204811-29a0f056d231
+	go.sia.tech/renterd v1.0.5
+	go.uber.org/fx v1.20.1
+	go.uber.org/zap v1.26.0
+	go.uber.org/zap/exp v0.2.0
+	golang.org/x/crypto v0.21.0
+	gorm.io/driver/mysql v1.5.4
+	gorm.io/gorm v1.25.7
+	lukechampine.com/blake3 v1.2.2-0.20240329192137-af604d0fbc33
+	nhooyr.io/websocket v1.8.10
 )
 
 require (
-	github.com/BurntSushi/toml v1.2.1 // indirect
-	github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 // indirect
-	github.com/CloudyKit/jet/v6 v6.2.0 // indirect
-	github.com/Joker/jade v1.1.3 // indirect
-	github.com/KyleBanks/depth v1.2.1 // indirect
-	github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06 // indirect
-	github.com/andybalholm/brotli v1.0.5 // indirect
-	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
-	github.com/aymerick/douceur v0.2.0 // indirect
-	github.com/blang/semver/v4 v4.0.0 // indirect
-	github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 // indirect
-	github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 // indirect
-	github.com/fatih/structs v1.1.0 // indirect
-	github.com/flosch/pongo2/v4 v4.0.2 // indirect
-	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/go-openapi/jsonpointer v0.19.6 // indirect
-	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/spec v0.20.9 // indirect
-	github.com/go-openapi/swag v0.22.3 // indirect
-	github.com/go-sql-driver/mysql v1.7.1 // indirect
-	github.com/gobwas/httphead v0.1.0 // indirect
-	github.com/gobwas/pool v0.2.1 // indirect
-	github.com/gobwas/ws v1.2.0 // indirect
-	github.com/goccy/go-json v0.9.11 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/uuid v1.3.0 // indirect
-	github.com/gorilla/css v1.0.0 // indirect
-	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.19.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.22.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.27.2 // indirect
+	github.com/aws/smithy-go v1.20.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
+	github.com/casbin/govaluate v1.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/dchest/threefish v0.0.0-20120919164726-3ecf4c494abf // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/fatih/color v1.14.1 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-openapi/jsonpointer v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.8 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/gorilla/websocket v1.5.1 // indirect
+	github.com/hashicorp/go-hclog v1.6.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/imdario/mergo v0.3.16 // indirect
-	github.com/iris-contrib/go.uuid v2.0.0+incompatible // indirect
-	github.com/iris-contrib/schema v0.0.6 // indirect
+	github.com/hashicorp/yamux v0.1.1 // indirect
+	github.com/hbollon/go-edlib v1.6.0 // indirect
+	github.com/invopop/yaml v0.2.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/jonboulle/clockwork v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/kataras/blocks v0.0.7 // indirect
-	github.com/kataras/golog v0.1.8 // indirect
-	github.com/kataras/neffos v0.0.21 // indirect
-	github.com/kataras/pio v0.0.11 // indirect
-	github.com/kataras/sitemap v0.0.6 // indirect
-	github.com/kataras/tunnel v0.0.4 // indirect
-	github.com/klauspost/compress v1.16.5 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
-	github.com/kr/pretty v0.3.1 // indirect
+	github.com/jpillora/backoff v1.0.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+	github.com/klauspost/reedsolomon v1.12.1 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
-	github.com/mailgun/raymond/v2 v2.0.48 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/mattn/go-sqlite3 v1.14.16 // indirect
-	github.com/mediocregopher/radix/v3 v3.8.1 // indirect
-	github.com/microcosm-cc/bluemonday v1.0.23 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
-	github.com/mr-tron/base58 v1.2.0 // indirect
-	github.com/multiformats/go-base32 v0.1.0 // indirect
-	github.com/multiformats/go-base36 v0.2.0 // indirect
-	github.com/nats-io/nats.go v1.25.0 // indirect
-	github.com/nats-io/nkeys v0.4.4 // indirect
-	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.7 // indirect
-	github.com/rogpeppe/go-internal v1.10.0 // indirect
-	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/schollz/closestmatch v2.1.0+incompatible // indirect
-	github.com/sergi/go-diff v1.1.0 // indirect
-	github.com/sirupsen/logrus v1.9.0 // indirect
-	github.com/spf13/afero v1.9.5 // indirect
-	github.com/spf13/cast v1.5.0 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/subosito/gotenv v1.4.2 // indirect
-	github.com/tdewolff/minify/v2 v2.12.5 // indirect
-	github.com/tdewolff/parse/v2 v2.6.5 // indirect
-	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
+	github.com/miekg/dns v1.1.58 // indirect
+	github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 // indirect
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
+	github.com/mr-tron/base58 v1.1.0 // indirect
+	github.com/multiformats/go-base32 v0.0.3 // indirect
+	github.com/multiformats/go-base36 v0.1.0 // indirect
+	github.com/multiformats/go-multibase v0.2.0 // indirect
+	github.com/oklog/run v1.0.0 // indirect
+	github.com/olebedev/emitter v0.0.0-20230411050614-349169dec2ba // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/perimeterx/marshmallow v1.1.5 // indirect
+	github.com/prometheus/client_golang v1.18.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.45.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/robfig/cron/v3 v3.0.1 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
-	github.com/yosssi/ace v0.0.5 // indirect
-	go.uber.org/atomic v1.11.0 // indirect
+	gitlab.com/NebulousLabs/bolt v1.4.4 // indirect
+	gitlab.com/NebulousLabs/encoding v0.0.0-20200604091946-456c3dc907fe // indirect
+	gitlab.com/NebulousLabs/entropy-mnemonics v0.0.0-20181018051301-7532f67e3500 // indirect
+	gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975 // indirect
+	gitlab.com/NebulousLabs/fastrand v0.0.0-20181126182046-603482d69e40 // indirect
+	gitlab.com/NebulousLabs/go-upnp v0.0.0-20211002182029-11da932010b6 // indirect
+	gitlab.com/NebulousLabs/log v0.0.0-20210609172545-77f6775350e2 // indirect
+	gitlab.com/NebulousLabs/merkletree v0.0.0-20200118113624-07fbf710afc4 // indirect
+	gitlab.com/NebulousLabs/persist v0.0.0-20200605115618-007e5e23d877 // indirect
+	gitlab.com/NebulousLabs/ratelimit v0.0.0-20200811080431-99b8f0768b2e // indirect
+	gitlab.com/NebulousLabs/siamux v0.0.2-0.20220630142132-142a1443a259 // indirect
+	gitlab.com/NebulousLabs/threadgroup v0.0.0-20200608151952-38921fbef213 // indirect
+	go.etcd.io/bbolt v1.3.8 // indirect
+	go.sia.tech/mux v1.2.0 // indirect
+	go.sia.tech/siad v1.5.10-0.20230228235644-3059c0b930ca // indirect
+	go.uber.org/dig v1.17.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/net v0.9.0 // indirect
-	golang.org/x/sys v0.8.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.8.0 // indirect
-	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	golang.org/x/exp v0.0.0-20231219180239-dc181d75b848 // indirect
+	golang.org/x/mod v0.16.0 // indirect
+	golang.org/x/net v0.22.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.19.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect
+	google.golang.org/grpc v1.62.0 // indirect
+	google.golang.org/protobuf v1.32.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	lukechampine.com/frand v1.4.2 // indirect
 )
 
-replace go.uber.org/multierr => go.uber.org/multierr v1.9.0
-
-replace (
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.39.0
-	go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.14.0
-	go.opentelemetry.io/otel/exporters/otlp/internal/retry => go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.12.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace => go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.12.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp => go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.12.0
-	go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v0.37.0
-	go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v1.12.0
-	go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.14.0
-	go.opentelemetry.io/proto/otlp => go.opentelemetry.io/proto/otlp v0.19.0
-)
+replace github.com/tus/tusd/v2 => github.com/LumeWeb/tusd/v2 v2.2.3-0.20240224143554-96925dd43120

import/import.go

@@ -0,0 +1,217 @@
+package _import
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"git.lumeweb.com/LumeWeb/portal/db/models"
+
+	"go.uber.org/fx"
+	"gorm.io/gorm"
+)
+
+var ErrNotFound = gorm.ErrRecordNotFound
+
+var _ ImportService = (*ImportServiceDefault)(nil)
+
+type ImportMetadata struct {
+	ID         uint
+	UserID     uint
+	Hash       []byte
+	Status     models.ImportStatus
+	Progress   float64
+	Protocol   string
+	ImporterIP string
+	Created    time.Time
+}
+
+type ImportService interface {
+	SaveImport(ctx context.Context, metadata ImportMetadata, skipExisting bool) error
+	GetImport(ctx context.Context, objectHash []byte) (ImportMetadata, error)
+	DeleteImport(ctx context.Context, objectHash []byte) error
+	UpdateProgress(ctx context.Context, objectHash []byte, stage int, totalStages int) error
+	UpdateStatus(ctx context.Context, objectHash []byte, status models.ImportStatus) error
+}
+
+func (u ImportMetadata) IsEmpty() bool {
+	if u.UserID != 0 || u.Protocol != "" || u.ImporterIP != "" || u.Status != "" {
+		return false
+	}
+
+	if !u.Created.IsZero() {
+		return false
+	}
+
+	if len(u.Hash) != 0 {
+		return false
+	}
+
+	return true
+}
+
+var Module = fx.Module("import",
+	fx.Provide(
+		fx.Annotate(
+			NewImportService,
+			fx.As(new(ImportService)),
+		),
+	),
+)
+
+type ImportServiceDefault struct {
+	db *gorm.DB
+}
+
+func (i ImportServiceDefault) UpdateProgress(ctx context.Context, objectHash []byte, stage int, totalStages int) error {
+	_import, err := i.GetImport(ctx, objectHash)
+	if err != nil {
+		return err
+	}
+
+	if _import.IsEmpty() {
+		return ErrNotFound
+	}
+
+	_import.Progress = float64(stage) / float64(totalStages) * 100.0
+
+	return i.SaveImport(ctx, _import, false)
+}
+
+func (i ImportServiceDefault) UpdateStatus(ctx context.Context, objectHash []byte, status models.ImportStatus) error {
+	_import, err := i.GetImport(ctx, objectHash)
+	if err != nil {
+		return err
+	}
+
+	if _import.IsEmpty() {
+		return ErrNotFound
+	}
+
+	_import.Status = status
+
+	return i.SaveImport(ctx, _import, false)
+}
+
+func (i ImportServiceDefault) SaveImport(ctx context.Context, metadata ImportMetadata, skipExisting bool) error {
+	var __import models.Import
+
+	__import.Hash = metadata.Hash
+
+	ret := i.db.WithContext(ctx).Model(&models.Import{}).Where(&__import).First(&__import)
+
+	if ret.Error != nil {
+		if errors.Is(ret.Error, gorm.ErrRecordNotFound) {
+			return i.createImport(ctx, metadata)
+		}
+		return ret.Error
+	}
+
+	if skipExisting {
+		return nil
+	}
+
+	changed := false
+
+	if __import.UserID != metadata.UserID {
+		__import.UserID = metadata.UserID
+		changed = true
+	}
+
+	if __import.Status != metadata.Status {
+		__import.Status = metadata.Status
+		changed = true
+	}
+
+	if __import.Progress != metadata.Progress {
+		__import.Progress = metadata.Progress
+		changed = true
+	}
+
+	if __import.Protocol != metadata.Protocol {
+		__import.Protocol = metadata.Protocol
+		changed = true
+	}
+
+	if __import.ImporterIP != metadata.ImporterIP {
+		__import.ImporterIP = metadata.ImporterIP
+		changed = true
+	}
+	if changed {
+		return i.db.Updates(&__import).Error
+	}
+
+	return nil
+}
+
+func (m *ImportServiceDefault) createImport(ctx context.Context, metadata ImportMetadata) error {
+	__import := models.Import{
+		UserID:     metadata.UserID,
+		Hash:       metadata.Hash,
+		Status:     metadata.Status,
+		Progress:   metadata.Progress,
+		Protocol:   metadata.Protocol,
+		ImporterIP: metadata.ImporterIP,
+	}
+
+	if __import.Status == "" {
+		__import.Status = models.ImportStatusQueued
+	}
+
+	return m.db.WithContext(ctx).Create(&__import).Error
+}
+
+func (i ImportServiceDefault) GetImport(ctx context.Context, objectHash []byte) (ImportMetadata, error) {
+	var _import models.Import
+
+	_import.Hash = objectHash
+
+	ret := i.db.WithContext(ctx).Model(&models.Import{}).Where(&_import).First(&_import)
+
+	if ret.Error != nil {
+		if errors.Is(ret.Error, gorm.ErrRecordNotFound) {
+			return ImportMetadata{}, ErrNotFound
+		}
+		return ImportMetadata{}, ret.Error
+
+	}
+
+	return ImportMetadata{
+		ID:         _import.ID,
+		UserID:     _import.UserID,
+		Hash:       _import.Hash,
+		Protocol:   _import.Protocol,
+		Status:     _import.Status,
+		Progress:   _import.Progress,
+		ImporterIP: _import.ImporterIP,
+		Created:    _import.CreatedAt,
+	}, nil
+}
+
+func (i ImportServiceDefault) DeleteImport(ctx context.Context, objectHash []byte) error {
+	var _import models.Import
+
+	_import.Hash = objectHash
+
+	ret := i.db.WithContext(ctx).Model(&models.Import{}).Where(&_import).Delete(&_import)
+
+	if ret.Error != nil {
+		if errors.Is(ret.Error, gorm.ErrRecordNotFound) {
+			return ErrNotFound
+		}
+		return ret.Error
+	}
+
+	return nil
+}
+
+type ImportServiceParams struct {
+	fx.In
+	Db *gorm.DB
+}
+
+func NewImportService(params ImportServiceParams) *ImportServiceDefault {
+	return &ImportServiceDefault{
+		db: params.Db,
+	}
+}

logger/logger.go

@@ -1,30 +1,45 @@
 package logger
 
 import (
-	"github.com/spf13/viper"
+	"os"
+
+	"git.lumeweb.com/LumeWeb/portal/config"
+
 	"go.uber.org/zap"
-	"log"
+	"go.uber.org/zap/zapcore"
 )
 
-var logger *zap.Logger
+func NewLogger(cm *config.Manager) (*zap.Logger, *zap.AtomicLevel) {
 
-func Init() {
-	var newLogger *zap.Logger
-	var err error
+	// Create a new atomic level
+	atomicLevel := zap.NewAtomicLevel()
 
-	if viper.GetBool("debug") {
-		newLogger, err = zap.NewDevelopment()
+	if cm != nil {
+		// Set initial log level, for example, info level
+		atomicLevel.SetLevel(mapLogLevel(cm.Config().Core.Log.Level))
 	} else {
-		newLogger, err = zap.NewProduction()
+		atomicLevel.SetLevel(mapLogLevel("debug"))
 	}
 
-	if err != nil {
-		log.Fatal(err)
+	// Create the logger with the atomic level
+	logger := zap.New(zapcore.NewCore(
+		zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()),
+		zapcore.Lock(os.Stdout),
+		atomicLevel,
+	), zap.AddCaller())
+
+	return logger, &atomicLevel
 }
 
-	logger = newLogger
+func mapLogLevel(level string) zapcore.Level {
+	switch level {
+	case "debug":
+		return zapcore.DebugLevel
+	case "info":
+		return zapcore.InfoLevel
+	case "warn":
+		return zapcore.WarnLevel
+	default:
+		return zapcore.ErrorLevel
 	}
-
-func Get() *zap.Logger {
-	return logger
 }

mailer/email.go

@@ -0,0 +1,69 @@
+package mailer
+
+import "github.com/wneessen/go-mail"
+
+type Email struct {
+	to      string
+	from    string
+	subject string
+	body    string
+}
+
+func (e *Email) To() string {
+	return e.to
+}
+
+func (e *Email) SetTo(to string) {
+	e.to = to
+}
+
+func (e *Email) From() string {
+	return e.from
+}
+
+func (e *Email) SetFrom(from string) {
+	e.from = from
+}
+
+func (e *Email) Subject() string {
+	return e.subject
+}
+
+func (e *Email) SetSubject(subject string) {
+	e.subject = subject
+}
+
+func (e *Email) Body() string {
+	return e.body
+}
+
+func (e *Email) SetBody(body string) {
+	e.body = body
+}
+
+func (e *Email) ToMessage() (*mail.Msg, error) {
+	msg :=
+		mail.NewMsg()
+
+	err := msg.From(e.from)
+	if err != nil {
+		return nil, err
+	}
+
+	err = msg.To(e.to)
+
+	if err != nil {
+		return nil, err
+	}
+
+	msg.SetBodyString("text/plain", e.body)
+
+	return msg, nil
+}
+
+func NewEmail(subject, body string) *Email {
+	return &Email{
+		subject: subject,
+		body:    body,
+	}
+}

mailer/mailer.go

@@ -0,0 +1,83 @@
+package mailer
+
+import (
+	"context"
+	"strings"
+
+	"git.lumeweb.com/LumeWeb/portal/config"
+	"github.com/wneessen/go-mail"
+	"go.uber.org/fx"
+	"go.uber.org/zap"
+)
+
+type TemplateData = map[string]interface{}
+
+var Module = fx.Module("mailer",
+	fx.Options(
+		fx.Provide(NewMailer),
+		fx.Provide(NewTemplateRegistry),
+		fx.Invoke(func(registry *TemplateRegistry) error {
+			return registry.loadTemplates()
+		}),
+	),
+)
+
+type Mailer struct {
+	config           *config.Manager
+	logger           *zap.Logger
+	client           *mail.Client
+	templateRegistry *TemplateRegistry
+}
+
+func (m *Mailer) TemplateSend(template string, subjectVars TemplateData, bodyVars TemplateData, to string) error {
+	email, err := m.templateRegistry.RenderTemplate(template, subjectVars, bodyVars)
+
+	if err != nil {
+		return err
+	}
+
+	email.SetFrom(m.config.Config().Core.Mail.From)
+	email.SetTo(to)
+
+	msg, err := email.ToMessage()
+	if err != nil {
+		return err
+	}
+
+	return m.client.DialAndSend(msg)
+}
+
+func NewMailer(lc fx.Lifecycle, config *config.Manager, logger *zap.Logger, templateRegistry *TemplateRegistry) (*Mailer, error) {
+	m := &Mailer{config: config, logger: logger, templateRegistry: templateRegistry}
+
+	lc.Append(fx.Hook{
+		OnStart: func(context.Context) error {
+			var options []mail.Option
+
+			if config.Config().Core.Mail.Port != 0 {
+				options = append(options, mail.WithPort(config.Config().Core.Mail.Port))
+			}
+
+			if config.Config().Core.Mail.AuthType != "" {
+				options = append(options, mail.WithSMTPAuth(mail.SMTPAuthType(strings.ToUpper(config.Config().Core.Mail.AuthType))))
+			}
+
+			if config.Config().Core.Mail.SSL {
+				options = append(options, mail.WithSSLPort(true))
+			}
+
+			options = append(options, mail.WithUsername(config.Config().Core.Mail.Username))
+			options = append(options, mail.WithPassword(config.Config().Core.Mail.Password))
+
+			client, err := mail.NewClient(config.Config().Core.Mail.Host, options...)
+			if err != nil {
+				return err
+			}
+
+			m.client = client
+			return nil
+		},
+	})
+
+	return m, nil
+}

mailer/templates.go

@@ -0,0 +1,97 @@
+package mailer
+
+import (
+	"embed"
+	"errors"
+	"io/fs"
+	"strings"
+	"text/template"
+)
+
+const EMAIL_FS_PREFIX = "templates/"
+
+const TPL_PASSWORD_RESET = "password_reset"
+const TPL_VERIFY_EMAIL = "verify_email"
+
+type EmailTemplate struct {
+	Subject *template.Template
+	Body    *template.Template
+}
+
+//go:embed templates/*
+var templateFS embed.FS
+
+var ErrTemplateNotFound = errors.New("template not found")
+
+type TemplateRegistry struct {
+	templates map[string]EmailTemplate
+}
+
+func NewTemplateRegistry() *TemplateRegistry {
+	return &TemplateRegistry{
+		templates: make(map[string]EmailTemplate),
+	}
+}
+
+func (tr *TemplateRegistry) loadTemplates() error {
+	subjectTemplates, err := fs.Glob(templateFS, EMAIL_FS_PREFIX+"*_subject*")
+
+	if err != nil {
+		return err
+	}
+
+	for _, subjectTemplate := range subjectTemplates {
+		templateName := strings.TrimPrefix(subjectTemplate, EMAIL_FS_PREFIX)
+		templateName = strings.TrimSuffix(templateName, "_subject.tpl")
+		bodyTemplate := strings.TrimSuffix(subjectTemplate, "_subject.tpl") + "_body.tpl"
+		bodyTemplate = strings.TrimPrefix(bodyTemplate, EMAIL_FS_PREFIX)
+
+		subjectContent, err := fs.ReadFile(templateFS, EMAIL_FS_PREFIX+templateName+"_subject.tpl")
+		if err != nil {
+			return err
+		}
+
+		subjectTmpl, err := template.New(templateName).Parse(string(subjectContent))
+		if err != nil {
+			return err
+		}
+
+		bodyContent, err := fs.ReadFile(templateFS, EMAIL_FS_PREFIX+bodyTemplate)
+		if err != nil {
+			return err
+		}
+
+		bodyTmpl, err := template.New(templateName).Parse(string(bodyContent))
+		if err != nil {
+			return err
+		}
+
+		tr.templates[templateName] = EmailTemplate{
+			Subject: subjectTmpl,
+			Body:    bodyTmpl,
+		}
+	}
+
+	return nil
+}
+
+func (tr *TemplateRegistry) RenderTemplate(templateName string, subjectVars TemplateData, bodyVars TemplateData) (*Email, error) {
+	tmpl, ok := tr.templates[templateName]
+	if !ok {
+		return nil, ErrTemplateNotFound
+	}
+
+	var subjectBuilder strings.Builder
+	err := tmpl.Subject.Execute(&subjectBuilder, subjectVars)
+	if err != nil {
+		return nil, err
+	}
+
+	var bodyBuilder strings.Builder
+	err = tmpl.Body.Execute(&bodyBuilder, bodyVars)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewEmail(subjectBuilder.String(), bodyBuilder.String()), nil
+}

mailer/templates/password_reset_body.tpl

@@ -0,0 +1,16 @@
+Dear {{if .FirstName}}{{.FirstName}}{{else}}{{.Email}}{{end}},
+
+You are receiving this email because we received a password reset request for your account. If you did not request a password reset, please ignore this email.
+
+To reset your password, please click the link below:
+{{.ResetURL}}
+
+This link will expire in {{.ExpireTime}} hours. If you did not request a password reset, no further action is required.
+
+If you're having trouble clicking the reset link, copy and paste the URL below into your web browser:
+{{.ResetURL}}
+
+Thank you for using {{.PortalName}}.
+
+Best regards,
+The {{.PortalName}} Team

mailer/templates/password_reset_subject.tpl

@@ -0,0 +1 @@
+Reset Your Password for {{.PortalName}}

mailer/templates/verify_email_body.tpl

@@ -0,0 +1,10 @@
+Dear {{if .FirstName}}{{.FirstName}}{{else}}{{.Email}}{{end}},
+
+Thank you for registering with {{.PortalName}}. To complete your registration and verify your email address, please go to the following link:
+
+{{.VerificationLink}}
+
+Please note, this link will expire in {{.ExpireTime}}. If you did not initiate this request, please ignore this email or contact our support team for assistance.
+
+Best regards,
+The {{.PortalName}} Team

mailer/templates/verify_email_subject.tpl

@@ -0,0 +1 @@
+Verify Your Email for {{.PortalName}}

main.go

@@ -1,106 +0,0 @@
-package main
-
-import (
-	"embed"
-	"git.lumeweb.com/LumeWeb/portal/config"
-	"git.lumeweb.com/LumeWeb/portal/controller"
-	"git.lumeweb.com/LumeWeb/portal/db"
-	_ "git.lumeweb.com/LumeWeb/portal/docs"
-	"git.lumeweb.com/LumeWeb/portal/logger"
-	"git.lumeweb.com/LumeWeb/portal/service/auth"
-	"git.lumeweb.com/LumeWeb/portal/service/files"
-	"git.lumeweb.com/LumeWeb/portal/tus"
-	"github.com/iris-contrib/swagger"
-	"github.com/iris-contrib/swagger/swaggerFiles"
-	"github.com/kataras/iris/v12"
-	"github.com/kataras/iris/v12/mvc"
-	"go.uber.org/zap"
-	"log"
-	"net/http"
-)
-
-// Embed a directory of static files for serving from the app's root path
-//
-//go:embed app/*
-var embedFrontend embed.FS
-
-//	@title			Lume Web Portal
-//	@version		1.0
-//	@description	A decentralized data storage portal for the open web
-
-//	@contact.name	Lume Web Project
-//	@contact.url	https://lumeweb.com
-//	@contact.email	contact@lumeweb.com
-
-//	@license.name	MIT
-//	@license.url	https://opensource.org/license/mit/
-
-// @externalDocs.description	OpenAPI
-// @externalDocs.url			https://swagger.io/resources/open-api/
-func main() {
-	// Initialize the configuration settings
-	config.Init()
-
-	// Initialize the database connection
-	db.Init()
-	logger.Init()
-	files.Init()
-	auth.Init()
-
-	// Create a new Iris app instance
-	app := iris.New()
-	// Enable Gzip compression for responses
-	app.Use(iris.Compression)
-
-	// Serve static files from the embedded directory at the app's root path
-	app.HandleDir("/", embedFrontend)
-
-	api := app.Party("/api")
-	v1 := api.Party("/v1")
-
-	// Register the AccountController with the MVC framework and attach it to the "/api/account" path
-	mvc.Configure(v1.Party("/account"), func(app *mvc.Application) {
-		app.Handle(new(controller.AccountController))
-	})
-
-	mvc.Configure(v1.Party("/auth"), func(app *mvc.Application) {
-		app.Handle(new(controller.AuthController))
-	})
-
-	mvc.Configure(v1.Party("/files"), func(app *mvc.Application) {
-		app.Handle(new(controller.FilesController))
-		app.Router.Use()
-	})
-
-	tusHandler := tus.Init()
-
-	v1.Any(tus.TUS_API_PATH+"/{fileparam:path}", iris.FromStd(http.StripPrefix(v1.GetRelPath()+tus.TUS_API_PATH+"/", tusHandler)))
-	v1.Post(tus.TUS_API_PATH, iris.FromStd(http.StripPrefix(v1.GetRelPath()+tus.TUS_API_PATH, tusHandler)))
-
-	swaggerConfig := swagger.Config{
-		// The url pointing to API definition.
-		URL:          "http://localhost:8080/swagger/doc.json",
-		DeepLinking:  true,
-		DocExpansion: "list",
-		DomID:        "#swagger-ui",
-		// The UI prefix URL (see route).
-		Prefix: "/swagger",
-	}
-	swaggerUI := swagger.Handler(swaggerFiles.Handler, swaggerConfig)
-
-	app.Get("/swagger", swaggerUI)
-	// And the wildcard one for index.html, *.js, *.css and e.t.c.
-	app.Get("/swagger/{any:path}", swaggerUI)
-
-	// Start the Iris app and listen for incoming requests on port 80
-	err := app.Listen(":8080", func(app *iris.Application) {
-		routes := app.GetRoutes()
-		for _, route := range routes {
-			log.Println(route)
-		}
-	})
-
-	if err != nil {
-		logger.Get().Error("Failed starting webserver proof", zap.Error(err))
-	}
-}

metadata/metadata.go

@@ -0,0 +1,174 @@
+package metadata
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"git.lumeweb.com/LumeWeb/portal/db/models"
+
+	"go.uber.org/fx"
+	"gorm.io/gorm"
+)
+
+var ErrNotFound = gorm.ErrRecordNotFound
+
+var _ MetadataService = (*MetadataServiceDefault)(nil)
+
+type UploadMetadata struct {
+	ID         uint      `json:"upload_id"`
+	UserID     uint      `json:"user_id"`
+	Hash       []byte    `json:"hash"`
+	MimeType   string    `json:"mime_type"`
+	Protocol   string    `json:"protocol"`
+	UploaderIP string    `json:"uploader_ip"`
+	Size       uint64    `json:"size"`
+	Created    time.Time `json:"created"`
+}
+
+func (u UploadMetadata) IsEmpty() bool {
+	if u.UserID != 0 || u.MimeType != "" || u.Protocol != "" || u.UploaderIP != "" || u.Size != 0 {
+		return false
+	}
+
+	if !u.Created.IsZero() {
+		return false
+	}
+
+	if len(u.Hash) != 0 {
+		return false
+	}
+
+	return true
+}
+
+var Module = fx.Module("metadata",
+	fx.Provide(
+		fx.Annotate(
+			NewMetadataService,
+			fx.As(new(MetadataService)),
+		),
+	),
+)
+
+type MetadataService interface {
+	SaveUpload(ctx context.Context, metadata UploadMetadata, skipExisting bool) error
+	GetUpload(ctx context.Context, objectHash []byte) (UploadMetadata, error)
+	DeleteUpload(ctx context.Context, objectHash []byte) error
+}
+
+type MetadataServiceDefault struct {
+	db *gorm.DB
+}
+
+type MetadataServiceParams struct {
+	fx.In
+	Db *gorm.DB
+}
+
+func NewMetadataService(params MetadataServiceParams) *MetadataServiceDefault {
+	return &MetadataServiceDefault{
+		db: params.Db,
+	}
+}
+
+func (m *MetadataServiceDefault) SaveUpload(ctx context.Context, metadata UploadMetadata, skipExisting bool) error {
+	var upload models.Upload
+
+	upload.Hash = metadata.Hash
+
+	ret := m.db.WithContext(ctx).Model(&models.Upload{}).Where(&upload).First(&upload)
+
+	if ret.Error != nil {
+		if errors.Is(ret.Error, gorm.ErrRecordNotFound) {
+			return m.createUpload(ctx, metadata)
+		}
+		return ret.Error
+	}
+
+	if skipExisting {
+		return nil
+	}
+
+	changed := false
+
+	if upload.UserID != metadata.UserID {
+		upload.UserID = metadata.UserID
+		changed = true
+	}
+
+	if upload.MimeType != metadata.MimeType {
+		upload.MimeType = metadata.MimeType
+		changed = true
+	}
+
+	if upload.Protocol != metadata.Protocol {
+		upload.Protocol = metadata.Protocol
+		changed = true
+	}
+
+	if upload.UploaderIP != metadata.UploaderIP {
+		upload.UploaderIP = metadata.UploaderIP
+		changed = true
+	}
+
+	if upload.Size != metadata.Size {
+		upload.Size = metadata.Size
+		changed = true
+	}
+
+	if changed {
+		return m.db.Updates(&upload).Error
+	}
+
+	return nil
+}
+
+func (m *MetadataServiceDefault) createUpload(ctx context.Context, metadata UploadMetadata) error {
+	upload := models.Upload{
+		UserID:     metadata.UserID,
+		Hash:       metadata.Hash,
+		MimeType:   metadata.MimeType,
+		Protocol:   metadata.Protocol,
+		UploaderIP: metadata.UploaderIP,
+		Size:       metadata.Size,
+	}
+
+	return m.db.WithContext(ctx).Create(&upload).Error
+}
+
+func (m *MetadataServiceDefault) GetUpload(ctx context.Context, objectHash []byte) (UploadMetadata, error) {
+	var upload models.Upload
+
+	upload.Hash = objectHash
+
+	ret := m.db.WithContext(ctx).Model(&models.Upload{}).Where(&upload).First(&upload)
+
+	if ret.Error != nil {
+		return UploadMetadata{}, ret.Error
+	}
+
+	return UploadMetadata{
+		ID:         upload.ID,
+		UserID:     upload.UserID,
+		Hash:       upload.Hash,
+		MimeType:   upload.MimeType,
+		Protocol:   upload.Protocol,
+		UploaderIP: upload.UploaderIP,
+		Size:       upload.Size,
+	}, nil
+}
+
+func (m *MetadataServiceDefault) DeleteUpload(ctx context.Context, objectHash []byte) error {
+	var upload models.Upload
+
+	upload.Hash = objectHash
+
+	ret := m.db.WithContext(ctx).Model(&models.Upload{}).Where(&upload).First(&upload)
+
+	if ret.Error != nil {
+		return ret.Error
+	}
+
+	return m.db.Delete(&upload).Error
+}