LumeWeb
/
portal
1
0
Fork 0
Code Issues 5 Pull Requests Packages Projects Releases Wiki Activity

refactor: Prune old code base to prepare for rewrite

This commit is contained in:
Derrick Hammer 2024-01-11 14:49:50 -05:00
parent d1b0aa5139
commit c7bce2ff23
Signed by: pcfreak30
GPG Key ID: C997C339BE476FF2
41 changed files with 1 additions and 2837 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
bao/bao.go
View File

@ -1,31 +0,0 @@
package bao
import (
"bufio"
_ "embed"
"io"
"lukechampine.com/blake3"
)
func ComputeTree(reader io.Reader, size int64) ([]byte, [32]byte, error) {
bufSize := blake3.BaoEncodedSize(int(size), true)
buf := bufferAt{buf: make([]byte, bufSize)}
hash, err := blake3.BaoEncode(&buf, bufio.NewReader(reader), size, true)
if err != nil {
return nil, [32]byte{}, err
}
return buf.buf, hash, nil
}
type bufferAt struct {
buf []byte
}
func (b *bufferAt) WriteAt(p []byte, off int64) (int, error) {
if copy(b.buf[off:], p) != len(p) {
panic("bad buffer size")
}
return len(p), nil
}

100
cid/cid.go
View File

@ -1,100 +0,0 @@
package cid
import (
"bytes"
"encoding/binary"
"encoding/hex"
"errors"
"github.com/multiformats/go-multibase"
)
var MAGIC_BYTES = []byte{0x26, 0x1f}
var (
ErrMissingEmptySize = errors.New("Missing or empty size")
ErrInvalidCIDMagic = errors.New("CID magic bytes missing or invalid")
)
type CID struct {
Hash [32]byte
Size uint64
}
func (c CID) StringHash() string {
return hex.EncodeToString(c.Hash[:])
}
func Encode(hash []byte, size uint64) (string, error) {
var hashBytes [32]byte
copy(hashBytes[:], hash)
return EncodeFixed(hashBytes, size)
}
func EncodeFixed(hash [32]byte, size uint64) (string, error) {
sizeBytes := make([]byte, 8)
binary.LittleEndian.PutUint64(sizeBytes, size)
prefixedHash := append(MAGIC_BYTES, hash[:]...)
prefixedHash = append(prefixedHash, sizeBytes...)
return multibase.Encode(multibase.Base58BTC, prefixedHash)
}
func EncodeString(hash string, size uint64) (string, error) {
hashBytes, err := hex.DecodeString(hash)
if err != nil {
return "", err
}
return Encode(hashBytes, size)
}
func Valid(cid string) (bool, error) {
_, err := maybeDecode(cid)
if err != nil {
return false, err
}
return true, nil
}
func Decode(cid string) (*CID, error) {
data, err := maybeDecode(cid)
if err != nil {
return &CID{}, err
}
data = data[len(MAGIC_BYTES):]
var hash [32]byte
copy(hash[:], data[:])
size := binary.LittleEndian.Uint64(data[32:])
return &CID{Hash: hash, Size: size}, nil
}
func maybeDecode(cid string) ([]byte, error) {
_, data, err := multibase.Decode(cid)
if err != nil {
return nil, err
}
if bytes.Compare(data[0:len(MAGIC_BYTES)], MAGIC_BYTES) != 0 {
return nil, ErrInvalidCIDMagic
}
sizeBytes := data[len(MAGIC_BYTES)+32:]
if len(sizeBytes) == 0 {
return nil, ErrMissingEmptySize
}
size := binary.LittleEndian.Uint64(sizeBytes)
if size == 0 {
return nil, ErrMissingEmptySize
}
return data, nil
}

59
config/config.go
View File

@ -1,59 +0,0 @@
package config
import (
"errors"
"fmt"
"github.com/spf13/pflag"
"github.com/spf13/viper"
"log"
)
var (
ConfigFilePaths = []string{
"/etc/lumeweb/portal/",
"$HOME/.lumeweb/portal/",
".",
}
)
func Init() {
viper.SetConfigName("config")
viper.SetConfigType("json")
for _, path := range ConfigFilePaths {
viper.AddConfigPath(path)
}
viper.SetEnvPrefix("LUME_WEB_PORTAL")
viper.AutomaticEnv()
pflag.String("database.type", "sqlite", "Database type")
pflag.String("database.host", "localhost", "Database host")
pflag.Int("database.port", 3306, "Database port")
pflag.String("database.user", "root", "Database user")
pflag.String("database.password", "", "Database password")
pflag.String("database.name", "lumeweb_portal", "Database name")
pflag.String("database.path", "./db.sqlite", "Database path for SQLite")
pflag.String("renterd-api-password", ".", "admin password for renterd")
pflag.Bool("debug", false, "enable debug mode")
pflag.Parse()
err := viper.BindPFlags(pflag.CommandLine)
if err != nil {
log.Fatalf("Fatal error arguments: %s \n", err)
return
}
err = viper.ReadInConfig()
if err != nil {
if errors.As(err, &viper.ConfigFileNotFoundError{}) {
// Config file not found, this is not an error.
fmt.Println("Config file not found, using default settings.")
} else {
// Other error, panic.
panic(fmt.Errorf("Fatal error config file: %s \n", err))
}
}
}

35
controller/account.go
View File

@ -1,35 +0,0 @@
package controller
import (
"git.lumeweb.com/LumeWeb/portal/controller/request"
"git.lumeweb.com/LumeWeb/portal/service/account"
"github.com/kataras/iris/v12"
)
type AccountController struct {
Controller
}
func (a *AccountController) PostRegister() {
ri, success := tryParseRequest(request.RegisterRequest{}, a.Ctx)
if !success {
return
}
r, _ := ri.(*request.RegisterRequest)
err := account.Register(r.Email, r.Password, r.Pubkey)
if err != nil {
if err == account.ErrQueryingAcct || err == account.ErrFailedCreateAccount {
a.Ctx.StopWithError(iris.StatusInternalServerError, err)
} else {
a.Ctx.StopWithError(iris.StatusBadRequest, err)
}
return
}
// Return a success response to the client.
a.Ctx.StatusCode(iris.StatusCreated)
}

112
controller/auth.go
View File

@ -1,112 +0,0 @@
package controller
import (
"git.lumeweb.com/LumeWeb/portal/controller/request"
"git.lumeweb.com/LumeWeb/portal/controller/response"
"git.lumeweb.com/LumeWeb/portal/middleware"
"git.lumeweb.com/LumeWeb/portal/service/auth"
"github.com/kataras/iris/v12"
)
type AuthController struct {
Controller
}
// PostLogin handles the POST /api/auth/login request to authenticate a user and return a JWT token.
func (a *AuthController) PostLogin() {
ri, success := tryParseRequest(request.LoginRequest{}, a.Ctx)
if !success {
return
}
r, _ := ri.(*request.LoginRequest)
token, err := auth.LoginWithPassword(r.Email, r.Password)
if err != nil {
if err == auth.ErrFailedGenerateToken {
a.Ctx.StopWithError(iris.StatusInternalServerError, err)
} else {
a.Ctx.StopWithError(iris.StatusUnauthorized, err)
}
return
}
a.respondJSON(&response.LoginResponse{Token: token})
}
// PostChallenge handles the POST /api/auth/pubkey/challenge request to generate a challenge for a user's public key.
func (a *AuthController) PostPubkeyChallenge() {
ri, success := tryParseRequest(request.PubkeyChallengeRequest{}, a.Ctx)
if !success {
return
}
r, _ := (ri).(*request.PubkeyChallengeRequest)
challenge, err := auth.GeneratePubkeyChallenge(r.Pubkey)
if err != nil {
if err == auth.ErrFailedGenerateKeyChallenge {
a.Ctx.StopWithError(iris.StatusInternalServerError, err)
} else {
a.Ctx.StopWithError(iris.StatusUnauthorized, err)
}
return
}
a.respondJSON(&response.ChallengeResponse{Challenge: challenge})
}
// PostKeyLogin handles the POST /api/auth/pubkey/login request to authenticate a user using a public key challenge and return a JWT token.
func (a *AuthController) PostPubkeyLogin() {
ri, success := tryParseRequest(request.PubkeyLoginRequest{}, a.Ctx)
if !success {
return
}
r, _ := ri.(*request.PubkeyLoginRequest)
token, err := auth.LoginWithPubkey(r.Pubkey, r.Challenge, r.Signature)
if err != nil {
if err == auth.ErrFailedGenerateKeyChallenge || err == auth.ErrFailedGenerateToken || err == auth.ErrFailedSaveToken {
a.Ctx.StopWithError(iris.StatusInternalServerError, err)
} else {
a.Ctx.StopWithError(iris.StatusUnauthorized, err)
}
return
}
a.respondJSON(&response.LoginResponse{Token: token})
}
// PostLogout handles the POST /api/auth/logout request to invalidate a JWT token.
func (a *AuthController) PostLogout() {
ri, success := tryParseRequest(request.LogoutRequest{}, a.Ctx)
if !success {
return
}
r, _ := ri.(*request.LogoutRequest)
err := auth.Logout(r.Token)
if err != nil {
a.Ctx.StopWithError(iris.StatusBadRequest, err)
return
}
// Return a success response to the client.
a.Ctx.StatusCode(iris.StatusNoContent)
}
func (a *AuthController) GetStatus() {
middleware.VerifyJwt(a.Ctx)
if a.Ctx.IsStopped() {
return
}
a.respondJSON(&response.AuthStatusResponse{Status: true})
}

72
controller/controller.go
View File

@ -1,72 +0,0 @@
package controller
import (
"git.lumeweb.com/LumeWeb/portal/controller/validators"
"git.lumeweb.com/LumeWeb/portal/logger"
"github.com/kataras/iris/v12"
"go.uber.org/zap"
)
func tryParseRequest(r interface{}, ctx iris.Context) (interface{}, bool) {
v, ok := r.(validators.Validatable)
if !ok {
return r, true
}
var d map[string]interface{}
// Read the logout request from the client.
if err := ctx.ReadJSON(&d); err != nil {
logger.Get().Debug("failed to parse request", zap.Error(err))
ctx.StopWithError(iris.StatusBadRequest, err)
return nil, false
}
data, err := v.Import(d)
if err != nil {
logger.Get().Debug("failed to parse request", zap.Error(err))
ctx.StopWithError(iris.StatusBadRequest, err)
return nil, false
}
if err := data.Validate(); err != nil {
logger.Get().Debug("failed to parse request", zap.Error(err))
ctx.StopWithError(iris.StatusBadRequest, err)
return nil, false
}
return data, true
}
func sendErrorCustom(ctx iris.Context, err error, customError error, irisError int) bool {
if err != nil {
if customError != nil {
err = customError
}
ctx.StopWithError(irisError, err)
return true
}
return false
}
func InternalError(ctx iris.Context, err error) bool {
return sendErrorCustom(ctx, err, nil, iris.StatusInternalServerError)
}
func internalErrorCustom(ctx iris.Context, err error, customError error) bool {
return sendErrorCustom(ctx, err, customError, iris.StatusInternalServerError)
}
func SendError(ctx iris.Context, err error, irisError int) bool {
return sendErrorCustom(ctx, err, nil, irisError)
}
type Controller struct {
Ctx iris.Context
}
func (c Controller) respondJSON(data interface{}) {
err := c.Ctx.JSON(data)
if err != nil {
logger.Get().Error("failed to generate response", zap.Error(err))
}
}

213
controller/files.go
View File

@ -1,213 +0,0 @@
package controller
import (
"errors"
"git.lumeweb.com/LumeWeb/portal/cid"
"git.lumeweb.com/LumeWeb/portal/controller/response"
"git.lumeweb.com/LumeWeb/portal/logger"
"git.lumeweb.com/LumeWeb/portal/middleware"
"git.lumeweb.com/LumeWeb/portal/service/auth"
"git.lumeweb.com/LumeWeb/portal/service/files"
"github.com/kataras/iris/v12"
"go.uber.org/zap"
"io"
)
var ErrStreamDone = errors.New("done")
type FilesController struct {
Controller
}
func (f *FilesController) BeginRequest(ctx iris.Context) {
middleware.VerifyJwt(ctx)
}
func (f *FilesController) EndRequest(ctx iris.Context) {
}
func (f *FilesController) PostUpload() {
ctx := f.Ctx
file, meta, err := f.Ctx.FormFile("file")
if internalErrorCustom(ctx, err, errors.New("invalid file data")) {
logger.Get().Debug("invalid file data", zap.Error(err))
return
}
upload, err := files.Upload(file, meta.Size, nil, auth.GetCurrentUserId(ctx))
if InternalError(ctx, err) {
logger.Get().Debug("failed uploading file", zap.Error(err))
return
}
err = files.Pin(upload.Hash, upload.AccountID)
if InternalError(ctx, err) {
logger.Get().Debug("failed pinning file", zap.Error(err))
return
}
cidString, err := cid.EncodeString(upload.Hash, uint64(meta.Size))
if InternalError(ctx, err) {
logger.Get().Debug("failed creating cid", zap.Error(err))
return
}
err = ctx.JSON(&response.UploadResponse{Cid: cidString})
if err != nil {
logger.Get().Error("failed to create response", zap.Error(err))
}
}
func (f *FilesController) GetDownloadBy(cidString string) {
ctx := f.Ctx
hashHex, valid := ValidateCid(cidString, true, ctx)
if !valid {
return
}
download, err := files.Download(hashHex)
if InternalError(ctx, err) {
logger.Get().Debug("failed fetching file", zap.Error(err))
return
}
err = PassThroughStream(download, ctx)
if err != ErrStreamDone && InternalError(ctx, err) {
logger.Get().Debug("failed streaming file", zap.Error(err))
}
}
func (f *FilesController) GetProofBy(cidString string) {
ctx := f.Ctx
hashHex, valid := ValidateCid(cidString, true, ctx)
if !valid {
return
}
proof, err := files.DownloadProof(hashHex)
if InternalError(ctx, err) {
logger.Get().Debug("failed fetching file proof", zap.Error(err))
return
}
err = PassThroughStream(proof, ctx)
if InternalError(ctx, err) {
logger.Get().Debug("failed streaming file proof", zap.Error(err))
}
}
func (f *FilesController) GetStatusBy(cidString string) {
ctx := f.Ctx
hashHex, valid := ValidateCid(cidString, false, ctx)
if !valid {
return
}
status := files.Status(hashHex)
var statusCode string
switch status {
case files.STATUS_UPLOADED:
statusCode = "uploaded"
break
case files.STATUS_UPLOADING:
statusCode = "uploading"
break
case files.STATUS_NOT_FOUND:
statusCode = "not_found"
break
}
f.respondJSON(&response.FileStatusResponse{Status: statusCode})
}
func (f *FilesController) PostPinBy(cidString string) {
ctx := f.Ctx
hashHex, valid := ValidateCid(cidString, true, ctx)
if !valid {
return
}
err := files.Pin(hashHex, auth.GetCurrentUserId(ctx))
if InternalError(ctx, err) {
logger.Get().Error(err.Error())
return
}
f.Ctx.StatusCode(iris.StatusCreated)
}
func (f *FilesController) GetUploadLimit() {
f.respondJSON(&response.UploadLimit{Limit: f.Ctx.Application().ConfigurationReadOnly().GetPostMaxMemory()})
}
func ValidateCid(cidString string, validateStatus bool, ctx iris.Context) (string, bool) {
_, err := cid.Valid(cidString)
if SendError(ctx, err, iris.StatusBadRequest) {
logger.Get().Debug("invalid cid", zap.Error(err))
return "", false
}
cidObject, _ := cid.Decode(cidString)
hashHex := cidObject.StringHash()
if validateStatus {
status := files.Status(hashHex)
if status == files.STATUS_NOT_FOUND {
err := errors.New("cid not found")
SendError(ctx, errors.New("cid not found"), iris.StatusNotFound)
logger.Get().Debug("cid not found", zap.Error(err))
return "", false
}
}
return hashHex, true
}
func PassThroughStream(stream io.Reader, ctx iris.Context) error {
closed := false
err := ctx.StreamWriter(func(w io.Writer) error {
if closed {
return ErrStreamDone
}
count, err := io.CopyN(w, stream, 1024)
if count == 0 || err == io.EOF {
err = stream.(io.Closer).Close()
if err != nil {
logger.Get().Error("failed closing stream", zap.Error(err))
return err
}
closed = true
return nil
}
if err != nil {
return err
}
return nil
})
if err == ErrStreamDone {
err = nil
}
return err
}

23
controller/request/login.go
View File

@ -1,23 +0,0 @@
package request
import (
"git.lumeweb.com/LumeWeb/portal/controller/validators"
validation "github.com/go-ozzo/ozzo-validation/v4"
"github.com/go-ozzo/ozzo-validation/v4/is"
)
type LoginRequest struct {
validatable validators.ValidatableImpl
Email string `json:"email"`
Password string `json:"password"`
}
func (r LoginRequest) Validate() error {
return validation.ValidateStruct(&r,
validation.Field(&r.Email, is.EmailFormat, validation.Required),
validation.Field(&r.Password, validation.Required),
)
}
func (r LoginRequest) Import(d map[string]interface{}) (validators.Validatable, error) {
return r.validatable.Import(d, r)
}

19
controller/request/logout.go
View File

@ -1,19 +0,0 @@
package request
import (
"git.lumeweb.com/LumeWeb/portal/controller/validators"
validation "github.com/go-ozzo/ozzo-validation/v4"
)
type LogoutRequest struct {
validatable validators.ValidatableImpl
Token string `json:"token"`
}
func (r LogoutRequest) Validate() error {
return validation.ValidateStruct(&r, validation.Field(&r.Token, validation.Required))
}
func (r LogoutRequest) Import(d map[string]interface{}) (validators.Validatable, error) {
return r.validatable.Import(d, r)
}

21
controller/request/pubkey_challenge.go
View File

@ -1,21 +0,0 @@
package request
import (
"git.lumeweb.com/LumeWeb/portal/controller/validators"
validation "github.com/go-ozzo/ozzo-validation/v4"
)
type PubkeyChallengeRequest struct {
validatable validators.ValidatableImpl
Pubkey string `json:"pubkey"`
}
func (r PubkeyChallengeRequest) Validate() error {
return validation.ValidateStruct(&r,
validation.Field(&r.Pubkey, validation.Required, validation.By(validators.CheckPubkeyValidator)),
)
}
func (r PubkeyChallengeRequest) Import(d map[string]interface{}) (validators.Validatable, error) {
return r.validatable.Import(d, r)
}

25
controller/request/pubkey_login.go
View File

@ -1,25 +0,0 @@
package request
import (
"git.lumeweb.com/LumeWeb/portal/controller/validators"
validation "github.com/go-ozzo/ozzo-validation/v4"
)
type PubkeyLoginRequest struct {
validatable validators.ValidatableImpl
Pubkey string `json:"pubkey"`
Challenge string `json:"challenge"`
Signature string `json:"signature"`
}
func (r PubkeyLoginRequest) Validate() error {
return validation.ValidateStruct(&r,
validation.Field(&r.Pubkey, validation.Required, validation.By(validators.CheckPubkeyValidator)),
validation.Field(&r.Challenge, validation.Required),
validation.Field(&r.Signature, validation.Required, validation.Length(128, 128)),
)
}
func (r PubkeyLoginRequest) Import(d map[string]interface{}) (validators.Validatable, error) {
return r.validatable.Import(d, r)
}

25
controller/request/register.go
View File

@ -1,25 +0,0 @@
package request
import (
"git.lumeweb.com/LumeWeb/portal/controller/validators"
validation "github.com/go-ozzo/ozzo-validation/v4"
"github.com/go-ozzo/ozzo-validation/v4/is"
)
type RegisterRequest struct {
validatable validators.ValidatableImpl
Email string `json:"email"`
Password string `json:"password"`
Pubkey string `json:"pubkey"`
}
func (r RegisterRequest) Validate() error {
return validation.ValidateStruct(&r,
validation.Field(&r.Email, validation.Required, is.EmailFormat),
validation.Field(&r.Pubkey, validation.When(len(r.Password) == 0, validation.Required, validation.By(validators.CheckPubkeyValidator))),
validation.Field(&r.Password, validation.When(len(r.Pubkey) == 0, validation.Required)),
)
}
func (r RegisterRequest) Import(d map[string]interface{}) (validators.Validatable, error) {
return r.validatable.Import(d, r)
}

5
controller/response/auth_status.go
View File

@ -1,5 +0,0 @@
package response
type AuthStatusResponse struct {
Status bool `json:"status"`
}

5
controller/response/challenge.go
View File

@ -1,5 +0,0 @@
package response
type ChallengeResponse struct {
Challenge string `json:"challenge"`
}

5
controller/response/file_status.go
View File

@ -1,5 +0,0 @@
package response
type FileStatusResponse struct {
Status string `json:"status"`
}

5
controller/response/login.go
View File

@ -1,5 +0,0 @@
package response
type LoginResponse struct {
Token string `json:"token"`
}

5
controller/response/upload.go
View File

@ -1,5 +0,0 @@
package response
type UploadResponse struct {
Cid string `json:"cid"`
}

5
controller/response/upload_limit.go
View File

@ -1,5 +0,0 @@
package response
type UploadLimit struct {
Limit int64 `json:"limit"`
}

43
controller/validators/validators.go
View File

@ -1,43 +0,0 @@
package validators
import (
"crypto/ed25519"
"encoding/hex"
"errors"
"fmt"
validation "github.com/go-ozzo/ozzo-validation/v4"
"github.com/imdario/mergo"
"reflect"
)
func CheckPubkeyValidator(value interface{}) error {
p, _ := value.(string)
pubkeyBytes, err := hex.DecodeString(p)
if err != nil {
return err
}
if len(pubkeyBytes) != ed25519.PublicKeySize {
return errors.New(fmt.Sprintf("pubkey must be %d bytes in hexadecimal format", ed25519.PublicKeySize))
}
return nil
}
type Validatable interface {
validation.Validatable
Import(d map[string]interface{}) (Validatable, error)
}
type ValidatableImpl struct {
}
func (v ValidatableImpl) Import(d map[string]interface{}, destType Validatable) (Validatable, error) {
instance := reflect.New(reflect.TypeOf(destType)).Interface().(Validatable)
// Perform the import logic
if err := mergo.Map(instance, d, mergo.WithOverride); err != nil {
return nil, err
}
return instance, nil
}

71
db/db.go
View File

@ -1,71 +0,0 @@
package db
import (
"fmt"
"git.lumeweb.com/LumeWeb/portal/model"
"github.com/spf13/viper"
"gorm.io/driver/mysql"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
// Declare a global variable to hold the database connection.
var db *gorm.DB
// Init initializes the database connection based on the app's configuration settings.
func Init() {
// If the database connection has already been initialized, panic.
if db != nil {
panic("DB already initialized")
}
// Retrieve database connection settings from the app's configuration using the viper library.
dbType := viper.GetString("database.type")
dbHost := viper.GetString("database.host")
dbPort := viper.GetInt("database.port")
dbSocket := viper.GetString("database.socket")
dbUser := viper.GetString("database.user")
dbPassword := viper.GetString("database.password")
dbName := viper.GetString("database.name")
dbPath := viper.GetString("database.path")
var err error
var dsn string
switch dbType {
// Connect to a MySQL database.
case "mysql":
if dbSocket != "" {
dsn = fmt.Sprintf("%s:%s@unix(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", dbUser, dbPassword, dbSocket, dbName)
} else {
dsn = fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local", dbUser, dbPassword, dbHost, dbPort, dbName)
}
db, err = gorm.Open(mysql.Open(dsn), &gorm.Config{})
// Connect to a SQLite database.
case "sqlite":
db, err = gorm.Open(sqlite.Open(dbPath), &gorm.Config{})
// If the database type is unsupported, panic.
default:
panic(fmt.Errorf("Unsupported database type: %s \n", dbType))
}
// If there was an error connecting to the database, panic.
if err != nil {
panic(fmt.Errorf("Failed to connect to database: %s \n", err))
}
// Automatically migrate the database schema based on the model definitions.
err = db.Migrator().AutoMigrate(&model.Account{}, &model.Key{}, &model.KeyChallenge{}, &model.LoginSession{}, &model.Upload{}, &model.Pin{}, &model.Tus{}, &model.Dnslink{})
if err != nil {
panic(fmt.Errorf("Database setup failed database type: %s \n", err))
}
}
// Get returns the database connection instance.
func Get() *gorm.DB {
return db
}
func Close() error {
instance, _ := db.DB()
return instance.Close()
}

227
dnslink/dnslink.go
View File

@ -1,227 +0,0 @@
package dnslink
import (
"errors"
"git.lumeweb.com/LumeWeb/portal/cid"
"git.lumeweb.com/LumeWeb/portal/controller"
"git.lumeweb.com/LumeWeb/portal/db"
"git.lumeweb.com/LumeWeb/portal/logger"
"git.lumeweb.com/LumeWeb/portal/model"
"git.lumeweb.com/LumeWeb/portal/service/files"
dnslink "github.com/dnslink-std/go"
"github.com/kataras/iris/v12"
"github.com/kataras/iris/v12/context"
"github.com/vmihailenco/msgpack/v5"
"go.uber.org/zap"
"io"
"path/filepath"
"strings"
)
var (
ErrFailedReadAppManifest = errors.New("failed to read app manifest")
ErrInvalidAppManifest = errors.New("invalid app manifest")
)
type CID string
type ExtraMetadata map[string]interface{}
type WebAppMetadata struct {
Schema string `msgpack:"$schema,omitempty"`
Type string `msgpack:"type"`
Name string `msgpack:"name,omitempty"`
TryFiles []string `msgpack:"tryFiles,omitempty"`
ErrorPages map[string]string `msgpack:"errorPages,omitempty"`
Paths map[string]PathContent `msgpack:"paths"`
ExtraMetadata ExtraMetadata `msgpack:"extraMetadata,omitempty"`
}
type PathContent struct {
CID CID `msgpack:"cid"`
ContentType string `msgpack:"contentType,omitempty"`
}
func Handler(ctx *context.Context) {
record := model.Dnslink{}
domain := ctx.Request().Host
if err := db.Get().Model(&model.Dnslink{Domain: domain}).First(&record).Error; err != nil {
ctx.StopWithStatus(iris.StatusNotFound)
return
}
ret, err := dnslink.Resolve(domain)
if err != nil {
switch e := err.(type) {
default:
ctx.StopWithStatus(iris.StatusInternalServerError)
return
case dnslink.DNSRCodeError:
if e.DNSRCode == 3 {
ctx.StopWithStatus(iris.StatusNotFound)
return
}
}
}
if ret.Links["sia"] == nil || len(ret.Links["sia"]) == 0 {
ctx.StopWithStatus(iris.StatusNotFound)
return
}
appManifest := ret.Links["sia"][0]
decodedCid, valid := controller.ValidateCid(appManifest.Identifier, true, ctx)
if !valid {
return
}
manifest := fetchManifest(ctx, decodedCid)
if manifest == nil {
return
}
path := ctx.Path()
if strings.HasSuffix(path, "/") || filepath.Ext(path) == "" {
var directoryIndex *PathContent
for _, indexFile := range manifest.TryFiles {
path, exists := manifest.Paths[indexFile]
if !exists {
continue
}
_, err := cid.Valid(string(manifest.Paths[indexFile].CID))
if err != nil {
continue
}
cidObject, _ := cid.Decode(string(path.CID))
hashHex := cidObject.StringHash()
status := files.Status(hashHex)
if status == files.STATUS_NOT_FOUND {
continue
}
if status == files.STATUS_UPLOADED {
directoryIndex = &path
break
}
}
if directoryIndex == nil {
ctx.StopWithStatus(iris.StatusNotFound)
return
}
file, err := fetchFile(directoryIndex)
if maybeHandleFileError(err, ctx) {
return
}
ctx.Header("Content-Type", directoryIndex.ContentType)
streamFile(file, ctx)
return
}
path = strings.TrimLeft(path, "/")
requestedPath, exists := manifest.Paths[path]
if !exists {
ctx.StopWithStatus(iris.StatusNotFound)
return
}
file, err := fetchFile(&requestedPath)
if maybeHandleFileError(err, ctx) {
return
}
ctx.Header("Content-Type", requestedPath.ContentType)
streamFile(file, ctx)
}
func maybeHandleFileError(err error, ctx *context.Context) bool {
if err != nil {
if err == files.ErrInvalidFile {
controller.SendError(ctx, err, iris.StatusNotFound)
return true
}
controller.SendError(ctx, err, iris.StatusInternalServerError)
}
return err != nil
}
func streamFile(stream io.Reader, ctx *context.Context) {
err := controller.PassThroughStream(stream, ctx)
if err != controller.ErrStreamDone && controller.InternalError(ctx, err) {
logger.Get().Debug("failed streaming file", zap.Error(err))
}
}
func fetchFile(path *PathContent) (io.Reader, error) {
_, err := cid.Valid(string(path.CID))
if err != nil {
return nil, err
}
cidObject, _ := cid.Decode(string(path.CID))
hashHex := cidObject.StringHash()
status := files.Status(hashHex)
if status == files.STATUS_NOT_FOUND {
return nil, errors.New("cid not found")
}
if status == files.STATUS_UPLOADED {
stream, err := files.Download(hashHex)
if err != nil {
return nil, err
}
return stream, nil
}
return nil, errors.New("cid not found")
}
func fetchManifest(ctx iris.Context, hash string) *WebAppMetadata {
stream, err := files.Download(hash)
if err != nil {
if errors.Is(err, files.ErrInvalidFile) {
controller.SendError(ctx, err, iris.StatusNotFound)
return nil
}
controller.SendError(ctx, err, iris.StatusInternalServerError)
}
var metadata WebAppMetadata
data, err := io.ReadAll(stream)
if err != nil {
logger.Get().Debug(ErrFailedReadAppManifest.Error(), zap.Error(err))
controller.SendError(ctx, ErrFailedReadAppManifest, iris.StatusInternalServerError)
return nil
}
err = msgpack.Unmarshal(data, &metadata)
if err != nil {
logger.Get().Debug(ErrFailedReadAppManifest.Error(), zap.Error(err))
controller.SendError(ctx, ErrFailedReadAppManifest, iris.StatusInternalServerError)
return nil
}
if metadata.Type != "web_app" {
logger.Get().Debug(ErrInvalidAppManifest.Error())
controller.SendError(ctx, ErrInvalidAppManifest, iris.StatusInternalServerError)
return nil
}
return &metadata
}

127
go.mod
View File

@ -1,128 +1,3 @@
module git.lumeweb.com/LumeWeb/portal
go 1.18
require (
github.com/dnslink-std/go v0.6.0
github.com/go-ozzo/ozzo-validation/v4 v4.3.0
github.com/go-resty/resty/v2 v2.7.0
github.com/golang-queue/queue v0.1.3
github.com/huandu/go-clone v1.6.0
github.com/imdario/mergo v0.3.16
github.com/iris-contrib/swagger v0.0.0-20230531125653-f4ee631290a7
github.com/kataras/iris/v12 v12.2.0
github.com/kataras/jwt v0.1.8
github.com/multiformats/go-multibase v0.2.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.16.0
github.com/swaggo/swag v1.16.1
github.com/tus/tusd v1.11.0
github.com/vmihailenco/msgpack/v5 v5.3.5
go.uber.org/zap v1.24.0
golang.org/x/crypto v0.10.0
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df
gorm.io/driver/mysql v1.5.1
gorm.io/driver/sqlite v1.5.2
gorm.io/gorm v1.25.2
lukechampine.com/blake3 v1.2.1
)
require (
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 // indirect
github.com/CloudyKit/jet/v6 v6.2.0 // indirect
github.com/Joker/jade v1.1.3 // indirect
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06 // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/bmizerany/pat v0.0.0-20210406213842-e4b6760bdd6f // indirect
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 // indirect
github.com/fatih/structs v1.1.0 // indirect
github.com/flosch/pongo2/v4 v4.0.2 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/spec v0.20.9 // indirect
github.com/go-openapi/swag v0.22.4 // indirect
github.com/go-sql-driver/mysql v1.7.1 // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.2.1 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/css v1.0.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/iris-contrib/go.uuid v2.0.0+incompatible // indirect
github.com/iris-contrib/schema v0.0.6 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/kataras/blocks v0.0.7 // indirect
github.com/kataras/golog v0.1.9 // indirect
github.com/kataras/neffos v0.0.21 // indirect
github.com/kataras/pio v0.0.12 // indirect
github.com/kataras/sitemap v0.0.6 // indirect
github.com/kataras/tunnel v0.0.4 // indirect
github.com/klauspost/compress v1.16.6 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mailgun/raymond/v2 v2.0.48 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-sqlite3 v1.14.17 // indirect
github.com/mediocregopher/radix/v3 v3.8.1 // indirect
github.com/microcosm-cc/bluemonday v1.0.24 // indirect
github.com/miekg/dns v1.1.43 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-base32 v0.1.0 // indirect
github.com/multiformats/go-base36 v0.2.0 // indirect
github.com/nats-io/nats.go v1.27.1 // indirect
github.com/nats-io/nkeys v0.4.4 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/schollz/closestmatch v2.1.0+incompatible // indirect
github.com/sergi/go-diff v1.1.0 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spf13/afero v1.9.5 // indirect
github.com/spf13/cast v1.5.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
github.com/tdewolff/minify/v2 v2.12.7 // indirect
github.com/tdewolff/parse/v2 v2.6.6 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/yosssi/ace v0.0.5 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/net v0.11.0 // indirect
golang.org/x/sys v0.9.0 // indirect
golang.org/x/text v0.10.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.10.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
replace go.uber.org/multierr => go.uber.org/multierr v1.9.0
replace (
github.com/tus/tusd => git.lumeweb.com/LumeWeb/tusd v1.11.1-0.20230629085530-7b20ce6a9ae5
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.39.0
go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.14.0
go.opentelemetry.io/otel/exporters/otlp/internal/retry => go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.12.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace => go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.12.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp => go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.12.0
go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v0.37.0
go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v1.12.0
go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.14.0
go.opentelemetry.io/proto/otlp => go.opentelemetry.io/proto/otlp v0.19.0
)
go 1.20

30
logger/logger.go
View File

@ -1,30 +0,0 @@
package logger
import (
"github.com/spf13/viper"
"go.uber.org/zap"
"log"
)
var logger *zap.Logger
func Init() {
var newLogger *zap.Logger
var err error
if viper.GetBool("debug") {
newLogger, err = zap.NewDevelopment()
} else {
newLogger, err = zap.NewProduction()
}
if err != nil {
log.Fatal(err)
}
logger = newLogger
}
func Get() *zap.Logger {
return logger
}

28
middleware/jwt.go
View File

@ -1,28 +0,0 @@
package middleware
import (
"git.lumeweb.com/LumeWeb/portal/service/account"
"git.lumeweb.com/LumeWeb/portal/service/auth"
"github.com/kataras/iris/v12"
)
func VerifyJwt(ctx iris.Context) {
token := auth.GetRequestAuthCode(ctx)
if len(token) == 0 {
ctx.StopWithError(iris.StatusUnauthorized, auth.ErrInvalidToken)
return
}
acct, err := auth.VerifyLoginToken(token)
if err != nil {
ctx.StopWithError(iris.StatusUnauthorized, auth.ErrInvalidToken)
return
}
err = ctx.SetUser(account.NewUser(acct))
if err != nil {
ctx.StopWithError(iris.StatusInternalServerError, err)
}
}

17
model/account.go
View File

@ -1,17 +0,0 @@
package model
import (
"gorm.io/gorm"
"time"
)
type Account struct {
gorm.Model
ID uint `gorm:"primaryKey" gorm:"AUTO_INCREMENT"`
Email string `gorm:"uniqueIndex"`
Password *string
CreatedAt time.Time
UpdatedAt time.Time
LoginTokens []LoginSession
Keys []Key
}

11
model/dnslink.go
View File

@ -1,11 +0,0 @@
package model
import (
"gorm.io/gorm"
)
type Dnslink struct {
gorm.Model
ID uint `gorm:"primaryKey" gorm:"AUTO_INCREMENT"`
Domain string `gorm:"uniqueIndex"`
}

16
model/key.go
View File

@ -1,16 +0,0 @@
package model
import (
"gorm.io/gorm"
"time"
)
type Key struct {
gorm.Model
ID uint `gorm:"primaryKey" gorm:"AUTO_INCREMENT"`
AccountID uint
Account Account
Pubkey string
CreatedAt time.Time
UpdatedAt time.Time
}

15
model/key_challenge.go
View File

@ -1,15 +0,0 @@
package model
import (
"gorm.io/gorm"
"time"
)
type KeyChallenge struct {
gorm.Model
ID uint `gorm:"primaryKey" gorm:"AUTO_INCREMENT"`
AccountID uint
Account Account
Challenge string `gorm:"not null"`
Expiration time.Time
}

22
model/login_session.go
View File

@ -1,22 +0,0 @@
package model
import (
"gorm.io/gorm"
"time"
)
type LoginSession struct {
gorm.Model
ID uint `gorm:"primaryKey" gorm:"AUTO_INCREMENT"`
AccountID uint
Account Account
Token string `gorm:"uniqueIndex"`
Expiration time.Time
CreatedAt time.Time
UpdatedAt time.Time
}
func (s *LoginSession) BeforeCreate(tx *gorm.DB) (err error) {
s.Expiration = time.Now().Add(time.Hour * 24)
return
}

12
model/pin.go
View File

@ -1,12 +0,0 @@
package model
import "gorm.io/gorm"
type Pin struct {
gorm.Model
ID uint `gorm:"primaryKey" gorm:"AUTO_INCREMENT"`
AccountID uint `gorm:"uniqueIndex:idx_account_upload"`
UploadID uint `gorm:"uniqueIndex:idx_account_upload"`
Account Account
Upload Upload
}

15
model/tus.go
View File

@ -1,15 +0,0 @@
package model
import (
"gorm.io/gorm"
)
type Tus struct {
gorm.Model
ID uint `gorm:"primaryKey" gorm:"AUTO_INCREMENT"`
UploadID string
Hash string
Info string
AccountID uint
Account Account
}

13
model/upload.go
View File

@ -1,13 +0,0 @@
package model
import (
"gorm.io/gorm"
)
type Upload struct {
gorm.Model
ID uint `gorm:"primaryKey" gorm:"AUTO_INCREMENT"`
AccountID uint `gorm:"index"`
Account Account
Hash string `gorm:"uniqueIndex"`
}

76
service/account/account.go
View File

@ -1,76 +0,0 @@
package account
import (
"errors"
"git.lumeweb.com/LumeWeb/portal/db"
"git.lumeweb.com/LumeWeb/portal/logger"
"git.lumeweb.com/LumeWeb/portal/model"
"go.uber.org/zap"
"gorm.io/gorm"
)
var (
ErrEmailExists = errors.New("Account with email already exists")
ErrPubkeyExists = errors.New("Account with pubkey already exists")
ErrQueryingAcct = errors.New("Error querying accounts")
ErrFailedHashPassword = errors.New("Failed to hash password")
ErrFailedCreateAccount = errors.New("Failed to create account")
)
func Register(email string, password string, pubkey string) error {
err := db.Get().Transaction(func(tx *gorm.DB) error {
existingAccount := model.Account{}
err := tx.Where("email = ?", email).First(&existingAccount).Error
if err == nil {
return ErrEmailExists
} else if !errors.Is(err, gorm.ErrRecordNotFound) {
return err
}
if len(pubkey) > 0 {
var count int64
err := tx.Model(&model.Key{}).Where("pubkey = ?", pubkey).Count(&count).Error
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return err
}
if count > 0 {
// An account with the same pubkey already exists.
// Return an error response to the client.
return ErrPubkeyExists
}
}
// Create a new Account model with the provided email and hashed password.
account := model.Account{
Email: email,
}
// Hash the password before saving it to the database.
if len(password) > 0 {
hashedPassword, err := hashPassword(password)
if err != nil {
return err
}
account.Password = &hashedPassword
}
if err := tx.Create(&account).Error; err != nil {
return err
}
if len(pubkey) > 0 {
if err := tx.Create(&model.Key{Account: account, Pubkey: pubkey}).Error; err != nil {
return err
}
}
return nil
})
if err != nil {
logger.Get().Error(ErrFailedCreateAccount.Error(), zap.Error(err))
return err
}
return nil
}

20
service/account/user.go
View File

@ -1,20 +0,0 @@
package account
import (
"git.lumeweb.com/LumeWeb/portal/model"
"github.com/kataras/iris/v12/context"
"strconv"
)
type User struct {
context.User
account *model.Account
}
func (u User) GetID() (string, error) {
return strconv.Itoa(int(u.account.ID)), nil
}
func NewUser(account *model.Account) *User {
return &User{account: account}
}

19
service/account/util.go
View File

@ -1,19 +0,0 @@
package account
import (
"git.lumeweb.com/LumeWeb/portal/logger"
"go.uber.org/zap"
"golang.org/x/crypto/bcrypt"
)
func hashPassword(password string) (string, error) {
// Generate a new bcrypt hash from the provided password.
hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
if err != nil {
logger.Get().Error(ErrFailedHashPassword.Error(), zap.Error(err))
return "", ErrFailedHashPassword
}
// Convert the hashed password to a string and return it.
return string(hashedPassword), nil
}

264
service/auth/auth.go
View File

@ -1,264 +0,0 @@
package auth
import (
"crypto/ed25519"
"crypto/x509"
"encoding/hex"
"encoding/pem"
"errors"
"git.lumeweb.com/LumeWeb/portal/config"
"git.lumeweb.com/LumeWeb/portal/db"
"git.lumeweb.com/LumeWeb/portal/logger"
"git.lumeweb.com/LumeWeb/portal/model"
"github.com/kataras/jwt"
"github.com/spf13/viper"
"go.uber.org/zap"
"os"
"path"
"path/filepath"
"strings"
"time"
)
var jwtKey = ed25519.PrivateKey{}
var blocklist *jwt.Blocklist
var (
ErrInvalidEmailPassword = errors.New("Invalid email or password")
ErrPubkeyOnly = errors.New("Only pubkey login is supported")
ErrFailedGenerateToken = errors.New("Failed to generate token")
ErrFailedGenerateKeyChallenge = errors.New("Failed to generate key challenge")
ErrFailedSignJwt = errors.New("Failed to sign jwt")
ErrFailedSaveToken = errors.New("Failed to sign token")
ErrFailedDeleteKeyChallenge = errors.New("Failed to delete key challenge")
ErrFailedInvalidateToken = errors.New("Failed to invalidate token")
ErrInvalidKeyChallenge = errors.New("Invalid key challenge")
ErrInvalidPubkey = errors.New("Invalid pubkey")
ErrInvalidSignature = errors.New("Invalid signature")
ErrInvalidToken = errors.New("Invalid token")
)
func Init() {
blocklist = jwt.NewBlocklist(0)
configFile := viper.ConfigFileUsed()
var jwtPemPath string
jwtPemName := "jwt.pem"
if configFile == "" {
jwtPemPath = path.Join(config.ConfigFilePaths[0], jwtPemName)
} else {
jwtPemPath = path.Join(filepath.Dir(configFile), jwtPemName)
}
if _, err := os.Stat(jwtPemPath); err != nil {
_, private, err := ed25519.GenerateKey(nil)
if err != nil {
logger.Get().Fatal("Failed to compute JWT private key", zap.Error(err))
}
privateBytes, err := x509.MarshalPKCS8PrivateKey(private)
if err != nil {
logger.Get().Fatal("Failed to create marshal private key", zap.Error(err))
}
var pemPrivateBlock = &pem.Block{
Type: "PRIVATE KEY",
Bytes: privateBytes,
}
pemPrivateFile, err := os.Create(jwtPemPath)
if err != nil {
logger.Get().Fatal("Failed to create empty file for JWT private PEM", zap.Error(err))
}
err = pem.Encode(pemPrivateFile, pemPrivateBlock)
if err != nil {
logger.Get().Fatal("Failed to write JWT private PEM", zap.Error(err))
}
jwtKey = private
} else {
data, err := os.ReadFile(jwtPemPath)
if err != nil {
logger.Get().Fatal("Failed to read JWT private PEM", zap.Error(err))
}
pemBlock, _ := pem.Decode(data)
if err != nil {
logger.Get().Fatal("Failed to decode JWT private PEM", zap.Error(err))
}
privateBytes, err := x509.ParsePKCS8PrivateKey(pemBlock.Bytes)
if err != nil {
logger.Get().Fatal("Failed to unmarshal JWT private PEM", zap.Error(err))
}
jwtKey = privateBytes.(ed25519.PrivateKey)
}
}
func LoginWithPassword(email string, password string) (string, error) {
// Retrieve the account for the given email.
account := model.Account{}
if err := db.Get().Model(&account).Where("email = ?", email).First(&account).Error; err != nil {
logger.Get().Debug(ErrInvalidEmailPassword.Error(), zap.String("email", email))
return "", ErrInvalidEmailPassword
}
if account.Password == nil || len(*account.Password) == 0 {
logger.Get().Debug(ErrPubkeyOnly.Error(), zap.String("email", email))
return "", ErrPubkeyOnly
}
// Verify the provided password against the hashed password stored in the database.
if err := verifyPassword(*account.Password, password); err != nil {
logger.Get().Debug(ErrPubkeyOnly.Error(), zap.String("email", email))
return "", ErrInvalidEmailPassword
}
// Generate a JWT token for the authenticated user.
token, err := generateAndSaveLoginToken(account.ID, 24*time.Hour)
if err != nil {
return "", err
}
return token, nil
}
func LoginWithPubkey(pubkey string, challenge string, signature string) (string, error) {
pubkey = strings.ToLower(pubkey)
signature = strings.ToLower(signature)
// Retrieve the key challenge for the given challenge.
challengeObj := model.KeyChallenge{}
if err := db.Get().Model(challengeObj).Where("challenge = ?", challenge).First(&challengeObj).Error; err != nil {
logger.Get().Debug(ErrInvalidKeyChallenge.Error(), zap.Error(err), zap.String("challenge", challenge))
return "", ErrInvalidKeyChallenge
}
verifiedToken, err := jwt.Verify(jwt.EdDSA, jwtKey, []byte(challenge), blocklist)
if err != nil {
logger.Get().Debug(ErrInvalidKeyChallenge.Error(), zap.Error(err), zap.String("challenge", challenge))
return "", ErrInvalidKeyChallenge
}
rawPubKey, err := hex.DecodeString(pubkey)
if err != nil {
logger.Get().Debug(ErrInvalidPubkey.Error(), zap.Error(err), zap.String("pubkey", pubkey))
return "", ErrInvalidPubkey
}
rawSignature, err := hex.DecodeString(signature)
if err != nil {
logger.Get().Debug(ErrInvalidPubkey.Error(), zap.Error(err), zap.String("signature", pubkey))
return "", ErrInvalidSignature
}
publicKeyDecoded := ed25519.PublicKey(rawPubKey)
// Verify the challenge signature.
if !ed25519.Verify(publicKeyDecoded, []byte(challenge), rawSignature) {
logger.Get().Debug(ErrInvalidKeyChallenge.Error(), zap.Error(err), zap.String("challenge", challenge))
return "", ErrInvalidKeyChallenge
}
// Generate a JWT token for the authenticated user.
token, err := generateAndSaveLoginToken(challengeObj.AccountID, 24*time.Hour)
if err != nil {
return "", err
}
err = blocklist.InvalidateToken(verifiedToken.Token, verifiedToken.StandardClaims)
if err != nil {
logger.Get().Error(ErrFailedInvalidateToken.Error(), zap.Error(err), zap.String("pubkey", pubkey), zap.ByteString("token", verifiedToken.Token), zap.String("challenge", challenge))
return "", ErrFailedInvalidateToken
}
if err := db.Get().Delete(&challengeObj).Error; err != nil {
logger.Get().Debug(ErrFailedDeleteKeyChallenge.Error(), zap.Error(err))
return "", ErrFailedDeleteKeyChallenge
}
return token, nil
}
func GeneratePubkeyChallenge(pubkey string) (string, error) {
pubkey = strings.ToLower(pubkey)
// Retrieve the account for the given email.
account := model.Key{}
if err := db.Get().Where("pubkey = ?", pubkey).First(&account).Error; err != nil {
logger.Get().Debug("failed to query pubkey", zap.Error(err))
return "", errors.New("invalid pubkey")
}
// Generate a random challenge string.
challenge, err := generateAndSaveChallengeToken(account.AccountID, time.Minute)
if err != nil {
logger.Get().Error(ErrFailedGenerateKeyChallenge.Error())
return "", ErrFailedGenerateKeyChallenge
}
return challenge, nil
}
func Logout(token string) error {
// Verify the provided token.
claims, err := jwt.Verify(jwt.EdDSA, jwtKey, []byte(token), blocklist)
if err != nil {
logger.Get().Debug(ErrInvalidToken.Error(), zap.Error(err))
return ErrInvalidToken
}
err = blocklist.InvalidateToken(claims.Token, claims.StandardClaims)
if err != nil {
logger.Get().Error(ErrFailedInvalidateToken.Error(), zap.Error(err), zap.String("token", token))
return ErrFailedInvalidateToken
}
// Retrieve the key challenge for the given challenge.
session := model.LoginSession{}
if err := db.Get().Model(session).Where("token = ?", token).First(&session).Error; err != nil {
logger.Get().Debug(ErrFailedInvalidateToken.Error(), zap.Error(err), zap.String("token", token))
return ErrFailedInvalidateToken
}
db.Get().Delete(&session)
return nil
}
func VerifyLoginToken(token string) (*model.Account, error) {
uvt, err := jwt.Decode([]byte(token))
if err != nil {
return nil, ErrInvalidToken
}
var claim jwt.Claims
err = uvt.Claims(&claim)
if err != nil {
return nil, ErrInvalidToken
}
session := model.LoginSession{}
if err := db.Get().Model(session).Preload("Account").Where("token = ?", token).First(&session).Error; err != nil {
logger.Get().Debug(ErrInvalidToken.Error(), zap.Error(err), zap.String("token", token))
return nil, ErrInvalidToken
}
_, err = jwt.Verify(jwt.EdDSA, jwtKey, []byte(token), blocklist)
if err != nil {
db.Get().Delete(&session)
return nil, err
}
return &session.Account, nil
}

142
service/auth/util.go
View File

@ -1,142 +0,0 @@
package auth
import (
"errors"
"git.lumeweb.com/LumeWeb/portal/db"
"git.lumeweb.com/LumeWeb/portal/logger"
"git.lumeweb.com/LumeWeb/portal/model"
"github.com/kataras/iris/v12"
"github.com/kataras/jwt"
"go.uber.org/zap"
"golang.org/x/crypto/bcrypt"
"strconv"
"strings"
"time"
)
// verifyPassword compares the provided plaintext password with a hashed password and returns an error if they don't match.
func verifyPassword(hashedPassword, password string) error {
err := bcrypt.CompareHashAndPassword([]byte(hashedPassword), []byte(password))
if err != nil {
return errors.New("invalid email or password")
}
return nil
}
// generateToken generates a JWT token for the given account ID.
func generateToken(maxAge time.Duration, ttype string) (string, error) {
// Define the JWT claims.
claim := jwt.Claims{
Expiry: time.Now().Add(time.Hour * 24).Unix(), // Token expires in 24 hours.
IssuedAt: time.Now().Unix(),
Audience: []string{ttype},
}
token, err := jwt.Sign(jwt.EdDSA, jwtKey, claim, jwt.MaxAge(maxAge))
if err != nil {
logger.Get().Error(ErrFailedSignJwt.Error(), zap.Error(err))
return "", err
}
return string(token), nil
}
func generateAndSaveLoginToken(accountID uint, maxAge time.Duration) (string, error) {
// Generate a JWT token for the authenticated user.
token, err := generateToken(maxAge, "auth")
if err != nil {
logger.Get().Error(ErrFailedGenerateToken.Error())
return "", ErrFailedGenerateToken
}
verifiedToken, _ := jwt.Verify(jwt.EdDSA, jwtKey, []byte(token), blocklist)
var claim *jwt.Claims
_ = verifiedToken.Claims(&claim)
// Save the token to the database.
session := model.LoginSession{
Account: model.Account{ID: accountID},
Token: token,
Expiration: claim.ExpiresAt(),
}
existingSession := model.LoginSession{}
err = db.Get().Where("token = ?", token).First(&existingSession).Error
if err == nil {
return token, nil
}
if err := db.Get().Create(&session).Error; err != nil {
if strings.Contains(err.Error(), "Duplicate entry") {
return token, nil
}
logger.Get().Error(ErrFailedSaveToken.Error(), zap.Uint("account_id", accountID), zap.Duration("max_age", maxAge))
return "", ErrFailedSaveToken
}
return token, nil
}
func generateAndSaveChallengeToken(accountID uint, maxAge time.Duration) (string, error) {
// Generate a JWT token for the authenticated user.
token, err := generateToken(maxAge, "challenge")
if err != nil {
logger.Get().Error(ErrFailedGenerateToken.Error(), zap.Error(err))
return "", ErrFailedGenerateToken
}
verifiedToken, err := jwt.Verify(jwt.EdDSA, jwtKey, []byte(token), blocklist)
if err != nil {
return "", ErrFailedGenerateToken
}
var claim *jwt.Claims
_ = verifiedToken.Claims(&claim)
// Save the token to the database.
keyChallenge := model.KeyChallenge{
AccountID: accountID,
Challenge: token,
Expiration: claim.ExpiresAt(),
}
if err := db.Get().Create(&keyChallenge).Error; err != nil {
logger.Get().Error(ErrFailedSaveToken.Error(), zap.Error(err))
return "", ErrFailedSaveToken
}
return token, nil
}
func GetRequestAuthCode(ctx iris.Context) string {
authHeader := ctx.GetHeader("Authorization")
if authHeader == "" {
return ""
}
// pure check: authorization header format must be Bearer {token}
authHeaderParts := strings.Split(authHeader, " ")
if len(authHeaderParts) != 2 || strings.ToLower(authHeaderParts[0]) != "bearer" {
return ""
}
return authHeaderParts[1]
}
func GetCurrentUserId(ctx iris.Context) uint {
usr := ctx.User()
if usr == nil {
return 0
}
sid, _ := usr.GetID()
userID, _ := strconv.Atoi(sid)
return uint(userID)
}

316
service/files/files.go
View File

@ -1,316 +0,0 @@
package files
import (
"bytes"
"context"
"encoding/hex"
"errors"
"fmt"
"git.lumeweb.com/LumeWeb/portal/bao"
"git.lumeweb.com/LumeWeb/portal/db"
"git.lumeweb.com/LumeWeb/portal/logger"
"git.lumeweb.com/LumeWeb/portal/model"
"git.lumeweb.com/LumeWeb/portal/shared"
"git.lumeweb.com/LumeWeb/portal/tusstore"
"github.com/go-resty/resty/v2"
"github.com/spf13/viper"
_ "github.com/tus/tusd/pkg/handler"
"go.uber.org/zap"
"gorm.io/gorm"
"io"
"strings"
)
const (
STATUS_UPLOADED = iota
STATUS_UPLOADING = iota
STATUS_NOT_FOUND = iota
)
var (
ErrAlreadyExists = errors.New("Upload already exists")
ErrFailedFetchObject = errors.New("Failed fetching object")
ErrFailedFetchObjectProof = errors.New("Failed fetching object proof")
ErrFailedFetchTusObject = errors.New("Failed fetching tus object")
ErrFailedHashFile = errors.New("Failed to hash file")
ErrFailedQueryTusUpload = errors.New("Failed to query tus uploads")
ErrFailedQueryUpload = errors.New("Failed to query uploads")
ErrFailedQueryPins = errors.New("Failed to query pins")
ErrFailedSaveUpload = errors.New("Failed saving upload to db")
ErrFailedSavePin = errors.New("Failed saving pin to db")
ErrFailedUpload = errors.New("Failed uploading object")
ErrFailedUploadProof = errors.New("Failed uploading object proof")
ErrFileExistsOutOfSync = errors.New("File already exists in network, but missing in database")
ErrFileHashMismatch = errors.New("File hash does not match provided file hash")
ErrInvalidFile = errors.New("Invalid file")
)
var client *resty.Client
func Init() {
client = resty.New()
client.SetBaseURL("http://localhost:9980/api")
client.SetBasicAuth("", viper.GetString("renterd-api-password"))
client.SetDisableWarn(true)
client.SetCloseConnection(true)
}
func Upload(r io.ReadSeeker, size int64, hash []byte, accountID uint) (model.Upload, error) {
var upload model.Upload
tree, hashBytes, err := bao.ComputeTree(r, size)
if err != nil {
logger.Get().Error(ErrFailedHashFile.Error(), zap.Error(err))
return upload, ErrFailedHashFile
}
if hash != nil {
if bytes.Compare(hashBytes[:], hash) != 0 {
logger.Get().Error(ErrFileHashMismatch.Error())
return upload, ErrFileHashMismatch
}
}
hashHex := hex.EncodeToString(hashBytes[:])
_, err = r.Seek(0, io.SeekStart)
if err != nil {
return upload, err
}
result := db.Get().Where(&model.Upload{Hash: hashHex}).First(&upload)
if !errors.Is(result.Error, gorm.ErrRecordNotFound) {
if err != nil {
logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(err))
return upload, ErrFailedQueryUpload
}
logger.Get().Info(ErrAlreadyExists.Error())
return upload, nil
}
objectExistsResult, err := client.R().Get(getBusObjectUrl(hashHex))
if err != nil {
logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(err))
return upload, ErrFailedQueryUpload
}
objectStatusCode := objectExistsResult.StatusCode()
if objectStatusCode == 500 {
bodyErr := objectExistsResult.String()
if !strings.Contains(bodyErr, "no slabs found") {
logger.Get().Error(ErrFailedFetchObject.Error(), zap.String("error", objectExistsResult.String()))
return upload, ErrFailedFetchObject
}
objectStatusCode = 404
}
proofExistsResult, err := client.R().Get(getBusProofUrl(hashHex))
if err != nil {
logger.Get().Error(ErrFailedFetchObjectProof.Error(), zap.Error(err))
return upload, ErrFailedFetchObjectProof
}
proofStatusCode := proofExistsResult.StatusCode()
if proofStatusCode == 500 {
bodyErr := proofExistsResult.String()
if !strings.Contains(bodyErr, "no slabs found") {
logger.Get().Error(ErrFailedFetchObjectProof.Error(), zap.String("error", proofExistsResult.String()))
return upload, ErrFailedFetchObjectProof
}
objectStatusCode = 404
}
if objectStatusCode != 404 && proofStatusCode != 404 {
logger.Get().Error(ErrFileExistsOutOfSync.Error(), zap.String("hash", hashHex))
return upload, ErrFileExistsOutOfSync
}
ret, err := client.R().SetBody(r).Put(getWorkerObjectUrl(hashHex))
if ret.StatusCode() != 200 {
logger.Get().Error(ErrFailedUpload.Error(), zap.String("error", ret.String()))
return upload, ErrFailedUpload
}
ret, err = client.R().SetBody(tree).Put(getWorkerProofUrl(hashHex))
if ret.StatusCode() != 200 {
logger.Get().Error(ErrFailedUploadProof.Error(), zap.String("error", ret.String()))
return upload, ErrFailedUpload
}
upload = model.Upload{
Hash: hashHex,
AccountID: accountID,
}
if err = db.Get().Create(&upload).Error; err != nil {
logger.Get().Error(ErrFailedSaveUpload.Error(), zap.Error(err))
return upload, ErrFailedSaveUpload
}
return upload, nil
}
func Download(hash string) (io.Reader, error) {
uploadItem := db.Get().Table("uploads").Where(&model.Upload{Hash: hash}).Row()
tusItem := db.Get().Table("tus").Where(&model.Tus{Hash: hash}).Row()
if uploadItem.Err() == nil {
fetch, err := client.R().SetDoNotParseResponse(true).Get(getWorkerObjectUrl(hash))
if err != nil {
logger.Get().Error(ErrFailedFetchObject.Error(), zap.Error(err))
return nil, ErrFailedFetchObject
}
return fetch.RawBody(), nil
} else if tusItem.Err() == nil {
var tusData model.Tus
err := tusItem.Scan(&tusData)
if err != nil {
logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(err))
return nil, ErrFailedQueryUpload
}
upload, err := getStore().GetUpload(context.Background(), tusData.UploadID)
if err != nil {
logger.Get().Error(ErrFailedQueryTusUpload.Error(), zap.Error(err))
return nil, ErrFailedQueryTusUpload
}
reader, err := upload.GetReader(context.Background())
if err != nil {
logger.Get().Error(ErrFailedFetchTusObject.Error(), zap.Error(err))
return nil, ErrFailedFetchTusObject
}
return reader, nil
} else {
logger.Get().Error(ErrInvalidFile.Error(), zap.String("hash", hash))
return nil, ErrInvalidFile
}
}
func DownloadProof(hash string) (io.Reader, error) {
uploadItem := db.Get().Model(&model.Upload{}).Where(&model.Upload{Hash: hash}).Row()
if uploadItem.Err() != nil {
logger.Get().Debug(ErrInvalidFile.Error(), zap.String("hash", hash))
return nil, ErrInvalidFile
}
fetch, err := client.R().SetDoNotParseResponse(true).Get(getWorkerProofUrl(hash))
if err != nil {
logger.Get().Error(ErrFailedFetchObject.Error(), zap.Error(err))
return nil, ErrFailedFetchObject
}
return fetch.RawBody(), nil
}
func Status(hash string) int {
var count int64
uploadItem := db.Get().Table("uploads").Where(&model.Upload{Hash: hash}).Count(&count)
if uploadItem.Error != nil && !errors.Is(uploadItem.Error, gorm.ErrRecordNotFound) {
logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(uploadItem.Error))
}
if count > 0 {
return STATUS_UPLOADED
}
tusItem := db.Get().Table("tus").Where(&model.Tus{Hash: hash}).Count(&count)
if tusItem.Error != nil && !errors.Is(tusItem.Error, gorm.ErrRecordNotFound) {
logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(tusItem.Error))
}
if count > 0 {
return STATUS_UPLOADING
}
return STATUS_NOT_FOUND
}
func objectUrlBuilder(hash string, bus bool, proof bool) string {
path := []string{}
if bus {
path = append(path, "bus")
} else {
path = append(path, "worker")
}
path = append(path, "objects")
name := "%s"
if proof {
name = name + ".obao"
}
path = append(path, name)
return fmt.Sprintf(strings.Join(path, "/"), hash)
}
func getBusObjectUrl(hash string) string {
return objectUrlBuilder(hash, true, false)
}
func getWorkerObjectUrl(hash string) string {
return objectUrlBuilder(hash, false, false)
}
func getWorkerProofUrl(hash string) string {
return objectUrlBuilder(hash, false, true)
}
func getBusProofUrl(hash string) string {
return objectUrlBuilder(hash, true, true)
}
func getStore() *tusstore.DbFileStore {
ret := shared.GetTusStore()
return (*ret).(*tusstore.DbFileStore)
}
func Pin(hash string, accountID uint) error {
var upload model.Upload
if result := db.Get().Model(&upload).Where("hash = ?", hash).First(&upload); result.Error != nil {
if !errors.Is(result.Error, gorm.ErrRecordNotFound) {
logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(result.Error))
}
return ErrFailedQueryUpload
}
var pin model.Pin
result := db.Get().Model(&pin).Where(&model.Pin{Upload: upload, AccountID: accountID}).First(&pin)
if result.Error != nil && !errors.Is(result.Error, gorm.ErrRecordNotFound) {
logger.Get().Error(ErrFailedQueryPins.Error(), zap.Error(result.Error))
return ErrFailedQueryPins
}
if result.Error == nil {
return nil
}
pin.AccountID = upload.AccountID
pin.Upload = upload
result = db.Get().Save(&pin)
if result.Error != nil {
logger.Get().Error(ErrFailedSavePin.Error(), zap.Error(result.Error))
return ErrFailedSavePin
}
return nil
}

51
shared/shared.go
View File

@ -1,51 +0,0 @@
package shared
import (
tusd "github.com/tus/tusd/pkg/handler"
_ "go.uber.org/zap"
)
type TusFunc func(upload *tusd.Upload) error
var tusQueue *interface{}
var tusStore *interface{}
var tusComposer *interface{}
var tusWorker TusFunc
type tusRequestContextKey int
const (
TusRequestContextKey tusRequestContextKey = iota
)
func SetTusQueue(q interface{}) {
tusQueue = &q
}
func GetTusQueue() *interface{} {
return tusQueue
}
func SetTusStore(s interface{}) {
tusStore = &s
}
func GetTusStore() *interface{} {
return tusStore
}
func SetTusComposer(c interface{}) {
tusComposer = &c
}
func GetTusComposer() *interface{} {
return tusComposer
}
func SetTusWorker(w TusFunc) {
tusWorker = w
}
func GetTusWorker() TusFunc {
return tusWorker
}

222
tus/tus.go
View File

@ -1,222 +0,0 @@
package tus
import (
"context"
"encoding/hex"
"encoding/json"
"errors"
"git.lumeweb.com/LumeWeb/portal/cid"
"git.lumeweb.com/LumeWeb/portal/db"
"git.lumeweb.com/LumeWeb/portal/logger"
"git.lumeweb.com/LumeWeb/portal/model"
"git.lumeweb.com/LumeWeb/portal/service/files"
"git.lumeweb.com/LumeWeb/portal/shared"
"git.lumeweb.com/LumeWeb/portal/tusstore"
"github.com/golang-queue/queue"
tusd "github.com/tus/tusd/pkg/handler"
"github.com/tus/tusd/pkg/memorylocker"
"go.uber.org/zap"
"golang.org/x/exp/slices"
"gorm.io/gorm"
"io"
"strconv"
)
const TUS_API_PATH = "/files/tus"
const HASH_META_HEADER = "hash"
func Init() *tusd.Handler {
store := &tusstore.DbFileStore{
Path: "/tmp",
}
shared.SetTusStore(store)
composer := tusd.NewStoreComposer()
composer.UseCore(store)
composer.UseConcater(store)
composer.UseLocker(memorylocker.New())
composer.UseTerminater(store)
shared.SetTusComposer(composer)
handler, err := tusd.NewHandler(tusd.Config{
BasePath: "/api/v1" + TUS_API_PATH,
StoreComposer: composer,
PreUploadCreateCallback: func(hook tusd.HookEvent) error {
hash := hook.Upload.MetaData[HASH_META_HEADER]
if len(hash) == 0 {
msg := "missing hash metadata"
logger.Get().Debug(msg)
return errors.New(msg)
}
var upload model.Upload
result := db.Get().Where(&model.Upload{Hash: hash}).First(&upload)
if (result.Error != nil && !errors.Is(result.Error, gorm.ErrRecordNotFound)) || result.RowsAffected > 0 {
hashBytes, err := hex.DecodeString(hash)
if err != nil {
logger.Get().Debug("invalid hash", zap.Error(err))
return err
}
cidString, err := cid.Encode(hashBytes, uint64(hook.Upload.Size))
if err != nil {
logger.Get().Debug("failed to create cid", zap.Error(err))
return err
}
resp, err := json.Marshal(UploadResponse{Cid: cidString})
if err != nil {
logger.Get().Error("failed to create response", zap.Error(err))
return err
}
return tusd.NewHTTPError(errors.New(string(resp)), 304)
}
return nil
},
})
if err != nil {
panic(err)
}
pool := queue.NewPool(5)
shared.SetTusQueue(pool)
shared.SetTusWorker(tusWorker)
go tusStartup()
return handler
}
func tusStartup() {
tusQueue := getQueue()
store := getStore()
rows, err := db.Get().Model(&model.Tus{}).Rows()
if err != nil {
logger.Get().Error("failed to query tus uploads", zap.Error(err))
}
defer rows.Close()
processedHashes := make([]string, 0)
for rows.Next() {
var tusUpload model.Tus
err := db.Get().ScanRows(rows, &tusUpload)
if err != nil {
logger.Get().Error("failed to scan tus records", zap.Error(err))
return
}
upload, err := store.GetUpload(nil, tusUpload.UploadID)
if err != nil {
logger.Get().Error("failed to query tus upload", zap.Error(err))
db.Get().Delete(&tusUpload)
continue
}
if slices.Contains(processedHashes, tusUpload.Hash) {
err := terminateUpload(upload)
if err != nil {
logger.Get().Error("failed to terminate tus upload", zap.Error(err))
}
continue
}
if err := tusQueue.QueueTask(func(ctx context.Context) error {
return tusWorker(&upload)
}); err != nil {
logger.Get().Error("failed to queue tus upload", zap.Error(err))
} else {
processedHashes = append(processedHashes, tusUpload.Hash)
}
}
}
func tusWorker(upload *tusd.Upload) error {
info, err := (*upload).GetInfo(context.Background())
if err != nil {
logger.Get().Error("failed to query tus upload metadata", zap.Error(err))
return err
}
file, err := (*upload).GetReader(context.Background())
if err != nil {
logger.Get().Error("failed reading upload", zap.Error(err))
return err
}
hashHex := info.MetaData[HASH_META_HEADER]
hashBytes, err := hex.DecodeString(hashHex)
if err != nil {
logger.Get().Error("failed decoding hash", zap.Error(err))
tErr := terminateUpload(*upload)
if tErr != nil {
return tErr
}
return err
}
uploader, _ := strconv.Atoi(info.Storage["uploader"])
newUpload, err := files.Upload(file.(io.ReadSeeker), info.Size, hashBytes, uint(uploader))
tErr := terminateUpload(*upload)
if tErr != nil {
return tErr
}
if err != nil {
return err
}
err = files.Pin(newUpload.Hash, newUpload.AccountID)
if err != nil {
return err
}
return nil
}
func terminateUpload(upload tusd.Upload) error {
err := getComposer().Terminater.AsTerminatableUpload(upload).Terminate(context.Background())
if err != nil {
logger.Get().Error("failed deleting tus upload", zap.Error(err))
}
if err != nil {
return err
}
return nil
}
type UploadResponse struct {
Cid string `json:"cid"`
}
func getQueue() *queue.Queue {
ret := shared.GetTusQueue()
return (*ret).(*queue.Queue)
}
func getStore() *tusstore.DbFileStore {
ret := shared.GetTusStore()
return (*ret).(*tusstore.DbFileStore)
}
func getComposer() *tusd.StoreComposer {
ret := shared.GetTusComposer()
return (*ret).(*tusd.StoreComposer)
}

316
tusstore/store.go
View File

@ -1,316 +0,0 @@
package tusstore
import (
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"fmt"
"git.lumeweb.com/LumeWeb/portal/db"
"git.lumeweb.com/LumeWeb/portal/logger"
"git.lumeweb.com/LumeWeb/portal/model"
"git.lumeweb.com/LumeWeb/portal/service/auth"
"git.lumeweb.com/LumeWeb/portal/shared"
"github.com/golang-queue/queue"
clone "github.com/huandu/go-clone"
"github.com/kataras/iris/v12"
"github.com/tus/tusd/pkg/handler"
"go.uber.org/zap"
"io"
"os"
"path/filepath"
"strconv"
)
var defaultFilePerm = os.FileMode(0664)
type DbFileStore struct {
// Relative or absolute path to store files in. DbFileStore does not check
// whether the path exists, use os.MkdirAll in this case on your own.
Path string
}
func (store DbFileStore) UseIn(composer *handler.StoreComposer) {
composer.UseCore(store)
composer.UseTerminater(store)
composer.UseConcater(store)
composer.UseLengthDeferrer(store)
}
func (store DbFileStore) NewUpload(ctx context.Context, info handler.FileInfo) (handler.Upload, error) {
if info.ID == "" {
info.ID = uid()
}
binPath := store.binPath(info.ID)
info.Storage = map[string]string{
"Type": "dbstore",
"Path": binPath,
}
// Create binary file with no content
file, err := os.OpenFile(binPath, os.O_CREATE|os.O_WRONLY, defaultFilePerm)
if err != nil {
if os.IsNotExist(err) {
err = fmt.Errorf("upload directory does not exist: %s", store.Path)
}
return nil, err
}
err = file.Close()
if err != nil {
return nil, err
}
if err != nil {
return nil, err
}
irisContext := ctx.Value(shared.TusRequestContextKey).(iris.Context)
upload := &fileUpload{
info: info,
binPath: binPath,
hash: info.MetaData["hash"],
uploader: auth.GetCurrentUserId(irisContext),
}
// writeInfo creates the file by itself if necessary
err = upload.writeInfo()
if err != nil {
return nil, err
}
return upload, nil
}
func (store DbFileStore) GetUpload(ctx context.Context, id string) (handler.Upload, error) {
info := handler.FileInfo{
ID: id,
}
fUpload := &fileUpload{info: info}
record, is404, err := fUpload.getInfo()
if err != nil {
if is404 {
// Interpret os.ErrNotExist as 404 Not Found
err = handler.ErrNotFound
}
return nil, err
}
if err := json.Unmarshal([]byte(record.Info), &info); err != nil {
logger.Get().Error("fail to parse upload meta", zap.Error(err))
return nil, err
}
fUpload.info = info
fUpload.hash = record.Hash
binPath := store.binPath(id)
stat, err := os.Stat(binPath)
if err != nil {
if os.IsNotExist(err) {
// Interpret os.ErrNotExist as 404 Not Found
err = handler.ErrNotFound
}
return nil, err
}
info.Offset = stat.Size()
fUpload.binPath = binPath
return fUpload, nil
}
func (store DbFileStore) AsTerminatableUpload(upload handler.Upload) handler.TerminatableUpload {
return upload.(*fileUpload)
}
func (store DbFileStore) AsLengthDeclarableUpload(upload handler.Upload) handler.LengthDeclarableUpload {
return upload.(*fileUpload)
}
func (store DbFileStore) AsConcatableUpload(upload handler.Upload) handler.ConcatableUpload {
return upload.(*fileUpload)
}
// binPath returns the path to the file storing the binary data.
func (store DbFileStore) binPath(id string) string {
return filepath.Join(store.Path, id)
}
type fileUpload struct {
// info stores the current information about the upload
info handler.FileInfo
// binPath is the path to the binary file (which has no extension)
binPath string
hash string
uploader uint
}
func (upload *fileUpload) GetInfo(ctx context.Context) (handler.FileInfo, error) {
info := clone.Clone(upload.info).(handler.FileInfo)
info.Storage["uploader"] = strconv.Itoa(int(upload.uploader))
return upload.info, nil
}
func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) {
file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm)
if err != nil {
return 0, err
}
defer file.Close()
n, err := io.Copy(file, src)
upload.info.Offset += n
err = upload.writeInfo()
if err != nil {
return 0, err
}
return n, err
}
func (upload *fileUpload) GetReader(ctx context.Context) (io.Reader, error) {
return os.Open(upload.binPath)
}
func (upload *fileUpload) Terminate(ctx context.Context) error {
tusUpload := &model.Tus{
UploadID: upload.info.ID,
}
ret := db.Get().Where(&tusUpload).Delete(&tusUpload)
if ret.Error != nil {
logger.Get().Error("failed to delete tus entry", zap.Error(ret.Error))
}
if err := os.Remove(upload.binPath); err != nil {
return err
}
return nil
}
func (upload *fileUpload) ConcatUploads(ctx context.Context, uploads []handler.Upload) (err error) {
file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm)
if err != nil {
return err
}
defer file.Close()
for _, partialUpload := range uploads {
fileUpload := partialUpload.(*fileUpload)
src, err := os.Open(fileUpload.binPath)
if err != nil {
return err
}
if _, err := io.Copy(file, src); err != nil {
return err
}
}
return
}
func (upload *fileUpload) DeclareLength(ctx context.Context, length int64) error {
upload.info.Size = length
upload.info.SizeIsDeferred = false
return upload.writeInfo()
}
// writeInfo updates the entire information. Everything will be overwritten.
func (upload *fileUpload) writeInfo() error {
data, err := json.Marshal(upload.info)
if err != nil {
return err
}
tusRecord, is404, err := upload.getInfo()
if err != nil && !is404 {
return err
}
if tusRecord != nil {
tusRecord.Info = string(data)
if ret := db.Get().Save(&tusRecord); ret.Error != nil {
logger.Get().Error("failed to update tus entry", zap.Error(ret.Error))
return ret.Error
}
return nil
}
tusRecord = &model.Tus{UploadID: upload.info.ID, Hash: upload.hash, Info: string(data), AccountID: upload.uploader}
if ret := db.Get().Create(&tusRecord); ret.Error != nil {
logger.Get().Error("failed to create tus entry", zap.Error(ret.Error))
return ret.Error
}
return nil
}
func (upload *fileUpload) getInfo() (*model.Tus, bool, error) {
var tusRecord model.Tus
result := db.Get().Where(&model.Tus{UploadID: upload.info.ID}).First(&tusRecord)
if result.Error != nil && result.Error.Error() != "record not found" {
logger.Get().Error("failed to query tus entry", zap.Error(result.Error))
return nil, false, result.Error
}
if result.Error != nil {
return nil, true, result.Error
}
return &tusRecord, false, nil
}
func (upload *fileUpload) FinishUpload(ctx context.Context) error {
if err := getQueue().QueueTask(func(ctx context.Context) error {
upload, err := getStore().GetUpload(nil, upload.info.ID)
if err != nil {
logger.Get().Error("failed to query tus upload", zap.Error(err))
return err
}
return shared.GetTusWorker()(&upload)
}); err != nil {
logger.Get().Error("failed to queue tus upload", zap.Error(err))
return err
}
return nil
}
func uid() string {
id := make([]byte, 16)
_, err := io.ReadFull(rand.Reader, id)
if err != nil {
// This is probably an appropriate way to handle errors from our source
// for random bits.
panic(err)
}
return hex.EncodeToString(id)
}
func getQueue() *queue.Queue {
ret := shared.GetTusQueue()
return (*ret).(*queue.Queue)
}
func getStore() *DbFileStore {
ret := shared.GetTusStore()
return (*ret).(*DbFileStore)
}