diff --git a/bao/bao.go b/bao/bao.go deleted file mode 100644 index d5ff1f6..0000000 --- a/bao/bao.go +++ /dev/null @@ -1,31 +0,0 @@ -package bao - -import ( - "bufio" - _ "embed" - "io" - "lukechampine.com/blake3" -) - -func ComputeTree(reader io.Reader, size int64) ([]byte, [32]byte, error) { - bufSize := blake3.BaoEncodedSize(int(size), true) - buf := bufferAt{buf: make([]byte, bufSize)} - - hash, err := blake3.BaoEncode(&buf, bufio.NewReader(reader), size, true) - if err != nil { - return nil, [32]byte{}, err - } - - return buf.buf, hash, nil -} - -type bufferAt struct { - buf []byte -} - -func (b *bufferAt) WriteAt(p []byte, off int64) (int, error) { - if copy(b.buf[off:], p) != len(p) { - panic("bad buffer size") - } - return len(p), nil -} diff --git a/cid/cid.go b/cid/cid.go deleted file mode 100644 index b7815a2..0000000 --- a/cid/cid.go +++ /dev/null @@ -1,100 +0,0 @@ -package cid - -import ( - "bytes" - "encoding/binary" - "encoding/hex" - "errors" - "github.com/multiformats/go-multibase" -) - -var MAGIC_BYTES = []byte{0x26, 0x1f} - -var ( - ErrMissingEmptySize = errors.New("Missing or empty size") - ErrInvalidCIDMagic = errors.New("CID magic bytes missing or invalid") -) - -type CID struct { - Hash [32]byte - Size uint64 -} - -func (c CID) StringHash() string { - return hex.EncodeToString(c.Hash[:]) -} - -func Encode(hash []byte, size uint64) (string, error) { - var hashBytes [32]byte - copy(hashBytes[:], hash) - - return EncodeFixed(hashBytes, size) -} - -func EncodeFixed(hash [32]byte, size uint64) (string, error) { - sizeBytes := make([]byte, 8) - binary.LittleEndian.PutUint64(sizeBytes, size) - - prefixedHash := append(MAGIC_BYTES, hash[:]...) - prefixedHash = append(prefixedHash, sizeBytes...) - - return multibase.Encode(multibase.Base58BTC, prefixedHash) -} - -func EncodeString(hash string, size uint64) (string, error) { - hashBytes, err := hex.DecodeString(hash) - if err != nil { - return "", err - } - - return Encode(hashBytes, size) -} - -func Valid(cid string) (bool, error) { - _, err := maybeDecode(cid) - if err != nil { - return false, err - } - - return true, nil - -} - -func Decode(cid string) (*CID, error) { - data, err := maybeDecode(cid) - if err != nil { - return &CID{}, err - } - - data = data[len(MAGIC_BYTES):] - var hash [32]byte - copy(hash[:], data[:]) - size := binary.LittleEndian.Uint64(data[32:]) - - return &CID{Hash: hash, Size: size}, nil -} - -func maybeDecode(cid string) ([]byte, error) { - _, data, err := multibase.Decode(cid) - if err != nil { - return nil, err - } - - if bytes.Compare(data[0:len(MAGIC_BYTES)], MAGIC_BYTES) != 0 { - return nil, ErrInvalidCIDMagic - } - - sizeBytes := data[len(MAGIC_BYTES)+32:] - - if len(sizeBytes) == 0 { - return nil, ErrMissingEmptySize - } - - size := binary.LittleEndian.Uint64(sizeBytes) - - if size == 0 { - return nil, ErrMissingEmptySize - } - - return data, nil -} diff --git a/config/config.go b/config/config.go deleted file mode 100644 index 5e18cc7..0000000 --- a/config/config.go +++ /dev/null @@ -1,59 +0,0 @@ -package config - -import ( - "errors" - "fmt" - "github.com/spf13/pflag" - "github.com/spf13/viper" - "log" -) - -var ( - ConfigFilePaths = []string{ - "/etc/lumeweb/portal/", - "$HOME/.lumeweb/portal/", - ".", - } -) - -func Init() { - viper.SetConfigName("config") - viper.SetConfigType("json") - - for _, path := range ConfigFilePaths { - viper.AddConfigPath(path) - } - - viper.SetEnvPrefix("LUME_WEB_PORTAL") - viper.AutomaticEnv() - - pflag.String("database.type", "sqlite", "Database type") - pflag.String("database.host", "localhost", "Database host") - pflag.Int("database.port", 3306, "Database port") - pflag.String("database.user", "root", "Database user") - pflag.String("database.password", "", "Database password") - pflag.String("database.name", "lumeweb_portal", "Database name") - pflag.String("database.path", "./db.sqlite", "Database path for SQLite") - pflag.String("renterd-api-password", ".", "admin password for renterd") - pflag.Bool("debug", false, "enable debug mode") - pflag.Parse() - - err := viper.BindPFlags(pflag.CommandLine) - - if err != nil { - log.Fatalf("Fatal error arguments: %s \n", err) - return - } - - err = viper.ReadInConfig() - if err != nil { - if errors.As(err, &viper.ConfigFileNotFoundError{}) { - // Config file not found, this is not an error. - fmt.Println("Config file not found, using default settings.") - } else { - // Other error, panic. - panic(fmt.Errorf("Fatal error config file: %s \n", err)) - } - } - -} diff --git a/controller/account.go b/controller/account.go deleted file mode 100644 index 49699d5..0000000 --- a/controller/account.go +++ /dev/null @@ -1,35 +0,0 @@ -package controller - -import ( - "git.lumeweb.com/LumeWeb/portal/controller/request" - "git.lumeweb.com/LumeWeb/portal/service/account" - "github.com/kataras/iris/v12" -) - -type AccountController struct { - Controller -} - -func (a *AccountController) PostRegister() { - ri, success := tryParseRequest(request.RegisterRequest{}, a.Ctx) - if !success { - return - } - - r, _ := ri.(*request.RegisterRequest) - - err := account.Register(r.Email, r.Password, r.Pubkey) - - if err != nil { - if err == account.ErrQueryingAcct || err == account.ErrFailedCreateAccount { - a.Ctx.StopWithError(iris.StatusInternalServerError, err) - } else { - a.Ctx.StopWithError(iris.StatusBadRequest, err) - } - - return - } - - // Return a success response to the client. - a.Ctx.StatusCode(iris.StatusCreated) -} diff --git a/controller/auth.go b/controller/auth.go deleted file mode 100644 index 4c7c213..0000000 --- a/controller/auth.go +++ /dev/null @@ -1,112 +0,0 @@ -package controller - -import ( - "git.lumeweb.com/LumeWeb/portal/controller/request" - "git.lumeweb.com/LumeWeb/portal/controller/response" - "git.lumeweb.com/LumeWeb/portal/middleware" - "git.lumeweb.com/LumeWeb/portal/service/auth" - "github.com/kataras/iris/v12" -) - -type AuthController struct { - Controller -} - -// PostLogin handles the POST /api/auth/login request to authenticate a user and return a JWT token. -func (a *AuthController) PostLogin() { - ri, success := tryParseRequest(request.LoginRequest{}, a.Ctx) - if !success { - return - } - - r, _ := ri.(*request.LoginRequest) - - token, err := auth.LoginWithPassword(r.Email, r.Password) - - if err != nil { - if err == auth.ErrFailedGenerateToken { - a.Ctx.StopWithError(iris.StatusInternalServerError, err) - } else { - a.Ctx.StopWithError(iris.StatusUnauthorized, err) - } - return - } - - a.respondJSON(&response.LoginResponse{Token: token}) -} - -// PostChallenge handles the POST /api/auth/pubkey/challenge request to generate a challenge for a user's public key. -func (a *AuthController) PostPubkeyChallenge() { - ri, success := tryParseRequest(request.PubkeyChallengeRequest{}, a.Ctx) - if !success { - return - } - - r, _ := (ri).(*request.PubkeyChallengeRequest) - - challenge, err := auth.GeneratePubkeyChallenge(r.Pubkey) - if err != nil { - if err == auth.ErrFailedGenerateKeyChallenge { - a.Ctx.StopWithError(iris.StatusInternalServerError, err) - } else { - a.Ctx.StopWithError(iris.StatusUnauthorized, err) - } - return - } - - a.respondJSON(&response.ChallengeResponse{Challenge: challenge}) -} - -// PostKeyLogin handles the POST /api/auth/pubkey/login request to authenticate a user using a public key challenge and return a JWT token. -func (a *AuthController) PostPubkeyLogin() { - ri, success := tryParseRequest(request.PubkeyLoginRequest{}, a.Ctx) - if !success { - return - } - - r, _ := ri.(*request.PubkeyLoginRequest) - - token, err := auth.LoginWithPubkey(r.Pubkey, r.Challenge, r.Signature) - - if err != nil { - if err == auth.ErrFailedGenerateKeyChallenge || err == auth.ErrFailedGenerateToken || err == auth.ErrFailedSaveToken { - a.Ctx.StopWithError(iris.StatusInternalServerError, err) - } else { - a.Ctx.StopWithError(iris.StatusUnauthorized, err) - } - return - } - - a.respondJSON(&response.LoginResponse{Token: token}) - -} - -// PostLogout handles the POST /api/auth/logout request to invalidate a JWT token. -func (a *AuthController) PostLogout() { - ri, success := tryParseRequest(request.LogoutRequest{}, a.Ctx) - if !success { - return - } - - r, _ := ri.(*request.LogoutRequest) - - err := auth.Logout(r.Token) - - if err != nil { - a.Ctx.StopWithError(iris.StatusBadRequest, err) - return - } - - // Return a success response to the client. - a.Ctx.StatusCode(iris.StatusNoContent) -} - -func (a *AuthController) GetStatus() { - middleware.VerifyJwt(a.Ctx) - - if a.Ctx.IsStopped() { - return - } - - a.respondJSON(&response.AuthStatusResponse{Status: true}) -} diff --git a/controller/controller.go b/controller/controller.go deleted file mode 100644 index b8f2a64..0000000 --- a/controller/controller.go +++ /dev/null @@ -1,72 +0,0 @@ -package controller - -import ( - "git.lumeweb.com/LumeWeb/portal/controller/validators" - "git.lumeweb.com/LumeWeb/portal/logger" - "github.com/kataras/iris/v12" - "go.uber.org/zap" -) - -func tryParseRequest(r interface{}, ctx iris.Context) (interface{}, bool) { - v, ok := r.(validators.Validatable) - - if !ok { - return r, true - } - - var d map[string]interface{} - - // Read the logout request from the client. - if err := ctx.ReadJSON(&d); err != nil { - logger.Get().Debug("failed to parse request", zap.Error(err)) - ctx.StopWithError(iris.StatusBadRequest, err) - return nil, false - } - - data, err := v.Import(d) - if err != nil { - logger.Get().Debug("failed to parse request", zap.Error(err)) - ctx.StopWithError(iris.StatusBadRequest, err) - return nil, false - } - - if err := data.Validate(); err != nil { - logger.Get().Debug("failed to parse request", zap.Error(err)) - ctx.StopWithError(iris.StatusBadRequest, err) - return nil, false - } - - return data, true -} - -func sendErrorCustom(ctx iris.Context, err error, customError error, irisError int) bool { - if err != nil { - if customError != nil { - err = customError - } - ctx.StopWithError(irisError, err) - return true - } - - return false -} -func InternalError(ctx iris.Context, err error) bool { - return sendErrorCustom(ctx, err, nil, iris.StatusInternalServerError) -} -func internalErrorCustom(ctx iris.Context, err error, customError error) bool { - return sendErrorCustom(ctx, err, customError, iris.StatusInternalServerError) -} -func SendError(ctx iris.Context, err error, irisError int) bool { - return sendErrorCustom(ctx, err, nil, irisError) -} - -type Controller struct { - Ctx iris.Context -} - -func (c Controller) respondJSON(data interface{}) { - err := c.Ctx.JSON(data) - if err != nil { - logger.Get().Error("failed to generate response", zap.Error(err)) - } -} diff --git a/controller/files.go b/controller/files.go deleted file mode 100644 index 955ecd3..0000000 --- a/controller/files.go +++ /dev/null @@ -1,213 +0,0 @@ -package controller - -import ( - "errors" - "git.lumeweb.com/LumeWeb/portal/cid" - "git.lumeweb.com/LumeWeb/portal/controller/response" - "git.lumeweb.com/LumeWeb/portal/logger" - "git.lumeweb.com/LumeWeb/portal/middleware" - "git.lumeweb.com/LumeWeb/portal/service/auth" - "git.lumeweb.com/LumeWeb/portal/service/files" - "github.com/kataras/iris/v12" - "go.uber.org/zap" - "io" -) - -var ErrStreamDone = errors.New("done") - -type FilesController struct { - Controller -} - -func (f *FilesController) BeginRequest(ctx iris.Context) { - middleware.VerifyJwt(ctx) -} -func (f *FilesController) EndRequest(ctx iris.Context) { -} - -func (f *FilesController) PostUpload() { - ctx := f.Ctx - - file, meta, err := f.Ctx.FormFile("file") - if internalErrorCustom(ctx, err, errors.New("invalid file data")) { - logger.Get().Debug("invalid file data", zap.Error(err)) - return - } - - upload, err := files.Upload(file, meta.Size, nil, auth.GetCurrentUserId(ctx)) - - if InternalError(ctx, err) { - logger.Get().Debug("failed uploading file", zap.Error(err)) - return - } - - err = files.Pin(upload.Hash, upload.AccountID) - - if InternalError(ctx, err) { - logger.Get().Debug("failed pinning file", zap.Error(err)) - return - } - - cidString, err := cid.EncodeString(upload.Hash, uint64(meta.Size)) - - if InternalError(ctx, err) { - logger.Get().Debug("failed creating cid", zap.Error(err)) - return - } - - err = ctx.JSON(&response.UploadResponse{Cid: cidString}) - - if err != nil { - logger.Get().Error("failed to create response", zap.Error(err)) - } -} - -func (f *FilesController) GetDownloadBy(cidString string) { - ctx := f.Ctx - - hashHex, valid := ValidateCid(cidString, true, ctx) - - if !valid { - return - } - - download, err := files.Download(hashHex) - if InternalError(ctx, err) { - logger.Get().Debug("failed fetching file", zap.Error(err)) - return - } - - err = PassThroughStream(download, ctx) - if err != ErrStreamDone && InternalError(ctx, err) { - logger.Get().Debug("failed streaming file", zap.Error(err)) - } -} - -func (f *FilesController) GetProofBy(cidString string) { - ctx := f.Ctx - - hashHex, valid := ValidateCid(cidString, true, ctx) - - if !valid { - return - } - - proof, err := files.DownloadProof(hashHex) - if InternalError(ctx, err) { - logger.Get().Debug("failed fetching file proof", zap.Error(err)) - return - } - - err = PassThroughStream(proof, ctx) - if InternalError(ctx, err) { - logger.Get().Debug("failed streaming file proof", zap.Error(err)) - } -} - -func (f *FilesController) GetStatusBy(cidString string) { - ctx := f.Ctx - - hashHex, valid := ValidateCid(cidString, false, ctx) - - if !valid { - return - } - - status := files.Status(hashHex) - - var statusCode string - - switch status { - case files.STATUS_UPLOADED: - statusCode = "uploaded" - break - case files.STATUS_UPLOADING: - statusCode = "uploading" - break - case files.STATUS_NOT_FOUND: - statusCode = "not_found" - break - } - - f.respondJSON(&response.FileStatusResponse{Status: statusCode}) - -} - -func (f *FilesController) PostPinBy(cidString string) { - ctx := f.Ctx - - hashHex, valid := ValidateCid(cidString, true, ctx) - - if !valid { - return - } - - err := files.Pin(hashHex, auth.GetCurrentUserId(ctx)) - if InternalError(ctx, err) { - logger.Get().Error(err.Error()) - return - } - - f.Ctx.StatusCode(iris.StatusCreated) -} - -func (f *FilesController) GetUploadLimit() { - f.respondJSON(&response.UploadLimit{Limit: f.Ctx.Application().ConfigurationReadOnly().GetPostMaxMemory()}) -} - -func ValidateCid(cidString string, validateStatus bool, ctx iris.Context) (string, bool) { - _, err := cid.Valid(cidString) - if SendError(ctx, err, iris.StatusBadRequest) { - logger.Get().Debug("invalid cid", zap.Error(err)) - return "", false - } - - cidObject, _ := cid.Decode(cidString) - hashHex := cidObject.StringHash() - - if validateStatus { - status := files.Status(hashHex) - - if status == files.STATUS_NOT_FOUND { - err := errors.New("cid not found") - SendError(ctx, errors.New("cid not found"), iris.StatusNotFound) - logger.Get().Debug("cid not found", zap.Error(err)) - return "", false - } - } - - return hashHex, true -} - -func PassThroughStream(stream io.Reader, ctx iris.Context) error { - closed := false - - err := ctx.StreamWriter(func(w io.Writer) error { - if closed { - return ErrStreamDone - } - - count, err := io.CopyN(w, stream, 1024) - if count == 0 || err == io.EOF { - err = stream.(io.Closer).Close() - if err != nil { - logger.Get().Error("failed closing stream", zap.Error(err)) - return err - } - closed = true - return nil - } - - if err != nil { - return err - } - - return nil - }) - - if err == ErrStreamDone { - err = nil - } - - return err -} diff --git a/controller/request/login.go b/controller/request/login.go deleted file mode 100644 index 33681bc..0000000 --- a/controller/request/login.go +++ /dev/null @@ -1,23 +0,0 @@ -package request - -import ( - "git.lumeweb.com/LumeWeb/portal/controller/validators" - validation "github.com/go-ozzo/ozzo-validation/v4" - "github.com/go-ozzo/ozzo-validation/v4/is" -) - -type LoginRequest struct { - validatable validators.ValidatableImpl - Email string `json:"email"` - Password string `json:"password"` -} - -func (r LoginRequest) Validate() error { - return validation.ValidateStruct(&r, - validation.Field(&r.Email, is.EmailFormat, validation.Required), - validation.Field(&r.Password, validation.Required), - ) -} -func (r LoginRequest) Import(d map[string]interface{}) (validators.Validatable, error) { - return r.validatable.Import(d, r) -} diff --git a/controller/request/logout.go b/controller/request/logout.go deleted file mode 100644 index e40dd73..0000000 --- a/controller/request/logout.go +++ /dev/null @@ -1,19 +0,0 @@ -package request - -import ( - "git.lumeweb.com/LumeWeb/portal/controller/validators" - validation "github.com/go-ozzo/ozzo-validation/v4" -) - -type LogoutRequest struct { - validatable validators.ValidatableImpl - Token string `json:"token"` -} - -func (r LogoutRequest) Validate() error { - return validation.ValidateStruct(&r, validation.Field(&r.Token, validation.Required)) -} - -func (r LogoutRequest) Import(d map[string]interface{}) (validators.Validatable, error) { - return r.validatable.Import(d, r) -} diff --git a/controller/request/pubkey_challenge.go b/controller/request/pubkey_challenge.go deleted file mode 100644 index 714feec..0000000 --- a/controller/request/pubkey_challenge.go +++ /dev/null @@ -1,21 +0,0 @@ -package request - -import ( - "git.lumeweb.com/LumeWeb/portal/controller/validators" - validation "github.com/go-ozzo/ozzo-validation/v4" -) - -type PubkeyChallengeRequest struct { - validatable validators.ValidatableImpl - Pubkey string `json:"pubkey"` -} - -func (r PubkeyChallengeRequest) Validate() error { - return validation.ValidateStruct(&r, - validation.Field(&r.Pubkey, validation.Required, validation.By(validators.CheckPubkeyValidator)), - ) -} - -func (r PubkeyChallengeRequest) Import(d map[string]interface{}) (validators.Validatable, error) { - return r.validatable.Import(d, r) -} diff --git a/controller/request/pubkey_login.go b/controller/request/pubkey_login.go deleted file mode 100644 index c18e536..0000000 --- a/controller/request/pubkey_login.go +++ /dev/null @@ -1,25 +0,0 @@ -package request - -import ( - "git.lumeweb.com/LumeWeb/portal/controller/validators" - validation "github.com/go-ozzo/ozzo-validation/v4" -) - -type PubkeyLoginRequest struct { - validatable validators.ValidatableImpl - Pubkey string `json:"pubkey"` - Challenge string `json:"challenge"` - Signature string `json:"signature"` -} - -func (r PubkeyLoginRequest) Validate() error { - return validation.ValidateStruct(&r, - validation.Field(&r.Pubkey, validation.Required, validation.By(validators.CheckPubkeyValidator)), - validation.Field(&r.Challenge, validation.Required), - validation.Field(&r.Signature, validation.Required, validation.Length(128, 128)), - ) -} - -func (r PubkeyLoginRequest) Import(d map[string]interface{}) (validators.Validatable, error) { - return r.validatable.Import(d, r) -} diff --git a/controller/request/register.go b/controller/request/register.go deleted file mode 100644 index 17831ab..0000000 --- a/controller/request/register.go +++ /dev/null @@ -1,25 +0,0 @@ -package request - -import ( - "git.lumeweb.com/LumeWeb/portal/controller/validators" - validation "github.com/go-ozzo/ozzo-validation/v4" - "github.com/go-ozzo/ozzo-validation/v4/is" -) - -type RegisterRequest struct { - validatable validators.ValidatableImpl - Email string `json:"email"` - Password string `json:"password"` - Pubkey string `json:"pubkey"` -} - -func (r RegisterRequest) Validate() error { - return validation.ValidateStruct(&r, - validation.Field(&r.Email, validation.Required, is.EmailFormat), - validation.Field(&r.Pubkey, validation.When(len(r.Password) == 0, validation.Required, validation.By(validators.CheckPubkeyValidator))), - validation.Field(&r.Password, validation.When(len(r.Pubkey) == 0, validation.Required)), - ) -} -func (r RegisterRequest) Import(d map[string]interface{}) (validators.Validatable, error) { - return r.validatable.Import(d, r) -} diff --git a/controller/response/auth_status.go b/controller/response/auth_status.go deleted file mode 100644 index b560c90..0000000 --- a/controller/response/auth_status.go +++ /dev/null @@ -1,5 +0,0 @@ -package response - -type AuthStatusResponse struct { - Status bool `json:"status"` -} diff --git a/controller/response/challenge.go b/controller/response/challenge.go deleted file mode 100644 index 109f467..0000000 --- a/controller/response/challenge.go +++ /dev/null @@ -1,5 +0,0 @@ -package response - -type ChallengeResponse struct { - Challenge string `json:"challenge"` -} diff --git a/controller/response/file_status.go b/controller/response/file_status.go deleted file mode 100644 index 472fc57..0000000 --- a/controller/response/file_status.go +++ /dev/null @@ -1,5 +0,0 @@ -package response - -type FileStatusResponse struct { - Status string `json:"status"` -} diff --git a/controller/response/login.go b/controller/response/login.go deleted file mode 100644 index 6a2afb2..0000000 --- a/controller/response/login.go +++ /dev/null @@ -1,5 +0,0 @@ -package response - -type LoginResponse struct { - Token string `json:"token"` -} diff --git a/controller/response/upload.go b/controller/response/upload.go deleted file mode 100644 index 0cb075d..0000000 --- a/controller/response/upload.go +++ /dev/null @@ -1,5 +0,0 @@ -package response - -type UploadResponse struct { - Cid string `json:"cid"` -} diff --git a/controller/response/upload_limit.go b/controller/response/upload_limit.go deleted file mode 100644 index 73d0fe1..0000000 --- a/controller/response/upload_limit.go +++ /dev/null @@ -1,5 +0,0 @@ -package response - -type UploadLimit struct { - Limit int64 `json:"limit"` -} diff --git a/controller/validators/validators.go b/controller/validators/validators.go deleted file mode 100644 index 5af093f..0000000 --- a/controller/validators/validators.go +++ /dev/null @@ -1,43 +0,0 @@ -package validators - -import ( - "crypto/ed25519" - "encoding/hex" - "errors" - "fmt" - validation "github.com/go-ozzo/ozzo-validation/v4" - "github.com/imdario/mergo" - "reflect" -) - -func CheckPubkeyValidator(value interface{}) error { - p, _ := value.(string) - pubkeyBytes, err := hex.DecodeString(p) - if err != nil { - return err - } - - if len(pubkeyBytes) != ed25519.PublicKeySize { - return errors.New(fmt.Sprintf("pubkey must be %d bytes in hexadecimal format", ed25519.PublicKeySize)) - } - - return nil -} - -type Validatable interface { - validation.Validatable - Import(d map[string]interface{}) (Validatable, error) -} - -type ValidatableImpl struct { -} - -func (v ValidatableImpl) Import(d map[string]interface{}, destType Validatable) (Validatable, error) { - instance := reflect.New(reflect.TypeOf(destType)).Interface().(Validatable) - // Perform the import logic - if err := mergo.Map(instance, d, mergo.WithOverride); err != nil { - return nil, err - } - - return instance, nil -} diff --git a/db/db.go b/db/db.go deleted file mode 100644 index a0c87fd..0000000 --- a/db/db.go +++ /dev/null @@ -1,71 +0,0 @@ -package db - -import ( - "fmt" - "git.lumeweb.com/LumeWeb/portal/model" - "github.com/spf13/viper" - "gorm.io/driver/mysql" - "gorm.io/driver/sqlite" - "gorm.io/gorm" -) - -// Declare a global variable to hold the database connection. -var db *gorm.DB - -// Init initializes the database connection based on the app's configuration settings. -func Init() { - // If the database connection has already been initialized, panic. - if db != nil { - panic("DB already initialized") - } - - // Retrieve database connection settings from the app's configuration using the viper library. - dbType := viper.GetString("database.type") - dbHost := viper.GetString("database.host") - dbPort := viper.GetInt("database.port") - dbSocket := viper.GetString("database.socket") - dbUser := viper.GetString("database.user") - dbPassword := viper.GetString("database.password") - dbName := viper.GetString("database.name") - dbPath := viper.GetString("database.path") - - var err error - var dsn string - switch dbType { - // Connect to a MySQL database. - case "mysql": - if dbSocket != "" { - dsn = fmt.Sprintf("%s:%s@unix(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", dbUser, dbPassword, dbSocket, dbName) - } else { - dsn = fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local", dbUser, dbPassword, dbHost, dbPort, dbName) - } - db, err = gorm.Open(mysql.Open(dsn), &gorm.Config{}) - // Connect to a SQLite database. - case "sqlite": - db, err = gorm.Open(sqlite.Open(dbPath), &gorm.Config{}) - // If the database type is unsupported, panic. - default: - panic(fmt.Errorf("Unsupported database type: %s \n", dbType)) - } - // If there was an error connecting to the database, panic. - if err != nil { - panic(fmt.Errorf("Failed to connect to database: %s \n", err)) - } - - // Automatically migrate the database schema based on the model definitions. - err = db.Migrator().AutoMigrate(&model.Account{}, &model.Key{}, &model.KeyChallenge{}, &model.LoginSession{}, &model.Upload{}, &model.Pin{}, &model.Tus{}, &model.Dnslink{}) - if err != nil { - panic(fmt.Errorf("Database setup failed database type: %s \n", err)) - } -} - -// Get returns the database connection instance. -func Get() *gorm.DB { - return db -} -func Close() error { - - instance, _ := db.DB() - - return instance.Close() -} diff --git a/dnslink/dnslink.go b/dnslink/dnslink.go deleted file mode 100644 index 1772a50..0000000 --- a/dnslink/dnslink.go +++ /dev/null @@ -1,227 +0,0 @@ -package dnslink - -import ( - "errors" - "git.lumeweb.com/LumeWeb/portal/cid" - "git.lumeweb.com/LumeWeb/portal/controller" - "git.lumeweb.com/LumeWeb/portal/db" - "git.lumeweb.com/LumeWeb/portal/logger" - "git.lumeweb.com/LumeWeb/portal/model" - "git.lumeweb.com/LumeWeb/portal/service/files" - dnslink "github.com/dnslink-std/go" - "github.com/kataras/iris/v12" - "github.com/kataras/iris/v12/context" - "github.com/vmihailenco/msgpack/v5" - "go.uber.org/zap" - "io" - "path/filepath" - "strings" -) - -var ( - ErrFailedReadAppManifest = errors.New("failed to read app manifest") - ErrInvalidAppManifest = errors.New("invalid app manifest") -) - -type CID string -type ExtraMetadata map[string]interface{} - -type WebAppMetadata struct { - Schema string `msgpack:"$schema,omitempty"` - Type string `msgpack:"type"` - Name string `msgpack:"name,omitempty"` - TryFiles []string `msgpack:"tryFiles,omitempty"` - ErrorPages map[string]string `msgpack:"errorPages,omitempty"` - Paths map[string]PathContent `msgpack:"paths"` - ExtraMetadata ExtraMetadata `msgpack:"extraMetadata,omitempty"` -} - -type PathContent struct { - CID CID `msgpack:"cid"` - ContentType string `msgpack:"contentType,omitempty"` -} - -func Handler(ctx *context.Context) { - record := model.Dnslink{} - - domain := ctx.Request().Host - - if err := db.Get().Model(&model.Dnslink{Domain: domain}).First(&record).Error; err != nil { - ctx.StopWithStatus(iris.StatusNotFound) - return - } - ret, err := dnslink.Resolve(domain) - if err != nil { - switch e := err.(type) { - default: - ctx.StopWithStatus(iris.StatusInternalServerError) - return - case dnslink.DNSRCodeError: - if e.DNSRCode == 3 { - ctx.StopWithStatus(iris.StatusNotFound) - return - } - } - } - - if ret.Links["sia"] == nil || len(ret.Links["sia"]) == 0 { - ctx.StopWithStatus(iris.StatusNotFound) - return - } - - appManifest := ret.Links["sia"][0] - - decodedCid, valid := controller.ValidateCid(appManifest.Identifier, true, ctx) - if !valid { - return - } - - manifest := fetchManifest(ctx, decodedCid) - if manifest == nil { - return - } - - path := ctx.Path() - - if strings.HasSuffix(path, "/") || filepath.Ext(path) == "" { - var directoryIndex *PathContent - for _, indexFile := range manifest.TryFiles { - path, exists := manifest.Paths[indexFile] - - if !exists { - continue - } - - _, err := cid.Valid(string(manifest.Paths[indexFile].CID)) - - if err != nil { - continue - } - - cidObject, _ := cid.Decode(string(path.CID)) - hashHex := cidObject.StringHash() - - status := files.Status(hashHex) - - if status == files.STATUS_NOT_FOUND { - continue - } - if status == files.STATUS_UPLOADED { - directoryIndex = &path - break - } - } - - if directoryIndex == nil { - ctx.StopWithStatus(iris.StatusNotFound) - return - } - - file, err := fetchFile(directoryIndex) - if maybeHandleFileError(err, ctx) { - return - } - ctx.Header("Content-Type", directoryIndex.ContentType) - streamFile(file, ctx) - return - } - - path = strings.TrimLeft(path, "/") - - requestedPath, exists := manifest.Paths[path] - - if !exists { - ctx.StopWithStatus(iris.StatusNotFound) - return - } - - file, err := fetchFile(&requestedPath) - if maybeHandleFileError(err, ctx) { - return - } - ctx.Header("Content-Type", requestedPath.ContentType) - streamFile(file, ctx) -} - -func maybeHandleFileError(err error, ctx *context.Context) bool { - if err != nil { - if err == files.ErrInvalidFile { - controller.SendError(ctx, err, iris.StatusNotFound) - return true - } - controller.SendError(ctx, err, iris.StatusInternalServerError) - } - - return err != nil -} - -func streamFile(stream io.Reader, ctx *context.Context) { - err := controller.PassThroughStream(stream, ctx) - if err != controller.ErrStreamDone && controller.InternalError(ctx, err) { - logger.Get().Debug("failed streaming file", zap.Error(err)) - } -} - -func fetchFile(path *PathContent) (io.Reader, error) { - _, err := cid.Valid(string(path.CID)) - - if err != nil { - return nil, err - } - - cidObject, _ := cid.Decode(string(path.CID)) - hashHex := cidObject.StringHash() - - status := files.Status(hashHex) - - if status == files.STATUS_NOT_FOUND { - return nil, errors.New("cid not found") - } - if status == files.STATUS_UPLOADED { - stream, err := files.Download(hashHex) - - if err != nil { - return nil, err - } - - return stream, nil - } - - return nil, errors.New("cid not found") -} - -func fetchManifest(ctx iris.Context, hash string) *WebAppMetadata { - stream, err := files.Download(hash) - - if err != nil { - if errors.Is(err, files.ErrInvalidFile) { - controller.SendError(ctx, err, iris.StatusNotFound) - return nil - } - controller.SendError(ctx, err, iris.StatusInternalServerError) - } - var metadata WebAppMetadata - - data, err := io.ReadAll(stream) - - if err != nil { - logger.Get().Debug(ErrFailedReadAppManifest.Error(), zap.Error(err)) - controller.SendError(ctx, ErrFailedReadAppManifest, iris.StatusInternalServerError) - return nil - } - - err = msgpack.Unmarshal(data, &metadata) - if err != nil { - logger.Get().Debug(ErrFailedReadAppManifest.Error(), zap.Error(err)) - controller.SendError(ctx, ErrFailedReadAppManifest, iris.StatusInternalServerError) - return nil - } - - if metadata.Type != "web_app" { - logger.Get().Debug(ErrInvalidAppManifest.Error()) - controller.SendError(ctx, ErrInvalidAppManifest, iris.StatusInternalServerError) - return nil - } - - return &metadata -} diff --git a/go.mod b/go.mod index c9394f8..1113226 100644 --- a/go.mod +++ b/go.mod @@ -1,128 +1,3 @@ module git.lumeweb.com/LumeWeb/portal -go 1.18 - -require ( - github.com/dnslink-std/go v0.6.0 - github.com/go-ozzo/ozzo-validation/v4 v4.3.0 - github.com/go-resty/resty/v2 v2.7.0 - github.com/golang-queue/queue v0.1.3 - github.com/huandu/go-clone v1.6.0 - github.com/imdario/mergo v0.3.16 - github.com/iris-contrib/swagger v0.0.0-20230531125653-f4ee631290a7 - github.com/kataras/iris/v12 v12.2.0 - github.com/kataras/jwt v0.1.8 - github.com/multiformats/go-multibase v0.2.0 - github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.16.0 - github.com/swaggo/swag v1.16.1 - github.com/tus/tusd v1.11.0 - github.com/vmihailenco/msgpack/v5 v5.3.5 - go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.10.0 - golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df - gorm.io/driver/mysql v1.5.1 - gorm.io/driver/sqlite v1.5.2 - gorm.io/gorm v1.25.2 - lukechampine.com/blake3 v1.2.1 -) - -require ( - github.com/BurntSushi/toml v1.3.2 // indirect - github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 // indirect - github.com/CloudyKit/jet/v6 v6.2.0 // indirect - github.com/Joker/jade v1.1.3 // indirect - github.com/KyleBanks/depth v1.2.1 // indirect - github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06 // indirect - github.com/andybalholm/brotli v1.0.5 // indirect - github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect - github.com/aymerick/douceur v0.2.0 // indirect - github.com/blang/semver/v4 v4.0.0 // indirect - github.com/bmizerany/pat v0.0.0-20210406213842-e4b6760bdd6f // indirect - github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 // indirect - github.com/fatih/structs v1.1.0 // indirect - github.com/flosch/pongo2/v4 v4.0.2 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/spec v0.20.9 // indirect - github.com/go-openapi/swag v0.22.4 // indirect - github.com/go-sql-driver/mysql v1.7.1 // indirect - github.com/gobwas/httphead v0.1.0 // indirect - github.com/gobwas/pool v0.2.1 // indirect - github.com/gobwas/ws v1.2.1 // indirect - github.com/goccy/go-json v0.10.2 // indirect - github.com/golang/snappy v0.0.4 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/gorilla/css v1.0.0 // indirect - github.com/gorilla/websocket v1.5.0 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/iris-contrib/go.uuid v2.0.0+incompatible // indirect - github.com/iris-contrib/schema v0.0.6 // indirect - github.com/jinzhu/inflection v1.0.0 // indirect - github.com/jinzhu/now v1.1.5 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/kataras/blocks v0.0.7 // indirect - github.com/kataras/golog v0.1.9 // indirect - github.com/kataras/neffos v0.0.21 // indirect - github.com/kataras/pio v0.0.12 // indirect - github.com/kataras/sitemap v0.0.6 // indirect - github.com/kataras/tunnel v0.0.4 // indirect - github.com/klauspost/compress v1.16.6 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect - github.com/magiconair/properties v1.8.7 // indirect - github.com/mailgun/raymond/v2 v2.0.48 // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/mattn/go-sqlite3 v1.14.17 // indirect - github.com/mediocregopher/radix/v3 v3.8.1 // indirect - github.com/microcosm-cc/bluemonday v1.0.24 // indirect - github.com/miekg/dns v1.1.43 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/mr-tron/base58 v1.2.0 // indirect - github.com/multiformats/go-base32 v0.1.0 // indirect - github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/nats-io/nats.go v1.27.1 // indirect - github.com/nats-io/nkeys v0.4.4 // indirect - github.com/nats-io/nuid v1.0.1 // indirect - github.com/pelletier/go-toml/v2 v2.0.8 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/schollz/closestmatch v2.1.0+incompatible // indirect - github.com/sergi/go-diff v1.1.0 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect - github.com/spf13/afero v1.9.5 // indirect - github.com/spf13/cast v1.5.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/subosito/gotenv v1.4.2 // indirect - github.com/tdewolff/minify/v2 v2.12.7 // indirect - github.com/tdewolff/parse/v2 v2.6.6 // indirect - github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect - github.com/yosssi/ace v0.0.5 // indirect - go.uber.org/atomic v1.11.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.11.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/text v0.10.0 // indirect - golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.10.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/protobuf v1.31.0 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) - -replace go.uber.org/multierr => go.uber.org/multierr v1.9.0 - -replace ( - github.com/tus/tusd => git.lumeweb.com/LumeWeb/tusd v1.11.1-0.20230629085530-7b20ce6a9ae5 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.39.0 - go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.14.0 - go.opentelemetry.io/otel/exporters/otlp/internal/retry => go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.12.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace => go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.12.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp => go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.12.0 - go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v0.37.0 - go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v1.12.0 - go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.14.0 - go.opentelemetry.io/proto/otlp => go.opentelemetry.io/proto/otlp v0.19.0 -) +go 1.20 diff --git a/logger/logger.go b/logger/logger.go deleted file mode 100644 index 72391c0..0000000 --- a/logger/logger.go +++ /dev/null @@ -1,30 +0,0 @@ -package logger - -import ( - "github.com/spf13/viper" - "go.uber.org/zap" - "log" -) - -var logger *zap.Logger - -func Init() { - var newLogger *zap.Logger - var err error - - if viper.GetBool("debug") { - newLogger, err = zap.NewDevelopment() - } else { - newLogger, err = zap.NewProduction() - } - - if err != nil { - log.Fatal(err) - } - - logger = newLogger -} - -func Get() *zap.Logger { - return logger -} diff --git a/middleware/jwt.go b/middleware/jwt.go deleted file mode 100644 index 64bec9d..0000000 --- a/middleware/jwt.go +++ /dev/null @@ -1,28 +0,0 @@ -package middleware - -import ( - "git.lumeweb.com/LumeWeb/portal/service/account" - "git.lumeweb.com/LumeWeb/portal/service/auth" - "github.com/kataras/iris/v12" -) - -func VerifyJwt(ctx iris.Context) { - token := auth.GetRequestAuthCode(ctx) - - if len(token) == 0 { - ctx.StopWithError(iris.StatusUnauthorized, auth.ErrInvalidToken) - return - } - - acct, err := auth.VerifyLoginToken(token) - - if err != nil { - ctx.StopWithError(iris.StatusUnauthorized, auth.ErrInvalidToken) - return - } - - err = ctx.SetUser(account.NewUser(acct)) - if err != nil { - ctx.StopWithError(iris.StatusInternalServerError, err) - } -} diff --git a/model/account.go b/model/account.go deleted file mode 100644 index 50be927..0000000 --- a/model/account.go +++ /dev/null @@ -1,17 +0,0 @@ -package model - -import ( - "gorm.io/gorm" - "time" -) - -type Account struct { - gorm.Model - ID uint `gorm:"primaryKey" gorm:"AUTO_INCREMENT"` - Email string `gorm:"uniqueIndex"` - Password *string - CreatedAt time.Time - UpdatedAt time.Time - LoginTokens []LoginSession - Keys []Key -} diff --git a/model/dnslink.go b/model/dnslink.go deleted file mode 100644 index ddb5cc2..0000000 --- a/model/dnslink.go +++ /dev/null @@ -1,11 +0,0 @@ -package model - -import ( - "gorm.io/gorm" -) - -type Dnslink struct { - gorm.Model - ID uint `gorm:"primaryKey" gorm:"AUTO_INCREMENT"` - Domain string `gorm:"uniqueIndex"` -} diff --git a/model/key.go b/model/key.go deleted file mode 100644 index dda0fc4..0000000 --- a/model/key.go +++ /dev/null @@ -1,16 +0,0 @@ -package model - -import ( - "gorm.io/gorm" - "time" -) - -type Key struct { - gorm.Model - ID uint `gorm:"primaryKey" gorm:"AUTO_INCREMENT"` - AccountID uint - Account Account - Pubkey string - CreatedAt time.Time - UpdatedAt time.Time -} diff --git a/model/key_challenge.go b/model/key_challenge.go deleted file mode 100644 index 20bed47..0000000 --- a/model/key_challenge.go +++ /dev/null @@ -1,15 +0,0 @@ -package model - -import ( - "gorm.io/gorm" - "time" -) - -type KeyChallenge struct { - gorm.Model - ID uint `gorm:"primaryKey" gorm:"AUTO_INCREMENT"` - AccountID uint - Account Account - Challenge string `gorm:"not null"` - Expiration time.Time -} diff --git a/model/login_session.go b/model/login_session.go deleted file mode 100644 index 13070a5..0000000 --- a/model/login_session.go +++ /dev/null @@ -1,22 +0,0 @@ -package model - -import ( - "gorm.io/gorm" - "time" -) - -type LoginSession struct { - gorm.Model - ID uint `gorm:"primaryKey" gorm:"AUTO_INCREMENT"` - AccountID uint - Account Account - Token string `gorm:"uniqueIndex"` - Expiration time.Time - CreatedAt time.Time - UpdatedAt time.Time -} - -func (s *LoginSession) BeforeCreate(tx *gorm.DB) (err error) { - s.Expiration = time.Now().Add(time.Hour * 24) - return -} diff --git a/model/pin.go b/model/pin.go deleted file mode 100644 index dd89788..0000000 --- a/model/pin.go +++ /dev/null @@ -1,12 +0,0 @@ -package model - -import "gorm.io/gorm" - -type Pin struct { - gorm.Model - ID uint `gorm:"primaryKey" gorm:"AUTO_INCREMENT"` - AccountID uint `gorm:"uniqueIndex:idx_account_upload"` - UploadID uint `gorm:"uniqueIndex:idx_account_upload"` - Account Account - Upload Upload -} diff --git a/model/tus.go b/model/tus.go deleted file mode 100644 index 7dec152..0000000 --- a/model/tus.go +++ /dev/null @@ -1,15 +0,0 @@ -package model - -import ( - "gorm.io/gorm" -) - -type Tus struct { - gorm.Model - ID uint `gorm:"primaryKey" gorm:"AUTO_INCREMENT"` - UploadID string - Hash string - Info string - AccountID uint - Account Account -} diff --git a/model/upload.go b/model/upload.go deleted file mode 100644 index 959037d..0000000 --- a/model/upload.go +++ /dev/null @@ -1,13 +0,0 @@ -package model - -import ( - "gorm.io/gorm" -) - -type Upload struct { - gorm.Model - ID uint `gorm:"primaryKey" gorm:"AUTO_INCREMENT"` - AccountID uint `gorm:"index"` - Account Account - Hash string `gorm:"uniqueIndex"` -} diff --git a/service/account/account.go b/service/account/account.go deleted file mode 100644 index 11a1bbc..0000000 --- a/service/account/account.go +++ /dev/null @@ -1,76 +0,0 @@ -package account - -import ( - "errors" - "git.lumeweb.com/LumeWeb/portal/db" - "git.lumeweb.com/LumeWeb/portal/logger" - "git.lumeweb.com/LumeWeb/portal/model" - "go.uber.org/zap" - "gorm.io/gorm" -) - -var ( - ErrEmailExists = errors.New("Account with email already exists") - ErrPubkeyExists = errors.New("Account with pubkey already exists") - ErrQueryingAcct = errors.New("Error querying accounts") - ErrFailedHashPassword = errors.New("Failed to hash password") - ErrFailedCreateAccount = errors.New("Failed to create account") -) - -func Register(email string, password string, pubkey string) error { - err := db.Get().Transaction(func(tx *gorm.DB) error { - existingAccount := model.Account{} - err := tx.Where("email = ?", email).First(&existingAccount).Error - if err == nil { - return ErrEmailExists - } else if !errors.Is(err, gorm.ErrRecordNotFound) { - return err - } - - if len(pubkey) > 0 { - var count int64 - err := tx.Model(&model.Key{}).Where("pubkey = ?", pubkey).Count(&count).Error - if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { - return err - } - if count > 0 { - // An account with the same pubkey already exists. - // Return an error response to the client. - return ErrPubkeyExists - } - } - - // Create a new Account model with the provided email and hashed password. - account := model.Account{ - Email: email, - } - - // Hash the password before saving it to the database. - if len(password) > 0 { - hashedPassword, err := hashPassword(password) - if err != nil { - return err - } - account.Password = &hashedPassword - } - - if err := tx.Create(&account).Error; err != nil { - return err - } - - if len(pubkey) > 0 { - if err := tx.Create(&model.Key{Account: account, Pubkey: pubkey}).Error; err != nil { - return err - } - } - - return nil - }) - - if err != nil { - logger.Get().Error(ErrFailedCreateAccount.Error(), zap.Error(err)) - return err - } - - return nil -} diff --git a/service/account/user.go b/service/account/user.go deleted file mode 100644 index fc4103a..0000000 --- a/service/account/user.go +++ /dev/null @@ -1,20 +0,0 @@ -package account - -import ( - "git.lumeweb.com/LumeWeb/portal/model" - "github.com/kataras/iris/v12/context" - "strconv" -) - -type User struct { - context.User - account *model.Account -} - -func (u User) GetID() (string, error) { - return strconv.Itoa(int(u.account.ID)), nil -} - -func NewUser(account *model.Account) *User { - return &User{account: account} -} diff --git a/service/account/util.go b/service/account/util.go deleted file mode 100644 index 95d3a33..0000000 --- a/service/account/util.go +++ /dev/null @@ -1,19 +0,0 @@ -package account - -import ( - "git.lumeweb.com/LumeWeb/portal/logger" - "go.uber.org/zap" - "golang.org/x/crypto/bcrypt" -) - -func hashPassword(password string) (string, error) { - // Generate a new bcrypt hash from the provided password. - hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) - if err != nil { - logger.Get().Error(ErrFailedHashPassword.Error(), zap.Error(err)) - return "", ErrFailedHashPassword - } - - // Convert the hashed password to a string and return it. - return string(hashedPassword), nil -} diff --git a/service/auth/auth.go b/service/auth/auth.go deleted file mode 100644 index a039496..0000000 --- a/service/auth/auth.go +++ /dev/null @@ -1,264 +0,0 @@ -package auth - -import ( - "crypto/ed25519" - "crypto/x509" - "encoding/hex" - "encoding/pem" - "errors" - "git.lumeweb.com/LumeWeb/portal/config" - "git.lumeweb.com/LumeWeb/portal/db" - "git.lumeweb.com/LumeWeb/portal/logger" - "git.lumeweb.com/LumeWeb/portal/model" - "github.com/kataras/jwt" - "github.com/spf13/viper" - "go.uber.org/zap" - "os" - "path" - "path/filepath" - "strings" - "time" -) - -var jwtKey = ed25519.PrivateKey{} - -var blocklist *jwt.Blocklist - -var ( - ErrInvalidEmailPassword = errors.New("Invalid email or password") - ErrPubkeyOnly = errors.New("Only pubkey login is supported") - ErrFailedGenerateToken = errors.New("Failed to generate token") - ErrFailedGenerateKeyChallenge = errors.New("Failed to generate key challenge") - ErrFailedSignJwt = errors.New("Failed to sign jwt") - ErrFailedSaveToken = errors.New("Failed to sign token") - ErrFailedDeleteKeyChallenge = errors.New("Failed to delete key challenge") - ErrFailedInvalidateToken = errors.New("Failed to invalidate token") - ErrInvalidKeyChallenge = errors.New("Invalid key challenge") - ErrInvalidPubkey = errors.New("Invalid pubkey") - ErrInvalidSignature = errors.New("Invalid signature") - ErrInvalidToken = errors.New("Invalid token") -) - -func Init() { - blocklist = jwt.NewBlocklist(0) - - configFile := viper.ConfigFileUsed() - - var jwtPemPath string - jwtPemName := "jwt.pem" - - if configFile == "" { - jwtPemPath = path.Join(config.ConfigFilePaths[0], jwtPemName) - } else { - jwtPemPath = path.Join(filepath.Dir(configFile), jwtPemName) - } - - if _, err := os.Stat(jwtPemPath); err != nil { - _, private, err := ed25519.GenerateKey(nil) - if err != nil { - logger.Get().Fatal("Failed to compute JWT private key", zap.Error(err)) - } - - privateBytes, err := x509.MarshalPKCS8PrivateKey(private) - - if err != nil { - logger.Get().Fatal("Failed to create marshal private key", zap.Error(err)) - } - - var pemPrivateBlock = &pem.Block{ - Type: "PRIVATE KEY", - Bytes: privateBytes, - } - - pemPrivateFile, err := os.Create(jwtPemPath) - - if err != nil { - logger.Get().Fatal("Failed to create empty file for JWT private PEM", zap.Error(err)) - } - - err = pem.Encode(pemPrivateFile, pemPrivateBlock) - if err != nil { - logger.Get().Fatal("Failed to write JWT private PEM", zap.Error(err)) - } - - jwtKey = private - } else { - data, err := os.ReadFile(jwtPemPath) - if err != nil { - logger.Get().Fatal("Failed to read JWT private PEM", zap.Error(err)) - } - - pemBlock, _ := pem.Decode(data) - if err != nil { - logger.Get().Fatal("Failed to decode JWT private PEM", zap.Error(err)) - } - - privateBytes, err := x509.ParsePKCS8PrivateKey(pemBlock.Bytes) - - if err != nil { - logger.Get().Fatal("Failed to unmarshal JWT private PEM", zap.Error(err)) - } - - jwtKey = privateBytes.(ed25519.PrivateKey) - } - -} - -func LoginWithPassword(email string, password string) (string, error) { - // Retrieve the account for the given email. - account := model.Account{} - if err := db.Get().Model(&account).Where("email = ?", email).First(&account).Error; err != nil { - logger.Get().Debug(ErrInvalidEmailPassword.Error(), zap.String("email", email)) - return "", ErrInvalidEmailPassword - } - - if account.Password == nil || len(*account.Password) == 0 { - logger.Get().Debug(ErrPubkeyOnly.Error(), zap.String("email", email)) - return "", ErrPubkeyOnly - } - - // Verify the provided password against the hashed password stored in the database. - if err := verifyPassword(*account.Password, password); err != nil { - logger.Get().Debug(ErrPubkeyOnly.Error(), zap.String("email", email)) - return "", ErrInvalidEmailPassword - } - - // Generate a JWT token for the authenticated user. - token, err := generateAndSaveLoginToken(account.ID, 24*time.Hour) - if err != nil { - return "", err - } - - return token, nil -} - -func LoginWithPubkey(pubkey string, challenge string, signature string) (string, error) { - pubkey = strings.ToLower(pubkey) - signature = strings.ToLower(signature) - - // Retrieve the key challenge for the given challenge. - challengeObj := model.KeyChallenge{} - if err := db.Get().Model(challengeObj).Where("challenge = ?", challenge).First(&challengeObj).Error; err != nil { - logger.Get().Debug(ErrInvalidKeyChallenge.Error(), zap.Error(err), zap.String("challenge", challenge)) - return "", ErrInvalidKeyChallenge - } - - verifiedToken, err := jwt.Verify(jwt.EdDSA, jwtKey, []byte(challenge), blocklist) - if err != nil { - logger.Get().Debug(ErrInvalidKeyChallenge.Error(), zap.Error(err), zap.String("challenge", challenge)) - return "", ErrInvalidKeyChallenge - } - - rawPubKey, err := hex.DecodeString(pubkey) - if err != nil { - logger.Get().Debug(ErrInvalidPubkey.Error(), zap.Error(err), zap.String("pubkey", pubkey)) - return "", ErrInvalidPubkey - } - - rawSignature, err := hex.DecodeString(signature) - if err != nil { - logger.Get().Debug(ErrInvalidPubkey.Error(), zap.Error(err), zap.String("signature", pubkey)) - return "", ErrInvalidSignature - } - - publicKeyDecoded := ed25519.PublicKey(rawPubKey) - - // Verify the challenge signature. - if !ed25519.Verify(publicKeyDecoded, []byte(challenge), rawSignature) { - logger.Get().Debug(ErrInvalidKeyChallenge.Error(), zap.Error(err), zap.String("challenge", challenge)) - return "", ErrInvalidKeyChallenge - } - - // Generate a JWT token for the authenticated user. - token, err := generateAndSaveLoginToken(challengeObj.AccountID, 24*time.Hour) - if err != nil { - return "", err - } - - err = blocklist.InvalidateToken(verifiedToken.Token, verifiedToken.StandardClaims) - if err != nil { - logger.Get().Error(ErrFailedInvalidateToken.Error(), zap.Error(err), zap.String("pubkey", pubkey), zap.ByteString("token", verifiedToken.Token), zap.String("challenge", challenge)) - return "", ErrFailedInvalidateToken - } - - if err := db.Get().Delete(&challengeObj).Error; err != nil { - logger.Get().Debug(ErrFailedDeleteKeyChallenge.Error(), zap.Error(err)) - return "", ErrFailedDeleteKeyChallenge - } - - return token, nil -} - -func GeneratePubkeyChallenge(pubkey string) (string, error) { - pubkey = strings.ToLower(pubkey) - - // Retrieve the account for the given email. - account := model.Key{} - if err := db.Get().Where("pubkey = ?", pubkey).First(&account).Error; err != nil { - logger.Get().Debug("failed to query pubkey", zap.Error(err)) - return "", errors.New("invalid pubkey") - } - - // Generate a random challenge string. - challenge, err := generateAndSaveChallengeToken(account.AccountID, time.Minute) - if err != nil { - logger.Get().Error(ErrFailedGenerateKeyChallenge.Error()) - return "", ErrFailedGenerateKeyChallenge - } - - return challenge, nil -} - -func Logout(token string) error { - // Verify the provided token. - claims, err := jwt.Verify(jwt.EdDSA, jwtKey, []byte(token), blocklist) - if err != nil { - logger.Get().Debug(ErrInvalidToken.Error(), zap.Error(err)) - return ErrInvalidToken - } - - err = blocklist.InvalidateToken(claims.Token, claims.StandardClaims) - if err != nil { - logger.Get().Error(ErrFailedInvalidateToken.Error(), zap.Error(err), zap.String("token", token)) - return ErrFailedInvalidateToken - } - - // Retrieve the key challenge for the given challenge. - session := model.LoginSession{} - if err := db.Get().Model(session).Where("token = ?", token).First(&session).Error; err != nil { - logger.Get().Debug(ErrFailedInvalidateToken.Error(), zap.Error(err), zap.String("token", token)) - return ErrFailedInvalidateToken - } - - db.Get().Delete(&session) - - return nil -} - -func VerifyLoginToken(token string) (*model.Account, error) { - uvt, err := jwt.Decode([]byte(token)) - if err != nil { - return nil, ErrInvalidToken - } - - var claim jwt.Claims - - err = uvt.Claims(&claim) - if err != nil { - return nil, ErrInvalidToken - } - - session := model.LoginSession{} - if err := db.Get().Model(session).Preload("Account").Where("token = ?", token).First(&session).Error; err != nil { - logger.Get().Debug(ErrInvalidToken.Error(), zap.Error(err), zap.String("token", token)) - return nil, ErrInvalidToken - } - - _, err = jwt.Verify(jwt.EdDSA, jwtKey, []byte(token), blocklist) - if err != nil { - db.Get().Delete(&session) - return nil, err - } - - return &session.Account, nil -} diff --git a/service/auth/util.go b/service/auth/util.go deleted file mode 100644 index a63f3b6..0000000 --- a/service/auth/util.go +++ /dev/null @@ -1,142 +0,0 @@ -package auth - -import ( - "errors" - "git.lumeweb.com/LumeWeb/portal/db" - "git.lumeweb.com/LumeWeb/portal/logger" - "git.lumeweb.com/LumeWeb/portal/model" - "github.com/kataras/iris/v12" - "github.com/kataras/jwt" - "go.uber.org/zap" - "golang.org/x/crypto/bcrypt" - "strconv" - "strings" - "time" -) - -// verifyPassword compares the provided plaintext password with a hashed password and returns an error if they don't match. -func verifyPassword(hashedPassword, password string) error { - err := bcrypt.CompareHashAndPassword([]byte(hashedPassword), []byte(password)) - if err != nil { - return errors.New("invalid email or password") - } - return nil -} - -// generateToken generates a JWT token for the given account ID. -func generateToken(maxAge time.Duration, ttype string) (string, error) { - // Define the JWT claims. - claim := jwt.Claims{ - Expiry: time.Now().Add(time.Hour * 24).Unix(), // Token expires in 24 hours. - IssuedAt: time.Now().Unix(), - Audience: []string{ttype}, - } - - token, err := jwt.Sign(jwt.EdDSA, jwtKey, claim, jwt.MaxAge(maxAge)) - - if err != nil { - logger.Get().Error(ErrFailedSignJwt.Error(), zap.Error(err)) - return "", err - } - - return string(token), nil -} - -func generateAndSaveLoginToken(accountID uint, maxAge time.Duration) (string, error) { - // Generate a JWT token for the authenticated user. - token, err := generateToken(maxAge, "auth") - if err != nil { - logger.Get().Error(ErrFailedGenerateToken.Error()) - return "", ErrFailedGenerateToken - } - - verifiedToken, _ := jwt.Verify(jwt.EdDSA, jwtKey, []byte(token), blocklist) - var claim *jwt.Claims - - _ = verifiedToken.Claims(&claim) - - // Save the token to the database. - session := model.LoginSession{ - Account: model.Account{ID: accountID}, - Token: token, - Expiration: claim.ExpiresAt(), - } - - existingSession := model.LoginSession{} - - err = db.Get().Where("token = ?", token).First(&existingSession).Error - if err == nil { - return token, nil - } - - if err := db.Get().Create(&session).Error; err != nil { - if strings.Contains(err.Error(), "Duplicate entry") { - return token, nil - } - logger.Get().Error(ErrFailedSaveToken.Error(), zap.Uint("account_id", accountID), zap.Duration("max_age", maxAge)) - return "", ErrFailedSaveToken - } - - return token, nil -} - -func generateAndSaveChallengeToken(accountID uint, maxAge time.Duration) (string, error) { - // Generate a JWT token for the authenticated user. - token, err := generateToken(maxAge, "challenge") - if err != nil { - logger.Get().Error(ErrFailedGenerateToken.Error(), zap.Error(err)) - return "", ErrFailedGenerateToken - } - - verifiedToken, err := jwt.Verify(jwt.EdDSA, jwtKey, []byte(token), blocklist) - - if err != nil { - return "", ErrFailedGenerateToken - } - - var claim *jwt.Claims - - _ = verifiedToken.Claims(&claim) - - // Save the token to the database. - keyChallenge := model.KeyChallenge{ - AccountID: accountID, - Challenge: token, - Expiration: claim.ExpiresAt(), - } - - if err := db.Get().Create(&keyChallenge).Error; err != nil { - logger.Get().Error(ErrFailedSaveToken.Error(), zap.Error(err)) - return "", ErrFailedSaveToken - } - - return token, nil -} - -func GetRequestAuthCode(ctx iris.Context) string { - authHeader := ctx.GetHeader("Authorization") - if authHeader == "" { - return "" - } - - // pure check: authorization header format must be Bearer {token} - authHeaderParts := strings.Split(authHeader, " ") - if len(authHeaderParts) != 2 || strings.ToLower(authHeaderParts[0]) != "bearer" { - return "" - } - - return authHeaderParts[1] -} - -func GetCurrentUserId(ctx iris.Context) uint { - usr := ctx.User() - - if usr == nil { - return 0 - } - - sid, _ := usr.GetID() - userID, _ := strconv.Atoi(sid) - - return uint(userID) -} diff --git a/service/files/files.go b/service/files/files.go deleted file mode 100644 index 4247973..0000000 --- a/service/files/files.go +++ /dev/null @@ -1,316 +0,0 @@ -package files - -import ( - "bytes" - "context" - "encoding/hex" - "errors" - "fmt" - "git.lumeweb.com/LumeWeb/portal/bao" - "git.lumeweb.com/LumeWeb/portal/db" - "git.lumeweb.com/LumeWeb/portal/logger" - "git.lumeweb.com/LumeWeb/portal/model" - "git.lumeweb.com/LumeWeb/portal/shared" - "git.lumeweb.com/LumeWeb/portal/tusstore" - "github.com/go-resty/resty/v2" - "github.com/spf13/viper" - _ "github.com/tus/tusd/pkg/handler" - "go.uber.org/zap" - "gorm.io/gorm" - "io" - "strings" -) - -const ( - STATUS_UPLOADED = iota - STATUS_UPLOADING = iota - STATUS_NOT_FOUND = iota -) - -var ( - ErrAlreadyExists = errors.New("Upload already exists") - ErrFailedFetchObject = errors.New("Failed fetching object") - ErrFailedFetchObjectProof = errors.New("Failed fetching object proof") - ErrFailedFetchTusObject = errors.New("Failed fetching tus object") - ErrFailedHashFile = errors.New("Failed to hash file") - ErrFailedQueryTusUpload = errors.New("Failed to query tus uploads") - ErrFailedQueryUpload = errors.New("Failed to query uploads") - ErrFailedQueryPins = errors.New("Failed to query pins") - ErrFailedSaveUpload = errors.New("Failed saving upload to db") - ErrFailedSavePin = errors.New("Failed saving pin to db") - ErrFailedUpload = errors.New("Failed uploading object") - ErrFailedUploadProof = errors.New("Failed uploading object proof") - ErrFileExistsOutOfSync = errors.New("File already exists in network, but missing in database") - ErrFileHashMismatch = errors.New("File hash does not match provided file hash") - ErrInvalidFile = errors.New("Invalid file") -) - -var client *resty.Client - -func Init() { - client = resty.New() - client.SetBaseURL("http://localhost:9980/api") - client.SetBasicAuth("", viper.GetString("renterd-api-password")) - client.SetDisableWarn(true) - client.SetCloseConnection(true) -} - -func Upload(r io.ReadSeeker, size int64, hash []byte, accountID uint) (model.Upload, error) { - var upload model.Upload - - tree, hashBytes, err := bao.ComputeTree(r, size) - - if err != nil { - logger.Get().Error(ErrFailedHashFile.Error(), zap.Error(err)) - return upload, ErrFailedHashFile - } - - if hash != nil { - if bytes.Compare(hashBytes[:], hash) != 0 { - logger.Get().Error(ErrFileHashMismatch.Error()) - return upload, ErrFileHashMismatch - } - } - - hashHex := hex.EncodeToString(hashBytes[:]) - - _, err = r.Seek(0, io.SeekStart) - - if err != nil { - return upload, err - } - - result := db.Get().Where(&model.Upload{Hash: hashHex}).First(&upload) - if !errors.Is(result.Error, gorm.ErrRecordNotFound) { - if err != nil { - logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(err)) - return upload, ErrFailedQueryUpload - } - - logger.Get().Info(ErrAlreadyExists.Error()) - return upload, nil - } - - objectExistsResult, err := client.R().Get(getBusObjectUrl(hashHex)) - - if err != nil { - logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(err)) - return upload, ErrFailedQueryUpload - } - - objectStatusCode := objectExistsResult.StatusCode() - - if objectStatusCode == 500 { - bodyErr := objectExistsResult.String() - if !strings.Contains(bodyErr, "no slabs found") { - logger.Get().Error(ErrFailedFetchObject.Error(), zap.String("error", objectExistsResult.String())) - return upload, ErrFailedFetchObject - } - - objectStatusCode = 404 - } - - proofExistsResult, err := client.R().Get(getBusProofUrl(hashHex)) - - if err != nil { - logger.Get().Error(ErrFailedFetchObjectProof.Error(), zap.Error(err)) - return upload, ErrFailedFetchObjectProof - } - - proofStatusCode := proofExistsResult.StatusCode() - - if proofStatusCode == 500 { - bodyErr := proofExistsResult.String() - if !strings.Contains(bodyErr, "no slabs found") { - logger.Get().Error(ErrFailedFetchObjectProof.Error(), zap.String("error", proofExistsResult.String())) - return upload, ErrFailedFetchObjectProof - } - - objectStatusCode = 404 - } - - if objectStatusCode != 404 && proofStatusCode != 404 { - logger.Get().Error(ErrFileExistsOutOfSync.Error(), zap.String("hash", hashHex)) - return upload, ErrFileExistsOutOfSync - } - - ret, err := client.R().SetBody(r).Put(getWorkerObjectUrl(hashHex)) - if ret.StatusCode() != 200 { - logger.Get().Error(ErrFailedUpload.Error(), zap.String("error", ret.String())) - return upload, ErrFailedUpload - } - - ret, err = client.R().SetBody(tree).Put(getWorkerProofUrl(hashHex)) - if ret.StatusCode() != 200 { - logger.Get().Error(ErrFailedUploadProof.Error(), zap.String("error", ret.String())) - return upload, ErrFailedUpload - } - - upload = model.Upload{ - Hash: hashHex, - AccountID: accountID, - } - - if err = db.Get().Create(&upload).Error; err != nil { - logger.Get().Error(ErrFailedSaveUpload.Error(), zap.Error(err)) - return upload, ErrFailedSaveUpload - } - - return upload, nil -} -func Download(hash string) (io.Reader, error) { - uploadItem := db.Get().Table("uploads").Where(&model.Upload{Hash: hash}).Row() - tusItem := db.Get().Table("tus").Where(&model.Tus{Hash: hash}).Row() - - if uploadItem.Err() == nil { - fetch, err := client.R().SetDoNotParseResponse(true).Get(getWorkerObjectUrl(hash)) - if err != nil { - logger.Get().Error(ErrFailedFetchObject.Error(), zap.Error(err)) - return nil, ErrFailedFetchObject - } - - return fetch.RawBody(), nil - } else if tusItem.Err() == nil { - var tusData model.Tus - err := tusItem.Scan(&tusData) - if err != nil { - logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(err)) - return nil, ErrFailedQueryUpload - } - - upload, err := getStore().GetUpload(context.Background(), tusData.UploadID) - if err != nil { - logger.Get().Error(ErrFailedQueryTusUpload.Error(), zap.Error(err)) - return nil, ErrFailedQueryTusUpload - } - - reader, err := upload.GetReader(context.Background()) - if err != nil { - logger.Get().Error(ErrFailedFetchTusObject.Error(), zap.Error(err)) - return nil, ErrFailedFetchTusObject - } - - return reader, nil - } else { - logger.Get().Error(ErrInvalidFile.Error(), zap.String("hash", hash)) - return nil, ErrInvalidFile - } -} - -func DownloadProof(hash string) (io.Reader, error) { - uploadItem := db.Get().Model(&model.Upload{}).Where(&model.Upload{Hash: hash}).Row() - - if uploadItem.Err() != nil { - logger.Get().Debug(ErrInvalidFile.Error(), zap.String("hash", hash)) - return nil, ErrInvalidFile - } - fetch, err := client.R().SetDoNotParseResponse(true).Get(getWorkerProofUrl(hash)) - if err != nil { - logger.Get().Error(ErrFailedFetchObject.Error(), zap.Error(err)) - return nil, ErrFailedFetchObject - } - - return fetch.RawBody(), nil -} - -func Status(hash string) int { - var count int64 - - uploadItem := db.Get().Table("uploads").Where(&model.Upload{Hash: hash}).Count(&count) - - if uploadItem.Error != nil && !errors.Is(uploadItem.Error, gorm.ErrRecordNotFound) { - logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(uploadItem.Error)) - } - - if count > 0 { - return STATUS_UPLOADED - } - - tusItem := db.Get().Table("tus").Where(&model.Tus{Hash: hash}).Count(&count) - - if tusItem.Error != nil && !errors.Is(tusItem.Error, gorm.ErrRecordNotFound) { - logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(tusItem.Error)) - } - - if count > 0 { - return STATUS_UPLOADING - } - - return STATUS_NOT_FOUND -} - -func objectUrlBuilder(hash string, bus bool, proof bool) string { - path := []string{} - if bus { - path = append(path, "bus") - } else { - path = append(path, "worker") - } - - path = append(path, "objects") - - name := "%s" - - if proof { - name = name + ".obao" - } - - path = append(path, name) - - return fmt.Sprintf(strings.Join(path, "/"), hash) -} - -func getBusObjectUrl(hash string) string { - return objectUrlBuilder(hash, true, false) -} -func getWorkerObjectUrl(hash string) string { - return objectUrlBuilder(hash, false, false) -} -func getWorkerProofUrl(hash string) string { - return objectUrlBuilder(hash, false, true) -} -func getBusProofUrl(hash string) string { - return objectUrlBuilder(hash, true, true) -} - -func getStore() *tusstore.DbFileStore { - ret := shared.GetTusStore() - return (*ret).(*tusstore.DbFileStore) -} - -func Pin(hash string, accountID uint) error { - var upload model.Upload - - if result := db.Get().Model(&upload).Where("hash = ?", hash).First(&upload); result.Error != nil { - if !errors.Is(result.Error, gorm.ErrRecordNotFound) { - logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(result.Error)) - } - return ErrFailedQueryUpload - } - - var pin model.Pin - - result := db.Get().Model(&pin).Where(&model.Pin{Upload: upload, AccountID: accountID}).First(&pin) - - if result.Error != nil && !errors.Is(result.Error, gorm.ErrRecordNotFound) { - logger.Get().Error(ErrFailedQueryPins.Error(), zap.Error(result.Error)) - return ErrFailedQueryPins - } - - if result.Error == nil { - return nil - } - - pin.AccountID = upload.AccountID - pin.Upload = upload - - result = db.Get().Save(&pin) - - if result.Error != nil { - logger.Get().Error(ErrFailedSavePin.Error(), zap.Error(result.Error)) - - return ErrFailedSavePin - } - - return nil -} diff --git a/shared/shared.go b/shared/shared.go deleted file mode 100644 index 08abce8..0000000 --- a/shared/shared.go +++ /dev/null @@ -1,51 +0,0 @@ -package shared - -import ( - tusd "github.com/tus/tusd/pkg/handler" - _ "go.uber.org/zap" -) - -type TusFunc func(upload *tusd.Upload) error - -var tusQueue *interface{} -var tusStore *interface{} -var tusComposer *interface{} -var tusWorker TusFunc - -type tusRequestContextKey int - -const ( - TusRequestContextKey tusRequestContextKey = iota -) - -func SetTusQueue(q interface{}) { - tusQueue = &q -} - -func GetTusQueue() *interface{} { - return tusQueue -} - -func SetTusStore(s interface{}) { - tusStore = &s -} - -func GetTusStore() *interface{} { - return tusStore -} - -func SetTusComposer(c interface{}) { - tusComposer = &c -} - -func GetTusComposer() *interface{} { - return tusComposer -} - -func SetTusWorker(w TusFunc) { - tusWorker = w -} - -func GetTusWorker() TusFunc { - return tusWorker -} diff --git a/tus/tus.go b/tus/tus.go deleted file mode 100644 index b4a4157..0000000 --- a/tus/tus.go +++ /dev/null @@ -1,222 +0,0 @@ -package tus - -import ( - "context" - "encoding/hex" - "encoding/json" - "errors" - "git.lumeweb.com/LumeWeb/portal/cid" - "git.lumeweb.com/LumeWeb/portal/db" - "git.lumeweb.com/LumeWeb/portal/logger" - "git.lumeweb.com/LumeWeb/portal/model" - "git.lumeweb.com/LumeWeb/portal/service/files" - "git.lumeweb.com/LumeWeb/portal/shared" - "git.lumeweb.com/LumeWeb/portal/tusstore" - "github.com/golang-queue/queue" - tusd "github.com/tus/tusd/pkg/handler" - "github.com/tus/tusd/pkg/memorylocker" - "go.uber.org/zap" - "golang.org/x/exp/slices" - "gorm.io/gorm" - "io" - "strconv" -) - -const TUS_API_PATH = "/files/tus" - -const HASH_META_HEADER = "hash" - -func Init() *tusd.Handler { - store := &tusstore.DbFileStore{ - Path: "/tmp", - } - - shared.SetTusStore(store) - - composer := tusd.NewStoreComposer() - composer.UseCore(store) - composer.UseConcater(store) - composer.UseLocker(memorylocker.New()) - composer.UseTerminater(store) - shared.SetTusComposer(composer) - - handler, err := tusd.NewHandler(tusd.Config{ - BasePath: "/api/v1" + TUS_API_PATH, - StoreComposer: composer, - PreUploadCreateCallback: func(hook tusd.HookEvent) error { - hash := hook.Upload.MetaData[HASH_META_HEADER] - - if len(hash) == 0 { - msg := "missing hash metadata" - logger.Get().Debug(msg) - return errors.New(msg) - } - - var upload model.Upload - result := db.Get().Where(&model.Upload{Hash: hash}).First(&upload) - if (result.Error != nil && !errors.Is(result.Error, gorm.ErrRecordNotFound)) || result.RowsAffected > 0 { - hashBytes, err := hex.DecodeString(hash) - if err != nil { - logger.Get().Debug("invalid hash", zap.Error(err)) - return err - } - - cidString, err := cid.Encode(hashBytes, uint64(hook.Upload.Size)) - - if err != nil { - logger.Get().Debug("failed to create cid", zap.Error(err)) - return err - } - - resp, err := json.Marshal(UploadResponse{Cid: cidString}) - - if err != nil { - logger.Get().Error("failed to create response", zap.Error(err)) - return err - } - - return tusd.NewHTTPError(errors.New(string(resp)), 304) - } - - return nil - }, - }) - if err != nil { - panic(err) - } - - pool := queue.NewPool(5) - - shared.SetTusQueue(pool) - shared.SetTusWorker(tusWorker) - - go tusStartup() - - return handler -} - -func tusStartup() { - tusQueue := getQueue() - store := getStore() - - rows, err := db.Get().Model(&model.Tus{}).Rows() - - if err != nil { - logger.Get().Error("failed to query tus uploads", zap.Error(err)) - } - - defer rows.Close() - - processedHashes := make([]string, 0) - - for rows.Next() { - var tusUpload model.Tus - err := db.Get().ScanRows(rows, &tusUpload) - if err != nil { - logger.Get().Error("failed to scan tus records", zap.Error(err)) - return - } - - upload, err := store.GetUpload(nil, tusUpload.UploadID) - if err != nil { - logger.Get().Error("failed to query tus upload", zap.Error(err)) - db.Get().Delete(&tusUpload) - continue - } - - if slices.Contains(processedHashes, tusUpload.Hash) { - err := terminateUpload(upload) - if err != nil { - logger.Get().Error("failed to terminate tus upload", zap.Error(err)) - } - continue - } - - if err := tusQueue.QueueTask(func(ctx context.Context) error { - return tusWorker(&upload) - }); err != nil { - logger.Get().Error("failed to queue tus upload", zap.Error(err)) - } else { - processedHashes = append(processedHashes, tusUpload.Hash) - } - } -} - -func tusWorker(upload *tusd.Upload) error { - info, err := (*upload).GetInfo(context.Background()) - if err != nil { - logger.Get().Error("failed to query tus upload metadata", zap.Error(err)) - return err - } - file, err := (*upload).GetReader(context.Background()) - if err != nil { - logger.Get().Error("failed reading upload", zap.Error(err)) - return err - } - - hashHex := info.MetaData[HASH_META_HEADER] - - hashBytes, err := hex.DecodeString(hashHex) - - if err != nil { - logger.Get().Error("failed decoding hash", zap.Error(err)) - tErr := terminateUpload(*upload) - - if tErr != nil { - return tErr - } - return err - } - - uploader, _ := strconv.Atoi(info.Storage["uploader"]) - - newUpload, err := files.Upload(file.(io.ReadSeeker), info.Size, hashBytes, uint(uploader)) - tErr := terminateUpload(*upload) - - if tErr != nil { - return tErr - } - - if err != nil { - return err - } - - err = files.Pin(newUpload.Hash, newUpload.AccountID) - if err != nil { - return err - } - - return nil -} - -func terminateUpload(upload tusd.Upload) error { - err := getComposer().Terminater.AsTerminatableUpload(upload).Terminate(context.Background()) - - if err != nil { - logger.Get().Error("failed deleting tus upload", zap.Error(err)) - } - - if err != nil { - return err - } - - return nil -} - -type UploadResponse struct { - Cid string `json:"cid"` -} - -func getQueue() *queue.Queue { - ret := shared.GetTusQueue() - return (*ret).(*queue.Queue) -} - -func getStore() *tusstore.DbFileStore { - ret := shared.GetTusStore() - return (*ret).(*tusstore.DbFileStore) -} -func getComposer() *tusd.StoreComposer { - ret := shared.GetTusComposer() - return (*ret).(*tusd.StoreComposer) -} diff --git a/tusstore/store.go b/tusstore/store.go deleted file mode 100644 index 73f5699..0000000 --- a/tusstore/store.go +++ /dev/null @@ -1,316 +0,0 @@ -package tusstore - -import ( - "context" - "crypto/rand" - "encoding/hex" - "encoding/json" - "fmt" - "git.lumeweb.com/LumeWeb/portal/db" - "git.lumeweb.com/LumeWeb/portal/logger" - "git.lumeweb.com/LumeWeb/portal/model" - "git.lumeweb.com/LumeWeb/portal/service/auth" - "git.lumeweb.com/LumeWeb/portal/shared" - "github.com/golang-queue/queue" - clone "github.com/huandu/go-clone" - "github.com/kataras/iris/v12" - "github.com/tus/tusd/pkg/handler" - "go.uber.org/zap" - "io" - "os" - "path/filepath" - "strconv" -) - -var defaultFilePerm = os.FileMode(0664) - -type DbFileStore struct { - // Relative or absolute path to store files in. DbFileStore does not check - // whether the path exists, use os.MkdirAll in this case on your own. - Path string -} - -func (store DbFileStore) UseIn(composer *handler.StoreComposer) { - composer.UseCore(store) - composer.UseTerminater(store) - composer.UseConcater(store) - composer.UseLengthDeferrer(store) -} - -func (store DbFileStore) NewUpload(ctx context.Context, info handler.FileInfo) (handler.Upload, error) { - if info.ID == "" { - info.ID = uid() - } - binPath := store.binPath(info.ID) - info.Storage = map[string]string{ - "Type": "dbstore", - "Path": binPath, - } - - // Create binary file with no content - file, err := os.OpenFile(binPath, os.O_CREATE|os.O_WRONLY, defaultFilePerm) - if err != nil { - if os.IsNotExist(err) { - err = fmt.Errorf("upload directory does not exist: %s", store.Path) - } - return nil, err - } - - err = file.Close() - if err != nil { - return nil, err - } - - if err != nil { - return nil, err - } - - irisContext := ctx.Value(shared.TusRequestContextKey).(iris.Context) - - upload := &fileUpload{ - info: info, - binPath: binPath, - hash: info.MetaData["hash"], - uploader: auth.GetCurrentUserId(irisContext), - } - - // writeInfo creates the file by itself if necessary - err = upload.writeInfo() - if err != nil { - return nil, err - } - - return upload, nil -} - -func (store DbFileStore) GetUpload(ctx context.Context, id string) (handler.Upload, error) { - info := handler.FileInfo{ - ID: id, - } - - fUpload := &fileUpload{info: info} - - record, is404, err := fUpload.getInfo() - if err != nil { - if is404 { - // Interpret os.ErrNotExist as 404 Not Found - err = handler.ErrNotFound - } - return nil, err - } - - if err := json.Unmarshal([]byte(record.Info), &info); err != nil { - logger.Get().Error("fail to parse upload meta", zap.Error(err)) - return nil, err - } - - fUpload.info = info - - fUpload.hash = record.Hash - binPath := store.binPath(id) - stat, err := os.Stat(binPath) - if err != nil { - if os.IsNotExist(err) { - // Interpret os.ErrNotExist as 404 Not Found - err = handler.ErrNotFound - } - return nil, err - } - - info.Offset = stat.Size() - - fUpload.binPath = binPath - - return fUpload, nil -} - -func (store DbFileStore) AsTerminatableUpload(upload handler.Upload) handler.TerminatableUpload { - return upload.(*fileUpload) -} - -func (store DbFileStore) AsLengthDeclarableUpload(upload handler.Upload) handler.LengthDeclarableUpload { - return upload.(*fileUpload) -} - -func (store DbFileStore) AsConcatableUpload(upload handler.Upload) handler.ConcatableUpload { - return upload.(*fileUpload) -} - -// binPath returns the path to the file storing the binary data. -func (store DbFileStore) binPath(id string) string { - return filepath.Join(store.Path, id) -} - -type fileUpload struct { - // info stores the current information about the upload - info handler.FileInfo - // binPath is the path to the binary file (which has no extension) - binPath string - hash string - uploader uint -} - -func (upload *fileUpload) GetInfo(ctx context.Context) (handler.FileInfo, error) { - info := clone.Clone(upload.info).(handler.FileInfo) - info.Storage["uploader"] = strconv.Itoa(int(upload.uploader)) - - return upload.info, nil -} - -func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { - file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) - if err != nil { - return 0, err - } - defer file.Close() - - n, err := io.Copy(file, src) - - upload.info.Offset += n - - err = upload.writeInfo() - if err != nil { - return 0, err - } - - return n, err -} - -func (upload *fileUpload) GetReader(ctx context.Context) (io.Reader, error) { - return os.Open(upload.binPath) -} - -func (upload *fileUpload) Terminate(ctx context.Context) error { - tusUpload := &model.Tus{ - UploadID: upload.info.ID, - } - - ret := db.Get().Where(&tusUpload).Delete(&tusUpload) - - if ret.Error != nil { - logger.Get().Error("failed to delete tus entry", zap.Error(ret.Error)) - } - - if err := os.Remove(upload.binPath); err != nil { - return err - } - - return nil -} - -func (upload *fileUpload) ConcatUploads(ctx context.Context, uploads []handler.Upload) (err error) { - file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) - if err != nil { - return err - } - defer file.Close() - - for _, partialUpload := range uploads { - fileUpload := partialUpload.(*fileUpload) - - src, err := os.Open(fileUpload.binPath) - if err != nil { - return err - } - - if _, err := io.Copy(file, src); err != nil { - return err - } - } - - return -} - -func (upload *fileUpload) DeclareLength(ctx context.Context, length int64) error { - upload.info.Size = length - upload.info.SizeIsDeferred = false - return upload.writeInfo() -} - -// writeInfo updates the entire information. Everything will be overwritten. -func (upload *fileUpload) writeInfo() error { - data, err := json.Marshal(upload.info) - if err != nil { - return err - } - - tusRecord, is404, err := upload.getInfo() - - if err != nil && !is404 { - return err - } - - if tusRecord != nil { - tusRecord.Info = string(data) - if ret := db.Get().Save(&tusRecord); ret.Error != nil { - logger.Get().Error("failed to update tus entry", zap.Error(ret.Error)) - - return ret.Error - } - - return nil - } - - tusRecord = &model.Tus{UploadID: upload.info.ID, Hash: upload.hash, Info: string(data), AccountID: upload.uploader} - - if ret := db.Get().Create(&tusRecord); ret.Error != nil { - logger.Get().Error("failed to create tus entry", zap.Error(ret.Error)) - - return ret.Error - } - - return nil -} - -func (upload *fileUpload) getInfo() (*model.Tus, bool, error) { - var tusRecord model.Tus - - result := db.Get().Where(&model.Tus{UploadID: upload.info.ID}).First(&tusRecord) - - if result.Error != nil && result.Error.Error() != "record not found" { - logger.Get().Error("failed to query tus entry", zap.Error(result.Error)) - return nil, false, result.Error - } - - if result.Error != nil { - return nil, true, result.Error - } - - return &tusRecord, false, nil -} - -func (upload *fileUpload) FinishUpload(ctx context.Context) error { - if err := getQueue().QueueTask(func(ctx context.Context) error { - upload, err := getStore().GetUpload(nil, upload.info.ID) - if err != nil { - logger.Get().Error("failed to query tus upload", zap.Error(err)) - return err - } - return shared.GetTusWorker()(&upload) - }); err != nil { - logger.Get().Error("failed to queue tus upload", zap.Error(err)) - return err - } - - return nil -} - -func uid() string { - id := make([]byte, 16) - _, err := io.ReadFull(rand.Reader, id) - if err != nil { - // This is probably an appropriate way to handle errors from our source - // for random bits. - panic(err) - } - return hex.EncodeToString(id) -} -func getQueue() *queue.Queue { - ret := shared.GetTusQueue() - return (*ret).(*queue.Queue) -} - -func getStore() *DbFileStore { - ret := shared.GetTusStore() - return (*ret).(*DbFileStore) -}