LumeWeb
/
portal
1
0
Fork 0
Code Issues 5 Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: LumeWeb:892f093d93348459d113041104d773fdd5124a8d
Branches Tags
LumeWeb:master
LumeWeb:develop
LumeWeb:v0.1.0-develop.3
LumeWeb:v0.1.0-develop.2
LumeWeb:v0.1.0-develop.1
LumeWeb:v0.0.1
..
compare: LumeWeb:d1d4f6b6790ebec29adcd74e91c584155ff566ce
Branches Tags
LumeWeb:develop
LumeWeb:master
LumeWeb:v0.1.0-develop.3
LumeWeb:v0.1.0-develop.2
LumeWeb:v0.1.0-develop.1
LumeWeb:v0.0.1

No commits in common. "892f093d93348459d113041104d773fdd5124a8d" and "d1d4f6b6790ebec29adcd74e91c584155ff566ce" have entirely different histories.
892f093d93 ... d1d4f6b679

3 changed files with 42 additions and 107 deletions
Show Stats Expand all files Collapse all files

7
controller/files.go
View File

@ -38,13 +38,6 @@ func (f *FilesController) PostUpload() {
return
}
err = files.Pin(upload.Hash, upload.AccountID)
if internalError(ctx, err) {
logger.Get().Debug("failed pinning file", zap.Error(err))
return
}
cidString, err := cid.EncodeString(upload.Hash, uint64(meta.Size))
if internalError(ctx, err) {

135
service/files/files.go
View File

@ -16,7 +16,6 @@ import (
"github.com/spf13/viper"
_ "github.com/tus/tusd/pkg/handler"
"go.uber.org/zap"
"gorm.io/gorm"
"io"
"strings"
)
@ -27,24 +26,6 @@ const (
STATUS_NOT_FOUND = iota
)
var (
ErrAlreadyExists = errors.New("Upload already exists")
ErrFailedFetchObject = errors.New("Failed fetching object")
ErrFailedFetchObjectProof = errors.New("Failed fetching object proof")
ErrFailedFetchTusObject = errors.New("Failed fetching tus object")
ErrFailedHashFile = errors.New("Failed to hash file")
ErrFailedQueryTusUpload = errors.New("Failed to query tus uploads")
ErrFailedQueryUpload = errors.New("Failed to query uploads")
ErrFailedQueryPins = errors.New("Failed to query pins")
ErrFailedSaveUpload = errors.New("Failed saving upload to db")
ErrFailedSavePin = errors.New("Failed saving pin to db")
ErrFailedUpload = errors.New("Failed uploading object")
ErrFailedUploadProof = errors.New("Failed uploading object proof")
ErrFileExistsOutOfSync = errors.New("File already exists in network, but missing in database")
ErrFileHashMismatch = errors.New("File hash does not match provided file hash")
ErrInvalidFile = errors.New("Invalid file")
)
var client *resty.Client
func Init() {
@ -60,14 +41,14 @@ func Upload(r io.ReadSeeker, size int64, hash []byte) (model.Upload, error) {
tree, hashBytes, err := bao.ComputeTree(r, size)
if err != nil {
logger.Get().Error(ErrFailedHashFile.Error(), zap.Error(err))
return upload, ErrFailedHashFile
logger.Get().Error("Failed to hash file", zap.Error(err))
return upload, err
}
if hash != nil {
if bytes.Compare(hashBytes[:], hash) != 0 {
logger.Get().Error(ErrFileHashMismatch.Error())
return upload, ErrFileHashMismatch
logger.Get().Error("File hash does not match provided file hash")
return upload, err
}
}
@ -80,15 +61,15 @@ func Upload(r io.ReadSeeker, size int64, hash []byte) (model.Upload, error) {
}
result := db.Get().Where(&model.Upload{Hash: hashHex}).First(&upload)
if (result.Error != nil && !errors.Is(result.Error, gorm.ErrRecordNotFound)) || result.RowsAffected > 0 {
if (result.Error != nil && result.Error.Error() != "record not found") || result.RowsAffected > 0 {
err := result.Row().Scan(&upload)
if err != nil {
logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(err))
return upload, ErrFailedQueryUpload
logger.Get().Error("Failed to query uploads table", zap.Error(err))
return upload, err
}
if result.RowsAffected > 0 && upload.ID > 0 {
logger.Get().Info(ErrAlreadyExists.Error())
logger.Get().Info("Upload already exists")
return upload, nil
}
}
@ -96,8 +77,8 @@ func Upload(r io.ReadSeeker, size int64, hash []byte) (model.Upload, error) {
objectExistsResult, err := client.R().Get(getBusObjectUrl(hashHex))
if err != nil {
logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(err))
return upload, ErrFailedQueryUpload
logger.Get().Error("Failed query object", zap.Error(err))
return upload, err
}
objectStatusCode := objectExistsResult.StatusCode()
@ -105,8 +86,8 @@ func Upload(r io.ReadSeeker, size int64, hash []byte) (model.Upload, error) {
if objectStatusCode == 500 {
bodyErr := objectExistsResult.String()
if !strings.Contains(bodyErr, "no slabs found") {
logger.Get().Error(ErrFailedFetchObject.Error(), zap.String("error", objectExistsResult.String()))
return upload, ErrFailedFetchObject
logger.Get().Error("Failed fetching object", zap.String("error", objectExistsResult.String()))
return upload, errors.New(fmt.Sprintf("error fetching file: %s", objectExistsResult.String()))
}
objectStatusCode = 404
@ -115,8 +96,8 @@ func Upload(r io.ReadSeeker, size int64, hash []byte) (model.Upload, error) {
proofExistsResult, err := client.R().Get(getBusProofUrl(hashHex))
if err != nil {
logger.Get().Error(ErrFailedFetchObjectProof.Error(), zap.Error(err))
return upload, ErrFailedFetchObjectProof
logger.Get().Error("Failed query object proof", zap.Error(err))
return upload, err
}
proofStatusCode := proofExistsResult.StatusCode()
@ -124,28 +105,31 @@ func Upload(r io.ReadSeeker, size int64, hash []byte) (model.Upload, error) {
if proofStatusCode == 500 {
bodyErr := proofExistsResult.String()
if !strings.Contains(bodyErr, "no slabs found") {
logger.Get().Error(ErrFailedFetchObjectProof.Error(), zap.String("error", proofExistsResult.String()))
return upload, ErrFailedFetchObjectProof
logger.Get().Error("Failed fetching object proof", zap.String("error", proofExistsResult.String()))
return upload, errors.New(fmt.Sprintf("error fetching file proof: %s", proofExistsResult.String()))
}
objectStatusCode = 404
}
if objectStatusCode != 404 && proofStatusCode != 404 {
logger.Get().Error(ErrFileExistsOutOfSync.Error(), zap.String("hash", hashHex))
return upload, ErrFileExistsOutOfSync
msg := "file already exists in network, but missing in database"
logger.Get().Error(msg)
return upload, errors.New(msg)
}
ret, err := client.R().SetBody(r).Put(getWorkerObjectUrl(hashHex))
if ret.StatusCode() != 200 {
logger.Get().Error(ErrFailedUpload.Error(), zap.String("error", ret.String()))
return upload, ErrFailedUpload
logger.Get().Error("Failed uploading object", zap.String("error", ret.String()))
err = errors.New(ret.String())
return upload, err
}
ret, err = client.R().SetBody(tree).Put(getWorkerProofUrl(hashHex))
if ret.StatusCode() != 200 {
logger.Get().Error(ErrFailedUploadProof.Error(), zap.String("error", ret.String()))
return upload, ErrFailedUpload
logger.Get().Error("Failed uploading proof", zap.String("error", ret.String()))
err = errors.New(ret.String())
return upload, err
}
upload = model.Upload{
@ -153,8 +137,8 @@ func Upload(r io.ReadSeeker, size int64, hash []byte) (model.Upload, error) {
}
if err = db.Get().Create(&upload).Error; err != nil {
logger.Get().Error(ErrFailedSaveUpload.Error(), zap.Error(err))
return upload, ErrFailedSaveUpload
logger.Get().Error("Failed adding upload to db", zap.Error(err))
return upload, err
}
return upload, nil
@ -166,8 +150,8 @@ func Download(hash string) (io.Reader, error) {
if uploadItem.Err() == nil {
fetch, err := client.R().SetDoNotParseResponse(true).Get(fmt.Sprintf("/worker/objects/%s", hash))
if err != nil {
logger.Get().Error(ErrFailedFetchObject.Error(), zap.Error(err))
return nil, ErrFailedFetchObject
logger.Get().Error("Failed downloading object", zap.Error(err))
return nil, err
}
return fetch.RawBody(), nil
@ -175,26 +159,26 @@ func Download(hash string) (io.Reader, error) {
var tusData model.Tus
err := tusItem.Scan(&tusData)
if err != nil {
logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(err))
return nil, ErrFailedQueryUpload
logger.Get().Error("Failed querying upload from db", zap.Error(err))
return nil, err
}
upload, err := getStore().GetUpload(context.Background(), tusData.UploadID)
if err != nil {
logger.Get().Error(ErrFailedQueryTusUpload.Error(), zap.Error(err))
return nil, ErrFailedQueryTusUpload
logger.Get().Error("Failed querying tus upload", zap.Error(err))
return nil, err
}
reader, err := upload.GetReader(context.Background())
if err != nil {
logger.Get().Error(ErrFailedFetchTusObject.Error(), zap.Error(err))
return nil, ErrFailedFetchTusObject
logger.Get().Error("Failed reading tus upload", zap.Error(err))
return nil, err
}
return reader, nil
} else {
logger.Get().Error(ErrInvalidFile.Error(), zap.String("hash", hash))
return nil, ErrInvalidFile
logger.Get().Error("invalid file")
return nil, errors.New("invalid file")
}
}
@ -203,8 +187,8 @@ func Status(hash string) int {
uploadItem := db.Get().Table("uploads").Where(&model.Upload{Hash: hash}).Count(&count)
if uploadItem.Error != nil && !errors.Is(uploadItem.Error, gorm.ErrRecordNotFound) {
logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(uploadItem.Error))
if uploadItem.Error != nil {
logger.Get().Error("Failed querying upload from db", zap.Error(uploadItem.Error))
}
if count > 0 {
@ -213,8 +197,8 @@ func Status(hash string) int {
tusItem := db.Get().Table("tus").Where(&model.Tus{Hash: hash}).Count(&count)
if tusItem.Error != nil && !errors.Is(tusItem.Error, gorm.ErrRecordNotFound) {
logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(tusItem.Error))
if tusItem.Error != nil {
logger.Get().Error("Failed querying upload from db", zap.Error(tusItem.Error))
}
if count > 0 {
@ -262,40 +246,3 @@ func getStore() *tusstore.DbFileStore {
ret := shared.GetTusStore()
return (*ret).(*tusstore.DbFileStore)
}
func Pin(hash string, accountID uint) error {
var upload model.Upload
if result := db.Get().Model(&upload).Where("hash = ?", hash).First(&upload); result.Error != nil {
if !errors.Is(result.Error, gorm.ErrRecordNotFound) {
logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(result.Error))
}
return ErrFailedQueryUpload
}
var pin model.Pin
result := db.Get().Model(&pin).Where(&model.Pin{Upload: upload, AccountID: accountID}).First(&pin)
if result.Error != nil && !errors.Is(result.Error, gorm.ErrRecordNotFound) {
logger.Get().Error(ErrFailedQueryPins.Error(), zap.Error(result.Error))
return ErrFailedQueryPins
}
if result.Error == nil {
return nil
}
pin.AccountID = upload.AccountID
pin.Upload = upload
result = db.Get().Save(&pin)
if result.Error != nil {
logger.Get().Error(ErrFailedSavePin.Error(), zap.Error(result.Error))
return ErrFailedSavePin
}
return nil
}

7
tus/tus.go
View File

@ -166,7 +166,7 @@ func tusWorker(upload *tusd.Upload) error {
return err
}
newUpload, err := files.Upload(file.(io.ReadSeeker), info.Size, hashBytes)
_, err = files.Upload(file.(io.ReadSeeker), info.Size, hashBytes)
tErr := terminateUpload(*upload)
if tErr != nil {
@ -177,11 +177,6 @@ func tusWorker(upload *tusd.Upload) error {
return err
}
err = files.Pin(newUpload.Hash, newUpload.AccountID)
if err != nil {
return err
}
return nil
}