package files import ( "bytes" "context" "encoding/hex" "errors" "fmt" "git.lumeweb.com/LumeWeb/portal/bao" "git.lumeweb.com/LumeWeb/portal/db" "git.lumeweb.com/LumeWeb/portal/logger" "git.lumeweb.com/LumeWeb/portal/model" "git.lumeweb.com/LumeWeb/portal/shared" "git.lumeweb.com/LumeWeb/portal/tusstore" "github.com/go-resty/resty/v2" "github.com/spf13/viper" _ "github.com/tus/tusd/pkg/handler" "go.uber.org/zap" "gorm.io/gorm" "io" "strings" ) const ( STATUS_UPLOADED = iota STATUS_UPLOADING = iota STATUS_NOT_FOUND = iota ) var ( ErrAlreadyExists = errors.New("Upload already exists") ErrFailedFetchObject = errors.New("Failed fetching object") ErrFailedFetchObjectProof = errors.New("Failed fetching object proof") ErrFailedFetchTusObject = errors.New("Failed fetching tus object") ErrFailedHashFile = errors.New("Failed to hash file") ErrFailedQueryTusUpload = errors.New("Failed to query tus uploads") ErrFailedQueryUpload = errors.New("Failed to query uploads") ErrFailedQueryPins = errors.New("Failed to query pins") ErrFailedSaveUpload = errors.New("Failed saving upload to db") ErrFailedSavePin = errors.New("Failed saving pin to db") ErrFailedUpload = errors.New("Failed uploading object") ErrFailedUploadProof = errors.New("Failed uploading object proof") ErrFileExistsOutOfSync = errors.New("File already exists in network, but missing in database") ErrFileHashMismatch = errors.New("File hash does not match provided file hash") ErrInvalidFile = errors.New("Invalid file") ) var client *resty.Client func Init() { client = resty.New() client.SetBaseURL("http://localhost:9980/api") client.SetBasicAuth("", viper.GetString("renterd-api-password")) client.SetDisableWarn(true) client.SetCloseConnection(true) } func Upload(r io.ReadSeeker, size int64, hash []byte, accountID uint) (model.Upload, error) { var upload model.Upload tree, hashBytes, err := bao.ComputeTree(r, size) if err != nil { logger.Get().Error(ErrFailedHashFile.Error(), zap.Error(err)) return upload, ErrFailedHashFile } if hash != nil { if bytes.Compare(hashBytes[:], hash) != 0 { logger.Get().Error(ErrFileHashMismatch.Error()) return upload, ErrFileHashMismatch } } hashHex := hex.EncodeToString(hashBytes[:]) _, err = r.Seek(0, io.SeekStart) if err != nil { return upload, err } result := db.Get().Where(&model.Upload{Hash: hashHex}).First(&upload) if !errors.Is(result.Error, gorm.ErrRecordNotFound) { if err != nil { logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(err)) return upload, ErrFailedQueryUpload } logger.Get().Info(ErrAlreadyExists.Error()) return upload, nil } objectExistsResult, err := client.R().Get(getBusObjectUrl(hashHex)) if err != nil { logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(err)) return upload, ErrFailedQueryUpload } objectStatusCode := objectExistsResult.StatusCode() if objectStatusCode == 500 { bodyErr := objectExistsResult.String() if !strings.Contains(bodyErr, "no slabs found") { logger.Get().Error(ErrFailedFetchObject.Error(), zap.String("error", objectExistsResult.String())) return upload, ErrFailedFetchObject } objectStatusCode = 404 } proofExistsResult, err := client.R().Get(getBusProofUrl(hashHex)) if err != nil { logger.Get().Error(ErrFailedFetchObjectProof.Error(), zap.Error(err)) return upload, ErrFailedFetchObjectProof } proofStatusCode := proofExistsResult.StatusCode() if proofStatusCode == 500 { bodyErr := proofExistsResult.String() if !strings.Contains(bodyErr, "no slabs found") { logger.Get().Error(ErrFailedFetchObjectProof.Error(), zap.String("error", proofExistsResult.String())) return upload, ErrFailedFetchObjectProof } objectStatusCode = 404 } if objectStatusCode != 404 && proofStatusCode != 404 { logger.Get().Error(ErrFileExistsOutOfSync.Error(), zap.String("hash", hashHex)) return upload, ErrFileExistsOutOfSync } ret, err := client.R().SetBody(r).Put(getWorkerObjectUrl(hashHex)) if ret.StatusCode() != 200 { logger.Get().Error(ErrFailedUpload.Error(), zap.String("error", ret.String())) return upload, ErrFailedUpload } ret, err = client.R().SetBody(tree).Put(getWorkerProofUrl(hashHex)) if ret.StatusCode() != 200 { logger.Get().Error(ErrFailedUploadProof.Error(), zap.String("error", ret.String())) return upload, ErrFailedUpload } upload = model.Upload{ Hash: hashHex, AccountID: accountID, } if err = db.Get().Create(&upload).Error; err != nil { logger.Get().Error(ErrFailedSaveUpload.Error(), zap.Error(err)) return upload, ErrFailedSaveUpload } return upload, nil } func Download(hash string) (io.Reader, error) { uploadItem := db.Get().Table("uploads").Where(&model.Upload{Hash: hash}).Row() tusItem := db.Get().Table("tus").Where(&model.Tus{Hash: hash}).Row() if uploadItem.Err() == nil { fetch, err := client.R().SetDoNotParseResponse(true).Get(getWorkerObjectUrl(hash)) if err != nil { logger.Get().Error(ErrFailedFetchObject.Error(), zap.Error(err)) return nil, ErrFailedFetchObject } return fetch.RawBody(), nil } else if tusItem.Err() == nil { var tusData model.Tus err := tusItem.Scan(&tusData) if err != nil { logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(err)) return nil, ErrFailedQueryUpload } upload, err := getStore().GetUpload(context.Background(), tusData.UploadID) if err != nil { logger.Get().Error(ErrFailedQueryTusUpload.Error(), zap.Error(err)) return nil, ErrFailedQueryTusUpload } reader, err := upload.GetReader(context.Background()) if err != nil { logger.Get().Error(ErrFailedFetchTusObject.Error(), zap.Error(err)) return nil, ErrFailedFetchTusObject } return reader, nil } else { logger.Get().Error(ErrInvalidFile.Error(), zap.String("hash", hash)) return nil, ErrInvalidFile } } func DownloadProof(hash string) (io.Reader, error) { uploadItem := db.Get().Model(&model.Upload{}).Where(&model.Upload{Hash: hash}).Row() if uploadItem.Err() != nil { logger.Get().Debug(ErrInvalidFile.Error(), zap.String("hash", hash)) return nil, ErrInvalidFile } fetch, err := client.R().SetDoNotParseResponse(true).Get(getWorkerProofUrl(hash)) if err != nil { logger.Get().Error(ErrFailedFetchObject.Error(), zap.Error(err)) return nil, ErrFailedFetchObject } return fetch.RawBody(), nil } func Status(hash string) int { var count int64 uploadItem := db.Get().Table("uploads").Where(&model.Upload{Hash: hash}).Count(&count) if uploadItem.Error != nil && !errors.Is(uploadItem.Error, gorm.ErrRecordNotFound) { logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(uploadItem.Error)) } if count > 0 { return STATUS_UPLOADED } tusItem := db.Get().Table("tus").Where(&model.Tus{Hash: hash}).Count(&count) if tusItem.Error != nil && !errors.Is(tusItem.Error, gorm.ErrRecordNotFound) { logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(tusItem.Error)) } if count > 0 { return STATUS_UPLOADING } return STATUS_NOT_FOUND } func objectUrlBuilder(hash string, bus bool, proof bool) string { path := []string{} if bus { path = append(path, "bus") } else { path = append(path, "worker") } path = append(path, "objects") name := "%s" if proof { name = name + ".obao" } path = append(path, name) return fmt.Sprintf(strings.Join(path, "/"), hash) } func getBusObjectUrl(hash string) string { return objectUrlBuilder(hash, true, false) } func getWorkerObjectUrl(hash string) string { return objectUrlBuilder(hash, false, false) } func getWorkerProofUrl(hash string) string { return objectUrlBuilder(hash, false, true) } func getBusProofUrl(hash string) string { return objectUrlBuilder(hash, true, true) } func getStore() *tusstore.DbFileStore { ret := shared.GetTusStore() return (*ret).(*tusstore.DbFileStore) } func Pin(hash string, accountID uint) error { var upload model.Upload if result := db.Get().Model(&upload).Where("hash = ?", hash).First(&upload); result.Error != nil { if !errors.Is(result.Error, gorm.ErrRecordNotFound) { logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(result.Error)) } return ErrFailedQueryUpload } var pin model.Pin result := db.Get().Model(&pin).Where(&model.Pin{Upload: upload, AccountID: accountID}).First(&pin) if result.Error != nil && !errors.Is(result.Error, gorm.ErrRecordNotFound) { logger.Get().Error(ErrFailedQueryPins.Error(), zap.Error(result.Error)) return ErrFailedQueryPins } if result.Error == nil { return nil } pin.AccountID = upload.AccountID pin.Upload = upload result = db.Get().Save(&pin) if result.Error != nil { logger.Get().Error(ErrFailedSavePin.Error(), zap.Error(result.Error)) return ErrFailedSavePin } return nil }