portal/service/files/files.go

318 lines
8.7 KiB
Go
Raw Normal View History

package files
import (
"bytes"
"context"
"encoding/hex"
"errors"
"fmt"
"git.lumeweb.com/LumeWeb/portal/bao"
"git.lumeweb.com/LumeWeb/portal/db"
"git.lumeweb.com/LumeWeb/portal/logger"
"git.lumeweb.com/LumeWeb/portal/model"
"git.lumeweb.com/LumeWeb/portal/shared"
2023-05-24 00:10:51 +00:00
"git.lumeweb.com/LumeWeb/portal/tusstore"
"github.com/go-resty/resty/v2"
2023-06-06 20:32:07 +00:00
"github.com/spf13/viper"
2023-05-24 00:10:51 +00:00
_ "github.com/tus/tusd/pkg/handler"
2023-05-19 13:04:47 +00:00
"go.uber.org/zap"
"gorm.io/gorm"
"io"
2023-05-19 13:04:47 +00:00
"strings"
)
2023-06-06 20:33:14 +00:00
const (
STATUS_UPLOADED = iota
STATUS_UPLOADING = iota
STATUS_NOT_FOUND = iota
)
var (
ErrAlreadyExists = errors.New("Upload already exists")
ErrFailedFetchObject = errors.New("Failed fetching object")
ErrFailedFetchObjectProof = errors.New("Failed fetching object proof")
ErrFailedFetchTusObject = errors.New("Failed fetching tus object")
ErrFailedHashFile = errors.New("Failed to hash file")
ErrFailedQueryTusUpload = errors.New("Failed to query tus uploads")
2023-06-09 11:04:06 +00:00
ErrFailedQueryUpload = errors.New("Failed to query uploads")
ErrFailedQueryPins = errors.New("Failed to query pins")
ErrFailedSaveUpload = errors.New("Failed saving upload to db")
2023-06-09 11:04:06 +00:00
ErrFailedSavePin = errors.New("Failed saving pin to db")
ErrFailedUpload = errors.New("Failed uploading object")
ErrFailedUploadProof = errors.New("Failed uploading object proof")
ErrFileExistsOutOfSync = errors.New("File already exists in network, but missing in database")
ErrFileHashMismatch = errors.New("File hash does not match provided file hash")
ErrInvalidFile = errors.New("Invalid file")
)
var client *resty.Client
func Init() {
client = resty.New()
2023-06-06 20:32:07 +00:00
client.SetBaseURL("http://localhost:9980/api")
client.SetBasicAuth("", viper.GetString("renterd-api-password"))
client.SetDisableWarn(true)
}
func Upload(r io.ReadSeeker, size int64, hash []byte) (model.Upload, error) {
var upload model.Upload
tree, hashBytes, err := bao.ComputeTree(r, size)
if err != nil {
logger.Get().Error(ErrFailedHashFile.Error(), zap.Error(err))
return upload, ErrFailedHashFile
}
if hash != nil {
if bytes.Compare(hashBytes[:], hash) != 0 {
logger.Get().Error(ErrFileHashMismatch.Error())
return upload, ErrFileHashMismatch
}
}
hashHex := hex.EncodeToString(hashBytes[:])
_, err = r.Seek(0, io.SeekStart)
if err != nil {
return upload, err
}
2023-05-11 19:25:31 +00:00
result := db.Get().Where(&model.Upload{Hash: hashHex}).First(&upload)
if (result.Error != nil && !errors.Is(result.Error, gorm.ErrRecordNotFound)) || result.RowsAffected > 0 {
err := result.Row().Scan(&upload)
if err != nil {
logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(err))
return upload, ErrFailedQueryUpload
}
if result.RowsAffected > 0 && upload.ID > 0 {
logger.Get().Info(ErrAlreadyExists.Error())
return upload, nil
}
}
objectExistsResult, err := client.R().Get(getBusObjectUrl(hashHex))
if err != nil {
logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(err))
return upload, ErrFailedQueryUpload
}
objectStatusCode := objectExistsResult.StatusCode()
if objectStatusCode == 500 {
bodyErr := objectExistsResult.String()
if !strings.Contains(bodyErr, "no slabs found") {
logger.Get().Error(ErrFailedFetchObject.Error(), zap.String("error", objectExistsResult.String()))
return upload, ErrFailedFetchObject
}
2023-05-17 17:33:22 +00:00
objectStatusCode = 404
}
proofExistsResult, err := client.R().Get(getBusProofUrl(hashHex))
if err != nil {
logger.Get().Error(ErrFailedFetchObjectProof.Error(), zap.Error(err))
return upload, ErrFailedFetchObjectProof
}
proofStatusCode := proofExistsResult.StatusCode()
if proofStatusCode == 500 {
bodyErr := proofExistsResult.String()
if !strings.Contains(bodyErr, "no slabs found") {
logger.Get().Error(ErrFailedFetchObjectProof.Error(), zap.String("error", proofExistsResult.String()))
return upload, ErrFailedFetchObjectProof
}
objectStatusCode = 404
}
if objectStatusCode != 404 && proofStatusCode != 404 {
logger.Get().Error(ErrFileExistsOutOfSync.Error(), zap.String("hash", hashHex))
return upload, ErrFileExistsOutOfSync
}
ret, err := client.R().SetBody(r).Put(getWorkerObjectUrl(hashHex))
if ret.StatusCode() != 200 {
logger.Get().Error(ErrFailedUpload.Error(), zap.String("error", ret.String()))
return upload, ErrFailedUpload
}
ret, err = client.R().SetBody(tree).Put(getWorkerProofUrl(hashHex))
if ret.StatusCode() != 200 {
logger.Get().Error(ErrFailedUploadProof.Error(), zap.String("error", ret.String()))
return upload, ErrFailedUpload
}
upload = model.Upload{
Hash: hashHex,
}
if err = db.Get().Create(&upload).Error; err != nil {
logger.Get().Error(ErrFailedSaveUpload.Error(), zap.Error(err))
return upload, ErrFailedSaveUpload
}
return upload, nil
}
func Download(hash string) (io.Reader, error) {
uploadItem := db.Get().Table("uploads").Where(&model.Upload{Hash: hash}).Row()
tusItem := db.Get().Table("tus").Where(&model.Tus{Hash: hash}).Row()
if uploadItem.Err() == nil {
2023-06-09 19:48:41 +00:00
fetch, err := client.R().SetDoNotParseResponse(true).Get(getWorkerProofUrl(hash))
if err != nil {
logger.Get().Error(ErrFailedFetchObject.Error(), zap.Error(err))
return nil, ErrFailedFetchObject
}
return fetch.RawBody(), nil
} else if tusItem.Err() == nil {
var tusData model.Tus
err := tusItem.Scan(&tusData)
if err != nil {
logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(err))
return nil, ErrFailedQueryUpload
}
2023-05-24 00:10:51 +00:00
upload, err := getStore().GetUpload(context.Background(), tusData.UploadID)
if err != nil {
logger.Get().Error(ErrFailedQueryTusUpload.Error(), zap.Error(err))
return nil, ErrFailedQueryTusUpload
}
reader, err := upload.GetReader(context.Background())
if err != nil {
logger.Get().Error(ErrFailedFetchTusObject.Error(), zap.Error(err))
return nil, ErrFailedFetchTusObject
}
return reader, nil
} else {
logger.Get().Error(ErrInvalidFile.Error(), zap.String("hash", hash))
return nil, ErrInvalidFile
}
}
2023-06-09 19:52:58 +00:00
func DownloadProof(hash string) (io.Reader, error) {
uploadItem := db.Get().Model(&model.Upload{}).Where(&model.Upload{Hash: hash}).Row()
if uploadItem.Err() != nil {
logger.Get().Debug(ErrInvalidFile.Error(), zap.String("hash", hash))
return nil, ErrInvalidFile
}
fetch, err := client.R().SetDoNotParseResponse(true).Get(getWorkerProofUrl(hash))
if err != nil {
logger.Get().Error(ErrFailedFetchObject.Error(), zap.Error(err))
return nil, ErrFailedFetchObject
}
return fetch.RawBody(), nil
}
2023-06-06 20:33:14 +00:00
func Status(hash string) int {
var count int64
uploadItem := db.Get().Table("uploads").Where(&model.Upload{Hash: hash}).Count(&count)
if uploadItem.Error != nil && !errors.Is(uploadItem.Error, gorm.ErrRecordNotFound) {
logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(uploadItem.Error))
2023-06-06 20:33:14 +00:00
}
if count > 0 {
return STATUS_UPLOADED
}
tusItem := db.Get().Table("tus").Where(&model.Tus{Hash: hash}).Count(&count)
if tusItem.Error != nil && !errors.Is(tusItem.Error, gorm.ErrRecordNotFound) {
logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(tusItem.Error))
2023-06-06 20:33:14 +00:00
}
if count > 0 {
return STATUS_UPLOADING
}
return STATUS_NOT_FOUND
}
func objectUrlBuilder(hash string, bus bool, proof bool) string {
path := []string{}
if bus {
path = append(path, "bus")
} else {
path = append(path, "worker")
}
path = append(path, "objects")
name := "%s"
if proof {
name = name + ".obao"
}
path = append(path, name)
return fmt.Sprintf(strings.Join(path, "/"), hash)
}
func getBusObjectUrl(hash string) string {
return objectUrlBuilder(hash, true, false)
}
func getWorkerObjectUrl(hash string) string {
return objectUrlBuilder(hash, false, false)
}
func getWorkerProofUrl(hash string) string {
return objectUrlBuilder(hash, false, true)
}
func getBusProofUrl(hash string) string {
return objectUrlBuilder(hash, true, true)
}
2023-05-24 00:10:51 +00:00
func getStore() *tusstore.DbFileStore {
ret := shared.GetTusStore()
return (*ret).(*tusstore.DbFileStore)
}
2023-06-09 11:04:06 +00:00
func Pin(hash string, accountID uint) error {
var upload model.Upload
if result := db.Get().Model(&upload).Where("hash = ?", hash).First(&upload); result.Error != nil {
if !errors.Is(result.Error, gorm.ErrRecordNotFound) {
logger.Get().Error(ErrFailedQueryUpload.Error(), zap.Error(result.Error))
}
return ErrFailedQueryUpload
}
var pin model.Pin
result := db.Get().Model(&pin).Where(&model.Pin{Upload: upload, AccountID: accountID}).First(&pin)
if result.Error != nil && !errors.Is(result.Error, gorm.ErrRecordNotFound) {
logger.Get().Error(ErrFailedQueryPins.Error(), zap.Error(result.Error))
return ErrFailedQueryPins
}
if result.Error == nil {
return nil
}
pin.AccountID = upload.AccountID
pin.Upload = upload
result = db.Get().Save(&pin)
if result.Error != nil {
logger.Get().Error(ErrFailedSavePin.Error(), zap.Error(result.Error))
return ErrFailedSavePin
}
return nil
}