From fc042570ab056c7aef9db4c9e6129a8d971373a6 Mon Sep 17 00:00:00 2001 From: Derrick Hammer Date: Sat, 17 Feb 2024 23:06:27 -0500 Subject: [PATCH] refactor: change to store the hash as a raw, but also make it unique with an index --- account/account.go | 6 ++---- api/s5/messages.go | 9 +-------- api/s5/s5.go | 3 +-- db/models/tus_upload.go | 2 +- db/models/upload.go | 2 +- metadata/metadata.go | 14 ++++---------- protocols/s5/tus.go | 23 ++++++----------------- 7 files changed, 16 insertions(+), 43 deletions(-) diff --git a/account/account.go b/account/account.go index 1423283..92c457a 100644 --- a/account/account.go +++ b/account/account.go @@ -2,7 +2,6 @@ package account import ( "crypto/ed25519" - "encoding/hex" "errors" "time" @@ -249,7 +248,7 @@ func (s AccountServiceDefault) AccountPins(id uint, createdAfter uint64) ([]mode return pins, nil } -func (s AccountServiceDefault) DeletePinByHash(hash string, userId uint) error { +func (s AccountServiceDefault) DeletePinByHash(hash []byte, userId uint) error { // Define a struct for the query condition uploadQuery := models.Upload{Hash: hash} @@ -282,9 +281,8 @@ func (s AccountServiceDefault) DeletePinByHash(hash string, userId uint) error { return nil } func (s AccountServiceDefault) PinByHash(hash []byte, userId uint) error { - hashStr := hex.EncodeToString(hash) // Define a struct for the query condition - uploadQuery := models.Upload{Hash: hashStr} + uploadQuery := models.Upload{Hash: hash} // Retrieve the upload ID for the given hash var uploadID uint diff --git a/api/s5/messages.go b/api/s5/messages.go index dec39e5..3f68ae8 100644 --- a/api/s5/messages.go +++ b/api/s5/messages.go @@ -1,7 +1,6 @@ package s5 import ( - "encoding/hex" "git.lumeweb.com/LumeWeb/libs5-go/encoding" "git.lumeweb.com/LumeWeb/libs5-go/types" "git.lumeweb.com/LumeWeb/portal/db/models" @@ -113,13 +112,7 @@ func (a AccountPinResponse) EncodeMsgpack(enc *msgpack.Encoder) error { } for _, pin := range a.Pins { - hash, err := hex.DecodeString(pin.Upload.Hash) - - if err != nil { - return err - } - - err = enc.EncodeBytes(encoding.MultihashFromBytes(hash, types.HashTypeBlake3).FullBytes()) + err = enc.EncodeBytes(encoding.MultihashFromBytes(pin.Upload.Hash, types.HashTypeBlake3).FullBytes()) if err != nil { return err } diff --git a/api/s5/s5.go b/api/s5/s5.go index 7974580..ec3452c 100644 --- a/api/s5/s5.go +++ b/api/s5/s5.go @@ -695,8 +695,7 @@ func (s *S5API) accountPinDelete(jc jape.Context) { return } - hash := hex.EncodeToString(decodedCid.Hash.HashBytes()) - if err := s.accounts.DeletePinByHash(hash, user); err != nil { + if err := s.accounts.DeletePinByHash(decodedCid.Hash.HashBytes(), user); err != nil { s.sendErrorResponse(jc, NewS5Error(ErrKeyStorageOperationFailed, err)) return } diff --git a/db/models/tus_upload.go b/db/models/tus_upload.go index 83d76ca..56456ef 100644 --- a/db/models/tus_upload.go +++ b/db/models/tus_upload.go @@ -4,7 +4,7 @@ import "gorm.io/gorm" type TusUpload struct { gorm.Model - Hash string `gorm:"uniqueIndex:idx_hash_deleted"` + Hash []byte `gorm:"type:binary(32);uniqueIndex:idx_hash_deleted"` MimeType string UploadID string `gorm:"uniqueIndex"` UploaderID uint diff --git a/db/models/upload.go b/db/models/upload.go index d3c9208..5b78310 100644 --- a/db/models/upload.go +++ b/db/models/upload.go @@ -5,7 +5,7 @@ import "gorm.io/gorm" type Upload struct { gorm.Model UserID uint - Hash string + Hash []byte `gorm:"type:binary(32);unique_index"` MimeType string Protocol string User User diff --git a/metadata/metadata.go b/metadata/metadata.go index e691f4c..924b6aa 100644 --- a/metadata/metadata.go +++ b/metadata/metadata.go @@ -2,7 +2,6 @@ package metadata import ( "context" - "encoding/hex" "errors" "time" @@ -73,7 +72,7 @@ func NewMetadataService(params MetadataServiceParams) *MetadataServiceDefault { func (m *MetadataServiceDefault) SaveUpload(ctx context.Context, metadata UploadMetadata) error { var upload models.Upload - upload.Hash = hex.EncodeToString(metadata.Hash) + upload.Hash = metadata.Hash ret := m.db.WithContext(ctx).Model(&models.Upload{}).Where(&upload).First(&upload) @@ -99,7 +98,7 @@ func (m *MetadataServiceDefault) SaveUpload(ctx context.Context, metadata Upload func (m *MetadataServiceDefault) GetUpload(ctx context.Context, objectHash []byte) (UploadMetadata, error) { var upload models.Upload - upload.Hash = hex.EncodeToString(objectHash) + upload.Hash = objectHash ret := m.db.WithContext(ctx).Model(&models.Upload{}).Where(&upload).First(&upload) @@ -109,14 +108,9 @@ func (m *MetadataServiceDefault) GetUpload(ctx context.Context, objectHash []byt } } - hash, err := hex.DecodeString(upload.Hash) - if err != nil { - return UploadMetadata{}, err - } - return UploadMetadata{ UserID: upload.UserID, - Hash: hash, + Hash: upload.Hash, MimeType: upload.MimeType, Protocol: upload.Protocol, UploaderIP: upload.UploaderIP, @@ -127,7 +121,7 @@ func (m *MetadataServiceDefault) GetUpload(ctx context.Context, objectHash []byt func (m *MetadataServiceDefault) DeleteUpload(ctx context.Context, objectHash []byte) error { var upload models.Upload - upload.Hash = hex.EncodeToString(objectHash) + upload.Hash = objectHash ret := m.db.WithContext(ctx).Model(&models.Upload{}).Where(&upload).First(&upload) diff --git a/protocols/s5/tus.go b/protocols/s5/tus.go index 1cdf3b4..b4fda80 100644 --- a/protocols/s5/tus.go +++ b/protocols/s5/tus.go @@ -178,19 +178,15 @@ func (t *TusHandler) Tus() *tusd.Handler { } func (t *TusHandler) UploadExists(ctx context.Context, hash []byte) (bool, models.TusUpload) { - hashStr := hex.EncodeToString(hash) - var upload models.TusUpload - result := t.db.WithContext(ctx).Model(&models.TusUpload{}).Where(&models.TusUpload{Hash: hashStr}).First(&upload) + result := t.db.WithContext(ctx).Model(&models.TusUpload{}).Where(&models.TusUpload{Hash: hash}).First(&upload) return result.RowsAffected > 0, upload } func (t *TusHandler) CreateUpload(ctx context.Context, hash []byte, uploadID string, uploaderID uint, uploaderIP string, protocol string) (*models.TusUpload, error) { - hashStr := hex.EncodeToString(hash) - upload := &models.TusUpload{ - Hash: hashStr, + Hash: hash, UploadID: uploadID, UploaderID: uploaderID, UploaderIP: uploaderIP, @@ -358,15 +354,8 @@ func (t *TusHandler) uploadTask(ctx context.Context, upload *models.TusUpload) e return err } - dbHash, err := hex.DecodeString(upload.Hash) - - if err != nil { - t.logger.Error("Could not decode proof", zap.Error(err)) - return err - } - - if !bytes.Equal(proof.Hash, dbHash) { - t.logger.Error("Hashes do not match", zap.Any("upload", upload), zap.Any("proof", proof), zap.Any("dbHash", dbHash)) + if !bytes.Equal(proof.Hash, upload.Hash) { + t.logger.Error("Hashes do not match", zap.Any("upload", upload), zap.Any("proof", proof), zap.Any("dbHash", hex.EncodeToString(upload.Hash))) return err } @@ -383,7 +372,7 @@ func (t *TusHandler) uploadTask(ctx context.Context, upload *models.TusUpload) e return tusUpload.GetReader(ctx) }, Bucket: upload.Protocol, - FileName: "/" + t.storageProtocol.EncodeFileName(dbHash), + FileName: "/" + t.storageProtocol.EncodeFileName(upload.Hash), Size: uint64(info.Size), }, proof) @@ -423,7 +412,7 @@ func (t *TusHandler) uploadTask(ctx context.Context, upload *models.TusUpload) e return err } - err = t.accounts.PinByHash(dbHash, upload.UploaderID) + err = t.accounts.PinByHash(upload.Hash, upload.UploaderID) if err != nil { t.logger.Error("Could not pin upload", zap.Error(err)) return err