refactor: change to store the hash as a raw, but also make it unique with an index

account/account.go

@@ -2,7 +2,6 @@ package account
 
 import (
 	"crypto/ed25519"
-	"encoding/hex"
 	"errors"
 	"time"
 
@@ -249,7 +248,7 @@ func (s AccountServiceDefault) AccountPins(id uint, createdAfter uint64) ([]mode
 	return pins, nil
 }
 
-func (s AccountServiceDefault) DeletePinByHash(hash string, userId uint) error {
+func (s AccountServiceDefault) DeletePinByHash(hash []byte, userId uint) error {
 	// Define a struct for the query condition
 	uploadQuery := models.Upload{Hash: hash}
 
@@ -282,9 +281,8 @@ func (s AccountServiceDefault) DeletePinByHash(hash string, userId uint) error {
 	return nil
 }
 func (s AccountServiceDefault) PinByHash(hash []byte, userId uint) error {
-	hashStr := hex.EncodeToString(hash)
 	// Define a struct for the query condition
-	uploadQuery := models.Upload{Hash: hashStr}
+	uploadQuery := models.Upload{Hash: hash}
 
 	// Retrieve the upload ID for the given hash
 	var uploadID uint

api/s5/messages.go

@@ -1,7 +1,6 @@
 package s5
 
 import (
-	"encoding/hex"
 	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
 	"git.lumeweb.com/LumeWeb/libs5-go/types"
 	"git.lumeweb.com/LumeWeb/portal/db/models"
@@ -113,13 +112,7 @@ func (a AccountPinResponse) EncodeMsgpack(enc *msgpack.Encoder) error {
 	}
 
 	for _, pin := range a.Pins {
-		hash, err := hex.DecodeString(pin.Upload.Hash)
+		err = enc.EncodeBytes(encoding.MultihashFromBytes(pin.Upload.Hash, types.HashTypeBlake3).FullBytes())
-
-		if err != nil {
-			return err
-		}
-
-		err = enc.EncodeBytes(encoding.MultihashFromBytes(hash, types.HashTypeBlake3).FullBytes())
 		if err != nil {
 			return err
 		}

api/s5/s5.go

@@ -695,8 +695,7 @@ func (s *S5API) accountPinDelete(jc jape.Context) {
 		return
 	}
 
-	hash := hex.EncodeToString(decodedCid.Hash.HashBytes())
+	if err := s.accounts.DeletePinByHash(decodedCid.Hash.HashBytes(), user); err != nil {
-	if err := s.accounts.DeletePinByHash(hash, user); err != nil {
 		s.sendErrorResponse(jc, NewS5Error(ErrKeyStorageOperationFailed, err))
 		return
 	}

db/models/tus_upload.go

@@ -4,7 +4,7 @@ import "gorm.io/gorm"
 
 type TusUpload struct {
 	gorm.Model
-	Hash       string `gorm:"uniqueIndex:idx_hash_deleted"`
+	Hash       []byte `gorm:"type:binary(32);uniqueIndex:idx_hash_deleted"`
 	MimeType   string
 	UploadID   string `gorm:"uniqueIndex"`
 	UploaderID uint

db/models/upload.go

@@ -5,7 +5,7 @@ import "gorm.io/gorm"
 type Upload struct {
 	gorm.Model
 	UserID     uint
-	Hash       string
+	Hash       []byte `gorm:"type:binary(32);unique_index"`
 	MimeType   string
 	Protocol   string
 	User       User

metadata/metadata.go

@@ -2,7 +2,6 @@ package metadata
 
 import (
 	"context"
-	"encoding/hex"
 	"errors"
 	"time"
 
@@ -73,7 +72,7 @@ func NewMetadataService(params MetadataServiceParams) *MetadataServiceDefault {
 func (m *MetadataServiceDefault) SaveUpload(ctx context.Context, metadata UploadMetadata) error {
 	var upload models.Upload
 
-	upload.Hash = hex.EncodeToString(metadata.Hash)
+	upload.Hash = metadata.Hash
 
 	ret := m.db.WithContext(ctx).Model(&models.Upload{}).Where(&upload).First(&upload)
 
@@ -99,7 +98,7 @@ func (m *MetadataServiceDefault) SaveUpload(ctx context.Context, metadata Upload
 func (m *MetadataServiceDefault) GetUpload(ctx context.Context, objectHash []byte) (UploadMetadata, error) {
 	var upload models.Upload
 
-	upload.Hash = hex.EncodeToString(objectHash)
+	upload.Hash = objectHash
 
 	ret := m.db.WithContext(ctx).Model(&models.Upload{}).Where(&upload).First(&upload)
 
@@ -109,14 +108,9 @@ func (m *MetadataServiceDefault) GetUpload(ctx context.Context, objectHash []byt
 		}
 	}
 
-	hash, err := hex.DecodeString(upload.Hash)
-	if err != nil {
-		return UploadMetadata{}, err
-	}
-
 	return UploadMetadata{
 		UserID:     upload.UserID,
-		Hash:       hash,
+		Hash:       upload.Hash,
 		MimeType:   upload.MimeType,
 		Protocol:   upload.Protocol,
 		UploaderIP: upload.UploaderIP,
@@ -127,7 +121,7 @@ func (m *MetadataServiceDefault) GetUpload(ctx context.Context, objectHash []byt
 func (m *MetadataServiceDefault) DeleteUpload(ctx context.Context, objectHash []byte) error {
 	var upload models.Upload
 
-	upload.Hash = hex.EncodeToString(objectHash)
+	upload.Hash = objectHash
 
 	ret := m.db.WithContext(ctx).Model(&models.Upload{}).Where(&upload).First(&upload)
 

protocols/s5/tus.go

@@ -178,19 +178,15 @@ func (t *TusHandler) Tus() *tusd.Handler {
 }
 
 func (t *TusHandler) UploadExists(ctx context.Context, hash []byte) (bool, models.TusUpload) {
-	hashStr := hex.EncodeToString(hash)
-
 	var upload models.TusUpload
-	result := t.db.WithContext(ctx).Model(&models.TusUpload{}).Where(&models.TusUpload{Hash: hashStr}).First(&upload)
+	result := t.db.WithContext(ctx).Model(&models.TusUpload{}).Where(&models.TusUpload{Hash: hash}).First(&upload)
 
 	return result.RowsAffected > 0, upload
 }
 
 func (t *TusHandler) CreateUpload(ctx context.Context, hash []byte, uploadID string, uploaderID uint, uploaderIP string, protocol string) (*models.TusUpload, error) {
-	hashStr := hex.EncodeToString(hash)
-
 	upload := &models.TusUpload{
-		Hash:       hashStr,
+		Hash:       hash,
 		UploadID:   uploadID,
 		UploaderID: uploaderID,
 		UploaderIP: uploaderIP,
@@ -358,15 +354,8 @@ func (t *TusHandler) uploadTask(ctx context.Context, upload *models.TusUpload) e
 		return err
 	}
 
-	dbHash, err := hex.DecodeString(upload.Hash)
+	if !bytes.Equal(proof.Hash, upload.Hash) {
-
+		t.logger.Error("Hashes do not match", zap.Any("upload", upload), zap.Any("proof", proof), zap.Any("dbHash", hex.EncodeToString(upload.Hash)))
-	if err != nil {
-		t.logger.Error("Could not decode proof", zap.Error(err))
-		return err
-	}
-
-	if !bytes.Equal(proof.Hash, dbHash) {
-		t.logger.Error("Hashes do not match", zap.Any("upload", upload), zap.Any("proof", proof), zap.Any("dbHash", dbHash))
 		return err
 	}
 
@@ -383,7 +372,7 @@ func (t *TusHandler) uploadTask(ctx context.Context, upload *models.TusUpload) e
 			return tusUpload.GetReader(ctx)
 		},
 		Bucket:   upload.Protocol,
-		FileName: "/" + t.storageProtocol.EncodeFileName(dbHash),
+		FileName: "/" + t.storageProtocol.EncodeFileName(upload.Hash),
 		Size:     uint64(info.Size),
 	}, proof)
 
@@ -423,7 +412,7 @@ func (t *TusHandler) uploadTask(ctx context.Context, upload *models.TusUpload) e
 		return err
 	}
 
-	err = t.accounts.PinByHash(dbHash, upload.UploaderID)
+	err = t.accounts.PinByHash(upload.Hash, upload.UploaderID)
 	if err != nil {
 		t.logger.Error("Could not pin upload", zap.Error(err))
 		return err