LumeWeb
/
portal
1
0
Fork 0
Code Issues 5 Pull Requests Packages Projects Releases Wiki Activity

feat: implement /s5/upload/directory

This commit is contained in:
Derrick Hammer 2024-01-17 14:46:37 -05:00
parent 8c4687fd67
commit 5fec2f08ff
Signed by: pcfreak30
GPG Key ID: C997C339BE476FF2
3 changed files with 195 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
api/s5.go
View File

@ -38,7 +38,8 @@ func getRoutes(h *s5.HttpHandler, portal interfaces.Portal) map[string]jape.Hand
"GET /s5/account/pins.bin": s5.AuthMiddleware(h.AccountPins, portal),
// Upload API
"POST /s5/upload": s5.AuthMiddleware(h.SmallFileUpload, portal),
"POST /s5/upload": s5.AuthMiddleware(h.SmallFileUpload, portal),
"POST /s5/upload/directory": s5.AuthMiddleware(h.DirectoryUpload, portal),
// Pins API
"POST /s5/pin/:cid": s5.AuthMiddleware(h.AccountPin, portal),

196
api/s5/http.go
View File

@ -9,6 +9,7 @@ import (
"errors"
"fmt"
"git.lumeweb.com/LumeWeb/libs5-go/encoding"
"git.lumeweb.com/LumeWeb/libs5-go/metadata"
"git.lumeweb.com/LumeWeb/libs5-go/types"
"git.lumeweb.com/LumeWeb/portal/db/models"
"git.lumeweb.com/LumeWeb/portal/interfaces"
@ -36,6 +37,7 @@ const (
errFailedToGetPins = "Failed to get pins"
errFailedToDelPin = "Failed to delete pin"
errFailedToAddPin = "Failed to add pin"
errorNotMultiform = "Not a multipart form"
)
var (
@ -54,6 +56,7 @@ var (
errFailedToGetPinsErr = errors.New(errFailedToGetPins)
errFailedToDelPinErr = errors.New(errFailedToDelPin)
errFailedToAddPinErr = errors.New(errFailedToAddPin)
errNotMultiformErr = errors.New(errorNotMultiform)
)
type HttpHandler struct {
@ -133,12 +136,6 @@ func (h *HttpHandler) SmallFileUpload(jc jape.Context) {
return
}
if err != nil {
_ = jc.Error(errUploadingFileErr, http.StatusInternalServerError)
h.portal.Logger().Error(errUploadingFile, zap.Error(err))
return
}
if exists, upload := h.portal.Storage().FileExists(hash); exists {
cid, err := encoding.CIDFromHash(hash, upload.Size, types.CIDTypeRaw, types.HashTypeBlake3)
if err != nil {
@ -677,6 +674,193 @@ func (h *HttpHandler) AccountPin(jc jape.Context) {
jc.ResponseWriter.WriteHeader(http.StatusNoContent)
}
func (h *HttpHandler) DirectoryUpload(jc jape.Context) {
var tryFiles []string
var errorPages map[int]string
var name string
if jc.DecodeForm("tryFiles", &tryFiles) != nil {
return
}
if jc.DecodeForm("errorPages", &errorPages) != nil {
return
}
if jc.DecodeForm("name", &name) != nil {
return
}
r := jc.Request
contentType := r.Header.Get("Content-Type")
errored := func(err error) {
_ = jc.Error(errUploadingFileErr, http.StatusInternalServerError)
h.portal.Logger().Error(errUploadingFile, zap.Error(err))
}
if !strings.HasPrefix(contentType, "multipart/form-data") {
_ = jc.Error(errNotMultiformErr, http.StatusBadRequest)
h.portal.Logger().Error(errorNotMultiform)
return
}
err := r.ParseMultipartForm(h.portal.Config().GetInt64("core.post-upload-limit"))
if jc.Check(errMultiformParse, err) != nil {
h.portal.Logger().Error(errMultiformParse, zap.Error(err))
return
}
uploadMap := make(map[string]models.Upload, len(r.MultipartForm.File))
mimeMap := make(map[string]string, len(r.MultipartForm.File))
for _, files := range r.MultipartForm.File {
for _, fileHeader := range files {
// Open the file.
file, err := fileHeader.Open()
if err != nil {
errored(err)
return
}
defer func(file multipart.File) {
err := file.Close()
if err != nil {
h.portal.Logger().Error(errClosingStream, zap.Error(err))
}
}(file)
var rs io.ReadSeeker
hash, err := h.portal.Storage().GetHash(rs)
_, err = rs.Seek(0, io.SeekStart)
if err != nil {
_ = jc.Error(errUploadingFileErr, http.StatusInternalServerError)
h.portal.Logger().Error(errUploadingFile, zap.Error(err))
return
}
if exists, upload := h.portal.Storage().FileExists(hash); exists {
uploadMap[fileHeader.Filename] = upload
continue
}
hash, err = h.portal.Storage().PutFile(rs, "s5", false)
if err != nil {
errored(err)
return
}
upload, err := h.portal.Storage().CreateUpload(hash, uint(jc.Request.Context().Value(AuthUserIDKey).(uint64)), jc.Request.RemoteAddr, uint64(fileHeader.Size), "s5")
if err != nil {
errored(err)
return
}
// Reset the read pointer back to the start of the file.
if _, err := file.Seek(0, io.SeekStart); err != nil {
errored(err)
return
}
// Read a snippet of the file to determine its MIME type.
buffer := make([]byte, 512) // 512 bytes should be enough for http.DetectContentType to determine the type
if _, err := file.Read(buffer); err != nil {
errored(err)
return
}
// Reset the read pointer back to the start of the file.
if _, err := file.Seek(0, 0); err != nil {
errored(err)
return
}
// Detect MIME type.
mimeType := http.DetectContentType(buffer)
uploadMap[fileHeader.Filename] = *upload
mimeMap[fileHeader.Filename] = mimeType
}
}
filesMap := make(map[string]metadata.WebAppMetadataFileReference, len(uploadMap))
for name, file := range uploadMap {
hashDecoded, err := hex.DecodeString(file.Hash)
if err != nil {
errored(err)
return
}
cid, err := encoding.CIDFromHash(hashDecoded, file.Size, types.CIDTypeRaw, types.HashTypeBlake3)
if err != nil {
errored(err)
return
}
filesMap[name] = *metadata.NewWebAppMetadataFileReference(cid, mimeMap[name])
}
app := metadata.NewWebAppMetadata(name, tryFiles, *metadata.NewExtraMetadata(map[int]interface{}{}), errorPages, filesMap)
appData, err := msgpack.Marshal(app)
if err != nil {
errored(err)
return
}
var rs = bytes.NewReader(appData)
hash, err := h.portal.Storage().GetHash(rs)
_, err = rs.Seek(0, io.SeekStart)
if err != nil {
_ = jc.Error(errUploadingFileErr, http.StatusInternalServerError)
h.portal.Logger().Error(errUploadingFile, zap.Error(err))
return
}
if exists, upload := h.portal.Storage().FileExists(hash); exists {
cid, err := encoding.CIDFromHash(hash, upload.Size, types.CIDTypeMetadataWebapp, types.HashTypeBlake3)
if err != nil {
_ = jc.Error(errUploadingFileErr, http.StatusInternalServerError)
h.portal.Logger().Error(errUploadingFile, zap.Error(err))
return
}
cidStr, err := cid.ToString()
if err != nil {
_ = jc.Error(errUploadingFileErr, http.StatusInternalServerError)
h.portal.Logger().Error(errUploadingFile, zap.Error(err))
return
}
jc.Encode(map[string]string{"hash": cidStr})
return
}
hash, err = h.portal.Storage().PutFile(rs, "s5", false)
if err != nil {
errored(err)
return
}
cid, err := encoding.CIDFromHash(hash, uint64(len(appData)), types.CIDTypeRaw, types.HashTypeBlake3)
if err != nil {
errored(err)
return
}
cidStr, err := cid.ToString()
if err != nil {
errored(err)
return
}
jc.Encode(&AppUploadResponse{CID: cidStr})
}
func setAuthCookie(jwt string, jc jape.Context) {
authCookie := http.Cookie{
Name: "s5-auth-token",

3
api/s5/messages.go
View File

@ -50,3 +50,6 @@ type AccountStats struct {
type AccountStatsTotal struct {
UsedStorage uint64 `json:"usedStorage"`
}
type AppUploadResponse struct {
CID string `json:"cid"`
}