From 5fec2f08ff11992ebd0fc01b6229bd1d2bb71be8 Mon Sep 17 00:00:00 2001 From: Derrick Hammer Date: Wed, 17 Jan 2024 14:46:37 -0500 Subject: [PATCH] feat: implement /s5/upload/directory --- api/s5.go | 3 +- api/s5/http.go | 196 +++++++++++++++++++++++++++++++++++++++++++-- api/s5/messages.go | 3 + 3 files changed, 195 insertions(+), 7 deletions(-) diff --git a/api/s5.go b/api/s5.go index 49ddf09..037f85e 100644 --- a/api/s5.go +++ b/api/s5.go @@ -38,7 +38,8 @@ func getRoutes(h *s5.HttpHandler, portal interfaces.Portal) map[string]jape.Hand "GET /s5/account/pins.bin": s5.AuthMiddleware(h.AccountPins, portal), // Upload API - "POST /s5/upload": s5.AuthMiddleware(h.SmallFileUpload, portal), + "POST /s5/upload": s5.AuthMiddleware(h.SmallFileUpload, portal), + "POST /s5/upload/directory": s5.AuthMiddleware(h.DirectoryUpload, portal), // Pins API "POST /s5/pin/:cid": s5.AuthMiddleware(h.AccountPin, portal), diff --git a/api/s5/http.go b/api/s5/http.go index 552693d..245f9ae 100644 --- a/api/s5/http.go +++ b/api/s5/http.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "git.lumeweb.com/LumeWeb/libs5-go/encoding" + "git.lumeweb.com/LumeWeb/libs5-go/metadata" "git.lumeweb.com/LumeWeb/libs5-go/types" "git.lumeweb.com/LumeWeb/portal/db/models" "git.lumeweb.com/LumeWeb/portal/interfaces" @@ -36,6 +37,7 @@ const ( errFailedToGetPins = "Failed to get pins" errFailedToDelPin = "Failed to delete pin" errFailedToAddPin = "Failed to add pin" + errorNotMultiform = "Not a multipart form" ) var ( @@ -54,6 +56,7 @@ var ( errFailedToGetPinsErr = errors.New(errFailedToGetPins) errFailedToDelPinErr = errors.New(errFailedToDelPin) errFailedToAddPinErr = errors.New(errFailedToAddPin) + errNotMultiformErr = errors.New(errorNotMultiform) ) type HttpHandler struct { @@ -133,12 +136,6 @@ func (h *HttpHandler) SmallFileUpload(jc jape.Context) { return } - if err != nil { - _ = jc.Error(errUploadingFileErr, http.StatusInternalServerError) - h.portal.Logger().Error(errUploadingFile, zap.Error(err)) - return - } - if exists, upload := h.portal.Storage().FileExists(hash); exists { cid, err := encoding.CIDFromHash(hash, upload.Size, types.CIDTypeRaw, types.HashTypeBlake3) if err != nil { @@ -677,6 +674,193 @@ func (h *HttpHandler) AccountPin(jc jape.Context) { jc.ResponseWriter.WriteHeader(http.StatusNoContent) } +func (h *HttpHandler) DirectoryUpload(jc jape.Context) { + var tryFiles []string + var errorPages map[int]string + var name string + + if jc.DecodeForm("tryFiles", &tryFiles) != nil { + return + } + + if jc.DecodeForm("errorPages", &errorPages) != nil { + return + } + + if jc.DecodeForm("name", &name) != nil { + return + } + + r := jc.Request + contentType := r.Header.Get("Content-Type") + + errored := func(err error) { + _ = jc.Error(errUploadingFileErr, http.StatusInternalServerError) + h.portal.Logger().Error(errUploadingFile, zap.Error(err)) + } + + if !strings.HasPrefix(contentType, "multipart/form-data") { + _ = jc.Error(errNotMultiformErr, http.StatusBadRequest) + h.portal.Logger().Error(errorNotMultiform) + return + } + + err := r.ParseMultipartForm(h.portal.Config().GetInt64("core.post-upload-limit")) + + if jc.Check(errMultiformParse, err) != nil { + h.portal.Logger().Error(errMultiformParse, zap.Error(err)) + return + } + + uploadMap := make(map[string]models.Upload, len(r.MultipartForm.File)) + mimeMap := make(map[string]string, len(r.MultipartForm.File)) + + for _, files := range r.MultipartForm.File { + for _, fileHeader := range files { + // Open the file. + file, err := fileHeader.Open() + if err != nil { + errored(err) + return + } + defer func(file multipart.File) { + err := file.Close() + if err != nil { + h.portal.Logger().Error(errClosingStream, zap.Error(err)) + } + }(file) + + var rs io.ReadSeeker + + hash, err := h.portal.Storage().GetHash(rs) + _, err = rs.Seek(0, io.SeekStart) + if err != nil { + _ = jc.Error(errUploadingFileErr, http.StatusInternalServerError) + h.portal.Logger().Error(errUploadingFile, zap.Error(err)) + return + } + + if exists, upload := h.portal.Storage().FileExists(hash); exists { + uploadMap[fileHeader.Filename] = upload + continue + } + + hash, err = h.portal.Storage().PutFile(rs, "s5", false) + + if err != nil { + errored(err) + return + } + + upload, err := h.portal.Storage().CreateUpload(hash, uint(jc.Request.Context().Value(AuthUserIDKey).(uint64)), jc.Request.RemoteAddr, uint64(fileHeader.Size), "s5") + + if err != nil { + errored(err) + return + } + + // Reset the read pointer back to the start of the file. + if _, err := file.Seek(0, io.SeekStart); err != nil { + errored(err) + return + } + + // Read a snippet of the file to determine its MIME type. + buffer := make([]byte, 512) // 512 bytes should be enough for http.DetectContentType to determine the type + if _, err := file.Read(buffer); err != nil { + errored(err) + return + } + + // Reset the read pointer back to the start of the file. + if _, err := file.Seek(0, 0); err != nil { + errored(err) + return + } + + // Detect MIME type. + mimeType := http.DetectContentType(buffer) + + uploadMap[fileHeader.Filename] = *upload + mimeMap[fileHeader.Filename] = mimeType + } + } + filesMap := make(map[string]metadata.WebAppMetadataFileReference, len(uploadMap)) + + for name, file := range uploadMap { + hashDecoded, err := hex.DecodeString(file.Hash) + if err != nil { + errored(err) + return + } + + cid, err := encoding.CIDFromHash(hashDecoded, file.Size, types.CIDTypeRaw, types.HashTypeBlake3) + if err != nil { + errored(err) + return + } + + filesMap[name] = *metadata.NewWebAppMetadataFileReference(cid, mimeMap[name]) + } + + app := metadata.NewWebAppMetadata(name, tryFiles, *metadata.NewExtraMetadata(map[int]interface{}{}), errorPages, filesMap) + + appData, err := msgpack.Marshal(app) + if err != nil { + errored(err) + return + } + + var rs = bytes.NewReader(appData) + + hash, err := h.portal.Storage().GetHash(rs) + _, err = rs.Seek(0, io.SeekStart) + if err != nil { + _ = jc.Error(errUploadingFileErr, http.StatusInternalServerError) + h.portal.Logger().Error(errUploadingFile, zap.Error(err)) + return + } + + if exists, upload := h.portal.Storage().FileExists(hash); exists { + cid, err := encoding.CIDFromHash(hash, upload.Size, types.CIDTypeMetadataWebapp, types.HashTypeBlake3) + if err != nil { + _ = jc.Error(errUploadingFileErr, http.StatusInternalServerError) + h.portal.Logger().Error(errUploadingFile, zap.Error(err)) + return + } + cidStr, err := cid.ToString() + if err != nil { + _ = jc.Error(errUploadingFileErr, http.StatusInternalServerError) + h.portal.Logger().Error(errUploadingFile, zap.Error(err)) + return + } + jc.Encode(map[string]string{"hash": cidStr}) + return + } + + hash, err = h.portal.Storage().PutFile(rs, "s5", false) + + if err != nil { + errored(err) + return + } + + cid, err := encoding.CIDFromHash(hash, uint64(len(appData)), types.CIDTypeRaw, types.HashTypeBlake3) + + if err != nil { + errored(err) + return + } + + cidStr, err := cid.ToString() + if err != nil { + errored(err) + return + } + + jc.Encode(&AppUploadResponse{CID: cidStr}) +} + func setAuthCookie(jwt string, jc jape.Context) { authCookie := http.Cookie{ Name: "s5-auth-token", diff --git a/api/s5/messages.go b/api/s5/messages.go index 527acd6..cd73130 100644 --- a/api/s5/messages.go +++ b/api/s5/messages.go @@ -50,3 +50,6 @@ type AccountStats struct { type AccountStatsTotal struct { UsedStorage uint64 `json:"usedStorage"` } +type AppUploadResponse struct { + CID string `json:"cid"` +}