From 12c10bf62f7b2914da31de768fb2387183ec370d Mon Sep 17 00:00:00 2001 From: Marius Date: Wed, 2 Mar 2022 00:36:49 +0100 Subject: [PATCH] v2: Rework hooks system (#516) * ci: Remove plugin hook handler * Rework error type from interface to struct * Avoid writing to http.ResponseWriter directly * Allow hooks to modify response * Add example for HTTP hooks using Python * Implement new plugin system using Hashicorp/go-plugin * Enable returning partial HTTPResponses * Remove some (unnecessary) error handling * Forward stdout and stderr from plugin to tusd * docs: Update examples * cli: Update filehooks to new system * cli: Renovate gRPC hooks * docs: Correct casing of gRPC * misc: Documentation, better examples, and code structure --- .gitignore | 2 + cmd/tusd/cli/flags.go | 6 +- cmd/tusd/cli/hooks.go | 83 ++-- cmd/tusd/cli/hooks/file.go | 52 ++- cmd/tusd/cli/hooks/grpc.go | 81 ++-- cmd/tusd/cli/hooks/hooks.go | 74 ++-- cmd/tusd/cli/hooks/http.go | 97 ++--- cmd/tusd/cli/hooks/plugin.go | 145 +++++-- cmd/tusd/cli/hooks/proto/v1/hook.proto | 77 ---- cmd/tusd/cli/hooks/proto/v2/hook.proto | 116 ++++++ docs/hooks.md | 6 +- docs/minio.txt | 3 + examples/README.md | 11 + examples/hooks/file/post-create | 14 + examples/hooks/file/post-finish | 11 + examples/hooks/file/post-receive | 15 + examples/hooks/file/post-terminate | 11 + examples/hooks/file/pre-create | 37 ++ examples/hooks/file/pre-finish | 10 + examples/hooks/grpc/Makefile | 2 + examples/hooks/grpc/hook_pb2.py | 139 +++++++ examples/hooks/grpc/hook_pb2_grpc.py | 70 ++++ examples/hooks/grpc/server.py | 57 +++ examples/hooks/http/server.py | 65 +++ examples/hooks/plugin/Makefile | 2 + examples/hooks/plugin/hook_handler.go | 87 ++++ examples/hooks/post-create | 8 - examples/hooks/post-finish | 4 - examples/hooks/post-receive | 8 - examples/hooks/post-terminate | 4 - examples/hooks/pre-create | 7 - go.mod | 1 + go.sum | 26 ++ pkg/handler/config.go | 18 +- pkg/handler/datastore.go | 3 + pkg/handler/error.go | 32 ++ pkg/handler/head_test.go | 6 +- pkg/handler/hooks.go | 25 ++ pkg/handler/http.go | 80 ++++ pkg/handler/metrics.go | 8 +- pkg/handler/unrouted_handler.go | 241 +++++------ pkg/proto/v1/hook.pb.go | 475 --------------------- pkg/proto/v2/hook.pb.go | 555 +++++++++++++++++++++++++ pkg/s3store/s3store.go | 2 +- pkg/s3store/s3store_test.go | 2 +- 45 files changed, 1808 insertions(+), 970 deletions(-) delete mode 100644 cmd/tusd/cli/hooks/proto/v1/hook.proto create mode 100644 cmd/tusd/cli/hooks/proto/v2/hook.proto create mode 100644 docs/minio.txt create mode 100644 examples/README.md create mode 100755 examples/hooks/file/post-create create mode 100755 examples/hooks/file/post-finish create mode 100755 examples/hooks/file/post-receive create mode 100755 examples/hooks/file/post-terminate create mode 100755 examples/hooks/file/pre-create create mode 100755 examples/hooks/file/pre-finish create mode 100644 examples/hooks/grpc/Makefile create mode 100644 examples/hooks/grpc/hook_pb2.py create mode 100644 examples/hooks/grpc/hook_pb2_grpc.py create mode 100644 examples/hooks/grpc/server.py create mode 100644 examples/hooks/http/server.py create mode 100644 examples/hooks/plugin/Makefile create mode 100644 examples/hooks/plugin/hook_handler.go delete mode 100755 examples/hooks/post-create delete mode 100755 examples/hooks/post-finish delete mode 100755 examples/hooks/post-receive delete mode 100755 examples/hooks/post-terminate delete mode 100755 examples/hooks/pre-create create mode 100644 pkg/handler/error.go create mode 100644 pkg/handler/hooks.go create mode 100644 pkg/handler/http.go delete mode 100644 pkg/proto/v1/hook.pb.go create mode 100644 pkg/proto/v2/hook.pb.go diff --git a/.gitignore b/.gitignore index 73f6e93..c23a6eb 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ node_modules/ .DS_Store ./tusd tusd_*_* +__pycache__/ +examples/hooks/plugin/hook_handler diff --git a/cmd/tusd/cli/flags.go b/cmd/tusd/cli/flags.go index 6c721ac..4f149ab 100644 --- a/cmd/tusd/cli/flags.go +++ b/cmd/tusd/cli/flags.go @@ -38,6 +38,7 @@ var Flags struct { AzObjectPrefix string AzEndpoint string EnabledHooksString string + PluginHookPath string FileHooksDir string HttpHooksEndpoint string HttpHooksForwardHeaders string @@ -46,8 +47,6 @@ var Flags struct { GrpcHooksEndpoint string GrpcHooksRetry int GrpcHooksBackoff int - HooksStopUploadCode int - PluginHookPath string EnabledHooks []hooks.HookType ShowVersion bool ExposeMetrics bool @@ -91,6 +90,7 @@ func ParseFlags() { flag.StringVar(&Flags.AzObjectPrefix, "azure-object-prefix", "", "Prefix for Azure object names") flag.StringVar(&Flags.AzEndpoint, "azure-endpoint", "", "Custom Endpoint to use for Azure BlockBlob Storage (requires azure-storage to be pass)") flag.StringVar(&Flags.EnabledHooksString, "hooks-enabled-events", "pre-create,post-create,post-receive,post-terminate,post-finish", "Comma separated list of enabled hook events (e.g. post-create,post-finish). Leave empty to enable default events") + flag.StringVar(&Flags.PluginHookPath, "hooks-plugin", "", "Path to a Go plugin for loading hook functions") flag.StringVar(&Flags.FileHooksDir, "hooks-dir", "", "Directory to search for available hooks scripts") flag.StringVar(&Flags.HttpHooksEndpoint, "hooks-http", "", "An HTTP endpoint to which hook events will be sent to") flag.StringVar(&Flags.HttpHooksForwardHeaders, "hooks-http-forward-headers", "", "List of HTTP request headers to be forwarded from the client request to the hook endpoint") @@ -99,8 +99,6 @@ func ParseFlags() { flag.StringVar(&Flags.GrpcHooksEndpoint, "hooks-grpc", "", "An gRPC endpoint to which hook events will be sent to") flag.IntVar(&Flags.GrpcHooksRetry, "hooks-grpc-retry", 3, "Number of times to retry on a server error or network timeout") flag.IntVar(&Flags.GrpcHooksBackoff, "hooks-grpc-backoff", 1, "Number of seconds to wait before retrying each retry") - flag.IntVar(&Flags.HooksStopUploadCode, "hooks-stop-code", 0, "Return code from post-receive hook which causes tusd to stop and delete the current upload. A zero value means that no uploads will be stopped") - flag.StringVar(&Flags.PluginHookPath, "hooks-plugin", "", "Path to a Go plugin for loading hook functions (only supported on Linux and macOS; highly EXPERIMENTAL and may BREAK in the future)") flag.BoolVar(&Flags.ShowVersion, "version", false, "Print tusd version information") flag.BoolVar(&Flags.ExposeMetrics, "expose-metrics", true, "Expose metrics about tusd usage") flag.StringVar(&Flags.MetricsPath, "metrics-path", "/metrics", "Path under which the metrics endpoint will be accessible") diff --git a/cmd/tusd/cli/hooks.go b/cmd/tusd/cli/hooks.go index 2c7bcf3..135ceaa 100644 --- a/cmd/tusd/cli/hooks.go +++ b/cmd/tusd/cli/hooks.go @@ -1,7 +1,6 @@ package cli import ( - "fmt" "strconv" "strings" @@ -20,27 +19,12 @@ func hookTypeInSlice(a hooks.HookType, list []hooks.HookType) bool { return false } -func hookCallback(typ hooks.HookType, info handler.HookEvent) error { - if output, err := invokeHookSync(typ, info, true); err != nil { - if hookErr, ok := err.(hooks.HookError); ok { - return hooks.NewHookError( - fmt.Errorf("%s hook failed: %s", typ, err), - hookErr.StatusCode(), - hookErr.Body(), - ) - } - return fmt.Errorf("%s hook failed: %s\n%s", typ, err, string(output)) - } - - return nil +func preCreateCallback(event handler.HookEvent) (handler.HTTPResponse, error) { + return invokeHookSync(hooks.HookPreCreate, event) } -func preCreateCallback(info handler.HookEvent) error { - return hookCallback(hooks.HookPreCreate, info) -} - -func preFinishCallback(info handler.HookEvent) error { - return hookCallback(hooks.HookPreFinish, info) +func preFinishCallback(event handler.HookEvent) (handler.HTTPResponse, error) { + return invokeHookSync(hooks.HookPreFinish, event) } func SetupHookMetrics() { @@ -113,35 +97,35 @@ func SetupPostHooks(handler *handler.Handler) { go func() { for { select { - case info := <-handler.CompleteUploads: - invokeHookAsync(hooks.HookPostFinish, info) - case info := <-handler.TerminatedUploads: - invokeHookAsync(hooks.HookPostTerminate, info) - case info := <-handler.UploadProgress: - invokeHookAsync(hooks.HookPostReceive, info) - case info := <-handler.CreatedUploads: - invokeHookAsync(hooks.HookPostCreate, info) + case event := <-handler.CompleteUploads: + invokeHookAsync(hooks.HookPostFinish, event) + case event := <-handler.TerminatedUploads: + invokeHookAsync(hooks.HookPostTerminate, event) + case event := <-handler.UploadProgress: + invokeHookAsync(hooks.HookPostReceive, event) + case event := <-handler.CreatedUploads: + invokeHookAsync(hooks.HookPostCreate, event) } } }() } -func invokeHookAsync(typ hooks.HookType, info handler.HookEvent) { +func invokeHookAsync(typ hooks.HookType, event handler.HookEvent) { go func() { // Error handling is taken care by the function. - _, _ = invokeHookSync(typ, info, false) + _, _ = invokeHookSync(typ, event) }() } -func invokeHookSync(typ hooks.HookType, info handler.HookEvent, captureOutput bool) ([]byte, error) { +func invokeHookSync(typ hooks.HookType, event handler.HookEvent) (httpRes handler.HTTPResponse, err error) { if !hookTypeInSlice(typ, Flags.EnabledHooks) { - return nil, nil + return httpRes, nil } MetricsHookInvocationsTotal.WithLabelValues(string(typ)).Add(1) - id := info.Upload.ID - size := info.Upload.Size + id := event.Upload.ID + size := event.Upload.Size switch typ { case hooks.HookPostFinish: @@ -151,28 +135,43 @@ func invokeHookSync(typ hooks.HookType, info handler.HookEvent, captureOutput bo } if hookHandler == nil { - return nil, nil + return httpRes, nil } - name := string(typ) if Flags.VerboseOutput { - logEv(stdout, "HookInvocationStart", "type", name, "id", id) + logEv(stdout, "HookInvocationStart", "type", string(typ), "id", id) } - output, returnCode, err := hookHandler.InvokeHook(typ, info, captureOutput) + hookRes, err := hookHandler.InvokeHook(hooks.HookRequest{ + Type: typ, + Event: event, + }) if err != nil { logEv(stderr, "HookInvocationError", "type", string(typ), "id", id, "error", err.Error()) MetricsHookErrorsTotal.WithLabelValues(string(typ)).Add(1) + return httpRes, err } else if Flags.VerboseOutput { logEv(stdout, "HookInvocationFinish", "type", string(typ), "id", id) } - if typ == hooks.HookPostReceive && Flags.HooksStopUploadCode != 0 && Flags.HooksStopUploadCode == returnCode { - logEv(stdout, "HookStopUpload", "id", id) + httpRes = hookRes.HTTPResponse - info.Upload.StopUpload() + // If the hook response includes the instruction to reject the upload, reuse the error code + // and message from ErrUploadRejectedByServer, but also include custom HTTP response values + if typ == hooks.HookPreCreate && hookRes.RejectUpload { + err := handler.ErrUploadRejectedByServer + err.HTTPResponse = err.HTTPResponse.MergeWith(httpRes) + + return httpRes, err } - return output, err + if typ == hooks.HookPostReceive && hookRes.StopUpload { + logEv(stdout, "HookStopUpload", "id", id) + + // TODO: Control response for PATCH request + event.Upload.StopUpload() + } + + return httpRes, err } diff --git a/cmd/tusd/cli/hooks/file.go b/cmd/tusd/cli/hooks/file.go index 3bcd03e..0ca60f9 100644 --- a/cmd/tusd/cli/hooks/file.go +++ b/cmd/tusd/cli/hooks/file.go @@ -3,11 +3,10 @@ package hooks import ( "bytes" "encoding/json" + "fmt" "os" "os/exec" "strconv" - - "github.com/tus/tusd/pkg/handler" ) type FileHook struct { @@ -18,43 +17,50 @@ func (_ FileHook) Setup() error { return nil } -func (h FileHook) InvokeHook(typ HookType, info handler.HookEvent, captureOutput bool) ([]byte, int, error) { - hookPath := h.Directory + string(os.PathSeparator) + string(typ) +func (h FileHook) InvokeHook(req HookRequest) (res HookResponse, err error) { + hookPath := h.Directory + string(os.PathSeparator) + string(req.Type) cmd := exec.Command(hookPath) env := os.Environ() - env = append(env, "TUS_ID="+info.Upload.ID) - env = append(env, "TUS_SIZE="+strconv.FormatInt(info.Upload.Size, 10)) - env = append(env, "TUS_OFFSET="+strconv.FormatInt(info.Upload.Offset, 10)) + env = append(env, "TUS_ID="+req.Event.Upload.ID) + env = append(env, "TUS_SIZE="+strconv.FormatInt(req.Event.Upload.Size, 10)) + env = append(env, "TUS_OFFSET="+strconv.FormatInt(req.Event.Upload.Offset, 10)) - jsonInfo, err := json.Marshal(info) + jsonReq, err := json.Marshal(req) if err != nil { - return nil, 0, err + return res, err } - reader := bytes.NewReader(jsonInfo) + reader := bytes.NewReader(jsonReq) cmd.Stdin = reader cmd.Env = env cmd.Dir = h.Directory cmd.Stderr = os.Stderr - // If `captureOutput` is true, this function will return the output (both, - // stderr and stdout), else it will use this process' stdout - var output []byte - if !captureOutput { - cmd.Stdout = os.Stdout - err = cmd.Run() - } else { - output, err = cmd.Output() - } + output, err := cmd.Output() - // Ignore the error, only, if the hook's file could not be found. This usually + // Ignore the error if the hook's file could not be found. This usually // means that the user is only using a subset of the available hooks. if os.IsNotExist(err) { - err = nil + return res, nil } - returnCode := cmd.ProcessState.ExitCode() + // Report error if the exit code was non-zero + if err, ok := err.(*exec.ExitError); ok { + return res, fmt.Errorf("unexpected return code %d from hook endpoint: %s", err.ProcessState.ExitCode(), string(output)) + } - return output, returnCode, err + if err != nil { + return res, err + } + + // Do not parse the output as JSON, if we received no output to reduce possible + // errors. + if len(output) > 0 { + if err = json.Unmarshal(output, &res); err != nil { + return res, fmt.Errorf("failed to parse hook response: %w, response was: %s", err, string(output)) + } + } + + return res, nil } diff --git a/cmd/tusd/cli/hooks/grpc.go b/cmd/tusd/cli/hooks/grpc.go index 2e8eef4..b9ac15c 100644 --- a/cmd/tusd/cli/hooks/grpc.go +++ b/cmd/tusd/cli/hooks/grpc.go @@ -5,17 +5,15 @@ import ( "time" grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry" - "github.com/tus/tusd/pkg/handler" - pb "github.com/tus/tusd/pkg/proto/v1" + pb "github.com/tus/tusd/pkg/proto/v2" "google.golang.org/grpc" - "google.golang.org/grpc/status" ) type GrpcHook struct { Endpoint string MaxRetries int Backoff int - Client pb.HookServiceClient + Client pb.HookHandlerClient } func (g *GrpcHook) Setup() error { @@ -31,44 +29,59 @@ func (g *GrpcHook) Setup() error { if err != nil { return err } - g.Client = pb.NewHookServiceClient(conn) + g.Client = pb.NewHookHandlerClient(conn) return nil } -func (g *GrpcHook) InvokeHook(typ HookType, info handler.HookEvent, captureOutput bool) ([]byte, int, error) { +func (g *GrpcHook) InvokeHook(hookReq HookRequest) (hookRes HookResponse, err error) { ctx := context.Background() - req := &pb.SendRequest{Hook: marshal(typ, info)} - resp, err := g.Client.Send(ctx, req) + req := marshal(hookReq) + res, err := g.Client.InvokeHook(ctx, req) if err != nil { - if e, ok := status.FromError(err); ok { - return nil, int(e.Code()), err - } - return nil, 2, err + return hookRes, err } - if captureOutput { - return resp.Response.GetValue(), 0, err - } - return nil, 0, err + + hookRes = unmarshal(res) + return hookRes, nil } -func marshal(typ HookType, info handler.HookEvent) *pb.Hook { - return &pb.Hook{ - Upload: &pb.Upload{ - Id: info.Upload.ID, - Size: info.Upload.Size, - SizeIsDeferred: info.Upload.SizeIsDeferred, - Offset: info.Upload.Offset, - MetaData: info.Upload.MetaData, - IsPartial: info.Upload.IsPartial, - IsFinal: info.Upload.IsFinal, - PartialUploads: info.Upload.PartialUploads, - Storage: info.Upload.Storage, +func marshal(hookReq HookRequest) *pb.HookRequest { + event := hookReq.Event + + return &pb.HookRequest{ + Type: string(hookReq.Type), + Event: &pb.Event{ + Upload: &pb.FileInfo{ + Id: event.Upload.ID, + Size: event.Upload.Size, + SizeIsDeferred: event.Upload.SizeIsDeferred, + Offset: event.Upload.Offset, + MetaData: event.Upload.MetaData, + IsPartial: event.Upload.IsPartial, + IsFinal: event.Upload.IsFinal, + PartialUploads: event.Upload.PartialUploads, + Storage: event.Upload.Storage, + }, + HttpRequest: &pb.HTTPRequest{ + Method: event.HTTPRequest.Method, + Uri: event.HTTPRequest.URI, + RemoteAddr: event.HTTPRequest.RemoteAddr, + // TODO: HEaders + }, }, - HttpRequest: &pb.HTTPRequest{ - Method: info.HTTPRequest.Method, - Uri: info.HTTPRequest.URI, - RemoteAddr: info.HTTPRequest.RemoteAddr, - }, - Name: string(typ), } } + +func unmarshal(res *pb.HookResponse) (hookRes HookResponse) { + hookRes.RejectUpload = res.RejectUpload + hookRes.StopUpload = res.StopUpload + + httpRes := res.HttpResponse + if httpRes != nil { + hookRes.HTTPResponse.StatusCode = int(httpRes.StatusCode) + hookRes.HTTPResponse.Headers = httpRes.Headers + hookRes.HTTPResponse.Body = httpRes.Body + } + + return hookRes +} diff --git a/cmd/tusd/cli/hooks/hooks.go b/cmd/tusd/cli/hooks/hooks.go index b72e3c3..90c669a 100644 --- a/cmd/tusd/cli/hooks/hooks.go +++ b/cmd/tusd/cli/hooks/hooks.go @@ -1,12 +1,58 @@ package hooks +// TODO: Move hooks into a package in /pkg + import ( "github.com/tus/tusd/pkg/handler" ) +// HookHandler is the main inferface to be implemented by all hook backends. type HookHandler interface { + // Setup is invoked once the hook backend is initalized. Setup() error - InvokeHook(typ HookType, info handler.HookEvent, captureOutput bool) ([]byte, int, error) + // InvokeHook is invoked for every hook that is executed. req contains the + // corresponding information about the hook type, the involved upload, and + // causing HTTP request. + // The return value res allows to stop or reject an upload, as well as modifying + // the HTTP response. See the documentation for HookResponse for more details. + // If err is not nil, the value of res will be ignored. err should only be + // non-nil if the hook failed to complete successfully. + InvokeHook(req HookRequest) (res HookResponse, err error) +} + +// HookRequest contains the information about the hook type, the involved upload, +// and causing HTTP request. +type HookRequest struct { + // Type is the name of the hook. + Type HookType + // Event contains the involved upload and causing HTTP request. + Event handler.HookEvent +} + +// HookResponse is the response after a hook is executed. +type HookResponse struct { + // HTTPResponse's fields can be filled to modify the HTTP response. + // This is only possible for pre-create, pre-finish and post-receive hooks. + // For other hooks this value is ignored. + // If multiple hooks modify the HTTP response, a later hook may overwrite the + // modified values from a previous hook (e.g. if multiple post-receive hooks + // are executed). + // Example usages: Send an error to the client if RejectUpload/StopUpload are + // set in the pre-create/post-receive hook. Send more information to the client + // in the pre-finish hook. + HTTPResponse handler.HTTPResponse + + // RejectUpload will cause the upload to be rejected and not be created during + // POST request. This value is only respected for pre-create hooks. For other hooks, + // it is ignored. Use the HTTPResponse field to send details about the rejection + // to the client. + RejectUpload bool + + // StopUpload will cause the upload to be stopped during a PATCH request. + // This value is only respected for post-receive hooks. For other hooks, + // it is ignored. Use the HTTPResponse field to send details about the stop + // to the client. + StopUpload bool } type HookType string @@ -21,29 +67,3 @@ const ( ) var AvailableHooks []HookType = []HookType{HookPreCreate, HookPostCreate, HookPostReceive, HookPostTerminate, HookPostFinish, HookPreFinish} - -type hookDataStore struct { - handler.DataStore -} - -type HookError struct { - error - statusCode int - body []byte -} - -func NewHookError(err error, statusCode int, body []byte) HookError { - return HookError{err, statusCode, body} -} - -func (herr HookError) StatusCode() int { - return herr.statusCode -} - -func (herr HookError) Body() []byte { - return herr.body -} - -func (herr HookError) Error() string { - return herr.error.Error() -} diff --git a/cmd/tusd/cli/hooks/http.go b/cmd/tusd/cli/hooks/http.go index 4cc3b87..af781de 100644 --- a/cmd/tusd/cli/hooks/http.go +++ b/cmd/tusd/cli/hooks/http.go @@ -8,8 +8,6 @@ import ( "net/http" "time" - "github.com/tus/tusd/pkg/handler" - "github.com/sethgrid/pester" ) @@ -18,35 +16,11 @@ type HttpHook struct { MaxRetries int Backoff int ForwardHeaders []string + + client *pester.Client } -func (_ HttpHook) Setup() error { - return nil -} - -func (h HttpHook) InvokeHook(typ HookType, info handler.HookEvent, captureOutput bool) ([]byte, int, error) { - jsonInfo, err := json.Marshal(info) - if err != nil { - return nil, 0, err - } - - req, err := http.NewRequest("POST", h.Endpoint, bytes.NewBuffer(jsonInfo)) - if err != nil { - return nil, 0, err - } - - for _, k := range h.ForwardHeaders { - // Lookup the Canonicalised version of the specified header - if vals, ok := info.HTTPRequest.Header[http.CanonicalHeaderKey(k)]; ok { - // but set the case specified by the user - req.Header[k] = vals - } - } - - req.Header.Set("Hook-Name", string(typ)) - req.Header.Set("Content-Type", "application/json") - - // TODO: Can we initialize this in Setup()? +func (h *HttpHook) Setup() error { // Use linear backoff strategy with the user defined values. client := pester.New() client.KeepLog = true @@ -55,24 +29,51 @@ func (h HttpHook) InvokeHook(typ HookType, info handler.HookEvent, captureOutput return time.Duration(h.Backoff) * time.Second } - resp, err := client.Do(req) - if err != nil { - return nil, 0, err - } - defer resp.Body.Close() + h.client = client - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, 0, err - } - - if resp.StatusCode >= http.StatusBadRequest { - return body, resp.StatusCode, NewHookError(fmt.Errorf("endpoint returned: %s", resp.Status), resp.StatusCode, body) - } - - if captureOutput { - return body, resp.StatusCode, err - } - - return nil, resp.StatusCode, err + return nil +} + +func (h HttpHook) InvokeHook(hookReq HookRequest) (hookRes HookResponse, err error) { + jsonInfo, err := json.Marshal(hookReq) + if err != nil { + return hookRes, err + } + + httpReq, err := http.NewRequest("POST", h.Endpoint, bytes.NewBuffer(jsonInfo)) + if err != nil { + return hookRes, err + } + + for _, k := range h.ForwardHeaders { + // Lookup the Canonicalised version of the specified header + if vals, ok := hookReq.Event.HTTPRequest.Header[http.CanonicalHeaderKey(k)]; ok { + // but set the case specified by the user + httpReq.Header[k] = vals + } + } + + httpReq.Header.Set("Content-Type", "application/json") + + httpRes, err := h.client.Do(httpReq) + if err != nil { + return hookRes, err + } + defer httpRes.Body.Close() + + httpBody, err := ioutil.ReadAll(httpRes.Body) + if err != nil { + return hookRes, err + } + + // Report an error, if the response has a non-2XX status code + if httpRes.StatusCode < http.StatusOK || httpRes.StatusCode >= http.StatusMultipleChoices { + return hookRes, fmt.Errorf("unexpected response code from hook endpoint (%d): %s", httpRes.StatusCode, string(httpBody)) + } + + if err = json.Unmarshal(httpBody, &hookRes); err != nil { + return hookRes, fmt.Errorf("failed to parse hook response: %w", err) + } + + return hookRes, nil } diff --git a/cmd/tusd/cli/hooks/plugin.go b/cmd/tusd/cli/hooks/plugin.go index 8821527..3b766e3 100644 --- a/cmd/tusd/cli/hooks/plugin.go +++ b/cmd/tusd/cli/hooks/plugin.go @@ -1,69 +1,122 @@ package hooks import ( - "fmt" - "plugin" + "log" + "net/rpc" + "os" + "os/exec" - "github.com/tus/tusd/pkg/handler" + "github.com/hashicorp/go-plugin" ) -type PluginHookHandler interface { - PreCreate(info handler.HookEvent) error - PostCreate(info handler.HookEvent) error - PostReceive(info handler.HookEvent) error - PostFinish(info handler.HookEvent) error - PostTerminate(info handler.HookEvent) error - PreFinish(info handler.HookEvent) error -} +// TODO: When the tusd process stops, the plugin does not get properly killed +// and lives on as a zombie process. type PluginHook struct { Path string - handler PluginHookHandler + handlerImpl HookHandler } func (h *PluginHook) Setup() error { - p, err := plugin.Open(h.Path) + // We're a host! Start by launching the plugin process. + client := plugin.NewClient(&plugin.ClientConfig{ + HandshakeConfig: handshakeConfig, + Plugins: pluginMap, + Cmd: exec.Command(h.Path), + SyncStdout: os.Stdout, + SyncStderr: os.Stderr, + //Logger: logger, + }) + //defer client.Kill() + + // Connect via RPC + rpcClient, err := client.Client() if err != nil { - return err + log.Fatal(err) } - symbol, err := p.Lookup("TusdHookHandler") + // Request the plugin + raw, err := rpcClient.Dispense("hookHandler") if err != nil { - return err + log.Fatal(err) } - handler, ok := symbol.(*PluginHookHandler) - if !ok { - return fmt.Errorf("hooks: could not cast TusdHookHandler from %s into PluginHookHandler interface", h.Path) - } + // We should have a HookHandler now! This feels like a normal interface + // implementation but is in fact over an RPC connection. + h.handlerImpl = raw.(HookHandler) - h.handler = *handler - return nil + return h.handlerImpl.Setup() } -func (h PluginHook) InvokeHook(typ HookType, info handler.HookEvent, captureOutput bool) ([]byte, int, error) { - var err error - switch typ { - case HookPostFinish: - err = h.handler.PostFinish(info) - case HookPostTerminate: - err = h.handler.PostTerminate(info) - case HookPostReceive: - err = h.handler.PostReceive(info) - case HookPostCreate: - err = h.handler.PostCreate(info) - case HookPreCreate: - err = h.handler.PreCreate(info) - case HookPreFinish: - err = h.handler.PreFinish(info) - default: - err = fmt.Errorf("hooks: unknown hook named %s", typ) - } - - if err != nil { - return nil, 1, err - } - - return nil, 0, nil +func (h *PluginHook) InvokeHook(req HookRequest) (HookResponse, error) { + return h.handlerImpl.InvokeHook(req) +} + +// handshakeConfigs are used to just do a basic handshake between +// a plugin and host. If the handshake fails, a user friendly error is shown. +// This prevents users from executing bad plugins or executing a plugin +// directory. It is a UX feature, not a security feature. +var handshakeConfig = plugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "TUSD_PLUGIN", + MagicCookieValue: "yes", +} + +// pluginMap is the map of plugins we can dispense. +var pluginMap = map[string]plugin.Plugin{ + "hookHandler": &HookHandlerPlugin{}, +} + +// Here is an implementation that talks over RPC +type HookHandlerRPC struct{ client *rpc.Client } + +func (g *HookHandlerRPC) Setup() error { + var res interface{} + err := g.client.Call("Plugin.Setup", new(interface{}), &res) + return err +} + +func (g *HookHandlerRPC) InvokeHook(req HookRequest) (res HookResponse, err error) { + err = g.client.Call("Plugin.InvokeHook", req, &res) + return res, err +} + +// Here is the RPC server that HookHandlerRPC talks to, conforming to +// the requirements of net/rpc +type HookHandlerRPCServer struct { + // This is the real implementation + Impl HookHandler +} + +func (s *HookHandlerRPCServer) Setup(args interface{}, resp *interface{}) error { + return s.Impl.Setup() +} + +func (s *HookHandlerRPCServer) InvokeHook(args HookRequest, resp *HookResponse) (err error) { + *resp, err = s.Impl.InvokeHook(args) + return err +} + +// This is the implementation of plugin.Plugin so we can serve/consume this +// +// This has two methods: Server must return an RPC server for this plugin +// type. We construct a HookHandlerRPCServer for this. +// +// Client must return an implementation of our interface that communicates +// over an RPC client. We return HookHandlerRPC for this. +// +// Ignore MuxBroker. That is used to create more multiplexed streams on our +// plugin connection and is a more advanced use case. +type HookHandlerPlugin struct { + // Impl Injection + Impl HookHandler +} + +func (p *HookHandlerPlugin) Server(*plugin.MuxBroker) (interface{}, error) { + return &HookHandlerRPCServer{Impl: p.Impl}, nil +} + +func (HookHandlerPlugin) Client(b *plugin.MuxBroker, c *rpc.Client) (interface{}, error) { + return &HookHandlerRPC{client: c}, nil } diff --git a/cmd/tusd/cli/hooks/proto/v1/hook.proto b/cmd/tusd/cli/hooks/proto/v1/hook.proto deleted file mode 100644 index 08afe79..0000000 --- a/cmd/tusd/cli/hooks/proto/v1/hook.proto +++ /dev/null @@ -1,77 +0,0 @@ -// If this file gets changed, you must recompile the generate package in pkg/proto. -// To do this, install the Go protobuf toolchain as mentioned in -// https://github.com/golang/protobuf#installation. -// Then use following command to recompile it with gRPC support: -// protoc --go_out=plugins=grpc:../../../../../pkg/proto/ v1/hook.proto -// In addition, it may be necessary to update the protobuf or gRPC dependencies as well. - -syntax = "proto3"; -package v1; - -import "google/protobuf/any.proto"; - -// Uploaded data -message Upload { - // Unique integer identifier of the uploaded file - string id = 1; - // Total file size in bytes specified in the NewUpload call - int64 Size = 2; - // Indicates whether the total file size is deferred until later - bool SizeIsDeferred = 3; - // Offset in bytes (zero-based) - int64 Offset = 4; - map metaData = 5; - // Indicates that this is a partial upload which will later be used to form - // a final upload by concatenation. Partial uploads should not be processed - // when they are finished since they are only incomplete chunks of files. - bool isPartial = 6; - // Indicates that this is a final upload - bool isFinal = 7; - // If the upload is a final one (see IsFinal) this will be a non-empty - // ordered slice containing the ids of the uploads of which the final upload - // will consist after concatenation. - repeated string partialUploads = 8; - // Storage contains information about where the data storage saves the upload, - // for example a file path. The available values vary depending on what data - // store is used. This map may also be nil. - map storage = 9; -} - -message HTTPRequest { - // Method is the HTTP method, e.g. POST or PATCH - string method = 1; - // URI is the full HTTP request URI, e.g. /files/fooo - string uri = 2; - // RemoteAddr contains the network address that sent the request - string remoteAddr = 3; -} - -// Hook's data -message Hook { - // Upload contains information about the upload that caused this hook - // to be fired. - Upload upload = 1; - // HTTPRequest contains details about the HTTP request that reached - // tusd. - HTTPRequest httpRequest = 2; - // The hook name - string name = 3; -} - -// Request data to send hook -message SendRequest { - // The hook data - Hook hook = 1; -} - -// Response that contains data for sended hook -message SendResponse { - // The response of the hook. - google.protobuf.Any response = 1; -} - -// The hook service definition. -service HookService { - // Sends a hook - rpc Send (SendRequest) returns (SendResponse) {} -} diff --git a/cmd/tusd/cli/hooks/proto/v2/hook.proto b/cmd/tusd/cli/hooks/proto/v2/hook.proto new file mode 100644 index 0000000..d792ce9 --- /dev/null +++ b/cmd/tusd/cli/hooks/proto/v2/hook.proto @@ -0,0 +1,116 @@ +// If this file gets changed, you must recompile the generate package in pkg/proto. +// To do this, install the Go protobuf toolchain as mentioned in +// https://github.com/golang/protobuf#installation. +// Then use following command to recompile it with gRPC support: +// protoc --go_out=plugins=grpc:../../../../../pkg/proto/ v2/hook.proto +// In addition, it may be necessary to update the protobuf or gRPC dependencies as well. + +syntax = "proto3"; +package v2; + +// HookRequest contains the information about the hook type, the involved upload, +// and causing HTTP request. +message HookRequest { + // Type is the name of the hook. + string type = 1; + + // Event contains the involved upload and causing HTTP request. + Event event = 2; +} + +// Event represents an event from tusd which can be handled by the application. +message Event { + // Upload contains information about the upload that caused this hook + // to be fired. + FileInfo upload = 1; + + // HTTPRequest contains details about the HTTP request that reached + // tusd. + HTTPRequest httpRequest = 2; +} + +// FileInfo contains information about a single upload resource. +message FileInfo { + // ID is the unique identifier of the upload resource. + string id = 1; + // Total file size in bytes specified in the NewUpload call + int64 size = 2; + // Indicates whether the total file size is deferred until later + bool sizeIsDeferred = 3; + // Offset in bytes (zero-based) + int64 offset = 4; + map metaData = 5; + // Indicates that this is a partial upload which will later be used to form + // a final upload by concatenation. Partial uploads should not be processed + // when they are finished since they are only incomplete chunks of files. + bool isPartial = 6; + // Indicates that this is a final upload + bool isFinal = 7; + // If the upload is a final one (see IsFinal) this will be a non-empty + // ordered slice containing the ids of the uploads of which the final upload + // will consist after concatenation. + repeated string partialUploads = 8; + // Storage contains information about where the data storage saves the upload, + // for example a file path. The available values vary depending on what data + // store is used. This map may also be nil. + map storage = 9; +} + +// HTTPRequest contains basic details of an incoming HTTP request. +message HTTPRequest { + // Method is the HTTP method, e.g. POST or PATCH. + string method = 1; + // URI is the full HTTP request URI, e.g. /files/fooo. + string uri = 2; + // RemoteAddr contains the network address that sent the request. + string remoteAddr = 3; + // Header contains all HTTP headers as present in the HTTP request. + map header = 4; +} + +// HookResponse is the response after a hook is executed. +message HookResponse { +// HTTPResponse's fields can be filled to modify the HTTP response. + // This is only possible for pre-create, pre-finish and post-receive hooks. + // For other hooks this value is ignored. + // If multiple hooks modify the HTTP response, a later hook may overwrite the + // modified values from a previous hook (e.g. if multiple post-receive hooks + // are executed). + // Example usages: Send an error to the client if RejectUpload/StopUpload are + // set in the pre-create/post-receive hook. Send more information to the client + // in the pre-finish hook. + HTTPResponse httpResponse = 1; + + // RejectUpload will cause the upload to be rejected and not be created during + // POST request. This value is only respected for pre-create hooks. For other hooks, + // it is ignored. Use the HTTPResponse field to send details about the rejection + // to the client. + bool rejectUpload = 2; + + // StopUpload will cause the upload to be stopped during a PATCH request. + // This value is only respected for post-receive hooks. For other hooks, + // it is ignored. Use the HTTPResponse field to send details about the stop + // to the client. + bool stopUpload = 3; +} + +// HTTPResponse contains basic details of an outgoing HTTP response. +message HTTPResponse { + // StatusCode is status code, e.g. 200 or 400. + int64 statusCode = 1; + // Headers contains additional HTTP headers for the response. + map headers = 2; + // Body is the response body. + string body = 3; +} + + +// The hook service definition. +service HookHandler { + // InvokeHook is invoked for every hook that is executed. HookRequest contains the + // corresponding information about the hook type, the involved upload, and + // causing HTTP request. + // The return value HookResponse allows to stop or reject an upload, as well as modifying + // the HTTP response. See the documentation for HookResponse for more details. + rpc InvokeHook (HookRequest) returns (HookResponse) {} +} diff --git a/docs/hooks.md b/docs/hooks.md index 2bdd6c6..cce914d 100644 --- a/docs/hooks.md +++ b/docs/hooks.md @@ -1,5 +1,7 @@ # Hooks +TODO: Update with new details + When integrating tusd into an application, it is important to establish a communication channel between the two components. The tusd binary accomplishes this by providing a system which triggers actions when certain events happen, such as an upload being created or finished. This simple-but-powerful system enables uses ranging from logging over validation and authorization to processing the uploaded files. When a specific action happens during an upload (pre-create, post-receive, post-finish, or post-terminate), the hook system enables tusd to fire off a specific event. Tusd provides two ways of doing this: @@ -211,9 +213,9 @@ $ # Retrying 5 times with a 2 second backoff $ tusd --hooks-http http://localhost:8081/write --hooks-http-retry 5 --hooks-http-backoff 2 ``` -## GRPC Hooks +## gRPC Hooks -GRPC Hooks are the third type of hooks supported by tusd. Like the others hooks, it is disabled by default. To enable it, pass the `--hooks-grpc` option to the tusd binary. The flag's value will be a gRPC endpoint, which the tusd binary will be sent to: +gRPC Hooks are the third type of hooks supported by tusd. Like the others hooks, it is disabled by default. To enable it, pass the `--hooks-grpc` option to the tusd binary. The flag's value will be a gRPC endpoint, which the tusd binary will be sent to: ```bash $ tusd --hooks-grpc localhost:8080 diff --git a/docs/minio.txt b/docs/minio.txt new file mode 100644 index 0000000..cdb8b9c --- /dev/null +++ b/docs/minio.txt @@ -0,0 +1,3 @@ +MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY ./minio server data + +AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY go run cmd/tusd/main.go -s3-bucket tusdtest.transloadit.com -s3-endpoint http://127.0.0.1:9000 -expose-pprof diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..5469050 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,11 @@ +# Examples + +This directory contains following examples: + +- `apache2.conf` is the recommended minimum configuration for an Apache2 proxy in front of tusd. +- `nginx.conf` is the recommended minimum configuration for an Nginx proxy in front of tusd. +- `server/` is an example of how to the tusd package embedded in your own Go application. +- `hooks/file/` are Bash scripts for file hook implementations. +- `hooks/http/` is a Python HTTP server as the HTTP hook implementation. +- `hooks/grpc/` is a Python gRPC server as the gRPC hook implementation. +- `hooks/plugin/` is a Go plugin usable with the plugin hooks. diff --git a/examples/hooks/file/post-create b/examples/hooks/file/post-create new file mode 100755 index 0000000..3facfc8 --- /dev/null +++ b/examples/hooks/file/post-create @@ -0,0 +1,14 @@ +#!/bin/sh + +# This example demonstrates how to read the hook event details +# from stdout and output debug messages. + +id="$TUS_ID" +size="$TUS_SIZE" + +# We use >&2 to write debugging output to stderr. tusd +# will forward these to its stderr. Any output from the +# hook on stdout will be captured by tusd and interpreted +# as a response. +echo "Upload created with ID ${id} and size ${size}" >&2 +cat /dev/stdin | jq . >&2 diff --git a/examples/hooks/file/post-finish b/examples/hooks/file/post-finish new file mode 100755 index 0000000..fed82a1 --- /dev/null +++ b/examples/hooks/file/post-finish @@ -0,0 +1,11 @@ +#!/bin/sh + +# This example demonstrates how to read the hook event details +# from environment variables, stdin, and output debug messages. + +# We use >&2 to write debugging output to stderr. tusd +# will forward these to its stderr. Any output from the +# hook on stdout will be captured by tusd and interpreted +# as a response. +echo "Upload $TUS_ID ($TUS_SIZE bytes) finished" >&2 +cat /dev/stdin | jq . >&2 diff --git a/examples/hooks/file/post-receive b/examples/hooks/file/post-receive new file mode 100755 index 0000000..f96d571 --- /dev/null +++ b/examples/hooks/file/post-receive @@ -0,0 +1,15 @@ +#!/bin/sh + +# This example demonstrates how to read the hook event details +# from environment variables and output debug messages. + +id="$TUS_ID" +offset="$TUS_OFFSET" +size="$TUS_SIZE" +progress=$((100 * $offset/$size)) + +# We use >&2 to write debugging output to stderr. tusd +# will forward these to its stderr. Any output from the +# hook on stdout will be captured by tusd and interpreted +# as a response. +echo "Upload ${id} is at ${progress}% (${offset}/${size})" >&2 diff --git a/examples/hooks/file/post-terminate b/examples/hooks/file/post-terminate new file mode 100755 index 0000000..93b4a00 --- /dev/null +++ b/examples/hooks/file/post-terminate @@ -0,0 +1,11 @@ +#!/bin/sh + +# This example demonstrates how to read the hook event details +# from environment variables, stdin, and output debug messages. + +# We use >&2 to write debugging output to stderr. tusd +# will forward these to its stderr. Any output from the +# hook on stdout will be captured by tusd and interpreted +# as a response. +echo "Upload $TUS_ID terminated" >&2 +cat /dev/stdin | jq . >&2 diff --git a/examples/hooks/file/pre-create b/examples/hooks/file/pre-create new file mode 100755 index 0000000..593edf2 --- /dev/null +++ b/examples/hooks/file/pre-create @@ -0,0 +1,37 @@ +#!/bin/sh + +# This example demonstrates how to read the hook event details +# from stdout, output debug messages, and reject a new upload based +# on custom constraints. Here, an upload will be rejected if the +# filename metadata is missing. Remove the following `exit 0` line +# to activate the constraint: +exit 0 + +hasFilename="$(cat /dev/stdin | jq '.Event.Upload.MetaData | has("filename")')" + +# We use >&2 to write debugging output to stderr. tusd +# will forward these to its stderr. Any output from the +# hook on stdout will be captured by tusd and interpreted +# as a response. +echo "Filename exists: $hasFilename" >&2 + +if [ "$hasFilename" == "false" ]; then + + # If the condition is not met, output a JSON object on stdout, + # that instructs tusd to reject the upload and respond with a custom + # HTTP error response. + cat <&2 to write debugging output to stderr. tusd +# will forward these to its stderr. Any output from the +# hook on stdout will be captured by tusd and interpreted +# as a response. +cat /dev/stdin | jq . >&2 diff --git a/examples/hooks/grpc/Makefile b/examples/hooks/grpc/Makefile new file mode 100644 index 0000000..f0d6ea4 --- /dev/null +++ b/examples/hooks/grpc/Makefile @@ -0,0 +1,2 @@ +hook_pb2.py: + python -m grpc_tools.protoc --proto_path=../../../cmd/tusd/cli/hooks/proto/v2/ hook.proto --python_out=. --grpc_python_out=. diff --git a/examples/hooks/grpc/hook_pb2.py b/examples/hooks/grpc/hook_pb2.py new file mode 100644 index 0000000..b0366d8 --- /dev/null +++ b/examples/hooks/grpc/hook_pb2.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: hook.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\nhook.proto\x12\x02v2\"5\n\x0bHookRequest\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x18\n\x05\x65vent\x18\x02 \x01(\x0b\x32\t.v2.Event\"K\n\x05\x45vent\x12\x1c\n\x06upload\x18\x01 \x01(\x0b\x32\x0c.v2.FileInfo\x12$\n\x0bhttpRequest\x18\x02 \x01(\x0b\x32\x0f.v2.HTTPRequest\"\xc3\x02\n\x08\x46ileInfo\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\x03\x12\x16\n\x0esizeIsDeferred\x18\x03 \x01(\x08\x12\x0e\n\x06offset\x18\x04 \x01(\x03\x12,\n\x08metaData\x18\x05 \x03(\x0b\x32\x1a.v2.FileInfo.MetaDataEntry\x12\x11\n\tisPartial\x18\x06 \x01(\x08\x12\x0f\n\x07isFinal\x18\x07 \x01(\x08\x12\x16\n\x0epartialUploads\x18\x08 \x03(\t\x12*\n\x07storage\x18\t \x03(\x0b\x32\x19.v2.FileInfo.StorageEntry\x1a/\n\rMetaDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a.\n\x0cStorageEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x9a\x01\n\x0bHTTPRequest\x12\x0e\n\x06method\x18\x01 \x01(\t\x12\x0b\n\x03uri\x18\x02 \x01(\t\x12\x12\n\nremoteAddr\x18\x03 \x01(\t\x12+\n\x06header\x18\x04 \x03(\x0b\x32\x1b.v2.HTTPRequest.HeaderEntry\x1a-\n\x0bHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"`\n\x0cHookResponse\x12&\n\x0chttpResponse\x18\x01 \x01(\x0b\x32\x10.v2.HTTPResponse\x12\x14\n\x0crejectUpload\x18\x02 \x01(\x08\x12\x12\n\nstopUpload\x18\x03 \x01(\x08\"\x90\x01\n\x0cHTTPResponse\x12\x12\n\nstatusCode\x18\x01 \x01(\x03\x12.\n\x07headers\x18\x02 \x03(\x0b\x32\x1d.v2.HTTPResponse.HeadersEntry\x12\x0c\n\x04\x62ody\x18\x03 \x01(\t\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x32@\n\x0bHookHandler\x12\x31\n\nInvokeHook\x12\x0f.v2.HookRequest\x1a\x10.v2.HookResponse\"\x00\x62\x06proto3') + + + +_HOOKREQUEST = DESCRIPTOR.message_types_by_name['HookRequest'] +_EVENT = DESCRIPTOR.message_types_by_name['Event'] +_FILEINFO = DESCRIPTOR.message_types_by_name['FileInfo'] +_FILEINFO_METADATAENTRY = _FILEINFO.nested_types_by_name['MetaDataEntry'] +_FILEINFO_STORAGEENTRY = _FILEINFO.nested_types_by_name['StorageEntry'] +_HTTPREQUEST = DESCRIPTOR.message_types_by_name['HTTPRequest'] +_HTTPREQUEST_HEADERENTRY = _HTTPREQUEST.nested_types_by_name['HeaderEntry'] +_HOOKRESPONSE = DESCRIPTOR.message_types_by_name['HookResponse'] +_HTTPRESPONSE = DESCRIPTOR.message_types_by_name['HTTPResponse'] +_HTTPRESPONSE_HEADERSENTRY = _HTTPRESPONSE.nested_types_by_name['HeadersEntry'] +HookRequest = _reflection.GeneratedProtocolMessageType('HookRequest', (_message.Message,), { + 'DESCRIPTOR' : _HOOKREQUEST, + '__module__' : 'hook_pb2' + # @@protoc_insertion_point(class_scope:v2.HookRequest) + }) +_sym_db.RegisterMessage(HookRequest) + +Event = _reflection.GeneratedProtocolMessageType('Event', (_message.Message,), { + 'DESCRIPTOR' : _EVENT, + '__module__' : 'hook_pb2' + # @@protoc_insertion_point(class_scope:v2.Event) + }) +_sym_db.RegisterMessage(Event) + +FileInfo = _reflection.GeneratedProtocolMessageType('FileInfo', (_message.Message,), { + + 'MetaDataEntry' : _reflection.GeneratedProtocolMessageType('MetaDataEntry', (_message.Message,), { + 'DESCRIPTOR' : _FILEINFO_METADATAENTRY, + '__module__' : 'hook_pb2' + # @@protoc_insertion_point(class_scope:v2.FileInfo.MetaDataEntry) + }) + , + + 'StorageEntry' : _reflection.GeneratedProtocolMessageType('StorageEntry', (_message.Message,), { + 'DESCRIPTOR' : _FILEINFO_STORAGEENTRY, + '__module__' : 'hook_pb2' + # @@protoc_insertion_point(class_scope:v2.FileInfo.StorageEntry) + }) + , + 'DESCRIPTOR' : _FILEINFO, + '__module__' : 'hook_pb2' + # @@protoc_insertion_point(class_scope:v2.FileInfo) + }) +_sym_db.RegisterMessage(FileInfo) +_sym_db.RegisterMessage(FileInfo.MetaDataEntry) +_sym_db.RegisterMessage(FileInfo.StorageEntry) + +HTTPRequest = _reflection.GeneratedProtocolMessageType('HTTPRequest', (_message.Message,), { + + 'HeaderEntry' : _reflection.GeneratedProtocolMessageType('HeaderEntry', (_message.Message,), { + 'DESCRIPTOR' : _HTTPREQUEST_HEADERENTRY, + '__module__' : 'hook_pb2' + # @@protoc_insertion_point(class_scope:v2.HTTPRequest.HeaderEntry) + }) + , + 'DESCRIPTOR' : _HTTPREQUEST, + '__module__' : 'hook_pb2' + # @@protoc_insertion_point(class_scope:v2.HTTPRequest) + }) +_sym_db.RegisterMessage(HTTPRequest) +_sym_db.RegisterMessage(HTTPRequest.HeaderEntry) + +HookResponse = _reflection.GeneratedProtocolMessageType('HookResponse', (_message.Message,), { + 'DESCRIPTOR' : _HOOKRESPONSE, + '__module__' : 'hook_pb2' + # @@protoc_insertion_point(class_scope:v2.HookResponse) + }) +_sym_db.RegisterMessage(HookResponse) + +HTTPResponse = _reflection.GeneratedProtocolMessageType('HTTPResponse', (_message.Message,), { + + 'HeadersEntry' : _reflection.GeneratedProtocolMessageType('HeadersEntry', (_message.Message,), { + 'DESCRIPTOR' : _HTTPRESPONSE_HEADERSENTRY, + '__module__' : 'hook_pb2' + # @@protoc_insertion_point(class_scope:v2.HTTPResponse.HeadersEntry) + }) + , + 'DESCRIPTOR' : _HTTPRESPONSE, + '__module__' : 'hook_pb2' + # @@protoc_insertion_point(class_scope:v2.HTTPResponse) + }) +_sym_db.RegisterMessage(HTTPResponse) +_sym_db.RegisterMessage(HTTPResponse.HeadersEntry) + +_HOOKHANDLER = DESCRIPTOR.services_by_name['HookHandler'] +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _FILEINFO_METADATAENTRY._options = None + _FILEINFO_METADATAENTRY._serialized_options = b'8\001' + _FILEINFO_STORAGEENTRY._options = None + _FILEINFO_STORAGEENTRY._serialized_options = b'8\001' + _HTTPREQUEST_HEADERENTRY._options = None + _HTTPREQUEST_HEADERENTRY._serialized_options = b'8\001' + _HTTPRESPONSE_HEADERSENTRY._options = None + _HTTPRESPONSE_HEADERSENTRY._serialized_options = b'8\001' + _HOOKREQUEST._serialized_start=18 + _HOOKREQUEST._serialized_end=71 + _EVENT._serialized_start=73 + _EVENT._serialized_end=148 + _FILEINFO._serialized_start=151 + _FILEINFO._serialized_end=474 + _FILEINFO_METADATAENTRY._serialized_start=379 + _FILEINFO_METADATAENTRY._serialized_end=426 + _FILEINFO_STORAGEENTRY._serialized_start=428 + _FILEINFO_STORAGEENTRY._serialized_end=474 + _HTTPREQUEST._serialized_start=477 + _HTTPREQUEST._serialized_end=631 + _HTTPREQUEST_HEADERENTRY._serialized_start=586 + _HTTPREQUEST_HEADERENTRY._serialized_end=631 + _HOOKRESPONSE._serialized_start=633 + _HOOKRESPONSE._serialized_end=729 + _HTTPRESPONSE._serialized_start=732 + _HTTPRESPONSE._serialized_end=876 + _HTTPRESPONSE_HEADERSENTRY._serialized_start=830 + _HTTPRESPONSE_HEADERSENTRY._serialized_end=876 + _HOOKHANDLER._serialized_start=878 + _HOOKHANDLER._serialized_end=942 +# @@protoc_insertion_point(module_scope) diff --git a/examples/hooks/grpc/hook_pb2_grpc.py b/examples/hooks/grpc/hook_pb2_grpc.py new file mode 100644 index 0000000..3d0d49d --- /dev/null +++ b/examples/hooks/grpc/hook_pb2_grpc.py @@ -0,0 +1,70 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +import hook_pb2 as hook__pb2 + + +class HookHandlerStub(object): + """The hook service definition. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.InvokeHook = channel.unary_unary( + '/v2.HookHandler/InvokeHook', + request_serializer=hook__pb2.HookRequest.SerializeToString, + response_deserializer=hook__pb2.HookResponse.FromString, + ) + + +class HookHandlerServicer(object): + """The hook service definition. + """ + + def InvokeHook(self, request, context): + """Sends a hook + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_HookHandlerServicer_to_server(servicer, server): + rpc_method_handlers = { + 'InvokeHook': grpc.unary_unary_rpc_method_handler( + servicer.InvokeHook, + request_deserializer=hook__pb2.HookRequest.FromString, + response_serializer=hook__pb2.HookResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'v2.HookHandler', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class HookHandler(object): + """The hook service definition. + """ + + @staticmethod + def InvokeHook(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v2.HookHandler/InvokeHook', + hook__pb2.HookRequest.SerializeToString, + hook__pb2.HookResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/examples/hooks/grpc/server.py b/examples/hooks/grpc/server.py new file mode 100644 index 0000000..05fd0e9 --- /dev/null +++ b/examples/hooks/grpc/server.py @@ -0,0 +1,57 @@ +import grpc +from concurrent import futures +import time +import hook_pb2_grpc as pb2_grpc +import hook_pb2 as pb2 + +class HookHandler(pb2_grpc.HookHandlerServicer): + + def __init__(self, *args, **kwargs): + pass + + def InvokeHook(self, hook_request, context): + # Print data from hook request for debugging + print('Received hook request:') + print(hook_request) + + # Prepare hook response structure + hook_response = pb2.HookResponse() + + # Example: Use the pre-create hook to check if a filename has been supplied + # using metadata. If not, the upload is rejected with a custom HTTP response. + if hook_request.type == 'pre-create': + filename = hook_request.event.upload.metaData['filename'] + if filename == "": + hook_response.rejectUpload = True + hook_response.httpResponse.statusCode = 400 + hook_response.httpResponse.body = 'no filename provided' + hook_response.httpResponse.headers['X-Some-Header'] = 'yes' + + # Example: Use the post-finish hook to print information about a completed upload, + # including its storage location. + if hook_request.type == 'post-finish': + id = hook_request.event.upload.id + size = hook_request.event.upload.size + storage = hook_request.event.upload.storage + + print(f'Upload {id} ({size} bytes) is finished. Find the file at:') + print(storage) + + # Print data of hook response for debugging + print('Responding with hook response:') + print(hook_response) + print('------') + print('') + + # Return the hook response to send back to tusd + return hook_response + +def serve(): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + pb2_grpc.add_HookHandlerServicer_to_server(HookHandler(), server) + server.add_insecure_port('[::]:8000') + server.start() + server.wait_for_termination() + +if __name__ == '__main__': + serve() diff --git a/examples/hooks/http/server.py b/examples/hooks/http/server.py new file mode 100644 index 0000000..8fe7faf --- /dev/null +++ b/examples/hooks/http/server.py @@ -0,0 +1,65 @@ +from http.server import HTTPServer, BaseHTTPRequestHandler +from io import BytesIO + +import json + +class HTTPHookHandler(BaseHTTPRequestHandler): + + def do_GET(self): + self.send_response(200) + self.end_headers() + self.wfile.write(b'Hello! This server only responds to POST requests') + + def do_POST(self): + # Read entire body as JSON object + content_length = int(self.headers['Content-Length']) + request_body = self.rfile.read(content_length) + hook_request = json.loads(request_body) + + # Print data from hook request for debugging + print('Received hook request:') + print(hook_request) + + # Prepare hook response structure + hook_response = { + 'HTTPResponse': { + 'Headers': {} + } + } + + # Example: Use the pre-create hook to check if a filename has been supplied + # using metadata. If not, the upload is rejected with a custom HTTP response. + if hook_request['Type'] == 'pre-create': + metaData = hook_request['Event']['Upload']['MetaData'] + if 'filename' not in metaData: + hook_response['RejectUpload'] = True + hook_response['HTTPResponse']['StatusCode'] = 400 + hook_response['HTTPResponse']['Body'] = 'no filename provided' + hook_response['HTTPResponse']['Headers']['X-Some-Header'] = 'yes' + + # Example: Use the post-finish hook to print information about a completed upload, + # including its storage location. + if hook_request['Type'] == 'post-finish': + id = hook_request['Event']['Upload']['ID'] + size = hook_request['Event']['Upload']['Size'] + storage = hook_request['Event']['Upload']['Storage'] + + print(f'Upload {id} ({size} bytes) is finished. Find the file at:') + print(storage) + + + # Print data of hook response for debugging + print('Responding with hook response:') + print(hook_response) + print('------') + print('') + + # Send the data from the hook response as JSON output + response_body = json.dumps(hook_response) + self.send_response(200) + self.end_headers() + self.wfile.write(response_body.encode()) + + +httpd = HTTPServer(('localhost', 8000), HTTPHookHandler) +httpd.serve_forever() diff --git a/examples/hooks/plugin/Makefile b/examples/hooks/plugin/Makefile new file mode 100644 index 0000000..8f3adbc --- /dev/null +++ b/examples/hooks/plugin/Makefile @@ -0,0 +1,2 @@ +hook_handler: hook_handler.go + go build -o hook_handler ./hook_handler.go diff --git a/examples/hooks/plugin/hook_handler.go b/examples/hooks/plugin/hook_handler.go new file mode 100644 index 0000000..38a3a6b --- /dev/null +++ b/examples/hooks/plugin/hook_handler.go @@ -0,0 +1,87 @@ +package main + +import ( + "fmt" + "log" + + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-plugin" + "github.com/tus/tusd/cmd/tusd/cli/hooks" +) + +// Here is the implementation of our hook handler +type MyHookHandler struct { + logger hclog.Logger +} + +// Setup is called once the plugin has been loaded by tusd. +func (g *MyHookHandler) Setup() error { + // Use the log package or the g.logger field to write debug messages. + // Do not write to stdout directly, as this is used for communication between + // tusd and the plugin. + log.Println("MyHookHandler.Setup is invoked") + return nil +} + +// InvokeHook is called for every hook that tusd fires. +func (g *MyHookHandler) InvokeHook(req hooks.HookRequest) (res hooks.HookResponse, err error) { + log.Println("MyHookHandler.InvokeHook is invoked") + + // Prepare hook response structure + res.HTTPResponse.Headers = make(map[string]string) + + // Example: Use the pre-create hook to check if a filename has been supplied + // using metadata. If not, the upload is rejected with a custom HTTP response. + + if req.Type == hooks.HookPreCreate { + if _, ok := req.Event.Upload.MetaData["filename"]; !ok { + res.RejectUpload = true + res.HTTPResponse.StatusCode = 400 + res.HTTPResponse.Body = "no filename provided" + res.HTTPResponse.Headers["X-Some-Header"] = "yes" + } + } + + // Example: Use the post-finish hook to print information about a completed upload, + // including its storage location. + if req.Type == hooks.HookPreFinish { + id := req.Event.Upload.ID + size := req.Event.Upload.Size + storage := req.Event.Upload.Storage + + log.Printf("Upload %s (%d bytes) is finished. Find the file at:\n", id, size) + log.Println(storage) + + } + + // Return the hook response to tusd. + return res, nil +} + +// handshakeConfigs are used to just do a basic handshake between +// a plugin and tusd. If the handshake fails, a user friendly error is shown. +// This prevents users from executing bad plugins or executing a plugin +// directory. It is a UX feature, not a security feature. +var handshakeConfig = plugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "TUSD_PLUGIN", + MagicCookieValue: "yes", +} + +func main() { + // 1. Initialize our handler. + myHandler := &MyHookHandler{} + + // 2. Construct the plugin map. The key must be "hookHandler". + var pluginMap = map[string]plugin.Plugin{ + "hookHandler": &hooks.HookHandlerPlugin{Impl: myHandler}, + } + + // 3. Expose the plugin to tusd. + plugin.Serve(&plugin.ServeConfig{ + HandshakeConfig: handshakeConfig, + Plugins: pluginMap, + }) + + fmt.Println("DOONE") +} diff --git a/examples/hooks/post-create b/examples/hooks/post-create deleted file mode 100755 index b9d79a2..0000000 --- a/examples/hooks/post-create +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -id="$TUS_ID" -offset="$TUS_OFFSET" -size="$TUS_SIZE" - -echo "Upload created with ID ${id} and size ${size}" -cat /dev/stdin | jq . diff --git a/examples/hooks/post-finish b/examples/hooks/post-finish deleted file mode 100755 index b66e16b..0000000 --- a/examples/hooks/post-finish +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -echo "Upload $TUS_ID ($TUS_SIZE bytes) finished" -cat /dev/stdin | jq . diff --git a/examples/hooks/post-receive b/examples/hooks/post-receive deleted file mode 100755 index 06b8ea0..0000000 --- a/examples/hooks/post-receive +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -id="$TUS_ID" -offset="$TUS_OFFSET" -size="$TUS_SIZE" -progress=$((100 * $offset/$size)) - -echo "Upload ${id} is at ${progress}% (${offset}/${size})" diff --git a/examples/hooks/post-terminate b/examples/hooks/post-terminate deleted file mode 100755 index c4e0b13..0000000 --- a/examples/hooks/post-terminate +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -echo "Upload $TUS_ID terminated" -cat /dev/stdin | jq . diff --git a/examples/hooks/pre-create b/examples/hooks/pre-create deleted file mode 100755 index e3fdced..0000000 --- a/examples/hooks/pre-create +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -filename=$(cat /dev/stdin | jq .Upload.MetaData.filename) -if [ -z "$filename" ]; then - echo "Error: no filename provided" - exit 1 -fi diff --git a/go.mod b/go.mod index 74482ca..d3790e2 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.5.2 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 + github.com/hashicorp/go-plugin v1.4.3 // indirect github.com/prometheus/client_golang v1.12.1 github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index f882fa1..5ad1d9e 100644 --- a/go.sum +++ b/go.sum @@ -118,6 +118,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -219,10 +221,17 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= +github.com/hashicorp/go-hclog v0.14.1 h1:nQcJDQwIAGnmoUWp8ubocEX40cCml/17YkF6csQLReU= +github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-plugin v1.4.3 h1:DXmvivbWD5qdiBts9TpBC7BYL1Aia5sxbRgQB+v6UZM= +github.com/hashicorp/go-plugin v1.4.3/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -247,10 +256,17 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 h1:7GoSOOW2jpsfkntVKaS2rAr1TJqfcxotyaUcuxoZSzg= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -260,6 +276,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= +github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -368,6 +386,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -443,6 +462,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -452,6 +472,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -617,6 +638,7 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -677,6 +699,10 @@ google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEc google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211016002631-37fc39342514 h1:Rp1vYDPD4TdkMH5S/bZbopsGCsWhPcrLBUwOVhAQCxM= +google.golang.org/genproto v0.0.0-20211016002631-37fc39342514/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= diff --git a/pkg/handler/config.go b/pkg/handler/config.go index ae9676b..04accc7 100644 --- a/pkg/handler/config.go +++ b/pkg/handler/config.go @@ -41,14 +41,18 @@ type Config struct { // response to POST requests. RespectForwardedHeaders bool // PreUploadCreateCallback will be invoked before a new upload is created, if the - // property is supplied. If the callback returns nil, the upload will be created. - // Otherwise the HTTP request will be aborted. This can be used to implement - // validation of upload metadata etc. - PreUploadCreateCallback func(hook HookEvent) error + // property is supplied. If the callback returns no error, the upload will be created + // and optional values from HTTPResponse will be contained in the HTTP response. + // If the error is non-nil, the upload will not be created. This can be used to implement + // validation of upload metadata etc. Furthermore, HTTPResponse will be ignored and + // the error value can contain values for the HTTP response. + PreUploadCreateCallback func(hook HookEvent) (HTTPResponse, error) // PreFinishResponseCallback will be invoked after an upload is completed but before - // a response is returned to the client. Error responses from the callback will be passed - // back to the client. This can be used to implement post-processing validation. - PreFinishResponseCallback func(hook HookEvent) error + // a response is returned to the client. This can be used to implement post-processing validation. + // If the callback returns no error, optional values from HTTPResponse will be contained in the HTTP response. + // If the error is non-nil, the error will be forwarded to the client. Furthermore, + // HTTPResponse will be ignored and the error value can contain values for the HTTP response. + PreFinishResponseCallback func(hook HookEvent) (HTTPResponse, error) } func (config *Config) validate() error { diff --git a/pkg/handler/datastore.go b/pkg/handler/datastore.go index 1eb4240..539eb4e 100644 --- a/pkg/handler/datastore.go +++ b/pkg/handler/datastore.go @@ -7,7 +7,9 @@ import ( type MetaData map[string]string +// FileInfo contains information about a single upload resource. type FileInfo struct { + // ID is the unique identifier of the upload resource. ID string // Total file size in bytes specified in the NewUpload call Size int64 @@ -41,6 +43,7 @@ type FileInfo struct { // more data. Furthermore, a response is sent to notify the client of the // interrupting and the upload is terminated (if supported by the data store), // so the upload cannot be resumed anymore. +// TODO: Allow passing in a HTTP Response func (f FileInfo) StopUpload() { if f.stopUpload != nil { f.stopUpload() diff --git a/pkg/handler/error.go b/pkg/handler/error.go new file mode 100644 index 0000000..f4a518e --- /dev/null +++ b/pkg/handler/error.go @@ -0,0 +1,32 @@ +package handler + +// Error represents an error with the intent to be sent in the HTTP +// response to the client. Therefore, it also contains a HTTPResponse, +// next to an error code and error message. +type Error struct { + ErrorCode string + Message string + HTTPResponse HTTPResponse +} + +func (e Error) Error() string { + return e.ErrorCode + ": " + e.Message +} + +// NewError constructs a new Error object with the given error code and message. +// The corresponding HTTP response will have the provided status code +// and a body consisting of the error details. +// responses. See the net/http package for standardized status codes. +func NewError(errCode string, message string, statusCode int) Error { + return Error{ + ErrorCode: errCode, + Message: message, + HTTPResponse: HTTPResponse{ + StatusCode: statusCode, + Body: errCode + ": " + message + "\n", + Headers: HTTPHeaders{ + "Content-Type": "text/plain; charset=utf-8", + }, + }, + } +} diff --git a/pkg/handler/head_test.go b/pkg/handler/head_test.go index b5c83bb..353c372 100644 --- a/pkg/handler/head_test.go +++ b/pkg/handler/head_test.go @@ -76,10 +76,8 @@ func TestHead(t *testing.T) { ReqHeader: map[string]string{ "Tus-Resumable": "1.0.0", }, - Code: http.StatusNotFound, - ResHeader: map[string]string{ - "Content-Length": "0", - }, + Code: http.StatusNotFound, + ResHeader: map[string]string{}, }).Run(handler, t) if res.Body.String() != "" { diff --git a/pkg/handler/hooks.go b/pkg/handler/hooks.go new file mode 100644 index 0000000..08fc76d --- /dev/null +++ b/pkg/handler/hooks.go @@ -0,0 +1,25 @@ +package handler + +import "net/http" + +// HookEvent represents an event from tusd which can be handled by the application. +type HookEvent struct { + // Upload contains information about the upload that caused this hook + // to be fired. + Upload FileInfo + // HTTPRequest contains details about the HTTP request that reached + // tusd. + HTTPRequest HTTPRequest +} + +func newHookEvent(info FileInfo, r *http.Request) HookEvent { + return HookEvent{ + Upload: info, + HTTPRequest: HTTPRequest{ + Method: r.Method, + URI: r.RequestURI, + RemoteAddr: r.RemoteAddr, + Header: r.Header, + }, + } +} diff --git a/pkg/handler/http.go b/pkg/handler/http.go new file mode 100644 index 0000000..93bf529 --- /dev/null +++ b/pkg/handler/http.go @@ -0,0 +1,80 @@ +package handler + +import ( + "net/http" + "strconv" +) + +// HTTPRequest contains basic details of an incoming HTTP request. +type HTTPRequest struct { + // Method is the HTTP method, e.g. POST or PATCH. + Method string + // URI is the full HTTP request URI, e.g. /files/fooo. + URI string + // RemoteAddr contains the network address that sent the request. + RemoteAddr string + // Header contains all HTTP headers as present in the HTTP request. + Header http.Header +} + +type HTTPHeaders map[string]string + +// HTTPResponse contains basic details of an outgoing HTTP response. +type HTTPResponse struct { + // StatusCode is status code, e.g. 200 or 400. + StatusCode int + // Body is the response body. + Body string + // Headers contains additional HTTP headers for the response. + // TODO: Uniform naming with HTTPRequest.Header + Headers HTTPHeaders +} + +// writeTo writes the HTTP response into w, as specified by the fields in resp. +func (resp HTTPResponse) writeTo(w http.ResponseWriter) { + headers := w.Header() + for key, value := range resp.Headers { + headers.Set(key, value) + } + + if len(resp.Body) > 0 { + headers.Set("Content-Length", strconv.Itoa(len(resp.Body))) + } + + w.WriteHeader(resp.StatusCode) + + if len(resp.Body) > 0 { + w.Write([]byte(resp.Body)) + } +} + +// MergeWith returns a copy of resp1, where non-default values from resp2 overwrite +// values from resp1. +func (resp1 HTTPResponse) MergeWith(resp2 HTTPResponse) HTTPResponse { + // Clone the response 1 and use it as a basis + newResp := resp1 + + // Take the status code and body from response 2 to + // overwrite values from response 1. + if resp2.StatusCode != 0 { + newResp.StatusCode = resp2.StatusCode + } + + if len(resp2.Body) > 0 { + newResp.Body = resp2.Body + } + + // For the headers, me must make a new map to avoid writing + // into the header map from response 1. + newResp.Headers = make(HTTPHeaders, len(resp1.Headers)+len(resp2.Headers)) + + for key, value := range resp1.Headers { + newResp.Headers[key] = value + } + + for key, value := range resp2.Headers { + newResp.Headers[key] = value + } + + return newResp +} diff --git a/pkg/handler/metrics.go b/pkg/handler/metrics.go index 699086e..57a536e 100644 --- a/pkg/handler/metrics.go +++ b/pkg/handler/metrics.go @@ -31,7 +31,7 @@ func (m Metrics) incRequestsTotal(method string) { // TODO: Rework to only store error code // incErrorsTotal increases the counter for this error atomically by one. -func (m Metrics) incErrorsTotal(err HTTPError) { +func (m Metrics) incErrorsTotal(err Error) { ptr := m.ErrorsTotal.retrievePointerFor(err) atomic.AddUint64(ptr, 1) } @@ -95,10 +95,10 @@ func newErrorsTotalMap() *ErrorsTotalMap { // retrievePointerFor returns (after creating it if necessary) the pointer to // the counter for the error. -func (e *ErrorsTotalMap) retrievePointerFor(err HTTPError) *uint64 { +func (e *ErrorsTotalMap) retrievePointerFor(err Error) *uint64 { serr := ErrorsTotalMapEntry{ - ErrorCode: err.ErrorCode(), - StatusCode: err.StatusCode(), + ErrorCode: err.ErrorCode, + StatusCode: err.HTTPResponse.StatusCode, } e.lock.RLock() diff --git a/pkg/handler/unrouted_handler.go b/pkg/handler/unrouted_handler.go index 7cb37a0..0975a05 100644 --- a/pkg/handler/unrouted_handler.go +++ b/pkg/handler/unrouted_handler.go @@ -23,103 +23,31 @@ var ( reMimeType = regexp.MustCompile(`^[a-z]+\/[a-z0-9\-\+\.]+$`) ) -// HTTPError represents an error with an additional status code attached -// which may be used when this error is sent in a HTTP response. -// See the net/http package for standardized status codes. -type HTTPError interface { - error - ErrorCode() string - StatusCode() int - Body() []byte -} - -type httpError struct { - errorCode string - message string - statusCode int -} - -func (err httpError) Error() string { - return err.errorCode + ": " + err.message -} - -func (err httpError) StatusCode() int { - return err.statusCode -} - -func (err httpError) ErrorCode() string { - return err.errorCode -} - -func (err httpError) Body() []byte { - return []byte(err.Error()) -} - -// NewHTTPError adds the given status code to the provided error and returns -// the new error instance. The status code may be used in corresponding HTTP -// responses. See the net/http package for standardized status codes. -func NewHTTPError(errCode string, message string, statusCode int) HTTPError { - return httpError{errCode, message, statusCode} -} - var ( - ErrUnsupportedVersion = NewHTTPError("ERR_UNSUPPORTED_VERSION", "missing, invalid or unsupported Tus-Resumable header", http.StatusPreconditionFailed) - ErrMaxSizeExceeded = NewHTTPError("ERR_MAX_SIZE_EXCEEDED", "maximum size exceeded", http.StatusRequestEntityTooLarge) - ErrInvalidContentType = NewHTTPError("ERR_INVALID_CONTENT_TYPE", "missing or invalid Content-Type header", http.StatusBadRequest) - ErrInvalidUploadLength = NewHTTPError("ERR_INVALID_UPLOAD_LENGTH", "missing or invalid Upload-Length header", http.StatusBadRequest) - ErrInvalidOffset = NewHTTPError("ERR_INVALID_OFFSET", "missing or invalid Upload-Offset header", http.StatusBadRequest) - ErrNotFound = NewHTTPError("ERR_UPLOAD_NOT_FOUND", "upload not found", http.StatusNotFound) - ErrFileLocked = NewHTTPError("ERR_UPLOAD_LOCKED", "file currently locked", http.StatusLocked) - ErrMismatchOffset = NewHTTPError("ERR_MISMATCHED_OFFSET", "mismatched offset", http.StatusConflict) - ErrSizeExceeded = NewHTTPError("ERR_UPLOAD_SIZE_EXCEEDED", "upload's size exceeded", http.StatusRequestEntityTooLarge) - ErrNotImplemented = NewHTTPError("ERR_NOT_IMPLEMENTED", "feature not implemented", http.StatusNotImplemented) - ErrUploadNotFinished = NewHTTPError("ERR_UPLOAD_NOT_FINISHED", "one of the partial uploads is not finished", http.StatusBadRequest) - ErrInvalidConcat = NewHTTPError("ERR_INVALID_CONCAT", "invalid Upload-Concat header", http.StatusBadRequest) - ErrModifyFinal = NewHTTPError("ERR_MODIFY_FINAL", "modifying a final upload is not allowed", http.StatusForbidden) - ErrUploadLengthAndUploadDeferLength = NewHTTPError("ERR_AMBIGUOUS_UPLOAD_LENGTH", "provided both Upload-Length and Upload-Defer-Length", http.StatusBadRequest) - ErrInvalidUploadDeferLength = NewHTTPError("ERR_INVALID_UPLOAD_LENGTH_DEFER", "invalid Upload-Defer-Length header", http.StatusBadRequest) - ErrUploadStoppedByServer = NewHTTPError("ERR_UPLOAD_STOPPED", "upload has been stopped by server", http.StatusBadRequest) + ErrUnsupportedVersion = NewError("ERR_UNSUPPORTED_VERSION", "missing, invalid or unsupported Tus-Resumable header", http.StatusPreconditionFailed) + ErrMaxSizeExceeded = NewError("ERR_MAX_SIZE_EXCEEDED", "maximum size exceeded", http.StatusRequestEntityTooLarge) + ErrInvalidContentType = NewError("ERR_INVALID_CONTENT_TYPE", "missing or invalid Content-Type header", http.StatusBadRequest) + ErrInvalidUploadLength = NewError("ERR_INVALID_UPLOAD_LENGTH", "missing or invalid Upload-Length header", http.StatusBadRequest) + ErrInvalidOffset = NewError("ERR_INVALID_OFFSET", "missing or invalid Upload-Offset header", http.StatusBadRequest) + ErrNotFound = NewError("ERR_UPLOAD_NOT_FOUND", "upload not found", http.StatusNotFound) + ErrFileLocked = NewError("ERR_UPLOAD_LOCKED", "file currently locked", http.StatusLocked) + ErrMismatchOffset = NewError("ERR_MISMATCHED_OFFSET", "mismatched offset", http.StatusConflict) + ErrSizeExceeded = NewError("ERR_UPLOAD_SIZE_EXCEEDED", "upload's size exceeded", http.StatusRequestEntityTooLarge) + ErrNotImplemented = NewError("ERR_NOT_IMPLEMENTED", "feature not implemented", http.StatusNotImplemented) + ErrUploadNotFinished = NewError("ERR_UPLOAD_NOT_FINISHED", "one of the partial uploads is not finished", http.StatusBadRequest) + ErrInvalidConcat = NewError("ERR_INVALID_CONCAT", "invalid Upload-Concat header", http.StatusBadRequest) + ErrModifyFinal = NewError("ERR_MODIFY_FINAL", "modifying a final upload is not allowed", http.StatusForbidden) + ErrUploadLengthAndUploadDeferLength = NewError("ERR_AMBIGUOUS_UPLOAD_LENGTH", "provided both Upload-Length and Upload-Defer-Length", http.StatusBadRequest) + ErrInvalidUploadDeferLength = NewError("ERR_INVALID_UPLOAD_LENGTH_DEFER", "invalid Upload-Defer-Length header", http.StatusBadRequest) + ErrUploadStoppedByServer = NewError("ERR_UPLOAD_STOPPED", "upload has been stopped by server", http.StatusBadRequest) + ErrUploadRejectedByServer = NewError("ERR_UPLOAD_REJECTED", "upload creation has been rejected by server", http.StatusBadRequest) // TODO: These two responses are 500 for backwards compatability. We should discuss // whether it is better to more them to 4XX status codes. - ErrReadTimeout = NewHTTPError("ERR_READ_TIMEOUT", "timeout while reading request body", http.StatusInternalServerError) - ErrConnectionReset = NewHTTPError("ERR_CONNECTION_RESET", "TCP connection reset by peer", http.StatusInternalServerError) + ErrReadTimeout = NewError("ERR_READ_TIMEOUT", "timeout while reading request body", http.StatusInternalServerError) + ErrConnectionReset = NewError("ERR_CONNECTION_RESET", "TCP connection reset by peer", http.StatusInternalServerError) ) -// HTTPRequest contains basic details of an incoming HTTP request. -type HTTPRequest struct { - // Method is the HTTP method, e.g. POST or PATCH - Method string - // URI is the full HTTP request URI, e.g. /files/fooo - URI string - // RemoteAddr contains the network address that sent the request - RemoteAddr string - // Header contains all HTTP headers as present in the HTTP request. - Header http.Header -} - -// HookEvent represents an event from tusd which can be handled by the application. -type HookEvent struct { - // Upload contains information about the upload that caused this hook - // to be fired. - Upload FileInfo - // HTTPRequest contains details about the HTTP request that reached - // tusd. - HTTPRequest HTTPRequest -} - -func newHookEvent(info FileInfo, r *http.Request) HookEvent { - return HookEvent{ - Upload: info, - HTTPRequest: HTTPRequest{ - Method: r.Method, - URI: r.RequestURI, - RemoteAddr: r.RemoteAddr, - Header: r.Header, - }, - } -} - // UnroutedHandler exposes methods to handle requests as part of the tus protocol, // such as PostFile, HeadFile, PatchFile and DelFile. In addition the GetFile method // is provided which is, however, not part of the specification. @@ -266,7 +194,9 @@ func (handler *UnroutedHandler) Middleware(h http.Handler) http.Handler { // will be ignored or interpreted as a rejection. // For example, the Presto engine, which is used in older versions of // Opera, Opera Mobile and Opera Mini, handles CORS this way. - handler.sendResp(w, r, http.StatusOK) + handler.sendResp(w, r, HTTPResponse{ + StatusCode: http.StatusOK, + }) return } @@ -353,11 +283,18 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) PartialUploads: partialUploadIDs, } + resp := HTTPResponse{ + StatusCode: http.StatusCreated, + Headers: HTTPHeaders{}, + } + if handler.config.PreUploadCreateCallback != nil { - if err := handler.config.PreUploadCreateCallback(newHookEvent(info, r)); err != nil { + resp2, err := handler.config.PreUploadCreateCallback(newHookEvent(info, r)) + if err != nil { handler.sendError(w, r, err) return } + resp = resp.MergeWith(resp2) } upload, err := handler.composer.Core.NewUpload(ctx, info) @@ -377,7 +314,7 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) // Add the Location header directly after creating the new resource to even // include it in cases of failure when an error is returned url := handler.absFileURL(r, id) - w.Header().Set("Location", url) + resp.Headers["Location"] = url handler.Metrics.incUploadsCreated() handler.log("UploadCreated", "id", id, "size", i64toa(size), "url", url) @@ -410,7 +347,8 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) defer lock.Unlock() } - if err := handler.writeChunk(ctx, upload, info, w, r); err != nil { + resp, err = handler.writeChunk(ctx, upload, info, resp, r) + if err != nil { handler.sendError(w, r, err) return } @@ -418,13 +356,14 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) // Directly finish the upload if the upload is empty (i.e. has a size of 0). // This statement is in an else-if block to avoid causing duplicate calls // to finishUploadIfComplete if an upload is empty and contains a chunk. - if err := handler.finishUploadIfComplete(ctx, upload, info, r); err != nil { + resp, err = handler.finishUploadIfComplete(ctx, upload, info, resp, r) + if err != nil { handler.sendError(w, r, err) return } } - handler.sendResp(w, r, http.StatusCreated) + handler.sendResp(w, r, resp) } // HeadFile returns the length and offset for the HEAD request @@ -459,9 +398,14 @@ func (handler *UnroutedHandler) HeadFile(w http.ResponseWriter, r *http.Request) return } + resp := HTTPResponse{ + StatusCode: http.StatusOK, + Headers: make(HTTPHeaders), + } + // Add Upload-Concat header if possible if info.IsPartial { - w.Header().Set("Upload-Concat", "partial") + resp.Headers["Upload-Concat"] = "partial" } if info.IsFinal { @@ -472,23 +416,23 @@ func (handler *UnroutedHandler) HeadFile(w http.ResponseWriter, r *http.Request) // Remove trailing space v = v[:len(v)-1] - w.Header().Set("Upload-Concat", v) + resp.Headers["Upload-Concat"] = v } if len(info.MetaData) != 0 { - w.Header().Set("Upload-Metadata", SerializeMetadataHeader(info.MetaData)) + resp.Headers["Upload-Metadata"] = SerializeMetadataHeader(info.MetaData) } if info.SizeIsDeferred { - w.Header().Set("Upload-Defer-Length", UploadLengthDeferred) + resp.Headers["Upload-Defer-Length"] = UploadLengthDeferred } else { - w.Header().Set("Upload-Length", strconv.FormatInt(info.Size, 10)) - w.Header().Set("Content-Length", strconv.FormatInt(info.Size, 10)) + resp.Headers["Upload-Length"] = strconv.FormatInt(info.Size, 10) + resp.Headers["Content-Length"] = strconv.FormatInt(info.Size, 10) } - w.Header().Set("Cache-Control", "no-store") - w.Header().Set("Upload-Offset", strconv.FormatInt(info.Offset, 10)) - handler.sendResp(w, r, http.StatusOK) + resp.Headers["Cache-Control"] = "no-store" + resp.Headers["Upload-Offset"] = strconv.FormatInt(info.Offset, 10) + handler.sendResp(w, r, resp) } // PatchFile adds a chunk to an upload. This operation is only allowed @@ -548,10 +492,15 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request return } + resp := HTTPResponse{ + StatusCode: http.StatusNoContent, + Headers: make(HTTPHeaders, 1), // Initialize map, so writeChunk can set the Upload-Offset header. + } + // Do not proxy the call to the data store if the upload is already completed if !info.SizeIsDeferred && info.Offset == info.Size { - w.Header().Set("Upload-Offset", strconv.FormatInt(offset, 10)) - handler.sendResp(w, r, http.StatusNoContent) + resp.Headers["Upload-Offset"] = strconv.FormatInt(offset, 10) + handler.sendResp(w, r, resp) return } @@ -580,18 +529,19 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request info.SizeIsDeferred = false } - if err := handler.writeChunk(ctx, upload, info, w, r); err != nil { + resp, err = handler.writeChunk(ctx, upload, info, resp, r) + if err != nil { handler.sendError(w, r, err) return } - handler.sendResp(w, r, http.StatusNoContent) + handler.sendResp(w, r, resp) } // writeChunk reads the body from the requests r and appends it to the upload // with the corresponding id. Afterwards, it will set the necessary response // headers but will not send the response. -func (handler *UnroutedHandler) writeChunk(ctx context.Context, upload Upload, info FileInfo, w http.ResponseWriter, r *http.Request) error { +func (handler *UnroutedHandler) writeChunk(ctx context.Context, upload Upload, info FileInfo, resp HTTPResponse, r *http.Request) (HTTPResponse, error) { // Get Content-Length if possible length := r.ContentLength offset := info.Offset @@ -599,7 +549,7 @@ func (handler *UnroutedHandler) writeChunk(ctx context.Context, upload Upload, i // Test if this upload fits into the file's size if !info.SizeIsDeferred && offset+length > info.Size { - return ErrSizeExceeded + return resp, ErrSizeExceeded } maxSize := info.Size - offset @@ -679,27 +629,27 @@ func (handler *UnroutedHandler) writeChunk(ctx context.Context, upload Upload, i handler.log("ChunkWriteComplete", "id", id, "bytesWritten", i64toa(bytesWritten)) if err != nil { - return err + return resp, err } // Send new offset to client newOffset := offset + bytesWritten - w.Header().Set("Upload-Offset", strconv.FormatInt(newOffset, 10)) + resp.Headers["Upload-Offset"] = strconv.FormatInt(newOffset, 10) handler.Metrics.incBytesReceived(uint64(bytesWritten)) info.Offset = newOffset - return handler.finishUploadIfComplete(ctx, upload, info, r) + return handler.finishUploadIfComplete(ctx, upload, info, resp, r) } // finishUploadIfComplete checks whether an upload is completed (i.e. upload offset // matches upload size) and if so, it will call the data store's FinishUpload // function and send the necessary message on the CompleteUpload channel. -func (handler *UnroutedHandler) finishUploadIfComplete(ctx context.Context, upload Upload, info FileInfo, r *http.Request) error { +func (handler *UnroutedHandler) finishUploadIfComplete(ctx context.Context, upload Upload, info FileInfo, resp HTTPResponse, r *http.Request) (HTTPResponse, error) { // If the upload is completed, ... if !info.SizeIsDeferred && info.Offset == info.Size { // ... allow custom mechanism to finish and cleanup the upload if err := upload.FinishUpload(ctx); err != nil { - return err + return resp, err } // ... send the info out to the channel @@ -710,13 +660,15 @@ func (handler *UnroutedHandler) finishUploadIfComplete(ctx context.Context, uplo handler.Metrics.incUploadsFinished() if handler.config.PreFinishResponseCallback != nil { - if err := handler.config.PreFinishResponseCallback(newHookEvent(info, r)); err != nil { - return err + resp2, err := handler.config.PreFinishResponseCallback(newHookEvent(info, r)) + if err != nil { + return resp, err } + resp = resp.MergeWith(resp2) } } - return nil + return resp, nil } // GetFile handles requests to download a file using a GET request. This is not @@ -752,16 +704,21 @@ func (handler *UnroutedHandler) GetFile(w http.ResponseWriter, r *http.Request) return } - // Set headers before sending responses - w.Header().Set("Content-Length", strconv.FormatInt(info.Offset, 10)) - contentType, contentDisposition := filterContentType(info) - w.Header().Set("Content-Type", contentType) - w.Header().Set("Content-Disposition", contentDisposition) + resp := HTTPResponse{ + StatusCode: http.StatusOK, + Headers: HTTPHeaders{ + "Content-Length": strconv.FormatInt(info.Offset, 10), + "Content-Type": contentType, + "Content-Disposition": contentDisposition, + }, + Body: "", // Body is intentionally left empty, and we copy it manually in later. + } // If no data has been uploaded yet, respond with an empty "204 No Content" status. if info.Offset == 0 { - handler.sendResp(w, r, http.StatusNoContent) + resp.StatusCode = http.StatusNoContent + handler.sendResp(w, r, resp) return } @@ -771,7 +728,7 @@ func (handler *UnroutedHandler) GetFile(w http.ResponseWriter, r *http.Request) return } - handler.sendResp(w, r, http.StatusOK) + handler.sendResp(w, r, resp) io.Copy(w, src) // Try to close the reader if the io.Closer interface is implemented @@ -888,7 +845,9 @@ func (handler *UnroutedHandler) DelFile(w http.ResponseWriter, r *http.Request) return } - handler.sendResp(w, r, http.StatusNoContent) + handler.sendResp(w, r, HTTPResponse{ + StatusCode: http.StatusNoContent, + }) } // terminateUpload passes a given upload to the DataStore's Terminater, @@ -950,33 +909,27 @@ func (handler *UnroutedHandler) sendError(w http.ResponseWriter, r *http.Request // err = nil //} - statusErr, ok := err.(HTTPError) + detailedErr, ok := err.(Error) if !ok { handler.log("InternalServerError", "message", err.Error(), "method", r.Method, "path", r.URL.Path, "requestId", getRequestId(r)) - statusErr = NewHTTPError("ERR_INTERNAL_SERVER_ERROR", err.Error(), http.StatusInternalServerError) + detailedErr = NewError("ERR_INTERNAL_SERVER_ERROR", err.Error(), http.StatusInternalServerError) } - reason := append(statusErr.Body(), '\n') + // If we are sending the response for a HEAD request, ensure that we are not including + // any response body. if r.Method == "HEAD" { - reason = nil + detailedErr.HTTPResponse.Body = "" } - // TODO: Allow JSON response - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.Header().Set("Content-Length", strconv.Itoa(len(reason))) - w.WriteHeader(statusErr.StatusCode()) - w.Write(reason) - - handler.log("ResponseOutgoing", "status", strconv.Itoa(statusErr.StatusCode()), "method", r.Method, "path", r.URL.Path, "error", statusErr.ErrorCode(), "requestId", getRequestId(r)) - - handler.Metrics.incErrorsTotal(statusErr) + handler.sendResp(w, r, detailedErr.HTTPResponse) + handler.Metrics.incErrorsTotal(detailedErr) } // sendResp writes the header to w with the specified status code. -func (handler *UnroutedHandler) sendResp(w http.ResponseWriter, r *http.Request, status int) { - w.WriteHeader(status) +func (handler *UnroutedHandler) sendResp(w http.ResponseWriter, r *http.Request, resp HTTPResponse) { + resp.writeTo(w) - handler.log("ResponseOutgoing", "status", strconv.Itoa(status), "method", r.Method, "path", r.URL.Path, "requestId", getRequestId(r)) + handler.log("ResponseOutgoing", "status", strconv.Itoa(resp.StatusCode), "method", r.Method, "path", r.URL.Path, "requestId", getRequestId(r), "body", resp.Body) } // Make an absolute URLs to the given upload id. If the base path is absolute diff --git a/pkg/proto/v1/hook.pb.go b/pkg/proto/v1/hook.pb.go deleted file mode 100644 index 922b840..0000000 --- a/pkg/proto/v1/hook.pb.go +++ /dev/null @@ -1,475 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: v1/hook.proto - -package v1 - -import ( - context "context" - fmt "fmt" - proto "github.com/golang/protobuf/proto" - any "github.com/golang/protobuf/ptypes/any" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package - -// Uploaded data -type Upload struct { - // Unique integer identifier of the uploaded file - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - // Total file size in bytes specified in the NewUpload call - Size int64 `protobuf:"varint,2,opt,name=Size,proto3" json:"Size,omitempty"` - // Indicates whether the total file size is deferred until later - SizeIsDeferred bool `protobuf:"varint,3,opt,name=SizeIsDeferred,proto3" json:"SizeIsDeferred,omitempty"` - // Offset in bytes (zero-based) - Offset int64 `protobuf:"varint,4,opt,name=Offset,proto3" json:"Offset,omitempty"` - MetaData map[string]string `protobuf:"bytes,5,rep,name=metaData,proto3" json:"metaData,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - // Indicates that this is a partial upload which will later be used to form - // a final upload by concatenation. Partial uploads should not be processed - // when they are finished since they are only incomplete chunks of files. - IsPartial bool `protobuf:"varint,6,opt,name=isPartial,proto3" json:"isPartial,omitempty"` - // Indicates that this is a final upload - IsFinal bool `protobuf:"varint,7,opt,name=isFinal,proto3" json:"isFinal,omitempty"` - // If the upload is a final one (see IsFinal) this will be a non-empty - // ordered slice containing the ids of the uploads of which the final upload - // will consist after concatenation. - PartialUploads []string `protobuf:"bytes,8,rep,name=partialUploads,proto3" json:"partialUploads,omitempty"` - // Storage contains information about where the data storage saves the upload, - // for example a file path. The available values vary depending on what data - // store is used. This map may also be nil. - Storage map[string]string `protobuf:"bytes,9,rep,name=storage,proto3" json:"storage,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Upload) Reset() { *m = Upload{} } -func (m *Upload) String() string { return proto.CompactTextString(m) } -func (*Upload) ProtoMessage() {} -func (*Upload) Descriptor() ([]byte, []int) { - return fileDescriptor_581082325ef044c1, []int{0} -} - -func (m *Upload) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Upload.Unmarshal(m, b) -} -func (m *Upload) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Upload.Marshal(b, m, deterministic) -} -func (m *Upload) XXX_Merge(src proto.Message) { - xxx_messageInfo_Upload.Merge(m, src) -} -func (m *Upload) XXX_Size() int { - return xxx_messageInfo_Upload.Size(m) -} -func (m *Upload) XXX_DiscardUnknown() { - xxx_messageInfo_Upload.DiscardUnknown(m) -} - -var xxx_messageInfo_Upload proto.InternalMessageInfo - -func (m *Upload) GetId() string { - if m != nil { - return m.Id - } - return "" -} - -func (m *Upload) GetSize() int64 { - if m != nil { - return m.Size - } - return 0 -} - -func (m *Upload) GetSizeIsDeferred() bool { - if m != nil { - return m.SizeIsDeferred - } - return false -} - -func (m *Upload) GetOffset() int64 { - if m != nil { - return m.Offset - } - return 0 -} - -func (m *Upload) GetMetaData() map[string]string { - if m != nil { - return m.MetaData - } - return nil -} - -func (m *Upload) GetIsPartial() bool { - if m != nil { - return m.IsPartial - } - return false -} - -func (m *Upload) GetIsFinal() bool { - if m != nil { - return m.IsFinal - } - return false -} - -func (m *Upload) GetPartialUploads() []string { - if m != nil { - return m.PartialUploads - } - return nil -} - -func (m *Upload) GetStorage() map[string]string { - if m != nil { - return m.Storage - } - return nil -} - -type HTTPRequest struct { - // Method is the HTTP method, e.g. POST or PATCH - Method string `protobuf:"bytes,1,opt,name=method,proto3" json:"method,omitempty"` - // URI is the full HTTP request URI, e.g. /files/fooo - Uri string `protobuf:"bytes,2,opt,name=uri,proto3" json:"uri,omitempty"` - // RemoteAddr contains the network address that sent the request - RemoteAddr string `protobuf:"bytes,3,opt,name=remoteAddr,proto3" json:"remoteAddr,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *HTTPRequest) Reset() { *m = HTTPRequest{} } -func (m *HTTPRequest) String() string { return proto.CompactTextString(m) } -func (*HTTPRequest) ProtoMessage() {} -func (*HTTPRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_581082325ef044c1, []int{1} -} - -func (m *HTTPRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_HTTPRequest.Unmarshal(m, b) -} -func (m *HTTPRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_HTTPRequest.Marshal(b, m, deterministic) -} -func (m *HTTPRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_HTTPRequest.Merge(m, src) -} -func (m *HTTPRequest) XXX_Size() int { - return xxx_messageInfo_HTTPRequest.Size(m) -} -func (m *HTTPRequest) XXX_DiscardUnknown() { - xxx_messageInfo_HTTPRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_HTTPRequest proto.InternalMessageInfo - -func (m *HTTPRequest) GetMethod() string { - if m != nil { - return m.Method - } - return "" -} - -func (m *HTTPRequest) GetUri() string { - if m != nil { - return m.Uri - } - return "" -} - -func (m *HTTPRequest) GetRemoteAddr() string { - if m != nil { - return m.RemoteAddr - } - return "" -} - -// Hook's data -type Hook struct { - // Upload contains information about the upload that caused this hook - // to be fired. - Upload *Upload `protobuf:"bytes,1,opt,name=upload,proto3" json:"upload,omitempty"` - // HTTPRequest contains details about the HTTP request that reached - // tusd. - HttpRequest *HTTPRequest `protobuf:"bytes,2,opt,name=httpRequest,proto3" json:"httpRequest,omitempty"` - // The hook name - Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Hook) Reset() { *m = Hook{} } -func (m *Hook) String() string { return proto.CompactTextString(m) } -func (*Hook) ProtoMessage() {} -func (*Hook) Descriptor() ([]byte, []int) { - return fileDescriptor_581082325ef044c1, []int{2} -} - -func (m *Hook) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Hook.Unmarshal(m, b) -} -func (m *Hook) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Hook.Marshal(b, m, deterministic) -} -func (m *Hook) XXX_Merge(src proto.Message) { - xxx_messageInfo_Hook.Merge(m, src) -} -func (m *Hook) XXX_Size() int { - return xxx_messageInfo_Hook.Size(m) -} -func (m *Hook) XXX_DiscardUnknown() { - xxx_messageInfo_Hook.DiscardUnknown(m) -} - -var xxx_messageInfo_Hook proto.InternalMessageInfo - -func (m *Hook) GetUpload() *Upload { - if m != nil { - return m.Upload - } - return nil -} - -func (m *Hook) GetHttpRequest() *HTTPRequest { - if m != nil { - return m.HttpRequest - } - return nil -} - -func (m *Hook) GetName() string { - if m != nil { - return m.Name - } - return "" -} - -// Request data to send hook -type SendRequest struct { - // The hook data - Hook *Hook `protobuf:"bytes,1,opt,name=hook,proto3" json:"hook,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *SendRequest) Reset() { *m = SendRequest{} } -func (m *SendRequest) String() string { return proto.CompactTextString(m) } -func (*SendRequest) ProtoMessage() {} -func (*SendRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_581082325ef044c1, []int{3} -} - -func (m *SendRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SendRequest.Unmarshal(m, b) -} -func (m *SendRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SendRequest.Marshal(b, m, deterministic) -} -func (m *SendRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_SendRequest.Merge(m, src) -} -func (m *SendRequest) XXX_Size() int { - return xxx_messageInfo_SendRequest.Size(m) -} -func (m *SendRequest) XXX_DiscardUnknown() { - xxx_messageInfo_SendRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_SendRequest proto.InternalMessageInfo - -func (m *SendRequest) GetHook() *Hook { - if m != nil { - return m.Hook - } - return nil -} - -// Response that contains data for sended hook -type SendResponse struct { - // The response of the hook. - Response *any.Any `protobuf:"bytes,1,opt,name=response,proto3" json:"response,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *SendResponse) Reset() { *m = SendResponse{} } -func (m *SendResponse) String() string { return proto.CompactTextString(m) } -func (*SendResponse) ProtoMessage() {} -func (*SendResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_581082325ef044c1, []int{4} -} - -func (m *SendResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SendResponse.Unmarshal(m, b) -} -func (m *SendResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SendResponse.Marshal(b, m, deterministic) -} -func (m *SendResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_SendResponse.Merge(m, src) -} -func (m *SendResponse) XXX_Size() int { - return xxx_messageInfo_SendResponse.Size(m) -} -func (m *SendResponse) XXX_DiscardUnknown() { - xxx_messageInfo_SendResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_SendResponse proto.InternalMessageInfo - -func (m *SendResponse) GetResponse() *any.Any { - if m != nil { - return m.Response - } - return nil -} - -func init() { - proto.RegisterType((*Upload)(nil), "v1.Upload") - proto.RegisterMapType((map[string]string)(nil), "v1.Upload.MetaDataEntry") - proto.RegisterMapType((map[string]string)(nil), "v1.Upload.StorageEntry") - proto.RegisterType((*HTTPRequest)(nil), "v1.HTTPRequest") - proto.RegisterType((*Hook)(nil), "v1.Hook") - proto.RegisterType((*SendRequest)(nil), "v1.SendRequest") - proto.RegisterType((*SendResponse)(nil), "v1.SendResponse") -} - -func init() { - proto.RegisterFile("v1/hook.proto", fileDescriptor_581082325ef044c1) -} - -var fileDescriptor_581082325ef044c1 = []byte{ - // 477 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x52, 0x4d, 0x6f, 0xd3, 0x40, - 0x10, 0x25, 0xb1, 0xeb, 0xd8, 0xe3, 0xb6, 0x54, 0xab, 0x0a, 0x96, 0xa8, 0x42, 0x96, 0x0f, 0xc8, - 0x52, 0x25, 0x07, 0x07, 0x0e, 0x28, 0x5c, 0xa8, 0x54, 0x50, 0x39, 0x20, 0xaa, 0x4d, 0x11, 0xe7, - 0x2d, 0xde, 0x24, 0x56, 0x1c, 0xaf, 0xbb, 0x5e, 0x5b, 0x0a, 0x3f, 0x8a, 0xdf, 0x88, 0xf6, 0xc3, - 0x8d, 0xe9, 0x8d, 0x93, 0x67, 0xde, 0xbc, 0x79, 0xf3, 0x3c, 0x3b, 0x70, 0xd2, 0x65, 0xb3, 0x0d, - 0xe7, 0xdb, 0xb4, 0x16, 0x5c, 0x72, 0x34, 0xee, 0xb2, 0xe9, 0xab, 0x35, 0xe7, 0xeb, 0x92, 0xcd, - 0x34, 0x72, 0xdf, 0xae, 0x66, 0xb4, 0xda, 0x9b, 0x72, 0xfc, 0xc7, 0x01, 0xef, 0x47, 0x5d, 0x72, - 0x9a, 0xa3, 0x53, 0x18, 0x17, 0x39, 0x1e, 0x45, 0xa3, 0x24, 0x20, 0xe3, 0x22, 0x47, 0x08, 0xdc, - 0x65, 0xf1, 0x9b, 0xe1, 0x71, 0x34, 0x4a, 0x1c, 0xa2, 0x63, 0xf4, 0x06, 0x4e, 0xd5, 0xf7, 0x6b, - 0x73, 0xcd, 0x56, 0x4c, 0x08, 0x96, 0x63, 0x27, 0x1a, 0x25, 0x3e, 0x79, 0x82, 0xa2, 0x17, 0xe0, - 0x7d, 0x5f, 0xad, 0x1a, 0x26, 0xb1, 0xab, 0xbb, 0x6d, 0x86, 0xde, 0x83, 0xbf, 0x63, 0x92, 0x5e, - 0x53, 0x49, 0xf1, 0x51, 0xe4, 0x24, 0xe1, 0x1c, 0xa7, 0x5d, 0x96, 0x1a, 0x07, 0xe9, 0x37, 0x5b, - 0xfa, 0x5c, 0x49, 0xb1, 0x27, 0x8f, 0x4c, 0x74, 0x01, 0x41, 0xd1, 0xdc, 0x52, 0x21, 0x0b, 0x5a, - 0x62, 0x4f, 0x0f, 0x3c, 0x00, 0x08, 0xc3, 0xa4, 0x68, 0xbe, 0x14, 0x15, 0x2d, 0xf1, 0x44, 0xd7, - 0xfa, 0x54, 0xb9, 0xad, 0x0d, 0xc9, 0x0c, 0x68, 0xb0, 0x1f, 0x39, 0x49, 0x40, 0x9e, 0xa0, 0x28, - 0x83, 0x49, 0x23, 0xb9, 0xa0, 0x6b, 0x86, 0x03, 0x6d, 0xea, 0xe5, 0xc0, 0xd4, 0xd2, 0x54, 0x8c, - 0xa7, 0x9e, 0x37, 0xfd, 0x08, 0x27, 0xff, 0xb8, 0x45, 0x67, 0xe0, 0x6c, 0xd9, 0xde, 0xae, 0x4f, - 0x85, 0xe8, 0x1c, 0x8e, 0x3a, 0x5a, 0xb6, 0x66, 0x81, 0x01, 0x31, 0xc9, 0x62, 0xfc, 0x61, 0x34, - 0x5d, 0xc0, 0xf1, 0x50, 0xf5, 0x7f, 0x7a, 0xe3, 0x9f, 0x10, 0xde, 0xdc, 0xdd, 0xdd, 0x12, 0xf6, - 0xd0, 0xb2, 0x46, 0xaa, 0x45, 0xef, 0x98, 0xdc, 0xf0, 0xfe, 0xe1, 0x6c, 0xa6, 0x24, 0x5b, 0x51, - 0xd8, 0x76, 0x15, 0xa2, 0xd7, 0x00, 0x82, 0xed, 0xb8, 0x64, 0x57, 0x79, 0x2e, 0xf4, 0xb3, 0x05, - 0x64, 0x80, 0xc4, 0x0f, 0xe0, 0xde, 0x70, 0xbe, 0x45, 0x31, 0x78, 0xad, 0xfe, 0x73, 0xad, 0x18, - 0xce, 0xe1, 0xb0, 0x0b, 0x62, 0x2b, 0x28, 0x83, 0x70, 0x23, 0x65, 0x6d, 0x4d, 0xe8, 0x29, 0xe1, - 0xfc, 0xb9, 0x22, 0x0e, 0xbc, 0x91, 0x21, 0x47, 0x5d, 0x53, 0x45, 0x77, 0xcc, 0x0e, 0xd6, 0x71, - 0x7c, 0x09, 0xe1, 0x92, 0x55, 0x79, 0x4f, 0xb9, 0x00, 0x57, 0x1d, 0xae, 0x9d, 0xeb, 0x6b, 0x39, - 0xce, 0xb7, 0x44, 0xa3, 0xf1, 0x27, 0x38, 0x36, 0xe4, 0xa6, 0xe6, 0x55, 0xc3, 0xd0, 0x5b, 0xf0, - 0x85, 0x8d, 0x6d, 0xc7, 0x79, 0x6a, 0xee, 0x3c, 0xed, 0xef, 0x3c, 0xbd, 0xaa, 0xf6, 0xe4, 0x91, - 0x35, 0x5f, 0x40, 0xa8, 0xf4, 0x96, 0x4c, 0x74, 0xc5, 0x2f, 0x86, 0x2e, 0xc1, 0x55, 0x82, 0x48, - 0xfb, 0x1e, 0xf8, 0x98, 0x9e, 0x1d, 0x00, 0xd3, 0x19, 0x3f, 0xbb, 0xf7, 0xb4, 0xe6, 0xbb, 0xbf, - 0x01, 0x00, 0x00, 0xff, 0xff, 0x8f, 0xd4, 0x14, 0x0d, 0x5e, 0x03, 0x00, 0x00, -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConnInterface - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion6 - -// HookServiceClient is the client API for HookService service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type HookServiceClient interface { - // Sends a hook - Send(ctx context.Context, in *SendRequest, opts ...grpc.CallOption) (*SendResponse, error) -} - -type hookServiceClient struct { - cc grpc.ClientConnInterface -} - -func NewHookServiceClient(cc grpc.ClientConnInterface) HookServiceClient { - return &hookServiceClient{cc} -} - -func (c *hookServiceClient) Send(ctx context.Context, in *SendRequest, opts ...grpc.CallOption) (*SendResponse, error) { - out := new(SendResponse) - err := c.cc.Invoke(ctx, "/v1.HookService/Send", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// HookServiceServer is the server API for HookService service. -type HookServiceServer interface { - // Sends a hook - Send(context.Context, *SendRequest) (*SendResponse, error) -} - -// UnimplementedHookServiceServer can be embedded to have forward compatible implementations. -type UnimplementedHookServiceServer struct { -} - -func (*UnimplementedHookServiceServer) Send(ctx context.Context, req *SendRequest) (*SendResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Send not implemented") -} - -func RegisterHookServiceServer(s *grpc.Server, srv HookServiceServer) { - s.RegisterService(&_HookService_serviceDesc, srv) -} - -func _HookService_Send_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(SendRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(HookServiceServer).Send(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/v1.HookService/Send", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(HookServiceServer).Send(ctx, req.(*SendRequest)) - } - return interceptor(ctx, in, info, handler) -} - -var _HookService_serviceDesc = grpc.ServiceDesc{ - ServiceName: "v1.HookService", - HandlerType: (*HookServiceServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "Send", - Handler: _HookService_Send_Handler, - }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "v1/hook.proto", -} diff --git a/pkg/proto/v2/hook.pb.go b/pkg/proto/v2/hook.pb.go new file mode 100644 index 0000000..65814d9 --- /dev/null +++ b/pkg/proto/v2/hook.pb.go @@ -0,0 +1,555 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: v2/hook.proto + +package v2 + +import ( + context "context" + fmt "fmt" + proto "github.com/golang/protobuf/proto" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package + +// Hook's data +type HookRequest struct { + Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` + Event *Event `protobuf:"bytes,2,opt,name=event,proto3" json:"event,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *HookRequest) Reset() { *m = HookRequest{} } +func (m *HookRequest) String() string { return proto.CompactTextString(m) } +func (*HookRequest) ProtoMessage() {} +func (*HookRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_938ab51c60d4b622, []int{0} +} + +func (m *HookRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_HookRequest.Unmarshal(m, b) +} +func (m *HookRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_HookRequest.Marshal(b, m, deterministic) +} +func (m *HookRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_HookRequest.Merge(m, src) +} +func (m *HookRequest) XXX_Size() int { + return xxx_messageInfo_HookRequest.Size(m) +} +func (m *HookRequest) XXX_DiscardUnknown() { + xxx_messageInfo_HookRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_HookRequest proto.InternalMessageInfo + +func (m *HookRequest) GetType() string { + if m != nil { + return m.Type + } + return "" +} + +func (m *HookRequest) GetEvent() *Event { + if m != nil { + return m.Event + } + return nil +} + +type Event struct { + Upload *FileInfo `protobuf:"bytes,1,opt,name=upload,proto3" json:"upload,omitempty"` + HttpRequest *HTTPRequest `protobuf:"bytes,2,opt,name=httpRequest,proto3" json:"httpRequest,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Event) Reset() { *m = Event{} } +func (m *Event) String() string { return proto.CompactTextString(m) } +func (*Event) ProtoMessage() {} +func (*Event) Descriptor() ([]byte, []int) { + return fileDescriptor_938ab51c60d4b622, []int{1} +} + +func (m *Event) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Event.Unmarshal(m, b) +} +func (m *Event) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Event.Marshal(b, m, deterministic) +} +func (m *Event) XXX_Merge(src proto.Message) { + xxx_messageInfo_Event.Merge(m, src) +} +func (m *Event) XXX_Size() int { + return xxx_messageInfo_Event.Size(m) +} +func (m *Event) XXX_DiscardUnknown() { + xxx_messageInfo_Event.DiscardUnknown(m) +} + +var xxx_messageInfo_Event proto.InternalMessageInfo + +func (m *Event) GetUpload() *FileInfo { + if m != nil { + return m.Upload + } + return nil +} + +func (m *Event) GetHttpRequest() *HTTPRequest { + if m != nil { + return m.HttpRequest + } + return nil +} + +// TODO: Keep consistent naming capitalization +// Uploaded data +type FileInfo struct { + // Unique integer identifier of the uploaded file + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // Total file size in bytes specified in the NewUpload call + Size int64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` + // Indicates whether the total file size is deferred until later + SizeIsDeferred bool `protobuf:"varint,3,opt,name=sizeIsDeferred,proto3" json:"sizeIsDeferred,omitempty"` + // Offset in bytes (zero-based) + Offset int64 `protobuf:"varint,4,opt,name=offset,proto3" json:"offset,omitempty"` + MetaData map[string]string `protobuf:"bytes,5,rep,name=metaData,proto3" json:"metaData,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Indicates that this is a partial upload which will later be used to form + // a final upload by concatenation. Partial uploads should not be processed + // when they are finished since they are only incomplete chunks of files. + IsPartial bool `protobuf:"varint,6,opt,name=isPartial,proto3" json:"isPartial,omitempty"` + // Indicates that this is a final upload + IsFinal bool `protobuf:"varint,7,opt,name=isFinal,proto3" json:"isFinal,omitempty"` + // If the upload is a final one (see IsFinal) this will be a non-empty + // ordered slice containing the ids of the uploads of which the final upload + // will consist after concatenation. + PartialUploads []string `protobuf:"bytes,8,rep,name=partialUploads,proto3" json:"partialUploads,omitempty"` + // Storage contains information about where the data storage saves the upload, + // for example a file path. The available values vary depending on what data + // store is used. This map may also be nil. + Storage map[string]string `protobuf:"bytes,9,rep,name=storage,proto3" json:"storage,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *FileInfo) Reset() { *m = FileInfo{} } +func (m *FileInfo) String() string { return proto.CompactTextString(m) } +func (*FileInfo) ProtoMessage() {} +func (*FileInfo) Descriptor() ([]byte, []int) { + return fileDescriptor_938ab51c60d4b622, []int{2} +} + +func (m *FileInfo) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_FileInfo.Unmarshal(m, b) +} +func (m *FileInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_FileInfo.Marshal(b, m, deterministic) +} +func (m *FileInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_FileInfo.Merge(m, src) +} +func (m *FileInfo) XXX_Size() int { + return xxx_messageInfo_FileInfo.Size(m) +} +func (m *FileInfo) XXX_DiscardUnknown() { + xxx_messageInfo_FileInfo.DiscardUnknown(m) +} + +var xxx_messageInfo_FileInfo proto.InternalMessageInfo + +func (m *FileInfo) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *FileInfo) GetSize() int64 { + if m != nil { + return m.Size + } + return 0 +} + +func (m *FileInfo) GetSizeIsDeferred() bool { + if m != nil { + return m.SizeIsDeferred + } + return false +} + +func (m *FileInfo) GetOffset() int64 { + if m != nil { + return m.Offset + } + return 0 +} + +func (m *FileInfo) GetMetaData() map[string]string { + if m != nil { + return m.MetaData + } + return nil +} + +func (m *FileInfo) GetIsPartial() bool { + if m != nil { + return m.IsPartial + } + return false +} + +func (m *FileInfo) GetIsFinal() bool { + if m != nil { + return m.IsFinal + } + return false +} + +func (m *FileInfo) GetPartialUploads() []string { + if m != nil { + return m.PartialUploads + } + return nil +} + +func (m *FileInfo) GetStorage() map[string]string { + if m != nil { + return m.Storage + } + return nil +} + +type HTTPRequest struct { + // Method is the HTTP method, e.g. POST or PATCH + Method string `protobuf:"bytes,1,opt,name=method,proto3" json:"method,omitempty"` + // URI is the full HTTP request URI, e.g. /files/fooo + Uri string `protobuf:"bytes,2,opt,name=uri,proto3" json:"uri,omitempty"` + // RemoteAddr contains the network address that sent the request + RemoteAddr string `protobuf:"bytes,3,opt,name=remoteAddr,proto3" json:"remoteAddr,omitempty"` + Header map[string]string `protobuf:"bytes,4,rep,name=header,proto3" json:"header,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *HTTPRequest) Reset() { *m = HTTPRequest{} } +func (m *HTTPRequest) String() string { return proto.CompactTextString(m) } +func (*HTTPRequest) ProtoMessage() {} +func (*HTTPRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_938ab51c60d4b622, []int{3} +} + +func (m *HTTPRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_HTTPRequest.Unmarshal(m, b) +} +func (m *HTTPRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_HTTPRequest.Marshal(b, m, deterministic) +} +func (m *HTTPRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_HTTPRequest.Merge(m, src) +} +func (m *HTTPRequest) XXX_Size() int { + return xxx_messageInfo_HTTPRequest.Size(m) +} +func (m *HTTPRequest) XXX_DiscardUnknown() { + xxx_messageInfo_HTTPRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_HTTPRequest proto.InternalMessageInfo + +func (m *HTTPRequest) GetMethod() string { + if m != nil { + return m.Method + } + return "" +} + +func (m *HTTPRequest) GetUri() string { + if m != nil { + return m.Uri + } + return "" +} + +func (m *HTTPRequest) GetRemoteAddr() string { + if m != nil { + return m.RemoteAddr + } + return "" +} + +func (m *HTTPRequest) GetHeader() map[string]string { + if m != nil { + return m.Header + } + return nil +} + +type HookResponse struct { + HttpResponse *HTTPResponse `protobuf:"bytes,1,opt,name=httpResponse,proto3" json:"httpResponse,omitempty"` + RejectUpload bool `protobuf:"varint,2,opt,name=rejectUpload,proto3" json:"rejectUpload,omitempty"` + StopUpload bool `protobuf:"varint,3,opt,name=stopUpload,proto3" json:"stopUpload,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *HookResponse) Reset() { *m = HookResponse{} } +func (m *HookResponse) String() string { return proto.CompactTextString(m) } +func (*HookResponse) ProtoMessage() {} +func (*HookResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_938ab51c60d4b622, []int{4} +} + +func (m *HookResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_HookResponse.Unmarshal(m, b) +} +func (m *HookResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_HookResponse.Marshal(b, m, deterministic) +} +func (m *HookResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_HookResponse.Merge(m, src) +} +func (m *HookResponse) XXX_Size() int { + return xxx_messageInfo_HookResponse.Size(m) +} +func (m *HookResponse) XXX_DiscardUnknown() { + xxx_messageInfo_HookResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_HookResponse proto.InternalMessageInfo + +func (m *HookResponse) GetHttpResponse() *HTTPResponse { + if m != nil { + return m.HttpResponse + } + return nil +} + +func (m *HookResponse) GetRejectUpload() bool { + if m != nil { + return m.RejectUpload + } + return false +} + +func (m *HookResponse) GetStopUpload() bool { + if m != nil { + return m.StopUpload + } + return false +} + +type HTTPResponse struct { + StatusCode int64 `protobuf:"varint,1,opt,name=statusCode,proto3" json:"statusCode,omitempty"` + Headers map[string]string `protobuf:"bytes,2,rep,name=headers,proto3" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Body string `protobuf:"bytes,3,opt,name=body,proto3" json:"body,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *HTTPResponse) Reset() { *m = HTTPResponse{} } +func (m *HTTPResponse) String() string { return proto.CompactTextString(m) } +func (*HTTPResponse) ProtoMessage() {} +func (*HTTPResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_938ab51c60d4b622, []int{5} +} + +func (m *HTTPResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_HTTPResponse.Unmarshal(m, b) +} +func (m *HTTPResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_HTTPResponse.Marshal(b, m, deterministic) +} +func (m *HTTPResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_HTTPResponse.Merge(m, src) +} +func (m *HTTPResponse) XXX_Size() int { + return xxx_messageInfo_HTTPResponse.Size(m) +} +func (m *HTTPResponse) XXX_DiscardUnknown() { + xxx_messageInfo_HTTPResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_HTTPResponse proto.InternalMessageInfo + +func (m *HTTPResponse) GetStatusCode() int64 { + if m != nil { + return m.StatusCode + } + return 0 +} + +func (m *HTTPResponse) GetHeaders() map[string]string { + if m != nil { + return m.Headers + } + return nil +} + +func (m *HTTPResponse) GetBody() string { + if m != nil { + return m.Body + } + return "" +} + +func init() { + proto.RegisterType((*HookRequest)(nil), "v2.HookRequest") + proto.RegisterType((*Event)(nil), "v2.Event") + proto.RegisterType((*FileInfo)(nil), "v2.FileInfo") + proto.RegisterMapType((map[string]string)(nil), "v2.FileInfo.MetaDataEntry") + proto.RegisterMapType((map[string]string)(nil), "v2.FileInfo.StorageEntry") + proto.RegisterType((*HTTPRequest)(nil), "v2.HTTPRequest") + proto.RegisterMapType((map[string]string)(nil), "v2.HTTPRequest.HeaderEntry") + proto.RegisterType((*HookResponse)(nil), "v2.HookResponse") + proto.RegisterType((*HTTPResponse)(nil), "v2.HTTPResponse") + proto.RegisterMapType((map[string]string)(nil), "v2.HTTPResponse.HeadersEntry") +} + +func init() { + proto.RegisterFile("v2/hook.proto", fileDescriptor_938ab51c60d4b622) +} + +var fileDescriptor_938ab51c60d4b622 = []byte{ + // 578 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0xdb, 0x6e, 0xd3, 0x40, + 0x10, 0xc5, 0x71, 0x73, 0xf1, 0x38, 0x2d, 0xd5, 0x0a, 0xa1, 0x25, 0xdc, 0x22, 0x0b, 0xa1, 0x3c, + 0x05, 0xd5, 0x45, 0x5c, 0xca, 0x0b, 0x97, 0xb6, 0x4a, 0x1f, 0x90, 0xaa, 0xa5, 0xbc, 0xb3, 0xc5, + 0x13, 0x62, 0xe2, 0x7a, 0xcd, 0xee, 0xc6, 0x52, 0xf8, 0x02, 0x3e, 0x08, 0x89, 0x4f, 0xe0, 0xb7, + 0xd0, 0x5e, 0x42, 0x9c, 0xbc, 0xe5, 0xc9, 0x3b, 0x67, 0xce, 0xcc, 0x9e, 0x3d, 0x3b, 0x5e, 0xd8, + 0xaf, 0xd3, 0x67, 0x33, 0x21, 0xe6, 0xe3, 0x4a, 0x0a, 0x2d, 0x48, 0xab, 0x4e, 0x93, 0xf7, 0x10, + 0x4f, 0x84, 0x98, 0x33, 0xfc, 0xb1, 0x40, 0xa5, 0x09, 0x81, 0x3d, 0xbd, 0xac, 0x90, 0x06, 0xc3, + 0x60, 0x14, 0x31, 0xbb, 0x26, 0x8f, 0xa1, 0x8d, 0x35, 0x96, 0x9a, 0xb6, 0x86, 0xc1, 0x28, 0x4e, + 0xa3, 0x71, 0x9d, 0x8e, 0xcf, 0x0c, 0xc0, 0x1c, 0x9e, 0x7c, 0x81, 0xb6, 0x8d, 0xc9, 0x13, 0xe8, + 0x2c, 0xaa, 0x42, 0xf0, 0xcc, 0xd6, 0xc7, 0x69, 0xdf, 0x50, 0xcf, 0xf3, 0x02, 0x2f, 0xca, 0xa9, + 0x60, 0x3e, 0x47, 0x8e, 0x20, 0x9e, 0x69, 0x5d, 0xf9, 0x2d, 0x7d, 0xd7, 0xdb, 0x86, 0x3a, 0xb9, + 0xba, 0xba, 0xf4, 0x30, 0x6b, 0x72, 0x92, 0xdf, 0x21, 0xf4, 0x56, 0x7d, 0xc8, 0x01, 0xb4, 0xf2, + 0xcc, 0x2b, 0x6c, 0xe5, 0x99, 0xd1, 0xac, 0xf2, 0x9f, 0x68, 0x1b, 0x85, 0xcc, 0xae, 0xc9, 0x53, + 0x38, 0x30, 0xdf, 0x0b, 0x75, 0x8a, 0x53, 0x94, 0x12, 0x33, 0x1a, 0x0e, 0x83, 0x51, 0x8f, 0x6d, + 0xa1, 0xe4, 0x2e, 0x74, 0xc4, 0x74, 0xaa, 0x50, 0xd3, 0x3d, 0x5b, 0xed, 0x23, 0xf2, 0x02, 0x7a, + 0x37, 0xa8, 0xf9, 0x29, 0xd7, 0x9c, 0xb6, 0x87, 0xe1, 0x28, 0x4e, 0x07, 0xcd, 0xb3, 0x8c, 0x3f, + 0xfa, 0xe4, 0x59, 0xa9, 0xe5, 0x92, 0xfd, 0xe7, 0x92, 0x07, 0x10, 0xe5, 0xea, 0x92, 0x4b, 0x9d, + 0xf3, 0x82, 0x76, 0xec, 0x96, 0x6b, 0x80, 0x50, 0xe8, 0xe6, 0xea, 0x3c, 0x2f, 0x79, 0x41, 0xbb, + 0x36, 0xb7, 0x0a, 0x8d, 0xde, 0xca, 0x91, 0x3e, 0x5b, 0x93, 0x14, 0xed, 0x0d, 0xc3, 0x51, 0xc4, + 0xb6, 0x50, 0x72, 0x0c, 0x5d, 0xa5, 0x85, 0xe4, 0xdf, 0x90, 0x46, 0x56, 0xd6, 0xbd, 0x0d, 0x59, + 0x9f, 0x5c, 0xce, 0xa9, 0x5a, 0x31, 0x07, 0x6f, 0x60, 0x7f, 0x43, 0x2f, 0x39, 0x84, 0x70, 0x8e, + 0x4b, 0x6f, 0xa1, 0x59, 0x92, 0x3b, 0xd0, 0xae, 0x79, 0xb1, 0x70, 0x26, 0x46, 0xcc, 0x05, 0x27, + 0xad, 0x57, 0xc1, 0xe0, 0x04, 0xfa, 0xcd, 0xae, 0xbb, 0xd4, 0x26, 0x7f, 0x03, 0x88, 0x1b, 0x77, + 0x6a, 0xdc, 0xbe, 0x41, 0x3d, 0x13, 0xab, 0xdb, 0xf3, 0x91, 0xe9, 0xb9, 0x90, 0xb9, 0xaf, 0x37, + 0x4b, 0xf2, 0x08, 0x40, 0xe2, 0x8d, 0xd0, 0xf8, 0x2e, 0xcb, 0xa4, 0xbd, 0xbb, 0x88, 0x35, 0x10, + 0x72, 0x0c, 0x9d, 0x19, 0xf2, 0x0c, 0x25, 0xdd, 0xb3, 0x36, 0xdc, 0xdf, 0x1a, 0x9f, 0xf1, 0xc4, + 0x66, 0x9d, 0x11, 0x9e, 0x3a, 0x78, 0x0d, 0x71, 0x03, 0xde, 0xe9, 0x24, 0xbf, 0x02, 0xe8, 0xbb, + 0xff, 0x44, 0x55, 0xa2, 0x54, 0x48, 0x9e, 0x43, 0xdf, 0x0d, 0xa8, 0x8b, 0xfd, 0xc0, 0x1f, 0xae, + 0x65, 0x38, 0x9c, 0x6d, 0xb0, 0x48, 0x02, 0x7d, 0x89, 0xdf, 0xf1, 0xab, 0x76, 0xf7, 0x69, 0xf7, + 0xe9, 0xb1, 0x0d, 0xcc, 0x1c, 0x5d, 0x69, 0x51, 0x79, 0x86, 0x1b, 0xdb, 0x06, 0x92, 0xfc, 0x31, + 0x52, 0x1a, 0x5b, 0xb8, 0x02, 0xae, 0x17, 0xea, 0x83, 0xc8, 0x9c, 0x90, 0x90, 0x35, 0x10, 0xf2, + 0x12, 0xba, 0xce, 0x00, 0x45, 0x5b, 0xd6, 0xac, 0x87, 0xdb, 0x2a, 0xbd, 0x5b, 0xca, 0xcf, 0x8d, + 0x67, 0x9b, 0x1f, 0xeb, 0x5a, 0x64, 0x4b, 0x6f, 0xbf, 0x5d, 0x9b, 0x71, 0x68, 0x92, 0x77, 0x31, + 0x31, 0x7d, 0xeb, 0xde, 0x9a, 0x09, 0x2f, 0xb3, 0x02, 0x25, 0x39, 0x02, 0xb8, 0x28, 0x6b, 0x31, + 0x47, 0x03, 0x12, 0xf7, 0x00, 0xac, 0x9f, 0xa2, 0xc1, 0xe1, 0x1a, 0x70, 0x2a, 0x93, 0x5b, 0xd7, + 0x1d, 0xfb, 0x70, 0x1d, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0xdc, 0x73, 0x61, 0x1c, 0xc9, 0x04, + 0x00, 0x00, +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConnInterface + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion6 + +// HookHandlerClient is the client API for HookHandler service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type HookHandlerClient interface { + // Sends a hook + InvokeHook(ctx context.Context, in *HookRequest, opts ...grpc.CallOption) (*HookResponse, error) +} + +type hookHandlerClient struct { + cc grpc.ClientConnInterface +} + +func NewHookHandlerClient(cc grpc.ClientConnInterface) HookHandlerClient { + return &hookHandlerClient{cc} +} + +func (c *hookHandlerClient) InvokeHook(ctx context.Context, in *HookRequest, opts ...grpc.CallOption) (*HookResponse, error) { + out := new(HookResponse) + err := c.cc.Invoke(ctx, "/v2.HookHandler/InvokeHook", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// HookHandlerServer is the server API for HookHandler service. +type HookHandlerServer interface { + // Sends a hook + InvokeHook(context.Context, *HookRequest) (*HookResponse, error) +} + +// UnimplementedHookHandlerServer can be embedded to have forward compatible implementations. +type UnimplementedHookHandlerServer struct { +} + +func (*UnimplementedHookHandlerServer) InvokeHook(ctx context.Context, req *HookRequest) (*HookResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method InvokeHook not implemented") +} + +func RegisterHookHandlerServer(s *grpc.Server, srv HookHandlerServer) { + s.RegisterService(&_HookHandler_serviceDesc, srv) +} + +func _HookHandler_InvokeHook_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(HookRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(HookHandlerServer).InvokeHook(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/v2.HookHandler/InvokeHook", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(HookHandlerServer).InvokeHook(ctx, req.(*HookRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _HookHandler_serviceDesc = grpc.ServiceDesc{ + ServiceName: "v2.HookHandler", + HandlerType: (*HookHandlerServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "InvokeHook", + Handler: _HookHandler_InvokeHook_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "v2/hook.proto", +} diff --git a/pkg/s3store/s3store.go b/pkg/s3store/s3store.go index 8d32d8e..e7f3d00 100644 --- a/pkg/s3store/s3store.go +++ b/pkg/s3store/s3store.go @@ -717,7 +717,7 @@ func (upload s3Upload) GetReader(ctx context.Context) (io.Reader, error) { }) if err == nil { // The multipart upload still exists, which means we cannot download it yet - return nil, handler.NewHTTPError("ERR_INCOMPLETE_UPLOAD", "cannot stream non-finished upload", http.StatusBadRequest) + return nil, handler.NewError("ERR_INCOMPLETE_UPLOAD", "cannot stream non-finished upload", http.StatusBadRequest) } if isAwsError(err, "NoSuchUpload") { diff --git a/pkg/s3store/s3store_test.go b/pkg/s3store/s3store_test.go index 0a37b95..7850248 100644 --- a/pkg/s3store/s3store_test.go +++ b/pkg/s3store/s3store_test.go @@ -576,7 +576,7 @@ func TestGetReaderNotFinished(t *testing.T) { content, err := upload.GetReader(context.Background()) assert.Nil(content) - assert.Equal("cannot stream non-finished upload", err.Error()) + assert.Equal("ERR_INCOMPLETE_UPLOAD: cannot stream non-finished upload", err.Error()) } func TestDeclareLength(t *testing.T) {