cli: Add pre-finish hook (#382)

* core: add new synchronous event: pre-finish

* docs: pre-finish hook event

* docs: Added information about CORS to the FAQ. (#384)

* Added information about CORS to the FAQ.

* docs: Expand explanation of CORS

Co-authored-by: Marius <marius@transloadit.com>

* cli: add header forwarding in HTTP hooks (#371)

* cli: add header forwarding in HTTP hooks

* docs: Reword header forwarding flag, add documentation

* Don't enable pre-finish hooks by default

* Enable pre-finish hooks for plugins

* docs: default enabled hooks

* Rename callback

Co-authored-by: josh-marshall-jax <52457971+josh-marshall-jax@users.noreply.github.com>
Co-authored-by: Marius <marius@transloadit.com>

cmd/tusd/cli/flags.go

@@ -53,7 +53,7 @@ func ParseFlags() {
 	flag.StringVar(&Flags.S3Endpoint, "s3-endpoint", "", "Endpoint to use S3 compatible implementations like minio (requires s3-bucket to be pass)")
 	flag.StringVar(&Flags.GCSBucket, "gcs-bucket", "", "Use Google Cloud Storage with this bucket as storage backend (requires the GCS_SERVICE_ACCOUNT_FILE environment variable to be set)")
 	flag.StringVar(&Flags.GCSObjectPrefix, "gcs-object-prefix", "", "Prefix for GCS object names (can't contain underscore character)")
-	flag.StringVar(&Flags.EnabledHooksString, "hooks-enabled-events", "", "Comma separated list of enabled hook events (e.g. post-create,post-finish). Leave empty to enable all events")
+	flag.StringVar(&Flags.EnabledHooksString, "hooks-enabled-events", "pre-create,post-create,post-receive,post-terminate,post-finish", "Comma separated list of enabled hook events (e.g. post-create,post-finish). Leave empty to enable all events")
 	flag.StringVar(&Flags.FileHooksDir, "hooks-dir", "", "Directory to search for available hooks scripts")
 	flag.StringVar(&Flags.HttpHooksEndpoint, "hooks-http", "", "An HTTP endpoint to which hook events will be sent to")
 	flag.StringVar(&Flags.HttpHooksForwardHeaders, "hooks-http-forward-headers", "", "List of HTTP request headers to be forwarded from the client request to the hook endpoint")

cmd/tusd/cli/hooks.go

@@ -20,27 +20,36 @@ func hookTypeInSlice(a hooks.HookType, list []hooks.HookType) bool {
 	return false
 }
 
-func preCreateCallback(info handler.HookEvent) error {
+func hookCallback(typ hooks.HookType, info handler.HookEvent) error {
-	if output, err := invokeHookSync(hooks.HookPreCreate, info, true); err != nil {
+	if output, err := invokeHookSync(typ, info, true); err != nil {
 		if hookErr, ok := err.(hooks.HookError); ok {
 			return hooks.NewHookError(
-				fmt.Errorf("pre-create hook failed: %s", err),
+				fmt.Errorf("%s hook failed: %s", typ, err),
 				hookErr.StatusCode(),
 				hookErr.Body(),
 			)
 		}
-		return fmt.Errorf("pre-create hook failed: %s\n%s", err, string(output))
+		return fmt.Errorf("%s hook failed: %s\n%s", typ, err, string(output))
 	}
 
 	return nil
 }
 
+func preCreateCallback(info handler.HookEvent) error {
+	return hookCallback(hooks.HookPreCreate, info)
+}
+
+func preFinishCallback(info handler.HookEvent) error {
+	return hookCallback(hooks.HookPreFinish, info)
+}
+
 func SetupHookMetrics() {
 	MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPostFinish)).Add(0)
 	MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPostTerminate)).Add(0)
 	MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPostReceive)).Add(0)
 	MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPostCreate)).Add(0)
 	MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPreCreate)).Add(0)
+	MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPreFinish)).Add(0)
 }
 
 func SetupPreHooks(config *handler.Config) error {
@@ -89,6 +98,7 @@ func SetupPreHooks(config *handler.Config) error {
 	}
 
 	config.PreUploadCreateCallback = preCreateCallback
+	config.PreFinishResponseCallback = preFinishCallback
 
 	return nil
 }

cmd/tusd/cli/hooks/hooks.go

@@ -17,9 +17,10 @@ const (
 	HookPostReceive   HookType = "post-receive"
 	HookPostCreate    HookType = "post-create"
 	HookPreCreate     HookType = "pre-create"
+	HookPreFinish     HookType = "pre-finish"
 )
 
-var AvailableHooks []HookType = []HookType{HookPreCreate, HookPostCreate, HookPostReceive, HookPostTerminate, HookPostFinish}
+var AvailableHooks []HookType = []HookType{HookPreCreate, HookPostCreate, HookPostReceive, HookPostTerminate, HookPostFinish, HookPreFinish}
 
 type hookDataStore struct {
 	handler.DataStore

cmd/tusd/cli/hooks/plugin.go

@@ -13,6 +13,7 @@ type PluginHookHandler interface {
 	PostReceive(info handler.HookEvent) error
 	PostFinish(info handler.HookEvent) error
 	PostTerminate(info handler.HookEvent) error
+	PreFinish(info handler.HookEvent) error
 }
 
 type PluginHook struct {
@@ -54,6 +55,8 @@ func (h PluginHook) InvokeHook(typ HookType, info handler.HookEvent, captureOutp
 		err = h.handler.PostCreate(info)
 	case HookPreCreate:
 		err = h.handler.PreCreate(info)
+	case HookPreFinish:
+		err = h.handler.PreFinish(info)
 	default:
 		err = fmt.Errorf("hooks: unknown hook named %s", typ)
 	}

docs/hooks.md

@@ -13,7 +13,7 @@ If not otherwise noted, all hooks are invoked in a *non-blocking* way, meaning t
 
 ## Blocking Hooks
 
-On the other hand, there are a few *blocking* hooks, such as caused by the `pre-create` event. Because their exit code will dictate whether tusd will accept the current incoming request, tusd will wait until the hook process has exited. Therefore, in order to keep the response times low, one should avoid to make time-consuming operations inside the processes for blocking hooks. 
+On the other hand, there are a few *blocking* hooks, such as caused by the `pre-create` and `pre-finish` events. Because their exit code will dictate whether tusd will accept the current incoming request, tusd will wait until the hook process has exited. Therefore, in order to keep the response times low, one should avoid to make time-consuming operations inside the processes for blocking hooks.
 
 ### Blocking File Hooks
 
@@ -33,6 +33,12 @@ This event will be triggered before an upload is created, allowing you to run ce
 
 This event will be triggered after an upload is created, allowing you to run certain routines. For example, notifying other parts of your system that a new upload has to be handled. At this point the upload may have received some data already since the invocation of these hooks may be delayed by a short duration. 
 
+### pre-finish
+
+This event will be triggered after an upload is fully finished but before a response has been returned to the client.
+This is a blocking hook, as such it can be used to validate or post-process an uploaded file.
+A non-zero exit code or HTTP response greater than `400` will return a HTTP 500 error to the client.
+
 ### post-finish
 
 This event will be triggered after an upload is fully finished, meaning that all chunks have been transfered and saved in the storage. After this point, no further modifications, except possible deletion, can be made to the upload entity and it may be desirable to use the file for further processing or notify other applications of the completions of this upload.

docs/usage-binary.md

@@ -67,7 +67,7 @@ Usage of tusd:
   -hooks-dir string
     	Directory to search for available hooks scripts
   -hooks-enabled-events string
-    	Comma separated list of enabled hook events (e.g. post-create,post-finish). Leave empty to enable all events
+    	Comma separated list of enabled hook events (e.g. post-create,post-finish). Leave empty to enable all events (default "pre-create,post-create,post-receive,post-terminate,post-finish")
   -hooks-grpc string
     	An gRPC endpoint to which hook events will be sent to
   -hooks-grpc-backoff int

pkg/handler/config.go

@@ -40,11 +40,15 @@ type Config struct {
 	// potentially set by proxies when generating an absolute URL in the
 	// response to POST requests.
 	RespectForwardedHeaders bool
-	// PreUploadreateCCallback will be invoked before a new upload is created, if the
+	// PreUploadCreateCallback will be invoked before a new upload is created, if the
 	// property is supplied. If the callback returns nil, the upload will be created.
 	// Otherwise the HTTP request will be aborted. This can be used to implement
 	// validation of upload metadata etc.
 	PreUploadCreateCallback func(hook HookEvent) error
+	// PreFinishResponseCallback will be invoked after an upload is completed but before
+	// a response is returned to the client. Error responses from the callback will be passed
+	// back to the client. This can be used to implement post-processing validation.
+	PreFinishResponseCallback func(hook HookEvent) error
 }
 
 func (config *Config) validate() error {

pkg/handler/unrouted_handler.go

@@ -688,6 +688,12 @@ func (handler *UnroutedHandler) finishUploadIfComplete(ctx context.Context, uplo
 		}
 
 		handler.Metrics.incUploadsFinished()
+
+		if handler.config.PreFinishResponseCallback != nil {
+			if err := handler.config.PreFinishResponseCallback(newHookEvent(info, r)); err != nil {
+				return err
+			}
+		}
 	}
 
 	return nil