From 9433afd70c194575028d4dd3302e9d4433933699 Mon Sep 17 00:00:00 2001 From: Marius Date: Sun, 25 Aug 2019 19:58:24 +0200 Subject: [PATCH 1/3] docs: Use consistent style for flags Closes https://github.com/tus/tusd/issues/238 --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3240056..b6a9619 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ snippet demonstrates how to start a tusd process which accepts tus uploads at `http://localhost:1080/files/` (notice the trailing slash) and stores them locally in the `./data` directory: ``` -$ tusd -dir ./data +$ tusd -dir=./data [tusd] Using './data' as directory storage. [tusd] Using 0.00MB as maximum size. [tusd] Using 0.0.0.0:1080 as address to listen. @@ -69,7 +69,7 @@ option): $ export AWS_ACCESS_KEY_ID=xxxxx $ export AWS_SECRET_ACCESS_KEY=xxxxx $ export AWS_REGION=eu-west-1 -$ tusd -s3-bucket my-test-bucket.com +$ tusd -s3-bucket=my-test-bucket.com [tusd] Using 's3://my-test-bucket.com' as S3 bucket for storage. [tusd] Using 0.00MB as maximum size. [tusd] Using 0.0.0.0:1080 as address to listen. @@ -84,7 +84,7 @@ enable this feature, supply the path to your account file containing the necessa ``` $ export GCS_SERVICE_ACCOUNT_FILE=./account.json -$ tusd -gcs-bucket my-test-bucket.com +$ tusd -gcs-bucket=my-test-bucket.com [tusd] Using 'gcs://my-test-bucket.com' as GCS bucket for storage. [tusd] Using 0.00MB as maximum size. [tusd] Using 0.0.0.0:1080 as address to listen. From f1fe5e26034504f084c97cd53cccc65271425095 Mon Sep 17 00:00:00 2001 From: Marius Date: Mon, 26 Aug 2019 20:34:37 +0200 Subject: [PATCH 2/3] cli: Add option to whitelist hooks (#302) --- README.md | 2 ++ cmd/tusd/cli/flags.go | 35 ++++++++++++++++++++++++++++++++++- cmd/tusd/cli/hooks.go | 14 +++++++++++++- cmd/tusd/cli/hooks/hooks.go | 2 ++ docs/hooks.md | 3 +++ 5 files changed, 54 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b6a9619..bd71a17 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,8 @@ Usage of tusd: Use Google Cloud Storage with this bucket as storage backend (requires the GCS_SERVICE_ACCOUNT_FILE environment variable to be set) -hooks-dir string Directory to search for available hooks scripts + -hooks-enabled-events string + Comma separated list of enabled hook events (e.g. post-create,post-finish). Leave empty to enable all events -hooks-http string An HTTP endpoint to which hook events will be sent to -hooks-http-backoff int diff --git a/cmd/tusd/cli/flags.go b/cmd/tusd/cli/flags.go index e39ddbd..8af3a47 100644 --- a/cmd/tusd/cli/flags.go +++ b/cmd/tusd/cli/flags.go @@ -4,6 +4,8 @@ import ( "flag" "path/filepath" "strings" + + "github.com/tus/tusd/cmd/tusd/cli/hooks" ) var Flags struct { @@ -20,12 +22,14 @@ var Flags struct { S3Endpoint string GCSBucket string GCSObjectPrefix string + EnabledHooksString string FileHooksDir string HttpHooksEndpoint string HttpHooksRetry int HttpHooksBackoff int HooksStopUploadCode int PluginHookPath string + EnabledHooks []hooks.HookType ShowVersion bool ExposeMetrics bool MetricsPath string @@ -50,6 +54,7 @@ func ParseFlags() { flag.StringVar(&Flags.S3Endpoint, "s3-endpoint", "", "Endpoint to use S3 compatible implementations like minio (requires s3-bucket to be pass)") flag.StringVar(&Flags.GCSBucket, "gcs-bucket", "", "Use Google Cloud Storage with this bucket as storage backend (requires the GCS_SERVICE_ACCOUNT_FILE environment variable to be set)") flag.StringVar(&Flags.GCSObjectPrefix, "gcs-object-prefix", "", "Prefix for GCS object names (can't contain underscore character)") + flag.StringVar(&Flags.EnabledHooksString, "hooks-enabled-events", "", "Comma separated list of enabled hook events (e.g. post-create,post-finish). Leave empty to enable all events") flag.StringVar(&Flags.FileHooksDir, "hooks-dir", "", "Directory to search for available hooks scripts") flag.StringVar(&Flags.HttpHooksEndpoint, "hooks-http", "", "An HTTP endpoint to which hook events will be sent to") flag.IntVar(&Flags.HttpHooksRetry, "hooks-http-retry", 3, "Number of times to retry on a 500 or network timeout") @@ -61,9 +66,10 @@ func ParseFlags() { flag.StringVar(&Flags.MetricsPath, "metrics-path", "/metrics", "Path under which the metrics endpoint will be accessible") flag.BoolVar(&Flags.BehindProxy, "behind-proxy", false, "Respect X-Forwarded-* and similar headers which may be set by proxies") flag.BoolVar(&Flags.VerboseOutput, "verbose", true, "Enable verbose logging output") - flag.Parse() + SetEnabledHooks() + if Flags.FileHooksDir != "" { Flags.FileHooksDir, _ = filepath.Abs(Flags.FileHooksDir) Flags.FileHooksInstalled = true @@ -89,3 +95,30 @@ func ParseFlags() { "Please remove underscore from the value", Flags.GCSObjectPrefix) } } + +func SetEnabledHooks() { + if Flags.EnabledHooksString != "" { + slc := strings.Split(Flags.EnabledHooksString, ",") + + for i, h := range slc { + slc[i] = strings.TrimSpace(h) + + if !hookTypeInSlice(hooks.HookType(h), hooks.AvailableHooks) { + stderr.Fatalf("Unknown hook event type in -hooks-enabled-events flag: %s", h) + } + + Flags.EnabledHooks = append(Flags.EnabledHooks, hooks.HookType(h)) + } + } + + if len(Flags.EnabledHooks) == 0 { + Flags.EnabledHooks = hooks.AvailableHooks + } + + var enabledHooksString []string + for _, h := range Flags.EnabledHooks { + enabledHooksString = append(enabledHooksString, string(h)) + } + + stdout.Printf("Enabled hook events: %s", strings.Join(enabledHooksString, ", ")) +} diff --git a/cmd/tusd/cli/hooks.go b/cmd/tusd/cli/hooks.go index 6c36651..faa91f6 100644 --- a/cmd/tusd/cli/hooks.go +++ b/cmd/tusd/cli/hooks.go @@ -10,6 +10,15 @@ import ( var hookHandler hooks.HookHandler = nil +func hookTypeInSlice(a hooks.HookType, list []hooks.HookType) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} + type hookDataStore struct { tusd.DataStore } @@ -62,7 +71,6 @@ func SetupPreHooks(composer *tusd.StoreComposer) error { composer.UseCore(hookDataStore{ DataStore: composer.Core, }) - return nil } @@ -91,6 +99,10 @@ func invokeHookAsync(typ hooks.HookType, info tusd.FileInfo) { } func invokeHookSync(typ hooks.HookType, info tusd.FileInfo, captureOutput bool) ([]byte, error) { + if !hookTypeInSlice(typ, Flags.EnabledHooks) { + return nil, nil + } + switch typ { case hooks.HookPostFinish: logEv(stdout, "UploadFinished", "id", info.ID, "size", strconv.FormatInt(info.Size, 10)) diff --git a/cmd/tusd/cli/hooks/hooks.go b/cmd/tusd/cli/hooks/hooks.go index 81a7ed4..9e97492 100644 --- a/cmd/tusd/cli/hooks/hooks.go +++ b/cmd/tusd/cli/hooks/hooks.go @@ -19,6 +19,8 @@ const ( HookPreCreate HookType = "pre-create" ) +var AvailableHooks []HookType = []HookType{HookPreCreate, HookPostCreate, HookPostReceive, HookPostTerminate, HookPostFinish} + type hookDataStore struct { tusd.DataStore } diff --git a/docs/hooks.md b/docs/hooks.md index b43e117..3fa9323 100644 --- a/docs/hooks.md +++ b/docs/hooks.md @@ -45,6 +45,9 @@ This event will be triggered after an upload has been terminated, meaning that t This event will be triggered for every running upload to indicate its current progress. It will be emitted whenever the server has received more data from the client but at most every second. The offset property will be set to the number of bytes which have been transfered to the server, at the time in total. Please be aware that this number may be higher than the number of bytes which have been stored by the data store! +## Whitelisting Hook Events + +The `--hooks-enabled-events` option for the tusd binary works as a whitelist for hook events and takes a comma separated list of hook events (for instance: `pre-create,post-create`). This can be useful to limit the number of hook executions and save resources if you are only interested in some events. If the `--hooks-enabled-events` option is omitted, all hook events are enabled. ## File Hooks ### The Hook Directory From 852b6fa01dbd508b088f26ae2d22b3ebfd1f2399 Mon Sep 17 00:00:00 2001 From: Marius Date: Thu, 5 Sep 2019 19:45:43 +0200 Subject: [PATCH 3/3] core: Allow dots in MIME file types As reported in https://github.com/tus/tusd/issues/304 --- get_test.go | 8 ++++---- unrouted_handler.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/get_test.go b/get_test.go index ef7f208..e3fcfa8 100644 --- a/get_test.go +++ b/get_test.go @@ -134,8 +134,8 @@ func TestGet(t *testing.T) { store.EXPECT().GetInfo("yes").Return(FileInfo{ Offset: 0, MetaData: map[string]string{ - "filetype": "text/html", - "filename": "invoice.html", + "filetype": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "filename": "invoice.docx", }, }, nil) @@ -148,8 +148,8 @@ func TestGet(t *testing.T) { URL: "yes", ResHeader: map[string]string{ "Content-Length": "0", - "Content-Type": "text/html", - "Content-Disposition": `attachment;filename="invoice.html"`, + "Content-Type": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "Content-Disposition": `attachment;filename="invoice.docx"`, }, Code: http.StatusNoContent, ResBody: "", diff --git a/unrouted_handler.go b/unrouted_handler.go index edd25f9..8392d55 100644 --- a/unrouted_handler.go +++ b/unrouted_handler.go @@ -24,7 +24,7 @@ var ( reExtractFileID = regexp.MustCompile(`([^/]+)\/?$`) reForwardedHost = regexp.MustCompile(`host=([^,]+)`) reForwardedProto = regexp.MustCompile(`proto=(https?)`) - reMimeType = regexp.MustCompile(`^[a-z]+\/[a-z\-\+]+$`) + reMimeType = regexp.MustCompile(`^[a-z]+\/[a-z\-\+\.]+$`) ) // HTTPError represents an error with an additional status code attached