cli: Add option for run hooks from Go plugin

Squashed commit of the following:

commit 1b80f51f94cf860ba8516baed4b65e9ded6441fe
Author: Marius <maerious@gmail.com>
Date:   Mon Jun 10 11:41:30 2019 +0200

    Minor improvements

commit 98daad5f9fa55895a7ae6397b5fcaa353e240954
Author: Marius <maerious@gmail.com>
Date:   Fri Jun 7 13:26:14 2019 +0200

    Extract File and Http hooks into own structs
This commit is contained in:
Marius 2019-06-10 11:42:19 +02:00
parent 10e0bb1fb9
commit 05d9a0ba98
9 changed files with 300 additions and 152 deletions

View File

@ -25,6 +25,7 @@ var Flags struct {
HttpHooksRetry int HttpHooksRetry int
HttpHooksBackoff int HttpHooksBackoff int
HooksStopUploadCode int HooksStopUploadCode int
PluginHookPath string
ShowVersion bool ShowVersion bool
ExposeMetrics bool ExposeMetrics bool
MetricsPath string MetricsPath string
@ -53,6 +54,7 @@ func ParseFlags() {
flag.IntVar(&Flags.HttpHooksRetry, "hooks-http-retry", 3, "Number of times to retry on a 500 or network timeout") flag.IntVar(&Flags.HttpHooksRetry, "hooks-http-retry", 3, "Number of times to retry on a 500 or network timeout")
flag.IntVar(&Flags.HttpHooksBackoff, "hooks-http-backoff", 1, "Number of seconds to wait before retrying each retry") flag.IntVar(&Flags.HttpHooksBackoff, "hooks-http-backoff", 1, "Number of seconds to wait before retrying each retry")
flag.IntVar(&Flags.HooksStopUploadCode, "hooks-stop-code", 0, "Return code from post-receive hook which causes tusd to stop and delete the current upload. A zero value means that no uploads will be stopped") flag.IntVar(&Flags.HooksStopUploadCode, "hooks-stop-code", 0, "Return code from post-receive hook which causes tusd to stop and delete the current upload. A zero value means that no uploads will be stopped")
flag.StringVar(&Flags.PluginHookPath, "hooks-plugin", "", "Path to a Go plugin for loading hook functions (only supported on Linux and macOS; highly EXPERIMENTAL and may BREAK in the future)")
flag.BoolVar(&Flags.ShowVersion, "version", false, "Print tusd version information") flag.BoolVar(&Flags.ShowVersion, "version", false, "Print tusd version information")
flag.BoolVar(&Flags.ExposeMetrics, "expose-metrics", true, "Expose metrics about tusd usage") flag.BoolVar(&Flags.ExposeMetrics, "expose-metrics", true, "Expose metrics about tusd usage")
flag.StringVar(&Flags.MetricsPath, "metrics-path", "/metrics", "Path under which the metrics endpoint will be accessible") flag.StringVar(&Flags.MetricsPath, "metrics-path", "/metrics", "Path under which the metrics endpoint will be accessible")

View File

@ -1,54 +1,27 @@
package cli package cli
import ( import (
"bytes"
"encoding/json"
"fmt" "fmt"
"io/ioutil"
"net/http"
"os"
"os/exec"
"strconv" "strconv"
"time"
"github.com/tus/tusd" "github.com/tus/tusd"
"github.com/tus/tusd/cmd/tusd/cli/hooks"
"github.com/sethgrid/pester"
) )
type HookType string var hookHandler hooks.HookHandler = nil
const (
HookPostFinish HookType = "post-finish"
HookPostTerminate HookType = "post-terminate"
HookPostReceive HookType = "post-receive"
HookPostCreate HookType = "post-create"
HookPreCreate HookType = "pre-create"
)
type hookDataStore struct { type hookDataStore struct {
tusd.DataStore tusd.DataStore
} }
type hookError struct {
error
statusCode int
body []byte
}
func (herr hookError) StatusCode() int {
return herr.statusCode
}
func (herr hookError) Body() []byte {
return herr.body
}
func (store hookDataStore) NewUpload(info tusd.FileInfo) (id string, err error) { func (store hookDataStore) NewUpload(info tusd.FileInfo) (id string, err error) {
if output, err := invokeHookSync(HookPreCreate, info, true); err != nil { if output, err := invokeHookSync(hooks.HookPreCreate, info, true); err != nil {
if hookErr, ok := err.(hookError); ok { if hookErr, ok := err.(hooks.HookError); ok {
hookErr.error = fmt.Errorf("pre-create hook failed: %s", err) return "", hooks.NewHookError(
return "", hookErr fmt.Errorf("pre-create hook failed: %s", err),
hookErr.StatusCode(),
hookErr.Body(),
)
} }
return "", fmt.Errorf("pre-create hook failed: %s\n%s", err, string(output)) return "", fmt.Errorf("pre-create hook failed: %s\n%s", err, string(output))
} }
@ -56,17 +29,41 @@ func (store hookDataStore) NewUpload(info tusd.FileInfo) (id string, err error)
} }
func SetupHookMetrics() { func SetupHookMetrics() {
MetricsHookErrorsTotal.WithLabelValues(string(HookPostFinish)).Add(0) MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPostFinish)).Add(0)
MetricsHookErrorsTotal.WithLabelValues(string(HookPostTerminate)).Add(0) MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPostTerminate)).Add(0)
MetricsHookErrorsTotal.WithLabelValues(string(HookPostReceive)).Add(0) MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPostReceive)).Add(0)
MetricsHookErrorsTotal.WithLabelValues(string(HookPostCreate)).Add(0) MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPostCreate)).Add(0)
MetricsHookErrorsTotal.WithLabelValues(string(HookPreCreate)).Add(0) MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPreCreate)).Add(0)
} }
func SetupPreHooks(composer *tusd.StoreComposer) { func SetupPreHooks(composer *tusd.StoreComposer) error {
if Flags.FileHooksDir != "" {
hookHandler = &hooks.FileHook{
Directory: Flags.FileHooksDir,
}
} else if Flags.HttpHooksEndpoint != "" {
hookHandler = &hooks.HttpHook{
Endpoint: Flags.HttpHooksEndpoint,
MaxRetries: Flags.HttpHooksRetry,
Backoff: Flags.HttpHooksBackoff,
}
} else if Flags.PluginHookPath != "" {
hookHandler = &hooks.PluginHook{
Path: Flags.PluginHookPath,
}
} else {
return nil
}
if err := hookHandler.Setup(); err != nil {
return err
}
composer.UseCore(hookDataStore{ composer.UseCore(hookDataStore{
DataStore: composer.Core, DataStore: composer.Core,
}) })
return nil
} }
func SetupPostHooks(handler *tusd.Handler) { func SetupPostHooks(handler *tusd.Handler) {
@ -74,50 +71,41 @@ func SetupPostHooks(handler *tusd.Handler) {
for { for {
select { select {
case info := <-handler.CompleteUploads: case info := <-handler.CompleteUploads:
invokeHook(HookPostFinish, info) invokeHookAsync(hooks.HookPostFinish, info)
case info := <-handler.TerminatedUploads: case info := <-handler.TerminatedUploads:
invokeHook(HookPostTerminate, info) invokeHookAsync(hooks.HookPostTerminate, info)
case info := <-handler.UploadProgress: case info := <-handler.UploadProgress:
invokeHook(HookPostReceive, info) invokeHookAsync(hooks.HookPostReceive, info)
case info := <-handler.CreatedUploads: case info := <-handler.CreatedUploads:
invokeHook(HookPostCreate, info) invokeHookAsync(hooks.HookPostCreate, info)
} }
} }
}() }()
} }
func invokeHook(typ HookType, info tusd.FileInfo) { func invokeHookAsync(typ hooks.HookType, info tusd.FileInfo) {
go func() { go func() {
// Error handling is taken care by the function. // Error handling is taken care by the function.
_, _ = invokeHookSync(typ, info, false) _, _ = invokeHookSync(typ, info, false)
}() }()
} }
func invokeHookSync(typ HookType, info tusd.FileInfo, captureOutput bool) ([]byte, error) { func invokeHookSync(typ hooks.HookType, info tusd.FileInfo, captureOutput bool) ([]byte, error) {
switch typ { switch typ {
case HookPostFinish: case hooks.HookPostFinish:
logEv(stdout, "UploadFinished", "id", info.ID, "size", strconv.FormatInt(info.Size, 10)) logEv(stdout, "UploadFinished", "id", info.ID, "size", strconv.FormatInt(info.Size, 10))
case HookPostTerminate: case hooks.HookPostTerminate:
logEv(stdout, "UploadTerminated", "id", info.ID) logEv(stdout, "UploadTerminated", "id", info.ID)
} }
if !Flags.FileHooksInstalled && !Flags.HttpHooksInstalled { if hookHandler == nil {
return nil, nil return nil, nil
} }
name := string(typ) name := string(typ)
logEv(stdout, "HookInvocationStart", "type", name, "id", info.ID) logEv(stdout, "HookInvocationStart", "type", name, "id", info.ID)
output := []byte{} output, returnCode, err := hookHandler.InvokeHook(typ, info, captureOutput)
err := error(nil)
returnCode := 0
if Flags.FileHooksInstalled {
output, returnCode, err = invokeFileHook(name, typ, info, captureOutput)
}
if Flags.HttpHooksInstalled {
output, returnCode, err = invokeHttpHook(name, typ, info, captureOutput)
}
if err != nil { if err != nil {
logEv(stderr, "HookInvocationError", "type", string(typ), "id", info.ID, "error", err.Error()) logEv(stderr, "HookInvocationError", "type", string(typ), "id", info.ID, "error", err.Error())
@ -126,7 +114,7 @@ func invokeHookSync(typ HookType, info tusd.FileInfo, captureOutput bool) ([]byt
logEv(stdout, "HookInvocationFinish", "type", string(typ), "id", info.ID) logEv(stdout, "HookInvocationFinish", "type", string(typ), "id", info.ID)
} }
if typ == HookPostReceive && Flags.HooksStopUploadCode != 0 && Flags.HooksStopUploadCode == returnCode { if typ == hooks.HookPostReceive && Flags.HooksStopUploadCode != 0 && Flags.HooksStopUploadCode == returnCode {
logEv(stdout, "HookStopUpload", "id", info.ID) logEv(stdout, "HookStopUpload", "id", info.ID)
info.StopUpload() info.StopUpload()
@ -134,88 +122,3 @@ func invokeHookSync(typ HookType, info tusd.FileInfo, captureOutput bool) ([]byt
return output, err return output, err
} }
func invokeHttpHook(name string, typ HookType, info tusd.FileInfo, captureOutput bool) ([]byte, int, error) {
jsonInfo, err := json.Marshal(info)
if err != nil {
return nil, 0, err
}
req, err := http.NewRequest("POST", Flags.HttpHooksEndpoint, bytes.NewBuffer(jsonInfo))
if err != nil {
return nil, 0, err
}
req.Header.Set("Hook-Name", name)
req.Header.Set("Content-Type", "application/json")
// Use linear backoff strategy with the user defined values.
client := pester.New()
client.KeepLog = true
client.MaxRetries = Flags.HttpHooksRetry
client.Backoff = func(_ int) time.Duration {
return time.Duration(Flags.HttpHooksBackoff) * time.Second
}
resp, err := client.Do(req)
if err != nil {
return nil, 0, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, 0, err
}
if resp.StatusCode >= http.StatusBadRequest {
return body, resp.StatusCode, hookError{fmt.Errorf("endpoint returned: %s", resp.Status), resp.StatusCode, body}
}
if captureOutput {
return body, resp.StatusCode, err
}
return nil, resp.StatusCode, err
}
func invokeFileHook(name string, typ HookType, info tusd.FileInfo, captureOutput bool) ([]byte, int, error) {
hookPath := Flags.FileHooksDir + string(os.PathSeparator) + name
cmd := exec.Command(hookPath)
env := os.Environ()
env = append(env, "TUS_ID="+info.ID)
env = append(env, "TUS_SIZE="+strconv.FormatInt(info.Size, 10))
env = append(env, "TUS_OFFSET="+strconv.FormatInt(info.Offset, 10))
jsonInfo, err := json.Marshal(info)
if err != nil {
return nil, 0, err
}
reader := bytes.NewReader(jsonInfo)
cmd.Stdin = reader
cmd.Env = env
cmd.Dir = Flags.FileHooksDir
cmd.Stderr = os.Stderr
// If `captureOutput` is true, this function will return the output (both,
// stderr and stdout), else it will use this process' stdout
var output []byte
if !captureOutput {
cmd.Stdout = os.Stdout
err = cmd.Run()
} else {
output, err = cmd.Output()
}
// Ignore the error, only, if the hook's file could not be found. This usually
// means that the user is only using a subset of the available hooks.
if os.IsNotExist(err) {
err = nil
}
returnCode := cmd.ProcessState.ExitCode()
return output, returnCode, err
}

View File

@ -0,0 +1,60 @@
package hooks
import (
"bytes"
"encoding/json"
"os"
"os/exec"
"strconv"
"github.com/tus/tusd"
)
type FileHook struct {
Directory string
}
func (_ FileHook) Setup() error {
return nil
}
func (h FileHook) InvokeHook(typ HookType, info tusd.FileInfo, captureOutput bool) ([]byte, int, error) {
hookPath := h.Directory + string(os.PathSeparator) + string(typ)
cmd := exec.Command(hookPath)
env := os.Environ()
env = append(env, "TUS_ID="+info.ID)
env = append(env, "TUS_SIZE="+strconv.FormatInt(info.Size, 10))
env = append(env, "TUS_OFFSET="+strconv.FormatInt(info.Offset, 10))
jsonInfo, err := json.Marshal(info)
if err != nil {
return nil, 0, err
}
reader := bytes.NewReader(jsonInfo)
cmd.Stdin = reader
cmd.Env = env
cmd.Dir = h.Directory
cmd.Stderr = os.Stderr
// If `captureOutput` is true, this function will return the output (both,
// stderr and stdout), else it will use this process' stdout
var output []byte
if !captureOutput {
cmd.Stdout = os.Stdout
err = cmd.Run()
} else {
output, err = cmd.Output()
}
// Ignore the error, only, if the hook's file could not be found. This usually
// means that the user is only using a subset of the available hooks.
if os.IsNotExist(err) {
err = nil
}
returnCode := cmd.ProcessState.ExitCode()
return output, returnCode, err
}

View File

@ -0,0 +1,46 @@
package hooks
import (
"github.com/tus/tusd"
)
type HookHandler interface {
Setup() error
InvokeHook(typ HookType, info tusd.FileInfo, captureOutput bool) ([]byte, int, error)
}
type HookType string
const (
HookPostFinish HookType = "post-finish"
HookPostTerminate HookType = "post-terminate"
HookPostReceive HookType = "post-receive"
HookPostCreate HookType = "post-create"
HookPreCreate HookType = "pre-create"
)
type hookDataStore struct {
tusd.DataStore
}
type HookError struct {
error
statusCode int
body []byte
}
func NewHookError(err error, statusCode int, body []byte) HookError {
return HookError{err, statusCode, body}
}
func (herr HookError) StatusCode() int {
return herr.statusCode
}
func (herr HookError) Body() []byte {
return herr.body
}
func (herr HookError) Error() string {
return herr.error.Error()
}

View File

@ -0,0 +1,69 @@
package hooks
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"time"
"github.com/tus/tusd"
"github.com/sethgrid/pester"
)
type HttpHook struct {
Endpoint string
MaxRetries int
Backoff int
}
func (_ HttpHook) Setup() error {
return nil
}
func (h HttpHook) InvokeHook(typ HookType, info tusd.FileInfo, captureOutput bool) ([]byte, int, error) {
jsonInfo, err := json.Marshal(info)
if err != nil {
return nil, 0, err
}
req, err := http.NewRequest("POST", h.Endpoint, bytes.NewBuffer(jsonInfo))
if err != nil {
return nil, 0, err
}
req.Header.Set("Hook-Name", string(typ))
req.Header.Set("Content-Type", "application/json")
// TODO: Can we initialize this in Setup()?
// Use linear backoff strategy with the user defined values.
client := pester.New()
client.KeepLog = true
client.MaxRetries = h.MaxRetries
client.Backoff = func(_ int) time.Duration {
return time.Duration(h.Backoff) * time.Second
}
resp, err := client.Do(req)
if err != nil {
return nil, 0, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, 0, err
}
if resp.StatusCode >= http.StatusBadRequest {
return body, resp.StatusCode, NewHookError(fmt.Errorf("endpoint returned: %s", resp.Status), resp.StatusCode, body)
}
if captureOutput {
return body, resp.StatusCode, err
}
return nil, resp.StatusCode, err
}

View File

@ -0,0 +1,66 @@
package hooks
import (
"fmt"
"plugin"
"github.com/tus/tusd"
)
type PluginHookHandler interface {
PreCreate(info tusd.FileInfo) error
PostCreate(info tusd.FileInfo) error
PostReceive(info tusd.FileInfo) error
PostFinish(info tusd.FileInfo) error
PostTerminate(info tusd.FileInfo) error
}
type PluginHook struct {
Path string
handler PluginHookHandler
}
func (h *PluginHook) Setup() error {
p, err := plugin.Open(h.Path)
if err != nil {
return err
}
symbol, err := p.Lookup("TusdHookHandler")
if err != nil {
return err
}
handler, ok := symbol.(*PluginHookHandler)
if !ok {
return fmt.Errorf("hooks: could not cast TusdHookHandler from %s into PluginHookHandler interface", h.Path)
}
h.handler = *handler
return nil
}
func (h PluginHook) InvokeHook(typ HookType, info tusd.FileInfo, captureOutput bool) ([]byte, int, error) {
var err error
switch typ {
case HookPostFinish:
err = h.handler.PostFinish(info)
case HookPostTerminate:
err = h.handler.PostTerminate(info)
case HookPostReceive:
err = h.handler.PostReceive(info)
case HookPostCreate:
err = h.handler.PostCreate(info)
case HookPreCreate:
err = h.handler.PreCreate(info)
default:
err = fmt.Errorf("hooks: unknown hook named %s", typ)
}
if err != nil {
return nil, 1, err
}
return nil, 0, nil
}

View File

@ -7,8 +7,8 @@ import (
"github.com/tus/tusd" "github.com/tus/tusd"
) )
var stdout = log.New(os.Stdout, "[tusd] ", 0) var stdout = log.New(os.Stdout, "[tusd] ", log.Ldate|log.Ltime)
var stderr = log.New(os.Stderr, "[tusd] ", 0) var stderr = log.New(os.Stderr, "[tusd] ", log.Ldate|log.Ltime)
func logEv(logOutput *log.Logger, eventName string, details ...string) { func logEv(logOutput *log.Logger, eventName string, details ...string) {
tusd.LogEvent(logOutput, eventName, details...) tusd.LogEvent(logOutput, eventName, details...)

View File

@ -15,7 +15,9 @@ import (
// specified, in which case a different socket creation and binding mechanism // specified, in which case a different socket creation and binding mechanism
// is put in place. // is put in place.
func Serve() { func Serve() {
SetupPreHooks(Composer) if err := SetupPreHooks(Composer); err != nil {
stderr.Fatalf("Unable to setup hooks for handler: %s", err)
}
handler, err := tusd.NewHandler(tusd.Config{ handler, err := tusd.NewHandler(tusd.Config{
MaxSize: Flags.MaxSize, MaxSize: Flags.MaxSize,

View File

@ -47,7 +47,7 @@ type Config struct {
func (config *Config) validate() error { func (config *Config) validate() error {
if config.Logger == nil { if config.Logger == nil {
config.Logger = log.New(os.Stdout, "[tusd] ", log.Ldate | log.Lmicroseconds) config.Logger = log.New(os.Stdout, "[tusd] ", log.Ldate | log.Ltime)
} }
base := config.BasePath base := config.BasePath