Add HTTP hooks

Closes https://github.com/tus/tusd/pull/120

cmd/tusd/cli/flags.go

@@ -15,13 +15,17 @@ var Flags struct {
 	Timeout           int64
 	S3Bucket          string
 	S3Endpoint        string
-	HooksDir      string
+	FileHooksDir      string
+	HttpHooksEndpoint string
+	HttpHooksRetry    int
+	HttpHooksBackoff  int
 	ShowVersion       bool
 	ExposeMetrics     bool
 	MetricsPath       string
 	BehindProxy       bool
 
-	HooksInstalled bool
+	FileHooksInstalled bool
+	HttpHooksInstalled bool
 }
 
 func ParseFlags() {
@@ -34,7 +38,10 @@ func ParseFlags() {
 	flag.Int64Var(&Flags.Timeout, "timeout", 30*1000, "Read timeout for connections in milliseconds.  A zero value means that reads will not timeout")
 	flag.StringVar(&Flags.S3Bucket, "s3-bucket", "", "Use AWS S3 with this bucket as storage backend (requires the AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_REGION environment variables to be set)")
 	flag.StringVar(&Flags.S3Endpoint, "s3-endpoint", "", "Endpoint to use S3 compatible implementations like minio (requires s3-bucket to be pass)")
-	flag.StringVar(&Flags.HooksDir, "hooks-dir", "", "Directory to search for available hooks scripts")
+	flag.StringVar(&Flags.FileHooksDir, "hooks-dir", "", "Directory to search for available hooks scripts")
+	flag.StringVar(&Flags.HttpHooksEndpoint, "hooks-http", "", "An HTTP endpoint to which hook events will be sent to")
+	flag.IntVar(&Flags.HttpHooksRetry, "hooks-http-retry", 3, "Number of times to retry on a 500 or network timeout")
+	flag.IntVar(&Flags.HttpHooksBackoff, "hooks-http-backoff", 1, "Number of seconds to wait before retrying each retry")
 	flag.BoolVar(&Flags.ShowVersion, "version", false, "Print tusd version information")
 	flag.BoolVar(&Flags.ExposeMetrics, "expose-metrics", true, "Expose metrics about tusd usage")
 	flag.StringVar(&Flags.MetricsPath, "metrics-path", "/metrics", "Path under which the metrics endpoint will be accessible")
@@ -42,11 +49,17 @@ func ParseFlags() {
 
 	flag.Parse()
 
-	if Flags.HooksDir != "" {
+	if Flags.FileHooksDir != "" {
-		Flags.HooksDir, _ = filepath.Abs(Flags.HooksDir)
+		Flags.FileHooksDir, _ = filepath.Abs(Flags.FileHooksDir)
-		Flags.HooksInstalled = true
+		Flags.FileHooksInstalled = true
 
-		stdout.Printf("Using '%s' for hooks", Flags.HooksDir)
+		stdout.Printf("Using '%s' for hooks", Flags.FileHooksDir)
+	}
+
+	if Flags.HttpHooksEndpoint != "" {
+		Flags.HttpHooksInstalled = true
+
+		stdout.Printf("Using '%s' as the endpoint for hooks", Flags.HttpHooksEndpoint)
 	}
 
 	if Flags.UploadDir == "" && Flags.S3Bucket == "" {

cmd/tusd/cli/hooks.go

@@ -4,11 +4,16 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"net/http"
 	"os"
 	"os/exec"
 	"strconv"
+	"time"
 
 	"github.com/tus/tusd"
+
+	"github.com/sethgrid/pester"
 )
 
 type HookType string
@@ -67,14 +72,78 @@ func invokeHookSync(typ HookType, info tusd.FileInfo, captureOutput bool) ([]byt
 		logEv("UploadTerminated", "id", info.ID)
 	}
 
-	if !Flags.HooksInstalled {
+	if !Flags.FileHooksInstalled && !Flags.HttpHooksInstalled {
 		return nil, nil
 	}
-
 	name := string(typ)
 	logEv("HookInvocationStart", "type", name, "id", info.ID)
 
-	cmd := exec.Command(Flags.HooksDir + "/" + name)
+	output := []byte{}
+	err := error(nil)
+
+	if Flags.FileHooksInstalled {
+		output, err = invokeFileHook(name, typ, info, captureOutput)
+	}
+
+	if Flags.HttpHooksInstalled {
+		output, err = invokeHttpHook(name, typ, info, captureOutput)
+	}
+
+	if err != nil {
+		logEv("HookInvocationError", "type", string(typ), "id", info.ID, "error", err.Error())
+	} else {
+		logEv("HookInvocationFinish", "type", string(typ), "id", info.ID)
+	}
+
+	return output, err
+}
+
+func invokeHttpHook(name string, typ HookType, info tusd.FileInfo, captureOutput bool) ([]byte, error) {
+	jsonInfo, err := json.Marshal(info)
+	if err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequest("POST", Flags.HttpHooksEndpoint, bytes.NewBuffer(jsonInfo))
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Hook-Name", name)
+	req.Header.Set("Content-Type", "application/json")
+
+	// Use linear backoff strategy with the user defined values.
+	client := pester.New()
+	client.KeepLog = true
+	client.MaxRetries = Flags.HttpHooksRetry
+	client.Backoff = func(_ int) time.Duration {
+		return time.Duration(Flags.HttpHooksBackoff) * time.Second
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode >= http.StatusBadRequest {
+		return body, fmt.Errorf("endpoint returned: %s\n%s", resp.Status, body)
+	}
+
+	if captureOutput {
+		return body, err
+	}
+
+	return nil, err
+}
+
+func invokeFileHook(name string, typ HookType, info tusd.FileInfo, captureOutput bool) ([]byte, error) {
+	cmd := exec.Command(Flags.FileHooksDir + "/" + name)
 	env := os.Environ()
 	env = append(env, "TUS_ID="+info.ID)
 	env = append(env, "TUS_SIZE="+strconv.FormatInt(info.Size, 10))
@@ -89,7 +158,7 @@ func invokeHookSync(typ HookType, info tusd.FileInfo, captureOutput bool) ([]byt
 	cmd.Stdin = reader
 
 	cmd.Env = env
-	cmd.Dir = Flags.HooksDir
+	cmd.Dir = Flags.FileHooksDir
 	cmd.Stderr = os.Stderr
 
 	// If `captureOutput` is true, this function will return the output (both,
@@ -102,12 +171,6 @@ func invokeHookSync(typ HookType, info tusd.FileInfo, captureOutput bool) ([]byt
 		output, err = cmd.Output()
 	}
 
-	if err != nil {
-		logEv("HookInvocationError", "type", string(typ), "id", info.ID, "error", err.Error())
-	} else {
-		logEv("HookInvocationFinish", "type", string(typ), "id", info.ID)
-	}
-
 	// Ignore the error, only, if the hook's file could not be found. This usually
 	// means that the user is only using a subset of the available hooks.
 	if os.IsNotExist(err) {

docs/hooks.md

@@ -1,15 +1,43 @@
 # Hooks
 
-When integrating tusd into an application, it is important to establish a communication channel between the two components. The tusd binary accomplishes this by providing a Hook system which will execute custom code when certain events happen, such as an upload being created or finished. While being simple, yet powerful enough, enabled uses ranging from logging over validation and authorization to processing the uploaded files.
+When integrating tusd into an application, it is important to establish a communication channel between the two components. The tusd binary accomplishes this by providing a system which triggers actions when certain events happen, such as an upload being created or finished. This simple-but-powerful system enables uses ranging from logging over validation and authorization to processing the uploaded files.
 
-If you have previously worked with the hook system provided by [Git](https://git-scm.com/book/it/v2/Customizing-Git-Git-Hooks), you will see a lot of parallels. If this does not apply to you, don't worry, it is not complicated. Before getting stated, it is good to have a high level overview of what a hook is actually: It is a regular file, located in a specific directory, which will be executed once a certain event occurs. This file can either be a script interpreted by a runtime, such as Bash or Python, or a compiled binary. When invoked, the process will be provided with information about the event triggering the occuring event and the associated uploaded.
+When a specific action happens during an upload (pre-create, post-receive, post-finish, or post-terminate), the hook system enables tusd to fire off a specific event. Tusd provides two ways of doing this:
 
-## The Hook Directory
+1. Execute an arbitrary file, mirroring the Git hook system, called File Hooks.
+2. Fire off an HTTP POST request to a custom endpoint, called HTTP Hooks.
 
-By default, the hook system is disabled. To enable it, pass the `--hook-dir` option to the tusd binary. The flag's value will be a path, the **hook directory**, relative to the current working directory, pointing to the folder containing the executable **hook files**:
+## Blocking and Non-Blocking Hooks
+
+If not otherwise noted, all hooks are invoked in a *non-blocking* way, meaning that tusd will not wait until the hook process has finished and exited. Therefore, the hook process is not able to influence how tusd may continue handling the current request, regardless of which exit code it may set. Furthermore, the hook process' stdout and stderr will be piped to tusd's stdout and stderr correspondingly, allowing one to use these channels for additional logging.
+
+On the other hand, there are a few *blocking* hooks, such as caused by the `pre-create` event. Because their exit code will dictate whether tusd will accept the current incoming request, tusd will wait until the hook process has exited. Therefore, in order to keep the response times low, one should avoid to make time-consuming operations inside the processes for blocking hooks. An exit code of `0` indicates that tusd should continue handling the request as normal. On the other hand, a non-zero exit code tells tusd to reject the request with a `500 Internal Server Error` response containing the process' output from stderr. For the sake of logging, the process' output from stdout will always be piped to tusd's stdout.
+
+## List of Available Hooks
+
+### pre-create
+
+This event will be triggered before an upload is created, allowing you to run certain routines. For example, validating that specific metadata values are set, or verifying that a corresponding entity belonging to the upload (e.g. a user) exists. Because this event will result in a blocking hook, you can determine whether the upload should be created or rejected using the exit code. An exit code of `0` will allow the upload to be created and continued as usual. A non-zero exit code will reject an upload creation request, making it a good place for authentication and authorization. Please be aware, that during this stage the upload ID will be an empty string as the entity has not been created and therefore this piece of information is not yet available.
+
+### post-finish
+
+This event will be triggered after an upload is fully finished, meaning that all chunks have been transfered and saved in the storage. After this point, no further modifications, except possible deletion, can be made to the upload entity and it may be desirable to use the file for further processing or notify other applications of the completions of this upload.
+
+### post-terminate
+
+This event will be triggered after an upload has been terminated, meaning that the upload has been totally stopped and all associating chunks have been fully removed from the storage. Therefore, one is not able to retrieve the upload's content anymore and one may wish to notify further applications that this upload will never be resumed nor finished.
+
+### post-receive
+
+This event will be triggered for every running upload to indicate its current progress. It will occur for each open PATCH request, every second. The offset property will be set to the number of bytes which have been transfered to the server, at the time in total. Please be aware that this number may be higher than the number of bytes which have been stored by the data store!
+
+
+## File Hooks
+### The Hook Directory
+By default, the file hook system is disabled. To enable it, pass the `--hooks-dir` option to the tusd binary. The flag's value will be a path, the **hook directory**, relative to the current working directory, pointing to the folder containing the executable **hook files**:
 
 ```bash
-$ tusd --hook-dir ./path/to/hooks/
+$ tusd --hooks-dir ./path/to/hooks/
 
 [tusd] Using './path/to/hooks/' for hooks
 [tusd] Using './data' as directory storage.
@@ -18,7 +46,7 @@ $ tusd --hook-dir ./path/to/hooks/
 
 If an event occurs, the tusd binary will look for a file, named exactly as the event, which will then be executed, as long as the object exists. In the example above, the binary `./path/to/hooks/pre-create` will be invoked, before an upload is created, which can be used to e.g. validate certain metadata. Please note, that the hook file *must not* have an extension, such as `.sh` or `.py`, or else tusd will not recognize and ignore it. A detailed list of all events can be found at the end of this document.
 
-## The Hook's Environment
+### The Hook's Environment
 
 The process of the hook files are provided with information about the event and the upload using to two methods:
 * The `TUS_ID` and `TUS_SIZE` environment variables will contain the upload ID and its size in bytes, which triggered the event. Please be aware, that in the `pre-create` hook the upload ID will be an empty string as the entity has not been created and therefore this piece of information is not yet available.
@@ -52,26 +80,56 @@ The process of the hook files are provided with information about the event and
 
 Be aware that this environment does *not* contain direct data from any HTTP request, in particular not any header values or cookies. If you would like to pass information from the client to the hook, such as authentication details, you may wish to use the [metadata system](http://tus.io/protocols/resumable-upload.html#upload-metadata).
 
-## Blocking and Non-Blocking Hooks
 
-If not otherwise noted, all hooks are invoked in a *non-blocking* way, meaning that tusd will not wait until the hook process has finished and exited. Therefore, the hook process is not able to influence how tusd may continue handling the current request, regardless of which exit code it may set. Furthermore, the hook process' stdout and stderr will be piped to tusd's stdout and stderr correspondingly, allowing one to use these channels for additional logging.
+## HTTP Hooks
 
-On the other hand, there are a few *blocking* hooks, such as caused by the `pre-create` event. Because their exit code will dictate whether tusd will accept the current incoming request, tusd will wait until the hook process has exited. Therefore, in order to keep the response times low, one should avoid to make time-consuming operations inside the processes for blocking hooks. An exit code of `0` indicates that tusd should continue handling the request as normal. On the other hand, a non-zero exit code tells tusd to reject the request with a `500 Internal Server Error` response containing the process' output from stderr. For the sake of logging, the process' output from stdout will always be piped to tusd's stdout.
+HTTP Hooks are the second type of hooks supported by tusd. Like the file hooks, it is disabled by default. To enable it, pass the `--hooks-http` option to the tusd binary. The flag's value will be an HTTP URL endpoint, which the tusd binary will issue POST requests to:
 
-## List of Available Hooks
+```bash
+$ tusd --hooks-http http://localhost:8081/write
 
-### pre-create
+[tusd] Using 'http://localhost:8081/write' as the endpoint for hooks
+[tusd] Using './data' as directory storage.
+...
+```
 
-This event will be triggered before an upload is created, allowing you to run certain routines. For example, validating that specific metadata values are set, or verifying that a corresponding entity belonging to the upload (e.g. a user) exists. Because this event will result in a blocking hook, you can determine whether the upload should be created or rejected using the exit code. An exit code of `0` will allow the upload to be created and continued as usual. A non-zero exit code will reject an upload creation request, making it a good place for authentication and authorization. Please be aware, that during this stage the upload ID will be an empty string as the entity has not been created and therefore this piece of information is not yet available.
+Note that the URL must include the `http://` prefix!
 
-### post-finish
+### Usage
 
-This event will be triggered after an upload is fully finished, meaning that all chunks have been transfered and saved in the storage. After this point, no further modifications, except possible deletion, can be made to the upload entity and it may be desirable to use the file for further processing or notify other applications of the completions of this upload.
+Tusd will issue a `POST` request to the specified URL endpoint, specifying the hook name, such as pre-create or post-finish, in the `Hook-Name` header and following body:
 
-### post-terminate
+```js
+{
+  // The upload's ID. Will be empty during the pre-create event
+  "ID": "14b1c4c77771671a8479bc0444bbc5ce",
+  // The upload's total size in bytes.
+  "Size": 46205,
+  // The upload's current offset in bytes.
+  "Offset": 1592,
+  // These properties will be set to true, if the upload as a final or partial
+  // one. See the Concatenation extension for details:
+  // http://tus.io/protocols/resumable-upload.html#concatenation
+  "IsFinal": false,
+  "IsPartial": false,
+  // If the upload is a final one, this value will be an array of upload IDs
+  // which are concatenated to produce the upload.
+  "PartialUploads": null,
+  // The upload's meta data which can be supplied by the clients as it wishes.
+  // All keys and values in this object will be strings.
+  // Be aware that it may contain maliciously crafted values and you must not
+  // trust it without escaping it first!
+  "MetaData": {
+    "filename": "transloadit.png"
+  }
+}
+```
 
-This event will be triggered after an upload has been terminated, meaning that the upload has been totally stopped and all associating chunks have been fully removed from the storage. Therefore, one is not able to retrieve the upload's content anymore and one may wish to notify further applications that this upload will never be resumed nor finished.
+### Configuration
 
-### post-receive
+Tusd uses the [Pester library](https://github.com/sethgrid/pester) to issue requests and handle retries. By default, tusd will retry 3 times on a `500 Internal Server Error` response or network error, with a 1 second backoff. This can be configured with the flags `--hooks-http-retry` and `--hooks-http-backoff`, like so:
 
-This event will be triggered for every running upload to indicate its current progress. It will occur for each open PATCH request, every second. The offset property will be set to the number of bytes which have been transfered to the server, at the time in total. Please be aware that this number may be higher than the number of bytes which have been stored by the data store!
+```
+$ # Retrying 5 times with a 2 second backoff
+$ tusd --hooks-http http://localhost:8081/write --hooks-http-retry 5 --hooks-http-backoff 2
+```

vendor/github.com/sethgrid/pester/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) SendGrid 2016 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/sethgrid/pester/README.md

@@ -0,0 +1,126 @@
+# pester
+
+`pester` wraps Go's standard lib http client to provide several options to increase resiliency in your request. If you experience poor network conditions or requests could experience varied delays, you can now pester the endpoint for data.
+- Send out multiple requests and get the first back (only used for GET calls)
+- Retry on errors
+- Backoff
+
+### Simple Example
+Use `pester` where you would use the http client calls. By default, pester will use a concurrency of 1, and retry the endpoint 3 times with the `DefaultBackoff` strategy of waiting 1 second between retries.
+```go
+/* swap in replacement, just switch
+   http.{Get|Post|PostForm|Head|Do} to
+   pester.{Get|Post|PostForm|Head|Do}
+*/
+resp, err := pester.Get("http://sethammons.com")
+```
+
+### Backoff Strategy
+Provide your own backoff strategy, or use one of the provided built in strategies:
+- `DefaultBackoff`: 1 second
+- `LinearBackoff`: n seconds where n is the retry number
+- `LinearJitterBackoff`: n seconds where n is the retry number, +/- 0-33%
+- `ExponentialBackoff`: n seconds where n is 2^(retry number)
+- `ExponentialJitterBackoff`: n seconds where n is 2^(retry number), +/- 0-33%
+
+```go
+client := pester.New()
+client.Backoff = func(retry int) time.Duration {
+    // set up something dynamic or use a look up table
+    return time.Duration(retry) * time.Minute
+}
+```
+
+### Complete example
+For a complete and working example, see the sample directory.
+`pester` allows you to use a constructor to control:
+- backoff strategy
+- reties
+- concurrency
+- keeping a log for debugging
+```go
+package main
+
+import (
+    "log"
+    "net/http"
+    "strings"
+
+    "github.com/sethgrid/pester"
+)
+
+func main() {
+    log.Println("Starting...")
+
+    { // drop in replacement for http.Get and other client methods
+        resp, err := pester.Get("http://example.com")
+        if err != nil {
+            log.Println("error GETing example.com", err)
+        }
+        defer resp.Body.Close()
+        log.Printf("example.com %s", resp.Status)
+    }
+
+    { // control the resiliency
+        client := pester.New()
+        client.Concurrency = 3
+        client.MaxRetries = 5
+        client.Backoff = pester.ExponentialBackoff
+        client.KeepLog = true
+
+        resp, err := client.Get("http://example.com")
+        if err != nil {
+            log.Println("error GETing example.com", client.LogString())
+        }
+        defer resp.Body.Close()
+        log.Printf("example.com %s", resp.Status)
+    }
+
+    { // use the pester version of http.Client.Do
+        req, err := http.NewRequest("POST", "http://example.com", strings.NewReader("data"))
+        if err != nil {
+            log.Fatal("Unable to create a new http request", err)
+        }
+        resp, err := pester.Do(req)
+        if err != nil {
+            log.Println("error POSTing example.com", err)
+        }
+        defer resp.Body.Close()
+        log.Printf("example.com %s", resp.Status)
+    }
+}
+
+```
+
+### Example Log
+`pester` also allows you to control the resiliency and can optionally log the errors.
+```go
+c := pester.New()
+c.KeepLog = true
+
+nonExistantURL := "http://localhost:9000/foo"
+_, _ = c.Get(nonExistantURL)
+
+fmt.Println(c.LogString())
+/*
+Output:
+
+1432402837 Get [GET] http://localhost:9000/foo request-0 retry-0 error: Get http://localhost:9000/foo: dial tcp 127.0.0.1:9000: connection refused
+1432402838 Get [GET] http://localhost:9000/foo request-0 retry-1 error: Get http://localhost:9000/foo: dial tcp 127.0.0.1:9000: connection refused
+1432402839 Get [GET] http://localhost:9000/foo request-0 retry-2 error: Get http://localhost:9000/foo: dial tcp 127.0.0.1:9000: connection refused
+*/
+```
+
+### Tests
+
+You can run tests in the root directory with `$ go test`. There is a benchmark-like test available with `$ cd benchmarks; go test`.
+You can see `pester` in action with `$ cd sample; go run main.go`.
+
+For watching open file descriptors, you can run `watch "lsof -i -P | grep main"` if you started the app with `go run main.go`.
+I did this for watching for FD leaks. My method was to alter `sample/main.go` to only run one case (`pester.Get with set backoff stategy, concurrency and retries increased`)
+and adding a sleep after the result came back. This let me verify if FDs were getting left open when they should have closed. If you know a better way, let me know!
+I was able to see that FDs are now closing when they should :)
+
+![Are we there yet?](http://butchbellah.com/wp-content/uploads/2012/06/Are-We-There-Yet.jpg)
+
+Are we there yet? Are we there yet? Are we there yet? Are we there yet? ...

vendor/github.com/sethgrid/pester/main.go

@@ -0,0 +1,423 @@
+package pester
+
+// pester provides additional resiliency over the standard http client methods by
+// allowing you to control concurrency, retries, and a backoff strategy.
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math"
+	"math/rand"
+	"net/http"
+	"net/url"
+	"sync"
+	"time"
+)
+
+// Client wraps the http client and exposes all the functionality of the http.Client.
+// Additionally, Client provides pester specific values for handling resiliency.
+type Client struct {
+	// wrap it to provide access to http built ins
+	hc *http.Client
+
+	Transport     http.RoundTripper
+	CheckRedirect func(req *http.Request, via []*http.Request) error
+	Jar           http.CookieJar
+	Timeout       time.Duration
+
+	// pester specific
+	Concurrency int
+	MaxRetries  int
+	Backoff     BackoffStrategy
+	KeepLog     bool
+
+	SuccessReqNum   int
+	SuccessRetryNum int
+
+	wg *sync.WaitGroup
+
+	sync.Mutex
+	ErrLog []ErrEntry
+}
+
+// ErrEntry is used to provide the LogString() data and is populated
+// each time an error happens if KeepLog is set.
+// ErrEntry.Retry is deprecated in favor of ErrEntry.Attempt
+type ErrEntry struct {
+	Time    time.Time
+	Method  string
+	URL     string
+	Verb    string
+	Request int
+	Retry   int
+	Attempt int
+	Err     error
+}
+
+// result simplifies the channel communication for concurrent request handling
+type result struct {
+	resp  *http.Response
+	err   error
+	req   int
+	retry int
+}
+
+// params represents all the params needed to run http client calls and pester errors
+type params struct {
+	method   string
+	verb     string
+	req      *http.Request
+	url      string
+	bodyType string
+	body     io.Reader
+	data     url.Values
+}
+
+// New constructs a new DefaultClient with sensible default values
+func New() *Client {
+	return &Client{
+		Concurrency: DefaultClient.Concurrency,
+		MaxRetries:  DefaultClient.MaxRetries,
+		Backoff:     DefaultClient.Backoff,
+		ErrLog:      DefaultClient.ErrLog,
+		wg:          &sync.WaitGroup{},
+	}
+}
+
+// NewExtendedClient allows you to pass in an http.Client that is previously set up
+// and extends it to have Pester's features of concurrency and retries.
+func NewExtendedClient(hc *http.Client) *Client {
+	c := New()
+	c.hc = hc
+	return c
+}
+
+// BackoffStrategy is used to determine how long a retry request should wait until attempted
+type BackoffStrategy func(retry int) time.Duration
+
+// DefaultClient provides sensible defaults
+var DefaultClient = &Client{Concurrency: 1, MaxRetries: 3, Backoff: DefaultBackoff, ErrLog: []ErrEntry{}}
+
+// DefaultBackoff always returns 1 second
+func DefaultBackoff(_ int) time.Duration {
+	return 1 * time.Second
+}
+
+// ExponentialBackoff returns ever increasing backoffs by a power of 2
+func ExponentialBackoff(i int) time.Duration {
+	return time.Duration(math.Pow(2, float64(i))) * time.Second
+}
+
+// ExponentialJitterBackoff returns ever increasing backoffs by a power of 2
+// with +/- 0-33% to prevent sychronized reuqests.
+func ExponentialJitterBackoff(i int) time.Duration {
+	return jitter(int(math.Pow(2, float64(i))))
+}
+
+// LinearBackoff returns increasing durations, each a second longer than the last
+func LinearBackoff(i int) time.Duration {
+	return time.Duration(i) * time.Second
+}
+
+// LinearJitterBackoff returns increasing durations, each a second longer than the last
+// with +/- 0-33% to prevent sychronized reuqests.
+func LinearJitterBackoff(i int) time.Duration {
+	return jitter(i)
+}
+
+// jitter keeps the +/- 0-33% logic in one place
+func jitter(i int) time.Duration {
+	ms := i * 1000
+
+	maxJitter := ms / 3
+
+	rand.Seed(time.Now().Unix())
+	jitter := rand.Intn(maxJitter + 1)
+
+	if rand.Intn(2) == 1 {
+		ms = ms + jitter
+	} else {
+		ms = ms - jitter
+	}
+
+	// a jitter of 0 messes up the time.Tick chan
+	if ms <= 0 {
+		ms = 1
+	}
+
+	return time.Duration(ms) * time.Millisecond
+}
+
+// Wait blocks until all pester requests have returned
+// Probably not that useful outside of testing.
+func (c *Client) Wait() {
+	c.wg.Wait()
+}
+
+// pester provides all the logic of retries, concurrency, backoff, and logging
+func (c *Client) pester(p params) (*http.Response, error) {
+	resultCh := make(chan result)
+	multiplexCh := make(chan result)
+	finishCh := make(chan struct{})
+
+	// track all requests that go out so we can close the late listener routine that closes late incoming response bodies
+	totalSentRequests := &sync.WaitGroup{}
+	totalSentRequests.Add(1)
+	defer totalSentRequests.Done()
+	allRequestsBackCh := make(chan struct{})
+	go func() {
+		totalSentRequests.Wait()
+		close(allRequestsBackCh)
+	}()
+
+	// GET calls should be idempotent and can make use
+	// of concurrency. Other verbs can mutate and should not
+	// make use of the concurrency feature
+	concurrency := c.Concurrency
+	if p.verb != "GET" {
+		concurrency = 1
+	}
+
+	c.Lock()
+	if c.hc == nil {
+		c.hc = &http.Client{}
+		c.hc.Transport = c.Transport
+		c.hc.CheckRedirect = c.CheckRedirect
+		c.hc.Jar = c.Jar
+		c.hc.Timeout = c.Timeout
+	}
+	c.Unlock()
+
+	// re-create the http client so we can leverage the std lib
+	httpClient := http.Client{
+		Transport:     c.hc.Transport,
+		CheckRedirect: c.hc.CheckRedirect,
+		Jar:           c.hc.Jar,
+		Timeout:       c.hc.Timeout,
+	}
+
+	// if we have a request body, we need to save it for later
+	var originalRequestBody []byte
+	var originalBody []byte
+	var err error
+	if p.req != nil && p.req.Body != nil {
+		originalRequestBody, err = ioutil.ReadAll(p.req.Body)
+		if err != nil {
+			return &http.Response{}, errors.New("error reading request body")
+		}
+		p.req.Body.Close()
+	}
+	if p.body != nil {
+		originalBody, err = ioutil.ReadAll(p.body)
+		if err != nil {
+			return &http.Response{}, errors.New("error reading body")
+		}
+	}
+
+	AttemptLimit := c.MaxRetries
+	if AttemptLimit <= 0 {
+		AttemptLimit = 1
+	}
+
+	for req := 0; req < concurrency; req++ {
+		c.wg.Add(1)
+		totalSentRequests.Add(1)
+		go func(n int, p params) {
+			defer c.wg.Done()
+			defer totalSentRequests.Done()
+
+			var err error
+			for i := 1; i <= AttemptLimit; i++ {
+				c.wg.Add(1)
+				defer c.wg.Done()
+				select {
+				case <-finishCh:
+					return
+				default:
+				}
+				resp := &http.Response{}
+
+				// rehydrate the body (it is drained each read)
+				if len(originalRequestBody) > 0 {
+					p.req.Body = ioutil.NopCloser(bytes.NewBuffer(originalRequestBody))
+				}
+				if len(originalBody) > 0 {
+					p.body = bytes.NewBuffer(originalBody)
+				}
+
+				// route the calls
+				switch p.method {
+				case "Do":
+					resp, err = httpClient.Do(p.req)
+				case "Get":
+					resp, err = httpClient.Get(p.url)
+				case "Head":
+					resp, err = httpClient.Head(p.url)
+				case "Post":
+					resp, err = httpClient.Post(p.url, p.bodyType, p.body)
+				case "PostForm":
+					resp, err = httpClient.PostForm(p.url, p.data)
+				}
+
+				// Early return if we have a valid result
+				// Only retry (ie, continue the loop) on 5xx status codes
+				if err == nil && resp.StatusCode < 500 {
+					multiplexCh <- result{resp: resp, err: err, req: n, retry: i}
+					return
+				}
+
+				c.log(ErrEntry{
+					Time:    time.Now(),
+					Method:  p.method,
+					Verb:    p.verb,
+					URL:     p.url,
+					Request: n,
+					Retry:   i + 1, // would remove, but would break backward compatibility
+					Attempt: i,
+					Err:     err,
+				})
+
+				// if it is the last iteration, grab the result (which is an error at this point)
+				if i == AttemptLimit {
+					multiplexCh <- result{resp: resp, err: err}
+					return
+				}
+
+				// if we are retrying, we should close this response body to free the fd
+				if resp != nil {
+					resp.Body.Close()
+				}
+
+				// prevent a 0 from causing the tick to block, pass additional microsecond
+				<-time.After(c.Backoff(i) + 1*time.Microsecond)
+			}
+		}(req, p)
+	}
+
+	// spin off the go routine so it can continually listen in on late results and close the response bodies
+	go func() {
+		gotFirstResult := false
+		for {
+			select {
+			case res := <-multiplexCh:
+				if !gotFirstResult {
+					gotFirstResult = true
+					close(finishCh)
+					resultCh <- res
+				} else if res.resp != nil {
+					// we only return one result to the caller; close all other response bodies that come back
+					// drain the body before close as to not prevent keepalive. see https://gist.github.com/mholt/eba0f2cc96658be0f717
+					io.Copy(ioutil.Discard, res.resp.Body)
+					res.resp.Body.Close()
+				}
+			case <-allRequestsBackCh:
+				// don't leave this goroutine running
+				return
+			}
+		}
+	}()
+
+	select {
+	case res := <-resultCh:
+		c.Lock()
+		defer c.Unlock()
+		c.SuccessReqNum = res.req
+		c.SuccessRetryNum = res.retry
+		return res.resp, res.err
+	}
+}
+
+// LogString provides a string representation of the errors the client has seen
+func (c *Client) LogString() string {
+	c.Lock()
+	defer c.Unlock()
+	var res string
+	for _, e := range c.ErrLog {
+		res += fmt.Sprintf("%d %s [%s] %s request-%d retry-%d error: %s\n",
+			e.Time.Unix(), e.Method, e.Verb, e.URL, e.Request, e.Retry, e.Err)
+	}
+	return res
+}
+
+// LogErrCount is a helper method used primarily for test validation
+func (c *Client) LogErrCount() int {
+	c.Lock()
+	defer c.Unlock()
+	return len(c.ErrLog)
+}
+
+// EmbedHTTPClient allows you to extend an existing Pester client with an
+// underlying http.Client, such as https://godoc.org/golang.org/x/oauth2/google#DefaultClient
+func (c *Client) EmbedHTTPClient(hc *http.Client) {
+	c.hc = hc
+}
+
+func (c *Client) log(e ErrEntry) {
+	if c.KeepLog {
+		c.Lock()
+		c.ErrLog = append(c.ErrLog, e)
+		c.Unlock()
+	}
+}
+
+// Do provides the same functionality as http.Client.Do
+func (c *Client) Do(req *http.Request) (resp *http.Response, err error) {
+	return c.pester(params{method: "Do", req: req, verb: req.Method, url: req.URL.String()})
+}
+
+// Get provides the same functionality as http.Client.Get
+func (c *Client) Get(url string) (resp *http.Response, err error) {
+	return c.pester(params{method: "Get", url: url, verb: "GET"})
+}
+
+// Head provides the same functionality as http.Client.Head
+func (c *Client) Head(url string) (resp *http.Response, err error) {
+	return c.pester(params{method: "Head", url: url, verb: "HEAD"})
+}
+
+// Post provides the same functionality as http.Client.Post
+func (c *Client) Post(url string, bodyType string, body io.Reader) (resp *http.Response, err error) {
+	return c.pester(params{method: "Post", url: url, bodyType: bodyType, body: body, verb: "POST"})
+}
+
+// PostForm provides the same functionality as http.Client.PostForm
+func (c *Client) PostForm(url string, data url.Values) (resp *http.Response, err error) {
+	return c.pester(params{method: "PostForm", url: url, data: data, verb: "POST"})
+}
+
+////////////////////////////////////////
+// Provide self-constructing variants //
+////////////////////////////////////////
+
+// Do provides the same functionality as http.Client.Do and creates its own constructor
+func Do(req *http.Request) (resp *http.Response, err error) {
+	c := New()
+	return c.Do(req)
+}
+
+// Get provides the same functionality as http.Client.Get and creates its own constructor
+func Get(url string) (resp *http.Response, err error) {
+	c := New()
+	return c.Get(url)
+}
+
+// Head provides the same functionality as http.Client.Head and creates its own constructor
+func Head(url string) (resp *http.Response, err error) {
+	c := New()
+	return c.Head(url)
+}
+
+// Post provides the same functionality as http.Client.Post and creates its own constructor
+func Post(url string, bodyType string, body io.Reader) (resp *http.Response, err error) {
+	c := New()
+	return c.Post(url, bodyType, body)
+}
+
+// PostForm provides the same functionality as http.Client.PostForm and creates its own constructor
+func PostForm(url string, data url.Values) (resp *http.Response, err error) {
+	c := New()
+	return c.PostForm(url, data)
+}

vendor/vendor.json

@@ -26,6 +26,12 @@
 			"revision": "792786c7400a136282c1664665ae0a8db921c6c2",
 			"revisionTime": "2016-01-10T10:55:54Z"
 		},
+		{
+			"checksumSHA1": "A8ptzv+SqJ1dAEwFZ2EDp+bwOeg=",
+			"path": "github.com/sethgrid/pester",
+			"revision": "2c5fb962da6113d0968907fd81dba3ca35151d1c",
+			"revisionTime": "2016-12-29T17:44:48Z"
+		},
 		{
 			"checksumSHA1": "JXUVA1jky8ZX8w09p2t5KLs97Nc=",
 			"path": "github.com/stretchr/testify/assert",