LumeWeb
/
tusd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add HTTP hooks 

Closes https://github.com/tus/tusd/pull/120
This commit is contained in:
Peixian Wang 2017-04-14 16:29:18 +02:00 committed by Marius
parent 535e0dff59
commit a7950fdadb
7 changed files with 759 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
cmd/tusd/cli/flags.go
View File

@ -6,22 +6,26 @@ import (
)
var Flags struct {
HttpHost string
HttpPort string
MaxSize int64
UploadDir string
StoreSize int64
Basepath string
Timeout int64
S3Bucket string
S3Endpoint string
HooksDir string
ShowVersion bool
ExposeMetrics bool
MetricsPath string
BehindProxy bool
HttpHost string
HttpPort string
MaxSize int64
UploadDir string
StoreSize int64
Basepath string
Timeout int64
S3Bucket string
S3Endpoint string
FileHooksDir string
HttpHooksEndpoint string
HttpHooksRetry int
HttpHooksBackoff int
ShowVersion bool
ExposeMetrics bool
MetricsPath string
BehindProxy bool
HooksInstalled bool
FileHooksInstalled bool
HttpHooksInstalled bool
}
func ParseFlags() {
@ -34,7 +38,10 @@ func ParseFlags() {
flag.Int64Var(&Flags.Timeout, "timeout", 30*1000, "Read timeout for connections in milliseconds. A zero value means that reads will not timeout")
flag.StringVar(&Flags.S3Bucket, "s3-bucket", "", "Use AWS S3 with this bucket as storage backend (requires the AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_REGION environment variables to be set)")
flag.StringVar(&Flags.S3Endpoint, "s3-endpoint", "", "Endpoint to use S3 compatible implementations like minio (requires s3-bucket to be pass)")
flag.StringVar(&Flags.HooksDir, "hooks-dir", "", "Directory to search for available hooks scripts")
flag.StringVar(&Flags.FileHooksDir, "hooks-dir", "", "Directory to search for available hooks scripts")
flag.StringVar(&Flags.HttpHooksEndpoint, "hooks-http", "", "An HTTP endpoint to which hook events will be sent to")
flag.IntVar(&Flags.HttpHooksRetry, "hooks-http-retry", 3, "Number of times to retry on a 500 or network timeout")
flag.IntVar(&Flags.HttpHooksBackoff, "hooks-http-backoff", 1, "Number of seconds to wait before retrying each retry")
flag.BoolVar(&Flags.ShowVersion, "version", false, "Print tusd version information")
flag.BoolVar(&Flags.ExposeMetrics, "expose-metrics", true, "Expose metrics about tusd usage")
flag.StringVar(&Flags.MetricsPath, "metrics-path", "/metrics", "Path under which the metrics endpoint will be accessible")
@ -42,11 +49,17 @@ func ParseFlags() {
flag.Parse()
if Flags.HooksDir != "" {
Flags.HooksDir, _ = filepath.Abs(Flags.HooksDir)
Flags.HooksInstalled = true
if Flags.FileHooksDir != "" {
Flags.FileHooksDir, _ = filepath.Abs(Flags.FileHooksDir)
Flags.FileHooksInstalled = true
stdout.Printf("Using '%s' for hooks", Flags.HooksDir)
stdout.Printf("Using '%s' for hooks", Flags.FileHooksDir)
}
if Flags.HttpHooksEndpoint != "" {
Flags.HttpHooksInstalled = true
stdout.Printf("Using '%s' as the endpoint for hooks", Flags.HttpHooksEndpoint)
}
if Flags.UploadDir == "" && Flags.S3Bucket == "" {

85
cmd/tusd/cli/hooks.go
View File

@ -4,11 +4,16 @@ import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"os"
"os/exec"
"strconv"
"time"
"github.com/tus/tusd"
"github.com/sethgrid/pester"
)
type HookType string
@ -26,7 +31,7 @@ type hookDataStore struct {
func (store hookDataStore) NewUpload(info tusd.FileInfo) (id string, err error) {
if output, err := invokeHookSync(HookPreCreate, info, true); err != nil {
return "", fmt.Errorf("pre-create hook failed: %s\n%s", err, string(output))
return "", fmt.Errorf("pre-create hook failed: %s\n%s", err, string(output))
}
return store.DataStore.NewUpload(info)
}
@ -67,14 +72,78 @@ func invokeHookSync(typ HookType, info tusd.FileInfo, captureOutput bool) ([]byt
logEv("UploadTerminated", "id", info.ID)
}
if !Flags.HooksInstalled {
if !Flags.FileHooksInstalled && !Flags.HttpHooksInstalled {
return nil, nil
}
name := string(typ)
logEv("HookInvocationStart", "type", name, "id", info.ID)
cmd := exec.Command(Flags.HooksDir + "/" + name)
output := []byte{}
err := error(nil)
if Flags.FileHooksInstalled {
output, err = invokeFileHook(name, typ, info, captureOutput)
}
if Flags.HttpHooksInstalled {
output, err = invokeHttpHook(name, typ, info, captureOutput)
}
if err != nil {
logEv("HookInvocationError", "type", string(typ), "id", info.ID, "error", err.Error())
} else {
logEv("HookInvocationFinish", "type", string(typ), "id", info.ID)
}
return output, err
}
func invokeHttpHook(name string, typ HookType, info tusd.FileInfo, captureOutput bool) ([]byte, error) {
jsonInfo, err := json.Marshal(info)
if err != nil {
return nil, err
}
req, err := http.NewRequest("POST", Flags.HttpHooksEndpoint, bytes.NewBuffer(jsonInfo))
if err != nil {
return nil, err
}
req.Header.Set("Hook-Name", name)
req.Header.Set("Content-Type", "application/json")
// Use linear backoff strategy with the user defined values.
client := pester.New()
client.KeepLog = true
client.MaxRetries = Flags.HttpHooksRetry
client.Backoff = func(_ int) time.Duration {
return time.Duration(Flags.HttpHooksBackoff) * time.Second
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode >= http.StatusBadRequest {
return body, fmt.Errorf("endpoint returned: %s\n%s", resp.Status, body)
}
if captureOutput {
return body, err
}
return nil, err
}
func invokeFileHook(name string, typ HookType, info tusd.FileInfo, captureOutput bool) ([]byte, error) {
cmd := exec.Command(Flags.FileHooksDir + "/" + name)
env := os.Environ()
env = append(env, "TUS_ID="+info.ID)
env = append(env, "TUS_SIZE="+strconv.FormatInt(info.Size, 10))
@ -89,7 +158,7 @@ func invokeHookSync(typ HookType, info tusd.FileInfo, captureOutput bool) ([]byt
cmd.Stdin = reader
cmd.Env = env
cmd.Dir = Flags.HooksDir
cmd.Dir = Flags.FileHooksDir
cmd.Stderr = os.Stderr
// If `captureOutput` is true, this function will return the output (both,
@ -102,12 +171,6 @@ func invokeHookSync(typ HookType, info tusd.FileInfo, captureOutput bool) ([]byt
output, err = cmd.Output()
}
if err != nil {
logEv("HookInvocationError", "type", string(typ), "id", info.ID, "error", err.Error())
} else {
logEv("HookInvocationFinish", "type", string(typ), "id", info.ID)
}
// Ignore the error, only, if the hook's file could not be found. This usually
// means that the user is only using a subset of the available hooks.
if os.IsNotExist(err) {

94
docs/hooks.md
View File

@ -1,15 +1,43 @@
# Hooks
When integrating tusd into an application, it is important to establish a communication channel between the two components. The tusd binary accomplishes this by providing a Hook system which will execute custom code when certain events happen, such as an upload being created or finished. While being simple, yet powerful enough, enabled uses ranging from logging over validation and authorization to processing the uploaded files.
When integrating tusd into an application, it is important to establish a communication channel between the two components. The tusd binary accomplishes this by providing a system which triggers actions when certain events happen, such as an upload being created or finished. This simple-but-powerful system enables uses ranging from logging over validation and authorization to processing the uploaded files.
If you have previously worked with the hook system provided by [Git](https://git-scm.com/book/it/v2/Customizing-Git-Git-Hooks), you will see a lot of parallels. If this does not apply to you, don't worry, it is not complicated. Before getting stated, it is good to have a high level overview of what a hook is actually: It is a regular file, located in a specific directory, which will be executed once a certain event occurs. This file can either be a script interpreted by a runtime, such as Bash or Python, or a compiled binary. When invoked, the process will be provided with information about the event triggering the occuring event and the associated uploaded.
When a specific action happens during an upload (pre-create, post-receive, post-finish, or post-terminate), the hook system enables tusd to fire off a specific event. Tusd provides two ways of doing this:
## The Hook Directory
1. Execute an arbitrary file, mirroring the Git hook system, called File Hooks.
2. Fire off an HTTP POST request to a custom endpoint, called HTTP Hooks.
By default, the hook system is disabled. To enable it, pass the `--hook-dir` option to the tusd binary. The flag's value will be a path, the **hook directory**, relative to the current working directory, pointing to the folder containing the executable **hook files**:
## Blocking and Non-Blocking Hooks
If not otherwise noted, all hooks are invoked in a *non-blocking* way, meaning that tusd will not wait until the hook process has finished and exited. Therefore, the hook process is not able to influence how tusd may continue handling the current request, regardless of which exit code it may set. Furthermore, the hook process' stdout and stderr will be piped to tusd's stdout and stderr correspondingly, allowing one to use these channels for additional logging.
On the other hand, there are a few *blocking* hooks, such as caused by the `pre-create` event. Because their exit code will dictate whether tusd will accept the current incoming request, tusd will wait until the hook process has exited. Therefore, in order to keep the response times low, one should avoid to make time-consuming operations inside the processes for blocking hooks. An exit code of `0` indicates that tusd should continue handling the request as normal. On the other hand, a non-zero exit code tells tusd to reject the request with a `500 Internal Server Error` response containing the process' output from stderr. For the sake of logging, the process' output from stdout will always be piped to tusd's stdout.
## List of Available Hooks
### pre-create
This event will be triggered before an upload is created, allowing you to run certain routines. For example, validating that specific metadata values are set, or verifying that a corresponding entity belonging to the upload (e.g. a user) exists. Because this event will result in a blocking hook, you can determine whether the upload should be created or rejected using the exit code. An exit code of `0` will allow the upload to be created and continued as usual. A non-zero exit code will reject an upload creation request, making it a good place for authentication and authorization. Please be aware, that during this stage the upload ID will be an empty string as the entity has not been created and therefore this piece of information is not yet available.
### post-finish
This event will be triggered after an upload is fully finished, meaning that all chunks have been transfered and saved in the storage. After this point, no further modifications, except possible deletion, can be made to the upload entity and it may be desirable to use the file for further processing or notify other applications of the completions of this upload.
### post-terminate
This event will be triggered after an upload has been terminated, meaning that the upload has been totally stopped and all associating chunks have been fully removed from the storage. Therefore, one is not able to retrieve the upload's content anymore and one may wish to notify further applications that this upload will never be resumed nor finished.
### post-receive
This event will be triggered for every running upload to indicate its current progress. It will occur for each open PATCH request, every second. The offset property will be set to the number of bytes which have been transfered to the server, at the time in total. Please be aware that this number may be higher than the number of bytes which have been stored by the data store!
## File Hooks
### The Hook Directory
By default, the file hook system is disabled. To enable it, pass the `--hooks-dir` option to the tusd binary. The flag's value will be a path, the **hook directory**, relative to the current working directory, pointing to the folder containing the executable **hook files**:
```bash
$ tusd --hook-dir ./path/to/hooks/
$ tusd --hooks-dir ./path/to/hooks/
[tusd] Using './path/to/hooks/' for hooks
[tusd] Using './data' as directory storage.
@ -18,7 +46,7 @@ $ tusd --hook-dir ./path/to/hooks/
If an event occurs, the tusd binary will look for a file, named exactly as the event, which will then be executed, as long as the object exists. In the example above, the binary `./path/to/hooks/pre-create` will be invoked, before an upload is created, which can be used to e.g. validate certain metadata. Please note, that the hook file *must not* have an extension, such as `.sh` or `.py`, or else tusd will not recognize and ignore it. A detailed list of all events can be found at the end of this document.
## The Hook's Environment
### The Hook's Environment
The process of the hook files are provided with information about the event and the upload using to two methods:
* The `TUS_ID` and `TUS_SIZE` environment variables will contain the upload ID and its size in bytes, which triggered the event. Please be aware, that in the `pre-create` hook the upload ID will be an empty string as the entity has not been created and therefore this piece of information is not yet available.
@ -52,26 +80,56 @@ The process of the hook files are provided with information about the event and
Be aware that this environment does *not* contain direct data from any HTTP request, in particular not any header values or cookies. If you would like to pass information from the client to the hook, such as authentication details, you may wish to use the [metadata system](http://tus.io/protocols/resumable-upload.html#upload-metadata).
## Blocking and Non-Blocking Hooks
If not otherwise noted, all hooks are invoked in a *non-blocking* way, meaning that tusd will not wait until the hook process has finished and exited. Therefore, the hook process is not able to influence how tusd may continue handling the current request, regardless of which exit code it may set. Furthermore, the hook process' stdout and stderr will be piped to tusd's stdout and stderr correspondingly, allowing one to use these channels for additional logging.
## HTTP Hooks
On the other hand, there are a few *blocking* hooks, such as caused by the `pre-create` event. Because their exit code will dictate whether tusd will accept the current incoming request, tusd will wait until the hook process has exited. Therefore, in order to keep the response times low, one should avoid to make time-consuming operations inside the processes for blocking hooks. An exit code of `0` indicates that tusd should continue handling the request as normal. On the other hand, a non-zero exit code tells tusd to reject the request with a `500 Internal Server Error` response containing the process' output from stderr. For the sake of logging, the process' output from stdout will always be piped to tusd's stdout.
HTTP Hooks are the second type of hooks supported by tusd. Like the file hooks, it is disabled by default. To enable it, pass the `--hooks-http` option to the tusd binary. The flag's value will be an HTTP URL endpoint, which the tusd binary will issue POST requests to:
## List of Available Hooks
```bash
$ tusd --hooks-http http://localhost:8081/write
### pre-create
[tusd] Using 'http://localhost:8081/write' as the endpoint for hooks
[tusd] Using './data' as directory storage.
...
```
This event will be triggered before an upload is created, allowing you to run certain routines. For example, validating that specific metadata values are set, or verifying that a corresponding entity belonging to the upload (e.g. a user) exists. Because this event will result in a blocking hook, you can determine whether the upload should be created or rejected using the exit code. An exit code of `0` will allow the upload to be created and continued as usual. A non-zero exit code will reject an upload creation request, making it a good place for authentication and authorization. Please be aware, that during this stage the upload ID will be an empty string as the entity has not been created and therefore this piece of information is not yet available.
Note that the URL must include the `http://` prefix!
### post-finish
### Usage
This event will be triggered after an upload is fully finished, meaning that all chunks have been transfered and saved in the storage. After this point, no further modifications, except possible deletion, can be made to the upload entity and it may be desirable to use the file for further processing or notify other applications of the completions of this upload.
Tusd will issue a `POST` request to the specified URL endpoint, specifying the hook name, such as pre-create or post-finish, in the `Hook-Name` header and following body:
### post-terminate
```js
{
// The upload's ID. Will be empty during the pre-create event
"ID": "14b1c4c77771671a8479bc0444bbc5ce",
// The upload's total size in bytes.
"Size": 46205,
// The upload's current offset in bytes.
"Offset": 1592,
// These properties will be set to true, if the upload as a final or partial
// one. See the Concatenation extension for details:
// http://tus.io/protocols/resumable-upload.html#concatenation
"IsFinal": false,
"IsPartial": false,
// If the upload is a final one, this value will be an array of upload IDs
// which are concatenated to produce the upload.
"PartialUploads": null,
// The upload's meta data which can be supplied by the clients as it wishes.
// All keys and values in this object will be strings.
// Be aware that it may contain maliciously crafted values and you must not
// trust it without escaping it first!
"MetaData": {
"filename": "transloadit.png"
}
}
```
This event will be triggered after an upload has been terminated, meaning that the upload has been totally stopped and all associating chunks have been fully removed from the storage. Therefore, one is not able to retrieve the upload's content anymore and one may wish to notify further applications that this upload will never be resumed nor finished.
### Configuration
### post-receive
Tusd uses the [Pester library](https://github.com/sethgrid/pester) to issue requests and handle retries. By default, tusd will retry 3 times on a `500 Internal Server Error` response or network error, with a 1 second backoff. This can be configured with the flags `--hooks-http-retry` and `--hooks-http-backoff`, like so:
This event will be triggered for every running upload to indicate its current progress. It will occur for each open PATCH request, every second. The offset property will be set to the number of bytes which have been transfered to the server, at the time in total. Please be aware that this number may be higher than the number of bytes which have been stored by the data store!
```
$ # Retrying 5 times with a 2 second backoff
$ tusd --hooks-http http://localhost:8081/write --hooks-http-retry 5 --hooks-http-backoff 2
```

21
vendor/github.com/sethgrid/pester/LICENSE.md generated vendored Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) SendGrid 2016
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

126
vendor/github.com/sethgrid/pester/README.md generated vendored Normal file
View File

@ -0,0 +1,126 @@
# pester
`pester` wraps Go's standard lib http client to provide several options to increase resiliency in your request. If you experience poor network conditions or requests could experience varied delays, you can now pester the endpoint for data.
- Send out multiple requests and get the first back (only used for GET calls)
- Retry on errors
- Backoff
### Simple Example
Use `pester` where you would use the http client calls. By default, pester will use a concurrency of 1, and retry the endpoint 3 times with the `DefaultBackoff` strategy of waiting 1 second between retries.
```go
/* swap in replacement, just switch
http.{Get|Post|PostForm|Head|Do} to
pester.{Get|Post|PostForm|Head|Do}
*/
resp, err := pester.Get("http://sethammons.com")
```
### Backoff Strategy
Provide your own backoff strategy, or use one of the provided built in strategies:
- `DefaultBackoff`: 1 second
- `LinearBackoff`: n seconds where n is the retry number
- `LinearJitterBackoff`: n seconds where n is the retry number, +/- 0-33%
- `ExponentialBackoff`: n seconds where n is 2^(retry number)
- `ExponentialJitterBackoff`: n seconds where n is 2^(retry number), +/- 0-33%
```go
client := pester.New()
client.Backoff = func(retry int) time.Duration {
// set up something dynamic or use a look up table
return time.Duration(retry) * time.Minute
}
```
### Complete example
For a complete and working example, see the sample directory.
`pester` allows you to use a constructor to control:
- backoff strategy
- reties
- concurrency
- keeping a log for debugging
```go
package main
import (
"log"
"net/http"
"strings"
"github.com/sethgrid/pester"
)
func main() {
log.Println("Starting...")
{ // drop in replacement for http.Get and other client methods
resp, err := pester.Get("http://example.com")
if err != nil {
log.Println("error GETing example.com", err)
}
defer resp.Body.Close()
log.Printf("example.com %s", resp.Status)
}
{ // control the resiliency
client := pester.New()
client.Concurrency = 3
client.MaxRetries = 5
client.Backoff = pester.ExponentialBackoff
client.KeepLog = true
resp, err := client.Get("http://example.com")
if err != nil {
log.Println("error GETing example.com", client.LogString())
}
defer resp.Body.Close()
log.Printf("example.com %s", resp.Status)
}
{ // use the pester version of http.Client.Do
req, err := http.NewRequest("POST", "http://example.com", strings.NewReader("data"))
if err != nil {
log.Fatal("Unable to create a new http request", err)
}
resp, err := pester.Do(req)
if err != nil {
log.Println("error POSTing example.com", err)
}
defer resp.Body.Close()
log.Printf("example.com %s", resp.Status)
}
}
```
### Example Log
`pester` also allows you to control the resiliency and can optionally log the errors.
```go
c := pester.New()
c.KeepLog = true
nonExistantURL := "http://localhost:9000/foo"
_, _ = c.Get(nonExistantURL)
fmt.Println(c.LogString())
/*
Output:
1432402837 Get [GET] http://localhost:9000/foo request-0 retry-0 error: Get http://localhost:9000/foo: dial tcp 127.0.0.1:9000: connection refused
1432402838 Get [GET] http://localhost:9000/foo request-0 retry-1 error: Get http://localhost:9000/foo: dial tcp 127.0.0.1:9000: connection refused
1432402839 Get [GET] http://localhost:9000/foo request-0 retry-2 error: Get http://localhost:9000/foo: dial tcp 127.0.0.1:9000: connection refused
*/
```
### Tests
You can run tests in the root directory with `$ go test`. There is a benchmark-like test available with `$ cd benchmarks; go test`.
You can see `pester` in action with `$ cd sample; go run main.go`.
For watching open file descriptors, you can run `watch "lsof -i -P | grep main"` if you started the app with `go run main.go`.
I did this for watching for FD leaks. My method was to alter `sample/main.go` to only run one case (`pester.Get with set backoff stategy, concurrency and retries increased`)
and adding a sleep after the result came back. This let me verify if FDs were getting left open when they should have closed. If you know a better way, let me know!
I was able to see that FDs are now closing when they should :)
![Are we there yet?](http://butchbellah.com/wp-content/uploads/2012/06/Are-We-There-Yet.jpg)
Are we there yet? Are we there yet? Are we there yet? Are we there yet? ...

423
vendor/github.com/sethgrid/pester/main.go generated vendored Normal file
View File

@ -0,0 +1,423 @@
package pester
// pester provides additional resiliency over the standard http client methods by
// allowing you to control concurrency, retries, and a backoff strategy.
import (
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
"math"
"math/rand"
"net/http"
"net/url"
"sync"
"time"
)
// Client wraps the http client and exposes all the functionality of the http.Client.
// Additionally, Client provides pester specific values for handling resiliency.
type Client struct {
// wrap it to provide access to http built ins
hc *http.Client
Transport http.RoundTripper
CheckRedirect func(req *http.Request, via []*http.Request) error
Jar http.CookieJar
Timeout time.Duration
// pester specific
Concurrency int
MaxRetries int
Backoff BackoffStrategy
KeepLog bool
SuccessReqNum int
SuccessRetryNum int
wg *sync.WaitGroup
sync.Mutex
ErrLog []ErrEntry
}
// ErrEntry is used to provide the LogString() data and is populated
// each time an error happens if KeepLog is set.
// ErrEntry.Retry is deprecated in favor of ErrEntry.Attempt
type ErrEntry struct {
Time time.Time
Method string
URL string
Verb string
Request int
Retry int
Attempt int
Err error
}
// result simplifies the channel communication for concurrent request handling
type result struct {
resp *http.Response
err error
req int
retry int
}
// params represents all the params needed to run http client calls and pester errors
type params struct {
method string
verb string
req *http.Request
url string
bodyType string
body io.Reader
data url.Values
}
// New constructs a new DefaultClient with sensible default values
func New() *Client {
return &Client{
Concurrency: DefaultClient.Concurrency,
MaxRetries: DefaultClient.MaxRetries,
Backoff: DefaultClient.Backoff,
ErrLog: DefaultClient.ErrLog,
wg: &sync.WaitGroup{},
}
}
// NewExtendedClient allows you to pass in an http.Client that is previously set up
// and extends it to have Pester's features of concurrency and retries.
func NewExtendedClient(hc *http.Client) *Client {
c := New()
c.hc = hc
return c
}
// BackoffStrategy is used to determine how long a retry request should wait until attempted
type BackoffStrategy func(retry int) time.Duration
// DefaultClient provides sensible defaults
var DefaultClient = &Client{Concurrency: 1, MaxRetries: 3, Backoff: DefaultBackoff, ErrLog: []ErrEntry{}}
// DefaultBackoff always returns 1 second
func DefaultBackoff(_ int) time.Duration {
return 1 * time.Second
}
// ExponentialBackoff returns ever increasing backoffs by a power of 2
func ExponentialBackoff(i int) time.Duration {
return time.Duration(math.Pow(2, float64(i))) * time.Second
}
// ExponentialJitterBackoff returns ever increasing backoffs by a power of 2
// with +/- 0-33% to prevent sychronized reuqests.
func ExponentialJitterBackoff(i int) time.Duration {
return jitter(int(math.Pow(2, float64(i))))
}
// LinearBackoff returns increasing durations, each a second longer than the last
func LinearBackoff(i int) time.Duration {
return time.Duration(i) * time.Second
}
// LinearJitterBackoff returns increasing durations, each a second longer than the last
// with +/- 0-33% to prevent sychronized reuqests.
func LinearJitterBackoff(i int) time.Duration {
return jitter(i)
}
// jitter keeps the +/- 0-33% logic in one place
func jitter(i int) time.Duration {
ms := i * 1000
maxJitter := ms / 3
rand.Seed(time.Now().Unix())
jitter := rand.Intn(maxJitter + 1)
if rand.Intn(2) == 1 {
ms = ms + jitter
} else {
ms = ms - jitter
}
// a jitter of 0 messes up the time.Tick chan
if ms <= 0 {
ms = 1
}
return time.Duration(ms) * time.Millisecond
}
// Wait blocks until all pester requests have returned
// Probably not that useful outside of testing.
func (c *Client) Wait() {
c.wg.Wait()
}
// pester provides all the logic of retries, concurrency, backoff, and logging
func (c *Client) pester(p params) (*http.Response, error) {
resultCh := make(chan result)
multiplexCh := make(chan result)
finishCh := make(chan struct{})
// track all requests that go out so we can close the late listener routine that closes late incoming response bodies
totalSentRequests := &sync.WaitGroup{}
totalSentRequests.Add(1)
defer totalSentRequests.Done()
allRequestsBackCh := make(chan struct{})
go func() {
totalSentRequests.Wait()
close(allRequestsBackCh)
}()
// GET calls should be idempotent and can make use
// of concurrency. Other verbs can mutate and should not
// make use of the concurrency feature
concurrency := c.Concurrency
if p.verb != "GET" {
concurrency = 1
}
c.Lock()
if c.hc == nil {
c.hc = &http.Client{}
c.hc.Transport = c.Transport
c.hc.CheckRedirect = c.CheckRedirect
c.hc.Jar = c.Jar
c.hc.Timeout = c.Timeout
}
c.Unlock()
// re-create the http client so we can leverage the std lib
httpClient := http.Client{
Transport: c.hc.Transport,
CheckRedirect: c.hc.CheckRedirect,
Jar: c.hc.Jar,
Timeout: c.hc.Timeout,
}
// if we have a request body, we need to save it for later
var originalRequestBody []byte
var originalBody []byte
var err error
if p.req != nil && p.req.Body != nil {
originalRequestBody, err = ioutil.ReadAll(p.req.Body)
if err != nil {
return &http.Response{}, errors.New("error reading request body")
}
p.req.Body.Close()
}
if p.body != nil {
originalBody, err = ioutil.ReadAll(p.body)
if err != nil {
return &http.Response{}, errors.New("error reading body")
}
}
AttemptLimit := c.MaxRetries
if AttemptLimit <= 0 {
AttemptLimit = 1
}
for req := 0; req < concurrency; req++ {
c.wg.Add(1)
totalSentRequests.Add(1)
go func(n int, p params) {
defer c.wg.Done()
defer totalSentRequests.Done()
var err error
for i := 1; i <= AttemptLimit; i++ {
c.wg.Add(1)
defer c.wg.Done()
select {
case <-finishCh:
return
default:
}
resp := &http.Response{}
// rehydrate the body (it is drained each read)
if len(originalRequestBody) > 0 {
p.req.Body = ioutil.NopCloser(bytes.NewBuffer(originalRequestBody))
}
if len(originalBody) > 0 {
p.body = bytes.NewBuffer(originalBody)
}
// route the calls
switch p.method {
case "Do":
resp, err = httpClient.Do(p.req)
case "Get":
resp, err = httpClient.Get(p.url)
case "Head":
resp, err = httpClient.Head(p.url)
case "Post":
resp, err = httpClient.Post(p.url, p.bodyType, p.body)
case "PostForm":
resp, err = httpClient.PostForm(p.url, p.data)
}
// Early return if we have a valid result
// Only retry (ie, continue the loop) on 5xx status codes
if err == nil && resp.StatusCode < 500 {
multiplexCh <- result{resp: resp, err: err, req: n, retry: i}
return
}
c.log(ErrEntry{
Time: time.Now(),
Method: p.method,
Verb: p.verb,
URL: p.url,
Request: n,
Retry: i + 1, // would remove, but would break backward compatibility
Attempt: i,
Err: err,
})
// if it is the last iteration, grab the result (which is an error at this point)
if i == AttemptLimit {
multiplexCh <- result{resp: resp, err: err}
return
}
// if we are retrying, we should close this response body to free the fd
if resp != nil {
resp.Body.Close()
}
// prevent a 0 from causing the tick to block, pass additional microsecond
<-time.After(c.Backoff(i) + 1*time.Microsecond)
}
}(req, p)
}
// spin off the go routine so it can continually listen in on late results and close the response bodies
go func() {
gotFirstResult := false
for {
select {
case res := <-multiplexCh:
if !gotFirstResult {
gotFirstResult = true
close(finishCh)
resultCh <- res
} else if res.resp != nil {
// we only return one result to the caller; close all other response bodies that come back
// drain the body before close as to not prevent keepalive. see https://gist.github.com/mholt/eba0f2cc96658be0f717
io.Copy(ioutil.Discard, res.resp.Body)
res.resp.Body.Close()
}
case <-allRequestsBackCh:
// don't leave this goroutine running
return
}
}
}()
select {
case res := <-resultCh:
c.Lock()
defer c.Unlock()
c.SuccessReqNum = res.req
c.SuccessRetryNum = res.retry
return res.resp, res.err
}
}
// LogString provides a string representation of the errors the client has seen
func (c *Client) LogString() string {
c.Lock()
defer c.Unlock()
var res string
for _, e := range c.ErrLog {
res += fmt.Sprintf("%d %s [%s] %s request-%d retry-%d error: %s\n",
e.Time.Unix(), e.Method, e.Verb, e.URL, e.Request, e.Retry, e.Err)
}
return res
}
// LogErrCount is a helper method used primarily for test validation
func (c *Client) LogErrCount() int {
c.Lock()
defer c.Unlock()
return len(c.ErrLog)
}
// EmbedHTTPClient allows you to extend an existing Pester client with an
// underlying http.Client, such as https://godoc.org/golang.org/x/oauth2/google#DefaultClient
func (c *Client) EmbedHTTPClient(hc *http.Client) {
c.hc = hc
}
func (c *Client) log(e ErrEntry) {
if c.KeepLog {
c.Lock()
c.ErrLog = append(c.ErrLog, e)
c.Unlock()
}
}
// Do provides the same functionality as http.Client.Do
func (c *Client) Do(req *http.Request) (resp *http.Response, err error) {
return c.pester(params{method: "Do", req: req, verb: req.Method, url: req.URL.String()})
}
// Get provides the same functionality as http.Client.Get
func (c *Client) Get(url string) (resp *http.Response, err error) {
return c.pester(params{method: "Get", url: url, verb: "GET"})
}
// Head provides the same functionality as http.Client.Head
func (c *Client) Head(url string) (resp *http.Response, err error) {
return c.pester(params{method: "Head", url: url, verb: "HEAD"})
}
// Post provides the same functionality as http.Client.Post
func (c *Client) Post(url string, bodyType string, body io.Reader) (resp *http.Response, err error) {
return c.pester(params{method: "Post", url: url, bodyType: bodyType, body: body, verb: "POST"})
}
// PostForm provides the same functionality as http.Client.PostForm
func (c *Client) PostForm(url string, data url.Values) (resp *http.Response, err error) {
return c.pester(params{method: "PostForm", url: url, data: data, verb: "POST"})
}
////////////////////////////////////////
// Provide self-constructing variants //
////////////////////////////////////////
// Do provides the same functionality as http.Client.Do and creates its own constructor
func Do(req *http.Request) (resp *http.Response, err error) {
c := New()
return c.Do(req)
}
// Get provides the same functionality as http.Client.Get and creates its own constructor
func Get(url string) (resp *http.Response, err error) {
c := New()
return c.Get(url)
}
// Head provides the same functionality as http.Client.Head and creates its own constructor
func Head(url string) (resp *http.Response, err error) {
c := New()
return c.Head(url)
}
// Post provides the same functionality as http.Client.Post and creates its own constructor
func Post(url string, bodyType string, body io.Reader) (resp *http.Response, err error) {
c := New()
return c.Post(url, bodyType, body)
}
// PostForm provides the same functionality as http.Client.PostForm and creates its own constructor
func PostForm(url string, data url.Values) (resp *http.Response, err error) {
c := New()
return c.PostForm(url, data)
}

6
vendor/vendor.json vendored
View File

@ -26,6 +26,12 @@
"revision": "792786c7400a136282c1664665ae0a8db921c6c2",
"revisionTime": "2016-01-10T10:55:54Z"
},
{
"checksumSHA1": "A8ptzv+SqJ1dAEwFZ2EDp+bwOeg=",
"path": "github.com/sethgrid/pester",
"revision": "2c5fb962da6113d0968907fd81dba3ca35151d1c",
"revisionTime": "2016-12-29T17:44:48Z"
},
{
"checksumSHA1": "JXUVA1jky8ZX8w09p2t5KLs97Nc=",
"path": "github.com/stretchr/testify/assert",