cli: Shutdown gracefully by default (#963)
* cli: Implement graceful shutdown * Make shutdown timeout configurable * Also shutdown gracefully for SIGTERM * Add comment * Add test for handler.InterruptRequestHandling * Add documentation
This commit is contained in:
parent
79f848e390
commit
3f042b97a0
|
@ -61,6 +61,7 @@ var Flags struct {
|
||||||
TLSCertFile string
|
TLSCertFile string
|
||||||
TLSKeyFile string
|
TLSKeyFile string
|
||||||
TLSMode string
|
TLSMode string
|
||||||
|
ShutdownTimeout int64
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseFlags() {
|
func ParseFlags() {
|
||||||
|
@ -115,6 +116,7 @@ func ParseFlags() {
|
||||||
flag.StringVar(&Flags.TLSCertFile, "tls-certificate", "", "Path to the file containing the x509 TLS certificate to be used. The file should also contain any intermediate certificates and the CA certificate.")
|
flag.StringVar(&Flags.TLSCertFile, "tls-certificate", "", "Path to the file containing the x509 TLS certificate to be used. The file should also contain any intermediate certificates and the CA certificate.")
|
||||||
flag.StringVar(&Flags.TLSKeyFile, "tls-key", "", "Path to the file containing the key for the TLS certificate.")
|
flag.StringVar(&Flags.TLSKeyFile, "tls-key", "", "Path to the file containing the key for the TLS certificate.")
|
||||||
flag.StringVar(&Flags.TLSMode, "tls-mode", "tls12", "Specify which TLS mode to use; valid modes are tls13, tls12, and tls12-strong.")
|
flag.StringVar(&Flags.TLSMode, "tls-mode", "tls12", "Specify which TLS mode to use; valid modes are tls13, tls12, and tls12-strong.")
|
||||||
|
flag.Int64Var(&Flags.ShutdownTimeout, "shutdown-timeout", 10*1000, "Timeout in milliseconds for closing connections gracefully during shutdown. After the timeout, tusd will exit regardless of any open connection.")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
SetEnabledHooks()
|
SetEnabledHooks()
|
||||||
|
|
|
@ -1,10 +1,15 @@
|
||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"crypto/tls"
|
"crypto/tls"
|
||||||
|
"errors"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
"strings"
|
"strings"
|
||||||
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/tus/tusd/v2/pkg/handler"
|
"github.com/tus/tusd/v2/pkg/handler"
|
||||||
|
@ -115,19 +120,19 @@ func Serve() {
|
||||||
stdout.Printf("You can now upload files to: %s://%s%s", protocol, address, basepath)
|
stdout.Printf("You can now upload files to: %s://%s%s", protocol, address, basepath)
|
||||||
}
|
}
|
||||||
|
|
||||||
// If we're not using TLS just start the server and, if http.Serve() returns, just return.
|
|
||||||
if protocol == "http" {
|
|
||||||
if err = http.Serve(listener, mux); err != nil {
|
|
||||||
stderr.Fatalf("Unable to serve: %s", err)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO: Move TLS handling into own file.
|
|
||||||
// Fall-through for TLS mode.
|
|
||||||
server := &http.Server{
|
server := &http.Server{
|
||||||
Handler: mux,
|
Handler: mux,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
shutdownComplete := setupSignalHandler(server, handler)
|
||||||
|
|
||||||
|
if protocol == "http" {
|
||||||
|
// Non-TLS mode
|
||||||
|
err = server.Serve(listener)
|
||||||
|
} else {
|
||||||
|
// TODO: Move TLS handling into own file.
|
||||||
|
// TLS mode
|
||||||
|
|
||||||
switch Flags.TLSMode {
|
switch Flags.TLSMode {
|
||||||
case TLS13:
|
case TLS13:
|
||||||
server.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS13}
|
server.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS13}
|
||||||
|
@ -170,7 +175,62 @@ func Serve() {
|
||||||
// Disable HTTP/2; the default non-TLS mode doesn't support it
|
// Disable HTTP/2; the default non-TLS mode doesn't support it
|
||||||
server.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler), 0)
|
server.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler), 0)
|
||||||
|
|
||||||
if err = server.ServeTLS(listener, Flags.TLSCertFile, Flags.TLSKeyFile); err != nil {
|
err = server.ServeTLS(listener, Flags.TLSCertFile, Flags.TLSKeyFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Note: http.Server.Serve and http.Server.ServeTLS always return a non-nil error code. So
|
||||||
|
// we can assume from here that `err != nil`
|
||||||
|
if err == http.ErrServerClosed {
|
||||||
|
// ErrServerClosed means that http.Server.Shutdown was called due to an interruption signal.
|
||||||
|
// We wait until the interruption procedure is complete or times out and then exit main.
|
||||||
|
<-shutdownComplete
|
||||||
|
} else {
|
||||||
|
// Any other error is relayed to the user.
|
||||||
stderr.Fatalf("Unable to serve: %s", err)
|
stderr.Fatalf("Unable to serve: %s", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func setupSignalHandler(server *http.Server, handler *handler.Handler) <-chan struct{} {
|
||||||
|
shutdownComplete := make(chan struct{})
|
||||||
|
|
||||||
|
// We read up to two signals, so use a capacity of 2 here to not miss any signal
|
||||||
|
c := make(chan os.Signal, 2)
|
||||||
|
|
||||||
|
// os.Interrupt is mapped to SIGINT on Unix and to the termination instructions on Windows.
|
||||||
|
// On Unix we also listen to SIGTERM.
|
||||||
|
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
|
||||||
|
|
||||||
|
// Signal to the handler that it should stop all long running requests if we shut down
|
||||||
|
server.RegisterOnShutdown(handler.InterruptRequestHandling)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
// First interrupt signal
|
||||||
|
<-c
|
||||||
|
stdout.Println("Received interrupt signal. Shutting down tusd...")
|
||||||
|
|
||||||
|
// Wait for second interrupt signal, while also shutting down the existing server
|
||||||
|
go func() {
|
||||||
|
<-c
|
||||||
|
stdout.Println("Received second interrupt signal. Exiting immediately!")
|
||||||
|
os.Exit(1)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Shutdown the server, but with a user-specified timeout
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(Flags.ShutdownTimeout)*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
err := server.Shutdown(ctx)
|
||||||
|
|
||||||
|
if err == nil {
|
||||||
|
stdout.Println("Shutdown completed. Goodbye!")
|
||||||
|
} else if errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
stderr.Println("Shutdown timeout exceeded. Exiting immediately!")
|
||||||
|
} else {
|
||||||
|
stderr.Printf("Failed to shutdown gracefully: %s\n", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
close(shutdownComplete)
|
||||||
|
}()
|
||||||
|
|
||||||
|
return shutdownComplete
|
||||||
|
}
|
||||||
|
|
|
@ -224,3 +224,13 @@ $ tusd -help
|
||||||
Print tusd version information
|
Print tusd version information
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Graceful shutdown
|
||||||
|
|
||||||
|
If tusd receives a SIGINT or SIGTERM signal, it will initiate a graceful shutdown. SIGINT is usually emitted by pressing Ctrl+C inside the terminal that is running tusd. SIGINT and SIGTERM can also be emitted using the [`kill(1)`](https://man7.org/linux/man-pages/man1/kill.1.html) utility on Unix. Signals in that sense do not exist on Windows, so please refer to the [Go documentation](https://pkg.go.dev/os/signal#hdr-Windows) on how different events are translated into signals on Windows.
|
||||||
|
|
||||||
|
Once the graceful shutdown is started, tusd will stop listening on its port and won't accept new connections anymore. Idle connections are closed down. Already running requests will be given a grace period to complete before their connections are closed as well. PATCH and POST requests with a request body are interrupted, so that data stores can gracefully finish saving all the received data until that point. If all requests have been completed, tusd will exit.
|
||||||
|
|
||||||
|
If not all requests have been completed in the period defined by the `-shutdown-timeout` flag, tusd will exit regardless. By default, tusd will give all requests 10 seconds to complete their processing. If you do not want to wait for requests, use `-shutdown-timeout=0`.
|
||||||
|
|
||||||
|
tusd will also immediately exit if it receives a second SIGINT or SIGTERM signal. It will also always exit immediately if a SIGKILL is received.
|
||||||
|
|
|
@ -31,6 +31,10 @@ func (r *bodyReader) Read(b []byte) (int, error) {
|
||||||
return 0, io.EOF
|
return 0, io.EOF
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO: Mask certain errors that we can safely ignore later on:
|
||||||
|
// io.EOF, io.UnexpectedEOF, io.ErrClosedPipe,
|
||||||
|
// read tcp 127.0.0.1:1080->127.0.0.1:56953: read: connection reset by peer,
|
||||||
|
// read tcp 127.0.0.1:1080->127.0.0.1:9375: i/o timeout
|
||||||
n, err := r.reader.Read(b)
|
n, err := r.reader.Read(b)
|
||||||
atomic.AddInt64(&r.bytesCounter, int64(n))
|
atomic.AddInt64(&r.bytesCounter, int64(n))
|
||||||
r.err = err
|
r.err = err
|
||||||
|
|
|
@ -682,4 +682,57 @@ func TestPatch(t *testing.T) {
|
||||||
ResBody: "ERR_INTERNAL_SERVER_ERROR: an error while reading the body\n",
|
ResBody: "ERR_INTERNAL_SERVER_ERROR: an error while reading the body\n",
|
||||||
}).Run(handler, t)
|
}).Run(handler, t)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
SubTest(t, "InterruptRequestHandling", func(t *testing.T, store *MockFullDataStore, composer *StoreComposer) {
|
||||||
|
ctrl := gomock.NewController(t)
|
||||||
|
defer ctrl.Finish()
|
||||||
|
upload := NewMockFullUpload(ctrl)
|
||||||
|
|
||||||
|
gomock.InOrder(
|
||||||
|
store.EXPECT().GetUpload(gomock.Any(), "yes").Return(upload, nil),
|
||||||
|
upload.EXPECT().GetInfo(gomock.Any()).Return(FileInfo{
|
||||||
|
ID: "yes",
|
||||||
|
Offset: 0,
|
||||||
|
Size: 100,
|
||||||
|
}, nil),
|
||||||
|
upload.EXPECT().WriteChunk(gomock.Any(), int64(0), NewReaderMatcher("first ")).Return(int64(6), nil),
|
||||||
|
)
|
||||||
|
|
||||||
|
handler, _ := NewHandler(Config{
|
||||||
|
StoreComposer: composer,
|
||||||
|
})
|
||||||
|
|
||||||
|
reader, writer := io.Pipe()
|
||||||
|
a := assert.New(t)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
writer.Write([]byte("first "))
|
||||||
|
|
||||||
|
handler.InterruptRequestHandling()
|
||||||
|
|
||||||
|
// Wait a short time to ensure that the goroutine in the PATCH
|
||||||
|
// handler has received and processed the stop event.
|
||||||
|
<-time.After(10 * time.Millisecond)
|
||||||
|
|
||||||
|
// Assert that the "request body" has been closed.
|
||||||
|
_, err := writer.Write([]byte("second "))
|
||||||
|
a.Equal(err, io.ErrClosedPipe)
|
||||||
|
}()
|
||||||
|
|
||||||
|
(&httpTest{
|
||||||
|
Method: "PATCH",
|
||||||
|
URL: "yes",
|
||||||
|
ReqHeader: map[string]string{
|
||||||
|
"Tus-Resumable": "1.0.0",
|
||||||
|
"Content-Type": "application/offset+octet-stream",
|
||||||
|
"Upload-Offset": "0",
|
||||||
|
},
|
||||||
|
ReqBody: reader,
|
||||||
|
Code: http.StatusInternalServerError,
|
||||||
|
ResHeader: map[string]string{
|
||||||
|
"Upload-Offset": "",
|
||||||
|
},
|
||||||
|
ResBody: "ERR_SERVER_SHUTDOWN: request has been interrupted because the server is shutting down\n",
|
||||||
|
}).Run(handler, t)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -43,6 +43,7 @@ var (
|
||||||
ErrUploadStoppedByServer = NewError("ERR_UPLOAD_STOPPED", "upload has been stopped by server", http.StatusBadRequest)
|
ErrUploadStoppedByServer = NewError("ERR_UPLOAD_STOPPED", "upload has been stopped by server", http.StatusBadRequest)
|
||||||
ErrUploadRejectedByServer = NewError("ERR_UPLOAD_REJECTED", "upload creation has been rejected by server", http.StatusBadRequest)
|
ErrUploadRejectedByServer = NewError("ERR_UPLOAD_REJECTED", "upload creation has been rejected by server", http.StatusBadRequest)
|
||||||
ErrUploadInterrupted = NewError("ERR_UPLAOD_INTERRUPTED", "upload has been interrupted by another request for this upload resource", http.StatusBadRequest)
|
ErrUploadInterrupted = NewError("ERR_UPLAOD_INTERRUPTED", "upload has been interrupted by another request for this upload resource", http.StatusBadRequest)
|
||||||
|
ErrServerShutdown = NewError("ERR_SERVER_SHUTDOWN", "request has been interrupted because the server is shutting down", http.StatusInternalServerError)
|
||||||
|
|
||||||
// TODO: These two responses are 500 for backwards compatability. We should discuss
|
// TODO: These two responses are 500 for backwards compatability. We should discuss
|
||||||
// whether it is better to more them to 4XX status codes.
|
// whether it is better to more them to 4XX status codes.
|
||||||
|
@ -60,6 +61,7 @@ type UnroutedHandler struct {
|
||||||
basePath string
|
basePath string
|
||||||
logger *log.Logger
|
logger *log.Logger
|
||||||
extensions string
|
extensions string
|
||||||
|
serverCtx chan struct{}
|
||||||
|
|
||||||
// CompleteUploads is used to send notifications whenever an upload is
|
// CompleteUploads is used to send notifications whenever an upload is
|
||||||
// completed by a user. The HookEvent will contain information about this
|
// completed by a user. The HookEvent will contain information about this
|
||||||
|
@ -126,11 +128,29 @@ func NewUnroutedHandler(config Config) (*UnroutedHandler, error) {
|
||||||
logger: config.Logger,
|
logger: config.Logger,
|
||||||
extensions: extensions,
|
extensions: extensions,
|
||||||
Metrics: newMetrics(),
|
Metrics: newMetrics(),
|
||||||
|
serverCtx: make(chan struct{}),
|
||||||
}
|
}
|
||||||
|
|
||||||
return handler, nil
|
return handler, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// InterruptRequestHandling attempts to interrupt long running requests, so
|
||||||
|
// the server can shutdown gracefully. This function should not be used on
|
||||||
|
// its own, but as part of http.Server.Shutdown. For example:
|
||||||
|
//
|
||||||
|
// server := &http.Server{
|
||||||
|
// Handler: handler,
|
||||||
|
// }
|
||||||
|
// server.RegisterOnShutdown(handler.InterruptRequestHandling)
|
||||||
|
// server.Shutdown(ctx)
|
||||||
|
//
|
||||||
|
// Note: currently, this function only interrupts POST and PATCH requests
|
||||||
|
// with a request body. In the future, this might be extended to HEAD, DELETE
|
||||||
|
// and GET requests.
|
||||||
|
func (handler UnroutedHandler) InterruptRequestHandling() {
|
||||||
|
close(handler.serverCtx)
|
||||||
|
}
|
||||||
|
|
||||||
// SupportedExtensions returns a comma-separated list of the supported tus extensions.
|
// SupportedExtensions returns a comma-separated list of the supported tus extensions.
|
||||||
// The availability of an extension usually depends on whether the provided data store
|
// The availability of an extension usually depends on whether the provided data store
|
||||||
// implements some additional interfaces.
|
// implements some additional interfaces.
|
||||||
|
@ -596,18 +616,28 @@ func (handler *UnroutedHandler) writeChunk(c *httpContext, resp HTTPResponse, up
|
||||||
// We use a context object to allow the hook system to cancel an upload
|
// We use a context object to allow the hook system to cancel an upload
|
||||||
uploadCtx, stopUpload := context.WithCancel(context.Background())
|
uploadCtx, stopUpload := context.WithCancel(context.Background())
|
||||||
info.stopUpload = stopUpload
|
info.stopUpload = stopUpload
|
||||||
|
|
||||||
// terminateUpload specifies whether the upload should be deleted after
|
// terminateUpload specifies whether the upload should be deleted after
|
||||||
// the write has finished
|
// the write has finished
|
||||||
terminateUpload := false
|
terminateUpload := false
|
||||||
|
|
||||||
|
serverShutDown := false
|
||||||
|
|
||||||
// Cancel the context when the function exits to ensure that the goroutine
|
// Cancel the context when the function exits to ensure that the goroutine
|
||||||
// is properly cleaned up
|
// is properly cleaned up
|
||||||
defer stopUpload()
|
defer stopUpload()
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
// Interrupt the Read() call from the request body
|
select {
|
||||||
<-uploadCtx.Done()
|
case <-uploadCtx.Done():
|
||||||
// TODO: Consider using CloseWithError function from BodyReader
|
// uploadCtx is done if the upload is stopped by a post-receive hook
|
||||||
terminateUpload = true
|
terminateUpload = true
|
||||||
|
case <-handler.serverCtx:
|
||||||
|
// serverCtx is closed if the server is being shut down
|
||||||
|
serverShutDown = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// interrupt the Read() calls from the request body
|
||||||
r.Body.Close()
|
r.Body.Close()
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
@ -639,6 +669,11 @@ func (handler *UnroutedHandler) writeChunk(c *httpContext, resp HTTPResponse, up
|
||||||
if terminateUpload {
|
if terminateUpload {
|
||||||
err = ErrUploadStoppedByServer
|
err = ErrUploadStoppedByServer
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If the server is closing down, send an error response indicating this.
|
||||||
|
if serverShutDown {
|
||||||
|
err = ErrServerShutdown
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
handler.log("ChunkWriteComplete", "id", id, "bytesWritten", i64toa(bytesWritten))
|
handler.log("ChunkWriteComplete", "id", id, "bytesWritten", i64toa(bytesWritten))
|
||||||
|
@ -1104,6 +1139,7 @@ func (handler *UnroutedHandler) lockUpload(c *httpContext, id string) (Lock, err
|
||||||
releaseLock := func() {
|
releaseLock := func() {
|
||||||
if c.body != nil {
|
if c.body != nil {
|
||||||
handler.log("UploadInterrupted", "id", id, "requestId", getRequestId(c.req))
|
handler.log("UploadInterrupted", "id", id, "requestId", getRequestId(c.req))
|
||||||
|
// TODO: Consider replacing this with a channel or a context
|
||||||
c.body.closeWithError(ErrUploadInterrupted)
|
c.body.closeWithError(ErrUploadInterrupted)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue