From b72a4d43d6c05d11fa9494b068cf465761e00bcf Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Mon, 16 Mar 2020 14:05:10 -0400 Subject: [PATCH] Remove completed chunk files inline to reduce disk space usage --- pkg/s3store/s3store.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/s3store/s3store.go b/pkg/s3store/s3store.go index 7b59956..7a2f572 100644 --- a/pkg/s3store/s3store.go +++ b/pkg/s3store/s3store.go @@ -396,9 +396,6 @@ func (upload s3Upload) WriteChunk(ctx context.Context, offset int64, src io.Read go chunkProducer.produce(optimalPartSize) for file := range fileChan { - defer os.Remove(file.Name()) - defer file.Close() - stat, err := file.Stat() if err != nil { return 0, err @@ -433,6 +430,13 @@ func (upload s3Upload) WriteChunk(ctx context.Context, offset int64, src io.Read offset += n bytesUploaded += n nextPartNum += 1 + + if err := os.Remove(file.Name()); err != nil { + return bytesUploaded, err + } + if err := file.Close(); err != nil { + return bytesUploaded, err + } } return bytesUploaded - incompletePartSize, chunkProducer.err