Take IsTruncated field of S3 ListParts API response into account (#148)
* Take IsTruncated field of S3 ListParts API response into account * Rename s3store.ListParts to ListAllParts * Use proper formatting + make listAllParts private + test listAllParts through TestGetInfo * Update TestFinishUpload to also test paged ListParts response
This commit is contained in:
parent
a51f5994bb
commit
1ad6187d6d
|
@ -226,17 +226,12 @@ func (store S3Store) WriteChunk(id string, offset int64, src io.Reader) (int64,
|
|||
bytesUploaded := int64(0)
|
||||
|
||||
// Get number of parts to generate next number
|
||||
listPtr, err := store.Service.ListParts(&s3.ListPartsInput{
|
||||
Bucket: aws.String(store.Bucket),
|
||||
Key: aws.String(uploadId),
|
||||
UploadId: aws.String(multipartId),
|
||||
})
|
||||
parts, err := store.listAllParts(id)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
list := *listPtr
|
||||
numParts := len(list.Parts)
|
||||
numParts := len(parts)
|
||||
nextPartNum := int64(numParts + 1)
|
||||
|
||||
for {
|
||||
|
@ -288,7 +283,7 @@ func (store S3Store) WriteChunk(id string, offset int64, src io.Reader) (int64,
|
|||
}
|
||||
|
||||
func (store S3Store) GetInfo(id string) (info tusd.FileInfo, err error) {
|
||||
uploadId, multipartId := splitIds(id)
|
||||
uploadId, _ := splitIds(id)
|
||||
|
||||
// Get file info stored in separate object
|
||||
res, err := store.Service.GetObject(&s3.GetObjectInput{
|
||||
|
@ -308,11 +303,7 @@ func (store S3Store) GetInfo(id string) (info tusd.FileInfo, err error) {
|
|||
}
|
||||
|
||||
// Get uploaded parts and their offset
|
||||
listPtr, err := store.Service.ListParts(&s3.ListPartsInput{
|
||||
Bucket: aws.String(store.Bucket),
|
||||
Key: aws.String(uploadId),
|
||||
UploadId: aws.String(multipartId),
|
||||
})
|
||||
parts, err := store.listAllParts(id)
|
||||
if err != nil {
|
||||
// Check if the error is caused by the upload not being found. This happens
|
||||
// when the multipart upload has already been completed or aborted. Since
|
||||
|
@ -326,11 +317,9 @@ func (store S3Store) GetInfo(id string) (info tusd.FileInfo, err error) {
|
|||
}
|
||||
}
|
||||
|
||||
list := *listPtr
|
||||
|
||||
offset := int64(0)
|
||||
|
||||
for _, part := range list.Parts {
|
||||
for _, part := range parts {
|
||||
offset += *part.Size
|
||||
}
|
||||
|
||||
|
@ -444,22 +433,17 @@ func (store S3Store) FinishUpload(id string) error {
|
|||
uploadId, multipartId := splitIds(id)
|
||||
|
||||
// Get uploaded parts
|
||||
listPtr, err := store.Service.ListParts(&s3.ListPartsInput{
|
||||
Bucket: aws.String(store.Bucket),
|
||||
Key: aws.String(uploadId),
|
||||
UploadId: aws.String(multipartId),
|
||||
})
|
||||
parts, err := store.listAllParts(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Transform the []*s3.Part slice to a []*s3.CompletedPart slice for the next
|
||||
// request.
|
||||
list := *listPtr
|
||||
parts := make([]*s3.CompletedPart, len(list.Parts))
|
||||
completedParts := make([]*s3.CompletedPart, len(parts))
|
||||
|
||||
for index, part := range list.Parts {
|
||||
parts[index] = &s3.CompletedPart{
|
||||
for index, part := range parts {
|
||||
completedParts[index] = &s3.CompletedPart{
|
||||
ETag: part.ETag,
|
||||
PartNumber: part.PartNumber,
|
||||
}
|
||||
|
@ -470,7 +454,7 @@ func (store S3Store) FinishUpload(id string) error {
|
|||
Key: aws.String(uploadId),
|
||||
UploadId: aws.String(multipartId),
|
||||
MultipartUpload: &s3.CompletedMultipartUpload{
|
||||
Parts: parts,
|
||||
Parts: completedParts,
|
||||
},
|
||||
})
|
||||
|
||||
|
@ -517,6 +501,33 @@ func (store S3Store) ConcatUploads(dest string, partialUploads []string) error {
|
|||
return store.FinishUpload(dest)
|
||||
}
|
||||
|
||||
func (store S3Store) listAllParts(id string) (parts []*s3.Part, err error) {
|
||||
uploadId, multipartId := splitIds(id)
|
||||
|
||||
partMarker := int64(0)
|
||||
for {
|
||||
// Get uploaded parts
|
||||
listPtr, err := store.Service.ListParts(&s3.ListPartsInput{
|
||||
Bucket: aws.String(store.Bucket),
|
||||
Key: aws.String(uploadId),
|
||||
UploadId: aws.String(multipartId),
|
||||
PartNumberMarker: aws.Int64(partMarker),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
parts = append(parts, (*listPtr).Parts...)
|
||||
|
||||
if listPtr.IsTruncated != nil && *listPtr.IsTruncated {
|
||||
partMarker = *listPtr.NextPartNumberMarker
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
return parts, nil
|
||||
}
|
||||
|
||||
func splitIds(id string) (uploadId, multipartId string) {
|
||||
index := strings.Index(id, "+")
|
||||
if index == -1 {
|
||||
|
|
|
@ -107,6 +107,7 @@ func TestGetInfo(t *testing.T) {
|
|||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("uploadId"),
|
||||
UploadId: aws.String("multipartId"),
|
||||
PartNumberMarker: aws.Int64(0),
|
||||
}).Return(&s3.ListPartsOutput{
|
||||
Parts: []*s3.Part{
|
||||
{
|
||||
|
@ -116,13 +117,27 @@ func TestGetInfo(t *testing.T) {
|
|||
Size: aws.Int64(200),
|
||||
},
|
||||
},
|
||||
NextPartNumberMarker: aws.Int64(2),
|
||||
IsTruncated: aws.Bool(true),
|
||||
}, nil),
|
||||
s3obj.EXPECT().ListParts(&s3.ListPartsInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("uploadId"),
|
||||
UploadId: aws.String("multipartId"),
|
||||
PartNumberMarker: aws.Int64(2),
|
||||
}).Return(&s3.ListPartsOutput{
|
||||
Parts: []*s3.Part{
|
||||
{
|
||||
Size: aws.Int64(100),
|
||||
},
|
||||
},
|
||||
}, nil),
|
||||
)
|
||||
|
||||
info, err := store.GetInfo("uploadId+multipartId")
|
||||
assert.Nil(err)
|
||||
assert.Equal(int64(500), info.Size)
|
||||
assert.Equal(int64(300), info.Offset)
|
||||
assert.Equal(int64(400), info.Offset)
|
||||
assert.Equal("uploadId+multipartId", info.ID)
|
||||
assert.Equal("hello", info.MetaData["foo"])
|
||||
assert.Equal("menü", info.MetaData["bar"])
|
||||
|
@ -147,6 +162,7 @@ func TestGetInfoFinished(t *testing.T) {
|
|||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("uploadId"),
|
||||
UploadId: aws.String("multipartId"),
|
||||
PartNumberMarker: aws.Int64(0),
|
||||
}).Return(nil, awserr.New("NoSuchUpload", "The specified upload does not exist.", nil)),
|
||||
)
|
||||
|
||||
|
@ -243,6 +259,7 @@ func TestFinishUpload(t *testing.T) {
|
|||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("uploadId"),
|
||||
UploadId: aws.String("multipartId"),
|
||||
PartNumberMarker: aws.Int64(0),
|
||||
}).Return(&s3.ListPartsOutput{
|
||||
Parts: []*s3.Part{
|
||||
{
|
||||
|
@ -256,6 +273,22 @@ func TestFinishUpload(t *testing.T) {
|
|||
PartNumber: aws.Int64(2),
|
||||
},
|
||||
},
|
||||
NextPartNumberMarker: aws.Int64(2),
|
||||
IsTruncated: aws.Bool(true),
|
||||
}, nil),
|
||||
s3obj.EXPECT().ListParts(&s3.ListPartsInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("uploadId"),
|
||||
UploadId: aws.String("multipartId"),
|
||||
PartNumberMarker: aws.Int64(2),
|
||||
}).Return(&s3.ListPartsOutput{
|
||||
Parts: []*s3.Part{
|
||||
{
|
||||
Size: aws.Int64(100),
|
||||
ETag: aws.String("foobar"),
|
||||
PartNumber: aws.Int64(3),
|
||||
},
|
||||
},
|
||||
}, nil),
|
||||
s3obj.EXPECT().CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
|
@ -271,6 +304,10 @@ func TestFinishUpload(t *testing.T) {
|
|||
ETag: aws.String("bar"),
|
||||
PartNumber: aws.Int64(2),
|
||||
},
|
||||
{
|
||||
ETag: aws.String("foobar"),
|
||||
PartNumber: aws.Int64(3),
|
||||
},
|
||||
},
|
||||
},
|
||||
}).Return(nil, nil),
|
||||
|
@ -301,6 +338,7 @@ func TestWriteChunk(t *testing.T) {
|
|||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("uploadId"),
|
||||
UploadId: aws.String("multipartId"),
|
||||
PartNumberMarker: aws.Int64(0),
|
||||
}).Return(&s3.ListPartsOutput{
|
||||
Parts: []*s3.Part{
|
||||
{
|
||||
|
@ -315,6 +353,7 @@ func TestWriteChunk(t *testing.T) {
|
|||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("uploadId"),
|
||||
UploadId: aws.String("multipartId"),
|
||||
PartNumberMarker: aws.Int64(0),
|
||||
}).Return(&s3.ListPartsOutput{
|
||||
Parts: []*s3.Part{
|
||||
{
|
||||
|
@ -372,6 +411,7 @@ func TestWriteChunkDropTooSmall(t *testing.T) {
|
|||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("uploadId"),
|
||||
UploadId: aws.String("multipartId"),
|
||||
PartNumberMarker: aws.Int64(0),
|
||||
}).Return(&s3.ListPartsOutput{
|
||||
Parts: []*s3.Part{
|
||||
{
|
||||
|
@ -386,6 +426,7 @@ func TestWriteChunkDropTooSmall(t *testing.T) {
|
|||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("uploadId"),
|
||||
UploadId: aws.String("multipartId"),
|
||||
PartNumberMarker: aws.Int64(0),
|
||||
}).Return(&s3.ListPartsOutput{
|
||||
Parts: []*s3.Part{
|
||||
{
|
||||
|
@ -423,6 +464,7 @@ func TestWriteChunkAllowTooSmallLast(t *testing.T) {
|
|||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("uploadId"),
|
||||
UploadId: aws.String("multipartId"),
|
||||
PartNumberMarker: aws.Int64(0),
|
||||
}).Return(&s3.ListPartsOutput{
|
||||
Parts: []*s3.Part{
|
||||
{
|
||||
|
@ -437,6 +479,7 @@ func TestWriteChunkAllowTooSmallLast(t *testing.T) {
|
|||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("uploadId"),
|
||||
UploadId: aws.String("multipartId"),
|
||||
PartNumberMarker: aws.Int64(0),
|
||||
}).Return(&s3.ListPartsOutput{
|
||||
Parts: []*s3.Part{
|
||||
{
|
||||
|
@ -579,6 +622,7 @@ func TestConcatUploads(t *testing.T) {
|
|||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("uploadId"),
|
||||
UploadId: aws.String("multipartId"),
|
||||
PartNumberMarker: aws.Int64(0),
|
||||
}).Return(&s3.ListPartsOutput{
|
||||
Parts: []*s3.Part{
|
||||
{
|
||||
|
|
Loading…
Reference in New Issue