qugalet/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-02-04 20:05:05 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[chore]: Bump github.com/minio/minio-go/v7 from 7.0.81 to 7.0.84 (#3728)

Browse source 
Bumps [github.com/minio/minio-go/v7](https://github.com/minio/minio-go) from 7.0.81 to 7.0.84.
- [Release notes](https://github.com/minio/minio-go/releases)
- [Commits](https://github.com/minio/minio-go/compare/v7.0.81...v7.0.84)

---
updated-dependencies:
- dependency-name: github.com/minio/minio-go/v7
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
This commit is contained in:
dependabot[bot] 2025-02-03 10:39:40 +00:00 committed by GitHub
parent 27844b7da2
commit acd3e80ae1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
32 changed files with 975 additions and 302 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -61,7 +61,7 @@ require (
github.com/k3a/html2text v1.2.1
github.com/microcosm-cc/bluemonday v1.0.27
github.com/miekg/dns v1.1.63
github.com/minio/minio-go/v7 v7.0.81
github.com/minio/minio-go/v7 v7.0.84
github.com/mitchellh/mapstructure v1.5.0
github.com/ncruces/go-sqlite3 v0.22.0
github.com/oklog/ulid v1.3.1

4
go.sum generated
View file

@ -403,8 +403,8 @@ github.com/miekg/dns v1.1.63 h1:8M5aAw6OMZfFXTT7K5V0Eu5YiiL8l7nUAkyN6C9YwaY=
github.com/miekg/dns v1.1.63/go.mod h1:6NGHfjhpmr5lt3XPLuyfDJi5AXbNIPM9PY6H6sF1Nfs=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.81 h1:SzhMN0TQ6T/xSBu6Nvw3M5M8voM+Ht8RH3hE8S7zxaA=
github.com/minio/minio-go/v7 v7.0.81/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
github.com/minio/minio-go/v7 v7.0.84 h1:D1HVmAF8JF8Bpi6IU4V9vIEj+8pc+xU88EWMs2yed0E=
github.com/minio/minio-go/v7 v7.0.84/go.mod h1:57YXpvc5l3rjPdhqNrDsvVlY0qPI6UTk1bflAe+9doY=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=

2
vendor/github.com/minio/minio-go/v7/api-copy-object.go generated vendored
View file

@ -68,7 +68,7 @@ func (c *Client) CopyObject(ctx context.Context, dst CopyDestOptions, src CopySr
Bucket: dst.Bucket,
Key: dst.Object,
LastModified: cpObjRes.LastModified,
ETag: trimEtag(resp.Header.Get("ETag")),
ETag: trimEtag(cpObjRes.ETag),
VersionID: resp.Header.Get(amzVersionID),
Expiration: expTime,
ExpirationRuleID: ruleID,

2
vendor/github.com/minio/minio-go/v7/api-datatypes.go generated vendored
View file

@ -147,6 +147,7 @@ type UploadInfo struct {
ChecksumCRC32C string
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC64NVME string
}
// RestoreInfo contains information of the restore operation of an archived object
@ -219,6 +220,7 @@ type ObjectInfo struct {
ChecksumCRC32C string
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC64NVME string
Internal *struct {
K int // Data blocks

4
vendor/github.com/minio/minio-go/v7/api-get-object.go generated vendored
View file

@ -318,7 +318,7 @@ func (o *Object) doGetRequest(request getRequest) (getResponse, error) {
response := <-o.resCh
// Return any error to the top level.
if response.Error != nil {
if response.Error != nil && response.Error != io.EOF {
return response, response.Error
}
@ -340,7 +340,7 @@ func (o *Object) doGetRequest(request getRequest) (getResponse, error) {
// Data are ready on the wire, no need to reinitiate connection in lower level
o.seekData = false
return response, nil
return response, response.Error
}
// setOffset - handles the setting of offsets for

2
vendor/github.com/minio/minio-go/v7/api-presigned.go generated vendored
View file

@ -140,7 +140,7 @@ func (c *Client) PresignedPostPolicy(ctx context.Context, p *PostPolicy) (u *url
}
// Get credentials from the configured credentials provider.
credValues, err := c.credsProvider.Get()
credValues, err := c.credsProvider.GetWithContext(c.CredContext())
if err != nil {
return nil, nil, err
}

28
vendor/github.com/minio/minio-go/v7/api-put-object-multipart.go generated vendored
View file

@ -83,10 +83,7 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj
// HTTPS connection.
hashAlgos, hashSums := c.hashMaterials(opts.SendContentMd5, !opts.DisableContentSha256)
if len(hashSums) == 0 {
if opts.UserMetadata == nil {
opts.UserMetadata = make(map[string]string, 1)
}
opts.UserMetadata["X-Amz-Checksum-Algorithm"] = opts.AutoChecksum.String()
addAutoChecksumHeaders(&opts)
}
// Initiate a new multipart upload.
@ -113,7 +110,6 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj
// Create checksums
// CRC32C is ~50% faster on AMD64 @ 30GB/s
var crcBytes []byte
customHeader := make(http.Header)
crc := opts.AutoChecksum.Hasher()
for partNumber <= totalPartsCount {
@ -154,7 +150,6 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj
crc.Write(buf[:length])
cSum := crc.Sum(nil)
customHeader.Set(opts.AutoChecksum.Key(), base64.StdEncoding.EncodeToString(cSum))
crcBytes = append(crcBytes, cSum...)
}
p := uploadPartParams{bucketName: bucketName, objectName: objectName, uploadID: uploadID, reader: rd, partNumber: partNumber, md5Base64: md5Base64, sha256Hex: sha256Hex, size: int64(length), sse: opts.ServerSideEncryption, streamSha256: !opts.DisableContentSha256, customHeader: customHeader}
@ -182,11 +177,13 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj
// Loop over total uploaded parts to save them in
// Parts array before completing the multipart request.
allParts := make([]ObjectPart, 0, len(partsInfo))
for i := 1; i < partNumber; i++ {
part, ok := partsInfo[i]
if !ok {
return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Missing part number %d", i))
}
allParts = append(allParts, part)
complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{
ETag: part.ETag,
PartNumber: part.PartNumber,
@ -194,6 +191,7 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj
ChecksumCRC32C: part.ChecksumCRC32C,
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
})
}
@ -203,12 +201,8 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj
ServerSideEncryption: opts.ServerSideEncryption,
AutoChecksum: opts.AutoChecksum,
}
if len(crcBytes) > 0 {
// Add hash of hashes.
crc.Reset()
crc.Write(crcBytes)
opts.UserMetadata = map[string]string{opts.AutoChecksum.Key(): base64.StdEncoding.EncodeToString(crc.Sum(nil))}
}
applyAutoChecksum(&opts, allParts)
uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, opts)
if err != nil {
return UploadInfo{}, err
@ -354,10 +348,11 @@ func (c *Client) uploadPart(ctx context.Context, p uploadPartParams) (ObjectPart
// Once successfully uploaded, return completed part.
h := resp.Header
objPart := ObjectPart{
ChecksumCRC32: h.Get("x-amz-checksum-crc32"),
ChecksumCRC32C: h.Get("x-amz-checksum-crc32c"),
ChecksumSHA1: h.Get("x-amz-checksum-sha1"),
ChecksumSHA256: h.Get("x-amz-checksum-sha256"),
ChecksumCRC32: h.Get(ChecksumCRC32.Key()),
ChecksumCRC32C: h.Get(ChecksumCRC32C.Key()),
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
}
objPart.Size = p.size
objPart.PartNumber = p.partNumber
@ -461,5 +456,6 @@ func (c *Client) completeMultipartUpload(ctx context.Context, bucketName, object
ChecksumSHA1: completeMultipartUploadResult.ChecksumSHA1,
ChecksumCRC32: completeMultipartUploadResult.ChecksumCRC32,
ChecksumCRC32C: completeMultipartUploadResult.ChecksumCRC32C,
ChecksumCRC64NVME: completeMultipartUploadResult.ChecksumCRC64NVME,
}, nil
}

63
vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go generated vendored
View file

@ -113,10 +113,7 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
}
withChecksum := c.trailingHeaderSupport
if withChecksum {
if opts.UserMetadata == nil {
opts.UserMetadata = make(map[string]string, 1)
}
opts.UserMetadata["X-Amz-Checksum-Algorithm"] = opts.AutoChecksum.String()
addAutoChecksumHeaders(&opts)
}
// Initiate a new multipart upload.
uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts)
@ -240,6 +237,7 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
// Gather the responses as they occur and update any
// progress bar.
allParts := make([]ObjectPart, 0, totalPartsCount)
for u := 1; u <= totalPartsCount; u++ {
select {
case <-ctx.Done():
@ -248,7 +246,7 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
if uploadRes.Error != nil {
return UploadInfo{}, uploadRes.Error
}
allParts = append(allParts, uploadRes.Part)
// Update the totalUploadedSize.
totalUploadedSize += uploadRes.Size
complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{
@ -258,6 +256,7 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
ChecksumCRC32C: uploadRes.Part.ChecksumCRC32C,
ChecksumSHA1: uploadRes.Part.ChecksumSHA1,
ChecksumSHA256: uploadRes.Part.ChecksumSHA256,
ChecksumCRC64NVME: uploadRes.Part.ChecksumCRC64NVME,
})
}
}
@ -275,15 +274,7 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
AutoChecksum: opts.AutoChecksum,
}
if withChecksum {
// Add hash of hashes.
crc := opts.AutoChecksum.Hasher()
for _, part := range complMultipartUpload.Parts {
cs, err := base64.StdEncoding.DecodeString(part.Checksum(opts.AutoChecksum))
if err == nil {
crc.Write(cs)
}
}
opts.UserMetadata = map[string]string{opts.AutoChecksum.KeyCapitalized(): base64.StdEncoding.EncodeToString(crc.Sum(nil))}
applyAutoChecksum(&opts, allParts)
}
uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, opts)
@ -312,10 +303,7 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
}
if !opts.SendContentMd5 {
if opts.UserMetadata == nil {
opts.UserMetadata = make(map[string]string, 1)
}
opts.UserMetadata["X-Amz-Checksum-Algorithm"] = opts.AutoChecksum.String()
addAutoChecksumHeaders(&opts)
}
// Calculate the optimal parts info for a given size.
@ -342,7 +330,6 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
// Create checksums
// CRC32C is ~50% faster on AMD64 @ 30GB/s
var crcBytes []byte
customHeader := make(http.Header)
crc := opts.AutoChecksum.Hasher()
md5Hash := c.md5Hasher()
@ -389,7 +376,6 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
crc.Write(buf[:length])
cSum := crc.Sum(nil)
customHeader.Set(opts.AutoChecksum.KeyCapitalized(), base64.StdEncoding.EncodeToString(cSum))
crcBytes = append(crcBytes, cSum...)
}
// Update progress reader appropriately to the latest offset
@ -420,11 +406,13 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
// Loop over total uploaded parts to save them in
// Parts array before completing the multipart request.
allParts := make([]ObjectPart, 0, len(partsInfo))
for i := 1; i < partNumber; i++ {
part, ok := partsInfo[i]
if !ok {
return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Missing part number %d", i))
}
allParts = append(allParts, part)
complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{
ETag: part.ETag,
PartNumber: part.PartNumber,
@ -432,6 +420,7 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
ChecksumCRC32C: part.ChecksumCRC32C,
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
})
}
@ -442,12 +431,7 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
ServerSideEncryption: opts.ServerSideEncryption,
AutoChecksum: opts.AutoChecksum,
}
if len(crcBytes) > 0 {
// Add hash of hashes.
crc.Reset()
crc.Write(crcBytes)
opts.UserMetadata = map[string]string{opts.AutoChecksum.KeyCapitalized(): base64.StdEncoding.EncodeToString(crc.Sum(nil))}
}
applyAutoChecksum(&opts, allParts)
uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, opts)
if err != nil {
return UploadInfo{}, err
@ -475,10 +459,7 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
opts.AutoChecksum = opts.Checksum
}
if !opts.SendContentMd5 {
if opts.UserMetadata == nil {
opts.UserMetadata = make(map[string]string, 1)
}
opts.UserMetadata["X-Amz-Checksum-Algorithm"] = opts.AutoChecksum.String()
addAutoChecksumHeaders(&opts)
}
// Cancel all when an error occurs.
@ -510,7 +491,6 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
// Create checksums
// CRC32C is ~50% faster on AMD64 @ 30GB/s
var crcBytes []byte
crc := opts.AutoChecksum.Hasher()
// Total data read and written to server. should be equal to 'size' at the end of the call.
@ -570,7 +550,6 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
crc.Write(buf[:length])
cSum := crc.Sum(nil)
customHeader.Set(opts.AutoChecksum.Key(), base64.StdEncoding.EncodeToString(cSum))
crcBytes = append(crcBytes, cSum...)
}
wg.Add(1)
@ -630,11 +609,13 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
// Loop over total uploaded parts to save them in
// Parts array before completing the multipart request.
allParts := make([]ObjectPart, 0, len(partsInfo))
for i := 1; i < partNumber; i++ {
part, ok := partsInfo[i]
if !ok {
return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Missing part number %d", i))
}
allParts = append(allParts, part)
complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{
ETag: part.ETag,
PartNumber: part.PartNumber,
@ -642,6 +623,7 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
ChecksumCRC32C: part.ChecksumCRC32C,
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
})
}
@ -652,12 +634,8 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
ServerSideEncryption: opts.ServerSideEncryption,
AutoChecksum: opts.AutoChecksum,
}
if len(crcBytes) > 0 {
// Add hash of hashes.
crc.Reset()
crc.Write(crcBytes)
opts.UserMetadata = map[string]string{opts.AutoChecksum.KeyCapitalized(): base64.StdEncoding.EncodeToString(crc.Sum(nil))}
}
applyAutoChecksum(&opts, allParts)
uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, opts)
if err != nil {
return UploadInfo{}, err
@ -823,9 +801,10 @@ func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string,
ExpirationRuleID: ruleID,
// Checksum values
ChecksumCRC32: h.Get("x-amz-checksum-crc32"),
ChecksumCRC32C: h.Get("x-amz-checksum-crc32c"),
ChecksumSHA1: h.Get("x-amz-checksum-sha1"),
ChecksumSHA256: h.Get("x-amz-checksum-sha256"),
ChecksumCRC32: h.Get(ChecksumCRC32.Key()),
ChecksumCRC32C: h.Get(ChecksumCRC32C.Key()),
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
}, nil
}

18
vendor/github.com/minio/minio-go/v7/api-put-object.go generated vendored
View file

@ -387,10 +387,7 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam
opts.AutoChecksum = opts.Checksum
}
if !opts.SendContentMd5 {
if opts.UserMetadata == nil {
opts.UserMetadata = make(map[string]string, 1)
}
opts.UserMetadata["X-Amz-Checksum-Algorithm"] = opts.AutoChecksum.String()
addAutoChecksumHeaders(&opts)
}
// Initiate a new multipart upload.
@ -417,7 +414,6 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam
// Create checksums
// CRC32C is ~50% faster on AMD64 @ 30GB/s
var crcBytes []byte
customHeader := make(http.Header)
crc := opts.AutoChecksum.Hasher()
@ -443,7 +439,6 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam
crc.Write(buf[:length])
cSum := crc.Sum(nil)
customHeader.Set(opts.AutoChecksum.Key(), base64.StdEncoding.EncodeToString(cSum))
crcBytes = append(crcBytes, cSum...)
}
// Update progress reader appropriately to the latest offset
@ -475,11 +470,13 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam
// Loop over total uploaded parts to save them in
// Parts array before completing the multipart request.
allParts := make([]ObjectPart, 0, len(partsInfo))
for i := 1; i < partNumber; i++ {
part, ok := partsInfo[i]
if !ok {
return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Missing part number %d", i))
}
allParts = append(allParts, part)
complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{
ETag: part.ETag,
PartNumber: part.PartNumber,
@ -487,6 +484,7 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam
ChecksumCRC32C: part.ChecksumCRC32C,
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
})
}
@ -497,12 +495,8 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam
ServerSideEncryption: opts.ServerSideEncryption,
AutoChecksum: opts.AutoChecksum,
}
if len(crcBytes) > 0 {
// Add hash of hashes.
crc.Reset()
crc.Write(crcBytes)
opts.UserMetadata = map[string]string{opts.AutoChecksum.KeyCapitalized(): base64.StdEncoding.EncodeToString(crc.Sum(nil))}
}
applyAutoChecksum(&opts, allParts)
uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, opts)
if err != nil {
return UploadInfo{}, err

46
vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go generated vendored
View file

@ -18,6 +18,7 @@
package minio
import (
"encoding/base64"
"encoding/xml"
"errors"
"io"
@ -280,6 +281,41 @@ type ObjectPart struct {
ChecksumCRC32C string
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC64NVME string
}
// Checksum will return the checksum for the given type.
// Will return the empty string if not set.
func (c ObjectPart) Checksum(t ChecksumType) string {
switch {
case t.Is(ChecksumCRC32C):
return c.ChecksumCRC32C
case t.Is(ChecksumCRC32):
return c.ChecksumCRC32
case t.Is(ChecksumSHA1):
return c.ChecksumSHA1
case t.Is(ChecksumSHA256):
return c.ChecksumSHA256
case t.Is(ChecksumCRC64NVME):
return c.ChecksumCRC64NVME
}
return ""
}
// ChecksumRaw returns the decoded checksum from the part.
func (c ObjectPart) ChecksumRaw(t ChecksumType) ([]byte, error) {
b64 := c.Checksum(t)
if b64 == "" {
return nil, errors.New("no checksum set")
}
decoded, err := base64.StdEncoding.DecodeString(b64)
if err != nil {
return nil, err
}
if len(decoded) != t.RawByteLen() {
return nil, errors.New("checksum length mismatch")
}
return decoded, nil
}
// ListObjectPartsResult container for ListObjectParts response.
@ -296,6 +332,12 @@ type ListObjectPartsResult struct {
NextPartNumberMarker int
MaxParts int
// ChecksumAlgorithm will be CRC32, CRC32C, etc.
ChecksumAlgorithm string
// ChecksumType is FULL_OBJECT or COMPOSITE (assume COMPOSITE when unset)
ChecksumType string
// Indicates whether the returned list of parts is truncated.
IsTruncated bool
ObjectParts []ObjectPart `xml:"Part"`
@ -324,6 +366,7 @@ type completeMultipartUploadResult struct {
ChecksumCRC32C string
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC64NVME string
}
// CompletePart sub container lists individual part numbers and their
@ -338,6 +381,7 @@ type CompletePart struct {
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
}
// Checksum will return the checksum for the given type.
@ -352,6 +396,8 @@ func (c CompletePart) Checksum(t ChecksumType) string {
return c.ChecksumSHA1
case t.Is(ChecksumSHA256):
return c.ChecksumSHA256
case t.Is(ChecksumCRC64NVME):
return c.ChecksumCRC64NVME
}
return ""
}

18
vendor/github.com/minio/minio-go/v7/api.go generated vendored
View file

@ -133,7 +133,7 @@ type Options struct {
// Global constants.
const (
libraryName = "minio-go"
libraryVersion = "v7.0.81"
libraryVersion = "v7.0.84"
)
// User Agent should always following the below style.
@ -602,7 +602,7 @@ func (c *Client) executeMethod(ctx context.Context, method string, metadata requ
var retryable bool // Indicates if request can be retried.
var bodySeeker io.Seeker // Extracted seeker from io.Reader.
var reqRetry = c.maxRetries // Indicates how many times we can retry the request
reqRetry := c.maxRetries // Indicates how many times we can retry the request
if metadata.contentBody != nil {
// Check if body is seekable then it is retryable.
@ -808,7 +808,7 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
}
// Get credentials from the configured credentials provider.
value, err := c.credsProvider.Get()
value, err := c.credsProvider.GetWithContext(c.CredContext())
if err != nil {
return nil, err
}
@ -1018,3 +1018,15 @@ func (c *Client) isVirtualHostStyleRequest(url url.URL, bucketName string) bool
// path style requests
return s3utils.IsVirtualHostSupported(url, bucketName)
}
// CredContext returns the context for fetching credentials
func (c *Client) CredContext() *credentials.CredContext {
httpClient := c.httpClient
if httpClient == nil {
httpClient = http.DefaultClient
}
return &credentials.CredContext{
Client: httpClient,
Endpoint: c.endpointURL.String(),
}
}

2
vendor/github.com/minio/minio-go/v7/bucket-cache.go generated vendored
View file

@ -212,7 +212,7 @@ func (c *Client) getBucketLocationRequest(ctx context.Context, bucketName string
c.setUserAgent(req)
// Get credentials from the configured credentials provider.
value, err := c.credsProvider.Get()
value, err := c.credsProvider.GetWithContext(c.CredContext())
if err != nil {
return nil, err
}

200
vendor/github.com/minio/minio-go/v7/checksum.go generated vendored
View file

@ -21,11 +21,15 @@
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"encoding/binary"
"errors"
"hash"
"hash/crc32"
"hash/crc64"
"io"
"math/bits"
"net/http"
"sort"
)
// ChecksumType contains information about the checksum type.
@ -41,23 +45,41 @@
ChecksumCRC32
// ChecksumCRC32C indicates a CRC32 checksum with Castagnoli table.
ChecksumCRC32C
// ChecksumCRC64NVME indicates CRC64 with 0xad93d23594c93659 polynomial.
ChecksumCRC64NVME
// Keep after all valid checksums
checksumLast
// ChecksumFullObject is a modifier that can be used on CRC32 and CRC32C
// to indicate full object checksums.
ChecksumFullObject
// checksumMask is a mask for valid checksum types.
checksumMask = checksumLast - 1
// ChecksumNone indicates no checksum.
ChecksumNone ChecksumType = 0
// ChecksumFullObjectCRC32 indicates full object CRC32
ChecksumFullObjectCRC32 = ChecksumCRC32 | ChecksumFullObject
// ChecksumFullObjectCRC32C indicates full object CRC32C
ChecksumFullObjectCRC32C = ChecksumCRC32C | ChecksumFullObject
amzChecksumAlgo = "x-amz-checksum-algorithm"
amzChecksumCRC32 = "x-amz-checksum-crc32"
amzChecksumCRC32C = "x-amz-checksum-crc32c"
amzChecksumSHA1 = "x-amz-checksum-sha1"
amzChecksumSHA256 = "x-amz-checksum-sha256"
amzChecksumCRC64NVME = "x-amz-checksum-crc64nvme"
)
// Base returns the base type, without modifiers.
func (c ChecksumType) Base() ChecksumType {
return c & checksumMask
}
// Is returns if c is all of t.
func (c ChecksumType) Is(t ChecksumType) bool {
return c&t == t
@ -75,10 +97,39 @@ func (c ChecksumType) Key() string {
return amzChecksumSHA1
case ChecksumSHA256:
return amzChecksumSHA256
case ChecksumCRC64NVME:
return amzChecksumCRC64NVME
}
return ""
}
// CanComposite will return if the checksum type can be used for composite multipart upload on AWS.
func (c ChecksumType) CanComposite() bool {
switch c & checksumMask {
case ChecksumSHA256, ChecksumSHA1, ChecksumCRC32, ChecksumCRC32C:
return true
}
return false
}
// CanMergeCRC will return if the checksum type can be used for multipart upload on AWS.
func (c ChecksumType) CanMergeCRC() bool {
switch c & checksumMask {
case ChecksumCRC32, ChecksumCRC32C, ChecksumCRC64NVME:
return true
}
return false
}
// FullObjectRequested will return if the checksum type indicates full object checksum was requested.
func (c ChecksumType) FullObjectRequested() bool {
switch c & (ChecksumFullObject | checksumMask) {
case ChecksumFullObjectCRC32C, ChecksumFullObjectCRC32, ChecksumCRC64NVME:
return true
}
return false
}
// KeyCapitalized returns the capitalized key as used in HTTP headers.
func (c ChecksumType) KeyCapitalized() string {
return http.CanonicalHeaderKey(c.Key())
@ -93,10 +144,17 @@ func (c ChecksumType) RawByteLen() int {
return sha1.Size
case ChecksumSHA256:
return sha256.Size
case ChecksumCRC64NVME:
return crc64.Size
}
return 0
}
const crc64NVMEPolynomial = 0xad93d23594c93659
// crc64 uses reversed polynomials.
var crc64Table = crc64.MakeTable(bits.Reverse64(crc64NVMEPolynomial))
// Hasher returns a hasher corresponding to the checksum type.
// Returns nil if no checksum.
func (c ChecksumType) Hasher() hash.Hash {
@ -109,13 +167,15 @@ func (c ChecksumType) Hasher() hash.Hash {
return sha1.New()
case ChecksumSHA256:
return sha256.New()
case ChecksumCRC64NVME:
return crc64.New(crc64Table)
}
return nil
}
// IsSet returns whether the type is valid and known.
func (c ChecksumType) IsSet() bool {
return bits.OnesCount32(uint32(c)) == 1
return bits.OnesCount32(uint32(c&checksumMask)) == 1
}
// SetDefault will set the checksum if not already set.
@ -125,6 +185,16 @@ func (c *ChecksumType) SetDefault(t ChecksumType) {
}
}
// EncodeToString the encoded hash value of the content provided in b.
func (c ChecksumType) EncodeToString(b []byte) string {
if !c.IsSet() {
return ""
}
h := c.Hasher()
h.Write(b)
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
// String returns the type as a string.
// CRC32, CRC32C, SHA1, and SHA256 for valid values.
// Empty string for unset and "<invalid>" if not valid.
@ -140,6 +210,8 @@ func (c ChecksumType) String() string {
return "SHA256"
case ChecksumNone:
return ""
case ChecksumCRC64NVME:
return "CRC64NVME"
}
return "<invalid>"
}
@ -221,3 +293,129 @@ func (c Checksum) Raw() []byte {
}
return c.r
}
// CompositeChecksum returns the composite checksum of all provided parts.
func (c ChecksumType) CompositeChecksum(p []ObjectPart) (*Checksum, error) {
if !c.CanComposite() {
return nil, errors.New("cannot do composite checksum")
}
sort.Slice(p, func(i, j int) bool {
return p[i].PartNumber < p[j].PartNumber
})
c = c.Base()
crcBytes := make([]byte, 0, len(p)*c.RawByteLen())
for _, part := range p {
pCrc, err := part.ChecksumRaw(c)
if err != nil {
return nil, err
}
crcBytes = append(crcBytes, pCrc...)
}
h := c.Hasher()
h.Write(crcBytes)
return &Checksum{Type: c, r: h.Sum(nil)}, nil
}
// FullObjectChecksum will return the full object checksum from provided parts.
func (c ChecksumType) FullObjectChecksum(p []ObjectPart) (*Checksum, error) {
if !c.CanMergeCRC() {
return nil, errors.New("cannot merge this checksum type")
}
c = c.Base()
sort.Slice(p, func(i, j int) bool {
return p[i].PartNumber < p[j].PartNumber
})
switch len(p) {
case 0:
return nil, errors.New("no parts given")
case 1:
check, err := p[0].ChecksumRaw(c)
if err != nil {
return nil, err
}
return &Checksum{
Type: c,
r: check,
}, nil
}
var merged uint32
var merged64 uint64
first, err := p[0].ChecksumRaw(c)
if err != nil {
return nil, err
}
sz := p[0].Size
switch c {
case ChecksumCRC32, ChecksumCRC32C:
merged = binary.BigEndian.Uint32(first)
case ChecksumCRC64NVME:
merged64 = binary.BigEndian.Uint64(first)
}
poly32 := uint32(crc32.IEEE)
if c.Is(ChecksumCRC32C) {
poly32 = crc32.Castagnoli
}
for _, part := range p[1:] {
if part.Size == 0 {
continue
}
sz += part.Size
pCrc, err := part.ChecksumRaw(c)
if err != nil {
return nil, err
}
switch c {
case ChecksumCRC32, ChecksumCRC32C:
merged = crc32Combine(poly32, merged, binary.BigEndian.Uint32(pCrc), part.Size)
case ChecksumCRC64NVME:
merged64 = crc64Combine(bits.Reverse64(crc64NVMEPolynomial), merged64, binary.BigEndian.Uint64(pCrc), part.Size)
}
}
var tmp [8]byte
switch c {
case ChecksumCRC32, ChecksumCRC32C:
binary.BigEndian.PutUint32(tmp[:], merged)
return &Checksum{
Type: c,
r: tmp[:4],
}, nil
case ChecksumCRC64NVME:
binary.BigEndian.PutUint64(tmp[:], merged64)
return &Checksum{
Type: c,
r: tmp[:8],
}, nil
default:
return nil, errors.New("unknown checksum type")
}
}
func addAutoChecksumHeaders(opts *PutObjectOptions) {
if opts.UserMetadata == nil {
opts.UserMetadata = make(map[string]string, 1)
}
opts.UserMetadata["X-Amz-Checksum-Algorithm"] = opts.AutoChecksum.String()
if opts.AutoChecksum.FullObjectRequested() {
opts.UserMetadata["X-Amz-Checksum-Type"] = "FULL_OBJECT"
}
}
func applyAutoChecksum(opts *PutObjectOptions, allParts []ObjectPart) {
if !opts.AutoChecksum.IsSet() {
return
}
if opts.AutoChecksum.CanComposite() && !opts.AutoChecksum.Is(ChecksumFullObject) {
// Add composite hash of hashes.
crc, err := opts.AutoChecksum.CompositeChecksum(allParts)
if err == nil {
opts.UserMetadata = map[string]string{opts.AutoChecksum.Key(): crc.Encoded()}
}
} else if opts.AutoChecksum.CanMergeCRC() {
crc, err := opts.AutoChecksum.FullObjectChecksum(allParts)
if err == nil {
opts.UserMetadata = map[string]string{opts.AutoChecksum.KeyCapitalized(): crc.Encoded(), "X-Amz-Checksum-Type": "FULL_OBJECT"}
}
}
}

109
vendor/github.com/minio/minio-go/v7/functional_tests.go generated vendored
View file

@ -2006,9 +2006,13 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
{cs: minio.ChecksumCRC32},
{cs: minio.ChecksumSHA1},
{cs: minio.ChecksumSHA256},
{cs: minio.ChecksumCRC64NVME},
}
for _, test := range tests {
if os.Getenv("MINT_NO_FULL_OBJECT") != "" && test.cs.FullObjectRequested() {
continue
}
bufSize := dataFileMap["datafile-10-kB"]
// Save the data
@ -2065,6 +2069,7 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"])
cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
// Read the data back
gopts := minio.GetObjectOptions{Checksum: true}
@ -2084,6 +2089,7 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
cmpChecksum(st.ChecksumSHA1, meta["x-amz-checksum-sha1"])
cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(st.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
if st.Size != int64(bufSize) {
logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err)
@ -2127,12 +2133,12 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
cmpChecksum(st.ChecksumSHA1, "")
cmpChecksum(st.ChecksumCRC32, "")
cmpChecksum(st.ChecksumCRC32C, "")
cmpChecksum(st.ChecksumCRC64NVME, "")
delete(args, "range")
delete(args, "metadata")
}
logSuccess(testName, function, args, startTime)
}
}
// Test PutObject with custom checksums.
@ -2173,13 +2179,16 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
tests := []struct {
cs minio.ChecksumType
}{
{cs: minio.ChecksumCRC64NVME},
{cs: minio.ChecksumCRC32C},
{cs: minio.ChecksumCRC32},
{cs: minio.ChecksumSHA1},
{cs: minio.ChecksumSHA256},
}
for _, test := range tests {
if os.Getenv("MINT_NO_FULL_OBJECT") != "" && test.cs.FullObjectRequested() {
continue
}
function := "PutObject(bucketName, objectName, reader,size, opts)"
bufSize := dataFileMap["datafile-10-kB"]
@ -2227,6 +2236,7 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"])
cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
// Read the data back
gopts := minio.GetObjectOptions{Checksum: true}
@ -2247,6 +2257,7 @@ function = "GetObject(...)"
cmpChecksum(st.ChecksumSHA1, meta["x-amz-checksum-sha1"])
cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
if st.Size != int64(bufSize) {
logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err)
@ -2291,6 +2302,7 @@ function = "GetObject( Range...)"
cmpChecksum(st.ChecksumSHA1, "")
cmpChecksum(st.ChecksumCRC32, "")
cmpChecksum(st.ChecksumCRC32C, "")
cmpChecksum(st.ChecksumCRC64NVME, "")
function = "GetObjectAttributes(...)"
s, err := c.GetObjectAttributes(context.Background(), bucketName, objectName, minio.ObjectAttributesOptions{})
@ -2305,9 +2317,8 @@ function = "GetObjectAttributes(...)"
delete(args, "range")
delete(args, "metadata")
}
logSuccess(testName, function, args, startTime)
}
}
// Test PutObject with custom checksums.
@ -2319,7 +2330,7 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
args := map[string]interface{}{
"bucketName": "",
"objectName": "",
"opts": fmt.Sprintf("minio.PutObjectOptions{UserMetadata: metadata, Progress: progress Checksum: %v}", trailing),
"opts": fmt.Sprintf("minio.PutObjectOptions{UserMetadata: metadata, Trailing: %v}", trailing),
}
if !isFullMode() {
@ -2344,14 +2355,18 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
return
}
hashMultiPart := func(b []byte, partSize int, hasher hash.Hash) string {
hashMultiPart := func(b []byte, partSize int, cs minio.ChecksumType) string {
r := bytes.NewReader(b)
hasher := cs.Hasher()
if cs.FullObjectRequested() {
partSize = len(b)
}
tmp := make([]byte, partSize)
parts := 0
var all []byte
for {
n, err := io.ReadFull(r, tmp)
if err != nil && err != io.ErrUnexpectedEOF {
if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
logError(testName, function, args, startTime, "", "Calc crc failed", err)
}
if n == 0 {
@ -2365,6 +2380,9 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
break
}
}
if parts == 1 {
return base64.StdEncoding.EncodeToString(hasher.Sum(nil))
}
hasher.Reset()
hasher.Write(all)
return fmt.Sprintf("%s-%d", base64.StdEncoding.EncodeToString(hasher.Sum(nil)), parts)
@ -2373,6 +2391,9 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
tests := []struct {
cs minio.ChecksumType
}{
{cs: minio.ChecksumFullObjectCRC32},
{cs: minio.ChecksumFullObjectCRC32C},
{cs: minio.ChecksumCRC64NVME},
{cs: minio.ChecksumCRC32C},
{cs: minio.ChecksumCRC32},
{cs: minio.ChecksumSHA1},
@ -2380,8 +2401,12 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
}
for _, test := range tests {
bufSize := dataFileMap["datafile-129-MB"]
if os.Getenv("MINT_NO_FULL_OBJECT") != "" && test.cs.FullObjectRequested() {
continue
}
args["section"] = "prep"
bufSize := dataFileMap["datafile-129-MB"]
// Save the data
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
args["objectName"] = objectName
@ -2405,7 +2430,7 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
reader.Close()
h := test.cs.Hasher()
h.Reset()
want := hashMultiPart(b, partSize, test.cs.Hasher())
want := hashMultiPart(b, partSize, test.cs)
var cs minio.ChecksumType
rd := io.Reader(io.NopCloser(bytes.NewReader(b)))
@ -2413,7 +2438,9 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
cs = test.cs
rd = bytes.NewReader(b)
}
// Set correct CRC.
args["section"] = "PutObject"
resp, err := c.PutObject(context.Background(), bucketName, objectName, rd, int64(bufSize), minio.PutObjectOptions{
DisableContentSha256: true,
DisableMultipart: false,
@ -2427,7 +2454,7 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
return
}
switch test.cs {
switch test.cs.Base() {
case minio.ChecksumCRC32C:
cmpChecksum(resp.ChecksumCRC32C, want)
case minio.ChecksumCRC32:
@ -2436,15 +2463,41 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
cmpChecksum(resp.ChecksumSHA1, want)
case minio.ChecksumSHA256:
cmpChecksum(resp.ChecksumSHA256, want)
case minio.ChecksumCRC64NVME:
cmpChecksum(resp.ChecksumCRC64NVME, want)
}
args["section"] = "HeadObject"
st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{Checksum: true})
if err != nil {
logError(testName, function, args, startTime, "", "StatObject failed", err)
return
}
switch test.cs.Base() {
case minio.ChecksumCRC32C:
cmpChecksum(st.ChecksumCRC32C, want)
case minio.ChecksumCRC32:
cmpChecksum(st.ChecksumCRC32, want)
case minio.ChecksumSHA1:
cmpChecksum(st.ChecksumSHA1, want)
case minio.ChecksumSHA256:
cmpChecksum(st.ChecksumSHA256, want)
case minio.ChecksumCRC64NVME:
cmpChecksum(st.ChecksumCRC64NVME, want)
}
args["section"] = "GetObjectAttributes"
s, err := c.GetObjectAttributes(context.Background(), bucketName, objectName, minio.ObjectAttributesOptions{})
if err != nil {
logError(testName, function, args, startTime, "", "GetObjectAttributes failed", err)
return
}
if strings.ContainsRune(want, '-') {
want = want[:strings.IndexByte(want, '-')]
}
switch test.cs {
// Full Object CRC does not return anything with GetObjectAttributes
case minio.ChecksumCRC32C:
cmpChecksum(s.Checksum.ChecksumCRC32C, want)
case minio.ChecksumCRC32:
@ -2460,13 +2513,14 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
gopts.PartNumber = 2
// We cannot use StatObject, since it ignores partnumber.
args["section"] = "GetObject-Part"
r, err := c.GetObject(context.Background(), bucketName, objectName, gopts)
if err != nil {
logError(testName, function, args, startTime, "", "GetObject failed", err)
return
}
io.Copy(io.Discard, r)
st, err := r.Stat()
st, err = r.Stat()
if err != nil {
logError(testName, function, args, startTime, "", "Stat failed", err)
return
@ -2478,6 +2532,7 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
want = base64.StdEncoding.EncodeToString(h.Sum(nil))
switch test.cs {
// Full Object CRC does not return any part CRC for whatever reason.
case minio.ChecksumCRC32C:
cmpChecksum(st.ChecksumCRC32C, want)
case minio.ChecksumCRC32:
@ -2486,12 +2541,17 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
cmpChecksum(st.ChecksumSHA1, want)
case minio.ChecksumSHA256:
cmpChecksum(st.ChecksumSHA256, want)
case minio.ChecksumCRC64NVME:
// AWS doesn't return part checksum, but may in the future.
if st.ChecksumCRC64NVME != "" {
cmpChecksum(st.ChecksumCRC64NVME, want)
}
}
delete(args, "metadata")
}
delete(args, "section")
logSuccess(testName, function, args, startTime)
}
}
// Test PutObject with trailing checksums.
@ -2688,9 +2748,8 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
}
delete(args, "metadata")
}
logSuccess(testName, function, args, startTime)
}
}
// Test PutObject with custom checksums.
@ -5324,21 +5383,11 @@ function := "PresignedPostPolicy(policy)"
defer cleanupBucket(bucketName, c)
// Generate 33K of data.
reader := getDataReader("datafile-33-kB")
defer reader.Close()
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
// Azure requires the key to not start with a number
metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "user")
metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "")
buf, err := io.ReadAll(reader)
if err != nil {
logError(testName, function, args, startTime, "", "ReadAll failed", err)
return
}
policy := minio.NewPostPolicy()
policy.SetBucket(bucketName)
policy.SetKey(objectName)
@ -5347,8 +5396,8 @@ function := "PresignedPostPolicy(policy)"
policy.SetContentLengthRange(10, 1024*1024)
policy.SetUserMetadata(metadataKey, metadataValue)
// Add CRC32C of the 33kB file that the policy will explicitly allow.
checksum := minio.ChecksumCRC32C.ChecksumBytes(buf)
// Add CRC32C of some data that the policy will explicitly allow.
checksum := minio.ChecksumCRC32C.ChecksumBytes([]byte{0x01, 0x02, 0x03})
err = policy.SetChecksum(checksum)
if err != nil {
logError(testName, function, args, startTime, "", "SetChecksum failed", err)
@ -5363,7 +5412,7 @@ function := "PresignedPostPolicy(policy)"
return
}
// At this stage, we have a policy that allows us to upload datafile-33-kB.
// At this stage, we have a policy that allows us to upload for a specific checksum.
// Test that uploading datafile-10-kB, with a different checksum, fails as expected
filePath := getMintDataDirFilePath("datafile-10-kB")
if filePath == "" {
@ -5456,7 +5505,7 @@ function := "PresignedPostPolicy(policy)"
// Normalize the response body, because S3 uses quotes around the policy condition components
// in the error message, MinIO does not.
resBodyStr := strings.ReplaceAll(string(resBody), `"`, "")
if !strings.Contains(resBodyStr, "Policy Condition failed: [eq, $x-amz-checksum-crc32c, aHnJMw==]") {
if !strings.Contains(resBodyStr, "Policy Condition failed: [eq, $x-amz-checksum-crc32c, 8TDyHg=") {
logError(testName, function, args, startTime, "", "Unexpected response body", errors.New(resBodyStr))
return
}

43
vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go generated vendored
View file

@ -76,7 +76,8 @@ type AssumeRoleResult struct {
type STSAssumeRole struct {
Expiry
// Required http Client to use when connecting to MinIO STS service.
// Optional http Client to use when connecting to MinIO STS service
// (overrides default client in CredContext)
Client *http.Client
// STS endpoint to fetch STS credentials.
@ -108,16 +109,10 @@ type STSAssumeRoleOptions struct {
// NewSTSAssumeRole returns a pointer to a new
// Credentials object wrapping the STSAssumeRole.
func NewSTSAssumeRole(stsEndpoint string, opts STSAssumeRoleOptions) (*Credentials, error) {
if stsEndpoint == "" {
return nil, errors.New("STS endpoint cannot be empty")
}
if opts.AccessKey == "" || opts.SecretKey == "" {
return nil, errors.New("AssumeRole credentials access/secretkey is mandatory")
}
return New(&STSAssumeRole{
Client: &http.Client{
Transport: http.DefaultTransport,
},
STSEndpoint: stsEndpoint,
Options: opts,
}), nil
@ -222,10 +217,30 @@ func getAssumeRoleCredentials(clnt *http.Client, endpoint string, opts STSAssume
return a, nil
}
// Retrieve retrieves credentials from the MinIO service.
// Error will be returned if the request fails.
func (m *STSAssumeRole) Retrieve() (Value, error) {
a, err := getAssumeRoleCredentials(m.Client, m.STSEndpoint, m.Options)
// RetrieveWithCredContext retrieves credentials from the MinIO service.
// Error will be returned if the request fails, optional cred context.
func (m *STSAssumeRole) RetrieveWithCredContext(cc *CredContext) (Value, error) {
if cc == nil {
cc = defaultCredContext
}
client := m.Client
if client == nil {
client = cc.Client
}
if client == nil {
client = defaultCredContext.Client
}
stsEndpoint := m.STSEndpoint
if stsEndpoint == "" {
stsEndpoint = cc.Endpoint
}
if stsEndpoint == "" {
return Value{}, errors.New("STS endpoint unknown")
}
a, err := getAssumeRoleCredentials(client, stsEndpoint, m.Options)
if err != nil {
return Value{}, err
}
@ -241,3 +256,9 @@ func (m *STSAssumeRole) Retrieve() (Value, error) {
SignerType: SignatureV4,
}, nil
}
// Retrieve retrieves credentials from the MinIO service.
// Error will be returned if the request fails.
func (m *STSAssumeRole) Retrieve() (Value, error) {
return m.RetrieveWithCredContext(nil)
}

18
vendor/github.com/minio/minio-go/v7/pkg/credentials/chain.go generated vendored
View file

@ -55,6 +55,24 @@ func NewChainCredentials(providers []Provider) *Credentials {
})
}
// RetrieveWithCredContext is like Retrieve with CredContext
func (c *Chain) RetrieveWithCredContext(cc *CredContext) (Value, error) {
for _, p := range c.Providers {
creds, _ := p.RetrieveWithCredContext(cc)
// Always prioritize non-anonymous providers, if any.
if creds.AccessKeyID == "" && creds.SecretAccessKey == "" {
continue
}
c.curr = p
return creds, nil
}
// At this point we have exhausted all the providers and
// are left without any credentials return anonymous.
return Value{
SignerType: SignatureAnonymous,
}, nil
}
// Retrieve returns the credentials value, returns no credentials(anonymous)
// if no credentials provider returned any value.
//

48
vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials.go generated vendored
View file

@ -18,6 +18,7 @@
package credentials
import (
"net/http"
"sync"
"time"
)
@ -30,6 +31,10 @@
defaultExpiryWindow = 0.8
)
// defaultCredContext is used when the credential context doesn't
// actually matter or the default context is suitable.
var defaultCredContext = &CredContext{Client: http.DefaultClient}
// A Value is the S3 credentials value for individual credential fields.
type Value struct {
// S3 Access key ID
@ -52,8 +57,17 @@ type Value struct {
// Value. A provider is required to manage its own Expired state, and what to
// be expired means.
type Provider interface {
// RetrieveWithCredContext returns nil if it successfully retrieved the
// value. Error is returned if the value were not obtainable, or empty.
// optionally takes CredContext for additional context to retrieve credentials.
RetrieveWithCredContext(cc *CredContext) (Value, error)
// Retrieve returns nil if it successfully retrieved the value.
// Error is returned if the value were not obtainable, or empty.
//
// Deprecated: Retrieve() exists for historical compatibility and should not
// be used. To get new credentials use the RetrieveWithCredContext function
// to ensure the proper context (i.e. HTTP client) will be used.
Retrieve() (Value, error)
// IsExpired returns if the credentials are no longer valid, and need
@ -61,6 +75,18 @@ type Provider interface {
IsExpired() bool
}
// CredContext is passed to the Retrieve function of a provider to provide
// some additional context to retrieve credentials.
type CredContext struct {
// Client specifies the HTTP client that should be used if an HTTP
// request is to be made to fetch the credentials.
Client *http.Client
// Endpoint specifies the MinIO endpoint that will be used if no
// explicit endpoint is provided.
Endpoint string
}
// A Expiry provides shared expiration logic to be used by credentials
// providers to implement expiry functionality.
//
@ -146,16 +172,36 @@ func New(provider Provider) *Credentials {
//
// If Credentials.Expire() was called the credentials Value will be force
// expired, and the next call to Get() will cause them to be refreshed.
//
// Deprecated: Get() exists for historical compatibility and should not be
// used. To get new credentials use the Credentials.GetWithContext function
// to ensure the proper context (i.e. HTTP client) will be used.
func (c *Credentials) Get() (Value, error) {
return c.GetWithContext(nil)
}
// GetWithContext returns the credentials value, or error if the
// credentials Value failed to be retrieved.
//
// Will return the cached credentials Value if it has not expired. If the
// credentials Value has expired the Provider's Retrieve() will be called
// to refresh the credentials.
//
// If Credentials.Expire() was called the credentials Value will be force
// expired, and the next call to Get() will cause them to be refreshed.
func (c *Credentials) GetWithContext(cc *CredContext) (Value, error) {
if c == nil {
return Value{}, nil
}
if cc == nil {
cc = defaultCredContext
}
c.Lock()
defer c.Unlock()
if c.isExpired() {
creds, err := c.provider.Retrieve()
creds, err := c.provider.RetrieveWithCredContext(cc)
if err != nil {
return Value{}, err
}

13
vendor/github.com/minio/minio-go/v7/pkg/credentials/env_aws.go generated vendored
View file

@ -37,8 +37,7 @@ func NewEnvAWS() *Credentials {
return New(&EnvAWS{})
}
// Retrieve retrieves the keys from the environment.
func (e *EnvAWS) Retrieve() (Value, error) {
func (e *EnvAWS) retrieve() (Value, error) {
e.retrieved = false
id := os.Getenv("AWS_ACCESS_KEY_ID")
@ -65,6 +64,16 @@ func (e *EnvAWS) Retrieve() (Value, error) {
}, nil
}
// Retrieve retrieves the keys from the environment.
func (e *EnvAWS) Retrieve() (Value, error) {
return e.retrieve()
}
// RetrieveWithCredContext is like Retrieve (no-op input of Cred Context)
func (e *EnvAWS) RetrieveWithCredContext(_ *CredContext) (Value, error) {
return e.retrieve()
}
// IsExpired returns if the credentials have been retrieved.
func (e *EnvAWS) IsExpired() bool {
return !e.retrieved

13
vendor/github.com/minio/minio-go/v7/pkg/credentials/env_minio.go generated vendored
View file

@ -38,8 +38,7 @@ func NewEnvMinio() *Credentials {
return New(&EnvMinio{})
}
// Retrieve retrieves the keys from the environment.
func (e *EnvMinio) Retrieve() (Value, error) {
func (e *EnvMinio) retrieve() (Value, error) {
e.retrieved = false
id := os.Getenv("MINIO_ROOT_USER")
@ -62,6 +61,16 @@ func (e *EnvMinio) Retrieve() (Value, error) {
}, nil
}
// Retrieve retrieves the keys from the environment.
func (e *EnvMinio) Retrieve() (Value, error) {
return e.retrieve()
}
// RetrieveWithCredContext is like Retrieve() (no-op input cred context)
func (e *EnvMinio) RetrieveWithCredContext(_ *CredContext) (Value, error) {
return e.retrieve()
}
// IsExpired returns if the credentials have been retrieved.
func (e *EnvMinio) IsExpired() bool {
return !e.retrieved

15
vendor/github.com/minio/minio-go/v7/pkg/credentials/file_aws_credentials.go generated vendored
View file

@ -71,9 +71,7 @@ func NewFileAWSCredentials(filename, profile string) *Credentials {
})
}
// Retrieve reads and extracts the shared credentials from the current
// users home directory.
func (p *FileAWSCredentials) Retrieve() (Value, error) {
func (p *FileAWSCredentials) retrieve() (Value, error) {
if p.Filename == "" {
p.Filename = os.Getenv("AWS_SHARED_CREDENTIALS_FILE")
if p.Filename == "" {
@ -142,6 +140,17 @@ func (p *FileAWSCredentials) Retrieve() (Value, error) {
}, nil
}
// Retrieve reads and extracts the shared credentials from the current
// users home directory.
func (p *FileAWSCredentials) Retrieve() (Value, error) {
return p.retrieve()
}
// RetrieveWithCredContext is like Retrieve(), cred context is no-op for File credentials
func (p *FileAWSCredentials) RetrieveWithCredContext(_ *CredContext) (Value, error) {
return p.retrieve()
}
// loadProfiles loads from the file pointed to by shared credentials filename for profile.
// The credentials retrieved from the profile will be returned or error. Error will be
// returned if it fails to read from the file, or the data is invalid.

15
vendor/github.com/minio/minio-go/v7/pkg/credentials/file_minio_client.go generated vendored
View file

@ -56,9 +56,7 @@ func NewFileMinioClient(filename, alias string) *Credentials {
})
}
// Retrieve reads and extracts the shared credentials from the current
// users home directory.
func (p *FileMinioClient) Retrieve() (Value, error) {
func (p *FileMinioClient) retrieve() (Value, error) {
if p.Filename == "" {
if value, ok := os.LookupEnv("MINIO_SHARED_CREDENTIALS_FILE"); ok {
p.Filename = value
@ -96,6 +94,17 @@ func (p *FileMinioClient) Retrieve() (Value, error) {
}, nil
}
// Retrieve reads and extracts the shared credentials from the current
// users home directory.
func (p *FileMinioClient) Retrieve() (Value, error) {
return p.retrieve()
}
// RetrieveWithCredContext - is like Retrieve()
func (p *FileMinioClient) RetrieveWithCredContext(_ *CredContext) (Value, error) {
return p.retrieve()
}
// IsExpired returns if the shared credentials have expired.
func (p *FileMinioClient) IsExpired() bool {
return !p.retrieved

44
vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go generated vendored
View file

@ -49,7 +49,8 @@
type IAM struct {
Expiry
// Required http Client to use when connecting to IAM metadata service.
// Optional http Client to use when connecting to IAM metadata service
// (overrides default client in CredContext)
Client *http.Client
// Custom endpoint to fetch IAM role credentials.
@ -90,17 +91,16 @@ type IAM struct {
// NewIAM returns a pointer to a new Credentials object wrapping the IAM.
func NewIAM(endpoint string) *Credentials {
return New(&IAM{
Client: &http.Client{
Transport: http.DefaultTransport,
},
Endpoint: endpoint,
})
}
// Retrieve retrieves credentials from the EC2 service.
// Error will be returned if the request fails, or unable to extract
// the desired
func (m *IAM) Retrieve() (Value, error) {
// RetrieveWithCredContext is like Retrieve with Cred Context
func (m *IAM) RetrieveWithCredContext(cc *CredContext) (Value, error) {
if cc == nil {
cc = defaultCredContext
}
token := os.Getenv("AWS_CONTAINER_AUTHORIZATION_TOKEN")
if token == "" {
token = m.Container.AuthorizationToken
@ -144,7 +144,16 @@ func (m *IAM) Retrieve() (Value, error) {
var roleCreds ec2RoleCredRespBody
var err error
client := m.Client
if client == nil {
client = cc.Client
}
if client == nil {
client = defaultCredContext.Client
}
endpoint := m.Endpoint
switch {
case identityFile != "":
if len(endpoint) == 0 {
@ -160,7 +169,7 @@ func (m *IAM) Retrieve() (Value, error) {
}
creds := &STSWebIdentity{
Client: m.Client,
Client: client,
STSEndpoint: endpoint,
GetWebIDTokenExpiry: func() (*WebIdentityToken, error) {
token, err := os.ReadFile(identityFile)
@ -174,7 +183,7 @@ func (m *IAM) Retrieve() (Value, error) {
roleSessionName: roleSessionName,
}
stsWebIdentityCreds, err := creds.Retrieve()
stsWebIdentityCreds, err := creds.RetrieveWithCredContext(cc)
if err == nil {
m.SetExpiration(creds.Expiration(), DefaultExpiryWindow)
}
@ -185,11 +194,11 @@ func (m *IAM) Retrieve() (Value, error) {
endpoint = fmt.Sprintf("%s%s", DefaultECSRoleEndpoint, relativeURI)
}
roleCreds, err = getEcsTaskCredentials(m.Client, endpoint, token)
roleCreds, err = getEcsTaskCredentials(client, endpoint, token)
case tokenFile != "" && fullURI != "":
endpoint = fullURI
roleCreds, err = getEKSPodIdentityCredentials(m.Client, endpoint, tokenFile)
roleCreds, err = getEKSPodIdentityCredentials(client, endpoint, tokenFile)
case fullURI != "":
if len(endpoint) == 0 {
@ -203,10 +212,10 @@ func (m *IAM) Retrieve() (Value, error) {
}
}
roleCreds, err = getEcsTaskCredentials(m.Client, endpoint, token)
roleCreds, err = getEcsTaskCredentials(client, endpoint, token)
default:
roleCreds, err = getCredentials(m.Client, endpoint)
roleCreds, err = getCredentials(client, endpoint)
}
if err != nil {
@ -224,6 +233,13 @@ func (m *IAM) Retrieve() (Value, error) {
}, nil
}
// Retrieve retrieves credentials from the EC2 service.
// Error will be returned if the request fails, or unable to extract
// the desired
func (m *IAM) Retrieve() (Value, error) {
return m.RetrieveWithCredContext(nil)
}
// A ec2RoleCredRespBody provides the shape for unmarshaling credential
// request responses.
type ec2RoleCredRespBody struct {

5
vendor/github.com/minio/minio-go/v7/pkg/credentials/static.go generated vendored
View file

@ -59,6 +59,11 @@ func (s *Static) Retrieve() (Value, error) {
return s.Value, nil
}
// RetrieveWithCredContext returns the static credentials.
func (s *Static) RetrieveWithCredContext(_ *CredContext) (Value, error) {
return s.Retrieve()
}
// IsExpired returns if the credentials are expired.
//
// For Static, the credentials never expired.

42
vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go generated vendored
View file

@ -72,7 +72,8 @@ type ClientGrantsToken struct {
type STSClientGrants struct {
Expiry
// Required http Client to use when connecting to MinIO STS service.
// Optional http Client to use when connecting to MinIO STS service.
// (overrides default client in CredContext)
Client *http.Client
// MinIO endpoint to fetch STS credentials.
@ -90,16 +91,10 @@ type STSClientGrants struct {
// NewSTSClientGrants returns a pointer to a new
// Credentials object wrapping the STSClientGrants.
func NewSTSClientGrants(stsEndpoint string, getClientGrantsTokenExpiry func() (*ClientGrantsToken, error)) (*Credentials, error) {
if stsEndpoint == "" {
return nil, errors.New("STS endpoint cannot be empty")
}
if getClientGrantsTokenExpiry == nil {
return nil, errors.New("Client grants access token and expiry retrieval function should be defined")
}
return New(&STSClientGrants{
Client: &http.Client{
Transport: http.DefaultTransport,
},
STSEndpoint: stsEndpoint,
GetClientGrantsTokenExpiry: getClientGrantsTokenExpiry,
}), nil
@ -162,10 +157,29 @@ func getClientGrantsCredentials(clnt *http.Client, endpoint string,
return a, nil
}
// Retrieve retrieves credentials from the MinIO service.
// Error will be returned if the request fails.
func (m *STSClientGrants) Retrieve() (Value, error) {
a, err := getClientGrantsCredentials(m.Client, m.STSEndpoint, m.GetClientGrantsTokenExpiry)
// RetrieveWithCredContext is like Retrieve() with cred context
func (m *STSClientGrants) RetrieveWithCredContext(cc *CredContext) (Value, error) {
if cc == nil {
cc = defaultCredContext
}
client := m.Client
if client == nil {
client = cc.Client
}
if client == nil {
client = defaultCredContext.Client
}
stsEndpoint := m.STSEndpoint
if stsEndpoint == "" {
stsEndpoint = cc.Endpoint
}
if stsEndpoint == "" {
return Value{}, errors.New("STS endpoint unknown")
}
a, err := getClientGrantsCredentials(client, stsEndpoint, m.GetClientGrantsTokenExpiry)
if err != nil {
return Value{}, err
}
@ -181,3 +195,9 @@ func (m *STSClientGrants) Retrieve() (Value, error) {
SignerType: SignatureV4,
}, nil
}
// Retrieve retrieves credentials from the MinIO service.
// Error will be returned if the request fails.
func (m *STSClientGrants) Retrieve() (Value, error) {
return m.RetrieveWithCredContext(nil)
}

36
vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_custom_identity.go generated vendored
View file

@ -53,6 +53,8 @@ type AssumeRoleWithCustomTokenResponse struct {
type CustomTokenIdentity struct {
Expiry
// Optional http Client to use when connecting to MinIO STS service.
// (overrides default client in CredContext)
Client *http.Client
// MinIO server STS endpoint to fetch STS credentials.
@ -69,9 +71,21 @@ type CustomTokenIdentity struct {
RequestedExpiry time.Duration
}
// Retrieve - to satisfy Provider interface; fetches credentials from MinIO.
func (c *CustomTokenIdentity) Retrieve() (value Value, err error) {
u, err := url.Parse(c.STSEndpoint)
// RetrieveWithCredContext with Retrieve optionally cred context
func (c *CustomTokenIdentity) RetrieveWithCredContext(cc *CredContext) (value Value, err error) {
if cc == nil {
cc = defaultCredContext
}
stsEndpoint := c.STSEndpoint
if stsEndpoint == "" {
stsEndpoint = cc.Endpoint
}
if stsEndpoint == "" {
return Value{}, errors.New("STS endpoint unknown")
}
u, err := url.Parse(stsEndpoint)
if err != nil {
return value, err
}
@ -92,7 +106,15 @@ func (c *CustomTokenIdentity) Retrieve() (value Value, err error) {
return value, err
}
resp, err := c.Client.Do(req)
client := c.Client
if client == nil {
client = cc.Client
}
if client == nil {
client = defaultCredContext.Client
}
resp, err := client.Do(req)
if err != nil {
return value, err
}
@ -118,11 +140,15 @@ func (c *CustomTokenIdentity) Retrieve() (value Value, err error) {
}, nil
}
// Retrieve - to satisfy Provider interface; fetches credentials from MinIO.
func (c *CustomTokenIdentity) Retrieve() (value Value, err error) {
return c.RetrieveWithCredContext(nil)
}
// NewCustomTokenCredentials - returns credentials using the
// AssumeRoleWithCustomToken STS API.
func NewCustomTokenCredentials(stsEndpoint, token, roleArn string, optFuncs ...CustomTokenOpt) (*Credentials, error) {
c := CustomTokenIdentity{
Client: &http.Client{Transport: http.DefaultTransport},
STSEndpoint: stsEndpoint,
Token: token,
RoleArn: roleArn,

40
vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go generated vendored
View file

@ -20,6 +20,7 @@
import (
"bytes"
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
@ -55,7 +56,8 @@ type LDAPIdentityResult struct {
type LDAPIdentity struct {
Expiry
// Required http Client to use when connecting to MinIO STS service.
// Optional http Client to use when connecting to MinIO STS service.
// (overrides default client in CredContext)
Client *http.Client
// Exported STS endpoint to fetch STS credentials.
@ -77,7 +79,6 @@ type LDAPIdentity struct {
// Identity.
func NewLDAPIdentity(stsEndpoint, ldapUsername, ldapPassword string, optFuncs ...LDAPIdentityOpt) (*Credentials, error) {
l := LDAPIdentity{
Client: &http.Client{Transport: http.DefaultTransport},
STSEndpoint: stsEndpoint,
LDAPUsername: ldapUsername,
LDAPPassword: ldapPassword,
@ -113,7 +114,6 @@ func LDAPIdentityExpiryOpt(d time.Duration) LDAPIdentityOpt {
// Deprecated: Use the `LDAPIdentityPolicyOpt` with `NewLDAPIdentity` instead.
func NewLDAPIdentityWithSessionPolicy(stsEndpoint, ldapUsername, ldapPassword, policy string) (*Credentials, error) {
return New(&LDAPIdentity{
Client: &http.Client{Transport: http.DefaultTransport},
STSEndpoint: stsEndpoint,
LDAPUsername: ldapUsername,
LDAPPassword: ldapPassword,
@ -121,10 +121,22 @@ func NewLDAPIdentityWithSessionPolicy(stsEndpoint, ldapUsername, ldapPassword, p
}), nil
}
// Retrieve gets the credential by calling the MinIO STS API for
// RetrieveWithCredContext gets the credential by calling the MinIO STS API for
// LDAP on the configured stsEndpoint.
func (k *LDAPIdentity) Retrieve() (value Value, err error) {
u, err := url.Parse(k.STSEndpoint)
func (k *LDAPIdentity) RetrieveWithCredContext(cc *CredContext) (value Value, err error) {
if cc == nil {
cc = defaultCredContext
}
stsEndpoint := k.STSEndpoint
if stsEndpoint == "" {
stsEndpoint = cc.Endpoint
}
if stsEndpoint == "" {
return Value{}, errors.New("STS endpoint unknown")
}
u, err := url.Parse(stsEndpoint)
if err != nil {
return value, err
}
@ -148,7 +160,15 @@ func (k *LDAPIdentity) Retrieve() (value Value, err error) {
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
resp, err := k.Client.Do(req)
client := k.Client
if client == nil {
client = cc.Client
}
if client == nil {
client = defaultCredContext.Client
}
resp, err := client.Do(req)
if err != nil {
return value, err
}
@ -188,3 +208,9 @@ func (k *LDAPIdentity) Retrieve() (value Value, err error) {
SignerType: SignatureV4,
}, nil
}
// Retrieve gets the credential by calling the MinIO STS API for
// LDAP on the configured stsEndpoint.
func (k *LDAPIdentity) Retrieve() (value Value, err error) {
return k.RetrieveWithCredContext(defaultCredContext)
}

100
vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_tls_identity.go generated vendored
View file

@ -20,8 +20,8 @@
"crypto/tls"
"encoding/xml"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strconv"
@ -36,7 +36,12 @@
// CertificateIdentityWithTransport returns a CertificateIdentityOption that
// customizes the STSCertificateIdentity with the given http.RoundTripper.
func CertificateIdentityWithTransport(t http.RoundTripper) CertificateIdentityOption {
return CertificateIdentityOption(func(i *STSCertificateIdentity) { i.Client.Transport = t })
return CertificateIdentityOption(func(i *STSCertificateIdentity) {
if i.Client == nil {
i.Client = &http.Client{}
}
i.Client.Transport = t
})
}
// CertificateIdentityWithExpiry returns a CertificateIdentityOption that
@ -53,6 +58,10 @@ func CertificateIdentityWithExpiry(livetime time.Duration) CertificateIdentityOp
type STSCertificateIdentity struct {
Expiry
// Optional http Client to use when connecting to MinIO STS service.
// (overrides default client in CredContext)
Client *http.Client
// STSEndpoint is the base URL endpoint of the STS API.
// For example, https://minio.local:9000
STSEndpoint string
@ -68,50 +77,18 @@ type STSCertificateIdentity struct {
// The default livetime is one hour.
S3CredentialLivetime time.Duration
// Client is the HTTP client used to authenticate and fetch
// S3 credentials.
//
// A custom TLS client configuration can be specified by
// using a custom http.Transport:
// Client: http.Client {
// Transport: &http.Transport{
// TLSClientConfig: &tls.Config{},
// },
// }
Client http.Client
// Certificate is the client certificate that is used for
// STS authentication.
Certificate tls.Certificate
}
var _ Provider = (*STSWebIdentity)(nil) // compiler check
// NewSTSCertificateIdentity returns a STSCertificateIdentity that authenticates
// to the given STS endpoint with the given TLS certificate and retrieves and
// rotates S3 credentials.
func NewSTSCertificateIdentity(endpoint string, certificate tls.Certificate, options ...CertificateIdentityOption) (*Credentials, error) {
if endpoint == "" {
return nil, errors.New("STS endpoint cannot be empty")
}
if _, err := url.Parse(endpoint); err != nil {
return nil, err
}
identity := &STSCertificateIdentity{
STSEndpoint: endpoint,
Client: http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
ForceAttemptHTTP2: true,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 5 * time.Second,
TLSClientConfig: &tls.Config{
Certificates: []tls.Certificate{certificate},
},
},
},
Certificate: certificate,
}
for _, option := range options {
option(identity)
@ -119,10 +96,21 @@ func NewSTSCertificateIdentity(endpoint string, certificate tls.Certificate, opt
return New(identity), nil
}
// Retrieve fetches a new set of S3 credentials from the configured
// STS API endpoint.
func (i *STSCertificateIdentity) Retrieve() (Value, error) {
endpointURL, err := url.Parse(i.STSEndpoint)
// RetrieveWithCredContext is Retrieve with cred context
func (i *STSCertificateIdentity) RetrieveWithCredContext(cc *CredContext) (Value, error) {
if cc == nil {
cc = defaultCredContext
}
stsEndpoint := i.STSEndpoint
if stsEndpoint == "" {
stsEndpoint = cc.Endpoint
}
if stsEndpoint == "" {
return Value{}, errors.New("STS endpoint unknown")
}
endpointURL, err := url.Parse(stsEndpoint)
if err != nil {
return Value{}, err
}
@ -145,7 +133,28 @@ func (i *STSCertificateIdentity) Retrieve() (Value, error) {
}
req.Form.Add("DurationSeconds", strconv.FormatUint(uint64(livetime.Seconds()), 10))
resp, err := i.Client.Do(req)
client := i.Client
if client == nil {
client = cc.Client
}
if client == nil {
client = defaultCredContext.Client
}
tr, ok := client.Transport.(*http.Transport)
if !ok {
return Value{}, fmt.Errorf("CredContext should contain an http.Transport value")
}
// Clone the HTTP transport (patch the TLS client certificate)
trCopy := tr.Clone()
trCopy.TLSClientConfig.Certificates = []tls.Certificate{i.Certificate}
// Clone the HTTP client (patch the HTTP transport)
clientCopy := *client
clientCopy.Transport = trCopy
resp, err := clientCopy.Do(req)
if err != nil {
return Value{}, err
}
@ -193,6 +202,11 @@ func (i *STSCertificateIdentity) Retrieve() (Value, error) {
}, nil
}
// Retrieve fetches a new set of S3 credentials from the configured STS API endpoint.
func (i *STSCertificateIdentity) Retrieve() (Value, error) {
return i.RetrieveWithCredContext(defaultCredContext)
}
// Expiration returns the expiration time of the current S3 credentials.
func (i *STSCertificateIdentity) Expiration() time.Time { return i.expiration }

46
vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go generated vendored
View file

@ -69,7 +69,8 @@ type WebIdentityToken struct {
type STSWebIdentity struct {
Expiry
// Required http Client to use when connecting to MinIO STS service.
// Optional http Client to use when connecting to MinIO STS service.
// (overrides default client in CredContext)
Client *http.Client
// Exported STS endpoint to fetch STS credentials.
@ -97,16 +98,10 @@ type STSWebIdentity struct {
// NewSTSWebIdentity returns a pointer to a new
// Credentials object wrapping the STSWebIdentity.
func NewSTSWebIdentity(stsEndpoint string, getWebIDTokenExpiry func() (*WebIdentityToken, error), opts ...func(*STSWebIdentity)) (*Credentials, error) {
if stsEndpoint == "" {
return nil, errors.New("STS endpoint cannot be empty")
}
if getWebIDTokenExpiry == nil {
return nil, errors.New("Web ID token and expiry retrieval function should be defined")
}
i := &STSWebIdentity{
Client: &http.Client{
Transport: http.DefaultTransport,
},
STSEndpoint: stsEndpoint,
GetWebIDTokenExpiry: getWebIDTokenExpiry,
}
@ -162,6 +157,10 @@ func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSession
// Usually set when server is using extended userInfo endpoint.
v.Set("WebIdentityAccessToken", idToken.AccessToken)
}
if idToken.RefreshToken != "" {
// Usually set when server is using extended userInfo endpoint.
v.Set("WebIdentityRefreshToken", idToken.RefreshToken)
}
if idToken.Expiry > 0 {
v.Set("DurationSeconds", fmt.Sprintf("%d", idToken.Expiry))
}
@ -215,10 +214,29 @@ func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSession
return a, nil
}
// Retrieve retrieves credentials from the MinIO service.
// Error will be returned if the request fails.
func (m *STSWebIdentity) Retrieve() (Value, error) {
a, err := getWebIdentityCredentials(m.Client, m.STSEndpoint, m.RoleARN, m.roleSessionName, m.Policy, m.GetWebIDTokenExpiry)
// RetrieveWithCredContext is like Retrieve with optional cred context.
func (m *STSWebIdentity) RetrieveWithCredContext(cc *CredContext) (Value, error) {
if cc == nil {
cc = defaultCredContext
}
client := m.Client
if client == nil {
client = cc.Client
}
if client == nil {
client = defaultCredContext.Client
}
stsEndpoint := m.STSEndpoint
if stsEndpoint == "" {
stsEndpoint = cc.Endpoint
}
if stsEndpoint == "" {
return Value{}, errors.New("STS endpoint unknown")
}
a, err := getWebIdentityCredentials(client, stsEndpoint, m.RoleARN, m.roleSessionName, m.Policy, m.GetWebIDTokenExpiry)
if err != nil {
return Value{}, err
}
@ -235,6 +253,12 @@ func (m *STSWebIdentity) Retrieve() (Value, error) {
}, nil
}
// Retrieve retrieves credentials from the MinIO service.
// Error will be returned if the request fails.
func (m *STSWebIdentity) Retrieve() (Value, error) {
return m.RetrieveWithCredContext(nil)
}
// Expiration returns the expiration time of the credentials
func (m *STSWebIdentity) Expiration() time.Time {
return m.expiration

20
vendor/github.com/minio/minio-go/v7/pkg/s3utils/utils.go generated vendored
View file

@ -118,53 +118,53 @@ func GetRegionFromURL(endpointURL url.URL) string {
if endpointURL == sentinelURL {
return ""
}
if endpointURL.Host == "s3-external-1.amazonaws.com" {
if endpointURL.Hostname() == "s3-external-1.amazonaws.com" {
return ""
}
// if elb's are used we cannot calculate which region it may be, just return empty.
if elbAmazonRegex.MatchString(endpointURL.Host) || elbAmazonCnRegex.MatchString(endpointURL.Host) {
if elbAmazonRegex.MatchString(endpointURL.Hostname()) || elbAmazonCnRegex.MatchString(endpointURL.Hostname()) {
return ""
}
// We check for FIPS dualstack matching first to avoid the non-greedy
// regex for FIPS non-dualstack matching a dualstack URL
parts := amazonS3HostFIPSDualStack.FindStringSubmatch(endpointURL.Host)
parts := amazonS3HostFIPSDualStack.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 1 {
return parts[1]
}
parts = amazonS3HostFIPS.FindStringSubmatch(endpointURL.Host)
parts = amazonS3HostFIPS.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 1 {
return parts[1]
}
parts = amazonS3HostDualStack.FindStringSubmatch(endpointURL.Host)
parts = amazonS3HostDualStack.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 1 {
return parts[1]
}
parts = amazonS3HostHyphen.FindStringSubmatch(endpointURL.Host)
parts = amazonS3HostHyphen.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 1 {
return parts[1]
}
parts = amazonS3ChinaHost.FindStringSubmatch(endpointURL.Host)
parts = amazonS3ChinaHost.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 1 {
return parts[1]
}
parts = amazonS3ChinaHostDualStack.FindStringSubmatch(endpointURL.Host)
parts = amazonS3ChinaHostDualStack.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 1 {
return parts[1]
}
parts = amazonS3HostDot.FindStringSubmatch(endpointURL.Host)
parts = amazonS3HostDot.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 1 {
return parts[1]
}
parts = amazonS3HostPrivateLink.FindStringSubmatch(endpointURL.Host)
parts = amazonS3HostPrivateLink.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 1 {
return parts[1]
}

1
vendor/github.com/minio/minio-go/v7/retry.go generated vendored
View file

@ -112,6 +112,7 @@ func isS3CodeRetryable(s3Code string) (ok bool) {
// List of HTTP status codes which are retryable.
var retryableHTTPStatusCodes = map[int]struct{}{
http.StatusRequestTimeout: {},
429: {}, // http.StatusTooManyRequests is not part of the Go 1.5 library, yet
499: {}, // client closed request, retry. A non-standard status code introduced by nginx.
http.StatusInternalServerError: {},

152
vendor/github.com/minio/minio-go/v7/utils.go generated vendored
View file

@ -378,10 +378,11 @@ func ToObjectInfo(bucketName, objectName string, h http.Header) (ObjectInfo, err
Restore: restore,
// Checksum values
ChecksumCRC32: h.Get("x-amz-checksum-crc32"),
ChecksumCRC32C: h.Get("x-amz-checksum-crc32c"),
ChecksumSHA1: h.Get("x-amz-checksum-sha1"),
ChecksumSHA256: h.Get("x-amz-checksum-sha256"),
ChecksumCRC32: h.Get(ChecksumCRC32.Key()),
ChecksumCRC32C: h.Get(ChecksumCRC32C.Key()),
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
}, nil
}
@ -698,3 +699,146 @@ func (h *hashReaderWrapper) Read(p []byte) (n int, err error) {
}
return n, err
}
// Following is ported from C to Go in 2016 by Justin Ruggles, with minimal alteration.
// Used uint for unsigned long. Used uint32 for input arguments in order to match
// the Go hash/crc32 package. zlib CRC32 combine (https://github.com/madler/zlib)
// Modified for hash/crc64 by Klaus Post, 2024.
func gf2MatrixTimes(mat []uint64, vec uint64) uint64 {
var sum uint64
for vec != 0 {
if vec&1 != 0 {
sum ^= mat[0]
}
vec >>= 1
mat = mat[1:]
}
return sum
}
func gf2MatrixSquare(square, mat []uint64) {
if len(square) != len(mat) {
panic("square matrix size mismatch")
}
for n := range mat {
square[n] = gf2MatrixTimes(mat, mat[n])
}
}
// crc32Combine returns the combined CRC-32 hash value of the two passed CRC-32
// hash values crc1 and crc2. poly represents the generator polynomial
// and len2 specifies the byte length that the crc2 hash covers.
func crc32Combine(poly uint32, crc1, crc2 uint32, len2 int64) uint32 {
// degenerate case (also disallow negative lengths)
if len2 <= 0 {
return crc1
}
even := make([]uint64, 32) // even-power-of-two zeros operator
odd := make([]uint64, 32) // odd-power-of-two zeros operator
// put operator for one zero bit in odd
odd[0] = uint64(poly) // CRC-32 polynomial
row := uint64(1)
for n := 1; n < 32; n++ {
odd[n] = row
row <<= 1
}
// put operator for two zero bits in even
gf2MatrixSquare(even, odd)
// put operator for four zero bits in odd
gf2MatrixSquare(odd, even)
// apply len2 zeros to crc1 (first square will put the operator for one
// zero byte, eight zero bits, in even)
crc1n := uint64(crc1)
for {
// apply zeros operator for this bit of len2
gf2MatrixSquare(even, odd)
if len2&1 != 0 {
crc1n = gf2MatrixTimes(even, crc1n)
}
len2 >>= 1
// if no more bits set, then done
if len2 == 0 {
break
}
// another iteration of the loop with odd and even swapped
gf2MatrixSquare(odd, even)
if len2&1 != 0 {
crc1n = gf2MatrixTimes(odd, crc1n)
}
len2 >>= 1
// if no more bits set, then done
if len2 == 0 {
break
}
}
// return combined crc
crc1n ^= uint64(crc2)
return uint32(crc1n)
}
func crc64Combine(poly uint64, crc1, crc2 uint64, len2 int64) uint64 {
// degenerate case (also disallow negative lengths)
if len2 <= 0 {
return crc1
}
even := make([]uint64, 64) // even-power-of-two zeros operator
odd := make([]uint64, 64) // odd-power-of-two zeros operator
// put operator for one zero bit in odd
odd[0] = poly // CRC-64 polynomial
row := uint64(1)
for n := 1; n < 64; n++ {
odd[n] = row
row <<= 1
}
// put operator for two zero bits in even
gf2MatrixSquare(even, odd)
// put operator for four zero bits in odd
gf2MatrixSquare(odd, even)
// apply len2 zeros to crc1 (first square will put the operator for one
// zero byte, eight zero bits, in even)
crc1n := crc1
for {
// apply zeros operator for this bit of len2
gf2MatrixSquare(even, odd)
if len2&1 != 0 {
crc1n = gf2MatrixTimes(even, crc1n)
}
len2 >>= 1
// if no more bits set, then done
if len2 == 0 {
break
}
// another iteration of the loop with odd and even swapped
gf2MatrixSquare(odd, even)
if len2&1 != 0 {
crc1n = gf2MatrixTimes(odd, crc1n)
}
len2 >>= 1
// if no more bits set, then done
if len2 == 0 {
break
}
}
// return combined crc
crc1n ^= crc2
return crc1n
}

2
vendor/modules.txt vendored
View file

@ -474,7 +474,7 @@ github.com/miekg/dns
# github.com/minio/md5-simd v1.1.2
## explicit; go 1.14
github.com/minio/md5-simd
# github.com/minio/minio-go/v7 v7.0.81
# github.com/minio/minio-go/v7 v7.0.84
## explicit; go 1.22
github.com/minio/minio-go/v7
github.com/minio/minio-go/v7/pkg/cors