/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2023 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "crypto/sha1" "crypto/sha256" "encoding/base64" "encoding/binary" "errors" "hash" "hash/crc32" "hash/crc64" "io" "math/bits" "net/http" "sort" ) // ChecksumType contains information about the checksum type. type ChecksumType uint32 const ( // ChecksumSHA256 indicates a SHA256 checksum. ChecksumSHA256 ChecksumType = 1 << iota // ChecksumSHA1 indicates a SHA-1 checksum. ChecksumSHA1 // ChecksumCRC32 indicates a CRC32 checksum with IEEE table. ChecksumCRC32 // ChecksumCRC32C indicates a CRC32 checksum with Castagnoli table. ChecksumCRC32C // ChecksumCRC64NVME indicates CRC64 with 0xad93d23594c93659 polynomial. ChecksumCRC64NVME // Keep after all valid checksums checksumLast // ChecksumFullObject is a modifier that can be used on CRC32 and CRC32C // to indicate full object checksums. ChecksumFullObject // checksumMask is a mask for valid checksum types. checksumMask = checksumLast - 1 // ChecksumNone indicates no checksum. ChecksumNone ChecksumType = 0 // ChecksumFullObjectCRC32 indicates full object CRC32 ChecksumFullObjectCRC32 = ChecksumCRC32 | ChecksumFullObject // ChecksumFullObjectCRC32C indicates full object CRC32C ChecksumFullObjectCRC32C = ChecksumCRC32C | ChecksumFullObject amzChecksumAlgo = "x-amz-checksum-algorithm" amzChecksumCRC32 = "x-amz-checksum-crc32" amzChecksumCRC32C = "x-amz-checksum-crc32c" amzChecksumSHA1 = "x-amz-checksum-sha1" amzChecksumSHA256 = "x-amz-checksum-sha256" amzChecksumCRC64NVME = "x-amz-checksum-crc64nvme" ) // Base returns the base type, without modifiers. func (c ChecksumType) Base() ChecksumType { return c & checksumMask } // Is returns if c is all of t. func (c ChecksumType) Is(t ChecksumType) bool { return c&t == t } // Key returns the header key. // returns empty string if invalid or none. func (c ChecksumType) Key() string { switch c & checksumMask { case ChecksumCRC32: return amzChecksumCRC32 case ChecksumCRC32C: return amzChecksumCRC32C case ChecksumSHA1: return amzChecksumSHA1 case ChecksumSHA256: return amzChecksumSHA256 case ChecksumCRC64NVME: return amzChecksumCRC64NVME } return "" } // CanComposite will return if the checksum type can be used for composite multipart upload on AWS. func (c ChecksumType) CanComposite() bool { switch c & checksumMask { case ChecksumSHA256, ChecksumSHA1, ChecksumCRC32, ChecksumCRC32C: return true } return false } // CanMergeCRC will return if the checksum type can be used for multipart upload on AWS. func (c ChecksumType) CanMergeCRC() bool { switch c & checksumMask { case ChecksumCRC32, ChecksumCRC32C, ChecksumCRC64NVME: return true } return false } // FullObjectRequested will return if the checksum type indicates full object checksum was requested. func (c ChecksumType) FullObjectRequested() bool { switch c & (ChecksumFullObject | checksumMask) { case ChecksumFullObjectCRC32C, ChecksumFullObjectCRC32, ChecksumCRC64NVME: return true } return false } // KeyCapitalized returns the capitalized key as used in HTTP headers. func (c ChecksumType) KeyCapitalized() string { return http.CanonicalHeaderKey(c.Key()) } // RawByteLen returns the size of the un-encoded checksum. func (c ChecksumType) RawByteLen() int { switch c & checksumMask { case ChecksumCRC32, ChecksumCRC32C: return 4 case ChecksumSHA1: return sha1.Size case ChecksumSHA256: return sha256.Size case ChecksumCRC64NVME: return crc64.Size } return 0 } const crc64NVMEPolynomial = 0xad93d23594c93659 // crc64 uses reversed polynomials. var crc64Table = crc64.MakeTable(bits.Reverse64(crc64NVMEPolynomial)) // Hasher returns a hasher corresponding to the checksum type. // Returns nil if no checksum. func (c ChecksumType) Hasher() hash.Hash { switch c & checksumMask { case ChecksumCRC32: return crc32.NewIEEE() case ChecksumCRC32C: return crc32.New(crc32.MakeTable(crc32.Castagnoli)) case ChecksumSHA1: return sha1.New() case ChecksumSHA256: return sha256.New() case ChecksumCRC64NVME: return crc64.New(crc64Table) } return nil } // IsSet returns whether the type is valid and known. func (c ChecksumType) IsSet() bool { return bits.OnesCount32(uint32(c&checksumMask)) == 1 } // SetDefault will set the checksum if not already set. func (c *ChecksumType) SetDefault(t ChecksumType) { if !c.IsSet() { *c = t } } // EncodeToString the encoded hash value of the content provided in b. func (c ChecksumType) EncodeToString(b []byte) string { if !c.IsSet() { return "" } h := c.Hasher() h.Write(b) return base64.StdEncoding.EncodeToString(h.Sum(nil)) } // String returns the type as a string. // CRC32, CRC32C, SHA1, and SHA256 for valid values. // Empty string for unset and "" if not valid. func (c ChecksumType) String() string { switch c & checksumMask { case ChecksumCRC32: return "CRC32" case ChecksumCRC32C: return "CRC32C" case ChecksumSHA1: return "SHA1" case ChecksumSHA256: return "SHA256" case ChecksumNone: return "" case ChecksumCRC64NVME: return "CRC64NVME" } return "" } // ChecksumReader reads all of r and returns a checksum of type c. // Returns any error that may have occurred while reading. func (c ChecksumType) ChecksumReader(r io.Reader) (Checksum, error) { h := c.Hasher() if h == nil { return Checksum{}, nil } _, err := io.Copy(h, r) if err != nil { return Checksum{}, err } return NewChecksum(c, h.Sum(nil)), nil } // ChecksumBytes returns a checksum of the content b with type c. func (c ChecksumType) ChecksumBytes(b []byte) Checksum { h := c.Hasher() if h == nil { return Checksum{} } n, err := h.Write(b) if err != nil || n != len(b) { // Shouldn't happen with these checksummers. return Checksum{} } return NewChecksum(c, h.Sum(nil)) } // Checksum is a type and encoded value. type Checksum struct { Type ChecksumType r []byte } // NewChecksum sets the checksum to the value of b, // which is the raw hash output. // If the length of c does not match t.RawByteLen, // a checksum with ChecksumNone is returned. func NewChecksum(t ChecksumType, b []byte) Checksum { if t.IsSet() && len(b) == t.RawByteLen() { return Checksum{Type: t, r: b} } return Checksum{} } // NewChecksumString sets the checksum to the value of s, // which is the base 64 encoded raw hash output. // If the length of c does not match t.RawByteLen, it is not added. func NewChecksumString(t ChecksumType, s string) Checksum { b, _ := base64.StdEncoding.DecodeString(s) if t.IsSet() && len(b) == t.RawByteLen() { return Checksum{Type: t, r: b} } return Checksum{} } // IsSet returns whether the checksum is valid and known. func (c Checksum) IsSet() bool { return c.Type.IsSet() && len(c.r) == c.Type.RawByteLen() } // Encoded returns the encoded value. // Returns the empty string if not set or valid. func (c Checksum) Encoded() string { if !c.IsSet() { return "" } return base64.StdEncoding.EncodeToString(c.r) } // Raw returns the raw checksum value if set. func (c Checksum) Raw() []byte { if !c.IsSet() { return nil } return c.r } // CompositeChecksum returns the composite checksum of all provided parts. func (c ChecksumType) CompositeChecksum(p []ObjectPart) (*Checksum, error) { if !c.CanComposite() { return nil, errors.New("cannot do composite checksum") } sort.Slice(p, func(i, j int) bool { return p[i].PartNumber < p[j].PartNumber }) c = c.Base() crcBytes := make([]byte, 0, len(p)*c.RawByteLen()) for _, part := range p { pCrc, err := part.ChecksumRaw(c) if err != nil { return nil, err } crcBytes = append(crcBytes, pCrc...) } h := c.Hasher() h.Write(crcBytes) return &Checksum{Type: c, r: h.Sum(nil)}, nil } // FullObjectChecksum will return the full object checksum from provided parts. func (c ChecksumType) FullObjectChecksum(p []ObjectPart) (*Checksum, error) { if !c.CanMergeCRC() { return nil, errors.New("cannot merge this checksum type") } c = c.Base() sort.Slice(p, func(i, j int) bool { return p[i].PartNumber < p[j].PartNumber }) switch len(p) { case 0: return nil, errors.New("no parts given") case 1: check, err := p[0].ChecksumRaw(c) if err != nil { return nil, err } return &Checksum{ Type: c, r: check, }, nil } var merged uint32 var merged64 uint64 first, err := p[0].ChecksumRaw(c) if err != nil { return nil, err } sz := p[0].Size switch c { case ChecksumCRC32, ChecksumCRC32C: merged = binary.BigEndian.Uint32(first) case ChecksumCRC64NVME: merged64 = binary.BigEndian.Uint64(first) } poly32 := uint32(crc32.IEEE) if c.Is(ChecksumCRC32C) { poly32 = crc32.Castagnoli } for _, part := range p[1:] { if part.Size == 0 { continue } sz += part.Size pCrc, err := part.ChecksumRaw(c) if err != nil { return nil, err } switch c { case ChecksumCRC32, ChecksumCRC32C: merged = crc32Combine(poly32, merged, binary.BigEndian.Uint32(pCrc), part.Size) case ChecksumCRC64NVME: merged64 = crc64Combine(bits.Reverse64(crc64NVMEPolynomial), merged64, binary.BigEndian.Uint64(pCrc), part.Size) } } var tmp [8]byte switch c { case ChecksumCRC32, ChecksumCRC32C: binary.BigEndian.PutUint32(tmp[:], merged) return &Checksum{ Type: c, r: tmp[:4], }, nil case ChecksumCRC64NVME: binary.BigEndian.PutUint64(tmp[:], merged64) return &Checksum{ Type: c, r: tmp[:8], }, nil default: return nil, errors.New("unknown checksum type") } } func addAutoChecksumHeaders(opts *PutObjectOptions) { if opts.UserMetadata == nil { opts.UserMetadata = make(map[string]string, 1) } opts.UserMetadata["X-Amz-Checksum-Algorithm"] = opts.AutoChecksum.String() if opts.AutoChecksum.FullObjectRequested() { opts.UserMetadata["X-Amz-Checksum-Type"] = "FULL_OBJECT" } } func applyAutoChecksum(opts *PutObjectOptions, allParts []ObjectPart) { if !opts.AutoChecksum.IsSet() { return } if opts.AutoChecksum.CanComposite() && !opts.AutoChecksum.Is(ChecksumFullObject) { // Add composite hash of hashes. crc, err := opts.AutoChecksum.CompositeChecksum(allParts) if err == nil { opts.UserMetadata = map[string]string{opts.AutoChecksum.Key(): crc.Encoded()} } } else if opts.AutoChecksum.CanMergeCRC() { crc, err := opts.AutoChecksum.FullObjectChecksum(allParts) if err == nil { opts.UserMetadata = map[string]string{opts.AutoChecksum.KeyCapitalized(): crc.Encoded(), "X-Amz-Checksum-Type": "FULL_OBJECT"} } } }