Compare commits

..

1 commit

Author SHA1 Message Date
Victor Dyotte 32593f6a2c
Merge 40c33ccc49 into 713e73300c 2024-09-24 20:13:59 +00:00
16 changed files with 216 additions and 439 deletions

4
go.mod
View file

@ -21,7 +21,7 @@ require (
codeberg.org/gruf/go-mutexes v1.5.1 codeberg.org/gruf/go-mutexes v1.5.1
codeberg.org/gruf/go-runners v1.6.2 codeberg.org/gruf/go-runners v1.6.2
codeberg.org/gruf/go-sched v1.2.3 codeberg.org/gruf/go-sched v1.2.3
codeberg.org/gruf/go-storage v0.2.0 codeberg.org/gruf/go-storage v0.1.2
codeberg.org/gruf/go-structr v0.8.9 codeberg.org/gruf/go-structr v0.8.9
codeberg.org/superseriousbusiness/exif-terminator v0.9.0 codeberg.org/superseriousbusiness/exif-terminator v0.9.0
github.com/DmitriyVTitov/size v1.5.0 github.com/DmitriyVTitov/size v1.5.0
@ -42,7 +42,7 @@ require (
github.com/k3a/html2text v1.2.1 github.com/k3a/html2text v1.2.1
github.com/microcosm-cc/bluemonday v1.0.27 github.com/microcosm-cc/bluemonday v1.0.27
github.com/miekg/dns v1.1.62 github.com/miekg/dns v1.1.62
github.com/minio/minio-go/v7 v7.0.77 github.com/minio/minio-go/v7 v7.0.76
github.com/mitchellh/mapstructure v1.5.0 github.com/mitchellh/mapstructure v1.5.0
github.com/ncruces/go-sqlite3 v0.18.3 github.com/ncruces/go-sqlite3 v0.18.3
github.com/oklog/ulid v1.3.1 github.com/oklog/ulid v1.3.1

8
go.sum
View file

@ -76,8 +76,8 @@ codeberg.org/gruf/go-runners v1.6.2 h1:oQef9niahfHu/wch14xNxlRMP8i+ABXH1Cb9PzZ4o
codeberg.org/gruf/go-runners v1.6.2/go.mod h1:Tq5PrZ/m/rBXbLZz0u5if+yP3nG5Sf6S8O/GnyEePeQ= codeberg.org/gruf/go-runners v1.6.2/go.mod h1:Tq5PrZ/m/rBXbLZz0u5if+yP3nG5Sf6S8O/GnyEePeQ=
codeberg.org/gruf/go-sched v1.2.3 h1:H5ViDxxzOBR3uIyGBCf0eH8b1L8wMybOXcdtUUTXZHk= codeberg.org/gruf/go-sched v1.2.3 h1:H5ViDxxzOBR3uIyGBCf0eH8b1L8wMybOXcdtUUTXZHk=
codeberg.org/gruf/go-sched v1.2.3/go.mod h1:vT9uB6KWFIIwnG9vcPY2a0alYNoqdL1mSzRM8I+PK7A= codeberg.org/gruf/go-sched v1.2.3/go.mod h1:vT9uB6KWFIIwnG9vcPY2a0alYNoqdL1mSzRM8I+PK7A=
codeberg.org/gruf/go-storage v0.2.0 h1:mKj3Lx6AavEkuXXtxqPhdq+akW9YwrnP16yQBF7K5ZI= codeberg.org/gruf/go-storage v0.1.2 h1:dIOVOKq1CJpRmuhbB8Zok3mmo8V6VV/nX5GLIm6hywA=
codeberg.org/gruf/go-storage v0.2.0/go.mod h1:o3GzMDE5QNUaRnm/daUzFqvuAaC4utlgXDXYO79sWKU= codeberg.org/gruf/go-storage v0.1.2/go.mod h1:LRDpFHqRJi0f+35c3ltBH2e/pGfwY5dGlNlgCJ/R1DA=
codeberg.org/gruf/go-structr v0.8.9 h1:OyiSspWYCeJOm356fFPd+bDRumPrard2VAUXAPqZiJ0= codeberg.org/gruf/go-structr v0.8.9 h1:OyiSspWYCeJOm356fFPd+bDRumPrard2VAUXAPqZiJ0=
codeberg.org/gruf/go-structr v0.8.9/go.mod h1:zkoXVrAnKosh8VFAsbP/Hhs8FmLBjbVVy5w/Ngm8ApM= codeberg.org/gruf/go-structr v0.8.9/go.mod h1:zkoXVrAnKosh8VFAsbP/Hhs8FmLBjbVVy5w/Ngm8ApM=
codeberg.org/superseriousbusiness/exif-terminator v0.9.0 h1:/EfyGI6HIrbkhFwgXGSjZ9o1kr/+k8v4mKdfXTH02Go= codeberg.org/superseriousbusiness/exif-terminator v0.9.0 h1:/EfyGI6HIrbkhFwgXGSjZ9o1kr/+k8v4mKdfXTH02Go=
@ -419,8 +419,8 @@ github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ=
github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.77 h1:GaGghJRg9nwDVlNbwYjSDJT1rqltQkBFDsypWX1v3Bw= github.com/minio/minio-go/v7 v7.0.76 h1:9nxHH2XDai61cT/EFhyIw/wW4vJfpPNvl7lSFpRt+Ng=
github.com/minio/minio-go/v7 v7.0.77/go.mod h1:AVM3IUN6WwKzmwBxVdjzhH8xq+f57JSbbvzqvUzR6eg= github.com/minio/minio-go/v7 v7.0.76/go.mod h1:AVM3IUN6WwKzmwBxVdjzhH8xq+f57JSbbvzqvUzR6eg=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=

View file

@ -216,18 +216,10 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
"png", "png",
) )
// Get mimetype for the file container
// type, falling back to generic data.
p.emoji.ImageContentType = getMimeType(ext)
// Set the known emoji static content type.
p.emoji.ImageStaticContentType = "image/png"
// Copy temporary file into storage at path. // Copy temporary file into storage at path.
filesz, err := p.mgr.state.Storage.PutFile(ctx, filesz, err := p.mgr.state.Storage.PutFile(ctx,
p.emoji.ImagePath, p.emoji.ImagePath,
temppath, temppath,
p.emoji.ImageContentType,
) )
if err != nil { if err != nil {
return gtserror.Newf("error writing emoji to storage: %w", err) return gtserror.Newf("error writing emoji to storage: %w", err)
@ -237,7 +229,6 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
staticsz, err := p.mgr.state.Storage.PutFile(ctx, staticsz, err := p.mgr.state.Storage.PutFile(ctx,
p.emoji.ImageStaticPath, p.emoji.ImageStaticPath,
staticpath, staticpath,
p.emoji.ImageStaticContentType,
) )
if err != nil { if err != nil {
return gtserror.Newf("error writing static to storage: %w", err) return gtserror.Newf("error writing static to storage: %w", err)
@ -265,6 +256,13 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
"png", "png",
) )
// Get mimetype for the file container
// type, falling back to generic data.
p.emoji.ImageContentType = getMimeType(ext)
// Set the known emoji static content type.
p.emoji.ImageStaticContentType = "image/png"
// We can now consider this cached. // We can now consider this cached.
p.emoji.Cached = util.Ptr(true) p.emoji.Cached = util.Ptr(true)

View file

@ -261,15 +261,10 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
ext, ext,
) )
// Get mimetype for the file container
// type, falling back to generic data.
p.media.File.ContentType = getMimeType(ext)
// Copy temporary file into storage at path. // Copy temporary file into storage at path.
filesz, err := p.mgr.state.Storage.PutFile(ctx, filesz, err := p.mgr.state.Storage.PutFile(ctx,
p.media.File.Path, p.media.File.Path,
temppath, temppath,
p.media.File.ContentType,
) )
if err != nil { if err != nil {
return gtserror.Newf("error writing media to storage: %w", err) return gtserror.Newf("error writing media to storage: %w", err)
@ -291,14 +286,10 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
thumbExt, thumbExt,
) )
// Determine thumbnail content-type from thumb ext.
p.media.Thumbnail.ContentType = getMimeType(thumbExt)
// Copy thumbnail file into storage at path. // Copy thumbnail file into storage at path.
thumbsz, err := p.mgr.state.Storage.PutFile(ctx, thumbsz, err := p.mgr.state.Storage.PutFile(ctx,
p.media.Thumbnail.Path, p.media.Thumbnail.Path,
thumbpath, thumbpath,
p.media.Thumbnail.ContentType,
) )
if err != nil { if err != nil {
return gtserror.Newf("error writing thumb to storage: %w", err) return gtserror.Newf("error writing thumb to storage: %w", err)
@ -307,6 +298,9 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
// Set final determined thumbnail size. // Set final determined thumbnail size.
p.media.Thumbnail.FileSize = int(thumbsz) p.media.Thumbnail.FileSize = int(thumbsz)
// Determine thumbnail content-type from thumb ext.
p.media.Thumbnail.ContentType = getMimeType(thumbExt)
// Generate a media attachment thumbnail URL. // Generate a media attachment thumbnail URL.
p.media.Thumbnail.URL = uris.URIForAttachment( p.media.Thumbnail.URL = uris.URIForAttachment(
p.media.AccountID, p.media.AccountID,
@ -326,6 +320,10 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
ext, ext,
) )
// Get mimetype for the file container
// type, falling back to generic data.
p.media.File.ContentType = getMimeType(ext)
// We can now consider this cached. // We can now consider this cached.
p.media.Cached = util.Ptr(true) p.media.Cached = util.Ptr(true)

View file

@ -97,39 +97,23 @@ func (d *Driver) Put(ctx context.Context, key string, value []byte) (int, error)
return d.Storage.WriteBytes(ctx, key, value) return d.Storage.WriteBytes(ctx, key, value)
} }
// PutFile moves the contents of file at path, to storage.Driver{} under given key (with content-type if supported). // PutStream writes the bytes from supplied reader at key in the storage
func (d *Driver) PutFile(ctx context.Context, key, filepath, contentType string) (int64, error) { func (d *Driver) PutStream(ctx context.Context, key string, r io.Reader) (int64, error) {
return d.Storage.WriteStream(ctx, key, r)
}
// PutFile moves the contents of file at path, to storage.Driver{} under given key.
func (d *Driver) PutFile(ctx context.Context, key string, filepath string) (int64, error) {
// Open file at path for reading. // Open file at path for reading.
file, err := os.Open(filepath) file, err := os.Open(filepath)
if err != nil { if err != nil {
return 0, gtserror.Newf("error opening file %s: %w", filepath, err) return 0, gtserror.Newf("error opening file %s: %w", filepath, err)
} }
var sz int64
switch d := d.Storage.(type) {
case *s3.S3Storage:
var info minio.UploadInfo
// For S3 storage, write the file but specifically pass in the
// content-type as an extra option. This handles the case of media
// being served via CDN redirect (where we don't handle content-type).
info, err = d.PutObject(ctx, key, file, minio.PutObjectOptions{
ContentType: contentType,
})
// Get size from
// uploaded info.
sz = info.Size
default:
// Write the file data to storage under key. Note // Write the file data to storage under key. Note
// that for disk.DiskStorage{} this should end up // that for disk.DiskStorage{} this should end up
// being a highly optimized Linux sendfile syscall. // being a highly optimized Linux sendfile syscall.
sz, err = d.WriteStream(ctx, key, file) sz, err := d.Storage.WriteStream(ctx, key, file)
}
// Wrap write error.
if err != nil { if err != nil {
err = gtserror.Newf("error writing file %s: %w", key, err) err = gtserror.Newf("error writing file %s: %w", key, err)
} }
@ -321,7 +305,11 @@ func NewS3Storage() (*Driver, error) {
Creds: credentials.NewStaticV4(access, secret, ""), Creds: credentials.NewStaticV4(access, secret, ""),
Secure: secure, Secure: secure,
}, },
GetOpts: minio.GetObjectOptions{},
PutOpts: minio.PutObjectOptions{},
PutChunkSize: 5 * 1024 * 1024, // 5MiB PutChunkSize: 5 * 1024 * 1024, // 5MiB
StatOpts: minio.StatObjectOptions{},
RemoveOpts: minio.RemoveObjectOptions{},
ListSize: 200, ListSize: 200,
}) })
if err != nil { if err != nil {

View file

@ -87,7 +87,7 @@ func (w *Workers) Start() {
w.Dereference.Start(n) w.Dereference.Start(n)
log.Infof(nil, "started %d dereference workers", n) log.Infof(nil, "started %d dereference workers", n)
n = maxprocs n = 4 * maxprocs
w.Processing.Start(n) w.Processing.Start(n)
log.Infof(nil, "started %d processing workers", n) log.Infof(nil, "started %d processing workers", n)
} }

View file

@ -3,9 +3,35 @@
import ( import (
"strings" "strings"
"codeberg.org/gruf/go-storage"
"codeberg.org/gruf/go-storage/internal"
"github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7"
) )
// transformS3Error transforms an error returned from S3Storage underlying
// minio.Core client, by wrapping where necessary with our own error types.
func transformS3Error(err error) error {
// Cast this to a minio error response
ersp, ok := err.(minio.ErrorResponse)
if ok {
switch ersp.Code {
case "NoSuchKey":
return internal.WrapErr(err, storage.ErrNotFound)
case "Conflict":
return internal.WrapErr(err, storage.ErrAlreadyExists)
default:
return err
}
}
// Check if error has an invalid object name prefix
if strings.HasPrefix(err.Error(), "Object name ") {
return internal.WrapErr(err, storage.ErrInvalidKey)
}
return err
}
func isNotFoundError(err error) bool { func isNotFoundError(err error) bool {
errRsp, ok := err.(minio.ErrorResponse) errRsp, ok := err.(minio.ErrorResponse)
return ok && errRsp.Code == "NoSuchKey" return ok && errRsp.Code == "NoSuchKey"

View file

@ -5,7 +5,6 @@
"context" "context"
"errors" "errors"
"io" "io"
"net/http"
"codeberg.org/gruf/go-storage" "codeberg.org/gruf/go-storage"
"codeberg.org/gruf/go-storage/internal" "codeberg.org/gruf/go-storage/internal"
@ -35,7 +34,12 @@ func DefaultConfig() Config {
// immutable default configuration. // immutable default configuration.
var defaultConfig = Config{ var defaultConfig = Config{
CoreOpts: minio.Options{}, CoreOpts: minio.Options{},
GetOpts: minio.GetObjectOptions{},
PutOpts: minio.PutObjectOptions{},
PutChunkOpts: minio.PutObjectPartOptions{},
PutChunkSize: 4 * 1024 * 1024, // 4MiB PutChunkSize: 4 * 1024 * 1024, // 4MiB
StatOpts: minio.StatObjectOptions{},
RemoveOpts: minio.RemoveObjectOptions{},
ListSize: 200, ListSize: 200,
} }
@ -46,11 +50,31 @@ type Config struct {
// passed during initialization. // passed during initialization.
CoreOpts minio.Options CoreOpts minio.Options
// GetOpts are S3 client options
// passed during .Read___() calls.
GetOpts minio.GetObjectOptions
// PutOpts are S3 client options
// passed during .Write___() calls.
PutOpts minio.PutObjectOptions
// PutChunkSize is the chunk size (in bytes) // PutChunkSize is the chunk size (in bytes)
// to use when sending a byte stream reader // to use when sending a byte stream reader
// of unknown size as a multi-part object. // of unknown size as a multi-part object.
PutChunkSize int64 PutChunkSize int64
// PutChunkOpts are S3 client options
// passed during chunked .Write___() calls.
PutChunkOpts minio.PutObjectPartOptions
// StatOpts are S3 client options
// passed during .Stat() calls.
StatOpts minio.StatObjectOptions
// RemoveOpts are S3 client options
// passed during .Remove() calls.
RemoveOpts minio.RemoveObjectOptions
// ListSize determines how many items // ListSize determines how many items
// to include in each list request, made // to include in each list request, made
// during calls to .WalkKeys(). // during calls to .WalkKeys().
@ -79,8 +103,12 @@ func getS3Config(cfg *Config) Config {
return Config{ return Config{
CoreOpts: cfg.CoreOpts, CoreOpts: cfg.CoreOpts,
GetOpts: cfg.GetOpts,
PutOpts: cfg.PutOpts,
PutChunkSize: cfg.PutChunkSize, PutChunkSize: cfg.PutChunkSize,
ListSize: cfg.ListSize, ListSize: cfg.ListSize,
StatOpts: cfg.StatOpts,
RemoveOpts: cfg.RemoveOpts,
} }
} }
@ -155,50 +183,36 @@ func (st *S3Storage) ReadBytes(ctx context.Context, key string) ([]byte, error)
// ReadStream: implements Storage.ReadStream(). // ReadStream: implements Storage.ReadStream().
func (st *S3Storage) ReadStream(ctx context.Context, key string) (io.ReadCloser, error) { func (st *S3Storage) ReadStream(ctx context.Context, key string) (io.ReadCloser, error) {
rc, _, _, err := st.GetObject(ctx, key, minio.GetObjectOptions{}) // Fetch object reader from S3 bucket
return rc, err rc, _, _, err := st.client.GetObject(
}
// GetObject wraps minio.Core{}.GetObject() to handle wrapping with our own storage library error types.
func (st *S3Storage) GetObject(ctx context.Context, key string, opts minio.GetObjectOptions) (io.ReadCloser, minio.ObjectInfo, http.Header, error) {
// Query bucket for object data and info.
rc, info, hdr, err := st.client.GetObject(
ctx, ctx,
st.bucket, st.bucket,
key, key,
opts, st.config.GetOpts,
) )
if err != nil { if err != nil {
if isNotFoundError(err) { if isNotFoundError(err) {
// Wrap not found errors as our not found type. // Wrap not found errors as our not found type.
err = internal.WrapErr(err, storage.ErrNotFound) err = internal.WrapErr(err, storage.ErrNotFound)
} else if isObjectNameError(err) { } else if !isObjectNameError(err) {
// Wrap object name errors as our invalid key type. // Wrap object name errors as our invalid key type.
err = internal.WrapErr(err, storage.ErrInvalidKey) err = internal.WrapErr(err, storage.ErrInvalidKey)
} }
return nil, transformS3Error(err)
} }
return rc, nil
return rc, info, hdr, err
} }
// WriteBytes: implements Storage.WriteBytes(). // WriteBytes: implements Storage.WriteBytes().
func (st *S3Storage) WriteBytes(ctx context.Context, key string, value []byte) (int, error) { func (st *S3Storage) WriteBytes(ctx context.Context, key string, value []byte) (int, error) {
info, err := st.PutObject(ctx, key, bytes.NewReader(value), minio.PutObjectOptions{}) n, err := st.WriteStream(ctx, key, bytes.NewReader(value))
return int(info.Size), err return int(n), err
} }
// WriteStream: implements Storage.WriteStream(). // WriteStream: implements Storage.WriteStream().
func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) (int64, error) { func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) (int64, error) {
info, err := st.PutObject(ctx, key, r, minio.PutObjectOptions{})
return info.Size, err
}
// PutObject wraps minio.Core{}.PutObject() to handle wrapping with our own storage library error types, and in the case of an io.Reader
// that does not implement ReaderSize{}, it will instead handle upload by using minio.Core{}.NewMultipartUpload() in chunks of PutChunkSize.
func (st *S3Storage) PutObject(ctx context.Context, key string, r io.Reader, opts minio.PutObjectOptions) (minio.UploadInfo, error) {
if rs, ok := r.(ReaderSize); ok { if rs, ok := r.(ReaderSize); ok {
// This reader supports providing us the size of // This reader supports providing us the size of
// the encompassed data, allowing us to perform // the encompassed data, allowing us to perform
@ -211,21 +225,22 @@ func (st *S3Storage) PutObject(ctx context.Context, key string, r io.Reader, opt
rs.Size(), rs.Size(),
"", "",
"", "",
opts, st.config.PutOpts,
) )
if err != nil { if err != nil {
if isConflictError(err) { if isConflictError(err) {
// Wrap conflict errors as our already exists type. // Wrap conflict errors as our already exists type.
err = internal.WrapErr(err, storage.ErrAlreadyExists) err = internal.WrapErr(err, storage.ErrAlreadyExists)
} else if isObjectNameError(err) { } else if !isObjectNameError(err) {
// Wrap object name errors as our invalid key type. // Wrap object name errors as our invalid key type.
err = internal.WrapErr(err, storage.ErrInvalidKey) err = internal.WrapErr(err, storage.ErrInvalidKey)
} }
return 0, err
} }
return info, err return info.Size, nil
} }
// Start a new multipart upload to get ID. // Start a new multipart upload to get ID.
@ -233,24 +248,24 @@ func (st *S3Storage) PutObject(ctx context.Context, key string, r io.Reader, opt
ctx, ctx,
st.bucket, st.bucket,
key, key,
opts, st.config.PutOpts,
) )
if err != nil { if err != nil {
if isConflictError(err) { if isConflictError(err) {
// Wrap conflict errors as our already exists type. // Wrap conflict errors as our already exists type.
err = internal.WrapErr(err, storage.ErrAlreadyExists) err = internal.WrapErr(err, storage.ErrAlreadyExists)
} else if isObjectNameError(err) { } else if !isObjectNameError(err) {
// Wrap object name errors as our invalid key type. // Wrap object name errors as our invalid key type.
err = internal.WrapErr(err, storage.ErrInvalidKey) err = internal.WrapErr(err, storage.ErrInvalidKey)
} }
return minio.UploadInfo{}, err return 0, transformS3Error(err)
} }
var ( var (
total = int64(0)
index = int(1) // parts index index = int(1) // parts index
total = int64(0)
parts []minio.CompletePart parts []minio.CompletePart
chunk = make([]byte, st.config.PutChunkSize) chunk = make([]byte, st.config.PutChunkSize)
rbuf = bytes.NewReader(nil) rbuf = bytes.NewReader(nil)
@ -281,7 +296,7 @@ func (st *S3Storage) PutObject(ctx context.Context, key string, r io.Reader, opt
// All other errors. // All other errors.
default: default:
return minio.UploadInfo{}, err return 0, err
} }
// Reset byte reader. // Reset byte reader.
@ -296,13 +311,10 @@ func (st *S3Storage) PutObject(ctx context.Context, key string, r io.Reader, opt
index, index,
rbuf, rbuf,
int64(n), int64(n),
minio.PutObjectPartOptions{ st.config.PutChunkOpts,
SSE: opts.ServerSideEncryption,
DisableContentSha256: opts.DisableContentSha256,
},
) )
if err != nil { if err != nil {
return minio.UploadInfo{}, err return 0, err
} }
// Append completed part to slice. // Append completed part to slice.
@ -315,104 +327,101 @@ func (st *S3Storage) PutObject(ctx context.Context, key string, r io.Reader, opt
ChecksumSHA256: pt.ChecksumSHA256, ChecksumSHA256: pt.ChecksumSHA256,
}) })
// Update total.
total += int64(n)
// Iterate. // Iterate.
index++ index++
// Update total size.
total += pt.Size
} }
// Complete this multi-part upload operation // Complete this multi-part upload operation
info, err := st.client.CompleteMultipartUpload( _, err = st.client.CompleteMultipartUpload(
ctx, ctx,
st.bucket, st.bucket,
key, key,
uploadID, uploadID,
parts, parts,
opts, st.config.PutOpts,
) )
if err != nil { if err != nil {
return minio.UploadInfo{}, err return 0, err
} }
// Set correct size. return total, nil
info.Size = total
return info, nil
} }
// Stat: implements Storage.Stat(). // Stat: implements Storage.Stat().
func (st *S3Storage) Stat(ctx context.Context, key string) (*storage.Entry, error) { func (st *S3Storage) Stat(ctx context.Context, key string) (*storage.Entry, error) {
info, err := st.StatObject(ctx, key, minio.StatObjectOptions{}) // Query object in S3 bucket.
if err != nil { stat, err := st.client.StatObject(
if errors.Is(err, storage.ErrNotFound) {
err = nil // mask not-found errors
}
return nil, err
}
return &storage.Entry{
Key: key,
Size: info.Size,
}, nil
}
// StatObject wraps minio.Core{}.StatObject() to handle wrapping with our own storage library error types.
func (st *S3Storage) StatObject(ctx context.Context, key string, opts minio.StatObjectOptions) (minio.ObjectInfo, error) {
// Query bucket for object info.
info, err := st.client.StatObject(
ctx, ctx,
st.bucket, st.bucket,
key, key,
opts, st.config.StatOpts,
) )
if err != nil { if err != nil {
if isNotFoundError(err) { if isNotFoundError(err) {
// Wrap not found errors as our not found type. // Ignore err return
err = internal.WrapErr(err, storage.ErrNotFound) // for not-found.
} else if isObjectNameError(err) { err = nil
} else if !isObjectNameError(err) {
// Wrap object name errors as our invalid key type. // Wrap object name errors as our invalid key type.
err = internal.WrapErr(err, storage.ErrInvalidKey) err = internal.WrapErr(err, storage.ErrInvalidKey)
} }
return nil, err
} }
return info, err return &storage.Entry{
Key: key,
Size: stat.Size,
}, nil
} }
// Remove: implements Storage.Remove(). // Remove: implements Storage.Remove().
func (st *S3Storage) Remove(ctx context.Context, key string) error { func (st *S3Storage) Remove(ctx context.Context, key string) error {
_, err := st.StatObject(ctx, key, minio.StatObjectOptions{}) // Query object in S3 bucket.
if err != nil { _, err := st.client.StatObject(
return err
}
return st.RemoveObject(ctx, key, minio.RemoveObjectOptions{})
}
// RemoveObject wraps minio.Core{}.RemoveObject() to handle wrapping with our own storage library error types.
func (st *S3Storage) RemoveObject(ctx context.Context, key string, opts minio.RemoveObjectOptions) error {
// Remove object from S3 bucket
err := st.client.RemoveObject(
ctx, ctx,
st.bucket, st.bucket,
key, key,
opts, st.config.StatOpts,
) )
if err != nil { if err != nil {
if isNotFoundError(err) { if isNotFoundError(err) {
// Wrap not found errors as our not found type. // Wrap not found errors as our not found type.
err = internal.WrapErr(err, storage.ErrNotFound) err = internal.WrapErr(err, storage.ErrNotFound)
} else if isObjectNameError(err) { } else if !isObjectNameError(err) {
// Wrap object name errors as our invalid key type. // Wrap object name errors as our invalid key type.
err = internal.WrapErr(err, storage.ErrInvalidKey) err = internal.WrapErr(err, storage.ErrInvalidKey)
} }
return err
}
// Remove object from S3 bucket
err = st.client.RemoveObject(
ctx,
st.bucket,
key,
st.config.RemoveOpts,
)
if err != nil {
if isNotFoundError(err) {
// Wrap not found errors as our not found type.
err = internal.WrapErr(err, storage.ErrNotFound)
} else if !isObjectNameError(err) {
// Wrap object name errors as our invalid key type.
err = internal.WrapErr(err, storage.ErrInvalidKey)
} }
return err return err
}
return nil
} }
// WalkKeys: implements Storage.WalkKeys(). // WalkKeys: implements Storage.WalkKeys().

View file

@ -108,9 +108,7 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
if err != nil { if err != nil {
return UploadInfo{}, err return UploadInfo{}, err
} }
if opts.Checksum.IsSet() {
opts.AutoChecksum = opts.Checksum
}
withChecksum := c.trailingHeaderSupport withChecksum := c.trailingHeaderSupport
if withChecksum { if withChecksum {
if opts.UserMetadata == nil { if opts.UserMetadata == nil {
@ -306,11 +304,6 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
return UploadInfo{}, err return UploadInfo{}, err
} }
if opts.Checksum.IsSet() {
opts.AutoChecksum = opts.Checksum
opts.SendContentMd5 = false
}
if !opts.SendContentMd5 { if !opts.SendContentMd5 {
if opts.UserMetadata == nil { if opts.UserMetadata == nil {
opts.UserMetadata = make(map[string]string, 1) opts.UserMetadata = make(map[string]string, 1)
@ -470,10 +463,7 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
if err = s3utils.CheckValidObjectName(objectName); err != nil { if err = s3utils.CheckValidObjectName(objectName); err != nil {
return UploadInfo{}, err return UploadInfo{}, err
} }
if opts.Checksum.IsSet() {
opts.SendContentMd5 = false
opts.AutoChecksum = opts.Checksum
}
if !opts.SendContentMd5 { if !opts.SendContentMd5 {
if opts.UserMetadata == nil { if opts.UserMetadata == nil {
opts.UserMetadata = make(map[string]string, 1) opts.UserMetadata = make(map[string]string, 1)
@ -565,7 +555,7 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
// Calculate md5sum. // Calculate md5sum.
customHeader := make(http.Header) customHeader := make(http.Header)
if !opts.SendContentMd5 { if !opts.SendContentMd5 {
// Add Checksum instead. // Add CRC32C instead.
crc.Reset() crc.Reset()
crc.Write(buf[:length]) crc.Write(buf[:length])
cSum := crc.Sum(nil) cSum := crc.Sum(nil)
@ -687,9 +677,6 @@ func (c *Client) putObject(ctx context.Context, bucketName, objectName string, r
if opts.SendContentMd5 && s3utils.IsGoogleEndpoint(*c.endpointURL) && size < 0 { if opts.SendContentMd5 && s3utils.IsGoogleEndpoint(*c.endpointURL) && size < 0 {
return UploadInfo{}, errInvalidArgument("MD5Sum cannot be calculated with size '-1'") return UploadInfo{}, errInvalidArgument("MD5Sum cannot be calculated with size '-1'")
} }
if opts.Checksum.IsSet() {
opts.SendContentMd5 = false
}
var readSeeker io.Seeker var readSeeker io.Seeker
if size > 0 { if size > 0 {
@ -759,6 +746,17 @@ func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string,
// Set headers. // Set headers.
customHeader := opts.Header() customHeader := opts.Header()
// Add CRC when client supports it, MD5 is not set, not Google and we don't add SHA256 to chunks.
addCrc := c.trailingHeaderSupport && md5Base64 == "" && !s3utils.IsGoogleEndpoint(*c.endpointURL) && (opts.DisableContentSha256 || c.secure)
if addCrc {
// If user has added checksums, don't add them ourselves.
for k := range opts.UserMetadata {
if strings.HasPrefix(strings.ToLower(k), "x-amz-checksum-") {
addCrc = false
}
}
}
// Populate request metadata. // Populate request metadata.
reqMetadata := requestMetadata{ reqMetadata := requestMetadata{
bucketName: bucketName, bucketName: bucketName,
@ -770,23 +768,10 @@ func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string,
contentSHA256Hex: sha256Hex, contentSHA256Hex: sha256Hex,
streamSha256: !opts.DisableContentSha256, streamSha256: !opts.DisableContentSha256,
} }
// Add CRC when client supports it, MD5 is not set, not Google and we don't add SHA256 to chunks.
addCrc := c.trailingHeaderSupport && md5Base64 == "" && !s3utils.IsGoogleEndpoint(*c.endpointURL) && (opts.DisableContentSha256 || c.secure)
if opts.Checksum.IsSet() {
reqMetadata.addCrc = &opts.Checksum
} else if addCrc {
// If user has added checksums, don't add them ourselves.
for k := range opts.UserMetadata {
if strings.HasPrefix(strings.ToLower(k), "x-amz-checksum-") {
addCrc = false
}
}
if addCrc { if addCrc {
opts.AutoChecksum.SetDefault(ChecksumCRC32C) opts.AutoChecksum.SetDefault(ChecksumCRC32C)
reqMetadata.addCrc = &opts.AutoChecksum reqMetadata.addCrc = &opts.AutoChecksum
} }
}
if opts.Internal.SourceVersionID != "" { if opts.Internal.SourceVersionID != "" {
if opts.Internal.SourceVersionID != nullVersionID { if opts.Internal.SourceVersionID != nullVersionID {
if _, err := uuid.Parse(opts.Internal.SourceVersionID); err != nil { if _, err := uuid.Parse(opts.Internal.SourceVersionID); err != nil {

View file

@ -94,13 +94,6 @@ type PutObjectOptions struct {
// If none is specified CRC32C is used, since it is generally the fastest. // If none is specified CRC32C is used, since it is generally the fastest.
AutoChecksum ChecksumType AutoChecksum ChecksumType
// Checksum will force a checksum of the specific type.
// This requires that the client was created with "TrailingHeaders:true" option,
// and that the destination server supports it.
// Unavailable with V2 signatures & Google endpoints.
// This will disable content MD5 checksums if set.
Checksum ChecksumType
// ConcurrentStreamParts will create NumThreads buffers of PartSize bytes, // ConcurrentStreamParts will create NumThreads buffers of PartSize bytes,
// fill them serially and upload them in parallel. // fill them serially and upload them in parallel.
// This can be used for faster uploads on non-seekable or slow-to-seek input. // This can be used for faster uploads on non-seekable or slow-to-seek input.
@ -247,7 +240,7 @@ func (opts PutObjectOptions) Header() (header http.Header) {
} }
// validate() checks if the UserMetadata map has standard headers or and raises an error if so. // validate() checks if the UserMetadata map has standard headers or and raises an error if so.
func (opts PutObjectOptions) validate(c *Client) (err error) { func (opts PutObjectOptions) validate() (err error) {
for k, v := range opts.UserMetadata { for k, v := range opts.UserMetadata {
if !httpguts.ValidHeaderFieldName(k) || isStandardHeader(k) || isSSEHeader(k) || isStorageClassHeader(k) || isMinioHeader(k) { if !httpguts.ValidHeaderFieldName(k) || isStandardHeader(k) || isSSEHeader(k) || isStorageClassHeader(k) || isMinioHeader(k) {
return errInvalidArgument(k + " unsupported user defined metadata name") return errInvalidArgument(k + " unsupported user defined metadata name")
@ -262,17 +255,6 @@ func (opts PutObjectOptions) validate(c *Client) (err error) {
if opts.LegalHold != "" && !opts.LegalHold.IsValid() { if opts.LegalHold != "" && !opts.LegalHold.IsValid() {
return errInvalidArgument(opts.LegalHold.String() + " unsupported legal-hold status") return errInvalidArgument(opts.LegalHold.String() + " unsupported legal-hold status")
} }
if opts.Checksum.IsSet() {
switch {
case !c.trailingHeaderSupport:
return errInvalidArgument("Checksum requires Client with TrailingHeaders enabled")
case c.overrideSignerType.IsV2():
return errInvalidArgument("Checksum cannot be used with v2 signatures")
case s3utils.IsGoogleEndpoint(*c.endpointURL):
return errInvalidArgument("Checksum cannot be used with GCS endpoints")
}
}
return nil return nil
} }
@ -309,7 +291,7 @@ func (c *Client) PutObject(ctx context.Context, bucketName, objectName string, r
return UploadInfo{}, errors.New("object size must be provided with disable multipart upload") return UploadInfo{}, errors.New("object size must be provided with disable multipart upload")
} }
err = opts.validate(c) err = opts.validate()
if err != nil { if err != nil {
return UploadInfo{}, err return UploadInfo{}, err
} }
@ -351,7 +333,7 @@ func (c *Client) putObjectCommon(ctx context.Context, bucketName, objectName str
return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, reader, opts) return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, reader, opts)
} }
if size <= int64(partSize) || opts.DisableMultipart { if size < int64(partSize) || opts.DisableMultipart {
return c.putObject(ctx, bucketName, objectName, reader, size, opts) return c.putObject(ctx, bucketName, objectName, reader, size, opts)
} }
@ -380,10 +362,6 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam
return UploadInfo{}, err return UploadInfo{}, err
} }
if opts.Checksum.IsSet() {
opts.SendContentMd5 = false
opts.AutoChecksum = opts.Checksum
}
if !opts.SendContentMd5 { if !opts.SendContentMd5 {
if opts.UserMetadata == nil { if opts.UserMetadata == nil {
opts.UserMetadata = make(map[string]string, 1) opts.UserMetadata = make(map[string]string, 1)

View file

@ -107,7 +107,7 @@ type readSeekCloser interface {
// Total size should be < 5TB. // Total size should be < 5TB.
// This function blocks until 'objs' is closed and the content has been uploaded. // This function blocks until 'objs' is closed and the content has been uploaded.
func (c Client) PutObjectsSnowball(ctx context.Context, bucketName string, opts SnowballOptions, objs <-chan SnowballObject) (err error) { func (c Client) PutObjectsSnowball(ctx context.Context, bucketName string, opts SnowballOptions, objs <-chan SnowballObject) (err error) {
err = opts.Opts.validate(&c) err = opts.Opts.validate()
if err != nil { if err != nil {
return err return err
} }

View file

@ -128,7 +128,7 @@ type Options struct {
// Global constants. // Global constants.
const ( const (
libraryName = "minio-go" libraryName = "minio-go"
libraryVersion = "v7.0.77" libraryVersion = "v7.0.76"
) )
// User Agent should always following the below style. // User Agent should always following the below style.
@ -661,7 +661,7 @@ func (c *Client) executeMethod(ctx context.Context, method string, metadata requ
// Initiate the request. // Initiate the request.
res, err = c.do(req) res, err = c.do(req)
if err != nil { if err != nil {
if isRequestErrorRetryable(ctx, err) { if isRequestErrorRetryable(err) {
// Retry the request // Retry the request
continue continue
} }

View file

@ -83,7 +83,7 @@ func createHTTPTransport() (transport *http.Transport) {
return nil return nil
} }
if mustParseBool(os.Getenv(enableHTTPS)) && mustParseBool(os.Getenv(skipCERTValidation)) { if mustParseBool(os.Getenv(skipCERTValidation)) {
transport.TLSClientConfig.InsecureSkipVerify = true transport.TLSClientConfig.InsecureSkipVerify = true
} }
@ -2334,7 +2334,7 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
} }
// Test PutObject with custom checksums. // Test PutObject with custom checksums.
func testPutObjectWithTrailingChecksums() { func testPutMultipartObjectWithChecksums() {
// initialize logging params // initialize logging params
startTime := time.Now() startTime := time.Now()
testName := getFuncName() testName := getFuncName()
@ -2342,7 +2342,7 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
args := map[string]interface{}{ args := map[string]interface{}{
"bucketName": "", "bucketName": "",
"objectName": "", "objectName": "",
"opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress, TrailChecksum: xxx}", "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}",
} }
if !isFullMode() { if !isFullMode() {
@ -2359,198 +2359,6 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
Transport: createHTTPTransport(), Transport: createHTTPTransport(),
Secure: mustParseBool(os.Getenv(enableHTTPS)), Secure: mustParseBool(os.Getenv(enableHTTPS)),
TrailingHeaders: true,
})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
return
}
// Enable tracing, write to stderr.
// c.TraceOn(os.Stderr)
// Set user agent.
c.SetAppInfo("MinIO-go-FunctionalTest", appVersion)
// Generate a new random bucket name.
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
args["bucketName"] = bucketName
// Make a new bucket.
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
if err != nil {
logError(testName, function, args, startTime, "", "Make bucket failed", err)
return
}
defer cleanupBucket(bucketName, c)
tests := []struct {
cs minio.ChecksumType
}{
{cs: minio.ChecksumCRC32C},
{cs: minio.ChecksumCRC32},
{cs: minio.ChecksumSHA1},
{cs: minio.ChecksumSHA256},
}
for _, test := range tests {
function := "PutObject(bucketName, objectName, reader,size, opts)"
bufSize := dataFileMap["datafile-10-kB"]
// Save the data
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
args["objectName"] = objectName
cmpChecksum := func(got, want string) {
if want != got {
logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got))
return
}
}
meta := map[string]string{}
reader := getDataReader("datafile-10-kB")
b, err := io.ReadAll(reader)
if err != nil {
logError(testName, function, args, startTime, "", "Read failed", err)
return
}
h := test.cs.Hasher()
h.Reset()
// Test with Wrong CRC.
args["metadata"] = meta
args["range"] = "false"
args["checksum"] = test.cs.String()
resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{
DisableMultipart: true,
DisableContentSha256: true,
UserMetadata: meta,
Checksum: test.cs,
})
if err != nil {
logError(testName, function, args, startTime, "", "PutObject failed", err)
return
}
h.Write(b)
meta[test.cs.Key()] = base64.StdEncoding.EncodeToString(h.Sum(nil))
cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"])
cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"])
cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
// Read the data back
gopts := minio.GetObjectOptions{Checksum: true}
function = "GetObject(...)"
r, err := c.GetObject(context.Background(), bucketName, objectName, gopts)
if err != nil {
logError(testName, function, args, startTime, "", "GetObject failed", err)
return
}
st, err := r.Stat()
if err != nil {
logError(testName, function, args, startTime, "", "Stat failed", err)
return
}
cmpChecksum(st.ChecksumSHA256, meta["x-amz-checksum-sha256"])
cmpChecksum(st.ChecksumSHA1, meta["x-amz-checksum-sha1"])
cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
if st.Size != int64(bufSize) {
logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err)
return
}
if err := r.Close(); err != nil {
logError(testName, function, args, startTime, "", "Object Close failed", err)
return
}
if err := r.Close(); err == nil {
logError(testName, function, args, startTime, "", "Object already closed, should respond with error", err)
return
}
function = "GetObject( Range...)"
args["range"] = "true"
err = gopts.SetRange(100, 1000)
if err != nil {
logError(testName, function, args, startTime, "", "SetRange failed", err)
return
}
r, err = c.GetObject(context.Background(), bucketName, objectName, gopts)
if err != nil {
logError(testName, function, args, startTime, "", "GetObject failed", err)
return
}
b, err = io.ReadAll(r)
if err != nil {
logError(testName, function, args, startTime, "", "Read failed", err)
return
}
st, err = r.Stat()
if err != nil {
logError(testName, function, args, startTime, "", "Stat failed", err)
return
}
// Range requests should return empty checksums...
cmpChecksum(st.ChecksumSHA256, "")
cmpChecksum(st.ChecksumSHA1, "")
cmpChecksum(st.ChecksumCRC32, "")
cmpChecksum(st.ChecksumCRC32C, "")
function = "GetObjectAttributes(...)"
s, err := c.GetObjectAttributes(context.Background(), bucketName, objectName, minio.ObjectAttributesOptions{})
if err != nil {
logError(testName, function, args, startTime, "", "GetObjectAttributes failed", err)
return
}
cmpChecksum(s.Checksum.ChecksumSHA256, meta["x-amz-checksum-sha256"])
cmpChecksum(s.Checksum.ChecksumSHA1, meta["x-amz-checksum-sha1"])
cmpChecksum(s.Checksum.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(s.Checksum.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
delete(args, "range")
delete(args, "metadata")
}
logSuccess(testName, function, args, startTime)
}
// Test PutObject with custom checksums.
func testPutMultipartObjectWithChecksums(trailing bool) {
// initialize logging params
startTime := time.Now()
testName := getFuncName()
function := "PutObject(bucketName, objectName, reader,size, opts)"
args := map[string]interface{}{
"bucketName": "",
"objectName": "",
"opts": fmt.Sprintf("minio.PutObjectOptions{UserMetadata: metadata, Progress: progress Checksum: %v}", trailing),
}
if !isFullMode() {
logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs")
return
}
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Instantiate new minio client object.
c, err := minio.New(os.Getenv(serverEndpoint),
&minio.Options{
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
Transport: createHTTPTransport(),
Secure: mustParseBool(os.Getenv(enableHTTPS)),
TrailingHeaders: trailing,
}) })
if err != nil { if err != nil {
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
@ -2637,20 +2445,14 @@ function := "PutObject(bucketName, objectName, reader,size, opts)"
h.Reset() h.Reset()
want := hashMultiPart(b, partSize, test.cs.Hasher()) want := hashMultiPart(b, partSize, test.cs.Hasher())
var cs minio.ChecksumType
rd := io.Reader(io.NopCloser(bytes.NewReader(b)))
if trailing {
cs = test.cs
rd = bytes.NewReader(b)
}
// Set correct CRC. // Set correct CRC.
resp, err := c.PutObject(context.Background(), bucketName, objectName, rd, int64(bufSize), minio.PutObjectOptions{
resp, err := c.PutObject(context.Background(), bucketName, objectName, io.NopCloser(bytes.NewReader(b)), int64(bufSize), minio.PutObjectOptions{
DisableContentSha256: true, DisableContentSha256: true,
DisableMultipart: false, DisableMultipart: false,
UserMetadata: nil, UserMetadata: nil,
PartSize: partSize, PartSize: partSize,
AutoChecksum: test.cs, AutoChecksum: test.cs,
Checksum: cs,
}) })
if err != nil { if err != nil {
logError(testName, function, args, startTime, "", "PutObject failed", err) logError(testName, function, args, startTime, "", "PutObject failed", err)
@ -3180,7 +2982,6 @@ function := "GetObjectAttributes(ctx, bucketName, objectName, opts)"
testFiles[i].UploadInfo, err = c.PutObject(context.Background(), v.Bucket, v.Object, reader, int64(bufSize), minio.PutObjectOptions{ testFiles[i].UploadInfo, err = c.PutObject(context.Background(), v.Bucket, v.Object, reader, int64(bufSize), minio.PutObjectOptions{
ContentType: v.ContentType, ContentType: v.ContentType,
SendContentMd5: v.SendContentMd5, SendContentMd5: v.SendContentMd5,
Checksum: minio.ChecksumCRC32C,
}) })
if err != nil { if err != nil {
logError(testName, function, args, startTime, "", "PutObject failed", err) logError(testName, function, args, startTime, "", "PutObject failed", err)
@ -3262,7 +3063,7 @@ function := "GetObjectAttributes(ctx, bucketName, objectName, opts)"
test: objectAttributesTestOptions{ test: objectAttributesTestOptions{
TestFileName: "file1", TestFileName: "file1",
StorageClass: "STANDARD", StorageClass: "STANDARD",
HasFullChecksum: true, HasFullChecksum: false,
}, },
} }
@ -3351,10 +3152,9 @@ function := "GetObjectAttributes(ctx, bucketName, objectName, opts)"
info, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ info, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{
ContentType: "content/custom", ContentType: "content/custom",
SendContentMd5: false, SendContentMd5: true,
ServerSideEncryption: sse, ServerSideEncryption: sse,
PartSize: uint64(bufSize) / 2, PartSize: uint64(bufSize) / 2,
Checksum: minio.ChecksumCRC32C,
}) })
if err != nil { if err != nil {
logError(testName, function, args, startTime, "", "PutObject failed", err) logError(testName, function, args, startTime, "", "PutObject failed", err)
@ -3374,9 +3174,9 @@ function := "GetObjectAttributes(ctx, bucketName, objectName, opts)"
ETag: info.ETag, ETag: info.ETag,
NumberOfParts: 2, NumberOfParts: 2,
ObjectSize: int(info.Size), ObjectSize: int(info.Size),
HasFullChecksum: true, HasFullChecksum: false,
HasParts: true, HasParts: true,
HasPartChecksums: true, HasPartChecksums: false,
}) })
if err != nil { if err != nil {
logError(testName, function, args, startTime, "", "Validating GetObjectsAttributes response failed", err) logError(testName, function, args, startTime, "", "Validating GetObjectsAttributes response failed", err)
@ -5794,12 +5594,18 @@ function := "PresignedPostPolicy(policy)"
} }
writer.Close() writer.Close()
transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
if err != nil {
logError(testName, function, args, startTime, "", "DefaultTransport failed", err)
return
}
httpClient := &http.Client{ httpClient := &http.Client{
// Setting a sensible time out of 30secs to wait for response // Setting a sensible time out of 30secs to wait for response
// headers. Request is pro-actively canceled after 30secs // headers. Request is pro-actively canceled after 30secs
// with no response. // with no response.
Timeout: 30 * time.Second, Timeout: 30 * time.Second,
Transport: createHTTPTransport(), Transport: transport,
} }
args["url"] = presignedPostPolicyURL.String() args["url"] = presignedPostPolicyURL.String()
@ -7713,7 +7519,7 @@ functionAll += ", " + function
return return
} }
transport := createHTTPTransport() transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
if err != nil { if err != nil {
logError(testName, function, args, startTime, "", "DefaultTransport failed", err) logError(testName, function, args, startTime, "", "DefaultTransport failed", err)
return return
@ -12644,12 +12450,18 @@ functionAll += ", " + function
return return
} }
transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
if err != nil {
logError(testName, function, args, startTime, "", "DefaultTransport failed", err)
return
}
httpClient := &http.Client{ httpClient := &http.Client{
// Setting a sensible time out of 30secs to wait for response // Setting a sensible time out of 30secs to wait for response
// headers. Request is pro-actively canceled after 30secs // headers. Request is pro-actively canceled after 30secs
// with no response. // with no response.
Timeout: 30 * time.Second, Timeout: 30 * time.Second,
Transport: createHTTPTransport(), Transport: transport,
} }
req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil) req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil)
@ -13744,9 +13556,14 @@ function := "SetBucketCors(bucketName, cors)"
bucketURL := c.EndpointURL().String() + "/" + bucketName + "/" bucketURL := c.EndpointURL().String() + "/" + bucketName + "/"
objectURL := bucketURL + objectName objectURL := bucketURL + objectName
transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
if err != nil {
logError(testName, function, args, startTime, "", "DefaultTransport failed", err)
return
}
httpClient := &http.Client{ httpClient := &http.Client{
Timeout: 30 * time.Second, Timeout: 30 * time.Second,
Transport: createHTTPTransport(), Transport: transport,
} }
errStrAccessForbidden := `<Error><Code>AccessForbidden</Code><Message>CORSResponse: This CORS request is not allowed. This is usually because the evalution of Origin, request method / Access-Control-Request-Method or Access-Control-Request-Headers are not whitelisted` errStrAccessForbidden := `<Error><Code>AccessForbidden</Code><Message>CORSResponse: This CORS request is not allowed. This is usually because the evalution of Origin, request method / Access-Control-Request-Method or Access-Control-Request-Headers are not whitelisted`
@ -14940,9 +14757,7 @@ func main() {
testCompose10KSourcesV2() testCompose10KSourcesV2()
testUserMetadataCopyingV2() testUserMetadataCopyingV2()
testPutObjectWithChecksums() testPutObjectWithChecksums()
testPutObjectWithTrailingChecksums() testPutMultipartObjectWithChecksums()
testPutMultipartObjectWithChecksums(false)
testPutMultipartObjectWithChecksums(true)
testPutObject0ByteV2() testPutObject0ByteV2()
testPutObjectNoLengthV2() testPutObjectNoLengthV2()
testPutObjectsUnknownV2() testPutObjectsUnknownV2()

View file

@ -301,25 +301,6 @@ func (p *PostPolicy) SetUserMetadata(key, value string) error {
return nil return nil
} }
// SetUserMetadataStartsWith - Set how an user metadata should starts with.
// Can be retrieved through a HEAD request or an event.
func (p *PostPolicy) SetUserMetadataStartsWith(key, value string) error {
if strings.TrimSpace(key) == "" || key == "" {
return errInvalidArgument("Key is empty")
}
headerName := fmt.Sprintf("x-amz-meta-%s", key)
policyCond := policyCondition{
matchType: "starts-with",
condition: fmt.Sprintf("$%s", headerName),
value: value,
}
if err := p.addNewPolicy(policyCond); err != nil {
return err
}
p.formData[headerName] = value
return nil
}
// SetChecksum sets the checksum of the request. // SetChecksum sets the checksum of the request.
func (p *PostPolicy) SetChecksum(c Checksum) { func (p *PostPolicy) SetChecksum(c Checksum) {
if c.IsSet() { if c.IsSet() {

View file

@ -129,10 +129,9 @@ func isHTTPStatusRetryable(httpStatusCode int) (ok bool) {
} }
// For now, all http Do() requests are retriable except some well defined errors // For now, all http Do() requests are retriable except some well defined errors
func isRequestErrorRetryable(ctx context.Context, err error) bool { func isRequestErrorRetryable(err error) bool {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
// Retry if internal timeout in the HTTP call. return false
return ctx.Err() == nil
} }
if ue, ok := err.(*url.Error); ok { if ue, ok := err.(*url.Error); ok {
e := ue.Unwrap() e := ue.Unwrap()

4
vendor/modules.txt vendored
View file

@ -65,7 +65,7 @@ codeberg.org/gruf/go-runners
# codeberg.org/gruf/go-sched v1.2.3 # codeberg.org/gruf/go-sched v1.2.3
## explicit; go 1.19 ## explicit; go 1.19
codeberg.org/gruf/go-sched codeberg.org/gruf/go-sched
# codeberg.org/gruf/go-storage v0.2.0 # codeberg.org/gruf/go-storage v0.1.2
## explicit; go 1.22 ## explicit; go 1.22
codeberg.org/gruf/go-storage codeberg.org/gruf/go-storage
codeberg.org/gruf/go-storage/disk codeberg.org/gruf/go-storage/disk
@ -491,7 +491,7 @@ github.com/miekg/dns
# github.com/minio/md5-simd v1.1.2 # github.com/minio/md5-simd v1.1.2
## explicit; go 1.14 ## explicit; go 1.14
github.com/minio/md5-simd github.com/minio/md5-simd
# github.com/minio/minio-go/v7 v7.0.77 # github.com/minio/minio-go/v7 v7.0.76
## explicit; go 1.21 ## explicit; go 1.21
github.com/minio/minio-go/v7 github.com/minio/minio-go/v7
github.com/minio/minio-go/v7/pkg/cors github.com/minio/minio-go/v7/pkg/cors