mirror of
https://github.com/superseriousbusiness/gotosocial.git
synced 2024-12-29 18:46:31 +00:00
9d0df426da
* feat: vendor minio client * feat: introduce storage package with s3 support * feat: serve s3 files directly this saves a lot of bandwith as the files are fetched from the object store directly * fix: use explicit local storage in tests * feat: integrate s3 storage with the main server * fix: add s3 config to cli tests * docs: explicitly set values in example config also adds license header to the storage package * fix: use better http status code on s3 redirect HTTP 302 Found is the best fit, as it signifies that the resource requested was found but not under its presumed URL 307/TemporaryRedirect would mean that this resource is usually located here, not in this case 303/SeeOther indicates that the redirection does not link to the requested resource but to another page * refactor: use context in storage driver interface
12226 lines
391 KiB
Go
12226 lines
391 KiB
Go
//go:build mint
|
|
// +build mint
|
|
|
|
/*
|
|
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
|
|
* Copyright 2015-2020 MinIO, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package main
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"hash/crc32"
|
|
"io"
|
|
"io/ioutil"
|
|
"math/rand"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"reflect"
|
|
"runtime"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/dustin/go-humanize"
|
|
jsoniter "github.com/json-iterator/go"
|
|
log "github.com/sirupsen/logrus"
|
|
|
|
"github.com/minio/minio-go/v7"
|
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
|
"github.com/minio/minio-go/v7/pkg/encrypt"
|
|
"github.com/minio/minio-go/v7/pkg/notification"
|
|
"github.com/minio/minio-go/v7/pkg/tags"
|
|
)
|
|
|
|
const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569"
|
|
const (
|
|
letterIdxBits = 6 // 6 bits to represent a letter index
|
|
letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
|
|
letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits
|
|
)
|
|
|
|
const (
|
|
serverEndpoint = "SERVER_ENDPOINT"
|
|
accessKey = "ACCESS_KEY"
|
|
secretKey = "SECRET_KEY"
|
|
enableHTTPS = "ENABLE_HTTPS"
|
|
enableKMS = "ENABLE_KMS"
|
|
)
|
|
|
|
type mintJSONFormatter struct{}
|
|
|
|
func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) {
|
|
data := make(log.Fields, len(entry.Data))
|
|
for k, v := range entry.Data {
|
|
switch v := v.(type) {
|
|
case error:
|
|
// Otherwise errors are ignored by `encoding/json`
|
|
// https://github.com/sirupsen/logrus/issues/137
|
|
data[k] = v.Error()
|
|
default:
|
|
data[k] = v
|
|
}
|
|
}
|
|
json := jsoniter.ConfigCompatibleWithStandardLibrary
|
|
serialized, err := json.Marshal(data)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err)
|
|
}
|
|
return append(serialized, '\n'), nil
|
|
}
|
|
|
|
var readFull = func(r io.Reader, buf []byte) (n int, err error) {
|
|
// ReadFull reads exactly len(buf) bytes from r into buf.
|
|
// It returns the number of bytes copied and an error if
|
|
// fewer bytes were read. The error is EOF only if no bytes
|
|
// were read. If an EOF happens after reading some but not
|
|
// all the bytes, ReadFull returns ErrUnexpectedEOF.
|
|
// On return, n == len(buf) if and only if err == nil.
|
|
// If r returns an error having read at least len(buf) bytes,
|
|
// the error is dropped.
|
|
for n < len(buf) && err == nil {
|
|
var nn int
|
|
nn, err = r.Read(buf[n:])
|
|
// Some spurious io.Reader's return
|
|
// io.ErrUnexpectedEOF when nn == 0
|
|
// this behavior is undocumented
|
|
// so we are on purpose not using io.ReadFull
|
|
// implementation because this can lead
|
|
// to custom handling, to avoid that
|
|
// we simply modify the original io.ReadFull
|
|
// implementation to avoid this issue.
|
|
// io.ErrUnexpectedEOF with nn == 0 really
|
|
// means that io.EOF
|
|
if err == io.ErrUnexpectedEOF && nn == 0 {
|
|
err = io.EOF
|
|
}
|
|
n += nn
|
|
}
|
|
if n >= len(buf) {
|
|
err = nil
|
|
} else if n > 0 && err == io.EOF {
|
|
err = io.ErrUnexpectedEOF
|
|
}
|
|
return
|
|
}
|
|
|
|
func cleanEmptyEntries(fields log.Fields) log.Fields {
|
|
cleanFields := log.Fields{}
|
|
for k, v := range fields {
|
|
if v != "" {
|
|
cleanFields[k] = v
|
|
}
|
|
}
|
|
return cleanFields
|
|
}
|
|
|
|
// log successful test runs
|
|
func successLogger(testName string, function string, args map[string]interface{}, startTime time.Time) *log.Entry {
|
|
// calculate the test case duration
|
|
duration := time.Since(startTime)
|
|
// log with the fields as per mint
|
|
fields := log.Fields{"name": "minio-go: " + testName, "function": function, "args": args, "duration": duration.Nanoseconds() / 1000000, "status": "PASS"}
|
|
return log.WithFields(cleanEmptyEntries(fields))
|
|
}
|
|
|
|
// As few of the features are not available in Gateway(s) currently, Check if err value is NotImplemented,
|
|
// and log as NA in that case and continue execution. Otherwise log as failure and return
|
|
func logError(testName string, function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) {
|
|
// If server returns NotImplemented we assume it is gateway mode and hence log it as info and move on to next tests
|
|
// Special case for ComposeObject API as it is implemented on client side and adds specific error details like `Error in upload-part-copy` in
|
|
// addition to NotImplemented error returned from server
|
|
if isErrNotImplemented(err) {
|
|
ignoredLog(testName, function, args, startTime, message).Info()
|
|
} else if isRunOnFail() {
|
|
failureLog(testName, function, args, startTime, alert, message, err).Error()
|
|
} else {
|
|
failureLog(testName, function, args, startTime, alert, message, err).Fatal()
|
|
}
|
|
}
|
|
|
|
// log failed test runs
|
|
func failureLog(testName string, function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry {
|
|
// calculate the test case duration
|
|
duration := time.Since(startTime)
|
|
var fields log.Fields
|
|
// log with the fields as per mint
|
|
if err != nil {
|
|
fields = log.Fields{
|
|
"name": "minio-go: " + testName, "function": function, "args": args,
|
|
"duration": duration.Nanoseconds() / 1000000, "status": "FAIL", "alert": alert, "message": message, "error": err,
|
|
}
|
|
} else {
|
|
fields = log.Fields{
|
|
"name": "minio-go: " + testName, "function": function, "args": args,
|
|
"duration": duration.Nanoseconds() / 1000000, "status": "FAIL", "alert": alert, "message": message,
|
|
}
|
|
}
|
|
return log.WithFields(cleanEmptyEntries(fields))
|
|
}
|
|
|
|
// log not applicable test runs
|
|
func ignoredLog(testName string, function string, args map[string]interface{}, startTime time.Time, alert string) *log.Entry {
|
|
// calculate the test case duration
|
|
duration := time.Since(startTime)
|
|
// log with the fields as per mint
|
|
fields := log.Fields{
|
|
"name": "minio-go: " + testName, "function": function, "args": args,
|
|
"duration": duration.Nanoseconds() / 1000000, "status": "NA", "alert": strings.Split(alert, " ")[0] + " is NotImplemented",
|
|
}
|
|
return log.WithFields(cleanEmptyEntries(fields))
|
|
}
|
|
|
|
// Delete objects in given bucket, recursively
|
|
func cleanupBucket(bucketName string, c *minio.Client) error {
|
|
// Create a done channel to control 'ListObjectsV2' go routine.
|
|
doneCh := make(chan struct{})
|
|
// Exit cleanly upon return.
|
|
defer close(doneCh)
|
|
// Iterate over all objects in the bucket via listObjectsV2 and delete
|
|
for objCh := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{Recursive: true}) {
|
|
if objCh.Err != nil {
|
|
return objCh.Err
|
|
}
|
|
if objCh.Key != "" {
|
|
err := c.RemoveObject(context.Background(), bucketName, objCh.Key, minio.RemoveObjectOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
for objPartInfo := range c.ListIncompleteUploads(context.Background(), bucketName, "", true) {
|
|
if objPartInfo.Err != nil {
|
|
return objPartInfo.Err
|
|
}
|
|
if objPartInfo.Key != "" {
|
|
err := c.RemoveIncompleteUpload(context.Background(), bucketName, objPartInfo.Key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
// objects are already deleted, clear the buckets now
|
|
err := c.RemoveBucket(context.Background(), bucketName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return err
|
|
}
|
|
|
|
func cleanupVersionedBucket(bucketName string, c *minio.Client) error {
|
|
doneCh := make(chan struct{})
|
|
defer close(doneCh)
|
|
for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) {
|
|
if obj.Err != nil {
|
|
return obj.Err
|
|
}
|
|
if obj.Key != "" {
|
|
err := c.RemoveObject(context.Background(), bucketName, obj.Key,
|
|
minio.RemoveObjectOptions{VersionID: obj.VersionID, GovernanceBypass: true})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
for objPartInfo := range c.ListIncompleteUploads(context.Background(), bucketName, "", true) {
|
|
if objPartInfo.Err != nil {
|
|
return objPartInfo.Err
|
|
}
|
|
if objPartInfo.Key != "" {
|
|
err := c.RemoveIncompleteUpload(context.Background(), bucketName, objPartInfo.Key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
// objects are already deleted, clear the buckets now
|
|
err := c.RemoveBucket(context.Background(), bucketName)
|
|
if err != nil {
|
|
for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) {
|
|
log.Println("found", obj.Key, obj.VersionID)
|
|
}
|
|
return err
|
|
}
|
|
return err
|
|
}
|
|
|
|
func isErrNotImplemented(err error) bool {
|
|
return minio.ToErrorResponse(err).Code == "NotImplemented"
|
|
}
|
|
|
|
func isRunOnFail() bool {
|
|
return os.Getenv("RUN_ON_FAIL") == "1"
|
|
}
|
|
|
|
func init() {
|
|
// If server endpoint is not set, all tests default to
|
|
// using https://play.min.io
|
|
if os.Getenv(serverEndpoint) == "" {
|
|
os.Setenv(serverEndpoint, "play.min.io")
|
|
os.Setenv(accessKey, "Q3AM3UQ867SPQQA43P2F")
|
|
os.Setenv(secretKey, "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG")
|
|
os.Setenv(enableHTTPS, "1")
|
|
}
|
|
}
|
|
|
|
var mintDataDir = os.Getenv("MINT_DATA_DIR")
|
|
|
|
func getMintDataDirFilePath(filename string) (fp string) {
|
|
if mintDataDir == "" {
|
|
return
|
|
}
|
|
return filepath.Join(mintDataDir, filename)
|
|
}
|
|
|
|
func newRandomReader(seed, size int64) io.Reader {
|
|
return io.LimitReader(rand.New(rand.NewSource(seed)), size)
|
|
}
|
|
|
|
func mustCrcReader(r io.Reader) uint32 {
|
|
crc := crc32.NewIEEE()
|
|
_, err := io.Copy(crc, r)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return crc.Sum32()
|
|
}
|
|
|
|
func crcMatches(r io.Reader, want uint32) error {
|
|
crc := crc32.NewIEEE()
|
|
_, err := io.Copy(crc, r)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
got := crc.Sum32()
|
|
if got != want {
|
|
return fmt.Errorf("crc mismatch, want %x, got %x", want, got)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func crcMatchesName(r io.Reader, name string) error {
|
|
want := dataFileCRC32[name]
|
|
crc := crc32.NewIEEE()
|
|
_, err := io.Copy(crc, r)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
got := crc.Sum32()
|
|
if got != want {
|
|
return fmt.Errorf("crc mismatch, want %x, got %x", want, got)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// read data from file if it exists or optionally create a buffer of particular size
|
|
func getDataReader(fileName string) io.ReadCloser {
|
|
if mintDataDir == "" {
|
|
size := int64(dataFileMap[fileName])
|
|
if _, ok := dataFileCRC32[fileName]; !ok {
|
|
dataFileCRC32[fileName] = mustCrcReader(newRandomReader(size, size))
|
|
}
|
|
return ioutil.NopCloser(newRandomReader(size, size))
|
|
}
|
|
reader, _ := os.Open(getMintDataDirFilePath(fileName))
|
|
if _, ok := dataFileCRC32[fileName]; !ok {
|
|
dataFileCRC32[fileName] = mustCrcReader(reader)
|
|
reader.Close()
|
|
reader, _ = os.Open(getMintDataDirFilePath(fileName))
|
|
}
|
|
return reader
|
|
}
|
|
|
|
// randString generates random names and prepends them with a known prefix.
|
|
func randString(n int, src rand.Source, prefix string) string {
|
|
b := make([]byte, n)
|
|
// A rand.Int63() generates 63 random bits, enough for letterIdxMax letters!
|
|
for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
|
|
if remain == 0 {
|
|
cache, remain = src.Int63(), letterIdxMax
|
|
}
|
|
if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
|
|
b[i] = letterBytes[idx]
|
|
i--
|
|
}
|
|
cache >>= letterIdxBits
|
|
remain--
|
|
}
|
|
return prefix + string(b[0:30-len(prefix)])
|
|
}
|
|
|
|
var dataFileMap = map[string]int{
|
|
"datafile-0-b": 0,
|
|
"datafile-1-b": 1,
|
|
"datafile-1-kB": 1 * humanize.KiByte,
|
|
"datafile-10-kB": 10 * humanize.KiByte,
|
|
"datafile-33-kB": 33 * humanize.KiByte,
|
|
"datafile-100-kB": 100 * humanize.KiByte,
|
|
"datafile-1.03-MB": 1056 * humanize.KiByte,
|
|
"datafile-1-MB": 1 * humanize.MiByte,
|
|
"datafile-5-MB": 5 * humanize.MiByte,
|
|
"datafile-6-MB": 6 * humanize.MiByte,
|
|
"datafile-11-MB": 11 * humanize.MiByte,
|
|
"datafile-65-MB": 65 * humanize.MiByte,
|
|
"datafile-129-MB": 129 * humanize.MiByte,
|
|
}
|
|
|
|
var dataFileCRC32 = map[string]uint32{}
|
|
|
|
func isFullMode() bool {
|
|
return os.Getenv("MINT_MODE") == "full"
|
|
}
|
|
|
|
func getFuncName() string {
|
|
return getFuncNameLoc(2)
|
|
}
|
|
|
|
func getFuncNameLoc(caller int) string {
|
|
pc, _, _, _ := runtime.Caller(caller)
|
|
return strings.TrimPrefix(runtime.FuncForPC(pc).Name(), "main.")
|
|
}
|
|
|
|
// Tests bucket re-create errors.
|
|
func testMakeBucketError() {
|
|
region := "eu-central-1"
|
|
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "MakeBucket(bucketName, region)"
|
|
// initialize logging params
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"region": region,
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket in 'eu-central-1'.
|
|
if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket Failed", err)
|
|
return
|
|
}
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err == nil {
|
|
logError(testName, function, args, startTime, "", "Bucket already exists", err)
|
|
return
|
|
}
|
|
// Verify valid error response from server.
|
|
if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" &&
|
|
minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" {
|
|
logError(testName, function, args, startTime, "", "Invalid error returned by server", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
func testMetadataSizeLimit() {
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "PutObject(bucketName, objectName, reader, objectSize, opts)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"opts.UserMetadata": "",
|
|
}
|
|
rand.Seed(startTime.Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client creation failed", err)
|
|
return
|
|
}
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Make bucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
const HeaderSizeLimit = 8 * 1024
|
|
const UserMetadataLimit = 2 * 1024
|
|
|
|
// Meta-data greater than the 2 KB limit of AWS - PUT calls with this meta-data should fail
|
|
metadata := make(map[string]string)
|
|
metadata["X-Amz-Meta-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+UserMetadataLimit-len("X-Amz-Meta-Mint-Test")))
|
|
args["metadata"] = fmt.Sprint(metadata)
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata})
|
|
if err == nil {
|
|
logError(testName, function, args, startTime, "", "Created object with user-defined metadata exceeding metadata size limits", nil)
|
|
return
|
|
}
|
|
|
|
// Meta-data (headers) greater than the 8 KB limit of AWS - PUT calls with this meta-data should fail
|
|
metadata = make(map[string]string)
|
|
metadata["X-Amz-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+HeaderSizeLimit-len("X-Amz-Mint-Test")))
|
|
args["metadata"] = fmt.Sprint(metadata)
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata})
|
|
if err == nil {
|
|
logError(testName, function, args, startTime, "", "Created object with headers exceeding header size limits", nil)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests various bucket supported formats.
|
|
func testMakeBucketRegions() {
|
|
region := "eu-central-1"
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "MakeBucket(bucketName, region)"
|
|
// initialize logging params
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"region": region,
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket in 'eu-central-1'.
|
|
if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
// Delete all objects and buckets
|
|
if err = cleanupBucket(bucketName, c); err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
|
|
return
|
|
}
|
|
|
|
// Make a new bucket with '.' in its name, in 'us-west-2'. This
|
|
// request is internally staged into a path style instead of
|
|
// virtual host style.
|
|
region = "us-west-2"
|
|
args["region"] = region
|
|
if err = c.MakeBucket(context.Background(), bucketName+".withperiod", minio.MakeBucketOptions{Region: region}); err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
// Delete all objects and buckets
|
|
if err = cleanupBucket(bucketName+".withperiod", c); err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
|
|
return
|
|
}
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test PutObject using a large data to trigger multipart readat
|
|
func testPutObjectReadAt() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "PutObject(bucketName, objectName, reader, opts)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"opts": "objectContentType",
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Make bucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
bufSize := dataFileMap["datafile-129-MB"]
|
|
reader := getDataReader("datafile-129-MB")
|
|
defer reader.Close()
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
// Object content type
|
|
objectContentType := "binary/octet-stream"
|
|
args["objectContentType"] = objectContentType
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: objectContentType})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Read the data back
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Get Object failed", err)
|
|
return
|
|
}
|
|
|
|
st, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat Object failed", err)
|
|
return
|
|
}
|
|
if st.Size != int64(bufSize) {
|
|
logError(testName, function, args, startTime, "", fmt.Sprintf("Number of bytes in stat does not match, expected %d got %d", bufSize, st.Size), err)
|
|
return
|
|
}
|
|
if st.ContentType != objectContentType && st.ContentType != "application/octet-stream" {
|
|
logError(testName, function, args, startTime, "", "Content types don't match", err)
|
|
return
|
|
}
|
|
if err := crcMatchesName(r, "datafile-129-MB"); err != nil {
|
|
logError(testName, function, args, startTime, "", "data CRC check failed", err)
|
|
return
|
|
}
|
|
if err := r.Close(); err != nil {
|
|
logError(testName, function, args, startTime, "", "Object Close failed", err)
|
|
return
|
|
}
|
|
if err := r.Close(); err == nil {
|
|
logError(testName, function, args, startTime, "", "Object is already closed, didn't return error on Close", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
func testListObjectVersions() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "ListObjectVersions(bucketName, prefix, recursive)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"prefix": "",
|
|
"recursive": "",
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Make bucket failed", err)
|
|
return
|
|
}
|
|
|
|
err = c.EnableVersioning(context.Background(), bucketName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Enable versioning failed", err)
|
|
return
|
|
}
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
bufSize := dataFileMap["datafile-10-kB"]
|
|
reader := getDataReader("datafile-10-kB")
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
reader.Close()
|
|
|
|
bufSize = dataFileMap["datafile-1-b"]
|
|
reader = getDataReader("datafile-1-b")
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
reader.Close()
|
|
|
|
err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Unexpected object deletion", err)
|
|
return
|
|
}
|
|
|
|
var deleteMarkers, versions int
|
|
|
|
objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
|
|
for info := range objectsInfo {
|
|
if info.Err != nil {
|
|
logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
|
|
return
|
|
}
|
|
if info.Key != objectName {
|
|
logError(testName, function, args, startTime, "", "Unexpected object name in listing objects", nil)
|
|
return
|
|
}
|
|
if info.VersionID == "" {
|
|
logError(testName, function, args, startTime, "", "Unexpected version id in listing objects", nil)
|
|
return
|
|
}
|
|
if info.IsDeleteMarker {
|
|
deleteMarkers++
|
|
if !info.IsLatest {
|
|
logError(testName, function, args, startTime, "", "Unexpected IsLatest field in listing objects", nil)
|
|
return
|
|
}
|
|
} else {
|
|
versions++
|
|
}
|
|
}
|
|
|
|
if deleteMarkers != 1 {
|
|
logError(testName, function, args, startTime, "", "Unexpected number of DeleteMarker elements in listing objects", nil)
|
|
return
|
|
}
|
|
|
|
if versions != 2 {
|
|
logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil)
|
|
return
|
|
}
|
|
|
|
// Delete all objects and their versions as long as the bucket itself
|
|
if err = cleanupVersionedBucket(bucketName, c); err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
func testStatObjectWithVersioning() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "StatObject"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Make bucket failed", err)
|
|
return
|
|
}
|
|
|
|
err = c.EnableVersioning(context.Background(), bucketName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Enable versioning failed", err)
|
|
return
|
|
}
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
bufSize := dataFileMap["datafile-10-kB"]
|
|
reader := getDataReader("datafile-10-kB")
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
reader.Close()
|
|
|
|
bufSize = dataFileMap["datafile-1-b"]
|
|
reader = getDataReader("datafile-1-b")
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
reader.Close()
|
|
|
|
objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
|
|
|
|
var results []minio.ObjectInfo
|
|
for info := range objectsInfo {
|
|
if info.Err != nil {
|
|
logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
|
|
return
|
|
}
|
|
results = append(results, info)
|
|
}
|
|
|
|
if len(results) != 2 {
|
|
logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil)
|
|
return
|
|
}
|
|
|
|
for i := 0; i < len(results); i++ {
|
|
opts := minio.StatObjectOptions{VersionID: results[i].VersionID}
|
|
statInfo, err := c.StatObject(context.Background(), bucketName, objectName, opts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "error during HEAD object", err)
|
|
return
|
|
}
|
|
if statInfo.VersionID == "" || statInfo.VersionID != results[i].VersionID {
|
|
logError(testName, function, args, startTime, "", "error during HEAD object, unexpected version id", err)
|
|
return
|
|
}
|
|
if statInfo.ETag != results[i].ETag {
|
|
logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err)
|
|
return
|
|
}
|
|
if statInfo.LastModified.Unix() != results[i].LastModified.Unix() {
|
|
logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err)
|
|
return
|
|
}
|
|
if statInfo.Size != results[i].Size {
|
|
logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
// Delete all objects and their versions as long as the bucket itself
|
|
if err = cleanupVersionedBucket(bucketName, c); err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
func testGetObjectWithVersioning() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "GetObject()"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Make bucket failed", err)
|
|
return
|
|
}
|
|
|
|
err = c.EnableVersioning(context.Background(), bucketName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Enable versioning failed", err)
|
|
return
|
|
}
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
// Save the contents of datafiles to check with GetObject() reader output later
|
|
var buffers [][]byte
|
|
testFiles := []string{"datafile-1-b", "datafile-10-kB"}
|
|
|
|
for _, testFile := range testFiles {
|
|
r := getDataReader(testFile)
|
|
buf, err := ioutil.ReadAll(r)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "unexpected failure", err)
|
|
return
|
|
}
|
|
r.Close()
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
buffers = append(buffers, buf)
|
|
}
|
|
|
|
objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
|
|
|
|
var results []minio.ObjectInfo
|
|
for info := range objectsInfo {
|
|
if info.Err != nil {
|
|
logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
|
|
return
|
|
}
|
|
results = append(results, info)
|
|
}
|
|
|
|
if len(results) != 2 {
|
|
logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil)
|
|
return
|
|
}
|
|
|
|
sort.SliceStable(results, func(i, j int) bool {
|
|
return results[i].Size < results[j].Size
|
|
})
|
|
|
|
sort.SliceStable(buffers, func(i, j int) bool {
|
|
return len(buffers[i]) < len(buffers[j])
|
|
})
|
|
|
|
for i := 0; i < len(results); i++ {
|
|
opts := minio.GetObjectOptions{VersionID: results[i].VersionID}
|
|
reader, err := c.GetObject(context.Background(), bucketName, objectName, opts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "error during GET object", err)
|
|
return
|
|
}
|
|
statInfo, err := reader.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "error during calling reader.Stat()", err)
|
|
return
|
|
}
|
|
if statInfo.ETag != results[i].ETag {
|
|
logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err)
|
|
return
|
|
}
|
|
if statInfo.LastModified.Unix() != results[i].LastModified.Unix() {
|
|
logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err)
|
|
return
|
|
}
|
|
if statInfo.Size != results[i].Size {
|
|
logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err)
|
|
return
|
|
}
|
|
|
|
tmpBuffer := bytes.NewBuffer([]byte{})
|
|
_, err = io.Copy(tmpBuffer, reader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "unexpected io.Copy()", err)
|
|
return
|
|
}
|
|
|
|
if !bytes.Equal(tmpBuffer.Bytes(), buffers[i]) {
|
|
logError(testName, function, args, startTime, "", "unexpected content of GetObject()", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
// Delete all objects and their versions as long as the bucket itself
|
|
if err = cleanupVersionedBucket(bucketName, c); err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
func testPutObjectWithVersioning() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "GetObject()"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Make bucket failed", err)
|
|
return
|
|
}
|
|
|
|
err = c.EnableVersioning(context.Background(), bucketName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Enable versioning failed", err)
|
|
return
|
|
}
|
|
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
const n = 10
|
|
// Read input...
|
|
|
|
// Save the data concurrently.
|
|
var wg sync.WaitGroup
|
|
wg.Add(n)
|
|
buffers := make([][]byte, n)
|
|
var errs [n]error
|
|
for i := 0; i < n; i++ {
|
|
r := newRandomReader(int64((1<<20)*i+i), int64(i))
|
|
buf, err := ioutil.ReadAll(r)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "unexpected failure", err)
|
|
return
|
|
}
|
|
buffers[i] = buf
|
|
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
_, errs[i] = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{PartSize: 5 << 20})
|
|
}(i)
|
|
}
|
|
wg.Wait()
|
|
for _, err := range errs {
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
|
|
var results []minio.ObjectInfo
|
|
for info := range objectsInfo {
|
|
if info.Err != nil {
|
|
logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
|
|
return
|
|
}
|
|
results = append(results, info)
|
|
}
|
|
|
|
if len(results) != n {
|
|
logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil)
|
|
return
|
|
}
|
|
|
|
sort.Slice(results, func(i, j int) bool {
|
|
return results[i].Size < results[j].Size
|
|
})
|
|
|
|
sort.Slice(buffers, func(i, j int) bool {
|
|
return len(buffers[i]) < len(buffers[j])
|
|
})
|
|
|
|
for i := 0; i < len(results); i++ {
|
|
opts := minio.GetObjectOptions{VersionID: results[i].VersionID}
|
|
reader, err := c.GetObject(context.Background(), bucketName, objectName, opts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "error during GET object", err)
|
|
return
|
|
}
|
|
statInfo, err := reader.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "error during calling reader.Stat()", err)
|
|
return
|
|
}
|
|
if statInfo.ETag != results[i].ETag {
|
|
logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err)
|
|
return
|
|
}
|
|
if statInfo.LastModified.Unix() != results[i].LastModified.Unix() {
|
|
logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err)
|
|
return
|
|
}
|
|
if statInfo.Size != results[i].Size {
|
|
logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err)
|
|
return
|
|
}
|
|
|
|
tmpBuffer := bytes.NewBuffer([]byte{})
|
|
_, err = io.Copy(tmpBuffer, reader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "unexpected io.Copy()", err)
|
|
return
|
|
}
|
|
|
|
if !bytes.Equal(tmpBuffer.Bytes(), buffers[i]) {
|
|
logError(testName, function, args, startTime, "", "unexpected content of GetObject()", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
// Delete all objects and their versions as long as the bucket itself
|
|
if err = cleanupVersionedBucket(bucketName, c); err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
func testCopyObjectWithVersioning() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObject()"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Make bucket failed", err)
|
|
return
|
|
}
|
|
|
|
err = c.EnableVersioning(context.Background(), bucketName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Enable versioning failed", err)
|
|
return
|
|
}
|
|
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
testFiles := []string{"datafile-1-b", "datafile-10-kB"}
|
|
for _, testFile := range testFiles {
|
|
r := getDataReader(testFile)
|
|
buf, err := ioutil.ReadAll(r)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "unexpected failure", err)
|
|
return
|
|
}
|
|
r.Close()
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
|
|
var infos []minio.ObjectInfo
|
|
for info := range objectsInfo {
|
|
if info.Err != nil {
|
|
logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
|
|
return
|
|
}
|
|
infos = append(infos, info)
|
|
}
|
|
|
|
sort.Slice(infos, func(i, j int) bool {
|
|
return infos[i].Size < infos[j].Size
|
|
})
|
|
|
|
reader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{VersionID: infos[0].VersionID})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject of the oldest version content failed", err)
|
|
return
|
|
}
|
|
|
|
oldestContent, err := ioutil.ReadAll(reader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Reading the oldest object version failed", err)
|
|
return
|
|
}
|
|
|
|
// Copy Source
|
|
srcOpts := minio.CopySrcOptions{
|
|
Bucket: bucketName,
|
|
Object: objectName,
|
|
VersionID: infos[0].VersionID,
|
|
}
|
|
args["src"] = srcOpts
|
|
|
|
dstOpts := minio.CopyDestOptions{
|
|
Bucket: bucketName,
|
|
Object: objectName + "-copy",
|
|
}
|
|
args["dst"] = dstOpts
|
|
|
|
// Perform the Copy
|
|
if _, err = c.CopyObject(context.Background(), dstOpts, srcOpts); err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Destination object
|
|
readerCopy, err := c.GetObject(context.Background(), bucketName, objectName+"-copy", minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
defer readerCopy.Close()
|
|
|
|
newestContent, err := ioutil.ReadAll(readerCopy)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Reading from GetObject reader failed", err)
|
|
return
|
|
}
|
|
|
|
if len(newestContent) == 0 || !bytes.Equal(oldestContent, newestContent) {
|
|
logError(testName, function, args, startTime, "", "Unexpected destination object content", err)
|
|
return
|
|
}
|
|
|
|
// Delete all objects and their versions as long as the bucket itself
|
|
if err = cleanupVersionedBucket(bucketName, c); err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
func testConcurrentCopyObjectWithVersioning() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObject()"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Make bucket failed", err)
|
|
return
|
|
}
|
|
|
|
err = c.EnableVersioning(context.Background(), bucketName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Enable versioning failed", err)
|
|
return
|
|
}
|
|
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
testFiles := []string{"datafile-10-kB"}
|
|
for _, testFile := range testFiles {
|
|
r := getDataReader(testFile)
|
|
buf, err := ioutil.ReadAll(r)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "unexpected failure", err)
|
|
return
|
|
}
|
|
r.Close()
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
|
|
var infos []minio.ObjectInfo
|
|
for info := range objectsInfo {
|
|
if info.Err != nil {
|
|
logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
|
|
return
|
|
}
|
|
infos = append(infos, info)
|
|
}
|
|
|
|
sort.Slice(infos, func(i, j int) bool {
|
|
return infos[i].Size < infos[j].Size
|
|
})
|
|
|
|
reader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{VersionID: infos[0].VersionID})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject of the oldest version content failed", err)
|
|
return
|
|
}
|
|
|
|
oldestContent, err := ioutil.ReadAll(reader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Reading the oldest object version failed", err)
|
|
return
|
|
}
|
|
|
|
// Copy Source
|
|
srcOpts := minio.CopySrcOptions{
|
|
Bucket: bucketName,
|
|
Object: objectName,
|
|
VersionID: infos[0].VersionID,
|
|
}
|
|
args["src"] = srcOpts
|
|
|
|
dstOpts := minio.CopyDestOptions{
|
|
Bucket: bucketName,
|
|
Object: objectName + "-copy",
|
|
}
|
|
args["dst"] = dstOpts
|
|
|
|
// Perform the Copy concurrently
|
|
const n = 10
|
|
var wg sync.WaitGroup
|
|
wg.Add(n)
|
|
var errs [n]error
|
|
for i := 0; i < n; i++ {
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
_, errs[i] = c.CopyObject(context.Background(), dstOpts, srcOpts)
|
|
}(i)
|
|
}
|
|
wg.Wait()
|
|
for _, err := range errs {
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObject failed", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
objectsInfo = c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: false, Prefix: dstOpts.Object})
|
|
infos = []minio.ObjectInfo{}
|
|
for info := range objectsInfo {
|
|
// Destination object
|
|
readerCopy, err := c.GetObject(context.Background(), bucketName, objectName+"-copy", minio.GetObjectOptions{VersionID: info.VersionID})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
defer readerCopy.Close()
|
|
|
|
newestContent, err := ioutil.ReadAll(readerCopy)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Reading from GetObject reader failed", err)
|
|
return
|
|
}
|
|
|
|
if len(newestContent) == 0 || !bytes.Equal(oldestContent, newestContent) {
|
|
logError(testName, function, args, startTime, "", "Unexpected destination object content", err)
|
|
return
|
|
}
|
|
infos = append(infos, info)
|
|
}
|
|
|
|
if len(infos) != n {
|
|
logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil)
|
|
return
|
|
}
|
|
|
|
// Delete all objects and their versions as long as the bucket itself
|
|
if err = cleanupVersionedBucket(bucketName, c); err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
func testComposeObjectWithVersioning() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "ComposeObject()"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Make bucket failed", err)
|
|
return
|
|
}
|
|
|
|
err = c.EnableVersioning(context.Background(), bucketName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Enable versioning failed", err)
|
|
return
|
|
}
|
|
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
// var testFiles = []string{"datafile-5-MB", "datafile-10-kB"}
|
|
testFiles := []string{"datafile-5-MB", "datafile-10-kB"}
|
|
var testFilesBytes [][]byte
|
|
|
|
for _, testFile := range testFiles {
|
|
r := getDataReader(testFile)
|
|
buf, err := ioutil.ReadAll(r)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "unexpected failure", err)
|
|
return
|
|
}
|
|
r.Close()
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
testFilesBytes = append(testFilesBytes, buf)
|
|
}
|
|
|
|
objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
|
|
|
|
var results []minio.ObjectInfo
|
|
for info := range objectsInfo {
|
|
if info.Err != nil {
|
|
logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
|
|
return
|
|
}
|
|
results = append(results, info)
|
|
}
|
|
|
|
sort.SliceStable(results, func(i, j int) bool {
|
|
return results[i].Size > results[j].Size
|
|
})
|
|
|
|
// Source objects to concatenate. We also specify decryption
|
|
// key for each
|
|
src1 := minio.CopySrcOptions{
|
|
Bucket: bucketName,
|
|
Object: objectName,
|
|
VersionID: results[0].VersionID,
|
|
}
|
|
|
|
src2 := minio.CopySrcOptions{
|
|
Bucket: bucketName,
|
|
Object: objectName,
|
|
VersionID: results[1].VersionID,
|
|
}
|
|
|
|
dst := minio.CopyDestOptions{
|
|
Bucket: bucketName,
|
|
Object: objectName + "-copy",
|
|
}
|
|
|
|
_, err = c.ComposeObject(context.Background(), dst, src1, src2)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ComposeObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Destination object
|
|
readerCopy, err := c.GetObject(context.Background(), bucketName, objectName+"-copy", minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject of the copy object failed", err)
|
|
return
|
|
}
|
|
defer readerCopy.Close()
|
|
|
|
copyContentBytes, err := ioutil.ReadAll(readerCopy)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Reading from the copy object reader failed", err)
|
|
return
|
|
}
|
|
|
|
var expectedContent []byte
|
|
for _, fileBytes := range testFilesBytes {
|
|
expectedContent = append(expectedContent, fileBytes...)
|
|
}
|
|
|
|
if len(copyContentBytes) == 0 || !bytes.Equal(copyContentBytes, expectedContent) {
|
|
logError(testName, function, args, startTime, "", "Unexpected destination object content", err)
|
|
return
|
|
}
|
|
|
|
// Delete all objects and their versions as long as the bucket itself
|
|
if err = cleanupVersionedBucket(bucketName, c); err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
func testRemoveObjectWithVersioning() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "DeleteObject()"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Make bucket failed", err)
|
|
return
|
|
}
|
|
|
|
err = c.EnableVersioning(context.Background(), bucketName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Enable versioning failed", err)
|
|
return
|
|
}
|
|
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader("datafile-10-kB"), int64(dataFileMap["datafile-10-kB"]), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
|
|
var version minio.ObjectInfo
|
|
for info := range objectsInfo {
|
|
if info.Err != nil {
|
|
logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
|
|
return
|
|
}
|
|
version = info
|
|
break
|
|
}
|
|
|
|
err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{VersionID: version.VersionID})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "DeleteObject failed", err)
|
|
return
|
|
}
|
|
|
|
objectsInfo = c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
|
|
for range objectsInfo {
|
|
logError(testName, function, args, startTime, "", "Unexpected versioning info, should not have any one ", err)
|
|
return
|
|
}
|
|
|
|
err = c.RemoveBucket(context.Background(), bucketName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
func testRemoveObjectsWithVersioning() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "DeleteObjects()"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Make bucket failed", err)
|
|
return
|
|
}
|
|
|
|
err = c.EnableVersioning(context.Background(), bucketName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Enable versioning failed", err)
|
|
return
|
|
}
|
|
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader("datafile-10-kB"), int64(dataFileMap["datafile-10-kB"]), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
objectsVersions := make(chan minio.ObjectInfo)
|
|
go func() {
|
|
objectsVersionsInfo := c.ListObjects(context.Background(), bucketName,
|
|
minio.ListObjectsOptions{WithVersions: true, Recursive: true})
|
|
for info := range objectsVersionsInfo {
|
|
if info.Err != nil {
|
|
logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
|
|
return
|
|
}
|
|
objectsVersions <- info
|
|
}
|
|
close(objectsVersions)
|
|
}()
|
|
|
|
removeErrors := c.RemoveObjects(context.Background(), bucketName, objectsVersions, minio.RemoveObjectsOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "DeleteObjects call failed", err)
|
|
return
|
|
}
|
|
|
|
for e := range removeErrors {
|
|
if e.Err != nil {
|
|
logError(testName, function, args, startTime, "", "Single delete operation failed", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
objectsVersionsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
|
|
for range objectsVersionsInfo {
|
|
logError(testName, function, args, startTime, "", "Unexpected versioning info, should not have any one ", err)
|
|
return
|
|
}
|
|
|
|
err = c.RemoveBucket(context.Background(), bucketName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
func testObjectTaggingWithVersioning() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "{Get,Set,Remove}ObjectTagging()"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Make bucket failed", err)
|
|
return
|
|
}
|
|
|
|
err = c.EnableVersioning(context.Background(), bucketName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Enable versioning failed", err)
|
|
return
|
|
}
|
|
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
for _, file := range []string{"datafile-1-b", "datafile-10-kB"} {
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader(file), int64(dataFileMap[file]), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
versionsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
|
|
|
|
var versions []minio.ObjectInfo
|
|
for info := range versionsInfo {
|
|
if info.Err != nil {
|
|
logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
|
|
return
|
|
}
|
|
versions = append(versions, info)
|
|
}
|
|
|
|
sort.SliceStable(versions, func(i, j int) bool {
|
|
return versions[i].Size < versions[j].Size
|
|
})
|
|
|
|
tagsV1 := map[string]string{"key1": "val1"}
|
|
t1, err := tags.MapToObjectTags(tagsV1)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObjectTagging (1) failed", err)
|
|
return
|
|
}
|
|
|
|
err = c.PutObjectTagging(context.Background(), bucketName, objectName, t1, minio.PutObjectTaggingOptions{VersionID: versions[0].VersionID})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObjectTagging (1) failed", err)
|
|
return
|
|
}
|
|
|
|
tagsV2 := map[string]string{"key2": "val2"}
|
|
t2, err := tags.MapToObjectTags(tagsV2)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObjectTagging (1) failed", err)
|
|
return
|
|
}
|
|
|
|
err = c.PutObjectTagging(context.Background(), bucketName, objectName, t2, minio.PutObjectTaggingOptions{VersionID: versions[1].VersionID})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObjectTagging (2) failed", err)
|
|
return
|
|
}
|
|
|
|
tagsEqual := func(tags1, tags2 map[string]string) bool {
|
|
for k1, v1 := range tags1 {
|
|
v2, found := tags2[k1]
|
|
if found {
|
|
if v1 != v2 {
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
gotTagsV1, err := c.GetObjectTagging(context.Background(), bucketName, objectName, minio.GetObjectTaggingOptions{VersionID: versions[0].VersionID})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObjectTagging failed", err)
|
|
return
|
|
}
|
|
|
|
if !tagsEqual(t1.ToMap(), gotTagsV1.ToMap()) {
|
|
logError(testName, function, args, startTime, "", "Unexpected tags content (1)", err)
|
|
return
|
|
}
|
|
|
|
gotTagsV2, err := c.GetObjectTagging(context.Background(), bucketName, objectName, minio.GetObjectTaggingOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObjectTaggingContext failed", err)
|
|
return
|
|
}
|
|
|
|
if !tagsEqual(t2.ToMap(), gotTagsV2.ToMap()) {
|
|
logError(testName, function, args, startTime, "", "Unexpected tags content (2)", err)
|
|
return
|
|
}
|
|
|
|
err = c.RemoveObjectTagging(context.Background(), bucketName, objectName, minio.RemoveObjectTaggingOptions{VersionID: versions[0].VersionID})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObjectTagging (2) failed", err)
|
|
return
|
|
}
|
|
|
|
emptyTags, err := c.GetObjectTagging(context.Background(), bucketName, objectName,
|
|
minio.GetObjectTaggingOptions{VersionID: versions[0].VersionID})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObjectTagging failed", err)
|
|
return
|
|
}
|
|
|
|
if len(emptyTags.ToMap()) != 0 {
|
|
logError(testName, function, args, startTime, "", "Unexpected tags content (2)", err)
|
|
return
|
|
}
|
|
|
|
// Delete all objects and their versions as long as the bucket itself
|
|
if err = cleanupVersionedBucket(bucketName, c); err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test PutObject using a large data to trigger multipart readat
|
|
func testPutObjectWithMetadata() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "PutObject(bucketName, objectName, reader,size, opts)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}",
|
|
}
|
|
|
|
if !isFullMode() {
|
|
ignoredLog(testName, function, args, startTime, "Skipping functional tests for short/quick runs").Info()
|
|
return
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Make bucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
bufSize := dataFileMap["datafile-129-MB"]
|
|
reader := getDataReader("datafile-129-MB")
|
|
defer reader.Close()
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
// Object custom metadata
|
|
customContentType := "custom/contenttype"
|
|
|
|
args["metadata"] = map[string][]string{
|
|
"Content-Type": {customContentType},
|
|
"X-Amz-Meta-CustomKey": {"extra spaces in value"},
|
|
}
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{
|
|
ContentType: customContentType,
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Read the data back
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
|
|
st, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
if st.Size != int64(bufSize) {
|
|
logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err)
|
|
return
|
|
}
|
|
if st.ContentType != customContentType && st.ContentType != "application/octet-stream" {
|
|
logError(testName, function, args, startTime, "", "ContentType does not match, expected "+customContentType+" got "+st.ContentType, err)
|
|
return
|
|
}
|
|
if err := crcMatchesName(r, "datafile-129-MB"); err != nil {
|
|
logError(testName, function, args, startTime, "", "data CRC check failed", err)
|
|
return
|
|
}
|
|
if err := r.Close(); err != nil {
|
|
logError(testName, function, args, startTime, "", "Object Close failed", err)
|
|
return
|
|
}
|
|
if err := r.Close(); err == nil {
|
|
logError(testName, function, args, startTime, "", "Object already closed, should respond with error", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
func testPutObjectWithContentLanguage() {
|
|
// initialize logging params
|
|
objectName := "test-object"
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "PutObject(bucketName, objectName, reader, size, opts)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectName": objectName,
|
|
"size": -1,
|
|
"opts": "",
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
data := []byte{}
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(data), int64(0), minio.PutObjectOptions{
|
|
ContentLanguage: "en",
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject failed", err)
|
|
return
|
|
}
|
|
|
|
if objInfo.Metadata.Get("Content-Language") != "en" {
|
|
logError(testName, function, args, startTime, "", "Expected content-language 'en' doesn't match with StatObject return value", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test put object with streaming signature.
|
|
func testPutObjectStreaming() {
|
|
// initialize logging params
|
|
objectName := "test-object"
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "PutObject(bucketName, objectName, reader,size,opts)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectName": objectName,
|
|
"size": -1,
|
|
"opts": "",
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Upload an object.
|
|
sizes := []int64{0, 64*1024 - 1, 64 * 1024}
|
|
|
|
for _, size := range sizes {
|
|
data := newRandomReader(size, size)
|
|
ui, err := c.PutObject(context.Background(), bucketName, objectName, data, int64(size), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err)
|
|
return
|
|
}
|
|
|
|
if ui.Size != size {
|
|
logError(testName, function, args, startTime, "", "PutObjectStreaming result has unexpected size", nil)
|
|
return
|
|
}
|
|
|
|
objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject failed", err)
|
|
return
|
|
}
|
|
if objInfo.Size != size {
|
|
logError(testName, function, args, startTime, "", "Unexpected size", err)
|
|
return
|
|
}
|
|
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test get object seeker from the end, using whence set to '2'.
|
|
func testGetObjectSeekEnd() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "GetObject(bucketName, objectName)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Generate 33K of data.
|
|
bufSize := dataFileMap["datafile-33-kB"]
|
|
reader := getDataReader("datafile-33-kB")
|
|
defer reader.Close()
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
buf, err := ioutil.ReadAll(reader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Read the data back
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
|
|
st, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(bufSize) {
|
|
logError(testName, function, args, startTime, "", "Number of bytes read does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err)
|
|
return
|
|
}
|
|
|
|
pos, err := r.Seek(-100, 2)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Object Seek failed", err)
|
|
return
|
|
}
|
|
if pos != st.Size-100 {
|
|
logError(testName, function, args, startTime, "", "Incorrect position", err)
|
|
return
|
|
}
|
|
buf2 := make([]byte, 100)
|
|
m, err := readFull(r, buf2)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Error reading through readFull", err)
|
|
return
|
|
}
|
|
if m != len(buf2) {
|
|
logError(testName, function, args, startTime, "", "Number of bytes dont match, expected "+string(len(buf2))+" got "+string(m), err)
|
|
return
|
|
}
|
|
hexBuf1 := fmt.Sprintf("%02x", buf[len(buf)-100:])
|
|
hexBuf2 := fmt.Sprintf("%02x", buf2[:m])
|
|
if hexBuf1 != hexBuf2 {
|
|
logError(testName, function, args, startTime, "", "Values at same index dont match", err)
|
|
return
|
|
}
|
|
pos, err = r.Seek(-100, 2)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Object Seek failed", err)
|
|
return
|
|
}
|
|
if pos != st.Size-100 {
|
|
logError(testName, function, args, startTime, "", "Incorrect position", err)
|
|
return
|
|
}
|
|
if err = r.Close(); err != nil {
|
|
logError(testName, function, args, startTime, "", "ObjectClose failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test get object reader to not throw error on being closed twice.
|
|
func testGetObjectClosedTwice() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "GetObject(bucketName, objectName)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Generate 33K of data.
|
|
bufSize := dataFileMap["datafile-33-kB"]
|
|
reader := getDataReader("datafile-33-kB")
|
|
defer reader.Close()
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Read the data back
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
|
|
st, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
if st.Size != int64(bufSize) {
|
|
logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err)
|
|
return
|
|
}
|
|
if err := crcMatchesName(r, "datafile-33-kB"); err != nil {
|
|
logError(testName, function, args, startTime, "", "data CRC check failed", err)
|
|
return
|
|
}
|
|
if err := r.Close(); err != nil {
|
|
logError(testName, function, args, startTime, "", "Object Close failed", err)
|
|
return
|
|
}
|
|
if err := r.Close(); err == nil {
|
|
logError(testName, function, args, startTime, "", "Already closed object. No error returned", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test RemoveObjects request where context cancels after timeout
|
|
func testRemoveObjectsContext() {
|
|
// Initialize logging params.
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "RemoveObjects(ctx, bucketName, objectsCh)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
}
|
|
|
|
// Seed random based on current tie.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
// Enable tracing, write to stdout.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Generate put data.
|
|
r := bytes.NewReader(bytes.Repeat([]byte("a"), 8))
|
|
|
|
// Multi remove of 20 objects.
|
|
nrObjects := 20
|
|
objectsCh := make(chan minio.ObjectInfo)
|
|
go func() {
|
|
defer close(objectsCh)
|
|
for i := 0; i < nrObjects; i++ {
|
|
objectName := "sample" + strconv.Itoa(i) + ".txt"
|
|
info, err := c.PutObject(context.Background(), bucketName, objectName, r, 8,
|
|
minio.PutObjectOptions{ContentType: "application/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
continue
|
|
}
|
|
objectsCh <- minio.ObjectInfo{
|
|
Key: info.Key,
|
|
VersionID: info.VersionID,
|
|
}
|
|
}
|
|
}()
|
|
// Set context to cancel in 1 nanosecond.
|
|
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
|
|
args["ctx"] = ctx
|
|
defer cancel()
|
|
|
|
// Call RemoveObjects API with short timeout.
|
|
errorCh := c.RemoveObjects(ctx, bucketName, objectsCh, minio.RemoveObjectsOptions{})
|
|
// Check for error.
|
|
select {
|
|
case r := <-errorCh:
|
|
if r.Err == nil {
|
|
logError(testName, function, args, startTime, "", "RemoveObjects should fail on short timeout", err)
|
|
return
|
|
}
|
|
}
|
|
// Set context with longer timeout.
|
|
ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
|
|
args["ctx"] = ctx
|
|
defer cancel()
|
|
// Perform RemoveObjects with the longer timeout. Expect the removals to succeed.
|
|
errorCh = c.RemoveObjects(ctx, bucketName, objectsCh, minio.RemoveObjectsOptions{})
|
|
select {
|
|
case r, more := <-errorCh:
|
|
if more || r.Err != nil {
|
|
logError(testName, function, args, startTime, "", "Unexpected error", r.Err)
|
|
return
|
|
}
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test removing multiple objects with Remove API
|
|
func testRemoveMultipleObjects() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "RemoveObjects(bucketName, objectsCh)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Enable tracing, write to stdout.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
r := bytes.NewReader(bytes.Repeat([]byte("a"), 8))
|
|
|
|
// Multi remove of 1100 objects
|
|
nrObjects := 200
|
|
|
|
objectsCh := make(chan minio.ObjectInfo)
|
|
|
|
go func() {
|
|
defer close(objectsCh)
|
|
// Upload objects and send them to objectsCh
|
|
for i := 0; i < nrObjects; i++ {
|
|
objectName := "sample" + strconv.Itoa(i) + ".txt"
|
|
info, err := c.PutObject(context.Background(), bucketName, objectName, r, 8,
|
|
minio.PutObjectOptions{ContentType: "application/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
continue
|
|
}
|
|
objectsCh <- minio.ObjectInfo{
|
|
Key: info.Key,
|
|
VersionID: info.VersionID,
|
|
}
|
|
}
|
|
}()
|
|
|
|
// Call RemoveObjects API
|
|
errorCh := c.RemoveObjects(context.Background(), bucketName, objectsCh, minio.RemoveObjectsOptions{})
|
|
|
|
// Check if errorCh doesn't receive any error
|
|
select {
|
|
case r, more := <-errorCh:
|
|
if more {
|
|
logError(testName, function, args, startTime, "", "Unexpected error", r.Err)
|
|
return
|
|
}
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test removing multiple objects and check for results
|
|
func testRemoveMultipleObjectsWithResult() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "RemoveObjects(bucketName, objectsCh)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Enable tracing, write to stdout.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupVersionedBucket(bucketName, c)
|
|
|
|
r := bytes.NewReader(bytes.Repeat([]byte("a"), 8))
|
|
|
|
nrObjects := 10
|
|
nrLockedObjects := 5
|
|
|
|
objectsCh := make(chan minio.ObjectInfo)
|
|
|
|
go func() {
|
|
defer close(objectsCh)
|
|
// Upload objects and send them to objectsCh
|
|
for i := 0; i < nrObjects; i++ {
|
|
objectName := "sample" + strconv.Itoa(i) + ".txt"
|
|
info, err := c.PutObject(context.Background(), bucketName, objectName, r, 8,
|
|
minio.PutObjectOptions{ContentType: "application/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
if i < nrLockedObjects {
|
|
// t := time.Date(2130, time.April, 25, 14, 0, 0, 0, time.UTC)
|
|
t := time.Now().Add(5 * time.Minute)
|
|
m := minio.RetentionMode(minio.Governance)
|
|
opts := minio.PutObjectRetentionOptions{
|
|
GovernanceBypass: false,
|
|
RetainUntilDate: &t,
|
|
Mode: &m,
|
|
VersionID: info.VersionID,
|
|
}
|
|
err = c.PutObjectRetention(context.Background(), bucketName, objectName, opts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Error setting retention", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
objectsCh <- minio.ObjectInfo{
|
|
Key: info.Key,
|
|
VersionID: info.VersionID,
|
|
}
|
|
}
|
|
}()
|
|
|
|
// Call RemoveObjects API
|
|
resultCh := c.RemoveObjectsWithResult(context.Background(), bucketName, objectsCh, minio.RemoveObjectsOptions{})
|
|
|
|
var foundNil, foundErr int
|
|
|
|
for {
|
|
// Check if errorCh doesn't receive any error
|
|
select {
|
|
case deleteRes, ok := <-resultCh:
|
|
if !ok {
|
|
goto out
|
|
}
|
|
if deleteRes.ObjectName == "" {
|
|
logError(testName, function, args, startTime, "", "Unexpected object name", nil)
|
|
return
|
|
}
|
|
if deleteRes.ObjectVersionID == "" {
|
|
logError(testName, function, args, startTime, "", "Unexpected object version ID", nil)
|
|
return
|
|
}
|
|
|
|
if deleteRes.Err == nil {
|
|
foundNil++
|
|
} else {
|
|
foundErr++
|
|
}
|
|
}
|
|
}
|
|
out:
|
|
if foundNil+foundErr != nrObjects {
|
|
logError(testName, function, args, startTime, "", "Unexpected number of results", nil)
|
|
return
|
|
}
|
|
|
|
if foundNil != nrObjects-nrLockedObjects {
|
|
logError(testName, function, args, startTime, "", "Unexpected number of nil errors", nil)
|
|
return
|
|
}
|
|
|
|
if foundErr != nrLockedObjects {
|
|
logError(testName, function, args, startTime, "", "Unexpected number of errors", nil)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests FPutObject of a big file to trigger multipart
|
|
func testFPutObjectMultipart() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "FPutObject(bucketName, objectName, fileName, opts)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"fileName": "",
|
|
"opts": "",
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload.
|
|
fileName := getMintDataDirFilePath("datafile-129-MB")
|
|
if fileName == "" {
|
|
// Make a temp file with minPartSize bytes of data.
|
|
file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest")
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "TempFile creation failed", err)
|
|
return
|
|
}
|
|
// Upload 2 parts to utilize all 3 'workers' in multipart and still have a part to upload.
|
|
if _, err = io.Copy(file, getDataReader("datafile-129-MB")); err != nil {
|
|
logError(testName, function, args, startTime, "", "Copy failed", err)
|
|
return
|
|
}
|
|
if err = file.Close(); err != nil {
|
|
logError(testName, function, args, startTime, "", "File Close failed", err)
|
|
return
|
|
}
|
|
fileName = file.Name()
|
|
args["fileName"] = fileName
|
|
}
|
|
totalSize := dataFileMap["datafile-129-MB"]
|
|
// Set base object name
|
|
objectName := bucketName + "FPutObject" + "-standard"
|
|
args["objectName"] = objectName
|
|
|
|
objectContentType := "testapplication/octet-stream"
|
|
args["objectContentType"] = objectContentType
|
|
|
|
// Perform standard FPutObject with contentType provided (Expecting application/octet-stream)
|
|
_, err = c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ContentType: objectContentType})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "FPutObject failed", err)
|
|
return
|
|
}
|
|
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
objInfo, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Unexpected error", err)
|
|
return
|
|
}
|
|
if objInfo.Size != int64(totalSize) {
|
|
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(totalSize))+" got "+string(objInfo.Size), err)
|
|
return
|
|
}
|
|
if objInfo.ContentType != objectContentType && objInfo.ContentType != "application/octet-stream" {
|
|
logError(testName, function, args, startTime, "", "ContentType doesn't match", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests FPutObject with null contentType (default = application/octet-stream)
|
|
func testFPutObject() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "FPutObject(bucketName, objectName, fileName, opts)"
|
|
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"fileName": "",
|
|
"opts": "",
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
location := "us-east-1"
|
|
|
|
// Make a new bucket.
|
|
args["bucketName"] = bucketName
|
|
args["location"] = location
|
|
function = "MakeBucket(bucketName, location)"
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: location})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Upload 3 parts worth of data to use all 3 of multiparts 'workers' and have an extra part.
|
|
// Use different data in part for multipart tests to check parts are uploaded in correct order.
|
|
fName := getMintDataDirFilePath("datafile-129-MB")
|
|
if fName == "" {
|
|
// Make a temp file with minPartSize bytes of data.
|
|
file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest")
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "TempFile creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Upload 3 parts to utilize all 3 'workers' in multipart and still have a part to upload.
|
|
if _, err = io.Copy(file, getDataReader("datafile-129-MB")); err != nil {
|
|
logError(testName, function, args, startTime, "", "File copy failed", err)
|
|
return
|
|
}
|
|
// Close the file pro-actively for windows.
|
|
if err = file.Close(); err != nil {
|
|
logError(testName, function, args, startTime, "", "File close failed", err)
|
|
return
|
|
}
|
|
defer os.Remove(file.Name())
|
|
fName = file.Name()
|
|
}
|
|
|
|
// Set base object name
|
|
function = "FPutObject(bucketName, objectName, fileName, opts)"
|
|
objectName := bucketName + "FPutObject"
|
|
args["objectName"] = objectName + "-standard"
|
|
args["fileName"] = fName
|
|
args["opts"] = minio.PutObjectOptions{ContentType: "application/octet-stream"}
|
|
|
|
// Perform standard FPutObject with contentType provided (Expecting application/octet-stream)
|
|
ui, err := c.FPutObject(context.Background(), bucketName, objectName+"-standard", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "FPutObject failed", err)
|
|
return
|
|
}
|
|
|
|
if ui.Size != int64(dataFileMap["datafile-129-MB"]) {
|
|
logError(testName, function, args, startTime, "", "FPutObject returned an unexpected upload size", err)
|
|
return
|
|
}
|
|
|
|
// Perform FPutObject with no contentType provided (Expecting application/octet-stream)
|
|
args["objectName"] = objectName + "-Octet"
|
|
_, err = c.FPutObject(context.Background(), bucketName, objectName+"-Octet", fName, minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "File close failed", err)
|
|
return
|
|
}
|
|
|
|
srcFile, err := os.Open(fName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "File open failed", err)
|
|
return
|
|
}
|
|
defer srcFile.Close()
|
|
// Add extension to temp file name
|
|
tmpFile, err := os.Create(fName + ".gtar")
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "File create failed", err)
|
|
return
|
|
}
|
|
_, err = io.Copy(tmpFile, srcFile)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "File copy failed", err)
|
|
return
|
|
}
|
|
tmpFile.Close()
|
|
|
|
// Perform FPutObject with no contentType provided (Expecting application/x-gtar)
|
|
args["objectName"] = objectName + "-GTar"
|
|
args["opts"] = minio.PutObjectOptions{}
|
|
_, err = c.FPutObject(context.Background(), bucketName, objectName+"-GTar", fName+".gtar", minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "FPutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Check headers
|
|
function = "StatObject(bucketName, objectName, opts)"
|
|
args["objectName"] = objectName + "-standard"
|
|
rStandard, err := c.StatObject(context.Background(), bucketName, objectName+"-standard", minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject failed", err)
|
|
return
|
|
}
|
|
if rStandard.ContentType != "application/octet-stream" {
|
|
logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rStandard.ContentType, err)
|
|
return
|
|
}
|
|
|
|
function = "StatObject(bucketName, objectName, opts)"
|
|
args["objectName"] = objectName + "-Octet"
|
|
rOctet, err := c.StatObject(context.Background(), bucketName, objectName+"-Octet", minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject failed", err)
|
|
return
|
|
}
|
|
if rOctet.ContentType != "application/octet-stream" {
|
|
logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rOctet.ContentType, err)
|
|
return
|
|
}
|
|
|
|
function = "StatObject(bucketName, objectName, opts)"
|
|
args["objectName"] = objectName + "-GTar"
|
|
rGTar, err := c.StatObject(context.Background(), bucketName, objectName+"-GTar", minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject failed", err)
|
|
return
|
|
}
|
|
if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" && rGTar.ContentType != "application/x-tar" {
|
|
logError(testName, function, args, startTime, "", "ContentType does not match, expected application/x-tar or application/octet-stream, got "+rGTar.ContentType, err)
|
|
return
|
|
}
|
|
|
|
os.Remove(fName + ".gtar")
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests FPutObject request when context cancels after timeout
|
|
func testFPutObjectContext() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "FPutObject(bucketName, objectName, fileName, opts)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"fileName": "",
|
|
"opts": "",
|
|
}
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Upload 1 parts worth of data to use multipart upload.
|
|
// Use different data in part for multipart tests to check parts are uploaded in correct order.
|
|
fName := getMintDataDirFilePath("datafile-1-MB")
|
|
if fName == "" {
|
|
// Make a temp file with 1 MiB bytes of data.
|
|
file, err := ioutil.TempFile(os.TempDir(), "FPutObjectContextTest")
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "TempFile creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Upload 1 parts to trigger multipart upload
|
|
if _, err = io.Copy(file, getDataReader("datafile-1-MB")); err != nil {
|
|
logError(testName, function, args, startTime, "", "File copy failed", err)
|
|
return
|
|
}
|
|
// Close the file pro-actively for windows.
|
|
if err = file.Close(); err != nil {
|
|
logError(testName, function, args, startTime, "", "File close failed", err)
|
|
return
|
|
}
|
|
defer os.Remove(file.Name())
|
|
fName = file.Name()
|
|
}
|
|
|
|
// Set base object name
|
|
objectName := bucketName + "FPutObjectContext"
|
|
args["objectName"] = objectName
|
|
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
|
|
args["ctx"] = ctx
|
|
defer cancel()
|
|
|
|
// Perform FPutObject with contentType provided (Expecting application/octet-stream)
|
|
_, err = c.FPutObject(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"})
|
|
if err == nil {
|
|
logError(testName, function, args, startTime, "", "FPutObject should fail on short timeout", err)
|
|
return
|
|
}
|
|
ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
|
|
defer cancel()
|
|
// Perform FPutObject with a long timeout. Expect the put object to succeed
|
|
_, err = c.FPutObject(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "FPutObject shouldn't fail on long timeout", err)
|
|
return
|
|
}
|
|
|
|
_, err = c.StatObject(context.Background(), bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests FPutObject request when context cancels after timeout
|
|
func testFPutObjectContextV2() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "FPutObjectContext(ctx, bucketName, objectName, fileName, opts)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"opts": "minio.PutObjectOptions{ContentType:objectContentType}",
|
|
}
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Upload 1 parts worth of data to use multipart upload.
|
|
// Use different data in part for multipart tests to check parts are uploaded in correct order.
|
|
fName := getMintDataDirFilePath("datafile-1-MB")
|
|
if fName == "" {
|
|
// Make a temp file with 1 MiB bytes of data.
|
|
file, err := ioutil.TempFile(os.TempDir(), "FPutObjectContextTest")
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Temp file creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Upload 1 parts to trigger multipart upload
|
|
if _, err = io.Copy(file, getDataReader("datafile-1-MB")); err != nil {
|
|
logError(testName, function, args, startTime, "", "File copy failed", err)
|
|
return
|
|
}
|
|
|
|
// Close the file pro-actively for windows.
|
|
if err = file.Close(); err != nil {
|
|
logError(testName, function, args, startTime, "", "File close failed", err)
|
|
return
|
|
}
|
|
defer os.Remove(file.Name())
|
|
fName = file.Name()
|
|
}
|
|
|
|
// Set base object name
|
|
objectName := bucketName + "FPutObjectContext"
|
|
args["objectName"] = objectName
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
|
|
args["ctx"] = ctx
|
|
defer cancel()
|
|
|
|
// Perform FPutObject with contentType provided (Expecting application/octet-stream)
|
|
_, err = c.FPutObject(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"})
|
|
if err == nil {
|
|
logError(testName, function, args, startTime, "", "FPutObject should fail on short timeout", err)
|
|
return
|
|
}
|
|
ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
|
|
defer cancel()
|
|
// Perform FPutObject with a long timeout. Expect the put object to succeed
|
|
_, err = c.FPutObject(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "FPutObject shouldn't fail on longer timeout", err)
|
|
return
|
|
}
|
|
|
|
_, err = c.StatObject(context.Background(), bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test validates putObject with context to see if request cancellation is honored.
|
|
func testPutObjectContext() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "PutObject(ctx, bucketName, objectName, fileName, opts)"
|
|
args := map[string]interface{}{
|
|
"ctx": "",
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"opts": "",
|
|
}
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Make a new bucket.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket call failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
bufSize := dataFileMap["datafile-33-kB"]
|
|
reader := getDataReader("datafile-33-kB")
|
|
defer reader.Close()
|
|
objectName := fmt.Sprintf("test-file-%v", rand.Uint32())
|
|
args["objectName"] = objectName
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
|
|
cancel()
|
|
args["ctx"] = ctx
|
|
args["opts"] = minio.PutObjectOptions{ContentType: "binary/octet-stream"}
|
|
|
|
_, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err == nil {
|
|
logError(testName, function, args, startTime, "", "PutObject should fail on short timeout", err)
|
|
return
|
|
}
|
|
|
|
ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
|
|
args["ctx"] = ctx
|
|
|
|
defer cancel()
|
|
reader = getDataReader("datafile-33-kB")
|
|
defer reader.Close()
|
|
_, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject with long timeout failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests get object with s3zip extensions.
|
|
func testGetObjectS3Zip() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "GetObject(bucketName, objectName)"
|
|
args := map[string]interface{}{"x-minio-extract": true}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer func() {
|
|
// Delete all objects and buckets
|
|
if err = cleanupBucket(bucketName, c); err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
|
|
return
|
|
}
|
|
}()
|
|
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + ".zip"
|
|
args["objectName"] = objectName
|
|
|
|
var zipFile bytes.Buffer
|
|
zw := zip.NewWriter(&zipFile)
|
|
rng := rand.New(rand.NewSource(0xc0cac01a))
|
|
const nFiles = 500
|
|
for i := 0; i <= nFiles; i++ {
|
|
if i == nFiles {
|
|
// Make one large, compressible file.
|
|
i = 1000000
|
|
}
|
|
b := make([]byte, i)
|
|
if i < nFiles {
|
|
rng.Read(b)
|
|
}
|
|
wc, err := zw.Create(fmt.Sprintf("test/small/file-%d.bin", i))
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "zw.Create failed", err)
|
|
return
|
|
}
|
|
wc.Write(b)
|
|
}
|
|
err = zw.Close()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "zw.Close failed", err)
|
|
return
|
|
}
|
|
buf := zipFile.Bytes()
|
|
|
|
// Save the data
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Read the data back
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
|
|
st, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat object failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(len(buf)) {
|
|
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(len(buf))+", got "+string(st.Size), err)
|
|
return
|
|
}
|
|
r.Close()
|
|
|
|
zr, err := zip.NewReader(bytes.NewReader(buf), int64(len(buf)))
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "zip.NewReader failed", err)
|
|
return
|
|
}
|
|
lOpts := minio.ListObjectsOptions{}
|
|
lOpts.Set("x-minio-extract", "true")
|
|
lOpts.Prefix = objectName + "/"
|
|
lOpts.Recursive = true
|
|
list := c.ListObjects(context.Background(), bucketName, lOpts)
|
|
listed := map[string]minio.ObjectInfo{}
|
|
for item := range list {
|
|
if item.Err != nil {
|
|
break
|
|
}
|
|
listed[item.Key] = item
|
|
}
|
|
if len(listed) == 0 {
|
|
// Assume we are running against non-minio.
|
|
args["SKIPPED"] = true
|
|
ignoredLog(testName, function, args, startTime, "s3zip does not appear to be present").Info()
|
|
return
|
|
}
|
|
|
|
for _, file := range zr.File {
|
|
if file.FileInfo().IsDir() {
|
|
continue
|
|
}
|
|
args["zipfile"] = file.Name
|
|
zfr, err := file.Open()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "file.Open failed", err)
|
|
return
|
|
}
|
|
want, err := ioutil.ReadAll(zfr)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "fzip file read failed", err)
|
|
return
|
|
}
|
|
|
|
opts := minio.GetObjectOptions{}
|
|
opts.Set("x-minio-extract", "true")
|
|
key := path.Join(objectName, file.Name)
|
|
r, err = c.GetObject(context.Background(), bucketName, key, opts)
|
|
if err != nil {
|
|
terr := minio.ToErrorResponse(err)
|
|
if terr.StatusCode != http.StatusNotFound {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
}
|
|
return
|
|
}
|
|
got, err := ioutil.ReadAll(r)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
r.Close()
|
|
if !bytes.Equal(want, got) {
|
|
logError(testName, function, args, startTime, "", "Content mismatch", err)
|
|
return
|
|
}
|
|
oi, ok := listed[key]
|
|
if !ok {
|
|
logError(testName, function, args, startTime, "", "Object Missing", fmt.Errorf("%s not present in listing", key))
|
|
return
|
|
}
|
|
if int(oi.Size) != len(got) {
|
|
logError(testName, function, args, startTime, "", "Object Size Incorrect", fmt.Errorf("listing %d, read %d", oi.Size, len(got)))
|
|
return
|
|
}
|
|
delete(listed, key)
|
|
}
|
|
delete(args, "zipfile")
|
|
if len(listed) > 0 {
|
|
logError(testName, function, args, startTime, "", "Extra listed objects", fmt.Errorf("left over: %v", listed))
|
|
return
|
|
}
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests get object ReaderSeeker interface methods.
|
|
func testGetObjectReadSeekFunctional() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "GetObject(bucketName, objectName)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer func() {
|
|
// Delete all objects and buckets
|
|
if err = cleanupBucket(bucketName, c); err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
|
|
return
|
|
}
|
|
}()
|
|
|
|
// Generate 33K of data.
|
|
bufSize := dataFileMap["datafile-33-kB"]
|
|
reader := getDataReader("datafile-33-kB")
|
|
defer reader.Close()
|
|
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
buf, err := ioutil.ReadAll(reader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
|
|
// Save the data
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Read the data back
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
|
|
st, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat object failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(bufSize) {
|
|
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
|
|
return
|
|
}
|
|
|
|
// This following function helps us to compare data from the reader after seek
|
|
// with the data from the original buffer
|
|
cmpData := func(r io.Reader, start, end int) {
|
|
if end-start == 0 {
|
|
return
|
|
}
|
|
buffer := bytes.NewBuffer([]byte{})
|
|
if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil {
|
|
if err != io.EOF {
|
|
logError(testName, function, args, startTime, "", "CopyN failed", err)
|
|
return
|
|
}
|
|
}
|
|
if !bytes.Equal(buf[start:end], buffer.Bytes()) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
// Generic seek error for errors other than io.EOF
|
|
seekErr := errors.New("seek error")
|
|
|
|
testCases := []struct {
|
|
offset int64
|
|
whence int
|
|
pos int64
|
|
err error
|
|
shouldCmp bool
|
|
start int
|
|
end int
|
|
}{
|
|
// Start from offset 0, fetch data and compare
|
|
{0, 0, 0, nil, true, 0, 0},
|
|
// Start from offset 2048, fetch data and compare
|
|
{2048, 0, 2048, nil, true, 2048, bufSize},
|
|
// Start from offset larger than possible
|
|
{int64(bufSize) + 1024, 0, 0, seekErr, false, 0, 0},
|
|
// Move to offset 0 without comparing
|
|
{0, 0, 0, nil, false, 0, 0},
|
|
// Move one step forward and compare
|
|
{1, 1, 1, nil, true, 1, bufSize},
|
|
// Move larger than possible
|
|
{int64(bufSize), 1, 0, seekErr, false, 0, 0},
|
|
// Provide negative offset with CUR_SEEK
|
|
{int64(-1), 1, 0, seekErr, false, 0, 0},
|
|
// Test with whence SEEK_END and with positive offset
|
|
{1024, 2, int64(bufSize) - 1024, io.EOF, true, 0, 0},
|
|
// Test with whence SEEK_END and with negative offset
|
|
{-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize},
|
|
// Test with whence SEEK_END and with large negative offset
|
|
{-int64(bufSize) * 2, 2, 0, seekErr, true, 0, 0},
|
|
}
|
|
|
|
for i, testCase := range testCases {
|
|
// Perform seek operation
|
|
n, err := r.Seek(testCase.offset, testCase.whence)
|
|
// We expect an error
|
|
if testCase.err == seekErr && err == nil {
|
|
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err)
|
|
return
|
|
}
|
|
// We expect a specific error
|
|
if testCase.err != seekErr && testCase.err != err {
|
|
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err)
|
|
return
|
|
}
|
|
// If we expect an error go to the next loop
|
|
if testCase.err != nil {
|
|
continue
|
|
}
|
|
// Check the returned seek pos
|
|
if n != testCase.pos {
|
|
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", number of bytes seeked does not match, expected "+string(testCase.pos)+", got "+string(n), err)
|
|
return
|
|
}
|
|
// Compare only if shouldCmp is activated
|
|
if testCase.shouldCmp {
|
|
cmpData(r, testCase.start, testCase.end)
|
|
}
|
|
}
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests get object ReaderAt interface methods.
|
|
func testGetObjectReadAtFunctional() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "GetObject(bucketName, objectName)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Generate 33K of data.
|
|
bufSize := dataFileMap["datafile-33-kB"]
|
|
reader := getDataReader("datafile-33-kB")
|
|
defer reader.Close()
|
|
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
buf, err := ioutil.ReadAll(reader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
|
|
// Save the data
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// read the data back
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
offset := int64(2048)
|
|
|
|
// read directly
|
|
buf1 := make([]byte, 512)
|
|
buf2 := make([]byte, 512)
|
|
buf3 := make([]byte, 512)
|
|
buf4 := make([]byte, 512)
|
|
|
|
// Test readAt before stat is called such that objectInfo doesn't change.
|
|
m, err := r.ReadAt(buf1, offset)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
if m != len(buf1) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf1, buf[offset:offset+512]) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
|
|
return
|
|
}
|
|
offset += 512
|
|
|
|
st, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(bufSize) {
|
|
logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
|
|
return
|
|
}
|
|
|
|
m, err = r.ReadAt(buf2, offset)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
if m != len(buf2) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf2, buf[offset:offset+512]) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
|
|
return
|
|
}
|
|
|
|
offset += 512
|
|
m, err = r.ReadAt(buf3, offset)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
if m != len(buf3) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf3, buf[offset:offset+512]) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
|
|
return
|
|
}
|
|
offset += 512
|
|
m, err = r.ReadAt(buf4, offset)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
if m != len(buf4) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf4, buf[offset:offset+512]) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
|
|
return
|
|
}
|
|
|
|
buf5 := make([]byte, len(buf))
|
|
// Read the whole object.
|
|
m, err = r.ReadAt(buf5, 0)
|
|
if err != nil {
|
|
if err != io.EOF {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
}
|
|
if m != len(buf5) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf, buf5) {
|
|
logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err)
|
|
return
|
|
}
|
|
|
|
buf6 := make([]byte, len(buf)+1)
|
|
// Read the whole object and beyond.
|
|
_, err = r.ReadAt(buf6, 0)
|
|
if err != nil {
|
|
if err != io.EOF {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Reproduces issue https://github.com/minio/minio-go/issues/1137
|
|
func testGetObjectReadAtWhenEOFWasReached() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "GetObject(bucketName, objectName)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Generate 33K of data.
|
|
bufSize := dataFileMap["datafile-33-kB"]
|
|
reader := getDataReader("datafile-33-kB")
|
|
defer reader.Close()
|
|
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
buf, err := ioutil.ReadAll(reader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
|
|
// Save the data
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// read the data back
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// read directly
|
|
buf1 := make([]byte, len(buf))
|
|
buf2 := make([]byte, 512)
|
|
|
|
m, err := r.Read(buf1)
|
|
if err != nil {
|
|
if err != io.EOF {
|
|
logError(testName, function, args, startTime, "", "Read failed", err)
|
|
return
|
|
}
|
|
}
|
|
if m != len(buf1) {
|
|
logError(testName, function, args, startTime, "", "Read read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf1, buf) {
|
|
logError(testName, function, args, startTime, "", "Incorrect count of Read data", err)
|
|
return
|
|
}
|
|
|
|
st, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(bufSize) {
|
|
logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
|
|
return
|
|
}
|
|
|
|
m, err = r.ReadAt(buf2, 512)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
if m != len(buf2) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf2, buf[512:1024]) {
|
|
logError(testName, function, args, startTime, "", "Incorrect count of ReadAt data", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test Presigned Post Policy
|
|
func testPresignedPostPolicy() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "PresignedPostPolicy(policy)"
|
|
args := map[string]interface{}{
|
|
"policy": "",
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
|
|
// Make a new bucket in 'us-east-1' (source bucket).
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Generate 33K of data.
|
|
reader := getDataReader("datafile-33-kB")
|
|
defer reader.Close()
|
|
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
// Azure requires the key to not start with a number
|
|
metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "user")
|
|
metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
|
|
buf, err := ioutil.ReadAll(reader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
|
|
// Save the data
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
policy := minio.NewPostPolicy()
|
|
|
|
if err := policy.SetBucket(""); err == nil {
|
|
logError(testName, function, args, startTime, "", "SetBucket did not fail for invalid conditions", err)
|
|
return
|
|
}
|
|
if err := policy.SetKey(""); err == nil {
|
|
logError(testName, function, args, startTime, "", "SetKey did not fail for invalid conditions", err)
|
|
return
|
|
}
|
|
if err := policy.SetKeyStartsWith(""); err == nil {
|
|
logError(testName, function, args, startTime, "", "SetKeyStartsWith did not fail for invalid conditions", err)
|
|
return
|
|
}
|
|
if err := policy.SetExpires(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)); err == nil {
|
|
logError(testName, function, args, startTime, "", "SetExpires did not fail for invalid conditions", err)
|
|
return
|
|
}
|
|
if err := policy.SetContentType(""); err == nil {
|
|
logError(testName, function, args, startTime, "", "SetContentType did not fail for invalid conditions", err)
|
|
return
|
|
}
|
|
if err := policy.SetContentTypeStartsWith(""); err == nil {
|
|
logError(testName, function, args, startTime, "", "SetContentTypeStartsWith did not fail for invalid conditions", err)
|
|
return
|
|
}
|
|
if err := policy.SetContentLengthRange(1024*1024, 1024); err == nil {
|
|
logError(testName, function, args, startTime, "", "SetContentLengthRange did not fail for invalid conditions", err)
|
|
return
|
|
}
|
|
if err := policy.SetUserMetadata("", ""); err == nil {
|
|
logError(testName, function, args, startTime, "", "SetUserMetadata did not fail for invalid conditions", err)
|
|
return
|
|
}
|
|
|
|
policy.SetBucket(bucketName)
|
|
policy.SetKey(objectName)
|
|
policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days
|
|
policy.SetContentType("binary/octet-stream")
|
|
policy.SetContentLengthRange(10, 1024*1024)
|
|
policy.SetUserMetadata(metadataKey, metadataValue)
|
|
args["policy"] = policy.String()
|
|
|
|
presignedPostPolicyURL, formData, err := c.PresignedPostPolicy(context.Background(), policy)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedPostPolicy failed", err)
|
|
return
|
|
}
|
|
|
|
var formBuf bytes.Buffer
|
|
writer := multipart.NewWriter(&formBuf)
|
|
for k, v := range formData {
|
|
writer.WriteField(k, v)
|
|
}
|
|
|
|
// Get a 33KB file to upload and test if set post policy works
|
|
filePath := getMintDataDirFilePath("datafile-33-kB")
|
|
if filePath == "" {
|
|
// Make a temp file with 33 KB data.
|
|
file, err := ioutil.TempFile(os.TempDir(), "PresignedPostPolicyTest")
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "TempFile creation failed", err)
|
|
return
|
|
}
|
|
if _, err = io.Copy(file, getDataReader("datafile-33-kB")); err != nil {
|
|
logError(testName, function, args, startTime, "", "Copy failed", err)
|
|
return
|
|
}
|
|
if err = file.Close(); err != nil {
|
|
logError(testName, function, args, startTime, "", "File Close failed", err)
|
|
return
|
|
}
|
|
filePath = file.Name()
|
|
}
|
|
|
|
// add file to post request
|
|
f, err := os.Open(filePath)
|
|
defer f.Close()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "File open failed", err)
|
|
return
|
|
}
|
|
w, err := writer.CreateFormFile("file", filePath)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CreateFormFile failed", err)
|
|
return
|
|
}
|
|
|
|
_, err = io.Copy(w, f)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Copy failed", err)
|
|
return
|
|
}
|
|
writer.Close()
|
|
|
|
transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "DefaultTransport failed", err)
|
|
return
|
|
}
|
|
|
|
httpClient := &http.Client{
|
|
// Setting a sensible time out of 30secs to wait for response
|
|
// headers. Request is pro-actively canceled after 30secs
|
|
// with no response.
|
|
Timeout: 30 * time.Second,
|
|
Transport: transport,
|
|
}
|
|
|
|
req, err := http.NewRequest(http.MethodPost, presignedPostPolicyURL.String(), bytes.NewReader(formBuf.Bytes()))
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Http request failed", err)
|
|
return
|
|
}
|
|
|
|
req.Header.Set("Content-Type", writer.FormDataContentType())
|
|
|
|
// make post request with correct form data
|
|
res, err := httpClient.Do(req)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Http request failed", err)
|
|
return
|
|
}
|
|
defer res.Body.Close()
|
|
if res.StatusCode != http.StatusNoContent {
|
|
logError(testName, function, args, startTime, "", "Http request failed", errors.New(res.Status))
|
|
return
|
|
}
|
|
|
|
// expected path should be absolute path of the object
|
|
var scheme string
|
|
if mustParseBool(os.Getenv(enableHTTPS)) {
|
|
scheme = "https://"
|
|
} else {
|
|
scheme = "http://"
|
|
}
|
|
|
|
expectedLocation := scheme + os.Getenv(serverEndpoint) + "/" + bucketName + "/" + objectName
|
|
expectedLocationBucketDNS := scheme + bucketName + "." + os.Getenv(serverEndpoint) + "/" + objectName
|
|
|
|
if val, ok := res.Header["Location"]; ok {
|
|
if val[0] != expectedLocation && val[0] != expectedLocationBucketDNS {
|
|
logError(testName, function, args, startTime, "", "Location in header response is incorrect", err)
|
|
return
|
|
}
|
|
} else {
|
|
logError(testName, function, args, startTime, "", "Location not found in header response", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests copy object
|
|
func testCopyObject() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObject(dst, src)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
|
|
// Make a new bucket in 'us-east-1' (source bucket).
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Make a new bucket in 'us-east-1' (destination bucket).
|
|
err = c.MakeBucket(context.Background(), bucketName+"-copy", minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
defer cleanupBucket(bucketName+"-copy", c)
|
|
|
|
// Generate 33K of data.
|
|
bufSize := dataFileMap["datafile-33-kB"]
|
|
reader := getDataReader("datafile-33-kB")
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
// Check the various fields of source object against destination object.
|
|
objInfo, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
|
|
// Copy Source
|
|
src := minio.CopySrcOptions{
|
|
Bucket: bucketName,
|
|
Object: objectName,
|
|
// Set copy conditions.
|
|
MatchETag: objInfo.ETag,
|
|
MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC),
|
|
}
|
|
args["src"] = src
|
|
|
|
dst := minio.CopyDestOptions{
|
|
Bucket: bucketName + "-copy",
|
|
Object: objectName + "-copy",
|
|
}
|
|
|
|
// Perform the Copy
|
|
if _, err = c.CopyObject(context.Background(), dst, src); err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Source object
|
|
r, err = c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Destination object
|
|
readerCopy, err := c.GetObject(context.Background(), bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Check the various fields of source object against destination object.
|
|
objInfo, err = r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
objInfoCopy, err := readerCopy.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
if objInfo.Size != objInfoCopy.Size {
|
|
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+", got "+string(objInfo.Size), err)
|
|
return
|
|
}
|
|
|
|
if err := crcMatchesName(r, "datafile-33-kB"); err != nil {
|
|
logError(testName, function, args, startTime, "", "data CRC check failed", err)
|
|
return
|
|
}
|
|
if err := crcMatchesName(readerCopy, "datafile-33-kB"); err != nil {
|
|
logError(testName, function, args, startTime, "", "copy data CRC check failed", err)
|
|
return
|
|
}
|
|
// Close all the get readers before proceeding with CopyObject operations.
|
|
r.Close()
|
|
readerCopy.Close()
|
|
|
|
// CopyObject again but with wrong conditions
|
|
src = minio.CopySrcOptions{
|
|
Bucket: bucketName,
|
|
Object: objectName,
|
|
MatchUnmodifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC),
|
|
NoMatchETag: objInfo.ETag,
|
|
}
|
|
|
|
// Perform the Copy which should fail
|
|
_, err = c.CopyObject(context.Background(), dst, src)
|
|
if err == nil {
|
|
logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err)
|
|
return
|
|
}
|
|
|
|
src = minio.CopySrcOptions{
|
|
Bucket: bucketName,
|
|
Object: objectName,
|
|
}
|
|
|
|
dst = minio.CopyDestOptions{
|
|
Bucket: bucketName,
|
|
Object: objectName,
|
|
ReplaceMetadata: true,
|
|
UserMetadata: map[string]string{
|
|
"Copy": "should be same",
|
|
},
|
|
}
|
|
args["dst"] = dst
|
|
args["src"] = src
|
|
|
|
_, err = c.CopyObject(context.Background(), dst, src)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObject shouldn't fail", err)
|
|
return
|
|
}
|
|
|
|
oi, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject failed", err)
|
|
return
|
|
}
|
|
|
|
stOpts := minio.StatObjectOptions{}
|
|
stOpts.SetMatchETag(oi.ETag)
|
|
objInfo, err = c.StatObject(context.Background(), bucketName, objectName, stOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObject ETag should match and not fail", err)
|
|
return
|
|
}
|
|
|
|
if objInfo.Metadata.Get("x-amz-meta-copy") != "should be same" {
|
|
logError(testName, function, args, startTime, "", "CopyObject modified metadata should match", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests SSE-C get object ReaderSeeker interface methods.
|
|
func testSSECEncryptedGetObjectReadSeekFunctional() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "GetObject(bucketName, objectName)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer func() {
|
|
// Delete all objects and buckets
|
|
if err = cleanupBucket(bucketName, c); err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
|
|
return
|
|
}
|
|
}()
|
|
|
|
// Generate 129MiB of data.
|
|
bufSize := dataFileMap["datafile-129-MB"]
|
|
reader := getDataReader("datafile-129-MB")
|
|
defer reader.Close()
|
|
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
buf, err := ioutil.ReadAll(reader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
|
|
// Save the data
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{
|
|
ContentType: "binary/octet-stream",
|
|
ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Read the data back
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{
|
|
ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
defer r.Close()
|
|
|
|
st, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat object failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(bufSize) {
|
|
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
|
|
return
|
|
}
|
|
|
|
// This following function helps us to compare data from the reader after seek
|
|
// with the data from the original buffer
|
|
cmpData := func(r io.Reader, start, end int) {
|
|
if end-start == 0 {
|
|
return
|
|
}
|
|
buffer := bytes.NewBuffer([]byte{})
|
|
if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil {
|
|
if err != io.EOF {
|
|
logError(testName, function, args, startTime, "", "CopyN failed", err)
|
|
return
|
|
}
|
|
}
|
|
if !bytes.Equal(buf[start:end], buffer.Bytes()) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
testCases := []struct {
|
|
offset int64
|
|
whence int
|
|
pos int64
|
|
err error
|
|
shouldCmp bool
|
|
start int
|
|
end int
|
|
}{
|
|
// Start from offset 0, fetch data and compare
|
|
{0, 0, 0, nil, true, 0, 0},
|
|
// Start from offset 2048, fetch data and compare
|
|
{2048, 0, 2048, nil, true, 2048, bufSize},
|
|
// Start from offset larger than possible
|
|
{int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0},
|
|
// Move to offset 0 without comparing
|
|
{0, 0, 0, nil, false, 0, 0},
|
|
// Move one step forward and compare
|
|
{1, 1, 1, nil, true, 1, bufSize},
|
|
// Move larger than possible
|
|
{int64(bufSize), 1, 0, io.EOF, false, 0, 0},
|
|
// Provide negative offset with CUR_SEEK
|
|
{int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0},
|
|
// Test with whence SEEK_END and with positive offset
|
|
{1024, 2, 0, io.EOF, false, 0, 0},
|
|
// Test with whence SEEK_END and with negative offset
|
|
{-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize},
|
|
// Test with whence SEEK_END and with large negative offset
|
|
{-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0},
|
|
// Test with invalid whence
|
|
{0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0},
|
|
}
|
|
|
|
for i, testCase := range testCases {
|
|
// Perform seek operation
|
|
n, err := r.Seek(testCase.offset, testCase.whence)
|
|
if err != nil && testCase.err == nil {
|
|
// We expected success.
|
|
logError(testName, function, args, startTime, "",
|
|
fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err)
|
|
return
|
|
}
|
|
if err == nil && testCase.err != nil {
|
|
// We expected failure, but got success.
|
|
logError(testName, function, args, startTime, "",
|
|
fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err)
|
|
return
|
|
}
|
|
if err != nil && testCase.err != nil {
|
|
if err.Error() != testCase.err.Error() {
|
|
// We expect a specific error
|
|
logError(testName, function, args, startTime, "",
|
|
fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err)
|
|
return
|
|
}
|
|
}
|
|
// Check the returned seek pos
|
|
if n != testCase.pos {
|
|
logError(testName, function, args, startTime, "",
|
|
fmt.Sprintf("Test %d, number of bytes seeked does not match, expected %d, got %d", i+1, testCase.pos, n), err)
|
|
return
|
|
}
|
|
// Compare only if shouldCmp is activated
|
|
if testCase.shouldCmp {
|
|
cmpData(r, testCase.start, testCase.end)
|
|
}
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests SSE-S3 get object ReaderSeeker interface methods.
|
|
func testSSES3EncryptedGetObjectReadSeekFunctional() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "GetObject(bucketName, objectName)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer func() {
|
|
// Delete all objects and buckets
|
|
if err = cleanupBucket(bucketName, c); err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
|
|
return
|
|
}
|
|
}()
|
|
|
|
// Generate 129MiB of data.
|
|
bufSize := dataFileMap["datafile-129-MB"]
|
|
reader := getDataReader("datafile-129-MB")
|
|
defer reader.Close()
|
|
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
buf, err := ioutil.ReadAll(reader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
|
|
// Save the data
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{
|
|
ContentType: "binary/octet-stream",
|
|
ServerSideEncryption: encrypt.NewSSE(),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Read the data back
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
defer r.Close()
|
|
|
|
st, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat object failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(bufSize) {
|
|
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
|
|
return
|
|
}
|
|
|
|
// This following function helps us to compare data from the reader after seek
|
|
// with the data from the original buffer
|
|
cmpData := func(r io.Reader, start, end int) {
|
|
if end-start == 0 {
|
|
return
|
|
}
|
|
buffer := bytes.NewBuffer([]byte{})
|
|
if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil {
|
|
if err != io.EOF {
|
|
logError(testName, function, args, startTime, "", "CopyN failed", err)
|
|
return
|
|
}
|
|
}
|
|
if !bytes.Equal(buf[start:end], buffer.Bytes()) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
testCases := []struct {
|
|
offset int64
|
|
whence int
|
|
pos int64
|
|
err error
|
|
shouldCmp bool
|
|
start int
|
|
end int
|
|
}{
|
|
// Start from offset 0, fetch data and compare
|
|
{0, 0, 0, nil, true, 0, 0},
|
|
// Start from offset 2048, fetch data and compare
|
|
{2048, 0, 2048, nil, true, 2048, bufSize},
|
|
// Start from offset larger than possible
|
|
{int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0},
|
|
// Move to offset 0 without comparing
|
|
{0, 0, 0, nil, false, 0, 0},
|
|
// Move one step forward and compare
|
|
{1, 1, 1, nil, true, 1, bufSize},
|
|
// Move larger than possible
|
|
{int64(bufSize), 1, 0, io.EOF, false, 0, 0},
|
|
// Provide negative offset with CUR_SEEK
|
|
{int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0},
|
|
// Test with whence SEEK_END and with positive offset
|
|
{1024, 2, 0, io.EOF, false, 0, 0},
|
|
// Test with whence SEEK_END and with negative offset
|
|
{-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize},
|
|
// Test with whence SEEK_END and with large negative offset
|
|
{-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0},
|
|
// Test with invalid whence
|
|
{0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0},
|
|
}
|
|
|
|
for i, testCase := range testCases {
|
|
// Perform seek operation
|
|
n, err := r.Seek(testCase.offset, testCase.whence)
|
|
if err != nil && testCase.err == nil {
|
|
// We expected success.
|
|
logError(testName, function, args, startTime, "",
|
|
fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err)
|
|
return
|
|
}
|
|
if err == nil && testCase.err != nil {
|
|
// We expected failure, but got success.
|
|
logError(testName, function, args, startTime, "",
|
|
fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err)
|
|
return
|
|
}
|
|
if err != nil && testCase.err != nil {
|
|
if err.Error() != testCase.err.Error() {
|
|
// We expect a specific error
|
|
logError(testName, function, args, startTime, "",
|
|
fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err)
|
|
return
|
|
}
|
|
}
|
|
// Check the returned seek pos
|
|
if n != testCase.pos {
|
|
logError(testName, function, args, startTime, "",
|
|
fmt.Sprintf("Test %d, number of bytes seeked does not match, expected %d, got %d", i+1, testCase.pos, n), err)
|
|
return
|
|
}
|
|
// Compare only if shouldCmp is activated
|
|
if testCase.shouldCmp {
|
|
cmpData(r, testCase.start, testCase.end)
|
|
}
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests SSE-C get object ReaderAt interface methods.
|
|
func testSSECEncryptedGetObjectReadAtFunctional() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "GetObject(bucketName, objectName)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Generate 129MiB of data.
|
|
bufSize := dataFileMap["datafile-129-MB"]
|
|
reader := getDataReader("datafile-129-MB")
|
|
defer reader.Close()
|
|
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
buf, err := ioutil.ReadAll(reader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
|
|
// Save the data
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{
|
|
ContentType: "binary/octet-stream",
|
|
ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// read the data back
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{
|
|
ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
defer r.Close()
|
|
|
|
offset := int64(2048)
|
|
|
|
// read directly
|
|
buf1 := make([]byte, 512)
|
|
buf2 := make([]byte, 512)
|
|
buf3 := make([]byte, 512)
|
|
buf4 := make([]byte, 512)
|
|
|
|
// Test readAt before stat is called such that objectInfo doesn't change.
|
|
m, err := r.ReadAt(buf1, offset)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
if m != len(buf1) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf1, buf[offset:offset+512]) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
|
|
return
|
|
}
|
|
offset += 512
|
|
|
|
st, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(bufSize) {
|
|
logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
|
|
return
|
|
}
|
|
|
|
m, err = r.ReadAt(buf2, offset)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
if m != len(buf2) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf2, buf[offset:offset+512]) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
|
|
return
|
|
}
|
|
offset += 512
|
|
m, err = r.ReadAt(buf3, offset)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
if m != len(buf3) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf3, buf[offset:offset+512]) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
|
|
return
|
|
}
|
|
offset += 512
|
|
m, err = r.ReadAt(buf4, offset)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
if m != len(buf4) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf4, buf[offset:offset+512]) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
|
|
return
|
|
}
|
|
|
|
buf5 := make([]byte, len(buf))
|
|
// Read the whole object.
|
|
m, err = r.ReadAt(buf5, 0)
|
|
if err != nil {
|
|
if err != io.EOF {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
}
|
|
if m != len(buf5) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf, buf5) {
|
|
logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err)
|
|
return
|
|
}
|
|
|
|
buf6 := make([]byte, len(buf)+1)
|
|
// Read the whole object and beyond.
|
|
_, err = r.ReadAt(buf6, 0)
|
|
if err != nil {
|
|
if err != io.EOF {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests SSE-S3 get object ReaderAt interface methods.
|
|
func testSSES3EncryptedGetObjectReadAtFunctional() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "GetObject(bucketName, objectName)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Generate 129MiB of data.
|
|
bufSize := dataFileMap["datafile-129-MB"]
|
|
reader := getDataReader("datafile-129-MB")
|
|
defer reader.Close()
|
|
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
buf, err := ioutil.ReadAll(reader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
|
|
// Save the data
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{
|
|
ContentType: "binary/octet-stream",
|
|
ServerSideEncryption: encrypt.NewSSE(),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// read the data back
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
defer r.Close()
|
|
|
|
offset := int64(2048)
|
|
|
|
// read directly
|
|
buf1 := make([]byte, 512)
|
|
buf2 := make([]byte, 512)
|
|
buf3 := make([]byte, 512)
|
|
buf4 := make([]byte, 512)
|
|
|
|
// Test readAt before stat is called such that objectInfo doesn't change.
|
|
m, err := r.ReadAt(buf1, offset)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
if m != len(buf1) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf1, buf[offset:offset+512]) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
|
|
return
|
|
}
|
|
offset += 512
|
|
|
|
st, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(bufSize) {
|
|
logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
|
|
return
|
|
}
|
|
|
|
m, err = r.ReadAt(buf2, offset)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
if m != len(buf2) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf2, buf[offset:offset+512]) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
|
|
return
|
|
}
|
|
offset += 512
|
|
m, err = r.ReadAt(buf3, offset)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
if m != len(buf3) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf3, buf[offset:offset+512]) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
|
|
return
|
|
}
|
|
offset += 512
|
|
m, err = r.ReadAt(buf4, offset)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
if m != len(buf4) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf4, buf[offset:offset+512]) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
|
|
return
|
|
}
|
|
|
|
buf5 := make([]byte, len(buf))
|
|
// Read the whole object.
|
|
m, err = r.ReadAt(buf5, 0)
|
|
if err != nil {
|
|
if err != io.EOF {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
}
|
|
if m != len(buf5) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf, buf5) {
|
|
logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err)
|
|
return
|
|
}
|
|
|
|
buf6 := make([]byte, len(buf)+1)
|
|
// Read the whole object and beyond.
|
|
_, err = r.ReadAt(buf6, 0)
|
|
if err != nil {
|
|
if err != io.EOF {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// testSSECEncryptionPutGet tests encryption with customer provided encryption keys
|
|
func testSSECEncryptionPutGet() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "PutEncryptedObject(bucketName, objectName, reader, sse)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"sse": "",
|
|
}
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
testCases := []struct {
|
|
buf []byte
|
|
}{
|
|
{buf: bytes.Repeat([]byte("F"), 1)},
|
|
{buf: bytes.Repeat([]byte("F"), 15)},
|
|
{buf: bytes.Repeat([]byte("F"), 16)},
|
|
{buf: bytes.Repeat([]byte("F"), 17)},
|
|
{buf: bytes.Repeat([]byte("F"), 31)},
|
|
{buf: bytes.Repeat([]byte("F"), 32)},
|
|
{buf: bytes.Repeat([]byte("F"), 33)},
|
|
{buf: bytes.Repeat([]byte("F"), 1024)},
|
|
{buf: bytes.Repeat([]byte("F"), 1024*2)},
|
|
{buf: bytes.Repeat([]byte("F"), 1024*1024)},
|
|
}
|
|
|
|
const password = "correct horse battery staple" // https://xkcd.com/936/
|
|
|
|
for i, testCase := range testCases {
|
|
// Generate a random object name
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
// Secured object
|
|
sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName))
|
|
args["sse"] = sse
|
|
|
|
// Put encrypted data
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Read the data back
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err)
|
|
return
|
|
}
|
|
defer r.Close()
|
|
|
|
// Compare the sent object with the received one
|
|
recvBuffer := bytes.NewBuffer([]byte{})
|
|
if _, err = io.Copy(recvBuffer, r); err != nil {
|
|
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err)
|
|
return
|
|
}
|
|
if recvBuffer.Len() != len(testCase.buf) {
|
|
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) {
|
|
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// TestEncryptionFPut tests encryption with customer specified encryption keys
|
|
func testSSECEncryptionFPut() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, sse)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"filePath": "",
|
|
"contentType": "",
|
|
"sse": "",
|
|
}
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Object custom metadata
|
|
customContentType := "custom/contenttype"
|
|
args["metadata"] = customContentType
|
|
|
|
testCases := []struct {
|
|
buf []byte
|
|
}{
|
|
{buf: bytes.Repeat([]byte("F"), 0)},
|
|
{buf: bytes.Repeat([]byte("F"), 1)},
|
|
{buf: bytes.Repeat([]byte("F"), 15)},
|
|
{buf: bytes.Repeat([]byte("F"), 16)},
|
|
{buf: bytes.Repeat([]byte("F"), 17)},
|
|
{buf: bytes.Repeat([]byte("F"), 31)},
|
|
{buf: bytes.Repeat([]byte("F"), 32)},
|
|
{buf: bytes.Repeat([]byte("F"), 33)},
|
|
{buf: bytes.Repeat([]byte("F"), 1024)},
|
|
{buf: bytes.Repeat([]byte("F"), 1024*2)},
|
|
{buf: bytes.Repeat([]byte("F"), 1024*1024)},
|
|
}
|
|
|
|
const password = "correct horse battery staple" // https://xkcd.com/936/
|
|
for i, testCase := range testCases {
|
|
// Generate a random object name
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
// Secured object
|
|
sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName))
|
|
args["sse"] = sse
|
|
|
|
// Generate a random file name.
|
|
fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
file, err := os.Create(fileName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "file create failed", err)
|
|
return
|
|
}
|
|
_, err = file.Write(testCase.buf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "file write failed", err)
|
|
return
|
|
}
|
|
file.Close()
|
|
// Put encrypted data
|
|
if _, err = c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil {
|
|
logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Read the data back
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err)
|
|
return
|
|
}
|
|
defer r.Close()
|
|
|
|
// Compare the sent object with the received one
|
|
recvBuffer := bytes.NewBuffer([]byte{})
|
|
if _, err = io.Copy(recvBuffer, r); err != nil {
|
|
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err)
|
|
return
|
|
}
|
|
if recvBuffer.Len() != len(testCase.buf) {
|
|
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) {
|
|
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err)
|
|
return
|
|
}
|
|
|
|
os.Remove(fileName)
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// testSSES3EncryptionPutGet tests SSE-S3 encryption
|
|
func testSSES3EncryptionPutGet() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "PutEncryptedObject(bucketName, objectName, reader, sse)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"sse": "",
|
|
}
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
testCases := []struct {
|
|
buf []byte
|
|
}{
|
|
{buf: bytes.Repeat([]byte("F"), 1)},
|
|
{buf: bytes.Repeat([]byte("F"), 15)},
|
|
{buf: bytes.Repeat([]byte("F"), 16)},
|
|
{buf: bytes.Repeat([]byte("F"), 17)},
|
|
{buf: bytes.Repeat([]byte("F"), 31)},
|
|
{buf: bytes.Repeat([]byte("F"), 32)},
|
|
{buf: bytes.Repeat([]byte("F"), 33)},
|
|
{buf: bytes.Repeat([]byte("F"), 1024)},
|
|
{buf: bytes.Repeat([]byte("F"), 1024*2)},
|
|
{buf: bytes.Repeat([]byte("F"), 1024*1024)},
|
|
}
|
|
|
|
for i, testCase := range testCases {
|
|
// Generate a random object name
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
// Secured object
|
|
sse := encrypt.NewSSE()
|
|
args["sse"] = sse
|
|
|
|
// Put encrypted data
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Read the data back without any encryption headers
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err)
|
|
return
|
|
}
|
|
defer r.Close()
|
|
|
|
// Compare the sent object with the received one
|
|
recvBuffer := bytes.NewBuffer([]byte{})
|
|
if _, err = io.Copy(recvBuffer, r); err != nil {
|
|
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err)
|
|
return
|
|
}
|
|
if recvBuffer.Len() != len(testCase.buf) {
|
|
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) {
|
|
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// TestSSES3EncryptionFPut tests server side encryption
|
|
func testSSES3EncryptionFPut() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, sse)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"filePath": "",
|
|
"contentType": "",
|
|
"sse": "",
|
|
}
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Object custom metadata
|
|
customContentType := "custom/contenttype"
|
|
args["metadata"] = customContentType
|
|
|
|
testCases := []struct {
|
|
buf []byte
|
|
}{
|
|
{buf: bytes.Repeat([]byte("F"), 0)},
|
|
{buf: bytes.Repeat([]byte("F"), 1)},
|
|
{buf: bytes.Repeat([]byte("F"), 15)},
|
|
{buf: bytes.Repeat([]byte("F"), 16)},
|
|
{buf: bytes.Repeat([]byte("F"), 17)},
|
|
{buf: bytes.Repeat([]byte("F"), 31)},
|
|
{buf: bytes.Repeat([]byte("F"), 32)},
|
|
{buf: bytes.Repeat([]byte("F"), 33)},
|
|
{buf: bytes.Repeat([]byte("F"), 1024)},
|
|
{buf: bytes.Repeat([]byte("F"), 1024*2)},
|
|
{buf: bytes.Repeat([]byte("F"), 1024*1024)},
|
|
}
|
|
|
|
for i, testCase := range testCases {
|
|
// Generate a random object name
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
// Secured object
|
|
sse := encrypt.NewSSE()
|
|
args["sse"] = sse
|
|
|
|
// Generate a random file name.
|
|
fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
file, err := os.Create(fileName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "file create failed", err)
|
|
return
|
|
}
|
|
_, err = file.Write(testCase.buf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "file write failed", err)
|
|
return
|
|
}
|
|
file.Close()
|
|
// Put encrypted data
|
|
if _, err = c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil {
|
|
logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Read the data back
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err)
|
|
return
|
|
}
|
|
defer r.Close()
|
|
|
|
// Compare the sent object with the received one
|
|
recvBuffer := bytes.NewBuffer([]byte{})
|
|
if _, err = io.Copy(recvBuffer, r); err != nil {
|
|
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err)
|
|
return
|
|
}
|
|
if recvBuffer.Len() != len(testCase.buf) {
|
|
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) {
|
|
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err)
|
|
return
|
|
}
|
|
|
|
os.Remove(fileName)
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
func testBucketNotification() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "SetBucketNotification(bucketName)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
}
|
|
|
|
if os.Getenv("NOTIFY_BUCKET") == "" ||
|
|
os.Getenv("NOTIFY_SERVICE") == "" ||
|
|
os.Getenv("NOTIFY_REGION") == "" ||
|
|
os.Getenv("NOTIFY_ACCOUNTID") == "" ||
|
|
os.Getenv("NOTIFY_RESOURCE") == "" {
|
|
ignoredLog(testName, function, args, startTime, "Skipped notification test as it is not configured").Info()
|
|
return
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable to debug
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
bucketName := os.Getenv("NOTIFY_BUCKET")
|
|
args["bucketName"] = bucketName
|
|
|
|
topicArn := notification.NewArn("aws", os.Getenv("NOTIFY_SERVICE"), os.Getenv("NOTIFY_REGION"), os.Getenv("NOTIFY_ACCOUNTID"), os.Getenv("NOTIFY_RESOURCE"))
|
|
queueArn := notification.NewArn("aws", "dummy-service", "dummy-region", "dummy-accountid", "dummy-resource")
|
|
|
|
topicConfig := notification.NewConfig(topicArn)
|
|
topicConfig.AddEvents(notification.ObjectCreatedAll, notification.ObjectRemovedAll)
|
|
topicConfig.AddFilterSuffix("jpg")
|
|
|
|
queueConfig := notification.NewConfig(queueArn)
|
|
queueConfig.AddEvents(notification.ObjectCreatedAll)
|
|
queueConfig.AddFilterPrefix("photos/")
|
|
|
|
config := notification.Configuration{}
|
|
config.AddTopic(topicConfig)
|
|
|
|
// Add the same topicConfig again, should have no effect
|
|
// because it is duplicated
|
|
config.AddTopic(topicConfig)
|
|
if len(config.TopicConfigs) != 1 {
|
|
logError(testName, function, args, startTime, "", "Duplicate entry added", err)
|
|
return
|
|
}
|
|
|
|
// Add and remove a queue config
|
|
config.AddQueue(queueConfig)
|
|
config.RemoveQueueByArn(queueArn)
|
|
|
|
err = c.SetBucketNotification(context.Background(), bucketName, config)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "SetBucketNotification failed", err)
|
|
return
|
|
}
|
|
|
|
config, err = c.GetBucketNotification(context.Background(), bucketName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetBucketNotification failed", err)
|
|
return
|
|
}
|
|
|
|
if len(config.TopicConfigs) != 1 {
|
|
logError(testName, function, args, startTime, "", "Topic config is empty", err)
|
|
return
|
|
}
|
|
|
|
if config.TopicConfigs[0].Filter.S3Key.FilterRules[0].Value != "jpg" {
|
|
logError(testName, function, args, startTime, "", "Couldn't get the suffix", err)
|
|
return
|
|
}
|
|
|
|
err = c.RemoveAllBucketNotification(context.Background(), bucketName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "RemoveAllBucketNotification failed", err)
|
|
return
|
|
}
|
|
|
|
// Delete all objects and buckets
|
|
if err = cleanupBucket(bucketName, c); err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests comprehensive list of all methods.
|
|
func testFunctional() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "testFunctional()"
|
|
functionAll := ""
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, nil, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable to debug
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
|
|
// Make a new bucket.
|
|
function = "MakeBucket(bucketName, region)"
|
|
functionAll = "MakeBucket(bucketName, region)"
|
|
args["bucketName"] = bucketName
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
// Generate a random file name.
|
|
fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
file, err := os.Create(fileName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "File creation failed", err)
|
|
return
|
|
}
|
|
for i := 0; i < 3; i++ {
|
|
buf := make([]byte, rand.Intn(1<<19))
|
|
_, err = file.Write(buf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "File write failed", err)
|
|
return
|
|
}
|
|
}
|
|
file.Close()
|
|
|
|
// Verify if bucket exits and you have access.
|
|
var exists bool
|
|
function = "BucketExists(bucketName)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
}
|
|
exists, err = c.BucketExists(context.Background(), bucketName)
|
|
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "BucketExists failed", err)
|
|
return
|
|
}
|
|
if !exists {
|
|
logError(testName, function, args, startTime, "", "Could not find the bucket", err)
|
|
return
|
|
}
|
|
|
|
// Asserting the default bucket policy.
|
|
function = "GetBucketPolicy(ctx, bucketName)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
}
|
|
nilPolicy, err := c.GetBucketPolicy(context.Background(), bucketName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err)
|
|
return
|
|
}
|
|
if nilPolicy != "" {
|
|
logError(testName, function, args, startTime, "", "policy should be set to nil", err)
|
|
return
|
|
}
|
|
|
|
// Set the bucket policy to 'public readonly'.
|
|
function = "SetBucketPolicy(bucketName, readOnlyPolicy)"
|
|
functionAll += ", " + function
|
|
|
|
readOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}`
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"bucketPolicy": readOnlyPolicy,
|
|
}
|
|
|
|
err = c.SetBucketPolicy(context.Background(), bucketName, readOnlyPolicy)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err)
|
|
return
|
|
}
|
|
// should return policy `readonly`.
|
|
function = "GetBucketPolicy(ctx, bucketName)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
}
|
|
_, err = c.GetBucketPolicy(context.Background(), bucketName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err)
|
|
return
|
|
}
|
|
|
|
// Make the bucket 'public writeonly'.
|
|
function = "SetBucketPolicy(bucketName, writeOnlyPolicy)"
|
|
functionAll += ", " + function
|
|
|
|
writeOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}`
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"bucketPolicy": writeOnlyPolicy,
|
|
}
|
|
err = c.SetBucketPolicy(context.Background(), bucketName, writeOnlyPolicy)
|
|
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err)
|
|
return
|
|
}
|
|
// should return policy `writeonly`.
|
|
function = "GetBucketPolicy(ctx, bucketName)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
}
|
|
|
|
_, err = c.GetBucketPolicy(context.Background(), bucketName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err)
|
|
return
|
|
}
|
|
|
|
// Make the bucket 'public read/write'.
|
|
function = "SetBucketPolicy(bucketName, readWritePolicy)"
|
|
functionAll += ", " + function
|
|
|
|
readWritePolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}`
|
|
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"bucketPolicy": readWritePolicy,
|
|
}
|
|
err = c.SetBucketPolicy(context.Background(), bucketName, readWritePolicy)
|
|
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err)
|
|
return
|
|
}
|
|
// should return policy `readwrite`.
|
|
function = "GetBucketPolicy(bucketName)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
}
|
|
_, err = c.GetBucketPolicy(context.Background(), bucketName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err)
|
|
return
|
|
}
|
|
|
|
// List all buckets.
|
|
function = "ListBuckets()"
|
|
functionAll += ", " + function
|
|
args = nil
|
|
buckets, err := c.ListBuckets(context.Background())
|
|
|
|
if len(buckets) == 0 {
|
|
logError(testName, function, args, startTime, "", "Found bucket list to be empty", err)
|
|
return
|
|
}
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ListBuckets failed", err)
|
|
return
|
|
}
|
|
|
|
// Verify if previously created bucket is listed in list buckets.
|
|
bucketFound := false
|
|
for _, bucket := range buckets {
|
|
if bucket.Name == bucketName {
|
|
bucketFound = true
|
|
}
|
|
}
|
|
|
|
// If bucket not found error out.
|
|
if !bucketFound {
|
|
logError(testName, function, args, startTime, "", "Bucket: "+bucketName+" not found", err)
|
|
return
|
|
}
|
|
|
|
objectName := bucketName + "unique"
|
|
|
|
// Generate data
|
|
buf := bytes.Repeat([]byte("f"), 1<<19)
|
|
|
|
function = "PutObject(bucketName, objectName, reader, contentType)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName,
|
|
"contentType": "",
|
|
}
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName + "-nolength",
|
|
"contentType": "binary/octet-stream",
|
|
}
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName+"-nolength", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Instantiate a done channel to close all listing.
|
|
doneCh := make(chan struct{})
|
|
defer close(doneCh)
|
|
|
|
objFound := false
|
|
isRecursive := true // Recursive is true.
|
|
|
|
function = "ListObjects(bucketName, objectName, isRecursive, doneCh)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName,
|
|
"isRecursive": isRecursive,
|
|
}
|
|
|
|
for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Prefix: objectName, Recursive: true}) {
|
|
if obj.Key == objectName {
|
|
objFound = true
|
|
break
|
|
}
|
|
}
|
|
if !objFound {
|
|
logError(testName, function, args, startTime, "", "Object "+objectName+" not found", err)
|
|
return
|
|
}
|
|
|
|
objFound = false
|
|
isRecursive = true // Recursive is true.
|
|
function = "ListObjects()"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName,
|
|
"isRecursive": isRecursive,
|
|
}
|
|
|
|
for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{Prefix: objectName, Recursive: isRecursive}) {
|
|
if obj.Key == objectName {
|
|
objFound = true
|
|
break
|
|
}
|
|
}
|
|
if !objFound {
|
|
logError(testName, function, args, startTime, "", "Object "+objectName+" not found", err)
|
|
return
|
|
}
|
|
|
|
incompObjNotFound := true
|
|
|
|
function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName,
|
|
"isRecursive": isRecursive,
|
|
}
|
|
|
|
for objIncompl := range c.ListIncompleteUploads(context.Background(), bucketName, objectName, isRecursive) {
|
|
if objIncompl.Key != "" {
|
|
incompObjNotFound = false
|
|
break
|
|
}
|
|
}
|
|
if !incompObjNotFound {
|
|
logError(testName, function, args, startTime, "", "Unexpected dangling incomplete upload found", err)
|
|
return
|
|
}
|
|
|
|
function = "GetObject(bucketName, objectName)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName,
|
|
}
|
|
newReader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
|
|
newReadBytes, err := ioutil.ReadAll(newReader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
|
|
if !bytes.Equal(newReadBytes, buf) {
|
|
logError(testName, function, args, startTime, "", "GetObject bytes mismatch", err)
|
|
return
|
|
}
|
|
newReader.Close()
|
|
|
|
function = "FGetObject(bucketName, objectName, fileName)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName,
|
|
"fileName": fileName + "-f",
|
|
}
|
|
err = c.FGetObject(context.Background(), bucketName, objectName, fileName+"-f", minio.GetObjectOptions{})
|
|
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "FGetObject failed", err)
|
|
return
|
|
}
|
|
|
|
function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": "",
|
|
"expires": 3600 * time.Second,
|
|
}
|
|
if _, err = c.PresignedHeadObject(context.Background(), bucketName, "", 3600*time.Second, nil); err == nil {
|
|
logError(testName, function, args, startTime, "", "PresignedHeadObject success", err)
|
|
return
|
|
}
|
|
|
|
// Generate presigned HEAD object url.
|
|
function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName,
|
|
"expires": 3600 * time.Second,
|
|
}
|
|
presignedHeadURL, err := c.PresignedHeadObject(context.Background(), bucketName, objectName, 3600*time.Second, nil)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err)
|
|
return
|
|
}
|
|
|
|
transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "DefaultTransport failed", err)
|
|
return
|
|
}
|
|
|
|
httpClient := &http.Client{
|
|
// Setting a sensible time out of 30secs to wait for response
|
|
// headers. Request is pro-actively canceled after 30secs
|
|
// with no response.
|
|
Timeout: 30 * time.Second,
|
|
Transport: transport,
|
|
}
|
|
|
|
req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedHeadObject request was incorrect", err)
|
|
return
|
|
}
|
|
|
|
// Verify if presigned url works.
|
|
resp, err := httpClient.Do(req)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect", err)
|
|
return
|
|
}
|
|
if resp.StatusCode != http.StatusOK {
|
|
logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect, status "+string(resp.StatusCode), err)
|
|
return
|
|
}
|
|
if resp.Header.Get("ETag") == "" {
|
|
logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect", err)
|
|
return
|
|
}
|
|
resp.Body.Close()
|
|
|
|
function = "PresignedGetObject(bucketName, objectName, expires, reqParams)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": "",
|
|
"expires": 3600 * time.Second,
|
|
}
|
|
_, err = c.PresignedGetObject(context.Background(), bucketName, "", 3600*time.Second, nil)
|
|
if err == nil {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject success", err)
|
|
return
|
|
}
|
|
|
|
// Generate presigned GET object url.
|
|
function = "PresignedGetObject(bucketName, objectName, expires, reqParams)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName,
|
|
"expires": 3600 * time.Second,
|
|
}
|
|
presignedGetURL, err := c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, nil)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Verify if presigned url works.
|
|
req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err)
|
|
return
|
|
}
|
|
|
|
resp, err = httpClient.Do(req)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
|
|
return
|
|
}
|
|
if resp.StatusCode != http.StatusOK {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err)
|
|
return
|
|
}
|
|
newPresignedBytes, err := ioutil.ReadAll(resp.Body)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
|
|
return
|
|
}
|
|
resp.Body.Close()
|
|
if !bytes.Equal(newPresignedBytes, buf) {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
|
|
return
|
|
}
|
|
|
|
// Set request parameters.
|
|
reqParams := make(url.Values)
|
|
reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"")
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName,
|
|
"expires": 3600 * time.Second,
|
|
"reqParams": reqParams,
|
|
}
|
|
presignedGetURL, err = c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, reqParams)
|
|
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Verify if presigned url works.
|
|
req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err)
|
|
return
|
|
}
|
|
|
|
resp, err = httpClient.Do(req)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
|
|
return
|
|
}
|
|
if resp.StatusCode != http.StatusOK {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err)
|
|
return
|
|
}
|
|
newPresignedBytes, err = ioutil.ReadAll(resp.Body)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(newPresignedBytes, buf) {
|
|
logError(testName, function, args, startTime, "", "Bytes mismatch for presigned GET URL", err)
|
|
return
|
|
}
|
|
if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" {
|
|
logError(testName, function, args, startTime, "", "wrong Content-Disposition received "+string(resp.Header.Get("Content-Disposition")), err)
|
|
return
|
|
}
|
|
|
|
function = "PresignedPutObject(bucketName, objectName, expires)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": "",
|
|
"expires": 3600 * time.Second,
|
|
}
|
|
_, err = c.PresignedPutObject(context.Background(), bucketName, "", 3600*time.Second)
|
|
if err == nil {
|
|
logError(testName, function, args, startTime, "", "PresignedPutObject success", err)
|
|
return
|
|
}
|
|
|
|
function = "PresignedPutObject(bucketName, objectName, expires)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName + "-presigned",
|
|
"expires": 3600 * time.Second,
|
|
}
|
|
presignedPutURL, err := c.PresignedPutObject(context.Background(), bucketName, objectName+"-presigned", 3600*time.Second)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedPutObject failed", err)
|
|
return
|
|
}
|
|
|
|
buf = bytes.Repeat([]byte("g"), 1<<19)
|
|
|
|
req, err = http.NewRequest(http.MethodPut, presignedPutURL.String(), bytes.NewReader(buf))
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Couldn't make HTTP request with PresignedPutObject URL", err)
|
|
return
|
|
}
|
|
|
|
resp, err = httpClient.Do(req)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedPutObject failed", err)
|
|
return
|
|
}
|
|
|
|
newReader, err = c.GetObject(context.Background(), bucketName, objectName+"-presigned", minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject after PresignedPutObject failed", err)
|
|
return
|
|
}
|
|
|
|
newReadBytes, err = ioutil.ReadAll(newReader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll after GetObject failed", err)
|
|
return
|
|
}
|
|
|
|
if !bytes.Equal(newReadBytes, buf) {
|
|
logError(testName, function, args, startTime, "", "Bytes mismatch", err)
|
|
return
|
|
}
|
|
|
|
function = "PresignHeader(method, bucketName, objectName, expires, reqParams, extraHeaders)"
|
|
functionAll += ", " + function
|
|
presignExtraHeaders := map[string][]string{
|
|
"mysecret": {"abcxxx"},
|
|
}
|
|
args = map[string]interface{}{
|
|
"method": "PUT",
|
|
"bucketName": bucketName,
|
|
"objectName": objectName + "-presign-custom",
|
|
"expires": 3600 * time.Second,
|
|
"extraHeaders": presignExtraHeaders,
|
|
}
|
|
presignedURL, err := c.PresignHeader(context.Background(), "PUT", bucketName, objectName+"-presign-custom", 3600*time.Second, nil, presignExtraHeaders)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Presigned failed", err)
|
|
return
|
|
}
|
|
|
|
// Generate data more than 32K
|
|
buf = bytes.Repeat([]byte("1"), rand.Intn(1<<10)+32*1024)
|
|
|
|
req, err = http.NewRequest(http.MethodPut, presignedURL.String(), bytes.NewReader(buf))
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "HTTP request to Presigned URL failed", err)
|
|
return
|
|
}
|
|
|
|
req.Header.Add("mysecret", "abcxxx")
|
|
resp, err = httpClient.Do(req)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "HTTP request to Presigned URL failed", err)
|
|
return
|
|
}
|
|
|
|
// Download the uploaded object to verify
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName + "-presign-custom",
|
|
}
|
|
newReader, err = c.GetObject(context.Background(), bucketName, objectName+"-presign-custom", minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject of uploaded custom-presigned object failed", err)
|
|
return
|
|
}
|
|
|
|
newReadBytes, err = ioutil.ReadAll(newReader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed during get on custom-presigned put object", err)
|
|
return
|
|
}
|
|
newReader.Close()
|
|
|
|
if !bytes.Equal(newReadBytes, buf) {
|
|
logError(testName, function, args, startTime, "", "Bytes mismatch on custom-presigned object upload verification", err)
|
|
return
|
|
}
|
|
|
|
function = "RemoveObject(bucketName, objectName)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName,
|
|
}
|
|
err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{})
|
|
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "RemoveObject failed", err)
|
|
return
|
|
}
|
|
args["objectName"] = objectName + "-f"
|
|
err = c.RemoveObject(context.Background(), bucketName, objectName+"-f", minio.RemoveObjectOptions{})
|
|
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "RemoveObject failed", err)
|
|
return
|
|
}
|
|
|
|
args["objectName"] = objectName + "-nolength"
|
|
err = c.RemoveObject(context.Background(), bucketName, objectName+"-nolength", minio.RemoveObjectOptions{})
|
|
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "RemoveObject failed", err)
|
|
return
|
|
}
|
|
|
|
args["objectName"] = objectName + "-presigned"
|
|
err = c.RemoveObject(context.Background(), bucketName, objectName+"-presigned", minio.RemoveObjectOptions{})
|
|
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "RemoveObject failed", err)
|
|
return
|
|
}
|
|
|
|
args["objectName"] = objectName + "-presign-custom"
|
|
err = c.RemoveObject(context.Background(), bucketName, objectName+"-presign-custom", minio.RemoveObjectOptions{})
|
|
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "RemoveObject failed", err)
|
|
return
|
|
}
|
|
|
|
function = "RemoveBucket(bucketName)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
}
|
|
err = c.RemoveBucket(context.Background(), bucketName)
|
|
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "RemoveBucket failed", err)
|
|
return
|
|
}
|
|
err = c.RemoveBucket(context.Background(), bucketName)
|
|
if err == nil {
|
|
logError(testName, function, args, startTime, "", "RemoveBucket did not fail for invalid bucket name", err)
|
|
return
|
|
}
|
|
if err.Error() != "The specified bucket does not exist" {
|
|
logError(testName, function, args, startTime, "", "RemoveBucket failed", err)
|
|
return
|
|
}
|
|
|
|
os.Remove(fileName)
|
|
os.Remove(fileName + "-f")
|
|
successLogger(testName, functionAll, args, startTime).Info()
|
|
}
|
|
|
|
// Test for validating GetObject Reader* methods functioning when the
|
|
// object is modified in the object store.
|
|
func testGetObjectModified() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "GetObject(bucketName, objectName)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Make a new bucket.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Upload an object.
|
|
objectName := "myobject"
|
|
args["objectName"] = objectName
|
|
content := "helloworld"
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, strings.NewReader(content), int64(len(content)), minio.PutObjectOptions{ContentType: "application/text"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err)
|
|
return
|
|
}
|
|
|
|
defer c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{})
|
|
|
|
reader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Failed to GetObject "+objectName+", from bucket "+bucketName, err)
|
|
return
|
|
}
|
|
defer reader.Close()
|
|
|
|
// Read a few bytes of the object.
|
|
b := make([]byte, 5)
|
|
n, err := reader.ReadAt(b, 0)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Failed to read object "+objectName+", from bucket "+bucketName+" at an offset", err)
|
|
return
|
|
}
|
|
|
|
// Upload different contents to the same object while object is being read.
|
|
newContent := "goodbyeworld"
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, strings.NewReader(newContent), int64(len(newContent)), minio.PutObjectOptions{ContentType: "application/text"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err)
|
|
return
|
|
}
|
|
|
|
// Confirm that a Stat() call in between doesn't change the Object's cached etag.
|
|
_, err = reader.Stat()
|
|
expectedError := "At least one of the pre-conditions you specified did not hold"
|
|
if err.Error() != expectedError {
|
|
logError(testName, function, args, startTime, "", "Expected Stat to fail with error "+expectedError+", but received "+err.Error(), err)
|
|
return
|
|
}
|
|
|
|
// Read again only to find object contents have been modified since last read.
|
|
_, err = reader.ReadAt(b, int64(n))
|
|
if err.Error() != expectedError {
|
|
logError(testName, function, args, startTime, "", "Expected ReadAt to fail with error "+expectedError+", but received "+err.Error(), err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test validates putObject to upload a file seeked at a given offset.
|
|
func testPutObjectUploadSeekedObject() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "PutObject(bucketName, objectName, fileToUpload, contentType)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"fileToUpload": "",
|
|
"contentType": "binary/octet-stream",
|
|
}
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Make a new bucket.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
var tempfile *os.File
|
|
|
|
if fileName := getMintDataDirFilePath("datafile-100-kB"); fileName != "" {
|
|
tempfile, err = os.Open(fileName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "File open failed", err)
|
|
return
|
|
}
|
|
args["fileToUpload"] = fileName
|
|
} else {
|
|
tempfile, err = ioutil.TempFile("", "minio-go-upload-test-")
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "TempFile create failed", err)
|
|
return
|
|
}
|
|
args["fileToUpload"] = tempfile.Name()
|
|
|
|
// Generate 100kB data
|
|
if _, err = io.Copy(tempfile, getDataReader("datafile-100-kB")); err != nil {
|
|
logError(testName, function, args, startTime, "", "File copy failed", err)
|
|
return
|
|
}
|
|
|
|
defer os.Remove(tempfile.Name())
|
|
|
|
// Seek back to the beginning of the file.
|
|
tempfile.Seek(0, 0)
|
|
}
|
|
length := 100 * humanize.KiByte
|
|
objectName := fmt.Sprintf("test-file-%v", rand.Uint32())
|
|
args["objectName"] = objectName
|
|
|
|
offset := length / 2
|
|
if _, err = tempfile.Seek(int64(offset), 0); err != nil {
|
|
logError(testName, function, args, startTime, "", "TempFile seek failed", err)
|
|
return
|
|
}
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, tempfile, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
tempfile.Close()
|
|
|
|
obj, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
defer obj.Close()
|
|
|
|
n, err := obj.Seek(int64(offset), 0)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Seek failed", err)
|
|
return
|
|
}
|
|
if n != int64(offset) {
|
|
logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(offset), n), err)
|
|
return
|
|
}
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName+"getobject", obj, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
st, err := c.StatObject(context.Background(), bucketName, objectName+"getobject", minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject failed", err)
|
|
return
|
|
}
|
|
if st.Size != int64(length-offset) {
|
|
logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(length-offset), n), err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests bucket re-create errors.
|
|
func testMakeBucketErrorV2() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "MakeBucket(bucketName, region)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"region": "eu-west-1",
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
region := "eu-west-1"
|
|
args["bucketName"] = bucketName
|
|
args["region"] = region
|
|
|
|
// Make a new bucket in 'eu-west-1'.
|
|
if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err == nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket did not fail for existing bucket name", err)
|
|
return
|
|
}
|
|
// Verify valid error response from server.
|
|
if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" &&
|
|
minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" {
|
|
logError(testName, function, args, startTime, "", "Invalid error returned by server", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test get object reader to not throw error on being closed twice.
|
|
func testGetObjectClosedTwiceV2() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "MakeBucket(bucketName, region)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"region": "eu-west-1",
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Generate 33K of data.
|
|
bufSize := dataFileMap["datafile-33-kB"]
|
|
reader := getDataReader("datafile-33-kB")
|
|
defer reader.Close()
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Read the data back
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
|
|
st, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(bufSize) {
|
|
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err)
|
|
return
|
|
}
|
|
if err := r.Close(); err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
if err := r.Close(); err == nil {
|
|
logError(testName, function, args, startTime, "", "Object is already closed, should return error", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests FPutObject hidden contentType setting
|
|
func testFPutObjectV2() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "FPutObject(bucketName, objectName, fileName, opts)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"fileName": "",
|
|
"opts": "",
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Make a temp file with 11*1024*1024 bytes of data.
|
|
file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest")
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "TempFile creation failed", err)
|
|
return
|
|
}
|
|
|
|
r := bytes.NewReader(bytes.Repeat([]byte("b"), 11*1024*1024))
|
|
n, err := io.CopyN(file, r, 11*1024*1024)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Copy failed", err)
|
|
return
|
|
}
|
|
if n != int64(11*1024*1024) {
|
|
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err)
|
|
return
|
|
}
|
|
|
|
// Close the file pro-actively for windows.
|
|
err = file.Close()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "File close failed", err)
|
|
return
|
|
}
|
|
|
|
// Set base object name
|
|
objectName := bucketName + "FPutObject"
|
|
args["objectName"] = objectName
|
|
args["fileName"] = file.Name()
|
|
|
|
// Perform standard FPutObject with contentType provided (Expecting application/octet-stream)
|
|
_, err = c.FPutObject(context.Background(), bucketName, objectName+"-standard", file.Name(), minio.PutObjectOptions{ContentType: "application/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "FPutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Perform FPutObject with no contentType provided (Expecting application/octet-stream)
|
|
args["objectName"] = objectName + "-Octet"
|
|
args["contentType"] = ""
|
|
|
|
_, err = c.FPutObject(context.Background(), bucketName, objectName+"-Octet", file.Name(), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "FPutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Add extension to temp file name
|
|
fileName := file.Name()
|
|
err = os.Rename(fileName, fileName+".gtar")
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Rename failed", err)
|
|
return
|
|
}
|
|
|
|
// Perform FPutObject with no contentType provided (Expecting application/x-gtar)
|
|
args["objectName"] = objectName + "-Octet"
|
|
args["contentType"] = ""
|
|
args["fileName"] = fileName + ".gtar"
|
|
|
|
_, err = c.FPutObject(context.Background(), bucketName, objectName+"-GTar", fileName+".gtar", minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "FPutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Check headers and sizes
|
|
rStandard, err := c.StatObject(context.Background(), bucketName, objectName+"-standard", minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject failed", err)
|
|
return
|
|
}
|
|
|
|
if rStandard.Size != 11*1024*1024 {
|
|
logError(testName, function, args, startTime, "", "Unexpected size", nil)
|
|
return
|
|
}
|
|
|
|
if rStandard.ContentType != "application/octet-stream" {
|
|
logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rStandard.ContentType, err)
|
|
return
|
|
}
|
|
|
|
rOctet, err := c.StatObject(context.Background(), bucketName, objectName+"-Octet", minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject failed", err)
|
|
return
|
|
}
|
|
if rOctet.ContentType != "application/octet-stream" {
|
|
logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rOctet.ContentType, err)
|
|
return
|
|
}
|
|
|
|
if rOctet.Size != 11*1024*1024 {
|
|
logError(testName, function, args, startTime, "", "Unexpected size", nil)
|
|
return
|
|
}
|
|
|
|
rGTar, err := c.StatObject(context.Background(), bucketName, objectName+"-GTar", minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject failed", err)
|
|
return
|
|
}
|
|
if rGTar.Size != 11*1024*1024 {
|
|
logError(testName, function, args, startTime, "", "Unexpected size", nil)
|
|
return
|
|
}
|
|
if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" && rGTar.ContentType != "application/x-tar" {
|
|
logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/x-tar , got "+rGTar.ContentType, err)
|
|
return
|
|
}
|
|
|
|
os.Remove(fileName + ".gtar")
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests various bucket supported formats.
|
|
func testMakeBucketRegionsV2() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "MakeBucket(bucketName, region)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"region": "eu-west-1",
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket in 'eu-central-1'.
|
|
if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "eu-west-1"}); err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
if err = cleanupBucket(bucketName, c); err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed while removing bucket recursively", err)
|
|
return
|
|
}
|
|
|
|
// Make a new bucket with '.' in its name, in 'us-west-2'. This
|
|
// request is internally staged into a path style instead of
|
|
// virtual host style.
|
|
if err = c.MakeBucket(context.Background(), bucketName+".withperiod", minio.MakeBucketOptions{Region: "us-west-2"}); err != nil {
|
|
args["bucketName"] = bucketName + ".withperiod"
|
|
args["region"] = "us-west-2"
|
|
logError(testName, function, args, startTime, "", "MakeBucket test with a bucket name with period, '.', failed", err)
|
|
return
|
|
}
|
|
|
|
// Delete all objects and buckets
|
|
if err = cleanupBucket(bucketName+".withperiod", c); err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed while removing bucket recursively", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests get object ReaderSeeker interface methods.
|
|
func testGetObjectReadSeekFunctionalV2() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "GetObject(bucketName, objectName)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Generate 33K of data.
|
|
bufSize := dataFileMap["datafile-33-kB"]
|
|
reader := getDataReader("datafile-33-kB")
|
|
defer reader.Close()
|
|
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
buf, err := ioutil.ReadAll(reader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
|
|
// Save the data.
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Read the data back
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
defer r.Close()
|
|
|
|
st, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(bufSize) {
|
|
logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err)
|
|
return
|
|
}
|
|
|
|
offset := int64(2048)
|
|
n, err := r.Seek(offset, 0)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Seek failed", err)
|
|
return
|
|
}
|
|
if n != offset {
|
|
logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err)
|
|
return
|
|
}
|
|
n, err = r.Seek(0, 1)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Seek failed", err)
|
|
return
|
|
}
|
|
if n != offset {
|
|
logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err)
|
|
return
|
|
}
|
|
_, err = r.Seek(offset, 2)
|
|
if err == nil {
|
|
logError(testName, function, args, startTime, "", "Seek on positive offset for whence '2' should error out", err)
|
|
return
|
|
}
|
|
n, err = r.Seek(-offset, 2)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Seek failed", err)
|
|
return
|
|
}
|
|
if n != st.Size-offset {
|
|
logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(st.Size-offset)+" got "+string(n), err)
|
|
return
|
|
}
|
|
|
|
var buffer1 bytes.Buffer
|
|
if _, err = io.CopyN(&buffer1, r, st.Size); err != nil {
|
|
if err != io.EOF {
|
|
logError(testName, function, args, startTime, "", "Copy failed", err)
|
|
return
|
|
}
|
|
}
|
|
if !bytes.Equal(buf[len(buf)-int(offset):], buffer1.Bytes()) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err)
|
|
return
|
|
}
|
|
|
|
// Seek again and read again.
|
|
n, err = r.Seek(offset-1, 0)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Seek failed", err)
|
|
return
|
|
}
|
|
if n != (offset - 1) {
|
|
logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset-1)+" got "+string(n), err)
|
|
return
|
|
}
|
|
|
|
var buffer2 bytes.Buffer
|
|
if _, err = io.CopyN(&buffer2, r, st.Size); err != nil {
|
|
if err != io.EOF {
|
|
logError(testName, function, args, startTime, "", "Copy failed", err)
|
|
return
|
|
}
|
|
}
|
|
// Verify now lesser bytes.
|
|
if !bytes.Equal(buf[2047:], buffer2.Bytes()) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests get object ReaderAt interface methods.
|
|
func testGetObjectReadAtFunctionalV2() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "GetObject(bucketName, objectName)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Generate 33K of data.
|
|
bufSize := dataFileMap["datafile-33-kB"]
|
|
reader := getDataReader("datafile-33-kB")
|
|
defer reader.Close()
|
|
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
buf, err := ioutil.ReadAll(reader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
|
|
// Save the data
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Read the data back
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
defer r.Close()
|
|
|
|
st, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(bufSize) {
|
|
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err)
|
|
return
|
|
}
|
|
|
|
offset := int64(2048)
|
|
|
|
// Read directly
|
|
buf2 := make([]byte, 512)
|
|
buf3 := make([]byte, 512)
|
|
buf4 := make([]byte, 512)
|
|
|
|
m, err := r.ReadAt(buf2, offset)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
if m != len(buf2) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+" got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf2, buf[offset:offset+512]) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
|
|
return
|
|
}
|
|
offset += 512
|
|
m, err = r.ReadAt(buf3, offset)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
if m != len(buf3) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+" got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf3, buf[offset:offset+512]) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
|
|
return
|
|
}
|
|
offset += 512
|
|
m, err = r.ReadAt(buf4, offset)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
if m != len(buf4) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+" got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf4, buf[offset:offset+512]) {
|
|
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
|
|
return
|
|
}
|
|
|
|
buf5 := make([]byte, bufSize)
|
|
// Read the whole object.
|
|
m, err = r.ReadAt(buf5, 0)
|
|
if err != nil {
|
|
if err != io.EOF {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
}
|
|
if m != len(buf5) {
|
|
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+" got "+string(m), err)
|
|
return
|
|
}
|
|
if !bytes.Equal(buf, buf5) {
|
|
logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err)
|
|
return
|
|
}
|
|
|
|
buf6 := make([]byte, bufSize+1)
|
|
// Read the whole object and beyond.
|
|
_, err = r.ReadAt(buf6, 0)
|
|
if err != nil {
|
|
if err != io.EOF {
|
|
logError(testName, function, args, startTime, "", "ReadAt failed", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Tests copy object
|
|
func testCopyObjectV2() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObject(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
|
|
// Make a new bucket in 'us-east-1' (source bucket).
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Make a new bucket in 'us-east-1' (destination bucket).
|
|
err = c.MakeBucket(context.Background(), bucketName+"-copy", minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
defer cleanupBucket(bucketName+"-copy", c)
|
|
|
|
// Generate 33K of data.
|
|
bufSize := dataFileMap["datafile-33-kB"]
|
|
reader := getDataReader("datafile-33-kB")
|
|
defer reader.Close()
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
// Check the various fields of source object against destination object.
|
|
objInfo, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
r.Close()
|
|
|
|
// Copy Source
|
|
src := minio.CopySrcOptions{
|
|
Bucket: bucketName,
|
|
Object: objectName,
|
|
MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC),
|
|
MatchETag: objInfo.ETag,
|
|
}
|
|
args["source"] = src
|
|
|
|
// Set copy conditions.
|
|
dst := minio.CopyDestOptions{
|
|
Bucket: bucketName + "-copy",
|
|
Object: objectName + "-copy",
|
|
}
|
|
args["destination"] = dst
|
|
|
|
// Perform the Copy
|
|
_, err = c.CopyObject(context.Background(), dst, src)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Source object
|
|
r, err = c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
// Destination object
|
|
readerCopy, err := c.GetObject(context.Background(), bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
// Check the various fields of source object against destination object.
|
|
objInfo, err = r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
objInfoCopy, err := readerCopy.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
if objInfo.Size != objInfoCopy.Size {
|
|
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+" got "+string(objInfo.Size), err)
|
|
return
|
|
}
|
|
|
|
// Close all the readers.
|
|
r.Close()
|
|
readerCopy.Close()
|
|
|
|
// CopyObject again but with wrong conditions
|
|
src = minio.CopySrcOptions{
|
|
Bucket: bucketName,
|
|
Object: objectName,
|
|
MatchUnmodifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC),
|
|
NoMatchETag: objInfo.ETag,
|
|
}
|
|
|
|
// Perform the Copy which should fail
|
|
_, err = c.CopyObject(context.Background(), dst, src)
|
|
if err == nil {
|
|
logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
func testComposeObjectErrorCasesWrapper(c *minio.Client) {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "ComposeObject(destination, sourceList)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
|
|
// Make a new bucket in 'us-east-1' (source bucket).
|
|
err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Test that more than 10K source objects cannot be
|
|
// concatenated.
|
|
srcArr := [10001]minio.CopySrcOptions{}
|
|
srcSlice := srcArr[:]
|
|
dst := minio.CopyDestOptions{
|
|
Bucket: bucketName,
|
|
Object: "object",
|
|
}
|
|
|
|
args["destination"] = dst
|
|
// Just explain about srcArr in args["sourceList"]
|
|
// to stop having 10,001 null headers logged
|
|
args["sourceList"] = "source array of 10,001 elements"
|
|
if _, err := c.ComposeObject(context.Background(), dst, srcSlice...); err == nil {
|
|
logError(testName, function, args, startTime, "", "Expected error in ComposeObject", err)
|
|
return
|
|
} else if err.Error() != "There must be as least one and up to 10000 source objects." {
|
|
logError(testName, function, args, startTime, "", "Got unexpected error", err)
|
|
return
|
|
}
|
|
|
|
// Create a source with invalid offset spec and check that
|
|
// error is returned:
|
|
// 1. Create the source object.
|
|
const badSrcSize = 5 * 1024 * 1024
|
|
buf := bytes.Repeat([]byte("1"), badSrcSize)
|
|
_, err = c.PutObject(context.Background(), bucketName, "badObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
// 2. Set invalid range spec on the object (going beyond
|
|
// object size)
|
|
badSrc := minio.CopySrcOptions{
|
|
Bucket: bucketName,
|
|
Object: "badObject",
|
|
MatchRange: true,
|
|
Start: 1,
|
|
End: badSrcSize,
|
|
}
|
|
|
|
// 3. ComposeObject call should fail.
|
|
if _, err := c.ComposeObject(context.Background(), dst, badSrc); err == nil {
|
|
logError(testName, function, args, startTime, "", "ComposeObject expected to fail", err)
|
|
return
|
|
} else if !strings.Contains(err.Error(), "has invalid segment-to-copy") {
|
|
logError(testName, function, args, startTime, "", "Got invalid error", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test expected error cases
|
|
func testComposeObjectErrorCasesV2() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "ComposeObject(destination, sourceList)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
testComposeObjectErrorCasesWrapper(c)
|
|
}
|
|
|
|
func testComposeMultipleSources(c *minio.Client) {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "ComposeObject(destination, sourceList)"
|
|
args := map[string]interface{}{
|
|
"destination": "",
|
|
"sourceList": "",
|
|
}
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
// Make a new bucket in 'us-east-1' (source bucket).
|
|
err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Upload a small source object
|
|
const srcSize = 1024 * 1024 * 5
|
|
buf := bytes.Repeat([]byte("1"), srcSize)
|
|
_, err = c.PutObject(context.Background(), bucketName, "srcObject", bytes.NewReader(buf), int64(srcSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// We will append 10 copies of the object.
|
|
srcs := []minio.CopySrcOptions{}
|
|
for i := 0; i < 10; i++ {
|
|
srcs = append(srcs, minio.CopySrcOptions{
|
|
Bucket: bucketName,
|
|
Object: "srcObject",
|
|
})
|
|
}
|
|
|
|
// make the last part very small
|
|
srcs[9].MatchRange = true
|
|
|
|
args["sourceList"] = srcs
|
|
|
|
dst := minio.CopyDestOptions{
|
|
Bucket: bucketName,
|
|
Object: "dstObject",
|
|
}
|
|
args["destination"] = dst
|
|
|
|
ui, err := c.ComposeObject(context.Background(), dst, srcs...)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ComposeObject failed", err)
|
|
return
|
|
}
|
|
|
|
if ui.Size != 9*srcSize+1 {
|
|
logError(testName, function, args, startTime, "", "ComposeObject returned unexpected size", err)
|
|
return
|
|
}
|
|
|
|
objProps, err := c.StatObject(context.Background(), bucketName, "dstObject", minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject failed", err)
|
|
return
|
|
}
|
|
|
|
if objProps.Size != 9*srcSize+1 {
|
|
logError(testName, function, args, startTime, "", "Size mismatched! Expected "+string(10000*srcSize)+" got "+string(objProps.Size), err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test concatenating multiple 10K objects V2
|
|
func testCompose10KSourcesV2() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "ComposeObject(destination, sourceList)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
testComposeMultipleSources(c)
|
|
}
|
|
|
|
func testEncryptedEmptyObject() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "PutObject(bucketName, objectName, reader, objectSize, opts)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
// Make a new bucket in 'us-east-1' (source bucket).
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
sse := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"object"))
|
|
|
|
// 1. create an sse-c encrypted object to copy by uploading
|
|
const srcSize = 0
|
|
var buf []byte // Empty buffer
|
|
args["objectName"] = "object"
|
|
_, err = c.PutObject(context.Background(), bucketName, "object", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ServerSideEncryption: sse})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject call failed", err)
|
|
return
|
|
}
|
|
|
|
// 2. Test CopyObject for an empty object
|
|
src := minio.CopySrcOptions{
|
|
Bucket: bucketName,
|
|
Object: "object",
|
|
Encryption: sse,
|
|
}
|
|
|
|
dst := minio.CopyDestOptions{
|
|
Bucket: bucketName,
|
|
Object: "new-object",
|
|
Encryption: sse,
|
|
}
|
|
|
|
if _, err = c.CopyObject(context.Background(), dst, src); err != nil {
|
|
function = "CopyObject(dst, src)"
|
|
logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject failed", err)
|
|
return
|
|
}
|
|
|
|
// 3. Test Key rotation
|
|
newSSE := encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"new-object"))
|
|
src = minio.CopySrcOptions{
|
|
Bucket: bucketName,
|
|
Object: "new-object",
|
|
Encryption: sse,
|
|
}
|
|
|
|
dst = minio.CopyDestOptions{
|
|
Bucket: bucketName,
|
|
Object: "new-object",
|
|
Encryption: newSSE,
|
|
}
|
|
|
|
if _, err = c.CopyObject(context.Background(), dst, src); err != nil {
|
|
function = "CopyObject(dst, src)"
|
|
logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject with key rotation failed", err)
|
|
return
|
|
}
|
|
|
|
// 4. Download the object.
|
|
reader, err := c.GetObject(context.Background(), bucketName, "new-object", minio.GetObjectOptions{ServerSideEncryption: newSSE})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
defer reader.Close()
|
|
|
|
decBytes, err := ioutil.ReadAll(reader)
|
|
if err != nil {
|
|
logError(testName, function, map[string]interface{}{}, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(decBytes, buf) {
|
|
logError(testName, function, map[string]interface{}{}, startTime, "", "Downloaded object doesn't match the empty encrypted object", err)
|
|
return
|
|
}
|
|
|
|
delete(args, "objectName")
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, sseDst encrypt.ServerSide) {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncNameLoc(2)
|
|
function := "CopyObject(destination, source)"
|
|
args := map[string]interface{}{}
|
|
var srcEncryption, dstEncryption encrypt.ServerSide
|
|
|
|
// Make a new bucket in 'us-east-1' (source bucket).
|
|
err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// 1. create an sse-c encrypted object to copy by uploading
|
|
const srcSize = 1024 * 1024
|
|
buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB
|
|
_, err = c.PutObject(context.Background(), bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{
|
|
ServerSideEncryption: sseSrc,
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject call failed", err)
|
|
return
|
|
}
|
|
|
|
if sseSrc != nil && sseSrc.Type() != encrypt.S3 {
|
|
srcEncryption = sseSrc
|
|
}
|
|
|
|
// 2. copy object and change encryption key
|
|
src := minio.CopySrcOptions{
|
|
Bucket: bucketName,
|
|
Object: "srcObject",
|
|
Encryption: srcEncryption,
|
|
}
|
|
args["source"] = src
|
|
|
|
dst := minio.CopyDestOptions{
|
|
Bucket: bucketName,
|
|
Object: "dstObject",
|
|
Encryption: sseDst,
|
|
}
|
|
args["destination"] = dst
|
|
|
|
_, err = c.CopyObject(context.Background(), dst, src)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObject failed", err)
|
|
return
|
|
}
|
|
|
|
if sseDst != nil && sseDst.Type() != encrypt.S3 {
|
|
dstEncryption = sseDst
|
|
}
|
|
// 3. get copied object and check if content is equal
|
|
coreClient := minio.Core{c}
|
|
reader, _, _, err := coreClient.GetObject(context.Background(), bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: dstEncryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
|
|
decBytes, err := ioutil.ReadAll(reader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(decBytes, buf) {
|
|
logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err)
|
|
return
|
|
}
|
|
reader.Close()
|
|
|
|
// Test key rotation for source object in-place.
|
|
var newSSE encrypt.ServerSide
|
|
if sseSrc != nil && sseSrc.Type() == encrypt.SSEC {
|
|
newSSE = encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"srcObject")) // replace key
|
|
}
|
|
if sseSrc != nil && sseSrc.Type() == encrypt.S3 {
|
|
newSSE = encrypt.NewSSE()
|
|
}
|
|
if newSSE != nil {
|
|
dst = minio.CopyDestOptions{
|
|
Bucket: bucketName,
|
|
Object: "srcObject",
|
|
Encryption: newSSE,
|
|
}
|
|
args["destination"] = dst
|
|
|
|
_, err = c.CopyObject(context.Background(), dst, src)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Get copied object and check if content is equal
|
|
reader, _, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
|
|
decBytes, err = ioutil.ReadAll(reader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(decBytes, buf) {
|
|
logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err)
|
|
return
|
|
}
|
|
reader.Close()
|
|
|
|
// Test in-place decryption.
|
|
dst = minio.CopyDestOptions{
|
|
Bucket: bucketName,
|
|
Object: "srcObject",
|
|
}
|
|
args["destination"] = dst
|
|
|
|
src = minio.CopySrcOptions{
|
|
Bucket: bucketName,
|
|
Object: "srcObject",
|
|
Encryption: newSSE,
|
|
}
|
|
args["source"] = src
|
|
_, err = c.CopyObject(context.Background(), dst, src)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObject Key rotation failed", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
// Get copied decrypted object and check if content is equal
|
|
reader, _, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
defer reader.Close()
|
|
|
|
decBytes, err = ioutil.ReadAll(reader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(decBytes, buf) {
|
|
logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test encrypted copy object
|
|
func testUnencryptedToSSECCopyObject() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObject(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
|
|
sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject"))
|
|
// c.TraceOn(os.Stderr)
|
|
testEncryptedCopyObjectWrapper(c, bucketName, nil, sseDst)
|
|
}
|
|
|
|
// Test encrypted copy object
|
|
func testUnencryptedToSSES3CopyObject() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObject(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
|
|
var sseSrc encrypt.ServerSide
|
|
sseDst := encrypt.NewSSE()
|
|
// c.TraceOn(os.Stderr)
|
|
testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
|
|
}
|
|
|
|
// Test encrypted copy object
|
|
func testUnencryptedToUnencryptedCopyObject() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObject(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
|
|
var sseSrc, sseDst encrypt.ServerSide
|
|
// c.TraceOn(os.Stderr)
|
|
testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
|
|
}
|
|
|
|
// Test encrypted copy object
|
|
func testEncryptedSSECToSSECCopyObject() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObject(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
|
|
sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject"))
|
|
sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject"))
|
|
// c.TraceOn(os.Stderr)
|
|
testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
|
|
}
|
|
|
|
// Test encrypted copy object
|
|
func testEncryptedSSECToSSES3CopyObject() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObject(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
|
|
sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject"))
|
|
sseDst := encrypt.NewSSE()
|
|
// c.TraceOn(os.Stderr)
|
|
testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
|
|
}
|
|
|
|
// Test encrypted copy object
|
|
func testEncryptedSSECToUnencryptedCopyObject() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObject(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
|
|
sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject"))
|
|
var sseDst encrypt.ServerSide
|
|
// c.TraceOn(os.Stderr)
|
|
testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
|
|
}
|
|
|
|
// Test encrypted copy object
|
|
func testEncryptedSSES3ToSSECCopyObject() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObject(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
|
|
sseSrc := encrypt.NewSSE()
|
|
sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject"))
|
|
// c.TraceOn(os.Stderr)
|
|
testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
|
|
}
|
|
|
|
// Test encrypted copy object
|
|
func testEncryptedSSES3ToSSES3CopyObject() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObject(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
|
|
sseSrc := encrypt.NewSSE()
|
|
sseDst := encrypt.NewSSE()
|
|
// c.TraceOn(os.Stderr)
|
|
testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
|
|
}
|
|
|
|
// Test encrypted copy object
|
|
func testEncryptedSSES3ToUnencryptedCopyObject() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObject(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
|
|
sseSrc := encrypt.NewSSE()
|
|
var sseDst encrypt.ServerSide
|
|
// c.TraceOn(os.Stderr)
|
|
testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
|
|
}
|
|
|
|
// Test encrypted copy object
|
|
func testEncryptedCopyObjectV2() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObject(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
|
|
sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject"))
|
|
sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject"))
|
|
// c.TraceOn(os.Stderr)
|
|
testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
|
|
}
|
|
|
|
func testDecryptedCopyObject() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObject(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
bucketName, objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-"), "object"
|
|
if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}); err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
encryption := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName))
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(bytes.Repeat([]byte("a"), 1024*1024)), 1024*1024, minio.PutObjectOptions{
|
|
ServerSideEncryption: encryption,
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject call failed", err)
|
|
return
|
|
}
|
|
|
|
src := minio.CopySrcOptions{
|
|
Bucket: bucketName,
|
|
Object: objectName,
|
|
Encryption: encrypt.SSECopy(encryption),
|
|
}
|
|
args["source"] = src
|
|
|
|
dst := minio.CopyDestOptions{
|
|
Bucket: bucketName,
|
|
Object: "decrypted-" + objectName,
|
|
}
|
|
args["destination"] = dst
|
|
|
|
if _, err = c.CopyObject(context.Background(), dst, src); err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObject failed", err)
|
|
return
|
|
}
|
|
if _, err = c.GetObject(context.Background(), bucketName, "decrypted-"+objectName, minio.GetObjectOptions{}); err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
func testSSECMultipartEncryptedToSSECCopyObjectPart() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObjectPart(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
client, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Instantiate new core client object.
|
|
c := minio.Core{client}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
defer cleanupBucket(bucketName, client)
|
|
// Make a buffer with 6MB of data
|
|
buf := bytes.Repeat([]byte("abcdef"), 1024*1024)
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
password := "correct horse battery staple"
|
|
srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName))
|
|
|
|
// Upload a 6MB object using multipart mechanism
|
|
uploadID, err := c.NewMultipartUpload(context.Background(), bucketName, objectName, minio.PutObjectOptions{ServerSideEncryption: srcencryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
var completeParts []minio.CompletePart
|
|
|
|
part, err := c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, bytes.NewReader(buf[:5*1024*1024]), 5*1024*1024, "", "", srcencryption)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObjectPart call failed", err)
|
|
return
|
|
}
|
|
completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag})
|
|
|
|
part, err = c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 2, bytes.NewReader(buf[5*1024*1024:]), 1024*1024, "", "", srcencryption)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObjectPart call failed", err)
|
|
return
|
|
}
|
|
completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag})
|
|
|
|
// Complete the multipart upload
|
|
_, err = c.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, completeParts, minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Stat the object and check its length matches
|
|
objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
|
|
destBucketName := bucketName
|
|
destObjectName := objectName + "-dest"
|
|
dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName))
|
|
|
|
uploadID, err = c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Content of the destination object will be two copies of
|
|
// `objectName` concatenated, followed by first byte of
|
|
// `objectName`.
|
|
metadata := make(map[string]string)
|
|
header := make(http.Header)
|
|
encrypt.SSECopy(srcencryption).Marshal(header)
|
|
dstencryption.Marshal(header)
|
|
for k, v := range header {
|
|
metadata[k] = v[0]
|
|
}
|
|
|
|
metadata["x-amz-copy-source-if-match"] = objInfo.ETag
|
|
|
|
// First of three parts
|
|
fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Second of three parts
|
|
sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Last of three parts
|
|
lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Complete the multipart upload
|
|
_, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Stat the object and check its length matches
|
|
objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
|
|
if objInfo.Size != (6*1024*1024)*2+1 {
|
|
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
|
|
return
|
|
}
|
|
|
|
// Now we read the data back
|
|
getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption}
|
|
getOpts.SetRange(0, 6*1024*1024-1)
|
|
r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf := make([]byte, 6*1024*1024)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf, buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in first 6MB", err)
|
|
return
|
|
}
|
|
|
|
getOpts.SetRange(6*1024*1024, 0)
|
|
r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf = make([]byte, 6*1024*1024+1)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf[:6*1024*1024], buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in second 6MB", err)
|
|
return
|
|
}
|
|
if getBuf[6*1024*1024] != buf[0] {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
|
|
// Do not need to remove destBucketName its same as bucketName.
|
|
}
|
|
|
|
// Test Core CopyObjectPart implementation
|
|
func testSSECEncryptedToSSECCopyObjectPart() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObjectPart(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
client, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Instantiate new core client object.
|
|
c := minio.Core{client}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
defer cleanupBucket(bucketName, client)
|
|
// Make a buffer with 5MB of data
|
|
buf := bytes.Repeat([]byte("abcde"), 1024*1024)
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
password := "correct horse battery staple"
|
|
srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName))
|
|
putmetadata := map[string]string{
|
|
"Content-Type": "binary/octet-stream",
|
|
}
|
|
opts := minio.PutObjectOptions{
|
|
UserMetadata: putmetadata,
|
|
ServerSideEncryption: srcencryption,
|
|
}
|
|
uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject call failed", err)
|
|
return
|
|
}
|
|
|
|
st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(len(buf)) {
|
|
logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
|
|
return
|
|
}
|
|
|
|
destBucketName := bucketName
|
|
destObjectName := objectName + "-dest"
|
|
dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName))
|
|
|
|
uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Content of the destination object will be two copies of
|
|
// `objectName` concatenated, followed by first byte of
|
|
// `objectName`.
|
|
metadata := make(map[string]string)
|
|
header := make(http.Header)
|
|
encrypt.SSECopy(srcencryption).Marshal(header)
|
|
dstencryption.Marshal(header)
|
|
for k, v := range header {
|
|
metadata[k] = v[0]
|
|
}
|
|
|
|
metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
|
|
|
|
// First of three parts
|
|
fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Second of three parts
|
|
sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Last of three parts
|
|
lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Complete the multipart upload
|
|
_, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Stat the object and check its length matches
|
|
objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
|
|
if objInfo.Size != (5*1024*1024)*2+1 {
|
|
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
|
|
return
|
|
}
|
|
|
|
// Now we read the data back
|
|
getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption}
|
|
getOpts.SetRange(0, 5*1024*1024-1)
|
|
r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf := make([]byte, 5*1024*1024)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf, buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
|
|
return
|
|
}
|
|
|
|
getOpts.SetRange(5*1024*1024, 0)
|
|
r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf = make([]byte, 5*1024*1024+1)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf[:5*1024*1024], buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
|
|
return
|
|
}
|
|
if getBuf[5*1024*1024] != buf[0] {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
|
|
// Do not need to remove destBucketName its same as bucketName.
|
|
}
|
|
|
|
// Test Core CopyObjectPart implementation for SSEC encrypted to unencrypted copy
|
|
func testSSECEncryptedToUnencryptedCopyPart() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObjectPart(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
client, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Instantiate new core client object.
|
|
c := minio.Core{client}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
defer cleanupBucket(bucketName, client)
|
|
// Make a buffer with 5MB of data
|
|
buf := bytes.Repeat([]byte("abcde"), 1024*1024)
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
password := "correct horse battery staple"
|
|
srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName))
|
|
|
|
opts := minio.PutObjectOptions{
|
|
UserMetadata: map[string]string{
|
|
"Content-Type": "binary/octet-stream",
|
|
},
|
|
ServerSideEncryption: srcencryption,
|
|
}
|
|
uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject call failed", err)
|
|
return
|
|
}
|
|
|
|
st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(len(buf)) {
|
|
logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
|
|
return
|
|
}
|
|
|
|
destBucketName := bucketName
|
|
destObjectName := objectName + "-dest"
|
|
var dstencryption encrypt.ServerSide
|
|
|
|
uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Content of the destination object will be two copies of
|
|
// `objectName` concatenated, followed by first byte of
|
|
// `objectName`.
|
|
metadata := make(map[string]string)
|
|
header := make(http.Header)
|
|
encrypt.SSECopy(srcencryption).Marshal(header)
|
|
for k, v := range header {
|
|
metadata[k] = v[0]
|
|
}
|
|
|
|
metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
|
|
|
|
// First of three parts
|
|
fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Second of three parts
|
|
sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Last of three parts
|
|
lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Complete the multipart upload
|
|
_, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Stat the object and check its length matches
|
|
objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
|
|
if objInfo.Size != (5*1024*1024)*2+1 {
|
|
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
|
|
return
|
|
}
|
|
|
|
// Now we read the data back
|
|
getOpts := minio.GetObjectOptions{}
|
|
getOpts.SetRange(0, 5*1024*1024-1)
|
|
r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf := make([]byte, 5*1024*1024)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf, buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
|
|
return
|
|
}
|
|
|
|
getOpts.SetRange(5*1024*1024, 0)
|
|
r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf = make([]byte, 5*1024*1024+1)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf[:5*1024*1024], buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
|
|
return
|
|
}
|
|
if getBuf[5*1024*1024] != buf[0] {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
|
|
// Do not need to remove destBucketName its same as bucketName.
|
|
}
|
|
|
|
// Test Core CopyObjectPart implementation for SSEC encrypted to SSE-S3 encrypted copy
|
|
func testSSECEncryptedToSSES3CopyObjectPart() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObjectPart(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
client, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Instantiate new core client object.
|
|
c := minio.Core{client}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
defer cleanupBucket(bucketName, client)
|
|
// Make a buffer with 5MB of data
|
|
buf := bytes.Repeat([]byte("abcde"), 1024*1024)
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
password := "correct horse battery staple"
|
|
srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName))
|
|
putmetadata := map[string]string{
|
|
"Content-Type": "binary/octet-stream",
|
|
}
|
|
opts := minio.PutObjectOptions{
|
|
UserMetadata: putmetadata,
|
|
ServerSideEncryption: srcencryption,
|
|
}
|
|
|
|
uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject call failed", err)
|
|
return
|
|
}
|
|
|
|
st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(len(buf)) {
|
|
logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
|
|
return
|
|
}
|
|
|
|
destBucketName := bucketName
|
|
destObjectName := objectName + "-dest"
|
|
dstencryption := encrypt.NewSSE()
|
|
|
|
uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Content of the destination object will be two copies of
|
|
// `objectName` concatenated, followed by first byte of
|
|
// `objectName`.
|
|
metadata := make(map[string]string)
|
|
header := make(http.Header)
|
|
encrypt.SSECopy(srcencryption).Marshal(header)
|
|
dstencryption.Marshal(header)
|
|
|
|
for k, v := range header {
|
|
metadata[k] = v[0]
|
|
}
|
|
|
|
metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
|
|
|
|
// First of three parts
|
|
fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Second of three parts
|
|
sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Last of three parts
|
|
lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Complete the multipart upload
|
|
_, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Stat the object and check its length matches
|
|
objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
|
|
if objInfo.Size != (5*1024*1024)*2+1 {
|
|
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
|
|
return
|
|
}
|
|
|
|
// Now we read the data back
|
|
getOpts := minio.GetObjectOptions{}
|
|
getOpts.SetRange(0, 5*1024*1024-1)
|
|
r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf := make([]byte, 5*1024*1024)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf, buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
|
|
return
|
|
}
|
|
|
|
getOpts.SetRange(5*1024*1024, 0)
|
|
r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf = make([]byte, 5*1024*1024+1)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf[:5*1024*1024], buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
|
|
return
|
|
}
|
|
if getBuf[5*1024*1024] != buf[0] {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
|
|
// Do not need to remove destBucketName its same as bucketName.
|
|
}
|
|
|
|
// Test Core CopyObjectPart implementation for unencrypted to SSEC encryption copy part
|
|
func testUnencryptedToSSECCopyObjectPart() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObjectPart(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
client, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Instantiate new core client object.
|
|
c := minio.Core{client}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
defer cleanupBucket(bucketName, client)
|
|
// Make a buffer with 5MB of data
|
|
buf := bytes.Repeat([]byte("abcde"), 1024*1024)
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
password := "correct horse battery staple"
|
|
putmetadata := map[string]string{
|
|
"Content-Type": "binary/octet-stream",
|
|
}
|
|
opts := minio.PutObjectOptions{
|
|
UserMetadata: putmetadata,
|
|
}
|
|
uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject call failed", err)
|
|
return
|
|
}
|
|
|
|
st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(len(buf)) {
|
|
logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
|
|
return
|
|
}
|
|
|
|
destBucketName := bucketName
|
|
destObjectName := objectName + "-dest"
|
|
dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName))
|
|
|
|
uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Content of the destination object will be two copies of
|
|
// `objectName` concatenated, followed by first byte of
|
|
// `objectName`.
|
|
metadata := make(map[string]string)
|
|
header := make(http.Header)
|
|
dstencryption.Marshal(header)
|
|
for k, v := range header {
|
|
metadata[k] = v[0]
|
|
}
|
|
|
|
metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
|
|
|
|
// First of three parts
|
|
fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Second of three parts
|
|
sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Last of three parts
|
|
lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Complete the multipart upload
|
|
_, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Stat the object and check its length matches
|
|
objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
|
|
if objInfo.Size != (5*1024*1024)*2+1 {
|
|
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
|
|
return
|
|
}
|
|
|
|
// Now we read the data back
|
|
getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption}
|
|
getOpts.SetRange(0, 5*1024*1024-1)
|
|
r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf := make([]byte, 5*1024*1024)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf, buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
|
|
return
|
|
}
|
|
|
|
getOpts.SetRange(5*1024*1024, 0)
|
|
r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf = make([]byte, 5*1024*1024+1)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf[:5*1024*1024], buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
|
|
return
|
|
}
|
|
if getBuf[5*1024*1024] != buf[0] {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
|
|
// Do not need to remove destBucketName its same as bucketName.
|
|
}
|
|
|
|
// Test Core CopyObjectPart implementation for unencrypted to unencrypted copy
|
|
func testUnencryptedToUnencryptedCopyPart() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObjectPart(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
client, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Instantiate new core client object.
|
|
c := minio.Core{client}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
defer cleanupBucket(bucketName, client)
|
|
// Make a buffer with 5MB of data
|
|
buf := bytes.Repeat([]byte("abcde"), 1024*1024)
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
putmetadata := map[string]string{
|
|
"Content-Type": "binary/octet-stream",
|
|
}
|
|
opts := minio.PutObjectOptions{
|
|
UserMetadata: putmetadata,
|
|
}
|
|
uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject call failed", err)
|
|
return
|
|
}
|
|
st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(len(buf)) {
|
|
logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
|
|
return
|
|
}
|
|
|
|
destBucketName := bucketName
|
|
destObjectName := objectName + "-dest"
|
|
|
|
uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Content of the destination object will be two copies of
|
|
// `objectName` concatenated, followed by first byte of
|
|
// `objectName`.
|
|
metadata := make(map[string]string)
|
|
header := make(http.Header)
|
|
for k, v := range header {
|
|
metadata[k] = v[0]
|
|
}
|
|
|
|
metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
|
|
|
|
// First of three parts
|
|
fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Second of three parts
|
|
sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Last of three parts
|
|
lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Complete the multipart upload
|
|
_, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Stat the object and check its length matches
|
|
objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
|
|
if objInfo.Size != (5*1024*1024)*2+1 {
|
|
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
|
|
return
|
|
}
|
|
|
|
// Now we read the data back
|
|
getOpts := minio.GetObjectOptions{}
|
|
getOpts.SetRange(0, 5*1024*1024-1)
|
|
r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf := make([]byte, 5*1024*1024)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf, buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
|
|
return
|
|
}
|
|
|
|
getOpts.SetRange(5*1024*1024, 0)
|
|
r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf = make([]byte, 5*1024*1024+1)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf[:5*1024*1024], buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
|
|
return
|
|
}
|
|
if getBuf[5*1024*1024] != buf[0] {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
|
|
// Do not need to remove destBucketName its same as bucketName.
|
|
}
|
|
|
|
// Test Core CopyObjectPart implementation for unencrypted to SSE-S3 encrypted copy
|
|
func testUnencryptedToSSES3CopyObjectPart() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObjectPart(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
client, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Instantiate new core client object.
|
|
c := minio.Core{client}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
defer cleanupBucket(bucketName, client)
|
|
// Make a buffer with 5MB of data
|
|
buf := bytes.Repeat([]byte("abcde"), 1024*1024)
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
opts := minio.PutObjectOptions{
|
|
UserMetadata: map[string]string{
|
|
"Content-Type": "binary/octet-stream",
|
|
},
|
|
}
|
|
uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject call failed", err)
|
|
return
|
|
}
|
|
st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(len(buf)) {
|
|
logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
|
|
return
|
|
}
|
|
|
|
destBucketName := bucketName
|
|
destObjectName := objectName + "-dest"
|
|
dstencryption := encrypt.NewSSE()
|
|
|
|
uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Content of the destination object will be two copies of
|
|
// `objectName` concatenated, followed by first byte of
|
|
// `objectName`.
|
|
metadata := make(map[string]string)
|
|
header := make(http.Header)
|
|
dstencryption.Marshal(header)
|
|
|
|
for k, v := range header {
|
|
metadata[k] = v[0]
|
|
}
|
|
|
|
metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
|
|
|
|
// First of three parts
|
|
fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Second of three parts
|
|
sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Last of three parts
|
|
lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Complete the multipart upload
|
|
_, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Stat the object and check its length matches
|
|
objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
|
|
if objInfo.Size != (5*1024*1024)*2+1 {
|
|
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
|
|
return
|
|
}
|
|
|
|
// Now we read the data back
|
|
getOpts := minio.GetObjectOptions{}
|
|
getOpts.SetRange(0, 5*1024*1024-1)
|
|
r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf := make([]byte, 5*1024*1024)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf, buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
|
|
return
|
|
}
|
|
|
|
getOpts.SetRange(5*1024*1024, 0)
|
|
r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf = make([]byte, 5*1024*1024+1)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf[:5*1024*1024], buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
|
|
return
|
|
}
|
|
if getBuf[5*1024*1024] != buf[0] {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
|
|
// Do not need to remove destBucketName its same as bucketName.
|
|
}
|
|
|
|
// Test Core CopyObjectPart implementation for SSE-S3 to SSEC encryption copy part
|
|
func testSSES3EncryptedToSSECCopyObjectPart() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObjectPart(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
client, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Instantiate new core client object.
|
|
c := minio.Core{client}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
defer cleanupBucket(bucketName, client)
|
|
// Make a buffer with 5MB of data
|
|
buf := bytes.Repeat([]byte("abcde"), 1024*1024)
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
password := "correct horse battery staple"
|
|
srcEncryption := encrypt.NewSSE()
|
|
opts := minio.PutObjectOptions{
|
|
UserMetadata: map[string]string{
|
|
"Content-Type": "binary/octet-stream",
|
|
},
|
|
ServerSideEncryption: srcEncryption,
|
|
}
|
|
uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject call failed", err)
|
|
return
|
|
}
|
|
|
|
st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcEncryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(len(buf)) {
|
|
logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
|
|
return
|
|
}
|
|
|
|
destBucketName := bucketName
|
|
destObjectName := objectName + "-dest"
|
|
dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName))
|
|
|
|
uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Content of the destination object will be two copies of
|
|
// `objectName` concatenated, followed by first byte of
|
|
// `objectName`.
|
|
metadata := make(map[string]string)
|
|
header := make(http.Header)
|
|
dstencryption.Marshal(header)
|
|
for k, v := range header {
|
|
metadata[k] = v[0]
|
|
}
|
|
|
|
metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
|
|
|
|
// First of three parts
|
|
fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Second of three parts
|
|
sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Last of three parts
|
|
lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Complete the multipart upload
|
|
_, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Stat the object and check its length matches
|
|
objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
|
|
if objInfo.Size != (5*1024*1024)*2+1 {
|
|
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
|
|
return
|
|
}
|
|
|
|
// Now we read the data back
|
|
getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption}
|
|
getOpts.SetRange(0, 5*1024*1024-1)
|
|
r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf := make([]byte, 5*1024*1024)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf, buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
|
|
return
|
|
}
|
|
|
|
getOpts.SetRange(5*1024*1024, 0)
|
|
r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf = make([]byte, 5*1024*1024+1)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf[:5*1024*1024], buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
|
|
return
|
|
}
|
|
if getBuf[5*1024*1024] != buf[0] {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
|
|
// Do not need to remove destBucketName its same as bucketName.
|
|
}
|
|
|
|
// Test Core CopyObjectPart implementation for unencrypted to unencrypted copy
|
|
func testSSES3EncryptedToUnencryptedCopyPart() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObjectPart(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
client, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Instantiate new core client object.
|
|
c := minio.Core{client}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
defer cleanupBucket(bucketName, client)
|
|
// Make a buffer with 5MB of data
|
|
buf := bytes.Repeat([]byte("abcde"), 1024*1024)
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
srcEncryption := encrypt.NewSSE()
|
|
opts := minio.PutObjectOptions{
|
|
UserMetadata: map[string]string{
|
|
"Content-Type": "binary/octet-stream",
|
|
},
|
|
ServerSideEncryption: srcEncryption,
|
|
}
|
|
uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject call failed", err)
|
|
return
|
|
}
|
|
st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcEncryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(len(buf)) {
|
|
logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
|
|
return
|
|
}
|
|
|
|
destBucketName := bucketName
|
|
destObjectName := objectName + "-dest"
|
|
|
|
uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Content of the destination object will be two copies of
|
|
// `objectName` concatenated, followed by first byte of
|
|
// `objectName`.
|
|
metadata := make(map[string]string)
|
|
header := make(http.Header)
|
|
for k, v := range header {
|
|
metadata[k] = v[0]
|
|
}
|
|
|
|
metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
|
|
|
|
// First of three parts
|
|
fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Second of three parts
|
|
sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Last of three parts
|
|
lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Complete the multipart upload
|
|
_, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Stat the object and check its length matches
|
|
objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
|
|
if objInfo.Size != (5*1024*1024)*2+1 {
|
|
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
|
|
return
|
|
}
|
|
|
|
// Now we read the data back
|
|
getOpts := minio.GetObjectOptions{}
|
|
getOpts.SetRange(0, 5*1024*1024-1)
|
|
r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf := make([]byte, 5*1024*1024)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf, buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
|
|
return
|
|
}
|
|
|
|
getOpts.SetRange(5*1024*1024, 0)
|
|
r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf = make([]byte, 5*1024*1024+1)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf[:5*1024*1024], buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
|
|
return
|
|
}
|
|
if getBuf[5*1024*1024] != buf[0] {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
|
|
// Do not need to remove destBucketName its same as bucketName.
|
|
}
|
|
|
|
// Test Core CopyObjectPart implementation for unencrypted to SSE-S3 encrypted copy
|
|
func testSSES3EncryptedToSSES3CopyObjectPart() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObjectPart(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
client, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Instantiate new core client object.
|
|
c := minio.Core{client}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
defer cleanupBucket(bucketName, client)
|
|
// Make a buffer with 5MB of data
|
|
buf := bytes.Repeat([]byte("abcde"), 1024*1024)
|
|
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
srcEncryption := encrypt.NewSSE()
|
|
opts := minio.PutObjectOptions{
|
|
UserMetadata: map[string]string{
|
|
"Content-Type": "binary/octet-stream",
|
|
},
|
|
ServerSideEncryption: srcEncryption,
|
|
}
|
|
|
|
uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject call failed", err)
|
|
return
|
|
}
|
|
st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcEncryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
if st.Size != int64(len(buf)) {
|
|
logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
|
|
return
|
|
}
|
|
|
|
destBucketName := bucketName
|
|
destObjectName := objectName + "-dest"
|
|
dstencryption := encrypt.NewSSE()
|
|
|
|
uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Content of the destination object will be two copies of
|
|
// `objectName` concatenated, followed by first byte of
|
|
// `objectName`.
|
|
metadata := make(map[string]string)
|
|
header := make(http.Header)
|
|
dstencryption.Marshal(header)
|
|
|
|
for k, v := range header {
|
|
metadata[k] = v[0]
|
|
}
|
|
|
|
metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
|
|
|
|
// First of three parts
|
|
fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Second of three parts
|
|
sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Last of three parts
|
|
lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
|
|
return
|
|
}
|
|
|
|
// Complete the multipart upload
|
|
_, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
|
|
return
|
|
}
|
|
|
|
// Stat the object and check its length matches
|
|
objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject call failed", err)
|
|
return
|
|
}
|
|
|
|
if objInfo.Size != (5*1024*1024)*2+1 {
|
|
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
|
|
return
|
|
}
|
|
|
|
// Now we read the data back
|
|
getOpts := minio.GetObjectOptions{}
|
|
getOpts.SetRange(0, 5*1024*1024-1)
|
|
r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf := make([]byte, 5*1024*1024)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf, buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
|
|
return
|
|
}
|
|
|
|
getOpts.SetRange(5*1024*1024, 0)
|
|
r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject call failed", err)
|
|
return
|
|
}
|
|
getBuf = make([]byte, 5*1024*1024+1)
|
|
_, err = readFull(r, getBuf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Read buffer failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(getBuf[:5*1024*1024], buf) {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
|
|
return
|
|
}
|
|
if getBuf[5*1024*1024] != buf[0] {
|
|
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
|
|
// Do not need to remove destBucketName its same as bucketName.
|
|
}
|
|
|
|
func testUserMetadataCopying() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObject(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// c.TraceOn(os.Stderr)
|
|
testUserMetadataCopyingWrapper(c)
|
|
}
|
|
|
|
func testUserMetadataCopyingWrapper(c *minio.Client) {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObject(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
// Make a new bucket in 'us-east-1' (source bucket).
|
|
err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
fetchMeta := func(object string) (h http.Header) {
|
|
objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
h = make(http.Header)
|
|
for k, vs := range objInfo.Metadata {
|
|
if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") {
|
|
h.Add(k, vs[0])
|
|
}
|
|
}
|
|
return h
|
|
}
|
|
|
|
// 1. create a client encrypted object to copy by uploading
|
|
const srcSize = 1024 * 1024
|
|
buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB
|
|
metadata := make(http.Header)
|
|
metadata.Set("x-amz-meta-myheader", "myvalue")
|
|
m := make(map[string]string)
|
|
m["x-amz-meta-myheader"] = "myvalue"
|
|
_, err = c.PutObject(context.Background(), bucketName, "srcObject",
|
|
bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{UserMetadata: m})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObjectWithMetadata failed", err)
|
|
return
|
|
}
|
|
if !reflect.DeepEqual(metadata, fetchMeta("srcObject")) {
|
|
logError(testName, function, args, startTime, "", "Metadata match failed", err)
|
|
return
|
|
}
|
|
|
|
// 2. create source
|
|
src := minio.CopySrcOptions{
|
|
Bucket: bucketName,
|
|
Object: "srcObject",
|
|
}
|
|
|
|
// 2.1 create destination with metadata set
|
|
dst1 := minio.CopyDestOptions{
|
|
Bucket: bucketName,
|
|
Object: "dstObject-1",
|
|
UserMetadata: map[string]string{"notmyheader": "notmyvalue"},
|
|
ReplaceMetadata: true,
|
|
}
|
|
|
|
// 3. Check that copying to an object with metadata set resets
|
|
// the headers on the copy.
|
|
args["source"] = src
|
|
args["destination"] = dst1
|
|
_, err = c.CopyObject(context.Background(), dst1, src)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObject failed", err)
|
|
return
|
|
}
|
|
|
|
expectedHeaders := make(http.Header)
|
|
expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue")
|
|
if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-1")) {
|
|
logError(testName, function, args, startTime, "", "Metadata match failed", err)
|
|
return
|
|
}
|
|
|
|
// 4. create destination with no metadata set and same source
|
|
dst2 := minio.CopyDestOptions{
|
|
Bucket: bucketName,
|
|
Object: "dstObject-2",
|
|
}
|
|
|
|
// 5. Check that copying to an object with no metadata set,
|
|
// copies metadata.
|
|
args["source"] = src
|
|
args["destination"] = dst2
|
|
_, err = c.CopyObject(context.Background(), dst2, src)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObject failed", err)
|
|
return
|
|
}
|
|
|
|
expectedHeaders = metadata
|
|
if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-2")) {
|
|
logError(testName, function, args, startTime, "", "Metadata match failed", err)
|
|
return
|
|
}
|
|
|
|
// 6. Compose a pair of sources.
|
|
dst3 := minio.CopyDestOptions{
|
|
Bucket: bucketName,
|
|
Object: "dstObject-3",
|
|
ReplaceMetadata: true,
|
|
}
|
|
|
|
function = "ComposeObject(destination, sources)"
|
|
args["source"] = []minio.CopySrcOptions{src, src}
|
|
args["destination"] = dst3
|
|
_, err = c.ComposeObject(context.Background(), dst3, src, src)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ComposeObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Check that no headers are copied in this case
|
|
if !reflect.DeepEqual(make(http.Header), fetchMeta("dstObject-3")) {
|
|
logError(testName, function, args, startTime, "", "Metadata match failed", err)
|
|
return
|
|
}
|
|
|
|
// 7. Compose a pair of sources with dest user metadata set.
|
|
dst4 := minio.CopyDestOptions{
|
|
Bucket: bucketName,
|
|
Object: "dstObject-4",
|
|
UserMetadata: map[string]string{"notmyheader": "notmyvalue"},
|
|
ReplaceMetadata: true,
|
|
}
|
|
|
|
function = "ComposeObject(destination, sources)"
|
|
args["source"] = []minio.CopySrcOptions{src, src}
|
|
args["destination"] = dst4
|
|
_, err = c.ComposeObject(context.Background(), dst4, src, src)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ComposeObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Check that no headers are copied in this case
|
|
expectedHeaders = make(http.Header)
|
|
expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue")
|
|
if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-4")) {
|
|
logError(testName, function, args, startTime, "", "Metadata match failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
func testUserMetadataCopyingV2() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "CopyObject(destination, source)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// c.TraceOn(os.Stderr)
|
|
testUserMetadataCopyingWrapper(c)
|
|
}
|
|
|
|
func testStorageClassMetadataPutObject() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
function := "testStorageClassMetadataPutObject()"
|
|
args := map[string]interface{}{}
|
|
testName := getFuncName()
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
|
|
// Make a new bucket in 'us-east-1' (source bucket).
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
fetchMeta := func(object string) (h http.Header) {
|
|
objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
h = make(http.Header)
|
|
for k, vs := range objInfo.Metadata {
|
|
if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") {
|
|
for _, v := range vs {
|
|
h.Add(k, v)
|
|
}
|
|
}
|
|
}
|
|
return h
|
|
}
|
|
|
|
metadata := make(http.Header)
|
|
metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
|
|
|
|
emptyMetadata := make(http.Header)
|
|
|
|
const srcSize = 1024 * 1024
|
|
buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, "srcObjectRRSClass",
|
|
bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Get the returned metadata
|
|
returnedMeta := fetchMeta("srcObjectRRSClass")
|
|
|
|
// The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways)
|
|
if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) {
|
|
logError(testName, function, args, startTime, "", "Metadata match failed", err)
|
|
return
|
|
}
|
|
|
|
metadata = make(http.Header)
|
|
metadata.Set("x-amz-storage-class", "STANDARD")
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, "srcObjectSSClass",
|
|
bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClass")) {
|
|
logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
func testStorageClassInvalidMetadataPutObject() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
function := "testStorageClassInvalidMetadataPutObject()"
|
|
args := map[string]interface{}{}
|
|
testName := getFuncName()
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
|
|
// Make a new bucket in 'us-east-1' (source bucket).
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
const srcSize = 1024 * 1024
|
|
buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, "srcObjectRRSClass",
|
|
bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "INVALID_STORAGE_CLASS"})
|
|
if err == nil {
|
|
logError(testName, function, args, startTime, "", "PutObject with invalid storage class passed, was expected to fail", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
func testStorageClassMetadataCopyObject() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
function := "testStorageClassMetadataCopyObject()"
|
|
args := map[string]interface{}{}
|
|
testName := getFuncName()
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
|
|
// Make a new bucket in 'us-east-1' (source bucket).
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
fetchMeta := func(object string) (h http.Header) {
|
|
objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{})
|
|
args["bucket"] = bucketName
|
|
args["object"] = object
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Stat failed", err)
|
|
return
|
|
}
|
|
h = make(http.Header)
|
|
for k, vs := range objInfo.Metadata {
|
|
if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") {
|
|
for _, v := range vs {
|
|
h.Add(k, v)
|
|
}
|
|
}
|
|
}
|
|
return h
|
|
}
|
|
|
|
metadata := make(http.Header)
|
|
metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
|
|
|
|
emptyMetadata := make(http.Header)
|
|
|
|
const srcSize = 1024 * 1024
|
|
buf := bytes.Repeat([]byte("abcde"), srcSize)
|
|
|
|
// Put an object with RRS Storage class
|
|
_, err = c.PutObject(context.Background(), bucketName, "srcObjectRRSClass",
|
|
bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Make server side copy of object uploaded in previous step
|
|
src := minio.CopySrcOptions{
|
|
Bucket: bucketName,
|
|
Object: "srcObjectRRSClass",
|
|
}
|
|
dst := minio.CopyDestOptions{
|
|
Bucket: bucketName,
|
|
Object: "srcObjectRRSClassCopy",
|
|
}
|
|
if _, err = c.CopyObject(context.Background(), dst, src); err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObject failed on RRS", err)
|
|
return
|
|
}
|
|
|
|
// Get the returned metadata
|
|
returnedMeta := fetchMeta("srcObjectRRSClassCopy")
|
|
|
|
// The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways)
|
|
if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) {
|
|
logError(testName, function, args, startTime, "", "Metadata match failed", err)
|
|
return
|
|
}
|
|
|
|
metadata = make(http.Header)
|
|
metadata.Set("x-amz-storage-class", "STANDARD")
|
|
|
|
// Put an object with Standard Storage class
|
|
_, err = c.PutObject(context.Background(), bucketName, "srcObjectSSClass",
|
|
bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Make server side copy of object uploaded in previous step
|
|
src = minio.CopySrcOptions{
|
|
Bucket: bucketName,
|
|
Object: "srcObjectSSClass",
|
|
}
|
|
dst = minio.CopyDestOptions{
|
|
Bucket: bucketName,
|
|
Object: "srcObjectSSClassCopy",
|
|
}
|
|
if _, err = c.CopyObject(context.Background(), dst, src); err != nil {
|
|
logError(testName, function, args, startTime, "", "CopyObject failed on SS", err)
|
|
return
|
|
}
|
|
// Fetch the meta data of copied object
|
|
if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClassCopy")) {
|
|
logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test put object with size -1 byte object.
|
|
func testPutObjectNoLengthV2() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "PutObject(bucketName, objectName, reader, size, opts)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"size": -1,
|
|
"opts": "",
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
objectName := bucketName + "unique"
|
|
args["objectName"] = objectName
|
|
|
|
bufSize := dataFileMap["datafile-129-MB"]
|
|
reader := getDataReader("datafile-129-MB")
|
|
defer reader.Close()
|
|
args["size"] = bufSize
|
|
|
|
// Upload an object.
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, -1, minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err)
|
|
return
|
|
}
|
|
|
|
st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(bufSize) {
|
|
logError(testName, function, args, startTime, "", "Expected upload object size "+string(bufSize)+" got "+string(st.Size), err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test put objects of unknown size.
|
|
func testPutObjectsUnknownV2() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "PutObject(bucketName, objectName, reader,size,opts)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"size": "",
|
|
"opts": "",
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Issues are revealed by trying to upload multiple files of unknown size
|
|
// sequentially (on 4GB machines)
|
|
for i := 1; i <= 4; i++ {
|
|
// Simulate that we could be receiving byte slices of data that we want
|
|
// to upload as a file
|
|
rpipe, wpipe := io.Pipe()
|
|
defer rpipe.Close()
|
|
go func() {
|
|
b := []byte("test")
|
|
wpipe.Write(b)
|
|
wpipe.Close()
|
|
}()
|
|
|
|
// Upload the object.
|
|
objectName := fmt.Sprintf("%sunique%d", bucketName, i)
|
|
args["objectName"] = objectName
|
|
|
|
ui, err := c.PutObject(context.Background(), bucketName, objectName, rpipe, -1, minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err)
|
|
return
|
|
}
|
|
|
|
if ui.Size != 4 {
|
|
logError(testName, function, args, startTime, "", "Expected upload object size "+string(4)+" got "+string(ui.Size), nil)
|
|
return
|
|
}
|
|
|
|
st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObjectStreaming failed", err)
|
|
return
|
|
}
|
|
|
|
if st.Size != int64(4) {
|
|
logError(testName, function, args, startTime, "", "Expected upload object size "+string(4)+" got "+string(st.Size), err)
|
|
return
|
|
}
|
|
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test put object with 0 byte object.
|
|
func testPutObject0ByteV2() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "PutObject(bucketName, objectName, reader, size, opts)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"size": 0,
|
|
"opts": "",
|
|
}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
objectName := bucketName + "unique"
|
|
args["objectName"] = objectName
|
|
args["opts"] = minio.PutObjectOptions{}
|
|
|
|
// Upload an object.
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err)
|
|
return
|
|
}
|
|
st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObjectWithSize failed", err)
|
|
return
|
|
}
|
|
if st.Size != 0 {
|
|
logError(testName, function, args, startTime, "", "Expected upload object size 0 but got "+string(st.Size), err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test expected error cases
|
|
func testComposeObjectErrorCases() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "ComposeObject(destination, sourceList)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
testComposeObjectErrorCasesWrapper(c)
|
|
}
|
|
|
|
// Test concatenating multiple 10K objects V4
|
|
func testCompose10KSources() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "ComposeObject(destination, sourceList)"
|
|
args := map[string]interface{}{}
|
|
|
|
// Instantiate new minio client object
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
|
|
return
|
|
}
|
|
|
|
testComposeMultipleSources(c)
|
|
}
|
|
|
|
// Tests comprehensive list of all methods.
|
|
func testFunctionalV2() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "testFunctionalV2()"
|
|
functionAll := ""
|
|
args := map[string]interface{}{}
|
|
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable to debug
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
location := "us-east-1"
|
|
// Make a new bucket.
|
|
function = "MakeBucket(bucketName, location)"
|
|
functionAll = "MakeBucket(bucketName, location)"
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"location": location,
|
|
}
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: location})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
// Generate a random file name.
|
|
fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
file, err := os.Create(fileName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "file create failed", err)
|
|
return
|
|
}
|
|
for i := 0; i < 3; i++ {
|
|
buf := make([]byte, rand.Intn(1<<19))
|
|
_, err = file.Write(buf)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "file write failed", err)
|
|
return
|
|
}
|
|
}
|
|
file.Close()
|
|
|
|
// Verify if bucket exits and you have access.
|
|
var exists bool
|
|
function = "BucketExists(bucketName)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
}
|
|
exists, err = c.BucketExists(context.Background(), bucketName)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "BucketExists failed", err)
|
|
return
|
|
}
|
|
if !exists {
|
|
logError(testName, function, args, startTime, "", "Could not find existing bucket "+bucketName, err)
|
|
return
|
|
}
|
|
|
|
// Make the bucket 'public read/write'.
|
|
function = "SetBucketPolicy(bucketName, bucketPolicy)"
|
|
functionAll += ", " + function
|
|
|
|
readWritePolicy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:ListBucketMultipartUploads", "s3:ListBucket"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::` + bucketName + `"],"Sid": ""}]}`
|
|
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"bucketPolicy": readWritePolicy,
|
|
}
|
|
err = c.SetBucketPolicy(context.Background(), bucketName, readWritePolicy)
|
|
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err)
|
|
return
|
|
}
|
|
|
|
// List all buckets.
|
|
function = "ListBuckets()"
|
|
functionAll += ", " + function
|
|
args = nil
|
|
buckets, err := c.ListBuckets(context.Background())
|
|
if len(buckets) == 0 {
|
|
logError(testName, function, args, startTime, "", "List buckets cannot be empty", err)
|
|
return
|
|
}
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ListBuckets failed", err)
|
|
return
|
|
}
|
|
|
|
// Verify if previously created bucket is listed in list buckets.
|
|
bucketFound := false
|
|
for _, bucket := range buckets {
|
|
if bucket.Name == bucketName {
|
|
bucketFound = true
|
|
}
|
|
}
|
|
|
|
// If bucket not found error out.
|
|
if !bucketFound {
|
|
logError(testName, function, args, startTime, "", "Bucket "+bucketName+"not found", err)
|
|
return
|
|
}
|
|
|
|
objectName := bucketName + "unique"
|
|
|
|
// Generate data
|
|
buf := bytes.Repeat([]byte("n"), rand.Intn(1<<19))
|
|
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName,
|
|
"contentType": "",
|
|
}
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject failed", err)
|
|
return
|
|
}
|
|
if st.Size != int64(len(buf)) {
|
|
logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(st.Size), err)
|
|
return
|
|
}
|
|
|
|
objectNameNoLength := objectName + "-nolength"
|
|
args["objectName"] = objectNameNoLength
|
|
_, err = c.PutObject(context.Background(), bucketName, objectNameNoLength, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
st, err = c.StatObject(context.Background(), bucketName, objectNameNoLength, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "StatObject failed", err)
|
|
return
|
|
}
|
|
if st.Size != int64(len(buf)) {
|
|
logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(st.Size), err)
|
|
return
|
|
}
|
|
|
|
// Instantiate a done channel to close all listing.
|
|
doneCh := make(chan struct{})
|
|
defer close(doneCh)
|
|
|
|
objFound := false
|
|
isRecursive := true // Recursive is true.
|
|
function = "ListObjects(bucketName, objectName, isRecursive, doneCh)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName,
|
|
"isRecursive": isRecursive,
|
|
}
|
|
for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Prefix: objectName, Recursive: isRecursive}) {
|
|
if obj.Key == objectName {
|
|
objFound = true
|
|
break
|
|
}
|
|
}
|
|
if !objFound {
|
|
logError(testName, function, args, startTime, "", "Could not find existing object "+objectName, err)
|
|
return
|
|
}
|
|
|
|
incompObjNotFound := true
|
|
function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName,
|
|
"isRecursive": isRecursive,
|
|
}
|
|
for objIncompl := range c.ListIncompleteUploads(context.Background(), bucketName, objectName, isRecursive) {
|
|
if objIncompl.Key != "" {
|
|
incompObjNotFound = false
|
|
break
|
|
}
|
|
}
|
|
if !incompObjNotFound {
|
|
logError(testName, function, args, startTime, "", "Unexpected dangling incomplete upload found", err)
|
|
return
|
|
}
|
|
|
|
function = "GetObject(bucketName, objectName)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName,
|
|
}
|
|
newReader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
|
|
newReadBytes, err := ioutil.ReadAll(newReader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
newReader.Close()
|
|
|
|
if !bytes.Equal(newReadBytes, buf) {
|
|
logError(testName, function, args, startTime, "", "Bytes mismatch", err)
|
|
return
|
|
}
|
|
|
|
function = "FGetObject(bucketName, objectName, fileName)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName,
|
|
"fileName": fileName + "-f",
|
|
}
|
|
err = c.FGetObject(context.Background(), bucketName, objectName, fileName+"-f", minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "FgetObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Generate presigned HEAD object url.
|
|
function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName,
|
|
"expires": 3600 * time.Second,
|
|
}
|
|
presignedHeadURL, err := c.PresignedHeadObject(context.Background(), bucketName, objectName, 3600*time.Second, nil)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err)
|
|
return
|
|
}
|
|
|
|
transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "DefaultTransport failed", err)
|
|
return
|
|
}
|
|
|
|
httpClient := &http.Client{
|
|
// Setting a sensible time out of 30secs to wait for response
|
|
// headers. Request is pro-actively canceled after 30secs
|
|
// with no response.
|
|
Timeout: 30 * time.Second,
|
|
Transport: transport,
|
|
}
|
|
|
|
req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedHeadObject URL head request failed", err)
|
|
return
|
|
}
|
|
|
|
// Verify if presigned url works.
|
|
resp, err := httpClient.Do(req)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedHeadObject URL head request failed", err)
|
|
return
|
|
}
|
|
if resp.StatusCode != http.StatusOK {
|
|
logError(testName, function, args, startTime, "", "PresignedHeadObject URL returns status "+string(resp.StatusCode), err)
|
|
return
|
|
}
|
|
if resp.Header.Get("ETag") == "" {
|
|
logError(testName, function, args, startTime, "", "Got empty ETag", err)
|
|
return
|
|
}
|
|
resp.Body.Close()
|
|
|
|
// Generate presigned GET object url.
|
|
function = "PresignedGetObject(bucketName, objectName, expires, reqParams)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName,
|
|
"expires": 3600 * time.Second,
|
|
}
|
|
presignedGetURL, err := c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, nil)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Verify if presigned url works.
|
|
req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err)
|
|
return
|
|
}
|
|
|
|
resp, err = httpClient.Do(req)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
|
|
return
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err)
|
|
return
|
|
}
|
|
newPresignedBytes, err := ioutil.ReadAll(resp.Body)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
resp.Body.Close()
|
|
if !bytes.Equal(newPresignedBytes, buf) {
|
|
logError(testName, function, args, startTime, "", "Bytes mismatch", err)
|
|
return
|
|
}
|
|
|
|
// Set request parameters.
|
|
reqParams := make(url.Values)
|
|
reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"")
|
|
// Generate presigned GET object url.
|
|
args["reqParams"] = reqParams
|
|
presignedGetURL, err = c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, reqParams)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Verify if presigned url works.
|
|
req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err)
|
|
return
|
|
}
|
|
|
|
resp, err = httpClient.Do(req)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
|
|
return
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err)
|
|
return
|
|
}
|
|
newPresignedBytes, err = ioutil.ReadAll(resp.Body)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
|
return
|
|
}
|
|
if !bytes.Equal(newPresignedBytes, buf) {
|
|
logError(testName, function, args, startTime, "", "Bytes mismatch", err)
|
|
return
|
|
}
|
|
// Verify content disposition.
|
|
if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" {
|
|
logError(testName, function, args, startTime, "", "wrong Content-Disposition received ", err)
|
|
return
|
|
}
|
|
|
|
function = "PresignedPutObject(bucketName, objectName, expires)"
|
|
functionAll += ", " + function
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName + "-presigned",
|
|
"expires": 3600 * time.Second,
|
|
}
|
|
presignedPutURL, err := c.PresignedPutObject(context.Background(), bucketName, objectName+"-presigned", 3600*time.Second)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PresignedPutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Generate data more than 32K
|
|
buf = bytes.Repeat([]byte("1"), rand.Intn(1<<10)+32*1024)
|
|
|
|
req, err = http.NewRequest(http.MethodPut, presignedPutURL.String(), bytes.NewReader(buf))
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err)
|
|
return
|
|
}
|
|
|
|
resp, err = httpClient.Do(req)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err)
|
|
return
|
|
}
|
|
|
|
// Download the uploaded object to verify
|
|
args = map[string]interface{}{
|
|
"bucketName": bucketName,
|
|
"objectName": objectName + "-presigned",
|
|
}
|
|
newReader, err = c.GetObject(context.Background(), bucketName, objectName+"-presigned", minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject of uploaded presigned object failed", err)
|
|
return
|
|
}
|
|
|
|
newReadBytes, err = ioutil.ReadAll(newReader)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "ReadAll failed during get on presigned put object", err)
|
|
return
|
|
}
|
|
newReader.Close()
|
|
|
|
if !bytes.Equal(newReadBytes, buf) {
|
|
logError(testName, function, args, startTime, "", "Bytes mismatch on presigned object upload verification", err)
|
|
return
|
|
}
|
|
|
|
function = "PresignHeader(method, bucketName, objectName, expires, reqParams, extraHeaders)"
|
|
functionAll += ", " + function
|
|
presignExtraHeaders := map[string][]string{
|
|
"mysecret": {"abcxxx"},
|
|
}
|
|
args = map[string]interface{}{
|
|
"method": "PUT",
|
|
"bucketName": bucketName,
|
|
"objectName": objectName + "-presign-custom",
|
|
"expires": 3600 * time.Second,
|
|
"extraHeaders": presignExtraHeaders,
|
|
}
|
|
_, err = c.PresignHeader(context.Background(), "PUT", bucketName, objectName+"-presign-custom", 3600*time.Second, nil, presignExtraHeaders)
|
|
if err == nil {
|
|
logError(testName, function, args, startTime, "", "Presigned with extra headers succeeded", err)
|
|
return
|
|
}
|
|
|
|
os.Remove(fileName)
|
|
os.Remove(fileName + "-f")
|
|
successLogger(testName, functionAll, args, startTime).Info()
|
|
}
|
|
|
|
// Test get object with GetObject with context
|
|
func testGetObjectContext() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "GetObject(ctx, bucketName, objectName)"
|
|
args := map[string]interface{}{
|
|
"ctx": "",
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
}
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
bufSize := dataFileMap["datafile-33-kB"]
|
|
reader := getDataReader("datafile-33-kB")
|
|
defer reader.Close()
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
|
|
args["ctx"] = ctx
|
|
cancel()
|
|
|
|
r, err := c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed unexpectedly", err)
|
|
return
|
|
}
|
|
|
|
if _, err = r.Stat(); err == nil {
|
|
logError(testName, function, args, startTime, "", "GetObject should fail on short timeout", err)
|
|
return
|
|
}
|
|
r.Close()
|
|
|
|
ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
|
|
args["ctx"] = ctx
|
|
defer cancel()
|
|
|
|
// Read the data back
|
|
r, err = c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed", err)
|
|
return
|
|
}
|
|
|
|
st, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "object Stat call failed", err)
|
|
return
|
|
}
|
|
if st.Size != int64(bufSize) {
|
|
logError(testName, function, args, startTime, "", "Number of bytes in stat does not match: want "+string(bufSize)+", got"+string(st.Size), err)
|
|
return
|
|
}
|
|
if err := r.Close(); err != nil {
|
|
logError(testName, function, args, startTime, "", "object Close() call failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test get object with FGetObject with a user provided context
|
|
func testFGetObjectContext() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "FGetObject(ctx, bucketName, objectName, fileName)"
|
|
args := map[string]interface{}{
|
|
"ctx": "",
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"fileName": "",
|
|
}
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
bufSize := dataFileMap["datafile-1-MB"]
|
|
reader := getDataReader("datafile-1-MB")
|
|
defer reader.Close()
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
|
|
args["ctx"] = ctx
|
|
defer cancel()
|
|
|
|
fileName := "tempfile-context"
|
|
args["fileName"] = fileName
|
|
// Read the data back
|
|
err = c.FGetObject(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{})
|
|
if err == nil {
|
|
logError(testName, function, args, startTime, "", "FGetObject should fail on short timeout", err)
|
|
return
|
|
}
|
|
ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
|
|
defer cancel()
|
|
|
|
// Read the data back
|
|
err = c.FGetObject(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "FGetObject with long timeout failed", err)
|
|
return
|
|
}
|
|
if err = os.Remove(fileName + "-fcontext"); err != nil {
|
|
logError(testName, function, args, startTime, "", "Remove file failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test get object with GetObject with a user provided context
|
|
func testGetObjectRanges() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "GetObject(ctx, bucketName, objectName, fileName)"
|
|
args := map[string]interface{}{
|
|
"ctx": "",
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"fileName": "",
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
defer cancel()
|
|
|
|
rng := rand.NewSource(time.Now().UnixNano())
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rng, "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
bufSize := dataFileMap["datafile-129-MB"]
|
|
reader := getDataReader("datafile-129-MB")
|
|
defer reader.Close()
|
|
// Save the data
|
|
objectName := randString(60, rng, "")
|
|
args["objectName"] = objectName
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
// Read the data back
|
|
tests := []struct {
|
|
start int64
|
|
end int64
|
|
}{
|
|
{
|
|
start: 1024,
|
|
end: 1024 + 1<<20,
|
|
},
|
|
{
|
|
start: 20e6,
|
|
end: 20e6 + 10000,
|
|
},
|
|
{
|
|
start: 40e6,
|
|
end: 40e6 + 10000,
|
|
},
|
|
{
|
|
start: 60e6,
|
|
end: 60e6 + 10000,
|
|
},
|
|
{
|
|
start: 80e6,
|
|
end: 80e6 + 10000,
|
|
},
|
|
{
|
|
start: 120e6,
|
|
end: int64(bufSize),
|
|
},
|
|
}
|
|
for _, test := range tests {
|
|
wantRC := getDataReader("datafile-129-MB")
|
|
io.CopyN(ioutil.Discard, wantRC, test.start)
|
|
want := mustCrcReader(io.LimitReader(wantRC, test.end-test.start+1))
|
|
opts := minio.GetObjectOptions{}
|
|
opts.SetRange(test.start, test.end)
|
|
args["opts"] = fmt.Sprintf("%+v", test)
|
|
obj, err := c.GetObject(ctx, bucketName, objectName, opts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "FGetObject with long timeout failed", err)
|
|
return
|
|
}
|
|
err = crcMatches(obj, want)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", fmt.Sprintf("GetObject offset %d -> %d", test.start, test.end), err)
|
|
return
|
|
}
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test get object ACLs with GetObjectACL with custom provided context
|
|
func testGetObjectACLContext() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "GetObjectACL(ctx, bucketName, objectName)"
|
|
args := map[string]interface{}{
|
|
"ctx": "",
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
}
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
bufSize := dataFileMap["datafile-1-MB"]
|
|
reader := getDataReader("datafile-1-MB")
|
|
defer reader.Close()
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
// Add meta data to add a canned acl
|
|
metaData := map[string]string{
|
|
"X-Amz-Acl": "public-read-write",
|
|
}
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName,
|
|
objectName, reader, int64(bufSize),
|
|
minio.PutObjectOptions{
|
|
ContentType: "binary/octet-stream",
|
|
UserMetadata: metaData,
|
|
})
|
|
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
args["ctx"] = ctx
|
|
defer cancel()
|
|
|
|
// Read the data back
|
|
objectInfo, getObjectACLErr := c.GetObjectACL(ctx, bucketName, objectName)
|
|
if getObjectACLErr != nil {
|
|
logError(testName, function, args, startTime, "", "GetObjectACL failed. ", getObjectACLErr)
|
|
return
|
|
}
|
|
|
|
s, ok := objectInfo.Metadata["X-Amz-Acl"]
|
|
if !ok {
|
|
logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Acl\"", nil)
|
|
return
|
|
}
|
|
|
|
if len(s) != 1 {
|
|
logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" canned acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil)
|
|
return
|
|
}
|
|
|
|
// Do a very limited testing if this is not AWS S3
|
|
if os.Getenv(serverEndpoint) != "s3.amazonaws.com" {
|
|
if s[0] != "private" {
|
|
logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" expected \"private\" but got"+fmt.Sprintf("%q", s[0]), nil)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
return
|
|
}
|
|
|
|
if s[0] != "public-read-write" {
|
|
logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" expected \"public-read-write\" but got"+fmt.Sprintf("%q", s[0]), nil)
|
|
return
|
|
}
|
|
|
|
bufSize = dataFileMap["datafile-1-MB"]
|
|
reader2 := getDataReader("datafile-1-MB")
|
|
defer reader2.Close()
|
|
// Save the data
|
|
objectName = randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
// Add meta data to add a canned acl
|
|
metaData = map[string]string{
|
|
"X-Amz-Grant-Read": "id=fooread@minio.go",
|
|
"X-Amz-Grant-Write": "id=foowrite@minio.go",
|
|
}
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, reader2, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
|
return
|
|
}
|
|
|
|
ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
|
|
args["ctx"] = ctx
|
|
defer cancel()
|
|
|
|
// Read the data back
|
|
objectInfo, getObjectACLErr = c.GetObjectACL(ctx, bucketName, objectName)
|
|
if getObjectACLErr == nil {
|
|
logError(testName, function, args, startTime, "", "GetObjectACL fail", getObjectACLErr)
|
|
return
|
|
}
|
|
|
|
if len(objectInfo.Metadata) != 3 {
|
|
logError(testName, function, args, startTime, "", "GetObjectACL fail expected \"3\" ACLs but got "+fmt.Sprintf(`"%d"`, len(objectInfo.Metadata)), nil)
|
|
return
|
|
}
|
|
|
|
s, ok = objectInfo.Metadata["X-Amz-Grant-Read"]
|
|
if !ok {
|
|
logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Read\"", nil)
|
|
return
|
|
}
|
|
|
|
if len(s) != 1 {
|
|
logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil)
|
|
return
|
|
}
|
|
|
|
if s[0] != "fooread@minio.go" {
|
|
logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"fooread@minio.go\" got "+fmt.Sprintf("%q", s), nil)
|
|
return
|
|
}
|
|
|
|
s, ok = objectInfo.Metadata["X-Amz-Grant-Write"]
|
|
if !ok {
|
|
logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Write\"", nil)
|
|
return
|
|
}
|
|
|
|
if len(s) != 1 {
|
|
logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil)
|
|
return
|
|
}
|
|
|
|
if s[0] != "foowrite@minio.go" {
|
|
logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"foowrite@minio.go\" got "+fmt.Sprintf("%q", s), nil)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test validates putObject with context to see if request cancellation is honored for V2.
|
|
func testPutObjectContextV2() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "PutObject(ctx, bucketName, objectName, reader, size, opts)"
|
|
args := map[string]interface{}{
|
|
"ctx": "",
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"size": "",
|
|
"opts": "",
|
|
}
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Make a new bucket.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
defer cleanupBucket(bucketName, c)
|
|
bufSize := dataFileMap["datatfile-33-kB"]
|
|
reader := getDataReader("datafile-33-kB")
|
|
defer reader.Close()
|
|
|
|
objectName := fmt.Sprintf("test-file-%v", rand.Uint32())
|
|
args["objectName"] = objectName
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
args["ctx"] = ctx
|
|
args["size"] = bufSize
|
|
defer cancel()
|
|
|
|
_, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject with short timeout failed", err)
|
|
return
|
|
}
|
|
|
|
ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
|
|
args["ctx"] = ctx
|
|
|
|
defer cancel()
|
|
reader = getDataReader("datafile-33-kB")
|
|
defer reader.Close()
|
|
_, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject with long timeout failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test get object with GetObject with custom context
|
|
func testGetObjectContextV2() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "GetObject(ctx, bucketName, objectName)"
|
|
args := map[string]interface{}{
|
|
"ctx": "",
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
}
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
bufSize := dataFileMap["datafile-33-kB"]
|
|
reader := getDataReader("datafile-33-kB")
|
|
defer reader.Close()
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject call failed", err)
|
|
return
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
|
|
args["ctx"] = ctx
|
|
cancel()
|
|
|
|
r, err := c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject failed unexpectedly", err)
|
|
return
|
|
}
|
|
if _, err = r.Stat(); err == nil {
|
|
logError(testName, function, args, startTime, "", "GetObject should fail on short timeout", err)
|
|
return
|
|
}
|
|
r.Close()
|
|
|
|
ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
|
|
defer cancel()
|
|
|
|
// Read the data back
|
|
r, err = c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "GetObject shouldn't fail on longer timeout", err)
|
|
return
|
|
}
|
|
|
|
st, err := r.Stat()
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "object Stat call failed", err)
|
|
return
|
|
}
|
|
if st.Size != int64(bufSize) {
|
|
logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(bufSize)+" got "+string(st.Size), err)
|
|
return
|
|
}
|
|
if err := r.Close(); err != nil {
|
|
logError(testName, function, args, startTime, "", " object Close() call failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test get object with FGetObject with custom context
|
|
func testFGetObjectContextV2() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "FGetObject(ctx, bucketName, objectName,fileName)"
|
|
args := map[string]interface{}{
|
|
"ctx": "",
|
|
"bucketName": "",
|
|
"objectName": "",
|
|
"fileName": "",
|
|
}
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket call failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
bufSize := dataFileMap["datatfile-1-MB"]
|
|
reader := getDataReader("datafile-1-MB")
|
|
defer reader.Close()
|
|
// Save the data
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "PutObject call failed", err)
|
|
return
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
|
|
args["ctx"] = ctx
|
|
defer cancel()
|
|
|
|
fileName := "tempfile-context"
|
|
args["fileName"] = fileName
|
|
|
|
// Read the data back
|
|
err = c.FGetObject(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{})
|
|
if err == nil {
|
|
logError(testName, function, args, startTime, "", "FGetObject should fail on short timeout", err)
|
|
return
|
|
}
|
|
ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
|
|
defer cancel()
|
|
|
|
// Read the data back
|
|
err = c.FGetObject(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "FGetObject call shouldn't fail on long timeout", err)
|
|
return
|
|
}
|
|
|
|
if err = os.Remove(fileName + "-fcontext"); err != nil {
|
|
logError(testName, function, args, startTime, "", "Remove file failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test list object v1 and V2
|
|
func testListObjects() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "ListObjects(bucketName, objectPrefix, recursive, doneCh)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectPrefix": "",
|
|
"recursive": "true",
|
|
}
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
defer cleanupBucket(bucketName, c)
|
|
|
|
testObjects := []struct {
|
|
name string
|
|
storageClass string
|
|
}{
|
|
// Special characters
|
|
{"foo bar", "STANDARD"},
|
|
{"foo-%", "STANDARD"},
|
|
{"random-object-1", "STANDARD"},
|
|
{"random-object-2", "REDUCED_REDUNDANCY"},
|
|
}
|
|
|
|
for i, object := range testObjects {
|
|
bufSize := dataFileMap["datafile-33-kB"]
|
|
reader := getDataReader("datafile-33-kB")
|
|
defer reader.Close()
|
|
_, err = c.PutObject(context.Background(), bucketName, object.name, reader, int64(bufSize),
|
|
minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: object.storageClass})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", fmt.Sprintf("PutObject %d call failed", i+1), err)
|
|
return
|
|
}
|
|
}
|
|
|
|
testList := func(listFn func(context.Context, string, minio.ListObjectsOptions) <-chan minio.ObjectInfo, bucket string, opts minio.ListObjectsOptions) {
|
|
var objCursor int
|
|
|
|
// check for object name and storage-class from listing object result
|
|
for objInfo := range listFn(context.Background(), bucket, opts) {
|
|
if objInfo.Err != nil {
|
|
logError(testName, function, args, startTime, "", "ListObjects failed unexpectedly", err)
|
|
return
|
|
}
|
|
if objInfo.Key != testObjects[objCursor].name {
|
|
logError(testName, function, args, startTime, "", "ListObjects does not return expected object name", err)
|
|
return
|
|
}
|
|
if objInfo.StorageClass != testObjects[objCursor].storageClass {
|
|
// Ignored as Gateways (Azure/GCS etc) wont return storage class
|
|
ignoredLog(testName, function, args, startTime, "ListObjects doesn't return expected storage class").Info()
|
|
}
|
|
objCursor++
|
|
}
|
|
|
|
if objCursor != len(testObjects) {
|
|
logError(testName, function, args, startTime, "", "ListObjects returned unexpected number of items", errors.New(""))
|
|
return
|
|
}
|
|
}
|
|
|
|
testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true, UseV1: true})
|
|
testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true})
|
|
testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true, WithMetadata: true})
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Test deleting multiple objects with object retention set in Governance mode
|
|
func testRemoveObjects() {
|
|
// initialize logging params
|
|
startTime := time.Now()
|
|
testName := getFuncName()
|
|
function := "RemoveObjects(bucketName, objectsCh, opts)"
|
|
args := map[string]interface{}{
|
|
"bucketName": "",
|
|
"objectPrefix": "",
|
|
"recursive": "true",
|
|
}
|
|
// Seed random based on current time.
|
|
rand.Seed(time.Now().Unix())
|
|
|
|
// Instantiate new minio client object.
|
|
c, err := minio.New(os.Getenv(serverEndpoint),
|
|
&minio.Options{
|
|
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
|
|
Secure: mustParseBool(os.Getenv(enableHTTPS)),
|
|
})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err)
|
|
return
|
|
}
|
|
|
|
// Enable tracing, write to stderr.
|
|
// c.TraceOn(os.Stderr)
|
|
|
|
// Set user agent.
|
|
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
|
|
|
// Generate a new random bucket name.
|
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
|
args["bucketName"] = bucketName
|
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
|
args["objectName"] = objectName
|
|
|
|
// Make a new bucket.
|
|
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
|
return
|
|
}
|
|
|
|
bufSize := dataFileMap["datafile-129-MB"]
|
|
reader := getDataReader("datafile-129-MB")
|
|
defer reader.Close()
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Error uploading object", err)
|
|
return
|
|
}
|
|
|
|
// Replace with smaller...
|
|
bufSize = dataFileMap["datafile-10-kB"]
|
|
reader = getDataReader("datafile-10-kB")
|
|
defer reader.Close()
|
|
|
|
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Error uploading object", err)
|
|
}
|
|
|
|
t := time.Date(2030, time.April, 25, 14, 0, 0, 0, time.UTC)
|
|
m := minio.RetentionMode(minio.Governance)
|
|
opts := minio.PutObjectRetentionOptions{
|
|
GovernanceBypass: false,
|
|
RetainUntilDate: &t,
|
|
Mode: &m,
|
|
}
|
|
err = c.PutObjectRetention(context.Background(), bucketName, objectName, opts)
|
|
if err != nil {
|
|
logError(testName, function, args, startTime, "", "Error setting retention", err)
|
|
return
|
|
}
|
|
|
|
objectsCh := make(chan minio.ObjectInfo)
|
|
// Send object names that are needed to be removed to objectsCh
|
|
go func() {
|
|
defer close(objectsCh)
|
|
// List all objects from a bucket-name with a matching prefix.
|
|
for object := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Recursive: true}) {
|
|
if object.Err != nil {
|
|
logError(testName, function, args, startTime, "", "Error listing objects", object.Err)
|
|
return
|
|
}
|
|
objectsCh <- object
|
|
}
|
|
}()
|
|
|
|
for rErr := range c.RemoveObjects(context.Background(), bucketName, objectsCh, minio.RemoveObjectsOptions{}) {
|
|
// Error is expected here because Retention is set on the object
|
|
// and RemoveObjects is called without Bypass Governance
|
|
if rErr.Err == nil {
|
|
logError(testName, function, args, startTime, "", "Expected error during deletion", nil)
|
|
return
|
|
}
|
|
}
|
|
|
|
objectsCh1 := make(chan minio.ObjectInfo)
|
|
|
|
// Send object names that are needed to be removed to objectsCh
|
|
go func() {
|
|
defer close(objectsCh1)
|
|
// List all objects from a bucket-name with a matching prefix.
|
|
for object := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Recursive: true}) {
|
|
if object.Err != nil {
|
|
logError(testName, function, args, startTime, "", "Error listing objects", object.Err)
|
|
return
|
|
}
|
|
objectsCh1 <- object
|
|
}
|
|
}()
|
|
|
|
opts1 := minio.RemoveObjectsOptions{
|
|
GovernanceBypass: true,
|
|
}
|
|
|
|
for rErr := range c.RemoveObjects(context.Background(), bucketName, objectsCh1, opts1) {
|
|
// Error is not expected here because Retention is set on the object
|
|
// and RemoveObjects is called with Bypass Governance
|
|
logError(testName, function, args, startTime, "", "Error detected during deletion", rErr.Err)
|
|
return
|
|
}
|
|
|
|
// Delete all objects and buckets
|
|
if err = cleanupVersionedBucket(bucketName, c); err != nil {
|
|
logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
|
|
return
|
|
}
|
|
|
|
successLogger(testName, function, args, startTime).Info()
|
|
}
|
|
|
|
// Convert string to bool and always return false if any error
|
|
func mustParseBool(str string) bool {
|
|
b, err := strconv.ParseBool(str)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
return b
|
|
}
|
|
|
|
func main() {
|
|
// Output to stdout instead of the default stderr
|
|
log.SetOutput(os.Stdout)
|
|
// create custom formatter
|
|
mintFormatter := mintJSONFormatter{}
|
|
// set custom formatter
|
|
log.SetFormatter(&mintFormatter)
|
|
// log Info or above -- success cases are Info level, failures are Fatal level
|
|
log.SetLevel(log.InfoLevel)
|
|
|
|
tls := mustParseBool(os.Getenv(enableHTTPS))
|
|
kms := mustParseBool(os.Getenv(enableKMS))
|
|
if os.Getenv(enableKMS) == "" {
|
|
// Default to KMS tests.
|
|
kms = true
|
|
}
|
|
|
|
// execute tests
|
|
if isFullMode() {
|
|
testMakeBucketErrorV2()
|
|
testGetObjectClosedTwiceV2()
|
|
testFPutObjectV2()
|
|
testMakeBucketRegionsV2()
|
|
testGetObjectReadSeekFunctionalV2()
|
|
testGetObjectReadAtFunctionalV2()
|
|
testGetObjectRanges()
|
|
testCopyObjectV2()
|
|
testFunctionalV2()
|
|
testComposeObjectErrorCasesV2()
|
|
testCompose10KSourcesV2()
|
|
testUserMetadataCopyingV2()
|
|
testPutObject0ByteV2()
|
|
testPutObjectNoLengthV2()
|
|
testPutObjectsUnknownV2()
|
|
testGetObjectContextV2()
|
|
testFPutObjectContextV2()
|
|
testFGetObjectContextV2()
|
|
testPutObjectContextV2()
|
|
testPutObjectWithVersioning()
|
|
testMakeBucketError()
|
|
testMakeBucketRegions()
|
|
testPutObjectWithMetadata()
|
|
testPutObjectReadAt()
|
|
testPutObjectStreaming()
|
|
testGetObjectSeekEnd()
|
|
testGetObjectClosedTwice()
|
|
testGetObjectS3Zip()
|
|
testRemoveMultipleObjects()
|
|
testRemoveMultipleObjectsWithResult()
|
|
testFPutObjectMultipart()
|
|
testFPutObject()
|
|
testGetObjectReadSeekFunctional()
|
|
testGetObjectReadAtFunctional()
|
|
testGetObjectReadAtWhenEOFWasReached()
|
|
testPresignedPostPolicy()
|
|
testCopyObject()
|
|
testComposeObjectErrorCases()
|
|
testCompose10KSources()
|
|
testUserMetadataCopying()
|
|
testBucketNotification()
|
|
testFunctional()
|
|
testGetObjectModified()
|
|
testPutObjectUploadSeekedObject()
|
|
testGetObjectContext()
|
|
testFPutObjectContext()
|
|
testFGetObjectContext()
|
|
testGetObjectACLContext()
|
|
testPutObjectContext()
|
|
testStorageClassMetadataPutObject()
|
|
testStorageClassInvalidMetadataPutObject()
|
|
testStorageClassMetadataCopyObject()
|
|
testPutObjectWithContentLanguage()
|
|
testListObjects()
|
|
testRemoveObjects()
|
|
testListObjectVersions()
|
|
testStatObjectWithVersioning()
|
|
testGetObjectWithVersioning()
|
|
testCopyObjectWithVersioning()
|
|
testConcurrentCopyObjectWithVersioning()
|
|
testComposeObjectWithVersioning()
|
|
testRemoveObjectWithVersioning()
|
|
testRemoveObjectsWithVersioning()
|
|
testObjectTaggingWithVersioning()
|
|
|
|
// SSE-C tests will only work over TLS connection.
|
|
if tls {
|
|
testSSECEncryptionPutGet()
|
|
testSSECEncryptionFPut()
|
|
testSSECEncryptedGetObjectReadAtFunctional()
|
|
testSSECEncryptedGetObjectReadSeekFunctional()
|
|
testEncryptedCopyObjectV2()
|
|
testEncryptedSSECToSSECCopyObject()
|
|
testEncryptedSSECToUnencryptedCopyObject()
|
|
testUnencryptedToSSECCopyObject()
|
|
testUnencryptedToUnencryptedCopyObject()
|
|
testEncryptedEmptyObject()
|
|
testDecryptedCopyObject()
|
|
testSSECEncryptedToSSECCopyObjectPart()
|
|
testSSECMultipartEncryptedToSSECCopyObjectPart()
|
|
testSSECEncryptedToUnencryptedCopyPart()
|
|
testUnencryptedToSSECCopyObjectPart()
|
|
testUnencryptedToUnencryptedCopyPart()
|
|
testEncryptedSSECToSSES3CopyObject()
|
|
testEncryptedSSES3ToSSECCopyObject()
|
|
testSSECEncryptedToSSES3CopyObjectPart()
|
|
testSSES3EncryptedToSSECCopyObjectPart()
|
|
}
|
|
|
|
// KMS tests
|
|
if kms {
|
|
testSSES3EncryptionPutGet()
|
|
testSSES3EncryptionFPut()
|
|
testSSES3EncryptedGetObjectReadAtFunctional()
|
|
testSSES3EncryptedGetObjectReadSeekFunctional()
|
|
testEncryptedSSES3ToSSES3CopyObject()
|
|
testEncryptedSSES3ToUnencryptedCopyObject()
|
|
testUnencryptedToSSES3CopyObject()
|
|
testUnencryptedToSSES3CopyObjectPart()
|
|
testSSES3EncryptedToUnencryptedCopyPart()
|
|
testSSES3EncryptedToSSES3CopyObjectPart()
|
|
}
|
|
} else {
|
|
testFunctional()
|
|
testFunctionalV2()
|
|
}
|
|
}
|