mirror of
https://github.com/superseriousbusiness/gotosocial.git
synced 2024-12-05 01:52:46 +00:00
468 lines
11 KiB
Go
468 lines
11 KiB
Go
|
package disk
|
||
|
|
||
|
import (
|
||
|
"bytes"
|
||
|
"context"
|
||
|
"errors"
|
||
|
"io"
|
||
|
"io/fs"
|
||
|
"os"
|
||
|
"path"
|
||
|
"strings"
|
||
|
"syscall"
|
||
|
|
||
|
"codeberg.org/gruf/go-fastcopy"
|
||
|
"codeberg.org/gruf/go-fastpath/v2"
|
||
|
"codeberg.org/gruf/go-storage"
|
||
|
"codeberg.org/gruf/go-storage/internal"
|
||
|
)
|
||
|
|
||
|
// ensure DiskStorage conforms to storage.Storage.
|
||
|
var _ storage.Storage = (*DiskStorage)(nil)
|
||
|
|
||
|
// DefaultConfig returns the default DiskStorage configuration.
|
||
|
func DefaultConfig() Config {
|
||
|
return defaultConfig
|
||
|
}
|
||
|
|
||
|
// immutable default configuration.
|
||
|
var defaultConfig = Config{
|
||
|
OpenRead: OpenArgs{syscall.O_RDONLY, 0o644},
|
||
|
OpenWrite: OpenArgs{syscall.O_CREAT | syscall.O_WRONLY, 0o644},
|
||
|
MkdirPerms: 0o755,
|
||
|
WriteBufSize: 4096,
|
||
|
}
|
||
|
|
||
|
// OpenArgs defines args passed
|
||
|
// in a syscall.Open() operation.
|
||
|
type OpenArgs struct {
|
||
|
Flags int
|
||
|
Perms uint32
|
||
|
}
|
||
|
|
||
|
// Config defines options to be
|
||
|
// used when opening a DiskStorage.
|
||
|
type Config struct {
|
||
|
|
||
|
// OpenRead are the arguments passed
|
||
|
// to syscall.Open() when opening a
|
||
|
// file for read operations.
|
||
|
OpenRead OpenArgs
|
||
|
|
||
|
// OpenWrite are the arguments passed
|
||
|
// to syscall.Open() when opening a
|
||
|
// file for write operations.
|
||
|
OpenWrite OpenArgs
|
||
|
|
||
|
// MkdirPerms are the permissions used
|
||
|
// when creating necessary sub-dirs in
|
||
|
// a storage key with slashes.
|
||
|
MkdirPerms uint32
|
||
|
|
||
|
// WriteBufSize is the buffer size
|
||
|
// to use when writing file streams.
|
||
|
WriteBufSize int
|
||
|
}
|
||
|
|
||
|
// getDiskConfig returns valid (and owned!) Config for given ptr.
|
||
|
func getDiskConfig(cfg *Config) Config {
|
||
|
if cfg == nil {
|
||
|
// use defaults.
|
||
|
return defaultConfig
|
||
|
}
|
||
|
|
||
|
// Ensure non-zero syscall args.
|
||
|
if cfg.OpenRead.Flags == 0 {
|
||
|
cfg.OpenRead.Flags = defaultConfig.OpenRead.Flags
|
||
|
}
|
||
|
if cfg.OpenRead.Perms == 0 {
|
||
|
cfg.OpenRead.Perms = defaultConfig.OpenRead.Perms
|
||
|
}
|
||
|
if cfg.OpenWrite.Flags == 0 {
|
||
|
cfg.OpenWrite.Flags = defaultConfig.OpenWrite.Flags
|
||
|
}
|
||
|
if cfg.OpenWrite.Perms == 0 {
|
||
|
cfg.OpenWrite.Perms = defaultConfig.OpenWrite.Perms
|
||
|
}
|
||
|
if cfg.MkdirPerms == 0 {
|
||
|
cfg.MkdirPerms = defaultConfig.MkdirPerms
|
||
|
}
|
||
|
|
||
|
// Ensure valid write buf.
|
||
|
if cfg.WriteBufSize <= 0 {
|
||
|
cfg.WriteBufSize = defaultConfig.WriteBufSize
|
||
|
}
|
||
|
|
||
|
return Config{
|
||
|
OpenRead: cfg.OpenRead,
|
||
|
OpenWrite: cfg.OpenWrite,
|
||
|
MkdirPerms: cfg.MkdirPerms,
|
||
|
WriteBufSize: cfg.WriteBufSize,
|
||
|
}
|
||
|
}
|
||
|
|
||
|
// DiskStorage is a Storage implementation
|
||
|
// that stores directly to a filesystem.
|
||
|
type DiskStorage struct {
|
||
|
path string // path is the root path of this store
|
||
|
pool fastcopy.CopyPool // pool is the prepared io copier with buffer pool
|
||
|
cfg Config // cfg is the supplied configuration for this store
|
||
|
}
|
||
|
|
||
|
// Open opens a DiskStorage instance for given folder path and configuration.
|
||
|
func Open(path string, cfg *Config) (*DiskStorage, error) {
|
||
|
// Check + set config defaults.
|
||
|
config := getDiskConfig(cfg)
|
||
|
|
||
|
// Clean provided storage path, ensure
|
||
|
// final '/' to help with path trimming.
|
||
|
pb := internal.GetPathBuilder()
|
||
|
path = pb.Clean(path) + "/"
|
||
|
internal.PutPathBuilder(pb)
|
||
|
|
||
|
// Ensure directories up-to path exist.
|
||
|
perms := fs.FileMode(config.MkdirPerms)
|
||
|
err := os.MkdirAll(path, perms)
|
||
|
if err != nil {
|
||
|
return nil, err
|
||
|
}
|
||
|
|
||
|
// Prepare DiskStorage.
|
||
|
st := &DiskStorage{
|
||
|
path: path,
|
||
|
cfg: config,
|
||
|
}
|
||
|
|
||
|
// Set fastcopy pool buffer size.
|
||
|
st.pool.Buffer(config.WriteBufSize)
|
||
|
|
||
|
return st, nil
|
||
|
}
|
||
|
|
||
|
// Clean: implements Storage.Clean().
|
||
|
func (st *DiskStorage) Clean(ctx context.Context) error {
|
||
|
// Check context still valid.
|
||
|
if err := ctx.Err(); err != nil {
|
||
|
return err
|
||
|
}
|
||
|
|
||
|
// Clean unused directories.
|
||
|
return cleanDirs(st.path, OpenArgs{
|
||
|
Flags: syscall.O_RDONLY,
|
||
|
})
|
||
|
}
|
||
|
|
||
|
// ReadBytes: implements Storage.ReadBytes().
|
||
|
func (st *DiskStorage) ReadBytes(ctx context.Context, key string) ([]byte, error) {
|
||
|
// Get stream reader for key
|
||
|
rc, err := st.ReadStream(ctx, key)
|
||
|
if err != nil {
|
||
|
return nil, err
|
||
|
}
|
||
|
|
||
|
// Read all data to memory.
|
||
|
data, err := io.ReadAll(rc)
|
||
|
if err != nil {
|
||
|
_ = rc.Close()
|
||
|
return nil, err
|
||
|
}
|
||
|
|
||
|
// Close storage stream reader.
|
||
|
if err := rc.Close(); err != nil {
|
||
|
return nil, err
|
||
|
}
|
||
|
|
||
|
return data, nil
|
||
|
}
|
||
|
|
||
|
// ReadStream: implements Storage.ReadStream().
|
||
|
func (st *DiskStorage) ReadStream(ctx context.Context, key string) (io.ReadCloser, error) {
|
||
|
// Generate file path for key.
|
||
|
kpath, err := st.Filepath(key)
|
||
|
if err != nil {
|
||
|
return nil, err
|
||
|
}
|
||
|
|
||
|
// Check context still valid.
|
||
|
if err := ctx.Err(); err != nil {
|
||
|
return nil, err
|
||
|
}
|
||
|
|
||
|
// Attempt to open file with read args.
|
||
|
file, err := open(kpath, st.cfg.OpenRead)
|
||
|
if err != nil {
|
||
|
|
||
|
if err == syscall.ENOENT {
|
||
|
// Translate not-found errors and wrap with key.
|
||
|
err = internal.ErrWithKey(storage.ErrNotFound, key)
|
||
|
}
|
||
|
|
||
|
return nil, err
|
||
|
}
|
||
|
|
||
|
return file, nil
|
||
|
}
|
||
|
|
||
|
// WriteBytes: implements Storage.WriteBytes().
|
||
|
func (st *DiskStorage) WriteBytes(ctx context.Context, key string, value []byte) (int, error) {
|
||
|
n, err := st.WriteStream(ctx, key, bytes.NewReader(value))
|
||
|
return int(n), err
|
||
|
}
|
||
|
|
||
|
// WriteStream: implements Storage.WriteStream().
|
||
|
func (st *DiskStorage) WriteStream(ctx context.Context, key string, stream io.Reader) (int64, error) {
|
||
|
// Acquire path builder buffer.
|
||
|
pb := internal.GetPathBuilder()
|
||
|
|
||
|
// Generate the file path for given key.
|
||
|
kpath, subdir, err := st.filepath(pb, key)
|
||
|
if err != nil {
|
||
|
return 0, err
|
||
|
}
|
||
|
|
||
|
// Done with path buffer.
|
||
|
internal.PutPathBuilder(pb)
|
||
|
|
||
|
// Check context still valid.
|
||
|
if err := ctx.Err(); err != nil {
|
||
|
return 0, err
|
||
|
}
|
||
|
|
||
|
if subdir {
|
||
|
// Get dir of key path.
|
||
|
dir := path.Dir(kpath)
|
||
|
|
||
|
// Note that subdir will only be set if
|
||
|
// the transformed key (without base path)
|
||
|
// contains any slashes. This is not a
|
||
|
// definitive check, but it allows us to
|
||
|
// skip a syscall if mkdirall not needed!
|
||
|
perms := fs.FileMode(st.cfg.MkdirPerms)
|
||
|
err = os.MkdirAll(dir, perms)
|
||
|
if err != nil {
|
||
|
return 0, err
|
||
|
}
|
||
|
}
|
||
|
|
||
|
// Attempt to open file with write args.
|
||
|
file, err := open(kpath, st.cfg.OpenWrite)
|
||
|
if err != nil {
|
||
|
|
||
|
if st.cfg.OpenWrite.Flags&syscall.O_EXCL != 0 &&
|
||
|
err == syscall.EEXIST {
|
||
|
// Translate already exists errors and wrap with key.
|
||
|
err = internal.ErrWithKey(storage.ErrAlreadyExists, key)
|
||
|
}
|
||
|
|
||
|
return 0, err
|
||
|
}
|
||
|
|
||
|
// Copy provided stream to file interface.
|
||
|
n, err := st.pool.Copy(file, stream)
|
||
|
if err != nil {
|
||
|
_ = file.Close()
|
||
|
return n, err
|
||
|
}
|
||
|
|
||
|
// Finally, close file.
|
||
|
return n, file.Close()
|
||
|
}
|
||
|
|
||
|
// Stat implements Storage.Stat().
|
||
|
func (st *DiskStorage) Stat(ctx context.Context, key string) (*storage.Entry, error) {
|
||
|
// Generate file path for key.
|
||
|
kpath, err := st.Filepath(key)
|
||
|
if err != nil {
|
||
|
return nil, err
|
||
|
}
|
||
|
|
||
|
// Check context still valid.
|
||
|
if err := ctx.Err(); err != nil {
|
||
|
return nil, err
|
||
|
}
|
||
|
|
||
|
// Stat file on disk.
|
||
|
stat, err := stat(kpath)
|
||
|
if stat == nil {
|
||
|
return nil, err
|
||
|
}
|
||
|
|
||
|
return &storage.Entry{
|
||
|
Key: key,
|
||
|
Size: stat.Size,
|
||
|
}, nil
|
||
|
}
|
||
|
|
||
|
// Remove implements Storage.Remove().
|
||
|
func (st *DiskStorage) Remove(ctx context.Context, key string) error {
|
||
|
// Generate file path for key.
|
||
|
kpath, err := st.Filepath(key)
|
||
|
if err != nil {
|
||
|
return err
|
||
|
}
|
||
|
|
||
|
// Check context still valid.
|
||
|
if err := ctx.Err(); err != nil {
|
||
|
return err
|
||
|
}
|
||
|
|
||
|
// Stat file on disk.
|
||
|
stat, err := stat(kpath)
|
||
|
if err != nil {
|
||
|
return err
|
||
|
}
|
||
|
|
||
|
// Not-found (or handled
|
||
|
// as) error situations.
|
||
|
if stat == nil {
|
||
|
return internal.ErrWithKey(storage.ErrNotFound, key)
|
||
|
} else if stat.Mode&syscall.S_IFREG == 0 {
|
||
|
err := errors.New("storage/disk: not a regular file")
|
||
|
return internal.ErrWithKey(err, key)
|
||
|
}
|
||
|
|
||
|
// Remove at path (we know this is file).
|
||
|
if err := unlink(kpath); err != nil {
|
||
|
|
||
|
if err == syscall.ENOENT {
|
||
|
// Translate not-found errors and wrap with key.
|
||
|
err = internal.ErrWithKey(storage.ErrNotFound, key)
|
||
|
}
|
||
|
|
||
|
return err
|
||
|
}
|
||
|
|
||
|
return nil
|
||
|
}
|
||
|
|
||
|
// WalkKeys implements Storage.WalkKeys().
|
||
|
func (st *DiskStorage) WalkKeys(ctx context.Context, opts storage.WalkKeysOpts) error {
|
||
|
if opts.Step == nil {
|
||
|
panic("nil step fn")
|
||
|
}
|
||
|
|
||
|
// Check context still valid.
|
||
|
if err := ctx.Err(); err != nil {
|
||
|
return err
|
||
|
}
|
||
|
|
||
|
// Acquire path builder for walk.
|
||
|
pb := internal.GetPathBuilder()
|
||
|
defer internal.PutPathBuilder(pb)
|
||
|
|
||
|
// Dir to walk.
|
||
|
dir := st.path
|
||
|
|
||
|
if opts.Prefix != "" {
|
||
|
// Convert key prefix to one of our storage filepaths.
|
||
|
pathprefix, subdir, err := st.filepath(pb, opts.Prefix)
|
||
|
if err != nil {
|
||
|
return internal.ErrWithMsg(err, "prefix error")
|
||
|
}
|
||
|
|
||
|
if subdir {
|
||
|
// Note that subdir will only be set if
|
||
|
// the transformed key (without base path)
|
||
|
// contains any slashes. This is not a
|
||
|
// definitive check, but it allows us to
|
||
|
// update the directory we walk in case
|
||
|
// it might narrow search parameters!
|
||
|
dir = path.Dir(pathprefix)
|
||
|
}
|
||
|
|
||
|
// Set updated storage
|
||
|
// path prefix in opts.
|
||
|
opts.Prefix = pathprefix
|
||
|
}
|
||
|
|
||
|
// Only need to open dirs as read-only.
|
||
|
args := OpenArgs{Flags: syscall.O_RDONLY}
|
||
|
|
||
|
return walkDir(pb, dir, args, func(kpath string, fsentry fs.DirEntry) error {
|
||
|
if !fsentry.Type().IsRegular() {
|
||
|
// Ignore anything but
|
||
|
// regular file types.
|
||
|
return nil
|
||
|
}
|
||
|
|
||
|
// Get full item path (without root).
|
||
|
kpath = pb.Join(kpath, fsentry.Name())
|
||
|
|
||
|
// Perform a fast filter check against storage path prefix (if set).
|
||
|
if opts.Prefix != "" && !strings.HasPrefix(kpath, opts.Prefix) {
|
||
|
return nil // ignore
|
||
|
}
|
||
|
|
||
|
// Storage key without base.
|
||
|
key := kpath[len(st.path):]
|
||
|
|
||
|
// Ignore filtered keys.
|
||
|
if opts.Filter != nil &&
|
||
|
!opts.Filter(key) {
|
||
|
return nil // ignore
|
||
|
}
|
||
|
|
||
|
// Load file info. This should already
|
||
|
// be loaded due to the underlying call
|
||
|
// to os.File{}.ReadDir() populating them.
|
||
|
info, err := fsentry.Info()
|
||
|
if err != nil {
|
||
|
return err
|
||
|
}
|
||
|
|
||
|
// Perform provided walk function
|
||
|
return opts.Step(storage.Entry{
|
||
|
Key: key,
|
||
|
Size: info.Size(),
|
||
|
})
|
||
|
})
|
||
|
}
|
||
|
|
||
|
// Filepath checks and returns a formatted Filepath for given key.
|
||
|
func (st *DiskStorage) Filepath(key string) (path string, err error) {
|
||
|
pb := internal.GetPathBuilder()
|
||
|
path, _, err = st.filepath(pb, key)
|
||
|
internal.PutPathBuilder(pb)
|
||
|
return
|
||
|
}
|
||
|
|
||
|
// filepath performs the "meat" of Filepath(), returning also if path *may* be a subdir of base.
|
||
|
func (st *DiskStorage) filepath(pb *fastpath.Builder, key string) (path string, subdir bool, err error) {
|
||
|
// Fast check for whether this may be a
|
||
|
// sub-directory. This is not a definitive
|
||
|
// check, it's only for a fastpath check.
|
||
|
subdir = strings.ContainsRune(key, '/')
|
||
|
|
||
|
// Build from base.
|
||
|
pb.Append(st.path)
|
||
|
pb.Append(key)
|
||
|
|
||
|
// Take COPY of bytes.
|
||
|
path = string(pb.B)
|
||
|
|
||
|
// Check for dir traversal outside base.
|
||
|
if isDirTraversal(st.path, path) {
|
||
|
err = internal.ErrWithKey(storage.ErrInvalidKey, key)
|
||
|
}
|
||
|
|
||
|
return
|
||
|
}
|
||
|
|
||
|
// isDirTraversal will check if rootPlusPath is a dir traversal outside of root,
|
||
|
// assuming that both are cleaned and that rootPlusPath is path.Join(root, somePath).
|
||
|
func isDirTraversal(root, rootPlusPath string) bool {
|
||
|
switch {
|
||
|
// Root is $PWD, check for traversal out of
|
||
|
case root == ".":
|
||
|
return strings.HasPrefix(rootPlusPath, "../")
|
||
|
|
||
|
// The path MUST be prefixed by root
|
||
|
case !strings.HasPrefix(rootPlusPath, root):
|
||
|
return true
|
||
|
|
||
|
// In all other cases, check not equal
|
||
|
default:
|
||
|
return len(root) == len(rootPlusPath)
|
||
|
}
|
||
|
}
|