update go-store to latest

This commit is contained in:
tsmethurst 2022-01-29 12:15:51 +01:00
parent 9aa364f1eb
commit 4e74c84148
8 changed files with 70 additions and 52 deletions

2
go.mod
View file

@ -4,7 +4,7 @@ go 1.17
require (
codeberg.org/gruf/go-runners v1.2.0
codeberg.org/gruf/go-store v1.3.2
codeberg.org/gruf/go-store v1.3.3
github.com/ReneKroon/ttlcache v1.7.0
github.com/buckket/go-blurhash v1.1.0
github.com/coreos/go-oidc/v3 v3.1.0

2
go.sum
View file

@ -71,6 +71,8 @@ codeberg.org/gruf/go-runners v1.2.0 h1:tkoPrwYMkVg1o/C4PGTR1YbC11XX4r06uLPOYajBs
codeberg.org/gruf/go-runners v1.2.0/go.mod h1:9gTrmMnO3d+50C+hVzcmGBf+zTuswReS278E2EMvnmw=
codeberg.org/gruf/go-store v1.3.2 h1:cLTMEqyK0uF/bt1ULkRR4h41Pdgxwvw3uxSpLUublHo=
codeberg.org/gruf/go-store v1.3.2/go.mod h1:g4+9h3wbwZ6IW0uhpw57xywcqiy4CIj0zQLqqtjEU1M=
codeberg.org/gruf/go-store v1.3.3 h1:fAP9FXy6HiLPxdD7cmpSzyfKXmVvZLjqn0m7HhxVT5M=
codeberg.org/gruf/go-store v1.3.3/go.mod h1:g4+9h3wbwZ6IW0uhpw57xywcqiy4CIj0zQLqqtjEU1M=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=

View file

@ -140,7 +140,7 @@ func OpenBlock(path string, cfg *BlockConfig) (*BlockStorage, error) {
}
// Open and acquire storage lock for path
lock, err := OpenLock(pb.Join(path, lockFile))
lock, err := OpenLock(pb.Join(path, LockFile))
if err != nil {
return nil, err
}

View file

@ -5,6 +5,8 @@
"io/fs"
"os"
"path"
_path "path"
"strings"
"syscall"
"codeberg.org/gruf/go-bytes"
@ -31,6 +33,11 @@ type DiskConfig struct {
// Overwrite allows overwriting values of stored keys in the storage
Overwrite bool
// LockFile allows specifying the filesystem path to use for the lockfile,
// providing only a filename it will store the lockfile within provided store
// path and nest the store under `path/store` to prevent access to lockfile
LockFile string
// Compression is the Compressor to use when reading / writing files, default is no compression
Compression Compressor
}
@ -57,11 +64,17 @@ func getDiskConfig(cfg *DiskConfig) DiskConfig {
cfg.WriteBufSize = DefaultDiskConfig.WriteBufSize
}
// Assume empty lockfile path == use default
if len(cfg.LockFile) < 1 {
cfg.LockFile = LockFile
}
// Return owned config copy
return DiskConfig{
Transform: cfg.Transform,
WriteBufSize: cfg.WriteBufSize,
Overwrite: cfg.Overwrite,
LockFile: cfg.LockFile,
Compression: cfg.Compression,
}
}
@ -76,16 +89,27 @@ type DiskStorage struct {
// OpenFile opens a DiskStorage instance for given folder path and configuration
func OpenFile(path string, cfg *DiskConfig) (*DiskStorage, error) {
// Get checked config
config := getDiskConfig(cfg)
// Acquire path builder
pb := util.GetPathBuilder()
defer util.PutPathBuilder(pb)
// Clean provided path, ensure ends in '/' (should
// be dir, this helps with file path trimming later)
storePath := pb.Join(path, "store") + "/"
// Clean provided store path, ensure
// ends in '/' to help later path trimming
storePath := pb.Clean(path) + "/"
// Get checked config
config := getDiskConfig(cfg)
// Clean provided lockfile path
lockfile := pb.Clean(config.LockFile)
// Check if lockfile is an *actual* path or just filename
if lockDir, _ := _path.Split(lockfile); len(lockDir) < 1 {
// Lockfile is a filename, store must be nested under
// $storePath/store to prevent access to the lockfile
storePath += "store/"
lockfile = pb.Join(path, lockfile)
}
// Attempt to open dir path
file, err := os.OpenFile(storePath, defaultFileROFlags, defaultDirPerms)
@ -118,7 +142,7 @@ func OpenFile(path string, cfg *DiskConfig) (*DiskStorage, error) {
}
// Open and acquire storage lock for path
lock, err := OpenLock(pb.Join(path, lockFile))
lock, err := OpenLock(lockfile)
if err != nil {
return nil, err
}
@ -347,9 +371,27 @@ func (st *DiskStorage) filepath(key string) (string, error) {
pb.AppendString(key)
// Check for dir traversal outside of root
if util.IsDirTraversal(st.path, pb.StringPtr()) {
if isDirTraversal(st.path, pb.StringPtr()) {
return "", ErrInvalidKey
}
return pb.String(), nil
}
// isDirTraversal will check if rootPlusPath is a dir traversal outside of root,
// assuming that both are cleaned and that rootPlusPath is path.Join(root, somePath)
func isDirTraversal(root, rootPlusPath string) bool {
switch {
// Root is $PWD, check for traversal out of
case root == ".":
return strings.HasPrefix(rootPlusPath, "../")
// The path MUST be prefixed by root
case !strings.HasPrefix(rootPlusPath, root):
return true
// In all other cases, check not equal
default:
return len(root) == len(rootPlusPath)
}
}

View file

@ -8,13 +8,8 @@
"codeberg.org/gruf/go-store/util"
)
// lockFile is our standard lockfile name.
var lockFile = "store.lock"
// IsLockKey returns whether storage key is our lockfile.
func IsLockKey(key string) bool {
return key == lockFile
}
// LockFile is our standard lockfile name.
const LockFile = "store.lock"
// Lock represents a filesystem lock to ensure only one storage instance open per path.
type Lock struct {

View file

@ -3,30 +3,10 @@
import (
"io/fs"
"os"
"strings"
"syscall"
"codeberg.org/gruf/go-fastpath"
)
// IsDirTraversal will check if rootPlusPath is a dir traversal outside of root,
// assuming that both are cleaned and that rootPlusPath is path.Join(root, somePath)
func IsDirTraversal(root string, rootPlusPath string) bool {
switch {
// Root is $PWD, check for traversal out of
case root == ".":
return strings.HasPrefix(rootPlusPath, "../")
// The path MUST be prefixed by root
case !strings.HasPrefix(rootPlusPath, root):
return true
// In all other cases, check not equal
default:
return len(root) == len(rootPlusPath)
}
}
// WalkDir traverses the dir tree of the supplied path, performing the supplied walkFn on each entry
func WalkDir(pb *fastpath.Builder, path string, walkFn func(string, fs.DirEntry)) error {
// Read supplied dir path
@ -100,14 +80,3 @@ func cleanDirs(pb *fastpath.Builder, path string) error {
}
return nil
}
// RetryOnEINTR is a low-level filesystem function for retrying syscalls on O_EINTR received
func RetryOnEINTR(do func() error) error {
for {
err := do()
if err == syscall.EINTR {
continue
}
return err
}
}

14
vendor/codeberg.org/gruf/go-store/util/sys.go generated vendored Normal file
View file

@ -0,0 +1,14 @@
package util
import "syscall"
// RetryOnEINTR is a low-level filesystem function for retrying syscalls on O_EINTR received
func RetryOnEINTR(do func() error) error {
for {
err := do()
if err == syscall.EINTR {
continue
}
return err
}
}

6
vendor/modules.txt vendored
View file

@ -16,15 +16,13 @@ codeberg.org/gruf/go-hashenc
# codeberg.org/gruf/go-mutexes v1.1.0
## explicit; go 1.14
codeberg.org/gruf/go-mutexes
# codeberg.org/gruf/go-nowish v1.1.0
## explicit; go 1.14
# codeberg.org/gruf/go-pools v1.0.2
## explicit; go 1.16
codeberg.org/gruf/go-pools
# codeberg.org/gruf/go-runners v1.2.0
## explicit; go 1.14
codeberg.org/gruf/go-runners
# codeberg.org/gruf/go-store v1.3.2
# codeberg.org/gruf/go-store v1.3.3
## explicit; go 1.14
codeberg.org/gruf/go-store/kv
codeberg.org/gruf/go-store/storage
@ -521,8 +519,6 @@ github.com/vmihailenco/tagparser/v2/internal/parser
# github.com/wagslane/go-password-validator v0.3.0
## explicit; go 1.16
github.com/wagslane/go-password-validator
# github.com/zeebo/blake3 v0.2.1
## explicit; go 1.13
# golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b
## explicit; go 1.17
golang.org/x/crypto/acme