pull in ncruces/go-sqlite3 v0.20.3 with tetratelabs/wazero v1.8.2 (#3574)

This commit is contained in:
kim 2024-11-26 16:25:48 +00:00 committed by GitHub
parent 6a8af42647
commit 61f8f1e0e3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
41 changed files with 374 additions and 226 deletions

4
go.mod
View file

@ -62,7 +62,7 @@ require (
github.com/miekg/dns v1.1.62
github.com/minio/minio-go/v7 v7.0.80
github.com/mitchellh/mapstructure v1.5.0
github.com/ncruces/go-sqlite3 v0.20.2
github.com/ncruces/go-sqlite3 v0.20.3
github.com/oklog/ulid v1.3.1
github.com/prometheus/client_golang v1.20.5
github.com/spf13/cobra v1.8.1
@ -73,7 +73,7 @@ require (
github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB.0.20230227143000-f4900831d6c8
github.com/tdewolff/minify/v2 v2.21.2
github.com/technologize/otel-go-contrib v1.1.1
github.com/tetratelabs/wazero v1.8.1
github.com/tetratelabs/wazero v1.8.2
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
github.com/ulule/limiter/v3 v3.11.2
github.com/uptrace/bun v1.2.6

8
go.sum generated
View file

@ -434,8 +434,8 @@ github.com/moul/http2curl v1.0.0 h1:dRMWoAtb+ePxMlLkrCbAqh4TlPHXvoGUSQ323/9Zahs=
github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/ncruces/go-sqlite3 v0.20.2 h1:cMLIwrLZQuCWVCEOowSqlIlpzgbag3jnYVW4NM5u01M=
github.com/ncruces/go-sqlite3 v0.20.2/go.mod h1:yL4ZNWGsr1/8pcLfpPW1RT1WFdvyeHonrgIwwi4rvkg=
github.com/ncruces/go-sqlite3 v0.20.3 h1:+4G4uEqOeusF0yRuQVUl9fuoEebUolwQSnBUjYBLYIw=
github.com/ncruces/go-sqlite3 v0.20.3/go.mod h1:ojLIAB243gtz68Eo283Ps+k9PyR3dvzS+9/RgId4+AA=
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
@ -553,8 +553,8 @@ github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03
github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8=
github.com/technologize/otel-go-contrib v1.1.1 h1:wZH9aSPNWZWIkEh3vfaKfMb15AJ80jJ1aVj/4GZdqIw=
github.com/technologize/otel-go-contrib v1.1.1/go.mod h1:dCN/wj2WyUO8aFZFdIN+6tfJHImjTML/8r2YVYAy3So=
github.com/tetratelabs/wazero v1.8.1 h1:NrcgVbWfkWvVc4UtT4LRLDf91PsOzDzefMdwhLfA550=
github.com/tetratelabs/wazero v1.8.1/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4=
github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
github.com/tidwall/btree v0.0.0-20191029221954-400434d76274 h1:G6Z6HvJuPjG6XfNGi/feOATzeJrfgTNJY+rGrHbA04E=
github.com/tidwall/btree v0.0.0-20191029221954-400434d76274/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
github.com/tidwall/buntdb v1.1.2 h1:noCrqQXL9EKMtcdwJcmuVKSEjqu1ua99RHHgbLTEHRo=

View file

@ -77,7 +77,7 @@ It also benefits greatly from [SQLite's](https://sqlite.org/testing.html) and
Every commit is [tested](https://github.com/ncruces/go-sqlite3/wiki/Test-matrix) on
Linux (amd64/arm64/386/riscv64/ppc64le/s390x), macOS (amd64/arm64),
Windows (amd64), FreeBSD (amd64), OpenBSD (amd64), NetBSD (amd64),
illumos (amd64), and Solaris (amd64).
DragonFly BSD (amd64), illumos (amd64), and Solaris (amd64).
The Go VFS is tested by running SQLite's
[mptest](https://github.com/sqlite/sqlite/blob/master/mptest/mptest.c).
@ -90,9 +90,20 @@ Perfomance of the [`database/sql`](https://pkg.go.dev/database/sql) driver is
The Wasm and VFS layers are also tested by running SQLite's
[speedtest1](https://github.com/sqlite/sqlite/blob/master/test/speedtest1.c).
### FAQ, issues, new features
For questions, please see [Discussions](https://github.com/ncruces/go-sqlite3/discussions/categories/q-a).
Also, post there if you used this driver for something interesting
([_"Show and tell"_](https://github.com/ncruces/go-sqlite3/discussions/categories/show-and-tell)),
have an [idea](https://github.com/ncruces/go-sqlite3/discussions/categories/ideas)…
The [Issue](https://github.com/ncruces/go-sqlite3/issues) tracker is for bugs we want fixed,
and features we're working on, planning to work on, or asking for help with.
### Alternatives
- [`modernc.org/sqlite`](https://pkg.go.dev/modernc.org/sqlite)
- [`crawshaw.io/sqlite`](https://pkg.go.dev/crawshaw.io/sqlite)
- [`github.com/mattn/go-sqlite3`](https://pkg.go.dev/github.com/mattn/go-sqlite3)
- [`github.com/zombiezen/go-sqlite`](https://pkg.go.dev/github.com/zombiezen/go-sqlite)
- [`github.com/zombiezen/go-sqlite`](https://pkg.go.dev/github.com/zombiezen/go-sqlite)

View file

@ -1,6 +1,6 @@
# Embeddable Wasm build of SQLite
This folder includes an embeddable Wasm build of SQLite 3.47.0 for use with
This folder includes an embeddable Wasm build of SQLite 3.47.1 for use with
[`github.com/ncruces/go-sqlite3`](https://pkg.go.dev/github.com/ncruces/go-sqlite3).
The following optional features are compiled in:

Binary file not shown.

View file

@ -9,5 +9,6 @@ golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8=
golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=

View file

@ -30,7 +30,6 @@ like SQLite.
You can also opt into a cross-platform locking implementation
with the `sqlite3_dotlk` build tag.
The only requirement is an atomic `os.Mkdir`.
Otherwise, file locking is not supported, and you must use
[`nolock=1`](https://sqlite.org/uri.html#urinolock)

View file

@ -101,6 +101,14 @@ func (c cksmFile) Pragma(name string, value string) (string, error) {
return "", _NOTFOUND
}
func (c cksmFile) DeviceCharacteristics() DeviceCharacteristic {
res := c.File.DeviceCharacteristics()
if c.verifyCksm {
res &^= IOCAP_SUBPAGE_READ
}
return res
}
func (c cksmFile) fileControl(ctx context.Context, mod api.Module, op _FcntlOpcode, pArg uint32) _ErrorCode {
switch op {
case _FCNTL_CKPT_START:

View file

@ -177,6 +177,7 @@ func (e _ErrorCode) Error() string {
IOCAP_POWERSAFE_OVERWRITE DeviceCharacteristic = 0x00001000
IOCAP_IMMUTABLE DeviceCharacteristic = 0x00002000
IOCAP_BATCH_ATOMIC DeviceCharacteristic = 0x00004000
IOCAP_SUBPAGE_READ DeviceCharacteristic = 0x00008000
)
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html

View file

@ -187,7 +187,7 @@ func (f *vfsFile) SectorSize() int {
}
func (f *vfsFile) DeviceCharacteristics() DeviceCharacteristic {
var res DeviceCharacteristic
res := IOCAP_SUBPAGE_READ
if osBatchAtomic(f.File) {
res |= IOCAP_BATCH_ATOMIC
}

View file

@ -15,9 +15,15 @@ func osGetSharedLock(file *os.File) _ErrorCode {
func osGetReservedLock(file *os.File) _ErrorCode {
rc := osLock(file, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK)
if rc == _BUSY {
// The documentation states the lock is upgraded by releasing the previous lock,
// then acquiring the new lock.
// This is a race, so return BUSY_SNAPSHOT to ensure the transaction is aborted.
// The documentation states that a lock is upgraded by
// releasing the previous lock, then acquiring the new lock.
// Going over the source code of various BSDs, though,
// with LOCK_NB, the lock is not released,
// and EAGAIN is returned holding the shared lock.
// Still, if we're already in a transaction, we want to abort it,
// so return BUSY_SNAPSHOT here. If there's no transaction active,
// SQLite will change this back to SQLITE_BUSY,
// and invoke the busy handler if appropriate.
return _BUSY_SNAPSHOT
}
return rc
@ -33,9 +39,11 @@ func osGetExclusiveLock(file *os.File, state *LockLevel) _ErrorCode {
func osDowngradeLock(file *os.File, _ LockLevel) _ErrorCode {
rc := osLock(file, unix.LOCK_SH|unix.LOCK_NB, _IOERR_RDLOCK)
if rc == _BUSY {
// The documentation states the lock is upgraded by releasing the previous lock,
// then acquiring the new lock.
// This is a race, so return IOERR_RDLOCK to ensure the transaction is aborted.
// The documentation states that a lock is downgraded by
// releasing the previous lock then acquiring the new lock.
// Going over the source code of various BSDs, though,
// with LOCK_SH|LOCK_NB this should never happen.
// Return IOERR_RDLOCK, as BUSY would cause an assert to fail.
return _IOERR_RDLOCK
}
return _OK
@ -50,7 +58,10 @@ func osReleaseLock(file *os.File, _ LockLevel) _ErrorCode {
}
func osCheckReservedLock(file *os.File) (bool, _ErrorCode) {
// Test the RESERVED lock.
// Test the RESERVED lock with fcntl(F_GETLK).
// This only works on systems where fcntl and flock are compatible.
// However, SQLite only calls this while holding a shared lock,
// so the difference is immaterial.
lock, rc := osTestLock(file, _RESERVED_BYTE, 1)
return lock == unix.F_WRLCK, rc
}

View file

@ -28,7 +28,8 @@ func osGetSharedLock(file *os.File) _ErrorCode {
name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
err := os.Mkdir(name+".lock", 0777)
f, err := os.OpenFile(name+".lock", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
f.Close()
if errors.Is(err, fs.ErrExist) {
return _BUSY // Another process has the lock.
}

View file

@ -50,14 +50,17 @@ func osGetExclusiveLock(file *os.File, state *LockLevel) _ErrorCode {
if rc != _OK {
// Reacquire the SHARED lock.
osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0)
if rc := osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0); rc != _OK {
// notest // this should never happen
return _IOERR_RDLOCK
}
}
return rc
}
func osDowngradeLock(file *os.File, state LockLevel) _ErrorCode {
if state >= LOCK_EXCLUSIVE {
// Release the EXCLUSIVE lock.
// Release the EXCLUSIVE lock while holding the PENDING lock.
osUnlock(file, _SHARED_FIRST, _SHARED_SIZE)
// Reacquire the SHARED lock.
@ -78,7 +81,7 @@ func osDowngradeLock(file *os.File, state LockLevel) _ErrorCode {
}
func osReleaseLock(file *os.File, state LockLevel) _ErrorCode {
// Release all locks.
// Release all locks, PENDING must be last.
if state >= LOCK_RESERVED {
osUnlock(file, _RESERVED_BYTE, 1)
}

View file

@ -14,52 +14,52 @@
"github.com/ncruces/go-sqlite3/internal/util"
)
type vfsShmFile struct {
type vfsShmParent struct {
*os.File
info os.FileInfo
refs int // +checklocks:vfsShmFilesMtx
refs int // +checklocks:vfsShmListMtx
lock [_SHM_NLOCK]int16 // +checklocks:Mutex
sync.Mutex
}
var (
// +checklocks:vfsShmFilesMtx
vfsShmFiles []*vfsShmFile
vfsShmFilesMtx sync.Mutex
// +checklocks:vfsShmListMtx
vfsShmList []*vfsShmParent
vfsShmListMtx sync.Mutex
)
type vfsShm struct {
*vfsShmFile
*vfsShmParent
path string
lock [_SHM_NLOCK]bool
regions []*util.MappedRegion
}
func (s *vfsShm) Close() error {
if s.vfsShmFile == nil {
if s.vfsShmParent == nil {
return nil
}
vfsShmFilesMtx.Lock()
defer vfsShmFilesMtx.Unlock()
vfsShmListMtx.Lock()
defer vfsShmListMtx.Unlock()
// Unlock everything.
s.shmLock(0, _SHM_NLOCK, _SHM_UNLOCK)
// Decrease reference count.
if s.vfsShmFile.refs > 0 {
s.vfsShmFile.refs--
s.vfsShmFile = nil
if s.vfsShmParent.refs > 0 {
s.vfsShmParent.refs--
s.vfsShmParent = nil
return nil
}
err := s.File.Close()
for i, g := range vfsShmFiles {
if g == s.vfsShmFile {
vfsShmFiles[i] = nil
s.vfsShmFile = nil
for i, g := range vfsShmList {
if g == s.vfsShmParent {
vfsShmList[i] = nil
s.vfsShmParent = nil
return err
}
}
@ -67,7 +67,7 @@ func (s *vfsShm) Close() error {
}
func (s *vfsShm) shmOpen() _ErrorCode {
if s.vfsShmFile != nil {
if s.vfsShmParent != nil {
return _OK
}
@ -85,13 +85,13 @@ func (s *vfsShm) shmOpen() _ErrorCode {
return _IOERR_FSTAT
}
vfsShmFilesMtx.Lock()
defer vfsShmFilesMtx.Unlock()
vfsShmListMtx.Lock()
defer vfsShmListMtx.Unlock()
// Find a shared file, increase the reference count.
for _, g := range vfsShmFiles {
for _, g := range vfsShmList {
if g != nil && os.SameFile(fi, g.info) {
s.vfsShmFile = g
s.vfsShmParent = g
g.refs++
return _OK
}
@ -107,18 +107,18 @@ func (s *vfsShm) shmOpen() _ErrorCode {
}
// Add the new shared file.
s.vfsShmFile = &vfsShmFile{
s.vfsShmParent = &vfsShmParent{
File: f,
info: fi,
}
f = nil // Don't close the file.
for i, g := range vfsShmFiles {
for i, g := range vfsShmList {
if g == nil {
vfsShmFiles[i] = s.vfsShmFile
vfsShmList[i] = s.vfsShmParent
return _OK
}
}
vfsShmFiles = append(vfsShmFiles, s.vfsShmFile)
vfsShmList = append(vfsShmList, s.vfsShmParent)
return _OK
}
@ -157,57 +157,11 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) _ErrorCode {
s.Lock()
defer s.Unlock()
switch {
case flags&_SHM_UNLOCK != 0:
for i := offset; i < offset+n; i++ {
if s.lock[i] {
if s.vfsShmFile.lock[i] == 0 {
panic(util.AssertErr())
}
if s.vfsShmFile.lock[i] <= 0 {
s.vfsShmFile.lock[i] = 0
} else {
s.vfsShmFile.lock[i]--
}
s.lock[i] = false
}
}
case flags&_SHM_SHARED != 0:
for i := offset; i < offset+n; i++ {
if s.lock[i] {
panic(util.AssertErr())
}
if s.vfsShmFile.lock[i]+1 <= 0 {
return _BUSY
}
}
for i := offset; i < offset+n; i++ {
s.vfsShmFile.lock[i]++
s.lock[i] = true
}
case flags&_SHM_EXCLUSIVE != 0:
for i := offset; i < offset+n; i++ {
if s.lock[i] {
panic(util.AssertErr())
}
if s.vfsShmFile.lock[i] != 0 {
return _BUSY
}
}
for i := offset; i < offset+n; i++ {
s.vfsShmFile.lock[i] = -1
s.lock[i] = true
}
default:
panic(util.AssertErr())
}
return _OK
return s.shmMemLock(offset, n, flags)
}
func (s *vfsShm) shmUnmap(delete bool) {
if s.vfsShmFile == nil {
if s.vfsShmParent == nil {
return
}

View file

@ -31,7 +31,10 @@
//
// https://sqlite.org/walformat.html#the_wal_index_file_format
func (s *vfsShm) shmAcquire() {
func (s *vfsShm) shmAcquire(ptr *_ErrorCode) {
if ptr != nil && *ptr != _OK {
return
}
if len(s.ptrs) == 0 || shmUnmodified(s.shadow[0][:], s.shared[0][:]) {
return
}
@ -69,7 +72,7 @@ func (s *vfsShm) shmRelease() {
func (s *vfsShm) shmBarrier() {
s.Lock()
s.shmAcquire()
s.shmAcquire(nil)
s.shmRelease()
s.Unlock()
}

View file

@ -13,22 +13,22 @@
"github.com/tetratelabs/wazero/api"
)
type vfsShmBuffer struct {
type vfsShmParent struct {
shared [][_WALINDEX_PGSZ]byte
refs int // +checklocks:vfsShmBuffersMtx
refs int // +checklocks:vfsShmListMtx
lock [_SHM_NLOCK]int16 // +checklocks:Mutex
sync.Mutex
}
var (
// +checklocks:vfsShmBuffersMtx
vfsShmBuffers = map[string]*vfsShmBuffer{}
vfsShmBuffersMtx sync.Mutex
// +checklocks:vfsShmListMtx
vfsShmList = map[string]*vfsShmParent{}
vfsShmListMtx sync.Mutex
)
type vfsShm struct {
*vfsShmBuffer
*vfsShmParent
mod api.Module
alloc api.Function
free api.Function
@ -40,20 +40,20 @@ type vfsShm struct {
}
func (s *vfsShm) Close() error {
if s.vfsShmBuffer == nil {
if s.vfsShmParent == nil {
return nil
}
vfsShmBuffersMtx.Lock()
defer vfsShmBuffersMtx.Unlock()
vfsShmListMtx.Lock()
defer vfsShmListMtx.Unlock()
// Unlock everything.
s.shmLock(0, _SHM_NLOCK, _SHM_UNLOCK)
// Decrease reference count.
if s.vfsShmBuffer.refs > 0 {
s.vfsShmBuffer.refs--
s.vfsShmBuffer = nil
if s.vfsShmParent.refs > 0 {
s.vfsShmParent.refs--
s.vfsShmParent = nil
return nil
}
@ -61,22 +61,22 @@ func (s *vfsShm) Close() error {
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return _IOERR_UNLOCK
}
delete(vfsShmBuffers, s.path)
s.vfsShmBuffer = nil
delete(vfsShmList, s.path)
s.vfsShmParent = nil
return nil
}
func (s *vfsShm) shmOpen() _ErrorCode {
if s.vfsShmBuffer != nil {
if s.vfsShmParent != nil {
return _OK
}
vfsShmBuffersMtx.Lock()
defer vfsShmBuffersMtx.Unlock()
vfsShmListMtx.Lock()
defer vfsShmListMtx.Unlock()
// Find a shared buffer, increase the reference count.
if g, ok := vfsShmBuffers[s.path]; ok {
s.vfsShmBuffer = g
if g, ok := vfsShmList[s.path]; ok {
s.vfsShmParent = g
g.refs++
return _OK
}
@ -92,8 +92,8 @@ func (s *vfsShm) shmOpen() _ErrorCode {
}
// Add the new shared buffer.
s.vfsShmBuffer = &vfsShmBuffer{}
vfsShmBuffers[s.path] = s.vfsShmBuffer
s.vfsShmParent = &vfsShmParent{}
vfsShmList[s.path] = s.vfsShmParent
return _OK
}
@ -112,7 +112,7 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
s.Lock()
defer s.Unlock()
defer s.shmAcquire()
defer s.shmAcquire(nil)
// Extend shared memory.
if int(id) >= len(s.shared) {
@ -125,7 +125,6 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
// Allocate shadow memory.
if int(id) >= len(s.shadow) {
s.shadow = append(s.shadow, make([][_WALINDEX_PGSZ]byte, int(id)-len(s.shadow)+1)...)
s.shadow[0][4] = 1 // force invalidation
}
// Allocate local memory.
@ -141,70 +140,26 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
s.ptrs = append(s.ptrs, uint32(s.stack[0]))
}
s.shadow[0][4] = 1
return s.ptrs[id], _OK
}
func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) _ErrorCode {
func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) (rc _ErrorCode) {
s.Lock()
defer s.Unlock()
switch {
case flags&_SHM_LOCK != 0:
defer s.shmAcquire()
defer s.shmAcquire(&rc)
case flags&_SHM_EXCLUSIVE != 0:
s.shmRelease()
}
switch {
case flags&_SHM_UNLOCK != 0:
for i := offset; i < offset+n; i++ {
if s.lock[i] {
if s.vfsShmBuffer.lock[i] == 0 {
panic(util.AssertErr())
}
if s.vfsShmBuffer.lock[i] <= 0 {
s.vfsShmBuffer.lock[i] = 0
} else {
s.vfsShmBuffer.lock[i]--
}
s.lock[i] = false
}
}
case flags&_SHM_SHARED != 0:
for i := offset; i < offset+n; i++ {
if s.lock[i] {
panic(util.AssertErr())
}
if s.vfsShmBuffer.lock[i]+1 <= 0 {
return _BUSY
}
}
for i := offset; i < offset+n; i++ {
s.vfsShmBuffer.lock[i]++
s.lock[i] = true
}
case flags&_SHM_EXCLUSIVE != 0:
for i := offset; i < offset+n; i++ {
if s.lock[i] {
panic(util.AssertErr())
}
if s.vfsShmBuffer.lock[i] != 0 {
return _BUSY
}
}
for i := offset; i < offset+n; i++ {
s.vfsShmBuffer.lock[i] = -1
s.lock[i] = true
}
default:
panic(util.AssertErr())
}
return _OK
return s.shmMemLock(offset, n, flags)
}
func (s *vfsShm) shmUnmap(delete bool) {
if s.vfsShmBuffer == nil {
if s.vfsShmParent == nil {
return
}
defer s.Close()

55
vendor/github.com/ncruces/go-sqlite3/vfs/shm_memlk.go generated vendored Normal file
View file

@ -0,0 +1,55 @@
//go:build ((freebsd || openbsd || netbsd || dragonfly || illumos) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !sqlite3_nosys) || sqlite3_flock || sqlite3_dotlk
package vfs
import "github.com/ncruces/go-sqlite3/internal/util"
// +checklocks:s.Mutex
func (s *vfsShm) shmMemLock(offset, n int32, flags _ShmFlag) _ErrorCode {
switch {
case flags&_SHM_UNLOCK != 0:
for i := offset; i < offset+n; i++ {
if s.lock[i] {
if s.vfsShmParent.lock[i] == 0 {
panic(util.AssertErr())
}
if s.vfsShmParent.lock[i] <= 0 {
s.vfsShmParent.lock[i] = 0
} else {
s.vfsShmParent.lock[i]--
}
s.lock[i] = false
}
}
case flags&_SHM_SHARED != 0:
for i := offset; i < offset+n; i++ {
if s.lock[i] {
panic(util.AssertErr())
}
if s.vfsShmParent.lock[i]+1 <= 0 {
return _BUSY
}
}
for i := offset; i < offset+n; i++ {
s.vfsShmParent.lock[i]++
s.lock[i] = true
}
case flags&_SHM_EXCLUSIVE != 0:
for i := offset; i < offset+n; i++ {
if s.lock[i] {
panic(util.AssertErr())
}
if s.vfsShmParent.lock[i] != 0 {
return _BUSY
}
}
for i := offset; i < offset+n; i++ {
s.vfsShmParent.lock[i] = -1
s.lock[i] = true
}
default:
panic(util.AssertErr())
}
return _OK
}

View file

@ -64,7 +64,7 @@ func (s *vfsShm) shmOpen() _ErrorCode {
return osReadLock(s.File, _SHM_DMS, 1, time.Millisecond)
}
func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, _ErrorCode) {
func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (_ uint32, rc _ErrorCode) {
// Ensure size is a multiple of the OS page size.
if size != _WALINDEX_PGSZ || (windows.Getpagesize()-1)&_WALINDEX_PGSZ != 0 {
return 0, _IOERR_SHMMAP
@ -78,7 +78,7 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
return 0, rc
}
defer s.shmAcquire()
defer s.shmAcquire(&rc)
// Check if file is big enough.
o, err := s.Seek(0, io.SeekEnd)
@ -107,7 +107,6 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
// Allocate shadow memory.
if int(id) >= len(s.shadow) {
s.shadow = append(s.shadow, make([][_WALINDEX_PGSZ]byte, int(id)-len(s.shadow)+1)...)
s.shadow[0][4] = 1 // force invalidation
}
// Allocate local memory.
@ -123,22 +122,23 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
s.ptrs = append(s.ptrs, uint32(s.stack[0]))
}
s.shadow[0][4] = 1
return s.ptrs[id], _OK
}
func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) _ErrorCode {
switch {
case flags&_SHM_LOCK != 0:
defer s.shmAcquire()
case flags&_SHM_EXCLUSIVE != 0:
s.shmRelease()
}
func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) (rc _ErrorCode) {
var timeout time.Duration
if s.blocking {
timeout = time.Millisecond
}
switch {
case flags&_SHM_LOCK != 0:
defer s.shmAcquire(&rc)
case flags&_SHM_EXCLUSIVE != 0:
s.shmRelease()
}
switch {
case flags&_SHM_UNLOCK != 0:
return osUnlock(s.File, _SHM_BASE+uint32(offset), uint32(n))

View file

@ -96,14 +96,21 @@ systems are ones we test, but that doesn't necessarily mean other operating
system versions won't work.
We currently test Linux (Ubuntu and scratch), MacOS and Windows as packaged by
[GitHub Actions][11], as well compilation of 32-bit Linux and 64-bit FreeBSD.
[GitHub Actions][11], as well as nested VMs running on Linux for FreeBSD, NetBSD,
OpenBSD, DragonFly BSD, illumos and Solaris.
We also test cross compilation for many `GOOS` and `GOARCH` combinations.
* Interpreter
* Linux is tested on amd64 (native) as well arm64 and riscv64 via emulation.
* MacOS and Windows are only tested on amd64.
* Windows, FreeBSD, NetBSD, OpenBSD, DragonFly BSD, illumos and Solaris are
tested only on amd64.
* macOS is tested only on arm64.
* Compiler
* Linux is tested on amd64 (native) as well arm64 via emulation.
* MacOS and Windows are only tested on amd64.
* Windows, FreeBSD, NetBSD, DragonFly BSD, illumos and Solaris are
tested only on amd64.
* macOS is tested only on arm64.
wazero has no dependencies and doesn't require CGO. This means it can also be
embedded in an application that doesn't use an operating system. This is a main

View file

@ -5,10 +5,15 @@
//
// Meanwhile, users who know their runtime.GOOS can operate with the compiler
// may choose to use NewRuntimeConfigCompiler explicitly.
//go:build (amd64 || arm64) && (darwin || linux || freebsd || windows)
//go:build (amd64 || arm64) && (linux || darwin || freebsd || netbsd || dragonfly || solaris || windows)
package wazero
import "github.com/tetratelabs/wazero/internal/platform"
func newRuntimeConfig() RuntimeConfig {
return NewRuntimeConfigCompiler()
if platform.CompilerSupported() {
return NewRuntimeConfigCompiler()
}
return NewRuntimeConfigInterpreter()
}

View file

@ -1,5 +1,5 @@
// This is the opposite constraint of config_supported.go
//go:build !(amd64 || arm64) || !(darwin || linux || freebsd || windows)
//go:build !(amd64 || arm64) || !(linux || darwin || freebsd || netbsd || dragonfly || solaris || windows)
package wazero

View file

@ -43,7 +43,7 @@ type LinearMemory interface {
}
// WithMemoryAllocator registers the given MemoryAllocator into the given
// context.Context.
// context.Context. The context must be passed when initializing a module.
func WithMemoryAllocator(ctx context.Context, allocator MemoryAllocator) context.Context {
if allocator != nil {
return context.WithValue(ctx, expctxkeys.MemoryAllocatorKey{}, allocator)

View file

@ -2196,7 +2196,7 @@ func (m *machine) Encode(ctx context.Context) (err error) {
}
// ResolveRelocations implements backend.Machine.
func (m *machine) ResolveRelocations(refToBinaryOffset []int, binary []byte, relocations []backend.RelocationInfo, _ []int) {
func (m *machine) ResolveRelocations(refToBinaryOffset []int, _ int, binary []byte, relocations []backend.RelocationInfo, _ []int) {
for _, r := range relocations {
offset := r.Offset
calleeFnOffset := refToBinaryOffset[r.FuncRef]

View file

@ -21,7 +21,7 @@
// trampolineIslandInterval is the range of the trampoline island.
// Half of the range is used for the trampoline island, and the other half is used for the function.
trampolineIslandInterval = maxUnconditionalBranchOffset / 2
trampolineIslandInterval = (maxUnconditionalBranchOffset - 1) / 2
// maxNumFunctions explicitly specifies the maximum number of functions that can be allowed in a single executable.
maxNumFunctions = trampolineIslandInterval >> 6
@ -42,12 +42,13 @@ func (m *machine) CallTrampolineIslandInfo(numFunctions int) (interval, size int
// ResolveRelocations implements backend.Machine ResolveRelocations.
func (m *machine) ResolveRelocations(
refToBinaryOffset []int,
importedFns int,
executable []byte,
relocations []backend.RelocationInfo,
callTrampolineIslandOffsets []int,
) {
for _, islandOffset := range callTrampolineIslandOffsets {
encodeCallTrampolineIsland(refToBinaryOffset, islandOffset, executable)
encodeCallTrampolineIsland(refToBinaryOffset, importedFns, islandOffset, executable)
}
for _, r := range relocations {
@ -71,11 +72,15 @@ func (m *machine) ResolveRelocations(
// encodeCallTrampolineIsland encodes a trampoline island for the given functions.
// Each island consists of a trampoline instruction sequence for each function.
// Each trampoline instruction sequence consists of 4 instructions + 32-bit immediate.
func encodeCallTrampolineIsland(refToBinaryOffset []int, islandOffset int, executable []byte) {
for i := 0; i < len(refToBinaryOffset); i++ {
func encodeCallTrampolineIsland(refToBinaryOffset []int, importedFns int, islandOffset int, executable []byte) {
// We skip the imported functions: they don't need trampolines
// and are not accounted for.
binaryOffsets := refToBinaryOffset[importedFns:]
for i := 0; i < len(binaryOffsets); i++ {
trampolineOffset := islandOffset + trampolineCallSize*i
fnOffset := refToBinaryOffset[i]
fnOffset := binaryOffsets[i]
diff := fnOffset - (trampolineOffset + 16)
if diff > math.MaxInt32 || diff < math.MinInt32 {
// This case even amd64 can't handle. 4GB is too big.

View file

@ -77,11 +77,13 @@
// ResolveRelocations resolves the relocations after emitting machine code.
// * refToBinaryOffset: the map from the function reference (ssa.FuncRef) to the executable offset.
// * importedFns: the max index of the imported functions at the beginning of refToBinaryOffset
// * executable: the binary to resolve the relocations.
// * relocations: the relocations to resolve.
// * callTrampolineIslandOffsets: the offsets of the trampoline islands in the executable.
ResolveRelocations(
refToBinaryOffset []int,
importedFns int,
executable []byte,
relocations []RelocationInfo,
callTrampolineIslandOffsets []int,

View file

@ -314,7 +314,7 @@ func (e *engine) compileModule(ctx context.Context, module *wasm.Module, listene
// Resolve relocations for local function calls.
if len(rels) > 0 {
machine.ResolveRelocations(refToBinaryOffset, executable, rels, callTrampolineIslandOffsets)
machine.ResolveRelocations(refToBinaryOffset, importedFns, executable, rels, callTrampolineIslandOffsets)
}
if runtime.GOARCH == "arm64" {

View file

@ -28,3 +28,8 @@ type CpuFeatureFlags interface {
CpuExtraFeatureAmd64ABM CpuFeature = 1 << 5
// Note: when adding new features, ensure that the feature is included in CpuFeatureFlags.Raw.
)
const (
// CpuFeatureArm64Atomic is the flag to query CpuFeatureFlags.Has for Large System Extensions capabilities on arm64
CpuFeatureArm64Atomic CpuFeature = 1 << 21
)

View file

@ -1,4 +1,4 @@
//go:build amd64 && !tinygo
//go:build gc
package platform
@ -12,7 +12,7 @@ type cpuFeatureFlags struct {
}
// cpuid exposes the CPUID instruction to the Go layer (https://www.amd.com/system/files/TechDocs/25481.pdf)
// implemented in impl_amd64.s
// implemented in cpuid_amd64.s
func cpuid(arg1, arg2 uint32) (eax, ebx, ecx, edx uint32)
// cpuidAsBitmap combines the result of invoking cpuid to uint64 bitmap.
@ -60,8 +60,9 @@ func (f *cpuFeatureFlags) HasExtra(cpuFeature CpuFeature) bool {
// Raw implements the same method on the CpuFeatureFlags interface.
func (f *cpuFeatureFlags) Raw() uint64 {
// Below, we only set the first 4 bits for the features we care about,
// instead of setting all the unnecessary bits obtained from the CPUID instruction.
// Below, we only set bits for the features we care about,
// instead of setting all the unnecessary bits obtained from the
// CPUID instruction.
var ret uint64
if f.Has(CpuFeatureAmd64SSE3) {
ret = 1 << 0

View file

@ -1,6 +1,9 @@
//go:build gc
#include "textflag.h"
// lifted from github.com/intel-go/cpuid and src/internal/cpu/cpu_x86.s
// func cpuid(arg1, arg2 uint32) (eax, ebx, ecx, edx uint32)
TEXT ·cpuid(SB), NOSPLIT, $0-24
MOVL arg1+0(FP), AX
@ -11,4 +14,3 @@ TEXT ·cpuid(SB), NOSPLIT, $0-24
MOVL CX, ecx+16(FP)
MOVL DX, edx+20(FP)
RET

View file

@ -0,0 +1,71 @@
//go:build gc
package platform
import "runtime"
// CpuFeatures exposes the capabilities for this CPU, queried via the Has, HasExtra methods.
var CpuFeatures = loadCpuFeatureFlags()
// cpuFeatureFlags implements CpuFeatureFlags interface.
type cpuFeatureFlags struct {
isar0 uint64
isar1 uint64
}
// implemented in cpuid_arm64.s
func getisar0() uint64
// implemented in cpuid_arm64.s
func getisar1() uint64
func loadCpuFeatureFlags() CpuFeatureFlags {
switch runtime.GOOS {
case "darwin", "windows":
// These OSes do not allow userland to read the instruction set attribute registers,
// but basically require atomic instructions:
// - "darwin" is the desktop version (mobile version is "ios"),
// and the M1 is a ARMv8.4.
// - "windows" requires them from Windows 11, see page 12
// https://download.microsoft.com/download/7/8/8/788bf5ab-0751-4928-a22c-dffdc23c27f2/Minimum%20Hardware%20Requirements%20for%20Windows%2011.pdf
return &cpuFeatureFlags{
isar0: uint64(CpuFeatureArm64Atomic),
isar1: 0,
}
case "linux", "freebsd":
// These OSes allow userland to read the instruction set attribute registers,
// which is otherwise restricted to EL0:
// https://kernel.org/doc/Documentation/arm64/cpu-feature-registers.txt
// See these for contents of the registers:
// https://developer.arm.com/documentation/ddi0601/latest/AArch64-Registers/ID-AA64ISAR0-EL1--AArch64-Instruction-Set-Attribute-Register-0
// https://developer.arm.com/documentation/ddi0601/latest/AArch64-Registers/ID-AA64ISAR1-EL1--AArch64-Instruction-Set-Attribute-Register-1
return &cpuFeatureFlags{
isar0: getisar0(),
isar1: getisar1(),
}
default:
return &cpuFeatureFlags{}
}
}
// Has implements the same method on the CpuFeatureFlags interface.
func (f *cpuFeatureFlags) Has(cpuFeature CpuFeature) bool {
return (f.isar0 & uint64(cpuFeature)) != 0
}
// HasExtra implements the same method on the CpuFeatureFlags interface.
func (f *cpuFeatureFlags) HasExtra(cpuFeature CpuFeature) bool {
return (f.isar1 & uint64(cpuFeature)) != 0
}
// Raw implements the same method on the CpuFeatureFlags interface.
func (f *cpuFeatureFlags) Raw() uint64 {
// Below, we only set bits for the features we care about,
// instead of setting all the unnecessary bits obtained from the
// instruction set attribute registers.
var ret uint64
if f.Has(CpuFeatureArm64Atomic) {
ret = 1 << 0
}
return ret
}

View file

@ -0,0 +1,21 @@
//go:build gc
#include "textflag.h"
// lifted from github.com/golang/sys and cpu/cpu_arm64.s
// func getisar0() uint64
TEXT ·getisar0(SB), NOSPLIT, $0-8
// get Instruction Set Attributes 0 into x0
// mrs x0, ID_AA64ISAR0_EL1 = d5380600
WORD $0xd5380600
MOVD R0, ret+0(FP)
RET
// func getisar1() uint64
TEXT ·getisar1(SB), NOSPLIT, $0-8
// get Instruction Set Attributes 1 into x0
// mrs x0, ID_AA64ISAR1_EL1 = d5380620
WORD $0xd5380620
MOVD R0, ret+0(FP)
RET

View file

@ -1,4 +1,4 @@
//go:build !amd64 || tinygo
//go:build !(amd64 || arm64) || !gc
package platform

View file

@ -1,5 +1,5 @@
// Separated from linux which has support for huge pages.
//go:build darwin || freebsd
//go:build darwin || freebsd || netbsd || dragonfly || solaris
package platform

View file

@ -1,10 +1,9 @@
//go:build (darwin || linux || freebsd) && !tinygo
//go:build (linux || darwin || freebsd || netbsd || dragonfly || solaris) && !tinygo
package platform
import (
"syscall"
"unsafe"
)
const (
@ -31,17 +30,3 @@ func mmapCodeSegmentARM64(size int) ([]byte, error) {
// The region must be RW: RW for writing native codes.
return mmapCodeSegment(size, mmapProtARM64)
}
// MprotectRX is like syscall.Mprotect with RX permission, defined locally so that freebsd compiles.
func MprotectRX(b []byte) (err error) {
var _p0 unsafe.Pointer
if len(b) > 0 {
_p0 = unsafe.Pointer(&b[0])
}
const prot = syscall.PROT_READ | syscall.PROT_EXEC
_, _, e1 := syscall.Syscall(syscall.SYS_MPROTECT, uintptr(_p0), uintptr(len(b)), uintptr(prot))
if e1 != 0 {
err = syscall.Errno(e1)
}
return
}

View file

@ -1,4 +1,4 @@
//go:build !(darwin || linux || freebsd || windows) || tinygo
//go:build !(linux || darwin || freebsd || netbsd || dragonfly || solaris || windows) || tinygo
package platform

View file

@ -0,0 +1,22 @@
//go:build (freebsd || netbsd || dragonfly) && !tinygo
package platform
import (
"syscall"
"unsafe"
)
// MprotectRX is like syscall.Mprotect with RX permission, defined locally so that BSD compiles.
func MprotectRX(b []byte) (err error) {
var _p0 unsafe.Pointer
if len(b) > 0 {
_p0 = unsafe.Pointer(&b[0])
}
const prot = syscall.PROT_READ | syscall.PROT_EXEC
_, _, e1 := syscall.Syscall(syscall.SYS_MPROTECT, uintptr(_p0), uintptr(len(b)), uintptr(prot))
if e1 != 0 {
err = syscall.Errno(e1)
}
return
}

View file

@ -0,0 +1,10 @@
//go:build (linux || darwin) && !tinygo
package platform
import "syscall"
// MprotectRX is like syscall.Mprotect with RX permission.
func MprotectRX(b []byte) (err error) {
return syscall.Mprotect(b, syscall.PROT_READ|syscall.PROT_EXEC)
}

View file

@ -0,0 +1,9 @@
//go:build solaris && !tinygo
package platform
import "syscall"
func MprotectRX(b []byte) error {
return syscall.ENOTSUP
}

View file

@ -11,15 +11,16 @@
// archRequirementsVerified is set by platform-specific init to true if the platform is supported
var archRequirementsVerified bool
// CompilerSupported is exported for tests and includes constraints here and also the assembler.
// CompilerSupported includes constraints here and also the assembler.
func CompilerSupported() bool {
switch runtime.GOOS {
case "darwin", "windows", "linux", "freebsd":
case "linux", "darwin", "freebsd", "netbsd", "dragonfly", "windows":
return archRequirementsVerified
case "solaris", "illumos":
return runtime.GOARCH == "amd64" && archRequirementsVerified
default:
return false
}
return archRequirementsVerified
}
// MmapCodeSegment copies the code into the executable region and returns the byte slice of the region.

View file

@ -2,6 +2,6 @@
// init verifies that the current CPU supports the required ARM64 features
func init() {
// No further checks currently needed.
archRequirementsVerified = true
// Ensure atomic instructions are supported.
archRequirementsVerified = CpuFeatures.Has(CpuFeatureArm64Atomic)
}

4
vendor/modules.txt vendored
View file

@ -520,7 +520,7 @@ github.com/modern-go/reflect2
# github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
## explicit
github.com/munnerz/goautoneg
# github.com/ncruces/go-sqlite3 v0.20.2
# github.com/ncruces/go-sqlite3 v0.20.3
## explicit; go 1.21
github.com/ncruces/go-sqlite3
github.com/ncruces/go-sqlite3/driver
@ -852,7 +852,7 @@ github.com/tdewolff/parse/v2/strconv
# github.com/technologize/otel-go-contrib v1.1.1
## explicit; go 1.17
github.com/technologize/otel-go-contrib/otelginmetrics
# github.com/tetratelabs/wazero v1.8.1
# github.com/tetratelabs/wazero v1.8.2
## explicit; go 1.21
github.com/tetratelabs/wazero
github.com/tetratelabs/wazero/api