Compare commits

...

4 commits

Author SHA1 Message Date
tobi 6d8e864ab0 Merge branch 'main' into domain_permission_subscriptions 2024-10-26 14:04:56 +02:00
tobi 82d27319b1 weeeeeee 2024-10-26 14:04:40 +02:00
kim 51cb6cae16
update go-sqlite3 => v0.20.0 (#3483) 2024-10-25 16:09:18 +00:00
kim d8a83860bc
[bugfix] incorrect /api/v_/instance domain uri fields (#3477)
* update instance v1 / v2 endpoints to set uri / domain correctly (i.e. account domain)

* fix tests relying on old instance behaviour
2024-10-22 14:47:28 +00:00
52 changed files with 1235 additions and 299 deletions

2
go.mod
View file

@ -44,7 +44,7 @@ require (
github.com/miekg/dns v1.1.62
github.com/minio/minio-go/v7 v7.0.78
github.com/mitchellh/mapstructure v1.5.0
github.com/ncruces/go-sqlite3 v0.19.0
github.com/ncruces/go-sqlite3 v0.20.0
github.com/oklog/ulid v1.3.1
github.com/prometheus/client_golang v1.20.5
github.com/spf13/cobra v1.8.1

4
go.sum generated
View file

@ -434,8 +434,8 @@ github.com/moul/http2curl v1.0.0 h1:dRMWoAtb+ePxMlLkrCbAqh4TlPHXvoGUSQ323/9Zahs=
github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/ncruces/go-sqlite3 v0.19.0 h1:yebbD/cP8Gf+7nKoUin2ATjnqJK2VvyS30d3xsjRp5k=
github.com/ncruces/go-sqlite3 v0.19.0/go.mod h1:yL4ZNWGsr1/8pcLfpPW1RT1WFdvyeHonrgIwwi4rvkg=
github.com/ncruces/go-sqlite3 v0.20.0 h1:/nBLvYxj7sk9S6y57nmMFvoQ/KJtGo0pNi8J80s8oJU=
github.com/ncruces/go-sqlite3 v0.20.0/go.mod h1:yL4ZNWGsr1/8pcLfpPW1RT1WFdvyeHonrgIwwi4rvkg=
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=

View file

@ -76,7 +76,8 @@ func (m *Module) deleteDomainPermissionDraft(
apiutil.JSON(c, http.StatusOK, domainPerm)
}
// getDomainPermissionDraft gets a single domain permission (block or allow).
// getDomainPermissionDraft gets a single
// domain permission draft (block or allow).
func (m *Module) getDomainPermissionDraft(
c *gin.Context,
permType gtsmodel.DomainPermissionType,
@ -104,17 +105,10 @@ func (m *Module) getDomainPermissionDraft(
return
}
export, errWithCode := apiutil.ParseDomainPermissionDraftExport(c.Query(apiutil.DomainPermissionDraftExportKey), false)
if errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
domainPerm, errWithCode := m.processor.Admin().DomainPermissionDraftGet(
c.Request.Context(),
permType,
domainPermID,
export,
)
if errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)

View file

@ -80,7 +80,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch1() {
}
suite.Equal(`{
"uri": "http://localhost:8080",
"uri": "localhost:8080",
"account_domain": "localhost:8080",
"title": "Example Instance",
"description": "<p>Here's a fuller description of the GoToSocial testrig instance.</p><p>This instance is for testing purposes only. It doesn't federate at all. Go check out <a href=\"https://github.com/superseriousbusiness/gotosocial/tree/main/testrig\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">https://github.com/superseriousbusiness/gotosocial/tree/main/testrig</a> and <a href=\"https://github.com/superseriousbusiness/gotosocial/blob/main/CONTRIBUTING.md#testing\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">https://github.com/superseriousbusiness/gotosocial/blob/main/CONTRIBUTING.md#testing</a></p><p>Users on this instance:</p><ul><li><span class=\"h-card\"><a href=\"http://localhost:8080/@admin\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>admin</span></a></span> (admin!).</li><li><span class=\"h-card\"><a href=\"http://localhost:8080/@1happyturtle\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>1happyturtle</span></a></span> (posts about turtles, we don't know why).</li><li><span class=\"h-card\"><a href=\"http://localhost:8080/@the_mighty_zork\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>the_mighty_zork</span></a></span> (who knows).</li></ul><p>If you need to edit the models for the testrig, you can do so at <code>internal/testmodels.go</code>.</p>",
@ -221,7 +221,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch2() {
}
suite.Equal(`{
"uri": "http://localhost:8080",
"uri": "localhost:8080",
"account_domain": "localhost:8080",
"title": "Geoff's Instance",
"description": "<p>Here's a fuller description of the GoToSocial testrig instance.</p><p>This instance is for testing purposes only. It doesn't federate at all. Go check out <a href=\"https://github.com/superseriousbusiness/gotosocial/tree/main/testrig\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">https://github.com/superseriousbusiness/gotosocial/tree/main/testrig</a> and <a href=\"https://github.com/superseriousbusiness/gotosocial/blob/main/CONTRIBUTING.md#testing\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">https://github.com/superseriousbusiness/gotosocial/blob/main/CONTRIBUTING.md#testing</a></p><p>Users on this instance:</p><ul><li><span class=\"h-card\"><a href=\"http://localhost:8080/@admin\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>admin</span></a></span> (admin!).</li><li><span class=\"h-card\"><a href=\"http://localhost:8080/@1happyturtle\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>1happyturtle</span></a></span> (posts about turtles, we don't know why).</li><li><span class=\"h-card\"><a href=\"http://localhost:8080/@the_mighty_zork\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>the_mighty_zork</span></a></span> (who knows).</li></ul><p>If you need to edit the models for the testrig, you can do so at <code>internal/testmodels.go</code>.</p>",
@ -362,7 +362,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch3() {
}
suite.Equal(`{
"uri": "http://localhost:8080",
"uri": "localhost:8080",
"account_domain": "localhost:8080",
"title": "GoToSocial Testrig Instance",
"description": "<p>Here's a fuller description of the GoToSocial testrig instance.</p><p>This instance is for testing purposes only. It doesn't federate at all. Go check out <a href=\"https://github.com/superseriousbusiness/gotosocial/tree/main/testrig\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">https://github.com/superseriousbusiness/gotosocial/tree/main/testrig</a> and <a href=\"https://github.com/superseriousbusiness/gotosocial/blob/main/CONTRIBUTING.md#testing\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">https://github.com/superseriousbusiness/gotosocial/blob/main/CONTRIBUTING.md#testing</a></p><p>Users on this instance:</p><ul><li><span class=\"h-card\"><a href=\"http://localhost:8080/@admin\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>admin</span></a></span> (admin!).</li><li><span class=\"h-card\"><a href=\"http://localhost:8080/@1happyturtle\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>1happyturtle</span></a></span> (posts about turtles, we don't know why).</li><li><span class=\"h-card\"><a href=\"http://localhost:8080/@the_mighty_zork\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>the_mighty_zork</span></a></span> (who knows).</li></ul><p>If you need to edit the models for the testrig, you can do so at <code>internal/testmodels.go</code>.</p>",
@ -554,7 +554,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch6() {
}
suite.Equal(`{
"uri": "http://localhost:8080",
"uri": "localhost:8080",
"account_domain": "localhost:8080",
"title": "GoToSocial Testrig Instance",
"description": "<p>Here's a fuller description of the GoToSocial testrig instance.</p><p>This instance is for testing purposes only. It doesn't federate at all. Go check out <a href=\"https://github.com/superseriousbusiness/gotosocial/tree/main/testrig\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">https://github.com/superseriousbusiness/gotosocial/tree/main/testrig</a> and <a href=\"https://github.com/superseriousbusiness/gotosocial/blob/main/CONTRIBUTING.md#testing\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">https://github.com/superseriousbusiness/gotosocial/blob/main/CONTRIBUTING.md#testing</a></p><p>Users on this instance:</p><ul><li><span class=\"h-card\"><a href=\"http://localhost:8080/@admin\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>admin</span></a></span> (admin!).</li><li><span class=\"h-card\"><a href=\"http://localhost:8080/@1happyturtle\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>1happyturtle</span></a></span> (posts about turtles, we don't know why).</li><li><span class=\"h-card\"><a href=\"http://localhost:8080/@the_mighty_zork\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>the_mighty_zork</span></a></span> (who knows).</li></ul><p>If you need to edit the models for the testrig, you can do so at <code>internal/testmodels.go</code>.</p>",
@ -717,7 +717,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch8() {
}
suite.Equal(`{
"uri": "http://localhost:8080",
"uri": "localhost:8080",
"account_domain": "localhost:8080",
"title": "GoToSocial Testrig Instance",
"description": "<p>Here's a fuller description of the GoToSocial testrig instance.</p><p>This instance is for testing purposes only. It doesn't federate at all. Go check out <a href=\"https://github.com/superseriousbusiness/gotosocial/tree/main/testrig\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">https://github.com/superseriousbusiness/gotosocial/tree/main/testrig</a> and <a href=\"https://github.com/superseriousbusiness/gotosocial/blob/main/CONTRIBUTING.md#testing\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">https://github.com/superseriousbusiness/gotosocial/blob/main/CONTRIBUTING.md#testing</a></p><p>Users on this instance:</p><ul><li><span class=\"h-card\"><a href=\"http://localhost:8080/@admin\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>admin</span></a></span> (admin!).</li><li><span class=\"h-card\"><a href=\"http://localhost:8080/@1happyturtle\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>1happyturtle</span></a></span> (posts about turtles, we don't know why).</li><li><span class=\"h-card\"><a href=\"http://localhost:8080/@the_mighty_zork\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>the_mighty_zork</span></a></span> (who knows).</li></ul><p>If you need to edit the models for the testrig, you can do so at <code>internal/testmodels.go</code>.</p>",
@ -899,7 +899,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch9() {
}
suite.Equal(`{
"uri": "http://localhost:8080",
"uri": "localhost:8080",
"account_domain": "localhost:8080",
"title": "GoToSocial Testrig Instance",
"description": "<p>Here's a fuller description of the GoToSocial testrig instance.</p><p>This instance is for testing purposes only. It doesn't federate at all. Go check out <a href=\"https://github.com/superseriousbusiness/gotosocial/tree/main/testrig\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">https://github.com/superseriousbusiness/gotosocial/tree/main/testrig</a> and <a href=\"https://github.com/superseriousbusiness/gotosocial/blob/main/CONTRIBUTING.md#testing\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">https://github.com/superseriousbusiness/gotosocial/blob/main/CONTRIBUTING.md#testing</a></p><p>Users on this instance:</p><ul><li><span class=\"h-card\"><a href=\"http://localhost:8080/@admin\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>admin</span></a></span> (admin!).</li><li><span class=\"h-card\"><a href=\"http://localhost:8080/@1happyturtle\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>1happyturtle</span></a></span> (posts about turtles, we don't know why).</li><li><span class=\"h-card\"><a href=\"http://localhost:8080/@the_mighty_zork\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>the_mighty_zork</span></a></span> (who knows).</li></ul><p>If you need to edit the models for the testrig, you can do so at <code>internal/testmodels.go</code>.</p>",

View file

@ -74,6 +74,9 @@ func (c *Caches) Init() {
c.initConversationLastStatusIDs()
c.initDomainAllow()
c.initDomainBlock()
c.initDomainPermissionDraft()
c.initDomainPermissionSubscription()
c.initDomainPermissionIgnore()
c.initEmoji()
c.initEmojiCategory()
c.initFilter()

View file

@ -73,6 +73,9 @@ type DBCaches struct {
// DomainPermissionSubscription provides access to the domain permission subscription database cache.
DomainPermissionSubscription StructCache[*gtsmodel.DomainPermissionSubscription]
// DomainPermissionIgnore provides access to the domain permission ignore database cache.
DomainPermissionIgnore *domain.Cache
// Emoji provides access to the gtsmodel Emoji database cache.
Emoji StructCache[*gtsmodel.Emoji]
@ -617,6 +620,10 @@ func (c *Caches) initDomainPermissionSubscription() {
})
}
func (c *Caches) initDomainPermissionIgnore() {
c.DB.DomainPermissionIgnore = new(domain.Cache)
}
func (c *Caches) initEmoji() {
// Calculate maximum cache size.
cap := calculateResultCacheMax(

View file

@ -23,6 +23,7 @@
"net/url"
"slices"
"github.com/miekg/dns"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
@ -593,15 +594,15 @@ func (d *domainDB) getDomainPermissionSubscription(
keyParts ...any,
) (*gtsmodel.DomainPermissionSubscription, error) {
// Fetch perm subscription from database cache with loader callback.
permDraft, err := d.state.Caches.DB.DomainPermissionSubscription.LoadOne(
permSub, err := d.state.Caches.DB.DomainPermissionSubscription.LoadOne(
lookup,
// Only called if not cached.
func() (*gtsmodel.DomainPermissionSubscription, error) {
var permDraft gtsmodel.DomainPermissionSubscription
if err := dbQuery(&permDraft); err != nil {
var permSub gtsmodel.DomainPermissionSubscription
if err := dbQuery(&permSub); err != nil {
return nil, err
}
return &permDraft, nil
return &permSub, nil
},
keyParts...,
)
@ -611,21 +612,21 @@ func() (*gtsmodel.DomainPermissionSubscription, error) {
if gtscontext.Barebones(ctx) {
// No need to fully populate.
return permDraft, nil
return permSub, nil
}
if permDraft.CreatedByAccount == nil {
if permSub.CreatedByAccount == nil {
// Not set, fetch from database.
permDraft.CreatedByAccount, err = d.state.DB.GetAccountByID(
permSub.CreatedByAccount, err = d.state.DB.GetAccountByID(
gtscontext.SetBarebones(ctx),
permDraft.CreatedByAccountID,
permSub.CreatedByAccountID,
)
if err != nil {
return nil, gtserror.Newf("error populating created by account: %w", err)
}
}
return permDraft, nil
return permSub, nil
}
func (d *domainDB) GetDomainPermissionSubscriptionByID(
@ -635,10 +636,10 @@ func (d *domainDB) GetDomainPermissionSubscriptionByID(
return d.getDomainPermissionSubscription(
ctx,
"ID",
func(permDraft *gtsmodel.DomainPermissionSubscription) error {
func(permSub *gtsmodel.DomainPermissionSubscription) error {
return d.db.
NewSelect().
Model(permDraft).
Model(permSub).
Where("? = ?", bun.Ident("domain_permission_subscription.id"), id).
Scan(ctx)
},
@ -743,14 +744,14 @@ func (d *domainDB) GetDomainPermissionSubscriptions(
// Allocate return slice (will be at most len permSubIDs).
permSubs := make([]*gtsmodel.DomainPermissionSubscription, 0, len(permSubIDs))
for _, id := range permSubIDs {
permDraft, err := d.GetDomainPermissionSubscriptionByID(ctx, id)
permSub, err := d.GetDomainPermissionSubscriptionByID(ctx, id)
if err != nil {
log.Errorf(ctx, "error getting domain permission subscription %q: %v", id, err)
continue
}
// Append to return slice
permSubs = append(permSubs, permDraft)
permSubs = append(permSubs, permSub)
}
return permSubs, nil
@ -799,3 +800,235 @@ func (d *domainDB) DeleteDomainPermissionSubscription(
return nil
}
func (d *domainDB) PutDomainPermissionIgnore(
ctx context.Context,
ignore *gtsmodel.DomainPermissionIgnore,
) error {
// Normalize the domain as punycode
var err error
ignore.Domain, err = util.Punify(ignore.Domain)
if err != nil {
return err
}
// Attempt to store domain perm ignore in DB
if _, err := d.db.NewInsert().
Model(ignore).
Exec(ctx); err != nil {
return err
}
// Clear the domain perm ignore cache (for later reload)
d.state.Caches.DB.DomainPermissionIgnore.Clear()
return nil
}
func (d *domainDB) IsDomainPermissionIgnored(ctx context.Context, domain string) (bool, error) {
// Normalize the domain as punycode
domain, err := util.Punify(domain)
if err != nil {
return false, err
}
// Check if our host and given domain are equal
// or part of the same second-level domain; we
// always ignore such perms as creating blocks
// or allows in such cases may break things.
if dns.CompareDomainName(domain, config.GetHost()) >= 2 {
return true, nil
}
// Func to scan list of all
// ignored domain perms from DB.
loadF := func() ([]string, error) {
var domains []string
if err := d.db.
NewSelect().
Table("domain_ignores").
Column("domain").
Scan(ctx, &domains); err != nil {
return nil, err
}
return domains, nil
}
// Check the cache for a domain perm ignore,
// hydrating the cache with loadF if necessary.
return d.state.Caches.DB.DomainPermissionIgnore.Matches(domain, loadF)
}
func (d *domainDB) GetDomainPermissionIgnoreByID(
ctx context.Context,
id string,
) (*gtsmodel.DomainPermissionIgnore, error) {
ignore := new(gtsmodel.DomainPermissionIgnore)
q := d.db.
NewSelect().
Model(ignore).
Where("? = ?", bun.Ident("domain_permission_ignore.id"), id)
if err := q.Scan(ctx); err != nil {
return nil, err
}
if gtscontext.Barebones(ctx) {
// No need to fully populate.
return ignore, nil
}
if ignore.CreatedByAccount == nil {
// Not set, fetch from database.
var err error
ignore.CreatedByAccount, err = d.state.DB.GetAccountByID(
gtscontext.SetBarebones(ctx),
ignore.CreatedByAccountID,
)
if err != nil {
return nil, gtserror.Newf("error populating created by account: %w", err)
}
}
return ignore, nil
}
func (d *domainDB) GetDomainPermissionIgnores(
ctx context.Context,
permType *gtsmodel.DomainPermissionType,
page *paging.Page,
) (
[]*gtsmodel.DomainPermissionIgnore,
error,
) {
var (
// Get paging params.
minID = page.GetMin()
maxID = page.GetMax()
limit = page.GetLimit()
order = page.GetOrder()
// Make educated guess for slice size
ignoreIDs = make([]string, 0, limit)
)
q := d.db.
NewSelect().
TableExpr(
"? AS ?",
bun.Ident("domain_permission_ignores"),
bun.Ident("domain_permission_ignore"),
).
// Select only IDs from table
Column("domain_permission_ignore.id")
// Return only items with id
// lower than provided maxID.
if maxID != "" {
q = q.Where(
"? < ?",
bun.Ident("domain_permission_ignore.id"),
maxID,
)
}
// Return only items with id
// greater than provided minID.
if minID != "" {
q = q.Where(
"? > ?",
bun.Ident("domain_permission_ignore.id"),
minID,
)
}
// Return only items with
// given subscription ID.
if permType != nil {
q = q.Where(
"? = ?",
bun.Ident("domain_permission_ignore.permission_type"),
*permType,
)
}
if limit > 0 {
// Limit amount of
// items returned.
q = q.Limit(limit)
}
if order == paging.OrderAscending {
// Page up.
q = q.OrderExpr(
"? ASC",
bun.Ident("domain_permission_ignore.id"),
)
} else {
// Page down.
q = q.OrderExpr(
"? DESC",
bun.Ident("domain_permission_ignore.id"),
)
}
if err := q.Scan(ctx, &ignoreIDs); err != nil {
return nil, err
}
// Catch case of no items early
if len(ignoreIDs) == 0 {
return nil, db.ErrNoEntries
}
// If we're paging up, we still want items
// to be sorted by ID desc, so reverse slice.
if order == paging.OrderAscending {
slices.Reverse(ignoreIDs)
}
// Allocate return slice (will be at most len permSubIDs).
ignores := make([]*gtsmodel.DomainPermissionIgnore, 0, len(ignoreIDs))
for _, id := range ignoreIDs {
ignore, err := d.GetDomainPermissionIgnoreByID(ctx, id)
if err != nil {
log.Errorf(ctx, "error getting domain permission ignore %q: %v", id, err)
continue
}
// Append to return slice
ignores = append(ignores, ignore)
}
return ignores, nil
}
func (d *domainDB) DeleteDomainPermissionIgnore(
ctx context.Context,
id string,
) error {
// Delete the permSub from DB.
q := d.db.NewDelete().
TableExpr(
"? AS ?",
bun.Ident("domain_permission_ignores"),
bun.Ident("domain_permission_ignore"),
).
Where(
"? = ?",
bun.Ident("domain_permission_ignore.id"),
id,
)
_, err := q.Exec(ctx)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return err
}
// Clear the domain perm ignore cache (for later reload)
d.state.Caches.DB.DomainPermissionIgnore.Clear()
return nil
}

View file

@ -46,6 +46,15 @@ func init() {
return err
}
// Create `domain_permission_ignores`.
if _, err := tx.
NewCreateTable().
Model((*gtsmodel.DomainPermissionIgnore)(nil)).
IfNotExists().
Exec(ctx); err != nil {
return err
}
// Create indexes. Indices. Indie sexes.
for table, indexes := range map[string]map[string][]string{
"domain_permission_drafts": {

View file

@ -27,8 +27,52 @@ type DomainPermissionDraft struct {
Domain string `bun:",nullzero,notnull,unique:domain_permission_drafts_permission_type_domain_subscription_id_uniq"` // Domain to block or allow. Eg. 'whatever.com'.
CreatedByAccountID string `bun:"type:CHAR(26),nullzero,notnull"` // Account ID of the creator of this subscription.
CreatedByAccount *Account `bun:"-"` // Account corresponding to createdByAccountID.
PrivateComment string `bun:""` // Private comment on this perm, viewable to admins.
PublicComment string `bun:""` // Public comment on this perm, viewable (optionally) by everyone.
PrivateComment string `bun:",nullzero"` // Private comment on this perm, viewable to admins.
PublicComment string `bun:",nullzero"` // Public comment on this perm, viewable (optionally) by everyone.
Obfuscate *bool `bun:",nullzero,notnull,default:false"` // Obfuscate domain name when displaying it publicly.
SubscriptionID string `bun:"type:CHAR(26),nullzero,unique:domain_permission_drafts_permission_type_domain_subscription_id_uniq"` // ID of the subscription that created this draft, if any.
}
func (d *DomainPermissionDraft) GetID() string {
return d.ID
}
func (d *DomainPermissionDraft) GetCreatedAt() time.Time {
return d.CreatedAt
}
func (d *DomainPermissionDraft) GetUpdatedAt() time.Time {
return d.UpdatedAt
}
func (d *DomainPermissionDraft) GetDomain() string {
return d.Domain
}
func (d *DomainPermissionDraft) GetCreatedByAccountID() string {
return d.CreatedByAccountID
}
func (d *DomainPermissionDraft) GetCreatedByAccount() *Account {
return d.CreatedByAccount
}
func (d *DomainPermissionDraft) GetPrivateComment() string {
return d.PrivateComment
}
func (d *DomainPermissionDraft) GetPublicComment() string {
return d.PublicComment
}
func (d *DomainPermissionDraft) GetObfuscate() *bool {
return d.Obfuscate
}
func (d *DomainPermissionDraft) GetSubscriptionID() string {
return d.SubscriptionID
}
func (d *DomainPermissionDraft) GetType() DomainPermissionType {
return DomainPermissionBlock
}

View file

@ -0,0 +1,32 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package gtsmodel
import "time"
// DomainPermissionIgnore represents one domain that should be ignored
// when domain permission (ignores) are created from subscriptions.
type DomainPermissionIgnore struct {
ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // ID of this item in the database.
CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // Time when this item was created.
PermissionType DomainPermissionType `bun:",notnull,unique:domain_permission_ignores_permission_type_domain_uniq"` // Permission type of the ignore.
Domain string `bun:",nullzero,notnull,unique:domain_permission_ignores_permission_type_domain_uniq"` // Domain to ignore. Eg. 'whatever.com'.
CreatedByAccountID string `bun:"type:CHAR(26),nullzero,notnull"` // Account ID of the creator of this ignore.
CreatedByAccount *Account `bun:"-"` // Account corresponding to createdByAccountID.
PrivateComment string `bun:",nullzero"` // Private comment on this ignore, viewable to admins.
}

View file

@ -32,8 +32,7 @@
)
// apiDomainPerm is a cheeky shortcut for returning
// the API version of the given domain permission
// (*gtsmodel.DomainBlock or *gtsmodel.DomainAllow),
// the API version of the given domain permission,
// or an appropriate error if something goes wrong.
func (p *Processor) apiDomainPerm(
ctx context.Context,
@ -333,3 +332,28 @@ func (p *Processor) DomainPermissionGet(
return p.apiDomainPerm(ctx, domainPerm, export)
}
func (p *Processor) DomainPermissionDraftGet(
ctx context.Context,
permissionType gtsmodel.DomainPermissionType,
id string,
) (*apimodel.DomainPermission, gtserror.WithCode) {
permDraft, err := p.state.DB.GetDomainPermissionDraftByID(ctx, id)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
err = gtserror.Newf(
"db error getting domain %s draft %s: %w",
permissionType.String(), id, err,
)
return nil, gtserror.NewErrorInternalError(err)
}
if permDraft == nil {
err = fmt.Errorf(
"no domain %s draft exists with id %s",
permissionType.String(), id,
)
return nil, gtserror.NewErrorNotFound(err, err.Error())
}
return p.apiDomainPerm(ctx, permDraft, false)
}

View file

@ -1523,9 +1523,15 @@ func (c *Converter) InstanceRuleToAdminAPIRule(r *gtsmodel.Rule) *apimodel.Admin
// InstanceToAPIV1Instance converts a gts instance into its api equivalent for serving at /api/v1/instance
func (c *Converter) InstanceToAPIV1Instance(ctx context.Context, i *gtsmodel.Instance) (*apimodel.InstanceV1, error) {
domain := i.Domain
accDomain := config.GetAccountDomain()
if accDomain != "" {
domain = accDomain
}
instance := &apimodel.InstanceV1{
URI: i.URI,
AccountDomain: config.GetAccountDomain(),
URI: domain,
AccountDomain: accDomain,
Title: i.Title,
Description: i.Description,
DescriptionText: i.DescriptionText,
@ -1642,9 +1648,15 @@ func (c *Converter) InstanceToAPIV1Instance(ctx context.Context, i *gtsmodel.Ins
// InstanceToAPIV2Instance converts a gts instance into its api equivalent for serving at /api/v2/instance
func (c *Converter) InstanceToAPIV2Instance(ctx context.Context, i *gtsmodel.Instance) (*apimodel.InstanceV2, error) {
domain := i.Domain
accDomain := config.GetAccountDomain()
if accDomain != "" {
domain = accDomain
}
instance := &apimodel.InstanceV2{
Domain: i.Domain,
AccountDomain: config.GetAccountDomain(),
Domain: domain,
AccountDomain: accDomain,
Title: i.Title,
Version: config.GetSoftwareVersion(),
SourceURL: instanceSourceURL,

View file

@ -1918,7 +1918,7 @@ func (suite *InternalToFrontendTestSuite) TestInstanceV1ToFrontend() {
// FIXME: "rules" is empty from the database, because it's not fetched through db.GetInstance
suite.Equal(`{
"uri": "http://localhost:8080",
"uri": "localhost:8080",
"account_domain": "localhost:8080",
"title": "GoToSocial Testrig Instance",
"description": "\u003cp\u003eHere's a fuller description of the GoToSocial testrig instance.\u003c/p\u003e\u003cp\u003eThis instance is for testing purposes only. It doesn't federate at all. Go check out \u003ca href=\"https://github.com/superseriousbusiness/gotosocial/tree/main/testrig\" rel=\"nofollow noreferrer noopener\" target=\"_blank\"\u003ehttps://github.com/superseriousbusiness/gotosocial/tree/main/testrig\u003c/a\u003e and \u003ca href=\"https://github.com/superseriousbusiness/gotosocial/blob/main/CONTRIBUTING.md#testing\" rel=\"nofollow noreferrer noopener\" target=\"_blank\"\u003ehttps://github.com/superseriousbusiness/gotosocial/blob/main/CONTRIBUTING.md#testing\u003c/a\u003e\u003c/p\u003e\u003cp\u003eUsers on this instance:\u003c/p\u003e\u003cul\u003e\u003cli\u003e\u003cspan class=\"h-card\"\u003e\u003ca href=\"http://localhost:8080/@admin\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\"\u003e@\u003cspan\u003eadmin\u003c/span\u003e\u003c/a\u003e\u003c/span\u003e (admin!).\u003c/li\u003e\u003cli\u003e\u003cspan class=\"h-card\"\u003e\u003ca href=\"http://localhost:8080/@1happyturtle\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\"\u003e@\u003cspan\u003e1happyturtle\u003c/span\u003e\u003c/a\u003e\u003c/span\u003e (posts about turtles, we don't know why).\u003c/li\u003e\u003cli\u003e\u003cspan class=\"h-card\"\u003e\u003ca href=\"http://localhost:8080/@the_mighty_zork\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\"\u003e@\u003cspan\u003ethe_mighty_zork\u003c/span\u003e\u003c/a\u003e\u003c/span\u003e (who knows).\u003c/li\u003e\u003c/ul\u003e\u003cp\u003eIf you need to edit the models for the testrig, you can do so at \u003ccode\u003einternal/testmodels.go\u003c/code\u003e.\u003c/p\u003e",

View file

@ -1,4 +1,4 @@
# Go bindings to SQLite using Wazero
# Go bindings to SQLite using wazero
[![Go Reference](https://pkg.go.dev/badge/image)](https://pkg.go.dev/github.com/ncruces/go-sqlite3)
[![Go Report](https://goreportcard.com/badge/github.com/ncruces/go-sqlite3)](https://goreportcard.com/report/github.com/ncruces/go-sqlite3)
@ -41,45 +41,6 @@ db.QueryRow(`SELECT sqlite_version()`).Scan(&version)
- [`github.com/ncruces/go-sqlite3/gormlite`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/gormlite)
provides a [GORM](https://gorm.io) driver.
### Extensions
- [`github.com/ncruces/go-sqlite3/ext/array`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/array)
provides the [`array`](https://sqlite.org/carray.html) table-valued function.
- [`github.com/ncruces/go-sqlite3/ext/blobio`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/blobio)
simplifies [incremental BLOB I/O](https://sqlite.org/c3ref/blob_open.html).
- [`github.com/ncruces/go-sqlite3/ext/bloom`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/bloom)
provides a [Bloom filter](https://github.com/nalgeon/sqlean/issues/27#issuecomment-1002267134) virtual table.
- [`github.com/ncruces/go-sqlite3/ext/closure`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/closure)
provides a transitive closure virtual table.
- [`github.com/ncruces/go-sqlite3/ext/csv`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/csv)
reads [comma-separated values](https://sqlite.org/csv.html).
- [`github.com/ncruces/go-sqlite3/ext/fileio`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/fileio)
reads, writes and lists files.
- [`github.com/ncruces/go-sqlite3/ext/hash`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/hash)
provides cryptographic hash functions.
- [`github.com/ncruces/go-sqlite3/ext/lines`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/lines)
reads data [line-by-line](https://github.com/asg017/sqlite-lines).
- [`github.com/ncruces/go-sqlite3/ext/pivot`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/pivot)
creates [pivot tables](https://github.com/jakethaw/pivot_vtab).
- [`github.com/ncruces/go-sqlite3/ext/regexp`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/regexp)
provides regular expression functions.
- [`github.com/ncruces/go-sqlite3/ext/statement`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/statement)
creates [parameterized views](https://github.com/0x09/sqlite-statement-vtab).
- [`github.com/ncruces/go-sqlite3/ext/stats`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/stats)
provides [statistics](https://www.oreilly.com/library/view/sql-in-a/9780596155322/ch04s02.html) functions.
- [`github.com/ncruces/go-sqlite3/ext/unicode`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/unicode)
provides [Unicode aware](https://sqlite.org/src/dir/ext/icu) functions.
- [`github.com/ncruces/go-sqlite3/ext/uuid`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/uuid)
generates [UUIDs](https://en.wikipedia.org/wiki/Universally_unique_identifier).
- [`github.com/ncruces/go-sqlite3/ext/zorder`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/zorder)
maps multidimensional data to one dimension.
- [`github.com/ncruces/go-sqlite3/vfs/adiantum`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/adiantum)
wraps a VFS to offer encryption at rest.
- [`github.com/ncruces/go-sqlite3/vfs/memdb`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/memdb)
implements an in-memory VFS.
- [`github.com/ncruces/go-sqlite3/vfs/readervfs`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs)
implements a VFS for immutable databases.
### Advanced features
- [incremental BLOB I/O](https://sqlite.org/c3ref/blob_open.html)
@ -92,7 +53,11 @@ db.QueryRow(`SELECT sqlite_version()`).Scan(&version)
- [math functions](https://sqlite.org/lang_mathfunc.html)
- [full-text search](https://sqlite.org/fts5.html)
- [geospatial search](https://sqlite.org/geopoly.html)
- [Unicode support](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/unicode)
- [statistics functions](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/stats)
- [encryption at rest](vfs/adiantum/README.md)
- [many extensions](ext/README.md)
- [custom VFSes](vfs/README.md#custom-vfses)
- [and more…](embed/README.md)
### Caveats

View file

@ -253,6 +253,7 @@ func (b *Blob) Seek(offset int64, whence int) (int64, error) {
//
// https://sqlite.org/c3ref/blob_reopen.html
func (b *Blob) Reopen(row int64) error {
b.c.checkInterrupt(b.c.handle)
err := b.c.error(b.c.call("sqlite3_blob_reopen", uint64(b.handle), uint64(row)))
b.bytes = int64(b.c.call("sqlite3_blob_bytes", uint64(b.handle)))
b.offset = 0

View file

@ -2,10 +2,12 @@
import (
"context"
"strconv"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/vfs"
"github.com/tetratelabs/wazero/api"
)
// Config makes configuration changes to a database connection.
@ -15,8 +17,19 @@
//
// https://sqlite.org/c3ref/db_config.html
func (c *Conn) Config(op DBConfig, arg ...bool) (bool, error) {
if op < DBCONFIG_ENABLE_FKEY || op > DBCONFIG_REVERSE_SCANORDER {
return false, MISUSE
}
// We need to call sqlite3_db_config, a variadic function.
// We only support the `int int*` variants.
// The int is a three-valued bool: -1 queries, 0/1 sets false/true.
// The int* points to where new state will be written to.
// The vararg is a pointer to an array containing these arguments:
// an int and an int* pointing to that int.
defer c.arena.mark()()
argsPtr := c.arena.new(2 * ptrlen)
argsPtr := c.arena.new(intlen + ptrlen)
var flag int
switch {
@ -63,18 +76,23 @@ func logCallback(ctx context.Context, mod api.Module, _, iCode, zMsg uint32) {
// https://sqlite.org/c3ref/file_control.html
func (c *Conn) FileControl(schema string, op FcntlOpcode, arg ...any) (any, error) {
defer c.arena.mark()()
ptr := c.arena.new(max(ptrlen, intlen))
var schemaPtr uint32
if schema != "" {
schemaPtr = c.arena.string(schema)
}
var rc uint64
var res any
switch op {
default:
return nil, MISUSE
case FCNTL_RESET_CACHE:
r := c.call("sqlite3_file_control",
rc = c.call("sqlite3_file_control",
uint64(c.handle), uint64(schemaPtr),
uint64(op), 0)
return nil, c.error(r)
case FCNTL_PERSIST_WAL, FCNTL_POWERSAFE_OVERWRITE:
var flag int
@ -84,70 +102,69 @@ func (c *Conn) FileControl(schema string, op FcntlOpcode, arg ...any) (any, erro
case arg[0]:
flag = 1
}
ptr := c.arena.new(4)
util.WriteUint32(c.mod, ptr, uint32(flag))
r := c.call("sqlite3_file_control",
rc = c.call("sqlite3_file_control",
uint64(c.handle), uint64(schemaPtr),
uint64(op), uint64(ptr))
return util.ReadUint32(c.mod, ptr) != 0, c.error(r)
res = util.ReadUint32(c.mod, ptr) != 0
case FCNTL_CHUNK_SIZE:
ptr := c.arena.new(4)
util.WriteUint32(c.mod, ptr, uint32(arg[0].(int)))
r := c.call("sqlite3_file_control",
rc = c.call("sqlite3_file_control",
uint64(c.handle), uint64(schemaPtr),
uint64(op), uint64(ptr))
return nil, c.error(r)
case FCNTL_RESERVE_BYTES:
bytes := -1
if len(arg) > 0 {
bytes = arg[0].(int)
}
ptr := c.arena.new(4)
util.WriteUint32(c.mod, ptr, uint32(bytes))
r := c.call("sqlite3_file_control",
rc = c.call("sqlite3_file_control",
uint64(c.handle), uint64(schemaPtr),
uint64(op), uint64(ptr))
return int(util.ReadUint32(c.mod, ptr)), c.error(r)
res = int(util.ReadUint32(c.mod, ptr))
case FCNTL_DATA_VERSION:
ptr := c.arena.new(4)
r := c.call("sqlite3_file_control",
rc = c.call("sqlite3_file_control",
uint64(c.handle), uint64(schemaPtr),
uint64(op), uint64(ptr))
return util.ReadUint32(c.mod, ptr), c.error(r)
res = util.ReadUint32(c.mod, ptr)
case FCNTL_LOCKSTATE:
ptr := c.arena.new(4)
r := c.call("sqlite3_file_control",
rc = c.call("sqlite3_file_control",
uint64(c.handle), uint64(schemaPtr),
uint64(op), uint64(ptr))
return vfs.LockLevel(util.ReadUint32(c.mod, ptr)), c.error(r)
res = vfs.LockLevel(util.ReadUint32(c.mod, ptr))
case FCNTL_VFS_POINTER:
ptr := c.arena.new(4)
r := c.call("sqlite3_file_control",
rc = c.call("sqlite3_file_control",
uint64(c.handle), uint64(schemaPtr),
uint64(op), uint64(ptr))
const zNameOffset = 16
ptr = util.ReadUint32(c.mod, ptr)
ptr = util.ReadUint32(c.mod, ptr+zNameOffset)
name := util.ReadString(c.mod, ptr, _MAX_NAME)
return vfs.Find(name), c.error(r)
if rc == _OK {
const zNameOffset = 16
ptr = util.ReadUint32(c.mod, ptr)
ptr = util.ReadUint32(c.mod, ptr+zNameOffset)
name := util.ReadString(c.mod, ptr, _MAX_NAME)
res = vfs.Find(name)
}
case FCNTL_FILE_POINTER, FCNTL_JOURNAL_POINTER:
ptr := c.arena.new(4)
r := c.call("sqlite3_file_control",
rc = c.call("sqlite3_file_control",
uint64(c.handle), uint64(schemaPtr),
uint64(op), uint64(ptr))
const fileHandleOffset = 4
ptr = util.ReadUint32(c.mod, ptr)
ptr = util.ReadUint32(c.mod, ptr+fileHandleOffset)
return util.GetHandle(c.ctx, ptr), c.error(r)
if rc == _OK {
const fileHandleOffset = 4
ptr = util.ReadUint32(c.mod, ptr)
ptr = util.ReadUint32(c.mod, ptr+fileHandleOffset)
res = util.GetHandle(c.ctx, ptr)
}
}
return nil, MISUSE
if err := c.error(rc); err != nil {
return nil, err
}
return res, nil
}
// Limit allows the size of various constructs to be
@ -234,10 +251,10 @@ func traceCallback(ctx context.Context, mod api.Module, evt TraceEvent, pDB, pAr
return rc
}
// WalCheckpoint checkpoints a WAL database.
// WALCheckpoint checkpoints a WAL database.
//
// https://sqlite.org/c3ref/wal_checkpoint_v2.html
func (c *Conn) WalCheckpoint(schema string, mode CheckpointMode) (nLog, nCkpt int, err error) {
func (c *Conn) WALCheckpoint(schema string, mode CheckpointMode) (nLog, nCkpt int, err error) {
defer c.arena.mark()()
nLogPtr := c.arena.new(ptrlen)
nCkptPtr := c.arena.new(ptrlen)
@ -250,19 +267,19 @@ func (c *Conn) WalCheckpoint(schema string, mode CheckpointMode) (nLog, nCkpt in
return nLog, nCkpt, c.error(r)
}
// WalAutoCheckpoint configures WAL auto-checkpoints.
// WALAutoCheckpoint configures WAL auto-checkpoints.
//
// https://sqlite.org/c3ref/wal_autocheckpoint.html
func (c *Conn) WalAutoCheckpoint(pages int) error {
func (c *Conn) WALAutoCheckpoint(pages int) error {
r := c.call("sqlite3_wal_autocheckpoint", uint64(c.handle), uint64(pages))
return c.error(r)
}
// WalHook registers a callback function to be invoked
// WALHook registers a callback function to be invoked
// each time data is committed to a database in WAL mode.
//
// https://sqlite.org/c3ref/wal_hook.html
func (c *Conn) WalHook(cb func(db *Conn, schema string, pages int) error) {
func (c *Conn) WALHook(cb func(db *Conn, schema string, pages int) error) {
var enable uint64
if cb != nil {
enable = 1
@ -311,3 +328,46 @@ func (c *Conn) SoftHeapLimit(n int64) int64 {
func (c *Conn) HardHeapLimit(n int64) int64 {
return int64(c.call("sqlite3_hard_heap_limit64", uint64(n)))
}
// EnableChecksums enables checksums on a database.
//
// https://sqlite.org/cksumvfs.html
func (c *Conn) EnableChecksums(schema string) error {
r, err := c.FileControl(schema, FCNTL_RESERVE_BYTES)
if err != nil {
return err
}
if r == 8 {
// Correct value, enabled.
return nil
}
if r == 0 {
// Default value, enable.
_, err = c.FileControl(schema, FCNTL_RESERVE_BYTES, 8)
if err != nil {
return err
}
r, err = c.FileControl(schema, FCNTL_RESERVE_BYTES)
if err != nil {
return err
}
}
if r != 8 {
// Invalid value.
return util.ErrorString("sqlite3: reserve bytes must be 8, is: " + strconv.Itoa(r.(int)))
}
// VACUUM the database.
if schema != "" {
err = c.Exec(`VACUUM ` + QuoteIdentifier(schema))
} else {
err = c.Exec(`VACUUM`)
}
if err != nil {
return err
}
// Checkpoint the WAL.
_, _, err = c.WALCheckpoint(schema, CHECKPOINT_RESTART)
return err
}

View file

@ -8,9 +8,10 @@
"strings"
"time"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/vfs"
"github.com/tetratelabs/wazero/api"
)
// Conn is a database connection handle.
@ -204,6 +205,7 @@ func (c *Conn) PrepareFlags(sql string, flags PrepareFlag) (stmt *Stmt, tail str
tailPtr := c.arena.new(ptrlen)
sqlPtr := c.arena.string(sql)
c.checkInterrupt(c.handle)
r := c.call("sqlite3_prepare_v3", uint64(c.handle),
uint64(sqlPtr), uint64(len(sql)+1), uint64(flags),
uint64(stmtPtr), uint64(tailPtr))
@ -457,8 +459,8 @@ func busyCallback(ctx context.Context, mod api.Module, pDB uint32, count int32)
// https://sqlite.org/c3ref/db_status.html
func (c *Conn) Status(op DBStatus, reset bool) (current, highwater int, err error) {
defer c.arena.mark()()
hiPtr := c.arena.new(4)
curPtr := c.arena.new(4)
hiPtr := c.arena.new(intlen)
curPtr := c.arena.new(intlen)
var i uint64
if reset {
@ -484,8 +486,8 @@ func (c *Conn) TableColumnMetadata(schema, table, column string) (declType, coll
declTypePtr := c.arena.new(ptrlen)
collSeqPtr := c.arena.new(ptrlen)
notNullPtr := c.arena.new(ptrlen)
primaryKeyPtr := c.arena.new(ptrlen)
autoIncPtr := c.arena.new(ptrlen)
primaryKeyPtr := c.arena.new(ptrlen)
if schema != "" {
schemaPtr = c.arena.string(schema)
}
@ -519,10 +521,3 @@ func (c *Conn) stmtsIter(yield func(*Stmt) bool) {
}
}
}
// DriverConn is implemented by the SQLite [database/sql] driver connection.
//
// Deprecated: use [github.com/ncruces/go-sqlite3/driver.Conn] instead.
type DriverConn interface {
Raw() *Conn
}

View file

@ -13,6 +13,7 @@
_MAX_FUNCTION_ARG = 100
ptrlen = 4
intlen = 4
)
// ErrorCode is a result code that [Error.Code] might return.
@ -177,6 +178,7 @@
DETERMINISTIC FunctionFlag = 0x000000800
DIRECTONLY FunctionFlag = 0x000080000
INNOCUOUS FunctionFlag = 0x000200000
SELFORDER1 FunctionFlag = 0x002000000
// SUBTYPE FunctionFlag = 0x000100000
// RESULT_SUBTYPE FunctionFlag = 0x001000000
)
@ -245,6 +247,7 @@
DBCONFIG_TRUSTED_SCHEMA DBConfig = 1017
DBCONFIG_STMT_SCANSTATUS DBConfig = 1018
DBCONFIG_REVERSE_SCANORDER DBConfig = 1019
// DBCONFIG_MAX DBConfig = 1019
)
// FcntlOpcode are the available opcodes for [Conn.FileControl].

View file

@ -1,6 +1,6 @@
# Embeddable Wasm build of SQLite
This folder includes an embeddable Wasm build of SQLite 3.46.1 for use with
This folder includes an embeddable Wasm build of SQLite 3.47.0 for use with
[`github.com/ncruces/go-sqlite3`](https://pkg.go.dev/github.com/ncruces/go-sqlite3).
The following optional features are compiled in:
@ -36,6 +36,6 @@ You can use your own custom build of SQLite.
Examples of custom builds of SQLite are:
- [`github.com/ncruces/go-sqlite3/embed/bcw2`](https://github.com/ncruces/go-sqlite3/tree/main/embed/bcw2)
built from a branch supporting [`BEGIN CONCURRENT`](https://sqlite.org/src/doc/begin-concurrent/doc/begin_concurrent.md)
and [Wal2](https://www.sqlite.org/cgi/src/doc/wal2/doc/wal2.md).
and [Wal2](https://sqlite.org/cgi/src/doc/wal2/doc/wal2.md).
- [`github.com/asg017/sqlite-vec-go-bindings/ncruces`](https://github.com/asg017/sqlite-vec-go-bindings)
which includes the [`sqlite-vec`](https://github.com/asg017/sqlite-vec) vector search extension.

Binary file not shown.

View file

@ -4,8 +4,9 @@
"context"
"sync"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
)
// CollationNeeded registers a callback to be invoked

View file

@ -4,6 +4,21 @@
import "github.com/tetratelabs/wazero/experimental"
func Virtual(cap, max uint64) experimental.LinearMemory {
return Slice(cap, max)
func NewMemory(cap, max uint64) experimental.LinearMemory {
return &sliceMemory{make([]byte, 0, cap)}
}
type sliceMemory struct {
buf []byte
}
func (b *sliceMemory) Free() {}
func (b *sliceMemory) Reallocate(size uint64) []byte {
if cap := uint64(cap(b.buf)); size > cap {
b.buf = append(b.buf[:cap], make([]byte, size-cap)...)
} else {
b.buf = b.buf[:size]
}
return b.buf
}

View file

@ -1,24 +0,0 @@
//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
package alloc
import "github.com/tetratelabs/wazero/experimental"
func Slice(cap, _ uint64) experimental.LinearMemory {
return &sliceMemory{make([]byte, 0, cap)}
}
type sliceMemory struct {
buf []byte
}
func (b *sliceMemory) Free() {}
func (b *sliceMemory) Reallocate(size uint64) []byte {
if cap := uint64(cap(b.buf)); size > cap {
b.buf = append(b.buf[:cap], make([]byte, size-cap)...)
} else {
b.buf = b.buf[:size]
}
return b.buf
}

View file

@ -9,7 +9,7 @@
"golang.org/x/sys/unix"
)
func Virtual(_, max uint64) experimental.LinearMemory {
func NewMemory(_, max uint64) experimental.LinearMemory {
// Round up to the page size.
rnd := uint64(unix.Getpagesize() - 1)
max = (max + rnd) &^ rnd

View file

@ -11,7 +11,7 @@
"golang.org/x/sys/windows"
)
func Virtual(_, max uint64) experimental.LinearMemory {
func NewMemory(_, max uint64) experimental.LinearMemory {
// Round up to the page size.
rnd := uint64(windows.Getpagesize() - 1)
max = (max + rnd) &^ rnd

View file

@ -1,22 +0,0 @@
package util
import "strings"
func ParseBool(s string) (b, ok bool) {
if len(s) == 0 {
return false, false
}
if s[0] == '0' {
return false, true
}
if '1' <= s[0] && s[0] <= '9' {
return true, true
}
switch strings.ToLower(s) {
case "true", "yes", "on":
return true, true
case "false", "no", "off":
return false, true
}
return false, false
}

View file

@ -0,0 +1,29 @@
package util
import "math"
func abs(n int) int {
if n < 0 {
return -n
}
return n
}
func GCD(m, n int) int {
for n != 0 {
m, n = n, m%n
}
return abs(m)
}
func LCM(m, n int) int {
if n == 0 {
return 0
}
return abs(n) * (abs(m) / GCD(m, n))
}
// https://developer.nvidia.com/blog/lerp-faster-cuda/
func Lerp(v0, v1, t float64) float64 {
return math.FMA(t, v1, math.FMA(-t, v0, v0))
}

View file

@ -1,4 +1,4 @@
//go:build unix && (amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
//go:build unix && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
package util
@ -7,17 +7,10 @@
"os"
"unsafe"
"github.com/ncruces/go-sqlite3/internal/alloc"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/experimental"
"golang.org/x/sys/unix"
)
func withAllocator(ctx context.Context) context.Context {
return experimental.WithMemoryAllocator(ctx,
experimental.MemoryAllocatorFunc(alloc.Virtual))
}
type mmapState struct {
regions []*MappedRegion
}

View file

@ -1,22 +1,5 @@
//go:build !unix || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
//go:build !unix || !(386 || arm || amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
package util
import (
"context"
"github.com/ncruces/go-sqlite3/internal/alloc"
"github.com/tetratelabs/wazero/experimental"
)
type mmapState struct{}
func withAllocator(ctx context.Context) context.Context {
return experimental.WithMemoryAllocator(ctx,
experimental.MemoryAllocatorFunc(func(cap, max uint64) experimental.LinearMemory {
if cap == max {
return alloc.Virtual(cap, max)
}
return alloc.Slice(cap, max)
}))
}

View file

@ -4,6 +4,8 @@
"context"
"github.com/tetratelabs/wazero/experimental"
"github.com/ncruces/go-sqlite3/internal/alloc"
)
type moduleKey struct{}
@ -14,7 +16,7 @@ type moduleState struct {
func NewContext(ctx context.Context) context.Context {
state := new(moduleState)
ctx = withAllocator(ctx)
ctx = experimental.WithMemoryAllocator(ctx, experimental.MemoryAllocatorFunc(alloc.NewMemory))
ctx = experimental.WithCloseNotifier(ctx, state)
ctx = context.WithValue(ctx, moduleKey{}, state)
return ctx

View file

@ -9,11 +9,12 @@
"sync"
"unsafe"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/vfs"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/experimental"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/vfs"
)
// Configure SQLite Wasm.
@ -49,10 +50,15 @@ func compileSQLite() {
cfg := RuntimeConfig
if cfg == nil {
cfg = wazero.NewRuntimeConfig()
if bits.UintSize >= 64 {
cfg = cfg.WithMemoryLimitPages(4096) // 256MB
} else {
cfg = cfg.WithMemoryLimitPages(512) // 32MB
}
}
cfg = cfg.WithCoreFeatures(api.CoreFeaturesV2 | experimental.CoreFeaturesThreads)
instance.runtime = wazero.NewRuntimeWithConfig(ctx,
cfg.WithCoreFeatures(api.CoreFeaturesV2|experimental.CoreFeaturesThreads))
instance.runtime = wazero.NewRuntimeWithConfig(ctx, cfg)
env := instance.runtime.NewHostModuleBuilder("env")
env = vfs.ExportHostFunctions(env)

View file

@ -8,8 +8,9 @@
"strconv"
"strings"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
)
// Txn is an in-progress database transaction.
@ -142,7 +143,7 @@ func (c *Conn) Savepoint() Savepoint {
// Names can be reused, but this makes catching bugs more likely.
name = QuoteIdentifier(name + "_" + strconv.Itoa(int(rand.Int31())))
err := c.txnExecInterrupted("SAVEPOINT " + name)
err := c.txnExecInterrupted(`SAVEPOINT ` + name)
if err != nil {
panic(err)
}
@ -186,7 +187,7 @@ func (s Savepoint) Release(errp *error) {
if s.c.GetAutocommit() { // There is nothing to commit.
return
}
*errp = s.c.Exec("RELEASE " + s.name)
*errp = s.c.Exec(`RELEASE ` + s.name)
if *errp == nil {
return
}
@ -198,8 +199,7 @@ func (s Savepoint) Release(errp *error) {
return
}
// ROLLBACK and RELEASE even if interrupted.
err := s.c.txnExecInterrupted("ROLLBACK TO " +
s.name + "; RELEASE " + s.name)
err := s.c.txnExecInterrupted(`ROLLBACK TO ` + s.name + `; RELEASE ` + s.name)
if err != nil {
panic(err)
}
@ -212,7 +212,7 @@ func (s Savepoint) Release(errp *error) {
// https://sqlite.org/lang_transaction.html
func (s Savepoint) Rollback() error {
// ROLLBACK even if interrupted.
return s.c.txnExecInterrupted("ROLLBACK TO " + s.name)
return s.c.txnExecInterrupted(`ROLLBACK TO ` + s.name)
}
func (c *Conn) txnExecInterrupted(sql string) error {

View file

@ -1,2 +1,2 @@
// Package osutil implements operating system utility functions.
// Package osutil implements operating system utilities.
package osutil

View file

@ -0,0 +1,9 @@
# SQLite utility functions
This package implements assorted SQLite utilities
useful to extension writers.
It also wraps a [parser](https://github.com/marcobambini/sqlite-createtable-parser)
for the [`CREATE`](https://sqlite.org/lang_createtable.html) and
[`ALTER TABLE`](https://sqlite.org/lang_altertable.html) commands,
created by [Marco Bambini](https://github.com/marcobambini).

View file

@ -0,0 +1,65 @@
package sql3util
import "strings"
// NamedArg splits an named arg into a key and value,
// around an equals sign.
// Spaces are trimmed around both key and value.
func NamedArg(arg string) (key, val string) {
key, val, _ = strings.Cut(arg, "=")
key = strings.TrimSpace(key)
val = strings.TrimSpace(val)
return
}
// Unquote unquotes a string.
//
// https://sqlite.org/lang_keywords.html
func Unquote(val string) string {
if len(val) < 2 {
return val
}
fst := val[0]
lst := val[len(val)-1]
rst := val[1 : len(val)-1]
if fst == '[' && lst == ']' {
return rst
}
if fst != lst {
return val
}
var old, new string
switch fst {
default:
return val
case '`':
old, new = "``", "`"
case '"':
old, new = `""`, `"`
case '\'':
old, new = `''`, `'`
}
return strings.ReplaceAll(rst, old, new)
}
// ParseBool parses a boolean.
//
// https://sqlite.org/pragma.html#syntax
func ParseBool(s string) (b, ok bool) {
if len(s) == 0 {
return false, false
}
if s[0] == '0' {
return false, true
}
if '1' <= s[0] && s[0] <= '9' {
return true, true
}
switch strings.ToLower(s) {
case "true", "yes", "on":
return true, true
case "false", "no", "off":
return false, true
}
return false, false
}

View file

@ -0,0 +1,61 @@
package sql3util
const (
_NONE = iota
_MEMORY
_SYNTAX
_UNSUPPORTEDSQL
)
type ConflictClause uint32
const (
CONFLICT_NONE ConflictClause = iota
CONFLICT_ROLLBACK
CONFLICT_ABORT
CONFLICT_FAIL
CONFLICT_IGNORE
CONFLICT_REPLACE
)
type OrderClause uint32
const (
ORDER_NONE OrderClause = iota
ORDER_ASC
ORDER_DESC
)
type FKAction uint32
const (
FKACTION_NONE FKAction = iota
FKACTION_SETNULL
FKACTION_SETDEFAULT
FKACTION_CASCADE
FKACTION_RESTRICT
FKACTION_NOACTION
)
type FKDefType uint32
const (
DEFTYPE_NONE FKDefType = iota
DEFTYPE_DEFERRABLE
DEFTYPE_DEFERRABLE_INITIALLY_DEFERRED
DEFTYPE_DEFERRABLE_INITIALLY_IMMEDIATE
DEFTYPE_NOTDEFERRABLE
DEFTYPE_NOTDEFERRABLE_INITIALLY_DEFERRED
DEFTYPE_NOTDEFERRABLE_INITIALLY_IMMEDIATE
)
type StatementType uint32
const (
CREATE_UNKNOWN StatementType = iota
CREATE_TABLE
ALTER_RENAME_TABLE
ALTER_RENAME_COLUMN
ALTER_ADD_COLUMN
ALTER_DROP_COLUMN
)

View file

@ -0,0 +1,210 @@
package sql3util
import (
"context"
_ "embed"
"sync"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
)
const (
errp = 4
sqlp = 8
)
var (
//go:embed parse/sql3parse_table.wasm
binary []byte
once sync.Once
runtime wazero.Runtime
compiled wazero.CompiledModule
)
// ParseTable parses a [CREATE] or [ALTER TABLE] command.
//
// [CREATE]: https://sqlite.org/lang_createtable.html
// [ALTER TABLE]: https://sqlite.org/lang_altertable.html
func ParseTable(sql string) (_ *Table, err error) {
once.Do(func() {
ctx := context.Background()
cfg := wazero.NewRuntimeConfigInterpreter()
runtime = wazero.NewRuntimeWithConfig(ctx, cfg)
compiled, err = runtime.CompileModule(ctx, binary)
})
if err != nil {
return nil, err
}
ctx := context.Background()
mod, err := runtime.InstantiateModule(ctx, compiled, wazero.NewModuleConfig().WithName(""))
if err != nil {
return nil, err
}
defer mod.Close(ctx)
if buf, ok := mod.Memory().Read(sqlp, uint32(len(sql))); ok {
copy(buf, sql)
}
stack := [...]uint64{sqlp, uint64(len(sql)), errp}
err = mod.ExportedFunction("sql3parse_table").CallWithStack(ctx, stack[:])
if err != nil {
return nil, err
}
c, _ := mod.Memory().ReadUint32Le(errp)
switch c {
case _MEMORY:
panic(util.OOMErr)
case _SYNTAX:
return nil, util.ErrorString("sql3parse: invalid syntax")
case _UNSUPPORTEDSQL:
return nil, util.ErrorString("sql3parse: unsupported SQL")
}
var tab Table
tab.load(mod, uint32(stack[0]), sql)
return &tab, nil
}
// Table holds metadata about a table.
type Table struct {
Name string
Schema string
Comment string
IsTemporary bool
IsIfNotExists bool
IsWithoutRowID bool
IsStrict bool
Columns []Column
Type StatementType
CurrentName string
NewName string
}
func (t *Table) load(mod api.Module, ptr uint32, sql string) {
t.Name = loadString(mod, ptr+0, sql)
t.Schema = loadString(mod, ptr+8, sql)
t.Comment = loadString(mod, ptr+16, sql)
t.IsTemporary = loadBool(mod, ptr+24)
t.IsIfNotExists = loadBool(mod, ptr+25)
t.IsWithoutRowID = loadBool(mod, ptr+26)
t.IsStrict = loadBool(mod, ptr+27)
t.Columns = loadSlice(mod, ptr+28, func(ptr uint32, res *Column) {
p, _ := mod.Memory().ReadUint32Le(ptr)
res.load(mod, p, sql)
})
t.Type = loadEnum[StatementType](mod, ptr+44)
t.CurrentName = loadString(mod, ptr+48, sql)
t.NewName = loadString(mod, ptr+56, sql)
}
// Column holds metadata about a column.
type Column struct {
Name string
Type string
Length string
ConstraintName string
Comment string
IsPrimaryKey bool
IsAutoIncrement bool
IsNotNull bool
IsUnique bool
PKOrder OrderClause
PKConflictClause ConflictClause
NotNullConflictClause ConflictClause
UniqueConflictClause ConflictClause
CheckExpr string
DefaultExpr string
CollateName string
ForeignKeyClause *ForeignKey
}
func (c *Column) load(mod api.Module, ptr uint32, sql string) {
c.Name = loadString(mod, ptr+0, sql)
c.Type = loadString(mod, ptr+8, sql)
c.Length = loadString(mod, ptr+16, sql)
c.ConstraintName = loadString(mod, ptr+24, sql)
c.Comment = loadString(mod, ptr+32, sql)
c.IsPrimaryKey = loadBool(mod, ptr+40)
c.IsAutoIncrement = loadBool(mod, ptr+41)
c.IsNotNull = loadBool(mod, ptr+42)
c.IsUnique = loadBool(mod, ptr+43)
c.PKOrder = loadEnum[OrderClause](mod, ptr+44)
c.PKConflictClause = loadEnum[ConflictClause](mod, ptr+48)
c.NotNullConflictClause = loadEnum[ConflictClause](mod, ptr+52)
c.UniqueConflictClause = loadEnum[ConflictClause](mod, ptr+56)
c.CheckExpr = loadString(mod, ptr+60, sql)
c.DefaultExpr = loadString(mod, ptr+68, sql)
c.CollateName = loadString(mod, ptr+76, sql)
if ptr, _ := mod.Memory().ReadUint32Le(ptr + 84); ptr != 0 {
c.ForeignKeyClause = &ForeignKey{}
c.ForeignKeyClause.load(mod, ptr, sql)
}
}
type ForeignKey struct {
Table string
Columns []string
OnDelete FKAction
OnUpdate FKAction
Match string
Deferrable FKDefType
}
func (f *ForeignKey) load(mod api.Module, ptr uint32, sql string) {
f.Table = loadString(mod, ptr+0, sql)
f.Columns = loadSlice(mod, ptr+8, func(ptr uint32, res *string) {
*res = loadString(mod, ptr, sql)
})
f.OnDelete = loadEnum[FKAction](mod, ptr+16)
f.OnUpdate = loadEnum[FKAction](mod, ptr+20)
f.Match = loadString(mod, ptr+24, sql)
f.Deferrable = loadEnum[FKDefType](mod, ptr+32)
}
func loadString(mod api.Module, ptr uint32, sql string) string {
off, _ := mod.Memory().ReadUint32Le(ptr + 0)
if off == 0 {
return ""
}
len, _ := mod.Memory().ReadUint32Le(ptr + 4)
return sql[off-sqlp : off+len-sqlp]
}
func loadSlice[T any](mod api.Module, ptr uint32, fn func(uint32, *T)) []T {
ref, _ := mod.Memory().ReadUint32Le(ptr + 4)
if ref == 0 {
return nil
}
len, _ := mod.Memory().ReadUint32Le(ptr + 0)
res := make([]T, len)
for i := range res {
fn(ref, &res[i])
ref += 4
}
return res
}
func loadEnum[T ~uint32](mod api.Module, ptr uint32) T {
val, _ := mod.Memory().ReadUint32Le(ptr)
return T(val)
}
func loadBool(mod api.Module, ptr uint32) bool {
val, _ := mod.Memory().ReadByte(ptr)
return val != 0
}

Binary file not shown.

View file

@ -0,0 +1,9 @@
// Package sql3util implements SQLite utilities.
package sql3util
// ValidPageSize returns true if s is a valid page size.
//
// https://sqlite.org/fileformat.html#pages
func ValidPageSize(s int) bool {
return 512 <= s && s <= 65536 && s&(s-1) == 0
}

View file

@ -4,7 +4,7 @@ This package implements the SQLite [OS Interface](https://sqlite.org/vfs.html) (
It replaces the default SQLite VFS with a **pure Go** implementation,
and exposes [interfaces](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#VFS)
that should allow you to implement your own custom VFSes.
that should allow you to implement your own [custom VFSes](#custom-vfses).
Since it is a from scratch reimplementation,
there are naturally some ways it deviates from the original.
@ -16,12 +16,12 @@ The main differences are [file locking](#file-locking) and [WAL mode](#write-ahe
POSIX advisory locks, which SQLite uses on Unix, are
[broken by design](https://github.com/sqlite/sqlite/blob/b74eb0/src/os_unix.c#L1073-L1161).
On Linux and macOS, this module uses
On Linux and macOS, this package uses
[OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html)
to synchronize access to database files.
OFD locks are fully compatible with POSIX advisory locks.
This module can also use
This package can also use
[BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2),
albeit with reduced concurrency (`BEGIN IMMEDIATE` behaves like `BEGIN EXCLUSIVE`).
On BSD, macOS, and illumos, BSD locks are fully compatible with POSIX advisory locks;
@ -30,7 +30,7 @@ elsewhere, they are very likely broken.
BSD locks are the default on BSD and illumos,
but you can opt into them with the `sqlite3_flock` build tag.
On Windows, this module uses `LockFileEx` and `UnlockFileEx`,
On Windows, this package uses `LockFileEx` and `UnlockFileEx`,
like SQLite.
Otherwise, file locking is not supported, and you must use
@ -46,18 +46,14 @@ to check if your build supports file locking.
### Write-Ahead Logging
On 64-bit little-endian Unix, this module uses `mmap` to implement
On little-endian Unix, this package uses `mmap` to implement
[shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index),
like SQLite.
To allow `mmap` to work, each connection needs to reserve up to 4GB of address space.
To limit the address space each connection reserves,
use [`WithMemoryLimitPages`](../tests/testcfg/testcfg.go).
With [BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2)
a WAL database can only be accessed by a single proccess.
Other processes that attempt to access a database locked with BSD locks,
will fail with the `SQLITE_PROTOCOL` error code.
will fail with the [`SQLITE_PROTOCOL`](https://sqlite.org/rescode.html#protocol) error code.
Otherwise, [WAL support is limited](https://sqlite.org/wal.html#noshm),
and `EXCLUSIVE` locking mode must be set to create, read, and write WAL databases.
@ -71,9 +67,22 @@ to check if your build supports shared memory.
### Batch-Atomic Write
On 64-bit Linux, this module supports [batch-atomic writes](https://sqlite.org/cgi/src/technote/714)
On 64-bit Linux, this package supports
[batch-atomic writes](https://sqlite.org/cgi/src/technote/714)
on the F2FS filesystem.
### Checksums
This package can be [configured](https://pkg.go.dev/github.com/ncruces/go-sqlite3#Conn.EnableChecksums)
to add an 8-byte checksum to the end of every page in an SQLite database.
The checksum is added as each page is written
and verified as each page is read.\
The checksum is intended to help detect database corruption
caused by random bit-flips in the mass storage device.
The implementation is compatible with SQLite's
[Checksum VFS Shim](https://sqlite.org/cksumvfs.html).
### Build Tags
The VFS can be customized with a few build tags:
@ -90,3 +99,14 @@ The VFS can be customized with a few build tags:
> [`unix-flock` VFS](https://sqlite.org/compile.html#enable_locking_style).
> If incompatible file locking is used, accessing databases concurrently with
> _other_ SQLite libraries will eventually corrupt data.
### Custom VFSes
- [`github.com/ncruces/go-sqlite3/vfs/adiantum`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/adiantum)
wraps a VFS to offer encryption at rest.
- [`github.com/ncruces/go-sqlite3/vfs/memdb`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/memdb)
implements an in-memory VFS.
- [`github.com/ncruces/go-sqlite3/vfs/readervfs`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs)
implements a VFS for immutable databases.
- [`github.com/ncruces/go-sqlite3/vfs/xts`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/xts)
wraps a VFS to offer encryption at rest.

View file

@ -49,6 +49,13 @@ type File interface {
DeviceCharacteristics() DeviceCharacteristic
}
// FileUnwrap should be implemented by a File
// that wraps another File implementation.
type FileUnwrap interface {
File
Unwrap() File
}
// FileLockState extends File to implement the
// SQLITE_FCNTL_LOCKSTATE file control opcode.
//
@ -58,6 +65,26 @@ type FileLockState interface {
LockState() LockLevel
}
// FilePersistentWAL extends File to implement the
// SQLITE_FCNTL_PERSIST_WAL file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpersistwal
type FilePersistentWAL interface {
File
PersistentWAL() bool
SetPersistentWAL(bool)
}
// FilePowersafeOverwrite extends File to implement the
// SQLITE_FCNTL_POWERSAFE_OVERWRITE file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpowersafeoverwrite
type FilePowersafeOverwrite interface {
File
PowersafeOverwrite() bool
SetPowersafeOverwrite(bool)
}
// FileChunkSize extends File to implement the
// SQLITE_FCNTL_CHUNK_SIZE file control opcode.
//
@ -94,26 +121,6 @@ type FileOverwrite interface {
Overwrite() error
}
// FilePersistentWAL extends File to implement the
// SQLITE_FCNTL_PERSIST_WAL file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpersistwal
type FilePersistentWAL interface {
File
PersistentWAL() bool
SetPersistentWAL(bool)
}
// FilePowersafeOverwrite extends File to implement the
// SQLITE_FCNTL_POWERSAFE_OVERWRITE file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpowersafeoverwrite
type FilePowersafeOverwrite interface {
File
PowersafeOverwrite() bool
SetPowersafeOverwrite(bool)
}
// FileCommitPhaseTwo extends File to implement the
// SQLITE_FCNTL_COMMIT_PHASETWO file control opcode.
//
@ -135,15 +142,6 @@ type FileBatchAtomicWrite interface {
RollbackAtomicWrite() error
}
// FilePragma extends File to implement the
// SQLITE_FCNTL_PRAGMA file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpragma
type FilePragma interface {
File
Pragma(name, value string) (string, error)
}
// FileCheckpoint extends File to implement the
// SQLITE_FCNTL_CKPT_START and SQLITE_FCNTL_CKPT_DONE
// file control opcodes.
@ -151,8 +149,17 @@ type FilePragma interface {
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlckptstart
type FileCheckpoint interface {
File
CheckpointDone() error
CheckpointStart() error
CheckpointStart()
CheckpointDone()
}
// FilePragma extends File to implement the
// SQLITE_FCNTL_PRAGMA file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpragma
type FilePragma interface {
File
Pragma(name, value string) (string, error)
}
// FileSharedMemory extends File to possibly implement
@ -171,5 +178,16 @@ type SharedMemory interface {
shmMap(context.Context, api.Module, int32, int32, bool) (uint32, _ErrorCode)
shmLock(int32, int32, _ShmFlag) _ErrorCode
shmUnmap(bool)
shmBarrier()
io.Closer
}
type blockingSharedMemory interface {
SharedMemory
shmEnableBlocking(block bool)
}
type fileControl interface {
File
fileControl(ctx context.Context, mod api.Module, op _FcntlOpcode, pArg uint32) _ErrorCode
}

149
vendor/github.com/ncruces/go-sqlite3/vfs/cksm.go generated vendored Normal file
View file

@ -0,0 +1,149 @@
package vfs
import (
"bytes"
"context"
_ "embed"
"encoding/binary"
"strconv"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/util/sql3util"
)
func cksmWrapFile(name *Filename, flags OpenFlag, file File) File {
// Checksum only main databases and WALs.
if flags&(OPEN_MAIN_DB|OPEN_WAL) == 0 {
return file
}
cksm := cksmFile{File: file}
if flags&OPEN_WAL != 0 {
main, _ := name.DatabaseFile().(cksmFile)
cksm.cksmFlags = main.cksmFlags
} else {
cksm.cksmFlags = new(cksmFlags)
cksm.isDB = true
}
return cksm
}
type cksmFile struct {
File
*cksmFlags
isDB bool
}
type cksmFlags struct {
computeCksm bool
verifyCksm bool
inCkpt bool
pageSize int
}
func (c cksmFile) ReadAt(p []byte, off int64) (n int, err error) {
n, err = c.File.ReadAt(p, off)
// SQLite is reading the header of a database file.
if c.isDB && off == 0 && len(p) >= 100 &&
bytes.HasPrefix(p, []byte("SQLite format 3\000")) {
c.init(p)
}
// Verify checksums.
if c.verifyCksm && !c.inCkpt && len(p) == c.pageSize {
cksm1 := cksmCompute(p[:len(p)-8])
cksm2 := *(*[8]byte)(p[len(p)-8:])
if cksm1 != cksm2 {
return 0, _IOERR_DATA
}
}
return n, err
}
func (c cksmFile) WriteAt(p []byte, off int64) (n int, err error) {
// SQLite is writing the first page of a database file.
if c.isDB && off == 0 && len(p) >= 100 &&
bytes.HasPrefix(p, []byte("SQLite format 3\000")) {
c.init(p)
}
// Compute checksums.
if c.computeCksm && !c.inCkpt && len(p) == c.pageSize {
*(*[8]byte)(p[len(p)-8:]) = cksmCompute(p[:len(p)-8])
}
return c.File.WriteAt(p, off)
}
func (c cksmFile) Pragma(name string, value string) (string, error) {
switch name {
case "checksum_verification":
b, ok := sql3util.ParseBool(value)
if ok {
c.verifyCksm = b && c.computeCksm
}
if !c.verifyCksm {
return "0", nil
}
return "1", nil
case "page_size":
if c.computeCksm {
// Do not allow page size changes on a checksum database.
return strconv.Itoa(c.pageSize), nil
}
}
return "", _NOTFOUND
}
func (c cksmFile) fileControl(ctx context.Context, mod api.Module, op _FcntlOpcode, pArg uint32) _ErrorCode {
switch op {
case _FCNTL_CKPT_START:
c.inCkpt = true
case _FCNTL_CKPT_DONE:
c.inCkpt = false
}
if rc := vfsFileControlImpl(ctx, mod, c, op, pArg); rc != _NOTFOUND {
return rc
}
return vfsFileControlImpl(ctx, mod, c.File, op, pArg)
}
func (f *cksmFlags) init(header []byte) {
f.pageSize = 256 * int(binary.LittleEndian.Uint16(header[16:18]))
if r := header[20] == 8; r != f.computeCksm {
f.computeCksm = r
f.verifyCksm = r
}
}
func cksmCompute(a []byte) (cksm [8]byte) {
var s1, s2 uint32
for len(a) >= 8 {
s1 += binary.LittleEndian.Uint32(a[0:4]) + s2
s2 += binary.LittleEndian.Uint32(a[4:8]) + s1
a = a[8:]
}
if len(a) != 0 {
panic(util.AssertErr())
}
binary.LittleEndian.PutUint32(cksm[0:4], s1)
binary.LittleEndian.PutUint32(cksm[4:8], s2)
return
}
func (c cksmFile) SharedMemory() SharedMemory {
if f, ok := c.File.(FileSharedMemory); ok {
return f.SharedMemory()
}
return nil
}
func (c cksmFile) Unwrap() File {
return c.File
}

View file

@ -51,6 +51,7 @@ func (e _ErrorCode) Error() string {
_IOERR_BEGIN_ATOMIC _ErrorCode = util.IOERR_BEGIN_ATOMIC
_IOERR_COMMIT_ATOMIC _ErrorCode = util.IOERR_COMMIT_ATOMIC
_IOERR_ROLLBACK_ATOMIC _ErrorCode = util.IOERR_ROLLBACK_ATOMIC
_IOERR_DATA _ErrorCode = util.IOERR_DATA
_BUSY_SNAPSHOT _ErrorCode = util.BUSY_SNAPSHOT
_CANTOPEN_FULLPATH _ErrorCode = util.CANTOPEN_FULLPATH
_CANTOPEN_ISDIR _ErrorCode = util.CANTOPEN_ISDIR

View file

@ -4,8 +4,9 @@
"context"
"net/url"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
)
// Filename is used by SQLite to pass filenames

View file

@ -1,4 +1,4 @@
//go:build (amd64 || arm64 || riscv64 || ppc64le) && !sqlite3_nosys
//go:build (amd64 || arm64 || riscv64) && !sqlite3_nosys
package vfs
@ -13,7 +13,7 @@
_F2FS_IOC_START_ATOMIC_WRITE = 62721
_F2FS_IOC_COMMIT_ATOMIC_WRITE = 62722
_F2FS_IOC_ABORT_ATOMIC_WRITE = 62725
_F2FS_IOC_GET_FEATURES = 2147808524
_F2FS_IOC_GET_FEATURES = 2147808524 // -2147158772
_F2FS_FEATURE_ATOMIC_WRITE = 4
)

View file

@ -1,4 +1,4 @@
//go:build !linux || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_nosys
//go:build !linux || !(amd64 || arm64 || riscv64) || sqlite3_nosys
package vfs

View file

@ -1,4 +1,4 @@
//go:build (darwin || linux) && (amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_flock || sqlite3_noshm || sqlite3_nosys)
//go:build (darwin || linux) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_flock || sqlite3_noshm || sqlite3_nosys)
package vfs
@ -6,11 +6,13 @@
"context"
"io"
"os"
"sync"
"time"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero/api"
"golang.org/x/sys/unix"
"github.com/ncruces/go-sqlite3/internal/util"
)
// SupportsSharedMemory is false on platforms that do not support shared memory.
@ -45,12 +47,15 @@ func NewSharedMemory(path string, flags OpenFlag) SharedMemory {
}
}
var _ blockingSharedMemory = &vfsShm{}
type vfsShm struct {
*os.File
path string
regions []*util.MappedRegion
readOnly bool
blocking bool
sync.Mutex
}
func (s *vfsShm) shmOpen() _ErrorCode {
@ -196,6 +201,12 @@ func (s *vfsShm) shmUnmap(delete bool) {
s.File = nil
}
func (s *vfsShm) shmBarrier() {
s.Lock()
//lint:ignore SA2001 memory barrier.
s.Unlock()
}
func (s *vfsShm) shmEnableBlocking(block bool) {
s.blocking = block
}

View file

@ -1,4 +1,4 @@
//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
package vfs
@ -8,9 +8,10 @@
"os"
"sync"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero/api"
"golang.org/x/sys/unix"
"github.com/ncruces/go-sqlite3/internal/util"
)
// SupportsSharedMemory is false on platforms that do not support shared memory.
@ -269,3 +270,9 @@ func (s *vfsShm) shmUnmap(delete bool) {
}
s.Close()
}
func (s *vfsShm) shmBarrier() {
s.lockMtx.Lock()
//lint:ignore SA2001 memory barrier.
s.lockMtx.Unlock()
}

View file

@ -1,4 +1,4 @@
//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(386 || arm || amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
package vfs

View file

@ -5,13 +5,15 @@
"crypto/rand"
"io"
"reflect"
"sync"
"strings"
"time"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/julianday"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/util/sql3util"
"github.com/ncruces/julianday"
)
// ExportHostFunctions is an internal API users need not call directly.
@ -146,7 +148,7 @@ func vfsOpen(ctx context.Context, mod api.Module, pVfs, zPath, pFile uint32, fla
}
if file, ok := file.(FilePowersafeOverwrite); ok {
if b, ok := util.ParseBool(name.URIParameter("psow")); ok {
if b, ok := sql3util.ParseBool(name.URIParameter("psow")); ok {
file.SetPowersafeOverwrite(b)
}
}
@ -157,6 +159,7 @@ func vfsOpen(ctx context.Context, mod api.Module, pVfs, zPath, pFile uint32, fla
if pOutFlags != 0 {
util.WriteUint32(mod, pOutFlags, uint32(flags))
}
file = cksmWrapFile(name, flags, file)
vfsFileRegister(ctx, mod, pFile, file)
return _OK
}
@ -235,20 +238,19 @@ func vfsCheckReservedLock(ctx context.Context, mod api.Module, pFile, pResOut ui
func vfsFileControl(ctx context.Context, mod api.Module, pFile uint32, op _FcntlOpcode, pArg uint32) _ErrorCode {
file := vfsFileGet(ctx, mod, pFile).(File)
if file, ok := file.(fileControl); ok {
return file.fileControl(ctx, mod, op, pArg)
}
return vfsFileControlImpl(ctx, mod, file, op, pArg)
}
func vfsFileControlImpl(ctx context.Context, mod api.Module, file File, op _FcntlOpcode, pArg uint32) _ErrorCode {
switch op {
case _FCNTL_LOCKSTATE:
if file, ok := file.(FileLockState); ok {
util.WriteUint32(mod, pArg, uint32(file.LockState()))
return _OK
}
case _FCNTL_LOCK_TIMEOUT:
if file, ok := file.(FileSharedMemory); ok {
if iface, ok := file.SharedMemory().(interface{ shmEnableBlocking(bool) }); ok {
if i := util.ReadUint32(mod, pArg); i == 0 || i == 1 {
iface.shmEnableBlocking(i != 0)
}
if lk := file.LockState(); lk <= LOCK_EXCLUSIVE {
util.WriteUint32(mod, pArg, uint32(lk))
return _OK
}
}
@ -329,15 +331,15 @@ func vfsFileControl(ctx context.Context, mod api.Module, pFile uint32, op _Fcntl
return vfsErrorCode(err, _IOERR_ROLLBACK_ATOMIC)
}
case _FCNTL_CKPT_DONE:
if file, ok := file.(FileCheckpoint); ok {
err := file.CheckpointDone()
return vfsErrorCode(err, _IOERR)
}
case _FCNTL_CKPT_START:
if file, ok := file.(FileCheckpoint); ok {
err := file.CheckpointStart()
return vfsErrorCode(err, _IOERR)
file.CheckpointStart()
return _OK
}
case _FCNTL_CKPT_DONE:
if file, ok := file.(FileCheckpoint); ok {
file.CheckpointDone()
return _OK
}
case _FCNTL_PRAGMA:
@ -349,7 +351,7 @@ func vfsFileControl(ctx context.Context, mod api.Module, pFile uint32, op _Fcntl
value = util.ReadString(mod, ptr, _MAX_SQL_LENGTH)
}
out, err := file.Pragma(name, value)
out, err := file.Pragma(strings.ToLower(name), value)
ret := vfsErrorCode(err, _ERROR)
if ret == _ERROR {
@ -366,6 +368,14 @@ func vfsFileControl(ctx context.Context, mod api.Module, pFile uint32, op _Fcntl
}
return ret
}
case _FCNTL_LOCK_TIMEOUT:
if file, ok := file.(FileSharedMemory); ok {
if shm, ok := file.SharedMemory().(blockingSharedMemory); ok {
shm.shmEnableBlocking(util.ReadUint32(mod, pArg) != 0)
return _OK
}
}
}
// Consider also implementing these opcodes (in use by SQLite):
@ -385,11 +395,9 @@ func vfsDeviceCharacteristics(ctx context.Context, mod api.Module, pFile uint32)
return file.DeviceCharacteristics()
}
var shmBarrier sync.Mutex
func vfsShmBarrier(ctx context.Context, mod api.Module, pFile uint32) {
shmBarrier.Lock()
defer shmBarrier.Unlock()
shm := vfsFileGet(ctx, mod, pFile).(FileSharedMemory).SharedMemory()
shm.shmBarrier()
}
func vfsShmMap(ctx context.Context, mod api.Module, pFile uint32, iRegion, szRegion int32, bExtend, pp uint32) _ErrorCode {

View file

@ -4,8 +4,9 @@
"context"
"reflect"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
)
// CreateModule registers a new virtual table module name.

3
vendor/modules.txt vendored
View file

@ -518,7 +518,7 @@ github.com/modern-go/reflect2
# github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
## explicit
github.com/munnerz/goautoneg
# github.com/ncruces/go-sqlite3 v0.19.0
# github.com/ncruces/go-sqlite3 v0.20.0
## explicit; go 1.21
github.com/ncruces/go-sqlite3
github.com/ncruces/go-sqlite3/driver
@ -526,6 +526,7 @@ github.com/ncruces/go-sqlite3/embed
github.com/ncruces/go-sqlite3/internal/alloc
github.com/ncruces/go-sqlite3/internal/util
github.com/ncruces/go-sqlite3/util/osutil
github.com/ncruces/go-sqlite3/util/sql3util
github.com/ncruces/go-sqlite3/vfs
github.com/ncruces/go-sqlite3/vfs/memdb
# github.com/ncruces/go-strftime v0.1.9