[feature] Default to WASM-based SQLite driver (#3053)

* [feature] Default to WASM-based SQLite driver

With 0.16 out this switches our default SQLite driver to the WASM-based
solution instead. So far the driver seems to perform just as well.
Switching our default should result in it getting a bit more testing
during the 0.17 development cycle.

* add the ol' john hancock

---------

Co-authored-by: tobi <tobi.smethurst@protonmail.com>

.drone.yml

@@ -42,7 +42,7 @@ steps:
         go test
         -failfast
         -timeout=20m
-        -tags "wasmsqlite3 netgo osusergo static_build kvformat timetzdata"
+        -tags "netgo osusergo static_build kvformat timetzdata"
         ./...
       - ./test/envparsing.sh
       - ./test/swagger.sh
@@ -204,6 +204,6 @@ steps:
 
 ---
 kind: signature
-hmac: 2e74313f4192b3e6daf6d1d00a7c3796019d93da7ce7e0a77208ccc3c37089b0
+hmac: 86ebddcd630792cac43aa92fa7f45118943c51b5157491d05eb480ac21762329
 
 ...

.goreleaser.yml

@@ -30,7 +30,7 @@ builds:
       - >-
         {{ if and (index .Env "DEBUG") (.Env.DEBUG) }}debugenv{{ end }}
       - >-
-        {{ if and (index .Env "WASMSQLITE3") (.Env.WASMSQLITE3) }}wasmsqlite3{{ end }}
+        {{ if and (index .Env "MODERNCSQLITE3") (.Env.MODERNCSQLITE3) }}moderncsqlite3{{ end }}
     env:
       - CGO_ENABLED=0
     goos:

internal/db/sqlite/driver.go

@@ -15,7 +15,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-//go:build !wasmsqlite3
+//go:build !moderncsqlite3
 
 package sqlite
 
@@ -23,19 +23,22 @@
 	"context"
 	"database/sql/driver"
 
-	"modernc.org/sqlite"
-
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+
+	"github.com/ncruces/go-sqlite3"
+	sqlite3driver "github.com/ncruces/go-sqlite3/driver"
+	_ "github.com/ncruces/go-sqlite3/embed"     // embed wasm binary
+	_ "github.com/ncruces/go-sqlite3/vfs/memdb" // include memdb vfs
 )
 
 // Driver is our own wrapper around the
-// sqlite.Driver{} type in order to wrap
+// driver.SQLite{} type in order to wrap
 // further SQL types with our own
 // functionality, e.g. err processing.
-type Driver struct{ sqlite.Driver }
+type Driver struct{ sqlite3driver.SQLite }
 
 func (d *Driver) Open(name string) (driver.Conn, error) {
-	conn, err := d.Driver.Open(name)
+	conn, err := d.SQLite.Open(name)
 	if err != nil {
 		err = processSQLiteError(err)
 		return nil, err
@@ -43,6 +46,30 @@ func (d *Driver) Open(name string) (driver.Conn, error) {
 	return &sqliteConn{conn.(connIface)}, nil
 }
 
+func (d *Driver) OpenConnector(name string) (driver.Connector, error) {
+	cc, err := d.SQLite.OpenConnector(name)
+	if err != nil {
+		return nil, err
+	}
+	return &sqliteConnector{driver: d, Connector: cc}, nil
+}
+
+type sqliteConnector struct {
+	driver *Driver
+	driver.Connector
+}
+
+func (c *sqliteConnector) Driver() driver.Driver { return c.driver }
+
+func (c *sqliteConnector) Connect(ctx context.Context) (driver.Conn, error) {
+	conn, err := c.Connector.Connect(ctx)
+	err = processSQLiteError(err)
+	if err != nil {
+		return nil, err
+	}
+	return &sqliteConn{conn.(connIface)}, nil
+}
+
 type sqliteConn struct{ connIface }
 
 func (c *sqliteConn) Begin() (driver.Tx, error) {
@@ -81,26 +108,16 @@ func (c *sqliteConn) ExecContext(ctx context.Context, query string, args []drive
 	return
 }
 
-func (c *sqliteConn) Query(query string, args []driver.Value) (driver.Rows, error) {
-	return c.QueryContext(context.Background(), query, db.ToNamedValues(args))
-}
-
-func (c *sqliteConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (rows driver.Rows, err error) {
-	rows, err = c.connIface.QueryContext(ctx, query, args)
-	err = processSQLiteError(err)
-	if err != nil {
-		return nil, err
-	}
-	return &sqliteRows{rows.(rowsIface)}, nil
-}
-
 func (c *sqliteConn) Close() (err error) {
+	// Get acces the underlying raw sqlite3 conn.
+	raw := c.connIface.(sqlite3.DriverConn).Raw()
+
 	// see: https://www.sqlite.org/pragma.html#pragma_optimize
 	const onClose = "PRAGMA analysis_limit=1000; PRAGMA optimize;"
-	_, _ = c.connIface.ExecContext(context.Background(), onClose, nil)
+	_ = raw.Exec(onClose)
 
-	// Finally, close the conn.
+	// Finally, close.
-	err = c.connIface.Close()
+	err = raw.Close()
 	return
 }
 
@@ -164,7 +181,7 @@ func (r *sqliteRows) Close() (err error) {
 }
 
 // connIface is the driver.Conn interface
-// types (and the like) that modernc.org/sqlite.conn
+// types (and the like) that go-sqlite3/driver.conn
 // conforms to. Useful so you don't need
 // to repeatedly perform checks yourself.
 type connIface interface {
@@ -172,11 +189,10 @@ type connIface interface {
 	driver.ConnBeginTx
 	driver.ConnPrepareContext
 	driver.ExecerContext
-	driver.QueryerContext
 }
 
 // StmtIface is the driver.Stmt interface
-// types (and the like) that modernc.org/sqlite.stmt
+// types (and the like) that go-sqlite3/driver.stmt
 // conforms to. Useful so you don't need
 // to repeatedly perform checks yourself.
 type stmtIface interface {
@@ -186,12 +202,10 @@ type stmtIface interface {
 }
 
 // RowsIface is the driver.Rows interface
-// types (and the like) that modernc.org/sqlite.rows
+// types (and the like) that go-sqlite3/driver.rows
 // conforms to. Useful so you don't need
 // to repeatedly perform checks yourself.
 type rowsIface interface {
 	driver.Rows
 	driver.RowsColumnTypeDatabaseTypeName
-	driver.RowsColumnTypeLength
-	driver.RowsColumnTypeScanType
 }

internal/db/sqlite/driver_moderncsqlite3.go

@@ -15,7 +15,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-//go:build wasmsqlite3
+//go:build moderncsqlite3
 
 package sqlite
 
@@ -23,22 +23,19 @@
 	"context"
 	"database/sql/driver"
 
-	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"modernc.org/sqlite"
 
-	"github.com/ncruces/go-sqlite3"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
-	sqlite3driver "github.com/ncruces/go-sqlite3/driver"
-	_ "github.com/ncruces/go-sqlite3/embed"     // embed wasm binary
-	_ "github.com/ncruces/go-sqlite3/vfs/memdb" // include memdb vfs
 )
 
 // Driver is our own wrapper around the
-// driver.SQLite{} type in order to wrap
+// sqlite.Driver{} type in order to wrap
 // further SQL types with our own
 // functionality, e.g. err processing.
-type Driver struct{ sqlite3driver.SQLite }
+type Driver struct{ sqlite.Driver }
 
 func (d *Driver) Open(name string) (driver.Conn, error) {
-	conn, err := d.SQLite.Open(name)
+	conn, err := d.Driver.Open(name)
 	if err != nil {
 		err = processSQLiteError(err)
 		return nil, err
@@ -46,30 +43,6 @@ func (d *Driver) Open(name string) (driver.Conn, error) {
 	return &sqliteConn{conn.(connIface)}, nil
 }
 
-func (d *Driver) OpenConnector(name string) (driver.Connector, error) {
-	cc, err := d.SQLite.OpenConnector(name)
-	if err != nil {
-		return nil, err
-	}
-	return &sqliteConnector{driver: d, Connector: cc}, nil
-}
-
-type sqliteConnector struct {
-	driver *Driver
-	driver.Connector
-}
-
-func (c *sqliteConnector) Driver() driver.Driver { return c.driver }
-
-func (c *sqliteConnector) Connect(ctx context.Context) (driver.Conn, error) {
-	conn, err := c.Connector.Connect(ctx)
-	err = processSQLiteError(err)
-	if err != nil {
-		return nil, err
-	}
-	return &sqliteConn{conn.(connIface)}, nil
-}
-
 type sqliteConn struct{ connIface }
 
 func (c *sqliteConn) Begin() (driver.Tx, error) {
@@ -108,16 +81,26 @@ func (c *sqliteConn) ExecContext(ctx context.Context, query string, args []drive
 	return
 }
 
-func (c *sqliteConn) Close() (err error) {
+func (c *sqliteConn) Query(query string, args []driver.Value) (driver.Rows, error) {
-	// Get acces the underlying raw sqlite3 conn.
+	return c.QueryContext(context.Background(), query, db.ToNamedValues(args))
-	raw := c.connIface.(sqlite3.DriverConn).Raw()
+}
 
+func (c *sqliteConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (rows driver.Rows, err error) {
+	rows, err = c.connIface.QueryContext(ctx, query, args)
+	err = processSQLiteError(err)
+	if err != nil {
+		return nil, err
+	}
+	return &sqliteRows{rows.(rowsIface)}, nil
+}
+
+func (c *sqliteConn) Close() (err error) {
 	// see: https://www.sqlite.org/pragma.html#pragma_optimize
 	const onClose = "PRAGMA analysis_limit=1000; PRAGMA optimize;"
-	_ = raw.Exec(onClose)
+	_, _ = c.connIface.ExecContext(context.Background(), onClose, nil)
 
-	// Finally, close.
+	// Finally, close the conn.
-	err = raw.Close()
+	err = c.connIface.Close()
 	return
 }
 
@@ -181,7 +164,7 @@ func (r *sqliteRows) Close() (err error) {
 }
 
 // connIface is the driver.Conn interface
-// types (and the like) that go-sqlite3/driver.conn
+// types (and the like) that modernc.org/sqlite.conn
 // conforms to. Useful so you don't need
 // to repeatedly perform checks yourself.
 type connIface interface {
@@ -189,10 +172,11 @@ type connIface interface {
 	driver.ConnBeginTx
 	driver.ConnPrepareContext
 	driver.ExecerContext
+	driver.QueryerContext
 }
 
 // StmtIface is the driver.Stmt interface
-// types (and the like) that go-sqlite3/driver.stmt
+// types (and the like) that modernc.org/sqlite.stmt
 // conforms to. Useful so you don't need
 // to repeatedly perform checks yourself.
 type stmtIface interface {
@@ -202,10 +186,12 @@ type stmtIface interface {
 }
 
 // RowsIface is the driver.Rows interface
-// types (and the like) that go-sqlite3/driver.rows
+// types (and the like) that modernc.org/sqlite.rows
 // conforms to. Useful so you don't need
 // to repeatedly perform checks yourself.
 type rowsIface interface {
 	driver.Rows
 	driver.RowsColumnTypeDatabaseTypeName
+	driver.RowsColumnTypeLength
+	driver.RowsColumnTypeScanType
 }

internal/db/sqlite/errors.go

@@ -15,7 +15,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-//go:build !wasmsqlite3
+//go:build !moderncsqlite3
 
 package sqlite
 
@@ -23,9 +23,7 @@
 	"database/sql/driver"
 	"fmt"
 
-	"modernc.org/sqlite"
+	"github.com/ncruces/go-sqlite3"
-	sqlite3 "modernc.org/sqlite/lib"
-
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 )
 
@@ -33,30 +31,30 @@
 // handle conversion to any of our common db types.
 func processSQLiteError(err error) error {
 	// Attempt to cast as sqlite error.
-	sqliteErr, ok := err.(*sqlite.Error)
+	sqliteErr, ok := err.(*sqlite3.Error)
 	if !ok {
 		return err
 	}
 
 	// Handle supplied error code:
-	switch sqliteErr.Code() {
+	switch sqliteErr.ExtendedCode() {
-	case sqlite3.SQLITE_CONSTRAINT_UNIQUE,
+	case sqlite3.CONSTRAINT_UNIQUE,
-		sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY:
+		sqlite3.CONSTRAINT_PRIMARYKEY:
 		return db.ErrAlreadyExists
 
-	// Busy should be very rare, but
+	// Busy should be very rare, but on
-	// on busy tell the database to close
+	// busy tell the database to close the
-	// the connection, re-open and re-attempt
+	// connection, re-open and re-attempt
-	// which should give a necessary timeout.
+	// which should give necessary timeout.
-	case sqlite3.SQLITE_BUSY,
+	case sqlite3.BUSY_RECOVERY,
-		sqlite3.SQLITE_BUSY_RECOVERY,
+		sqlite3.BUSY_SNAPSHOT:
-		sqlite3.SQLITE_BUSY_SNAPSHOT:
 		return driver.ErrBadConn
 	}
 
 	// Wrap the returned error with the code and
 	// extended code for easier debugging later.
-	return fmt.Errorf("%w (code=%d)", err,
+	return fmt.Errorf("%w (code=%d extended=%d)", err,
 		sqliteErr.Code(),
+		sqliteErr.ExtendedCode(),
 	)
 }

internal/db/sqlite/errors_moderncsqlite3.go

@@ -15,7 +15,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-//go:build wasmsqlite3
+//go:build moderncsqlite3
 
 package sqlite
 
@@ -23,7 +23,9 @@
 	"database/sql/driver"
 	"fmt"
 
-	"github.com/ncruces/go-sqlite3"
+	"modernc.org/sqlite"
+	sqlite3 "modernc.org/sqlite/lib"
+
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 )
 
@@ -31,30 +33,30 @@
 // handle conversion to any of our common db types.
 func processSQLiteError(err error) error {
 	// Attempt to cast as sqlite error.
-	sqliteErr, ok := err.(*sqlite3.Error)
+	sqliteErr, ok := err.(*sqlite.Error)
 	if !ok {
 		return err
 	}
 
 	// Handle supplied error code:
-	switch sqliteErr.ExtendedCode() {
+	switch sqliteErr.Code() {
-	case sqlite3.CONSTRAINT_UNIQUE,
+	case sqlite3.SQLITE_CONSTRAINT_UNIQUE,
-		sqlite3.CONSTRAINT_PRIMARYKEY:
+		sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY:
 		return db.ErrAlreadyExists
 
-	// Busy should be very rare, but on
+	// Busy should be very rare, but
-	// busy tell the database to close the
+	// on busy tell the database to close
-	// connection, re-open and re-attempt
+	// the connection, re-open and re-attempt
-	// which should give necessary timeout.
+	// which should give a necessary timeout.
-	case sqlite3.BUSY_RECOVERY,
+	case sqlite3.SQLITE_BUSY,
-		sqlite3.BUSY_SNAPSHOT:
+		sqlite3.SQLITE_BUSY_RECOVERY,
+		sqlite3.SQLITE_BUSY_SNAPSHOT:
 		return driver.ErrBadConn
 	}
 
 	// Wrap the returned error with the code and
 	// extended code for easier debugging later.
-	return fmt.Errorf("%w (code=%d extended=%d)", err,
+	return fmt.Errorf("%w (code=%d)", err,
 		sqliteErr.Code(),
-		sqliteErr.ExtendedCode(),
 	)
 }

scripts/build.sh

@@ -15,14 +15,14 @@ GO_GCFLAGS=${GO_GCFLAGS-}
     GO_BUILDTAGS="${GO_BUILDTAGS} debugenv"
 
 # Available Go build tags, with explanation, followed by benefits of enabling it:
-# - kvformat:    enables prettier output of log fields                       (slightly better performance)
+# - kvformat:       enables prettier output of log fields                       (slightly better performance)
-# - timetzdata:  embed timezone database inside binary                       (allow setting local time inside Docker containers, at cost of 450KB)
+# - timetzdata:     embed timezone database inside binary                       (allow setting local time inside Docker containers, at cost of 450KB)
-# - notracing:   disables compiling-in otel tracing support                 (reduced binary size, better performance)
+# - notracing:      disables compiling-in otel tracing support                  (reduced binary size, better performance)
-# - nometrics:   disables compiling-in otel metrics support                 (reduced binary size, better performance)
+# - nometrics:      disables compiling-in otel metrics support                  (reduced binary size, better performance)
-# - noerrcaller: disables caller function prefix in errors                   (slightly better performance, at cost of err readability)
+# - noerrcaller:    disables caller function prefix in errors                   (slightly better performance, at cost of err readability)
-# - debug:       enables /debug/pprof endpoint                               (adds debug, at performance cost)
+# - debug:          enables /debug/pprof endpoint                               (adds debug, at performance cost)
-# - debugenv:    enables /debug/pprof endpoint if DEBUG=1 env during runtime (adds debug, at performance cost)
+# - debugenv:       enables /debug/pprof endpoint if DEBUG=1 env during runtime (adds debug, at performance cost)
-# - wasmsqlite3: uses SQLite through WASM instead of the C-to-Go transpilation (experimental)
+# - moderncsqlite3: reverts to using the C-to-Go transpiled SQLite driver       (disables the WASM-based SQLite driver)
 log_exec env CGO_ENABLED=0 go build -trimpath -v \
                        -tags "${GO_BUILDTAGS}" \
                        -ldflags="${GO_LDFLAGS}" \