[feature] Tune sqlite pragmas (#1349)

* sqlite pragma tuning

* use formatuint

* add sqlite busy timeout

* fix incorrect cache size format

* update envparsing test

* add sqlite tuning flags to cli

* set sqlite timeout to 30s default

docs/configuration/database.md

@@ -107,4 +107,36 @@ db-tls-mode: "disable"
 # Examples: ["/path/to/some/cert.crt"]
 # Default: ""
 db-tls-ca-cert: ""
+
+# String. SQLite journaling mode.
+# SQLite only -- unused otherwise.
+# If set to empty string, the sqlite default will be used.
+# See: https://www.sqlite.org/pragma.html#pragma_journal_mode
+# Examples: ["DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF"]
+# Default: "WAL"
+db-sqlite-journal-mode: "WAL"
+
+# String. SQLite synchronous mode.
+# SQLite only -- unused otherwise.
+# If set to empty string, the sqlite default will be used.
+# See: https://www.sqlite.org/pragma.html#pragma_synchronous
+# Examples: ["OFF", "NORMAL", "FULL", "EXTRA"]
+# Default: "NORMAL"
+db-sqlite-synchronous: "NORMAL"
+
+# Byte size. SQlite cache size.
+# SQLite only -- unused otherwise.
+# If set to empty string or zero, the sqlite default will be used.
+# See: https://www.sqlite.org/pragma.html#pragma_cache_size
+# Examples: ["32MiB", "0", "64MiB"]
+# Default: "64MiB"
+db-sqlite-cache-size: "64MiB"
+
+# Duration. SQlite busy timeout.
+# SQLite only -- unused otherwise.
+# If set to empty string or zero, the sqlite default will be used.
+# See: https://www.sqlite.org/pragma.html#pragma_busy_timeout
+# Examples: ["0s", "1s", "30s", "1m", "5m"]
+# Default: "5s"
+db-sqlite-busy-timeout: "30s"
 ```

example/config.yaml

@@ -164,6 +164,38 @@ db-tls-mode: "disable"
 # Default: ""
 db-tls-ca-cert: ""
 
+# String. SQLite journaling mode.
+# SQLite only -- unused otherwise.
+# If set to empty string, the sqlite default will be used.
+# See: https://www.sqlite.org/pragma.html#pragma_journal_mode
+# Examples: ["DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF"]
+# Default: "WAL"
+db-sqlite-journal-mode: "WAL"
+
+# String. SQLite synchronous mode.
+# SQLite only -- unused otherwise.
+# If set to empty string, the sqlite default will be used.
+# See: https://www.sqlite.org/pragma.html#pragma_synchronous
+# Examples: ["OFF", "NORMAL", "FULL", "EXTRA"]
+# Default: "NORMAL"
+db-sqlite-synchronous: "NORMAL"
+
+# Byte size. SQlite cache size.
+# SQLite only -- unused otherwise.
+# If set to empty string or zero, the sqlite default will be used.
+# See: https://www.sqlite.org/pragma.html#pragma_cache_size
+# Examples: ["32MiB", "0", "64MiB"]
+# Default: "64MiB"
+db-sqlite-cache-size: "64MiB"
+
+# Duration. SQlite busy timeout.
+# SQLite only -- unused otherwise.
+# If set to empty string or zero, the sqlite default will be used.
+# See: https://www.sqlite.org/pragma.html#pragma_busy_timeout
+# Examples: ["0s", "1s", "30s", "1m", "5m"]
+# Default: "5s"
+db-sqlite-busy-timeout: "30s"
+
 cache:
   gts:
     ###########################

internal/config/config.go

@@ -58,14 +58,18 @@ type Configuration struct {
 	TrustedProxies  []string `name:"trusted-proxies" usage:"Proxies to trust when parsing x-forwarded headers into real IPs."`
 	SoftwareVersion string   `name:"software-version" usage:""`
 
-	DbType      string `name:"db-type" usage:"Database type: eg., postgres"`
+	DbType              string        `name:"db-type" usage:"Database type: eg., postgres"`
-	DbAddress   string `name:"db-address" usage:"Database ipv4 address, hostname, or filename"`
+	DbAddress           string        `name:"db-address" usage:"Database ipv4 address, hostname, or filename"`
-	DbPort      int    `name:"db-port" usage:"Database port"`
+	DbPort              int           `name:"db-port" usage:"Database port"`
-	DbUser      string `name:"db-user" usage:"Database username"`
+	DbUser              string        `name:"db-user" usage:"Database username"`
-	DbPassword  string `name:"db-password" usage:"Database password"`
+	DbPassword          string        `name:"db-password" usage:"Database password"`
-	DbDatabase  string `name:"db-database" usage:"Database name"`
+	DbDatabase          string        `name:"db-database" usage:"Database name"`
-	DbTLSMode   string `name:"db-tls-mode" usage:"Database tls mode"`
+	DbTLSMode           string        `name:"db-tls-mode" usage:"Database tls mode"`
-	DbTLSCACert string `name:"db-tls-ca-cert" usage:"Path to CA cert for db tls connection"`
+	DbTLSCACert         string        `name:"db-tls-ca-cert" usage:"Path to CA cert for db tls connection"`
+	DbSqliteJournalMode string        `name:"db-sqlite-journal-mode" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_journal_mode"`
+	DbSqliteSynchronous string        `name:"db-sqlite-synchronous" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_synchronous"`
+	DbSqliteCacheSize   bytesize.Size `name:"db-sqlite-cache-size" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_cache_size"`
+	DbSqliteBusyTimeout time.Duration `name:"db-sqlite-busy-timeout" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_busy_timeout"`
 
 	WebTemplateBaseDir string `name:"web-template-base-dir" usage:"Basedir for html templating files for rendering pages and composing emails."`
 	WebAssetBaseDir    string `name:"web-asset-base-dir" usage:"Directory to serve static assets from, accessible at example.org/assets/"`

internal/config/defaults.go

@@ -40,14 +40,18 @@
 	Port:            8080,
 	TrustedProxies:  []string{"127.0.0.1/32", "::1"}, // localhost
 
-	DbType:      "postgres",
+	DbType:              "postgres",
-	DbAddress:   "",
+	DbAddress:           "",
-	DbPort:      5432,
+	DbPort:              5432,
-	DbUser:      "",
+	DbUser:              "",
-	DbPassword:  "",
+	DbPassword:          "",
-	DbDatabase:  "gotosocial",
+	DbDatabase:          "gotosocial",
-	DbTLSMode:   "disable",
+	DbTLSMode:           "disable",
-	DbTLSCACert: "",
+	DbTLSCACert:         "",
+	DbSqliteJournalMode: "WAL",
+	DbSqliteSynchronous: "NORMAL",
+	DbSqliteCacheSize:   64 * bytesize.MiB,
+	DbSqliteBusyTimeout: time.Second * 30,
 
 	WebTemplateBaseDir: "./web/template/",
 	WebAssetBaseDir:    "./web/assets/",

internal/config/flags.go

@@ -51,6 +51,10 @@ func (s *ConfigState) AddGlobalFlags(cmd *cobra.Command) {
 		cmd.PersistentFlags().String(DbDatabaseFlag(), cfg.DbDatabase, fieldtag("DbDatabase", "usage"))
 		cmd.PersistentFlags().String(DbTLSModeFlag(), cfg.DbTLSMode, fieldtag("DbTLSMode", "usage"))
 		cmd.PersistentFlags().String(DbTLSCACertFlag(), cfg.DbTLSCACert, fieldtag("DbTLSCACert", "usage"))
+		cmd.PersistentFlags().String(DbSqliteJournalModeFlag(), cfg.DbSqliteJournalMode, fieldtag("DbSqliteJournalMode", "usage"))
+		cmd.PersistentFlags().String(DbSqliteSynchronousFlag(), cfg.DbSqliteSynchronous, fieldtag("DbSqliteSynchronous", "usage"))
+		cmd.PersistentFlags().Uint64(DbSqliteCacheSizeFlag(), uint64(cfg.DbSqliteCacheSize), fieldtag("DbSqliteCacheSize", "usage"))
+		cmd.PersistentFlags().Duration(DbSqliteBusyTimeoutFlag(), cfg.DbSqliteBusyTimeout, fieldtag("DbSqliteBusyTimeout", "usage"))
 	})
 }
 

internal/config/helpers.gen.go

@@ -524,6 +524,106 @@ func GetDbTLSCACert() string { return global.GetDbTLSCACert() }
 // SetDbTLSCACert safely sets the value for global configuration 'DbTLSCACert' field
 func SetDbTLSCACert(v string) { global.SetDbTLSCACert(v) }
 
+// GetDbSqliteJournalMode safely fetches the Configuration value for state's 'DbSqliteJournalMode' field
+func (st *ConfigState) GetDbSqliteJournalMode() (v string) {
+	st.mutex.Lock()
+	v = st.config.DbSqliteJournalMode
+	st.mutex.Unlock()
+	return
+}
+
+// SetDbSqliteJournalMode safely sets the Configuration value for state's 'DbSqliteJournalMode' field
+func (st *ConfigState) SetDbSqliteJournalMode(v string) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.DbSqliteJournalMode = v
+	st.reloadToViper()
+}
+
+// DbSqliteJournalModeFlag returns the flag name for the 'DbSqliteJournalMode' field
+func DbSqliteJournalModeFlag() string { return "db-sqlite-journal-mode" }
+
+// GetDbSqliteJournalMode safely fetches the value for global configuration 'DbSqliteJournalMode' field
+func GetDbSqliteJournalMode() string { return global.GetDbSqliteJournalMode() }
+
+// SetDbSqliteJournalMode safely sets the value for global configuration 'DbSqliteJournalMode' field
+func SetDbSqliteJournalMode(v string) { global.SetDbSqliteJournalMode(v) }
+
+// GetDbSqliteSynchronous safely fetches the Configuration value for state's 'DbSqliteSynchronous' field
+func (st *ConfigState) GetDbSqliteSynchronous() (v string) {
+	st.mutex.Lock()
+	v = st.config.DbSqliteSynchronous
+	st.mutex.Unlock()
+	return
+}
+
+// SetDbSqliteSynchronous safely sets the Configuration value for state's 'DbSqliteSynchronous' field
+func (st *ConfigState) SetDbSqliteSynchronous(v string) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.DbSqliteSynchronous = v
+	st.reloadToViper()
+}
+
+// DbSqliteSynchronousFlag returns the flag name for the 'DbSqliteSynchronous' field
+func DbSqliteSynchronousFlag() string { return "db-sqlite-synchronous" }
+
+// GetDbSqliteSynchronous safely fetches the value for global configuration 'DbSqliteSynchronous' field
+func GetDbSqliteSynchronous() string { return global.GetDbSqliteSynchronous() }
+
+// SetDbSqliteSynchronous safely sets the value for global configuration 'DbSqliteSynchronous' field
+func SetDbSqliteSynchronous(v string) { global.SetDbSqliteSynchronous(v) }
+
+// GetDbSqliteCacheSize safely fetches the Configuration value for state's 'DbSqliteCacheSize' field
+func (st *ConfigState) GetDbSqliteCacheSize() (v bytesize.Size) {
+	st.mutex.Lock()
+	v = st.config.DbSqliteCacheSize
+	st.mutex.Unlock()
+	return
+}
+
+// SetDbSqliteCacheSize safely sets the Configuration value for state's 'DbSqliteCacheSize' field
+func (st *ConfigState) SetDbSqliteCacheSize(v bytesize.Size) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.DbSqliteCacheSize = v
+	st.reloadToViper()
+}
+
+// DbSqliteCacheSizeFlag returns the flag name for the 'DbSqliteCacheSize' field
+func DbSqliteCacheSizeFlag() string { return "db-sqlite-cache-size" }
+
+// GetDbSqliteCacheSize safely fetches the value for global configuration 'DbSqliteCacheSize' field
+func GetDbSqliteCacheSize() bytesize.Size { return global.GetDbSqliteCacheSize() }
+
+// SetDbSqliteCacheSize safely sets the value for global configuration 'DbSqliteCacheSize' field
+func SetDbSqliteCacheSize(v bytesize.Size) { global.SetDbSqliteCacheSize(v) }
+
+// GetDbSqliteBusyTimeout safely fetches the Configuration value for state's 'DbSqliteBusyTimeout' field
+func (st *ConfigState) GetDbSqliteBusyTimeout() (v time.Duration) {
+	st.mutex.Lock()
+	v = st.config.DbSqliteBusyTimeout
+	st.mutex.Unlock()
+	return
+}
+
+// SetDbSqliteBusyTimeout safely sets the Configuration value for state's 'DbSqliteBusyTimeout' field
+func (st *ConfigState) SetDbSqliteBusyTimeout(v time.Duration) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.DbSqliteBusyTimeout = v
+	st.reloadToViper()
+}
+
+// DbSqliteBusyTimeoutFlag returns the flag name for the 'DbSqliteBusyTimeout' field
+func DbSqliteBusyTimeoutFlag() string { return "db-sqlite-busy-timeout" }
+
+// GetDbSqliteBusyTimeout safely fetches the value for global configuration 'DbSqliteBusyTimeout' field
+func GetDbSqliteBusyTimeout() time.Duration { return global.GetDbSqliteBusyTimeout() }
+
+// SetDbSqliteBusyTimeout safely sets the value for global configuration 'DbSqliteBusyTimeout' field
+func SetDbSqliteBusyTimeout(v time.Duration) { global.SetDbSqliteBusyTimeout(v) }
+
 // GetWebTemplateBaseDir safely fetches the Configuration value for state's 'WebTemplateBaseDir' field
 func (st *ConfigState) GetWebTemplateBaseDir() (v string) {
 	st.mutex.Lock()

internal/db/bundb/bundb.go

@@ -28,9 +28,11 @@
 	"fmt"
 	"os"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
+	"codeberg.org/gruf/go-bytesize"
 	"github.com/google/uuid"
 	"github.com/jackc/pgx/v4"
 	"github.com/jackc/pgx/v4/stdlib"
@@ -49,22 +51,6 @@
 	"modernc.org/sqlite"
 )
 
-const (
-	dbTypePostgres = "postgres"
-	dbTypeSqlite   = "sqlite"
-
-	// dbTLSModeDisable does not attempt to make a TLS connection to the database.
-	dbTLSModeDisable = "disable"
-	// dbTLSModeEnable attempts to make a TLS connection to the database, but doesn't fail if
-	// the certificate passed by the database isn't verified.
-	dbTLSModeEnable = "enable"
-	// dbTLSModeRequire attempts to make a TLS connection to the database, and requires
-	// that the certificate presented by the database is valid.
-	dbTLSModeRequire = "require"
-	// dbTLSModeUnset means that the TLS mode has not been set.
-	dbTLSModeUnset = ""
-)
-
 var registerTables = []interface{}{
 	&gtsmodel.AccountToEmoji{},
 	&gtsmodel.StatusToEmoji{},
@@ -127,26 +113,34 @@ func doMigration(ctx context.Context, db *bun.DB) error {
 func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) {
 	var conn *DBConn
 	var err error
-	dbType := strings.ToLower(config.GetDbType())
+	t := strings.ToLower(config.GetDbType())
 
-	switch dbType {
+	switch t {
-	case dbTypePostgres:
+	case "postgres":
 		conn, err = pgConn(ctx)
 		if err != nil {
 			return nil, err
 		}
-	case dbTypeSqlite:
+	case "sqlite":
 		conn, err = sqliteConn(ctx)
 		if err != nil {
 			return nil, err
 		}
 	default:
-		return nil, fmt.Errorf("database type %s not supported for bundb", dbType)
+		return nil, fmt.Errorf("database type %s not supported for bundb", t)
 	}
 
 	// Add database query hook
 	conn.DB.AddQueryHook(queryHook{})
 
+	// execute sqlite pragmas *after* adding database hook;
+	// this allows the pragma queries to be logged
+	if t == "sqlite" {
+		if err := sqlitePragmas(ctx, conn); err != nil {
+			return nil, err
+		}
+	}
+
 	// table registration is needed for many-to-many, see:
 	// https://bun.uptrace.dev/orm/many-to-many-relation/
 	for _, t := range registerTables {
@@ -230,29 +224,29 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) {
 
 func sqliteConn(ctx context.Context) (*DBConn, error) {
 	// validate db address has actually been set
-	dbAddress := config.GetDbAddress()
+	address := config.GetDbAddress()
-	if dbAddress == "" {
+	if address == "" {
 		return nil, fmt.Errorf("'%s' was not set when attempting to start sqlite", config.DbAddressFlag())
 	}
 
 	// Drop anything fancy from DB address
-	dbAddress = strings.Split(dbAddress, "?")[0]
+	address = strings.Split(address, "?")[0]
-	dbAddress = strings.TrimPrefix(dbAddress, "file:")
+	address = strings.TrimPrefix(address, "file:")
 
 	// Append our own SQLite preferences
-	dbAddress = "file:" + dbAddress + "?cache=shared"
+	address = "file:" + address
 
 	var inMem bool
 
-	if dbAddress == "file::memory:?cache=shared" {
+	if address == "file::memory:" {
-		dbAddress = fmt.Sprintf("file:%s?mode=memory&cache=shared", uuid.NewString())
+		address = fmt.Sprintf("file:%s?mode=memory&cache=shared", uuid.NewString())
-		log.Infof("using in-memory database address " + dbAddress)
+		log.Infof("using in-memory database address " + address)
 		log.Warn("sqlite in-memory database should only be used for debugging")
 		inMem = true
 	}
 
 	// Open new DB instance
-	sqldb, err := sql.Open("sqlite", dbAddress)
+	sqldb, err := sql.Open("sqlite", address)
 	if err != nil {
 		if errWithCode, ok := err.(*sqlite.Error); ok {
 			err = errors.New(sqlite.ErrorCodeString[errWithCode.Code()])
@@ -260,8 +254,6 @@ func sqliteConn(ctx context.Context) (*DBConn, error) {
 		return nil, fmt.Errorf("could not open sqlite db: %s", err)
 	}
 
-	tweakConnectionValues(sqldb)
-
 	if inMem {
 		// don't close connections on disconnect -- otherwise
 		// the SQLite database will be deleted when there
@@ -269,6 +261,7 @@ func sqliteConn(ctx context.Context) (*DBConn, error) {
 		sqldb.SetConnMaxLifetime(0)
 	}
 
+	// Wrap Bun database conn in our own wrapper
 	conn := WrapDBConn(bun.NewDB(sqldb, sqlitedialect.New()))
 
 	// ping to check the db is there and listening
@@ -278,11 +271,56 @@ func sqliteConn(ctx context.Context) (*DBConn, error) {
 		}
 		return nil, fmt.Errorf("sqlite ping: %s", err)
 	}
-
 	log.Info("connected to SQLITE database")
+
 	return conn, nil
 }
 
+func sqlitePragmas(ctx context.Context, conn *DBConn) error {
+	var pragmas [][]string
+	if mode := config.GetDbSqliteJournalMode(); mode != "" {
+		// Set the user provided SQLite journal mode
+		pragmas = append(pragmas, []string{"journal_mode", mode})
+	}
+
+	if mode := config.GetDbSqliteSynchronous(); mode != "" {
+		// Set the user provided SQLite synchronous mode
+		pragmas = append(pragmas, []string{"synchronous", mode})
+	}
+
+	if size := config.GetDbSqliteCacheSize(); size > 0 {
+		// Set the user provided SQLite cache size (in kibibytes)
+		// Prepend a '-' character to this to indicate to sqlite
+		// that we're giving kibibytes rather than num pages.
+		// https://www.sqlite.org/pragma.html#pragma_cache_size
+		s := "-" + strconv.FormatUint(uint64(size/bytesize.KiB), 10)
+		pragmas = append(pragmas, []string{"cache_size", s})
+	}
+
+	if timeout := config.GetDbSqliteBusyTimeout(); timeout > 0 {
+		t := strconv.FormatInt(timeout.Milliseconds(), 10)
+		pragmas = append(pragmas, []string{"busy_timeout", t})
+	}
+
+	for _, p := range pragmas {
+		pk := p[0]
+		pv := p[1]
+
+		if _, err := conn.DB.ExecContext(ctx, "PRAGMA ?=?", bun.Ident(pk), bun.Safe(pv)); err != nil {
+			return fmt.Errorf("error executing sqlite pragma %s: %w", pk, err)
+		}
+
+		var res string
+		if err := conn.DB.NewRaw("PRAGMA ?", bun.Ident(pk)).Scan(ctx, &res); err != nil {
+			return fmt.Errorf("error scanning sqlite pragma %s: %w", pv, err)
+		}
+
+		log.Infof("sqlite pragma %s set to %s", pk, res)
+	}
+
+	return nil
+}
+
 func pgConn(ctx context.Context) (*DBConn, error) {
 	opts, err := deriveBunDBPGOptions() //nolint:contextcheck
 	if err != nil {
@@ -291,7 +329,10 @@ func pgConn(ctx context.Context) (*DBConn, error) {
 
 	sqldb := stdlib.OpenDB(*opts)
 
-	tweakConnectionValues(sqldb)
+	// https://bun.uptrace.dev/postgres/running-bun-in-production.html#database-sql
+	maxOpenConns := 4 * runtime.GOMAXPROCS(0)
+	sqldb.SetMaxOpenConns(maxOpenConns)
+	sqldb.SetMaxIdleConns(maxOpenConns)
 
 	conn := WrapDBConn(bun.NewDB(sqldb, pgdialect.New()))
 
@@ -311,10 +352,6 @@ func pgConn(ctx context.Context) (*DBConn, error) {
 // deriveBunDBPGOptions takes an application config and returns either a ready-to-use set of options
 // with sensible defaults, or an error if it's not satisfied by the provided config.
 func deriveBunDBPGOptions() (*pgx.ConnConfig, error) {
-	if strings.ToUpper(config.GetDbType()) != db.DBTypePostgres {
-		return nil, fmt.Errorf("expected db type of %s but got %s", db.DBTypePostgres, config.DbTypeFlag())
-	}
-
 	// these are all optional, the db adapter figures out defaults
 	address := config.GetDbAddress()
 
@@ -326,14 +363,14 @@ func deriveBunDBPGOptions() (*pgx.ConnConfig, error) {
 
 	var tlsConfig *tls.Config
 	switch config.GetDbTLSMode() {
-	case dbTLSModeDisable, dbTLSModeUnset:
+	case "", "disable":
 		break // nothing to do
-	case dbTLSModeEnable:
+	case "enable":
 		/* #nosec G402 */
 		tlsConfig = &tls.Config{
 			InsecureSkipVerify: true,
 		}
-	case dbTLSModeRequire:
+	case "require":
 		tlsConfig = &tls.Config{
 			InsecureSkipVerify: false,
 			ServerName:         address,
@@ -397,13 +434,6 @@ func deriveBunDBPGOptions() (*pgx.ConnConfig, error) {
 	return cfg, nil
 }
 
-// https://bun.uptrace.dev/postgres/running-bun-in-production.html#database-sql
-func tweakConnectionValues(sqldb *sql.DB) {
-	maxOpenConns := 4 * runtime.GOMAXPROCS(0)
-	sqldb.SetMaxOpenConns(maxOpenConns)
-	sqldb.SetMaxIdleConns(maxOpenConns)
-}
-
 /*
 	CONVERSION FUNCTIONS
 */

test/envparsing.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"advanced-throttling-multiplier":-1,"application-name":"gts","bind-address":"127.0.0.1","cache":{"gts":{"account-max-size":99,"account-sweep-freq":1000000000,"account-ttl":10800000000000,"block-max-size":100,"block-sweep-freq":10000000000,"block-ttl":300000000000,"domain-block-max-size":1000,"domain-block-sweep-freq":60000000000,"domain-block-ttl":86400000000000,"emoji-category-max-size":100,"emoji-category-sweep-freq":10000000000,"emoji-category-ttl":300000000000,"emoji-max-size":500,"emoji-sweep-freq":10000000000,"emoji-ttl":300000000000,"mention-max-size":500,"mention-sweep-freq":10000000000,"mention-ttl":300000000000,"notification-max-size":500,"notification-sweep-freq":10000000000,"notification-ttl":300000000000,"report-max-size":100,"report-sweep-freq":10000000000,"report-ttl":300000000000,"status-max-size":500,"status-sweep-freq":10000000000,"status-ttl":300000000000,"tombstone-max-size":100,"tombstone-sweep-freq":10000000000,"tombstone-ttl":300000000000,"user-max-size":100,"user-sweep-freq":10000000000,"user-ttl":300000000000}},"config-path":"internal/config/testdata/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-password":"hunter2","db-port":6969,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","dry-run":false,"email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-public-timeline":true,"instance-expose-suspended":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-link-existing":true,"oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-proxy":true,"storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}'
+EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"advanced-throttling-multiplier":-1,"application-name":"gts","bind-address":"127.0.0.1","cache":{"gts":{"account-max-size":99,"account-sweep-freq":1000000000,"account-ttl":10800000000000,"block-max-size":100,"block-sweep-freq":10000000000,"block-ttl":300000000000,"domain-block-max-size":1000,"domain-block-sweep-freq":60000000000,"domain-block-ttl":86400000000000,"emoji-category-max-size":100,"emoji-category-sweep-freq":10000000000,"emoji-category-ttl":300000000000,"emoji-max-size":500,"emoji-sweep-freq":10000000000,"emoji-ttl":300000000000,"mention-max-size":500,"mention-sweep-freq":10000000000,"mention-ttl":300000000000,"notification-max-size":500,"notification-sweep-freq":10000000000,"notification-ttl":300000000000,"report-max-size":100,"report-sweep-freq":10000000000,"report-ttl":300000000000,"status-max-size":500,"status-sweep-freq":10000000000,"status-ttl":300000000000,"tombstone-max-size":100,"tombstone-sweep-freq":10000000000,"tombstone-ttl":300000000000,"user-max-size":100,"user-sweep-freq":10000000000,"user-ttl":300000000000}},"config-path":"internal/config/testdata/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-password":"hunter2","db-port":6969,"db-sqlite-busy-timeout":1000000000,"db-sqlite-cache-size":0,"db-sqlite-journal-mode":"DELETE","db-sqlite-synchronous":"FULL","db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","dry-run":false,"email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-public-timeline":true,"instance-expose-suspended":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-link-existing":true,"oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-proxy":true,"storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}'
 
 # Set all the environment variables to 
 # ensure that these are parsed without panic
@@ -22,6 +22,10 @@ GTS_DB_PORT=6969 \
 GTS_DB_USER='sex-haver' \
 GTS_DB_PASSWORD='hunter2' \
 GTS_DB_DATABASE='gotosocial_prod' \
+GTS_DB_SQLITE_JOURNAL_MODE='DELETE' \
+GTS_DB_SQLITE_SYNCHRONOUS='FULL' \
+GTS_DB_SQLITE_CACHE_SIZE=0 \
+GTS_DB_SQLITE_BUSY_TIMEOUT='1s' \
 GTS_TLS_MODE='' \
 GTS_DB_TLS_CA_CERT='' \
 GTS_WEB_TEMPLATE_BASE_DIR='/root' \

testrig/config.go

@@ -19,6 +19,9 @@
 package testrig
 
 import (
+	"time"
+
+	"codeberg.org/gruf/go-bytesize"
 	"github.com/coreos/go-oidc/v3/oidc"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 )
@@ -43,12 +46,16 @@ func InitTestConfig() {
 	Port:            8080,
 	TrustedProxies:  []string{"127.0.0.1/32", "::1"},
 
-	DbType:     "sqlite",
+	DbType:              "sqlite",
-	DbAddress:  ":memory:",
+	DbAddress:           ":memory:",
-	DbPort:     5432,
+	DbPort:              5432,
-	DbUser:     "postgres",
+	DbUser:              "postgres",
-	DbPassword: "postgres",
+	DbPassword:          "postgres",
-	DbDatabase: "postgres",
+	DbDatabase:          "postgres",
+	DbSqliteJournalMode: "WAL",
+	DbSqliteSynchronous: "NORMAL",
+	DbSqliteCacheSize:   64 * bytesize.MiB,
+	DbSqliteBusyTimeout: time.Second * 30,
 
 	WebTemplateBaseDir: "./web/template/",
 	WebAssetBaseDir:    "./web/assets/",

testrig/db.go

@@ -73,15 +73,11 @@
 // value as the port instead.
 func NewTestDB() db.DB {
 	if alternateAddress := os.Getenv("GTS_DB_ADDRESS"); alternateAddress != "" {
-		config.Config(func(cfg *config.Configuration) {
+		config.SetDbAddress(alternateAddress)
-			cfg.DbAddress = alternateAddress
-		})
 	}
 
 	if alternateDBType := os.Getenv("GTS_DB_TYPE"); alternateDBType != "" {
-		config.Config(func(cfg *config.Configuration) {
+		config.SetDbType(alternateDBType)
-			cfg.DbType = alternateDBType
-		})
 	}
 
 	if alternateDBPort := os.Getenv("GTS_DB_PORT"); alternateDBPort != "" {
@@ -89,9 +85,7 @@ func NewTestDB() db.DB {
 		if err != nil {
 			panic(err)
 		}
-		config.Config(func(cfg *config.Configuration) {
+		config.SetDbPort(int(port))
-			cfg.DbPort = int(port)
-		})
 	}
 
 	var state state.State