[feature] Add admin media prune orphaned CLI command (#1146)

* add FilePath regex

* add `admin media prune orphaned` command

* add prune orphaned function to media manager

* don't mark flag as required

* document admin media prune orphaned cmd

* oh envparsing.sh you coy minx

cmd/gotosocial/action/admin/media/prune/orphaned.go

@@ -0,0 +1,72 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package prune
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
+	"github.com/superseriousbusiness/gotosocial/internal/log"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+	gtsstorage "github.com/superseriousbusiness/gotosocial/internal/storage"
+)
+
+// Orphaned prunes orphaned media from storage.
+var Orphaned action.GTSAction = func(ctx context.Context) error {
+	dbService, err := bundb.NewBunDBService(ctx)
+	if err != nil {
+		return fmt.Errorf("error creating dbservice: %s", err)
+	}
+
+	storage, err := gtsstorage.AutoConfig()
+	if err != nil {
+		return fmt.Errorf("error creating storage backend: %w", err)
+	}
+
+	manager, err := media.NewManager(dbService, storage)
+	if err != nil {
+		return fmt.Errorf("error instantiating mediamanager: %s", err)
+	}
+
+	dry := config.GetAdminMediaPruneDryRun()
+
+	pruned, err := manager.PruneOrphaned(ctx, dry)
+	if err != nil {
+		return fmt.Errorf("error pruning: %s", err)
+	}
+
+	if dry {
+		log.Infof("DRY RUN: %d stored items are orphaned and eligible to be pruned", pruned)
+	} else {
+		log.Infof("%d stored items were orphaned and pruned", pruned)
+	}
+
+	if err := storage.Close(); err != nil {
+		return fmt.Errorf("error closing storage backend: %w", err)
+	}
+
+	if err := dbService.Stop(ctx); err != nil {
+		return fmt.Errorf("error closing dbservice: %s", err)
+	}
+
+	return nil
+}

cmd/gotosocial/admin.go

@@ -21,6 +21,7 @@
 import (
 	"github.com/spf13/cobra"
 	"github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action/admin/account"
+	"github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action/admin/media/prune"
 	"github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action/admin/trans"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 )
@@ -152,5 +153,40 @@ func adminCommands() *cobra.Command {
 	config.AddAdminTrans(adminImportCmd)
 	adminCmd.AddCommand(adminImportCmd)
 
+	/*
+		ADMIN MEDIA COMMANDS
+	*/
+
+	adminMediaCmd := &cobra.Command{
+		Use:   "media",
+		Short: "admin commands related stored media attachments/emojis",
+	}
+
+	/*
+		ADMIN MEDIA PRUNE COMMANDS
+	*/
+	adminMediaPruneCmd := &cobra.Command{
+		Use:   "prune",
+		Short: "admin commands for pruning unused/orphaned media from storage",
+	}
+	config.AddAdminMediaPrune(adminMediaPruneCmd)
+
+	adminMediaPruneOrphanedCmd := &cobra.Command{
+		Use:   "orphaned",
+		Short: "prune orphaned media from storage",
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			return preRun(preRunArgs{cmd: cmd})
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return run(cmd.Context(), prune.Orphaned)
+		},
+	}
+	config.AddAdminMediaPrune(adminMediaPruneOrphanedCmd)
+	adminMediaPruneCmd.AddCommand(adminMediaPruneOrphanedCmd)
+
+	adminMediaCmd.AddCommand(adminMediaPruneCmd)
+
+	adminCmd.AddCommand(adminMediaCmd)
+
 	return adminCmd
 }

docs/admin/cli.md

@@ -35,7 +35,7 @@ You can set these options using environment variables, passing them as CLI flags
 
 ## gotosocial admin
 
-Contains `account` subcommands.
+Contains `account`, `export`, `import`, and `media` subcommands.
 
 ### gotosocial admin account create
 
@@ -254,3 +254,36 @@ Example:
 ```bash
 gotosocial admin import --path example.json --config-path config.yaml
 ```
+
+### gotosocial admin media prune orphaned
+
+This command can be used to prune orphaned media from your GoToSocial.
+
+Orphaned media is defined as media that is in storage under a key that matches the format used by GoToSocial, but which does not have a corresponding database entry. This is useful for excising files that may be remaining from a previous installation, or files that were placed in storage mistakenly.
+
+**This command only works when GoToSocial is not running, since it acquires an exclusive lock on storage. Stop GoToSocial first before running this command!**
+
+```text
+prune orphaned media from storage
+
+Usage:
+  gotosocial admin media prune orphaned [flags]
+
+Flags:
+      --dry-run   perform a dry run and only log number of items eligible for pruning (default true)
+  -h, --help      help for orphaned
+```
+
+By default, this command performs a dry run, which will log how many items can be pruned. To do it for real, add `--dry-run=false` to the command.
+
+Example (dry run):
+
+```bash
+gotosocial admin media prune orphaned
+```
+
+Example (for real):
+
+```bash
+gotosocial admin media prune orphaned --dry-run=false
+```

internal/config/config.go

@@ -125,14 +125,15 @@ type Configuration struct {
 	SyslogProtocol string `name:"syslog-protocol" usage:"Protocol to use when directing logs to syslog. Leave empty to connect to local syslog."`
 	SyslogAddress  string `name:"syslog-address" usage:"Address:port to send syslog logs to. Leave empty to connect to local syslog."`
 
-	// TODO: move these elsewhere, these are more ephemeral vs long-running flags like above
-	AdminAccountUsername string `name:"username" usage:"the username to create/delete/etc"`
-	AdminAccountEmail    string `name:"email" usage:"the email address of this account"`
-	AdminAccountPassword string `name:"password" usage:"the password to set for this account"`
-	AdminTransPath       string `name:"path" usage:"the path of the file to import from/export to"`
-
 	AdvancedCookiesSamesite   string `name:"advanced-cookies-samesite" usage:"'strict' or 'lax', see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite"`
 	AdvancedRateLimitRequests int    `name:"advanced-rate-limit-requests" usage:"Amount of HTTP requests to permit within a 5 minute window. 0 or less turns rate limiting off."`
+
+	// TODO: move these elsewhere, these are more ephemeral vs long-running flags like above
+	AdminAccountUsername  string `name:"username" usage:"the username to create/delete/etc"`
+	AdminAccountEmail     string `name:"email" usage:"the email address of this account"`
+	AdminAccountPassword  string `name:"password" usage:"the password to set for this account"`
+	AdminTransPath        string `name:"path" usage:"the path of the file to import from/export to"`
+	AdminMediaPruneDryRun bool   `name:"dry-run" usage:"perform a dry run and only log number of items eligible for pruning"`
 }
 
 // MarshalMap will marshal current Configuration into a map structure (useful for JSON).

internal/config/flags.go

@@ -178,3 +178,10 @@ func AddAdminTrans(cmd *cobra.Command) {
 		panic(err)
 	}
 }
+
+// AddAdminMediaPrune attaches flags pertaining to media storage prune commands.
+func AddAdminMediaPrune(cmd *cobra.Command) {
+	name := AdminMediaPruneDryRunFlag()
+	usage := fieldtag("AdminMediaPruneDryRun", "usage")
+	cmd.Flags().Bool(name, true, usage)
+}

internal/config/helpers.gen.go

@@ -1745,6 +1745,56 @@ func GetSyslogAddress() string { return global.GetSyslogAddress() }
 // SetSyslogAddress safely sets the value for global configuration 'SyslogAddress' field
 func SetSyslogAddress(v string) { global.SetSyslogAddress(v) }
 
+// GetAdvancedCookiesSamesite safely fetches the Configuration value for state's 'AdvancedCookiesSamesite' field
+func (st *ConfigState) GetAdvancedCookiesSamesite() (v string) {
+	st.mutex.Lock()
+	v = st.config.AdvancedCookiesSamesite
+	st.mutex.Unlock()
+	return
+}
+
+// SetAdvancedCookiesSamesite safely sets the Configuration value for state's 'AdvancedCookiesSamesite' field
+func (st *ConfigState) SetAdvancedCookiesSamesite(v string) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.AdvancedCookiesSamesite = v
+	st.reloadToViper()
+}
+
+// AdvancedCookiesSamesiteFlag returns the flag name for the 'AdvancedCookiesSamesite' field
+func AdvancedCookiesSamesiteFlag() string { return "advanced-cookies-samesite" }
+
+// GetAdvancedCookiesSamesite safely fetches the value for global configuration 'AdvancedCookiesSamesite' field
+func GetAdvancedCookiesSamesite() string { return global.GetAdvancedCookiesSamesite() }
+
+// SetAdvancedCookiesSamesite safely sets the value for global configuration 'AdvancedCookiesSamesite' field
+func SetAdvancedCookiesSamesite(v string) { global.SetAdvancedCookiesSamesite(v) }
+
+// GetAdvancedRateLimitRequests safely fetches the Configuration value for state's 'AdvancedRateLimitRequests' field
+func (st *ConfigState) GetAdvancedRateLimitRequests() (v int) {
+	st.mutex.Lock()
+	v = st.config.AdvancedRateLimitRequests
+	st.mutex.Unlock()
+	return
+}
+
+// SetAdvancedRateLimitRequests safely sets the Configuration value for state's 'AdvancedRateLimitRequests' field
+func (st *ConfigState) SetAdvancedRateLimitRequests(v int) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.AdvancedRateLimitRequests = v
+	st.reloadToViper()
+}
+
+// AdvancedRateLimitRequestsFlag returns the flag name for the 'AdvancedRateLimitRequests' field
+func AdvancedRateLimitRequestsFlag() string { return "advanced-rate-limit-requests" }
+
+// GetAdvancedRateLimitRequests safely fetches the value for global configuration 'AdvancedRateLimitRequests' field
+func GetAdvancedRateLimitRequests() int { return global.GetAdvancedRateLimitRequests() }
+
+// SetAdvancedRateLimitRequests safely sets the value for global configuration 'AdvancedRateLimitRequests' field
+func SetAdvancedRateLimitRequests(v int) { global.SetAdvancedRateLimitRequests(v) }
+
 // GetAdminAccountUsername safely fetches the Configuration value for state's 'AdminAccountUsername' field
 func (st *ConfigState) GetAdminAccountUsername() (v string) {
 	st.mutex.Lock()
@@ -1845,52 +1895,27 @@ func GetAdminTransPath() string { return global.GetAdminTransPath() }
 // SetAdminTransPath safely sets the value for global configuration 'AdminTransPath' field
 func SetAdminTransPath(v string) { global.SetAdminTransPath(v) }
 
-// GetAdvancedCookiesSamesite safely fetches the Configuration value for state's 'AdvancedCookiesSamesite' field
+// GetAdminMediaPruneDryRun safely fetches the Configuration value for state's 'AdminMediaPruneDryRun' field
-func (st *ConfigState) GetAdvancedCookiesSamesite() (v string) {
+func (st *ConfigState) GetAdminMediaPruneDryRun() (v bool) {
 	st.mutex.Lock()
-	v = st.config.AdvancedCookiesSamesite
+	v = st.config.AdminMediaPruneDryRun
 	st.mutex.Unlock()
 	return
 }
 
-// SetAdvancedCookiesSamesite safely sets the Configuration value for state's 'AdvancedCookiesSamesite' field
+// SetAdminMediaPruneDryRun safely sets the Configuration value for state's 'AdminMediaPruneDryRun' field
-func (st *ConfigState) SetAdvancedCookiesSamesite(v string) {
+func (st *ConfigState) SetAdminMediaPruneDryRun(v bool) {
 	st.mutex.Lock()
 	defer st.mutex.Unlock()
-	st.config.AdvancedCookiesSamesite = v
+	st.config.AdminMediaPruneDryRun = v
 	st.reloadToViper()
 }
 
-// AdvancedCookiesSamesiteFlag returns the flag name for the 'AdvancedCookiesSamesite' field
+// AdminMediaPruneDryRunFlag returns the flag name for the 'AdminMediaPruneDryRun' field
-func AdvancedCookiesSamesiteFlag() string { return "advanced-cookies-samesite" }
+func AdminMediaPruneDryRunFlag() string { return "dry-run" }
 
-// GetAdvancedCookiesSamesite safely fetches the value for global configuration 'AdvancedCookiesSamesite' field
+// GetAdminMediaPruneDryRun safely fetches the value for global configuration 'AdminMediaPruneDryRun' field
-func GetAdvancedCookiesSamesite() string { return global.GetAdvancedCookiesSamesite() }
+func GetAdminMediaPruneDryRun() bool { return global.GetAdminMediaPruneDryRun() }
 
-// SetAdvancedCookiesSamesite safely sets the value for global configuration 'AdvancedCookiesSamesite' field
+// SetAdminMediaPruneDryRun safely sets the value for global configuration 'AdminMediaPruneDryRun' field
-func SetAdvancedCookiesSamesite(v string) { global.SetAdvancedCookiesSamesite(v) }
+func SetAdminMediaPruneDryRun(v bool) { global.SetAdminMediaPruneDryRun(v) }
-
-// GetAdvancedRateLimitRequests safely fetches the Configuration value for state's 'AdvancedRateLimitRequests' field
-func (st *ConfigState) GetAdvancedRateLimitRequests() (v int) {
-	st.mutex.Lock()
-	v = st.config.AdvancedRateLimitRequests
-	st.mutex.Unlock()
-	return
-}
-
-// SetAdvancedRateLimitRequests safely sets the Configuration value for state's 'AdvancedRateLimitRequests' field
-func (st *ConfigState) SetAdvancedRateLimitRequests(v int) {
-	st.mutex.Lock()
-	defer st.mutex.Unlock()
-	st.config.AdvancedRateLimitRequests = v
-	st.reloadToViper()
-}
-
-// AdvancedRateLimitRequestsFlag returns the flag name for the 'AdvancedRateLimitRequests' field
-func AdvancedRateLimitRequestsFlag() string { return "advanced-rate-limit-requests" }
-
-// GetAdvancedRateLimitRequests safely fetches the value for global configuration 'AdvancedRateLimitRequests' field
-func GetAdvancedRateLimitRequests() int { return global.GetAdvancedRateLimitRequests() }
-
-// SetAdvancedRateLimitRequests safely sets the value for global configuration 'AdvancedRateLimitRequests' field
-func SetAdvancedRateLimitRequests(v int) { global.SetAdvancedRateLimitRequests(v) }

internal/media/manager.go

@@ -91,6 +91,12 @@ type Manager interface {
 	//
 	// The returned int is the amount of media that was pruned by this function.
 	PruneUnusedLocalAttachments(ctx context.Context) (int, error)
+	// PruneOrphaned prunes files that exist in storage but which do not have a corresponding
+	// entry in the database.
+	//
+	// If dry is true, then nothing will be changed, only the amount that *would* be removed
+	// is returned to the caller.
+	PruneOrphaned(ctx context.Context, dry bool) (int, error)
 
 	// Stop stops the underlying worker pool of the manager. It should be called
 	// when closing GoToSocial in order to cleanly finish any in-progress jobs.

internal/media/pruneorphaned.go

@@ -0,0 +1,127 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package media
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/log"
+	"github.com/superseriousbusiness/gotosocial/internal/regexes"
+)
+
+func (m *manager) PruneOrphaned(ctx context.Context, dry bool) (int, error) {
+	var totalPruned int
+
+	// keys in storage will look like the following:
+	// `[ACCOUNT_ID]/[MEDIA_TYPE]/[MEDIA_SIZE]/[FILE_NAME]`
+	// we can filter out keys we're not interested in by
+	// matching through a regex
+	var matchCount int
+	match := func(storageKey string) bool {
+		if regexes.FilePath.MatchString(storageKey) {
+			matchCount++
+			return true
+		}
+		return false
+	}
+
+	log.Info("checking storage keys for orphaned pruning candidates...")
+	iterator, err := m.storage.Iterator(ctx, match)
+	if err != nil {
+		return 0, fmt.Errorf("PruneOrphaned: error getting storage iterator: %s", err)
+	}
+
+	// make sure we have some keys, and also advance
+	// the iterator to the first non-empty key
+	if !iterator.Next() {
+		return 0, nil
+	}
+
+	// for each key in the iterator, check if entry is orphaned
+	log.Info("got %d orphaned pruning candidates, checking for orphaned status, please wait...")
+	var checkedKeys int
+	orphanedKeys := make([]string, 0, matchCount)
+	for key := iterator.Key(); iterator.Next(); key = iterator.Key() {
+		if m.orphaned(ctx, key) {
+			orphanedKeys = append(orphanedKeys, key)
+		}
+		checkedKeys++
+		if checkedKeys%50 == 0 {
+			log.Infof("checked %d of %d orphaned pruning candidates...", checkedKeys, matchCount)
+		}
+	}
+	iterator.Release()
+
+	if !dry {
+		// the real deal, we have to delete stuff
+		for _, key := range orphanedKeys {
+			log.Infof("key %s corresponds to orphaned media, will remove it now", key)
+			if err := m.storage.Delete(ctx, key); err != nil {
+				log.Errorf("error deleting item with key %s from storage: %s", key, err)
+				continue
+			}
+			totalPruned++
+		}
+	} else {
+		// just a dry run, don't delete anything
+		for _, key := range orphanedKeys {
+			log.Infof("DRY RUN: key %s corresponds to orphaned media which would be deleted", key)
+			totalPruned++
+		}
+	}
+
+	return totalPruned, nil
+}
+
+func (m *manager) orphaned(ctx context.Context, key string) bool {
+	pathParts := regexes.FilePath.FindStringSubmatch(key)
+	if len(pathParts) != 6 {
+		return false
+	}
+
+	mediaType := pathParts[2]
+	mediaID := pathParts[4]
+
+	var orphaned bool
+	switch Type(mediaType) {
+	case TypeAttachment, TypeHeader, TypeAvatar:
+		if _, err := m.db.GetAttachmentByID(ctx, mediaID); err != nil {
+			if errors.Is(err, db.ErrNoEntries) {
+				orphaned = true
+			} else {
+				log.Errorf("orphaned: error calling GetAttachmentByID: %s", err)
+			}
+		}
+	case TypeEmoji:
+		if _, err := m.db.GetEmojiByID(ctx, mediaID); err != nil {
+			if errors.Is(err, db.ErrNoEntries) {
+				orphaned = true
+			} else {
+				log.Errorf("orphaned: error calling GetEmojiByID: %s", err)
+			}
+		}
+	default:
+		orphaned = true
+	}
+
+	return orphaned
+}

internal/media/pruneorphaned_test.go

@@ -0,0 +1,82 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package media_test
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+)
+
+type PruneOrphanedTestSuite struct {
+	MediaStandardTestSuite
+}
+
+func (suite *PruneOrphanedTestSuite) TestPruneOrphanedDry() {
+	// add a big orphan panda to store
+	b, err := os.ReadFile("./test/big-panda.gif")
+	if err != nil {
+		panic(err)
+	}
+
+	pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachments/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif"
+	if err := suite.storage.PutStream(context.Background(), pandaPath, bytes.NewBuffer(b)); err != nil {
+		panic(err)
+	}
+
+	// dry run should show up 1 orphaned panda
+	totalPruned, err := suite.manager.PruneOrphaned(context.Background(), true)
+	suite.NoError(err)
+	suite.Equal(1, totalPruned)
+
+	// panda should still be in storage
+	hasKey, err := suite.storage.Has(context.Background(), pandaPath)
+	suite.NoError(err)
+	suite.True(hasKey)
+}
+
+func (suite *PruneOrphanedTestSuite) TestPruneOrphanedMoist() {
+	// add a big orphan panda to store
+	b, err := os.ReadFile("./test/big-panda.gif")
+	if err != nil {
+		panic(err)
+	}
+
+	pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachments/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif"
+	if err := suite.storage.PutStream(context.Background(), pandaPath, bytes.NewBuffer(b)); err != nil {
+		panic(err)
+	}
+
+	// should show up 1 orphaned panda
+	totalPruned, err := suite.manager.PruneOrphaned(context.Background(), false)
+	suite.NoError(err)
+	suite.Equal(1, totalPruned)
+
+	// panda should no longer be in storage
+	hasKey, err := suite.storage.Has(context.Background(), pandaPath)
+	suite.NoError(err)
+	suite.False(hasKey)
+}
+
+func TestPruneOrphanedTestSuite(t *testing.T) {
+	suite.Run(t, &PruneOrphanedTestSuite{})
+}

internal/regexes/regexes.go

@@ -140,6 +140,13 @@
 	// BlockPath parses a path that validates and captures the username part and the ulid part
 	// from eg /users/example_username/blocks/01F7XT5JZW1WMVSW1KADS8PVDH
 	BlockPath = regexp.MustCompile(blockPath)
+
+	filePath = fmt.Sprintf(`^(%s)/([a-z]+)/([a-z]+)/(%s)\.([a-z]+)$`, ulid, ulid)
+	// FilePath parses a file storage path of the form [ACCOUNT_ID]/[MEDIA_TYPE]/[MEDIA_SIZE]/[FILE_NAME]
+	// eg 01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpeg
+	// It captures the account id, media type, media size, file name, and file extension, eg
+	// `01F8MH1H7YV1Z7D2C8K2730QBF`, `attachment`, `small`, `01F8MH8RMYQ6MSNY3JM2XT1CQ5`, `jpeg`.
+	FilePath = regexp.MustCompile(filePath)
 )
 
 // bufpool is a memory pool of byte buffers for use in our regex utility functions.

test/envparsing.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"application-name":"gts","bind-address":"127.0.0.1","config-path":"internal/config/testdata/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-password":"hunter2","db-port":6969,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-public-timeline":true,"instance-expose-suspended":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-proxy":true,"storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}'
+EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"application-name":"gts","bind-address":"127.0.0.1","config-path":"internal/config/testdata/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-password":"hunter2","db-port":6969,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","dry-run":false,"email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-public-timeline":true,"instance-expose-suspended":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-proxy":true,"storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}'
 
 # Set all the environment variables to 
 # ensure that these are parsed without panic