compiling now

internal/api/client/admin/emojicreate.go

@@ -27,7 +27,6 @@
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/api"
 	"github.com/superseriousbusiness/gotosocial/internal/api/model"
-	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/validate"
 )
@@ -133,10 +132,5 @@ func validateCreateEmoji(form *model.EmojiCreateRequest) error {
 		return errors.New("no emoji given")
 	}
 
-	// a very superficial check to see if the media size limit is exceeded
-	if form.Image.Size > media.EmojiMaxBytes {
-		return fmt.Errorf("file size limit exceeded: limit is %d bytes but emoji was %d bytes", media.EmojiMaxBytes, form.Image.Size)
-	}
-
 	return validate.EmojiShortcode(form.Shortcode)
 }

internal/db/bundb/errors.go

@@ -35,7 +35,7 @@ func processSQLiteError(err error) db.Error {
 
 	// Handle supplied error code:
 	switch sqliteErr.Code() {
-	case sqlite3.SQLITE_CONSTRAINT_UNIQUE:
+	case sqlite3.SQLITE_CONSTRAINT_UNIQUE, sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY:
 		return db.ErrAlreadyExists
 	default:
 		return err

internal/federation/dereferencing/account.go

@@ -246,25 +246,49 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount *
 	}
 
 	if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) {
-		a, err := d.mediaManager.ProcessRemoteHeaderOrAvatar(ctx, t, &gtsmodel.MediaAttachment{
+		avatarIRI, err := url.Parse(targetAccount.AvatarRemoteURL)
-			RemoteURL: targetAccount.AvatarRemoteURL,
-			Avatar:    true,
-		}, targetAccount.ID)
 		if err != nil {
-			return fmt.Errorf("error processing avatar for user: %s", err)
+			return err
 		}
-		targetAccount.AvatarMediaAttachmentID = a.ID
+
+		data, err := t.DereferenceMedia(ctx, avatarIRI)
+		if err != nil {
+			return err
+		}
+
+		media, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, targetAccount.AvatarRemoteURL)
+		if err != nil {
+			return err
+		}
+
+		if err := media.SetAsAvatar(ctx); err != nil {
+			return err
+		}
+
+		targetAccount.AvatarMediaAttachmentID = media.AttachmentID()
 	}
 
 	if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) {
-		a, err := d.mediaManager.ProcessRemoteHeaderOrAvatar(ctx, t, &gtsmodel.MediaAttachment{
+		headerIRI, err := url.Parse(targetAccount.HeaderRemoteURL)
-			RemoteURL: targetAccount.HeaderRemoteURL,
-			Header:    true,
-		}, targetAccount.ID)
 		if err != nil {
-			return fmt.Errorf("error processing header for user: %s", err)
+			return err
 		}
-		targetAccount.HeaderMediaAttachmentID = a.ID
+
+		data, err := t.DereferenceMedia(ctx, headerIRI)
+		if err != nil {
+			return err
+		}
+
+		media, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, targetAccount.HeaderRemoteURL)
+		if err != nil {
+			return err
+		}
+
+		if err := media.SetAsHeader(ctx); err != nil {
+			return err
+		}
+
+		targetAccount.HeaderMediaAttachmentID = media.AttachmentID()
 	}
 	return nil
 }

internal/federation/dereferencing/attachment.go

@@ -1,102 +0,0 @@
-/*
-   GoToSocial
-   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
-
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-package dereferencing
-
-import (
-	"context"
-	"fmt"
-	"net/url"
-
-	"github.com/sirupsen/logrus"
-	"github.com/superseriousbusiness/gotosocial/internal/db"
-	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-)
-
-func (d *deref) GetRemoteAttachment(ctx context.Context, requestingUsername string, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) {
-	if minAttachment.RemoteURL == "" {
-		return nil, fmt.Errorf("GetRemoteAttachment: minAttachment remote URL was empty")
-	}
-	remoteAttachmentURL := minAttachment.RemoteURL
-
-	l := logrus.WithFields(logrus.Fields{
-		"username":            requestingUsername,
-		"remoteAttachmentURL": remoteAttachmentURL,
-	})
-
-	// return early if we already have the attachment somewhere
-	maybeAttachment := &gtsmodel.MediaAttachment{}
-	where := []db.Where{
-		{
-			Key:   "remote_url",
-			Value: remoteAttachmentURL,
-		},
-	}
-
-	if err := d.db.GetWhere(ctx, where, maybeAttachment); err == nil {
-		// we already the attachment in the database
-		l.Debugf("GetRemoteAttachment: attachment already exists with id %s", maybeAttachment.ID)
-		return maybeAttachment, nil
-	}
-
-	a, err := d.RefreshAttachment(ctx, requestingUsername, minAttachment)
-	if err != nil {
-		return nil, fmt.Errorf("GetRemoteAttachment: error refreshing attachment: %s", err)
-	}
-
-	if err := d.db.Put(ctx, a); err != nil {
-		if err != db.ErrAlreadyExists {
-			return nil, fmt.Errorf("GetRemoteAttachment: error inserting attachment: %s", err)
-		}
-	}
-
-	return a, nil
-}
-
-func (d *deref) RefreshAttachment(ctx context.Context, requestingUsername string, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) {
-	// it just doesn't exist or we have to refresh
-	if minAttachment.AccountID == "" {
-		return nil, fmt.Errorf("RefreshAttachment: minAttachment account ID was empty")
-	}
-
-	if minAttachment.File.ContentType == "" {
-		return nil, fmt.Errorf("RefreshAttachment: minAttachment.file.contentType was empty")
-	}
-
-	t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername)
-	if err != nil {
-		return nil, fmt.Errorf("RefreshAttachment: error creating transport: %s", err)
-	}
-
-	derefURI, err := url.Parse(minAttachment.RemoteURL)
-	if err != nil {
-		return nil, err
-	}
-
-	attachmentBytes, err := t.DereferenceMedia(ctx, derefURI, minAttachment.File.ContentType)
-	if err != nil {
-		return nil, fmt.Errorf("RefreshAttachment: error dereferencing media: %s", err)
-	}
-
-	a, err := d.mediaManager.ProcessAttachment(ctx, attachmentBytes, minAttachment)
-	if err != nil {
-		return nil, fmt.Errorf("RefreshAttachment: error processing attachment: %s", err)
-	}
-
-	return a, nil
-}

internal/federation/dereferencing/dereferencer.go

@@ -41,34 +41,7 @@ type Dereferencer interface {
 
 	GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error)
 
-	// GetRemoteAttachment takes a minimal attachment struct and converts it into a fully fleshed out attachment, stored in the database and instance storage.
+	GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string) (*media.Media, error)
-	//
-	// The parameter minAttachment must have at least the following fields defined:
-	//   * minAttachment.RemoteURL
-	//   * minAttachment.AccountID
-	//   * minAttachment.File.ContentType
-	//
-	// The returned attachment will have an ID generated for it, so no need to generate one beforehand.
-	// A blurhash will also be generated for the attachment.
-	//
-	// Most other fields will be preserved on the passed attachment, including:
-	//   * minAttachment.StatusID
-	//   * minAttachment.CreatedAt
-	//   * minAttachment.UpdatedAt
-	//   * minAttachment.FileMeta
-	//   * minAttachment.AccountID
-	//   * minAttachment.Description
-	//   * minAttachment.ScheduledStatusID
-	//   * minAttachment.Thumbnail.RemoteURL
-	//   * minAttachment.Avatar
-	//   * minAttachment.Header
-	//
-	// GetRemoteAttachment will return early if an attachment with the same value as minAttachment.RemoteURL
-	// is found in the database -- then that attachment will be returned and nothing else will be changed or stored.
-	GetRemoteAttachment(ctx context.Context, requestingUsername string, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error)
-	// RefreshAttachment is like GetRemoteAttachment, but the attachment will always be dereferenced again,
-	// whether or not it was already stored in the database.
-	RefreshAttachment(ctx context.Context, requestingUsername string, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error)
 
 	DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error
 	DereferenceThread(ctx context.Context, username string, statusIRI *url.URL) error

internal/federation/dereferencing/media.go

@@ -0,0 +1,55 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+)
+
+func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string) (*media.Media, error) {
+	if accountID == "" {
+		return nil, fmt.Errorf("RefreshAttachment: minAttachment account ID was empty")
+	}
+
+	t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername)
+	if err != nil {
+		return nil, fmt.Errorf("RefreshAttachment: error creating transport: %s", err)
+	}
+
+	derefURI, err := url.Parse(remoteURL)
+	if err != nil {
+		return nil, err
+	}
+
+	data, err := t.DereferenceMedia(ctx, derefURI)
+	if err != nil {
+		return nil, fmt.Errorf("RefreshAttachment: error dereferencing media: %s", err)
+	}
+
+	m, err := d.mediaManager.ProcessMedia(ctx, data, accountID, remoteURL)
+	if err != nil {
+		return nil, fmt.Errorf("RefreshAttachment: error processing attachment: %s", err)
+	}
+
+	return m, nil
+}

internal/federation/dereferencing/media_test.go

@@ -31,6 +31,8 @@ type AttachmentTestSuite struct {
 }
 
 func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() {
+	ctx := context.Background()
+	
 	fetchingAccount := suite.testAccounts["local_account_1"]
 
 	attachmentOwner := "01FENS9F666SEQ6TYQWEEY78GM"
@@ -39,18 +41,12 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() {
 	attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg"
 	attachmentDescription := "It's a cute plushie."
 
-	minAttachment := &gtsmodel.MediaAttachment{
+	media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL)
-		RemoteURL: attachmentURL,
-		AccountID: attachmentOwner,
-		StatusID:  attachmentStatus,
-		File: gtsmodel.File{
-			ContentType: attachmentContentType,
-		},
-		Description: attachmentDescription,
-	}
-
-	attachment, err := suite.dereferencer.GetRemoteAttachment(context.Background(), fetchingAccount.Username, minAttachment)
 	suite.NoError(err)
+
+	attachment, err := media.LoadAttachment(ctx)
+	suite.NoError(err)
+
 	suite.NotNil(attachment)
 
 	suite.Equal(attachmentOwner, attachment.AccountID)

internal/federation/dereferencing/status.go

@@ -393,9 +393,15 @@ func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel.
 		a.AccountID = status.AccountID
 		a.StatusID = status.ID
 
-		attachment, err := d.GetRemoteAttachment(ctx, requestingUsername, a)
+		media, err := d.GetRemoteMedia(ctx, requestingUsername, a.AccountID, a.RemoteURL)
 		if err != nil {
-			logrus.Errorf("populateStatusAttachments: couldn't get remote attachment %s: %s", a.RemoteURL, err)
+			logrus.Errorf("populateStatusAttachments: couldn't get remote media %s: %s", a.RemoteURL, err)
+			continue
+		}
+
+		attachment, err := media.LoadAttachment(ctx)
+		if err != nil {
+			logrus.Errorf("populateStatusAttachments: couldn't load remote attachment %s: %s", a.RemoteURL, err)
 			continue
 		}
 

internal/media/manager.go

@@ -37,7 +37,17 @@
 
 // Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs.
 type Manager interface {
-	ProcessMedia(ctx context.Context, data []byte, accountID string) (*Media, error)
+	// ProcessMedia begins the process of decoding and storing the given data as a piece of media (aka an attachment).
+	// It will return a pointer to a Media struct upon which further actions can be performed, such as getting
+	// the finished media, thumbnail, decoded bytes, attachment, and setting additional fields.
+	//
+	// accountID should be the account that the media belongs to.
+	//
+	// RemoteURL is optional, and can be an empty string. Setting this to a non-empty string indicates that
+	// the piece of media originated on a remote instance and has been dereferenced to be cached locally.
+	ProcessMedia(ctx context.Context, data []byte, accountID string, remoteURL string) (*Media, error)
+
+	ProcessEmoji(ctx context.Context, data []byte, accountID string, remoteURL string) (*Media, error)
 }
 
 type manager struct {
@@ -70,7 +80,7 @@ func New(database db.DB, storage *kv.KVStore) (Manager, error) {
 	INTERFACE FUNCTIONS
 */
 
-func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID string) (*Media, error) {
+func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID string, remoteURL string) (*Media, error) {
 	contentType, err := parseContentType(data)
 	if err != nil {
 		return nil, err
@@ -85,7 +95,7 @@ func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID strin
 
 	switch mainType {
 	case mimeImage:
-		media, err := m.preProcessImage(ctx, data, contentType, accountID)
+		media, err := m.preProcessImage(ctx, data, contentType, accountID, remoteURL)
 		if err != nil {
 			return nil, err
 		}
@@ -97,7 +107,7 @@ func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID strin
 				return
 			default:
 				// start preloading the media for the caller's convenience
-				media.PreLoad(innerCtx)
+				media.preLoad(innerCtx)
 			}
 		})
 
@@ -107,8 +117,12 @@ func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID strin
 	}
 }
 
+func (m *manager) ProcessEmoji(ctx context.Context, data []byte, accountID string, remoteURL string) (*Media, error)  {
+	return nil, nil
+}
+
 // preProcessImage initializes processing
-func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType string, accountID string) (*Media, error) {
+func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType string, accountID string, remoteURL string) (*Media, error) {
 	if !supportedImage(contentType) {
 		return nil, fmt.Errorf("image type %s not supported", contentType)
 	}
@@ -128,6 +142,7 @@ func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType
 		ID:         id,
 		UpdatedAt:  time.Now(),
 		URL:        uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeOriginal), id, extension),
+		RemoteURL:  remoteURL,
 		Type:       gtsmodel.FileTypeImage,
 		AccountID:  accountID,
 		Processing: 0,

internal/media/manager_test.go

@@ -0,0 +1,4 @@
+package media_test
+
+
+

internal/media/media.go

@@ -1,9 +1,28 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package media
 
 import (
 	"context"
 	"fmt"
 	"sync"
+	"time"
 
 	"codeberg.org/gruf/go-store/kv"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -26,7 +45,8 @@ type Media struct {
 		attachment will be updated incrementally as media goes through processing
 	*/
 
-	attachment *gtsmodel.MediaAttachment
+	attachment *gtsmodel.MediaAttachment // will only be set if the media is an attachment
+	emoji      *gtsmodel.Emoji           // will only be set if the media is an emoji
 	rawData    []byte
 
 	/*
@@ -86,17 +106,10 @@ func (m *Media) Thumb(ctx context.Context) (*ImageMeta, error) {
 		m.attachment.Thumbnail.FileSize = thumb.size
 
 		// put or update the attachment in the database
-		if err := m.database.Put(ctx, m.attachment); err != nil {
+		if err := putOrUpdateAttachment(ctx, m.database, m.attachment); err != nil {
-			if err != db.ErrAlreadyExists {
+			m.err = err
-				m.err = fmt.Errorf("error putting attachment: %s", err)
 			m.thumbstate = errored
-				return nil, m.err
+			return nil, err
-			}
-			if err := m.database.UpdateByPrimaryKey(ctx, m.attachment); err != nil {
-				m.err = fmt.Errorf("error updating attachment: %s", err)
-				m.thumbstate = errored
-				return nil, m.err
-			}
 		}
 
 		// set the thumbnail of this media
@@ -148,6 +161,30 @@ func (m *Media) FullSize(ctx context.Context) (*ImageMeta, error) {
 			return nil, err
 		}
 
+		// put the full size in storage
+		if err := m.storage.Put(m.attachment.File.Path, decoded.image); err != nil {
+			m.err = fmt.Errorf("error storing full size image: %s", err)
+			m.fullSizeState = errored
+			return nil, m.err
+		}
+
+		// set appropriate fields on the attachment based on the image we derived
+		m.attachment.FileMeta.Original = gtsmodel.Original{
+			Width:  decoded.width,
+			Height: decoded.height,
+			Size:   decoded.size,
+			Aspect: decoded.aspect,
+		}
+		m.attachment.File.FileSize = decoded.size
+		m.attachment.File.UpdatedAt = time.Now()
+
+		// put or update the attachment in the database
+		if err := putOrUpdateAttachment(ctx, m.database, m.attachment); err != nil {
+			m.err = err
+			m.fullSizeState = errored
+			return nil, err
+		}
+
 		// set the fullsize of this media
 		m.fullSize = decoded
 
@@ -163,17 +200,46 @@ func (m *Media) FullSize(ctx context.Context) (*ImageMeta, error) {
 	return nil, fmt.Errorf("full size processing status %d unknown", m.fullSizeState)
 }
 
-// PreLoad begins the process of deriving the thumbnail and encoding the full-size image.
+func (m *Media) SetAsAvatar(ctx context.Context) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	m.attachment.Avatar = true
+	return putOrUpdateAttachment(ctx, m.database, m.attachment)
+}
+
+func (m *Media) SetAsHeader(ctx context.Context) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	m.attachment.Header = true
+	return putOrUpdateAttachment(ctx, m.database, m.attachment)
+}
+
+func (m *Media) SetStatusID(ctx context.Context, statusID string) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	m.attachment.StatusID = statusID
+	return putOrUpdateAttachment(ctx, m.database, m.attachment)
+}
+
+// AttachmentID returns the ID of the underlying media attachment without blocking processing.
+func (m *Media) AttachmentID() string {
+	return m.attachment.ID
+}
+
+// preLoad begins the process of deriving the thumbnail and encoding the full-size image.
 // It does this in a non-blocking way, so you can call it and then come back later and check
 // if it's finished.
-func (m *Media) PreLoad(ctx context.Context) {
+func (m *Media) preLoad(ctx context.Context) {
 	go m.Thumb(ctx)
 	go m.FullSize(ctx)
 }
 
 // Load is the blocking equivalent of pre-load. It makes sure the thumbnail and full-size image
 // have been processed, then it returns the full-size image.
-func (m *Media) Load(ctx context.Context) (*gtsmodel.MediaAttachment, error) {
+func (m *Media) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAttachment, error) {
 	if _, err := m.Thumb(ctx); err != nil {
 		return nil, err
 	}
@@ -184,3 +250,20 @@ func (m *Media) Load(ctx context.Context) (*gtsmodel.MediaAttachment, error) {
 
 	return m.attachment, nil
 }
+
+func (m *Media) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) {
+	return nil, nil
+}
+
+func putOrUpdateAttachment(ctx context.Context, database db.DB, attachment *gtsmodel.MediaAttachment) error {
+	if err := database.Put(ctx, attachment); err != nil {
+		if err != db.ErrAlreadyExists {
+			return fmt.Errorf("putOrUpdateAttachment: proper error while putting attachment: %s", err)
+		}
+		if err := database.UpdateByPrimaryKey(ctx, attachment); err != nil {
+			return fmt.Errorf("putOrUpdateAttachment: error while updating attachment: %s", err)
+		}
+	}
+
+	return nil
+}

internal/media/media_test.go

@@ -0,0 +1,65 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package media_test
+
+import (
+	"testing"
+
+	"codeberg.org/gruf/go-store/kv"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type MediaStandardTestSuite struct {
+	suite.Suite
+
+	db      db.DB
+	storage *kv.KVStore
+	manager media.Manager
+}
+
+func (suite *MediaStandardTestSuite) SetupSuite() {
+	testrig.InitTestLog()
+	testrig.InitTestConfig()
+
+	suite.db = testrig.NewTestDB()
+	suite.storage = testrig.NewTestStorage()
+}
+
+func (suite *MediaStandardTestSuite) SetupTest() {
+	testrig.StandardStorageSetup(suite.storage, "../../testrig/media")
+	testrig.StandardDBSetup(suite.db, nil)
+
+	m, err := media.New(suite.db, suite.storage)
+	if err != nil {
+		panic(err)
+	}
+	suite.manager = m
+}
+
+func (suite *MediaStandardTestSuite) TearDownTest() {
+	testrig.StandardDBTeardown(suite.db)
+	testrig.StandardStorageTeardown(suite.storage)
+}
+
+func TestMediaStandardTestSuite(t *testing.T) {
+	suite.Run(t, &MediaStandardTestSuite{})
+}

internal/processing/account/update.go

@@ -33,7 +33,6 @@
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/text"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
@@ -140,31 +139,40 @@ func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHead
 	var err error
 	maxImageSize := viper.GetInt(config.Keys.MediaImageMaxSize)
 	if int(avatar.Size) > maxImageSize {
-		err = fmt.Errorf("avatar with size %d exceeded max image size of %d bytes", avatar.Size, maxImageSize)
+		err = fmt.Errorf("UpdateAvatar: avatar with size %d exceeded max image size of %d bytes", avatar.Size, maxImageSize)
 		return nil, err
 	}
 	f, err := avatar.Open()
 	if err != nil {
-		return nil, fmt.Errorf("could not read provided avatar: %s", err)
+		return nil, fmt.Errorf("UpdateAvatar: could not read provided avatar: %s", err)
 	}
 
 	// extract the bytes
 	buf := new(bytes.Buffer)
 	size, err := io.Copy(buf, f)
 	if err != nil {
-		return nil, fmt.Errorf("could not read provided avatar: %s", err)
+		return nil, fmt.Errorf("UpdateAvatar: could not read provided avatar: %s", err)
 	}
 	if size == 0 {
-		return nil, errors.New("could not read provided avatar: size 0 bytes")
+		return nil, errors.New("UpdateAvatar: could not read provided avatar: size 0 bytes")
+	}
+
+	// we're done with the FileHeader now
+	if err := f.Close(); err != nil {
+		return nil, fmt.Errorf("UpdateAvatar: error closing multipart fileheader: %s", err)
 	}
 
 	// do the setting
-	avatarInfo, err := p.mediaManager.ProcessHeaderOrAvatar(ctx, buf.Bytes(), accountID, media.TypeAvatar, "")
+	media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), accountID, "")
 	if err != nil {
-		return nil, fmt.Errorf("error processing avatar: %s", err)
+		return nil, fmt.Errorf("UpdateAvatar: error processing avatar: %s", err)
 	}
 
-	return avatarInfo, f.Close()
+	if err := media.SetAsAvatar(ctx); err != nil {
+		return nil, fmt.Errorf("UpdateAvatar: error setting media as avatar: %s", err)
+	}
+
+	return media.LoadAttachment(ctx)
 }
 
 // UpdateHeader does the dirty work of checking the header part of an account update form,
@@ -174,31 +182,40 @@ func (p *processor) UpdateHeader(ctx context.Context, header *multipart.FileHead
 	var err error
 	maxImageSize := viper.GetInt(config.Keys.MediaImageMaxSize)
 	if int(header.Size) > maxImageSize {
-		err = fmt.Errorf("header with size %d exceeded max image size of %d bytes", header.Size, maxImageSize)
+		err = fmt.Errorf("UpdateHeader: header with size %d exceeded max image size of %d bytes", header.Size, maxImageSize)
 		return nil, err
 	}
 	f, err := header.Open()
 	if err != nil {
-		return nil, fmt.Errorf("could not read provided header: %s", err)
+		return nil, fmt.Errorf("UpdateHeader: could not read provided header: %s", err)
 	}
 
 	// extract the bytes
 	buf := new(bytes.Buffer)
 	size, err := io.Copy(buf, f)
 	if err != nil {
-		return nil, fmt.Errorf("could not read provided header: %s", err)
+		return nil, fmt.Errorf("UpdateHeader: could not read provided header: %s", err)
 	}
 	if size == 0 {
-		return nil, errors.New("could not read provided header: size 0 bytes")
+		return nil, errors.New("UpdateHeader: could not read provided header: size 0 bytes")
+	}
+
+	// we're done with the FileHeader now
+	if err := f.Close(); err != nil {
+		return nil, fmt.Errorf("UpdateHeader: error closing multipart fileheader: %s", err)
 	}
 
 	// do the setting
-	headerInfo, err := p.mediaManager.ProcessHeaderOrAvatar(ctx, buf.Bytes(), accountID, media.TypeHeader, "")
+	media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), accountID, "")
 	if err != nil {
-		return nil, fmt.Errorf("error processing header: %s", err)
+		return nil, fmt.Errorf("UpdateHeader: error processing header: %s", err)
 	}
 
-	return headerInfo, f.Close()
+	if err := media.SetAsHeader(ctx); err != nil {
+		return nil, fmt.Errorf("UpdateHeader: error setting media as header: %s", err)
+	}
+
+	return media.LoadAttachment(ctx)
 }
 
 func (p *processor) processNote(ctx context.Context, note string, accountID string) (string, error) {

internal/processing/admin/emoji.go

@@ -27,7 +27,6 @@
 
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/id"
 )
 
 func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, user *gtsmodel.User, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, error) {
@@ -49,26 +48,20 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account,
 		return nil, errors.New("could not read provided emoji: size 0 bytes")
 	}
 
-	// allow the mediaManager to work its magic of processing the emoji bytes, and putting them in whatever storage backend we're using
+	media, err := p.mediaManager.ProcessEmoji(ctx, buf.Bytes(), account.ID, "")
-	emoji, err := p.mediaManager.ProcessLocalEmoji(ctx, buf.Bytes(), form.Shortcode)
+	if err != nil {
-	if err != nil {
+		return nil, err
-		return nil, fmt.Errorf("error reading emoji: %s", err)
+	}
-	}
+
-
+	emoji, err := media.LoadEmoji(ctx)
-	emojiID, err := id.NewULID()
 	if err != nil {
 		return nil, err
 	}
-	emoji.ID = emojiID
 
 	apiEmoji, err := p.tc.EmojiToAPIEmoji(ctx, emoji)
 	if err != nil {
 		return nil, fmt.Errorf("error converting emoji to apitype: %s", err)
 	}
 
-	if err := p.db.Put(ctx, emoji); err != nil {
-		return nil, fmt.Errorf("database error while processing emoji: %s", err)
-	}
-
 	return &apiEmoji, nil
 }

internal/processing/media/create.go

@@ -44,13 +44,13 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form
 		return nil, errors.New("could not read provided attachment: size 0 bytes")
 	}
 
-	// process the media and load it immediately
+	// process the media attachment and load it immediately
-	media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), account.ID)
+	media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), account.ID, "")
 	if err != nil {
 		return nil, err
 	}
 
-	attachment, err := media.Load(ctx)
+	attachment, err := media.LoadAttachment(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -62,10 +62,5 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form
 		return nil, fmt.Errorf("error parsing media attachment to frontend type: %s", err)
 	}
 
-	// now we can confidently put the attachment in the database
-	if err := p.db.Put(ctx, attachment); err != nil {
-		return nil, fmt.Errorf("error storing media attachment in db: %s", err)
-	}
-
 	return &apiAttachment, nil
 }

internal/transport/derefmedia.go

@@ -28,18 +28,15 @@
 	"github.com/sirupsen/logrus"
 )
 
-func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL, expectedContentType string) ([]byte, error) {
+func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) ([]byte, error) {
 	l := logrus.WithField("func", "DereferenceMedia")
 	l.Debugf("performing GET to %s", iri.String())
 	req, err := http.NewRequestWithContext(ctx, "GET", iri.String(), nil)
 	if err != nil {
 		return nil, err
 	}
-	if expectedContentType == "" {
+
-		req.Header.Add("Accept", "*/*")
+	req.Header.Add("Accept", "*/*") // we don't know what kind of media we're going to get here
-	} else {
-		req.Header.Add("Accept", expectedContentType)
-	}
 	req.Header.Add("Date", t.clock.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
 	req.Header.Add("User-Agent", fmt.Sprintf("%s %s", t.appAgent, t.gofedAgent))
 	req.Header.Set("Host", iri.Host)

internal/transport/transport.go

@@ -34,7 +34,7 @@
 type Transport interface {
 	pub.Transport
 	// DereferenceMedia fetches the bytes of the given media attachment IRI, with the expectedContentType.
-	DereferenceMedia(ctx context.Context, iri *url.URL, expectedContentType string) ([]byte, error)
+	DereferenceMedia(ctx context.Context, iri *url.URL) ([]byte, error)
 	// DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo.
 	DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error)
 	// Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body.

testrig/mediahandler.go

@@ -26,5 +26,9 @@
 
 // NewTestMediaManager returns a media handler with the default test config, and the given db and storage.
 func NewTestMediaManager(db db.DB, storage *kv.KVStore) media.Manager {
-	return media.New(db, storage)
+	m, err := media.New(db, storage)
+	if err != nil {
+		panic(err)
+	}
+	return m
 }