Mention fixup (#167)

* rework mention creation a bit

* rework mention creation a bit

* tidy up status dereferencing

* start adding tests for dereferencing

* fixups

* fix

* review changes

internal/ap/extract.go

@@ -242,8 +242,9 @@ func ExtractImageURL(i WithImage) (*url.URL, error) {
 // ExtractSummary extracts the summary/content warning of an interface.
 func ExtractSummary(i WithSummary) (string, error) {
 	summaryProp := i.GetActivityStreamsSummary()
-	if summaryProp == nil {
+	if summaryProp == nil || summaryProp.Len() == 0 {
-		return "", errors.New("summary property was nil")
+		// no summary to speak of
+		return "", nil
 	}
 
 	for iter := summaryProp.Begin(); iter != summaryProp.End(); iter = iter.Next() {
@@ -544,12 +545,12 @@ func ExtractMentions(i WithTag) ([]*gtsmodel.Mention, error) {
 
 		mentionable, ok := t.(Mentionable)
 		if !ok {
-			continue
+			return nil, errors.New("mention was not convertable to ap.Mentionable")
 		}
 
 		mention, err := ExtractMention(mentionable)
 		if err != nil {
-			continue
+			return nil, err
 		}
 
 		mentions = append(mentions, mention)

internal/cliactions/server/server.go

@@ -111,7 +111,7 @@
 	}
 
 	// build converters and util
-	typeConverter := typeutils.NewConverter(c, dbService)
+	typeConverter := typeutils.NewConverter(c, dbService, log)
 	timelineManager := timelineprocessing.NewManager(dbService, typeConverter, c, log)
 
 	// build backend handlers

internal/db/bundb/instance.go

@@ -41,12 +41,12 @@ func (i *instanceDB) CountInstanceUsers(ctx context.Context, domain string) (int
 		Where("username != ?", domain).
 		Where("? IS NULL", bun.Ident("suspended_at"))
 
-		if domain == i.config.Host {
+	if domain == i.config.Host {
-			// if the domain is *this* domain, just count where the domain field is null
+		// if the domain is *this* domain, just count where the domain field is null
-			q = q.WhereGroup(" AND ", whereEmptyOrNull("domain"))
+		q = q.WhereGroup(" AND ", whereEmptyOrNull("domain"))
-		} else {
+	} else {
-			q = q.Where("domain = ?", domain)
+		q = q.Where("domain = ?", domain)
-		}
+	}
 
 	count, err := q.Count(ctx)
 

internal/federation/dereferencing/attachment.go

@@ -0,0 +1,84 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (d *deref) GetRemoteAttachment(ctx context.Context, requestingUsername string, remoteAttachmentURI *url.URL, ownerAccountID string, statusID string, expectedContentType string) (*gtsmodel.MediaAttachment, error) {
+	l := d.log.WithFields(logrus.Fields{
+		"username":            requestingUsername,
+		"remoteAttachmentURI": remoteAttachmentURI,
+	})
+
+	maybeAttachment := &gtsmodel.MediaAttachment{}
+	where := []db.Where{
+		{
+			Key:   "remote_url",
+			Value: remoteAttachmentURI.String(),
+		},
+	}
+
+	if err := d.db.GetWhere(ctx, where, maybeAttachment); err == nil {
+		// we already the attachment in the database
+		l.Debugf("GetRemoteAttachment: attachment already exists with id %s", maybeAttachment.ID)
+		return maybeAttachment, nil
+	}
+
+	a, err := d.RefreshAttachment(ctx, requestingUsername, remoteAttachmentURI, ownerAccountID, expectedContentType)
+	if err != nil {
+		return nil, fmt.Errorf("GetRemoteAttachment: error refreshing attachment: %s", err)
+	}
+
+	a.StatusID = statusID
+	if err := d.db.Put(ctx, a); err != nil {
+		if err != db.ErrAlreadyExists {
+			return nil, fmt.Errorf("GetRemoteAttachment: error inserting attachment: %s", err)
+		}
+	}
+
+	return a, nil
+}
+
+func (d *deref) RefreshAttachment(ctx context.Context, requestingUsername string, remoteAttachmentURI *url.URL, ownerAccountID string, expectedContentType string) (*gtsmodel.MediaAttachment, error) {
+	// it just doesn't exist or we have to refresh
+	t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername)
+	if err != nil {
+		return nil, fmt.Errorf("RefreshAttachment: error creating transport: %s", err)
+	}
+
+	attachmentBytes, err := t.DereferenceMedia(ctx, remoteAttachmentURI, expectedContentType)
+	if err != nil {
+		return nil, fmt.Errorf("RefreshAttachment: error dereferencing media: %s", err)
+	}
+
+	a, err := d.mediaHandler.ProcessAttachment(ctx, attachmentBytes, ownerAccountID, remoteAttachmentURI.String())
+	if err != nil {
+		return nil, fmt.Errorf("RefreshAttachment: error processing attachment: %s", err)
+	}
+
+	return a, nil
+}

internal/federation/dereferencing/dereferencer.go

@@ -43,6 +43,9 @@ type Dereferencer interface {
 
 	GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error)
 
+	GetRemoteAttachment(ctx context.Context, username string, remoteAttachmentURI *url.URL, ownerAccountID string, statusID string, expectedContentType string) (*gtsmodel.MediaAttachment, error)
+	RefreshAttachment(ctx context.Context, requestingUsername string, remoteAttachmentURI *url.URL, ownerAccountID string, expectedContentType string) (*gtsmodel.MediaAttachment, error)
+
 	DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error
 	DereferenceThread(ctx context.Context, username string, statusIRI *url.URL) error
 

internal/federation/dereferencing/dereferencer_test.go

@@ -0,0 +1,42 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing_test
+
+import (
+	"github.com/go-fed/activity/streams/vocab"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+type DereferencerStandardTestSuite struct {
+	suite.Suite
+	config *config.Config
+	db     db.DB
+	log    *logrus.Logger
+
+	testRemoteStatuses map[string]vocab.ActivityStreamsNote
+	testRemoteAccounts map[string]vocab.ActivityStreamsPerson
+	testAccounts       map[string]*gtsmodel.Account
+
+	dereferencer dereferencing.Dereferencer
+}

internal/federation/dereferencing/status.go

@@ -24,12 +24,12 @@
 	"errors"
 	"fmt"
 	"net/url"
+	"strings"
 
 	"github.com/go-fed/activity/streams"
 	"github.com/go-fed/activity/streams/vocab"
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
-	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
 )
@@ -229,8 +229,7 @@ func (d *deref) dereferenceStatusable(ctx context.Context, username string, remo
 // 2. Hashtags.
 // 3. Emojis.
 // 4. Mentions.
-// 5. Posting account.
+// 5. Replied-to-status.
-// 6. Replied-to-status.
 //
 // SIDE EFFECTS:
 // This function will deference all of the above, insert them in the database as necessary,
@@ -243,117 +242,113 @@ func (d *deref) populateStatusFields(ctx context.Context, status *gtsmodel.Statu
 	})
 	l.Debug("entering function")
 
-	// make sure we have a status URI and that the domain in question isn't blocked
+	statusIRI, err := url.Parse(status.URI)
-	statusURI, err := url.Parse(status.URI)
 	if err != nil {
-		return fmt.Errorf("DereferenceStatusFields: couldn't parse status URI %s: %s", status.URI, err)
+		return fmt.Errorf("populateStatusFields: couldn't parse status URI %s: %s", status.URI, err)
-	}
-	if blocked, err := d.db.IsDomainBlocked(ctx, statusURI.Host); blocked || err != nil {
-		return fmt.Errorf("DereferenceStatusFields: domain %s is blocked", statusURI.Host)
 	}
 
-	// we can continue -- create a new transport here because we'll probably need it
+	blocked, err := d.db.IsURIBlocked(ctx, statusIRI)
-	t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername)
 	if err != nil {
-		return fmt.Errorf("error creating transport: %s", err)
+		return fmt.Errorf("populateStatusFields: error checking blocked status of %s: %s", statusIRI, err)
+	}
+	if blocked {
+		return fmt.Errorf("populateStatusFields: domain %s is blocked", statusIRI)
 	}
 
 	// in case the status doesn't have an id yet (ie., it hasn't entered the database yet), then create one
 	if status.ID == "" {
 		newID, err := id.NewULIDFromTime(status.CreatedAt)
 		if err != nil {
-			return err
+			return fmt.Errorf("populateStatusFields: error creating ulid for status: %s", err)
 		}
 		status.ID = newID
 	}
 
 	// 1. Media attachments.
-	//
+	if err := d.populateStatusAttachments(ctx, status, requestingUsername); err != nil {
-	// At this point we should know:
+		return fmt.Errorf("populateStatusFields: error populating status attachments: %s", err)
-	// * the media type of the file we're looking for (a.File.ContentType)
-	// * the blurhash (a.Blurhash)
-	// * the file type (a.Type)
-	// * the remote URL (a.RemoteURL)
-	// This should be enough to pass along to the media processor.
-	attachmentIDs := []string{}
-	for _, a := range status.Attachments {
-		l.Tracef("dereferencing attachment: %+v", a)
-
-		// it might have been processed elsewhere so check first if it's already in the database or not
-		maybeAttachment := &gtsmodel.MediaAttachment{}
-		err := d.db.GetWhere(ctx, []db.Where{{Key: "remote_url", Value: a.RemoteURL}}, maybeAttachment)
-		if err == nil {
-			// we already have it in the db, dereferenced, no need to do it again
-			l.Tracef("attachment already exists with id %s", maybeAttachment.ID)
-			attachmentIDs = append(attachmentIDs, maybeAttachment.ID)
-			continue
-		}
-		if err != db.ErrNoEntries {
-			// we have a real error
-			return fmt.Errorf("error checking db for existence of attachment with remote url %s: %s", a.RemoteURL, err)
-		}
-		// it just doesn't exist yet so carry on
-		l.Debug("attachment doesn't exist yet, calling ProcessRemoteAttachment", a)
-		deferencedAttachment, err := d.mediaHandler.ProcessRemoteAttachment(ctx, t, a, status.AccountID)
-		if err != nil {
-			l.Errorf("error dereferencing status attachment: %s", err)
-			continue
-		}
-		l.Debugf("dereferenced attachment: %+v", deferencedAttachment)
-		deferencedAttachment.StatusID = status.ID
-		deferencedAttachment.Description = a.Description
-		if err := d.db.Put(ctx, deferencedAttachment); err != nil {
-			return fmt.Errorf("error inserting dereferenced attachment with remote url %s: %s", a.RemoteURL, err)
-		}
-		attachmentIDs = append(attachmentIDs, deferencedAttachment.ID)
 	}
-	status.AttachmentIDs = attachmentIDs
 
 	// 2. Hashtags
+	// TODO
 
 	// 3. Emojis
+	// TODO
 
 	// 4. Mentions
+	if err := d.populateStatusMentions(ctx, status, requestingUsername); err != nil {
+		return fmt.Errorf("populateStatusFields: error populating status mentions: %s", err)
+	}
+
+	// 5. Replied-to-status.
+	if err := d.populateStatusRepliedTo(ctx, status, requestingUsername); err != nil {
+		return fmt.Errorf("populateStatusFields: error populating status repliedTo: %s", err)
+	}
+
+	return nil
+}
+
+func (d *deref) populateStatusMentions(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error {
+	l := d.log
+
 	// At this point, mentions should have the namestring and mentionedAccountURI set on them.
-	//
+	// We can use these to find the accounts.
-	// We should dereference any accounts mentioned here which we don't have in our db yet, by their URI.
+
 	mentionIDs := []string{}
+	newMentions := []*gtsmodel.Mention{}
 	for _, m := range status.Mentions {
 		if m.ID != "" {
 			// we've already populated this mention, since it has an ID
-			l.Debug("mention already populated")
+			l.Debug("populateStatusMentions: mention already populated")
+			mentionIDs = append(mentionIDs, m.ID)
+			newMentions = append(newMentions, m)
 			continue
 		}
 
 		if m.TargetAccountURI == "" {
-			// can't do anything with this mention
+			l.Debug("populateStatusMentions: target URI not set on mention")
-			l.Debug("target URI not set on mention")
 			continue
 		}
 
 		targetAccountURI, err := url.Parse(m.TargetAccountURI)
 		if err != nil {
-			l.Debugf("error parsing mentioned account uri %s: %s", m.TargetAccountURI, err)
+			l.Debugf("populateStatusMentions: error parsing mentioned account uri %s: %s", m.TargetAccountURI, err)
 			continue
 		}
 
 		var targetAccount *gtsmodel.Account
-		if a, err := d.db.GetAccountByURL(ctx, targetAccountURI.String()); err == nil {
+		errs := []string{}
-			targetAccount = a
+
-		} else if a, _, err := d.GetRemoteAccount(ctx, requestingUsername, targetAccountURI, false); err == nil {
+		// check if account is in the db already
-			targetAccount = a
+		if a, err := d.db.GetAccountByURL(ctx, targetAccountURI.String()); err != nil {
+			errs = append(errs, err.Error())
 		} else {
-			// we can't find the target account so bail
+			l.Debugf("populateStatusMentions: got target account %s with id %s through GetAccountByURL", targetAccountURI, a.ID)
-			l.Debug("can't retrieve account targeted by mention")
+			targetAccount = a
+		}
+
+		if targetAccount == nil {
+			// we didn't find the account in our database already
+			// check if we can get the account remotely (dereference it)
+			if a, _, err := d.GetRemoteAccount(ctx, requestingUsername, targetAccountURI, false); err != nil {
+				errs = append(errs, err.Error())
+			} else {
+				l.Debugf("populateStatusMentions: got target account %s with id %s through GetRemoteAccount", targetAccountURI, a.ID)
+				targetAccount = a
+			}
+		}
+
+		if targetAccount == nil {
+			l.Debugf("populateStatusMentions: couldn't get target account %s: %s", m.TargetAccountURI, strings.Join(errs, " : "))
 			continue
 		}
 
 		mID, err := id.NewRandomULID()
 		if err != nil {
-			return err
+			return fmt.Errorf("populateStatusMentions: error generating ulid: %s", err)
 		}
 
-		m = &gtsmodel.Mention{
+		newMention := &gtsmodel.Mention{
 			ID:               mID,
 			StatusID:         status.ID,
 			Status:           m.Status,
@@ -369,32 +364,91 @@ func (d *deref) populateStatusFields(ctx context.Context, status *gtsmodel.Statu
 			TargetAccountURL: targetAccount.URL,
 		}
 
-		if err := d.db.Put(ctx, m); err != nil {
+		if err := d.db.Put(ctx, newMention); err != nil {
-			return fmt.Errorf("error creating mention: %s", err)
+			return fmt.Errorf("populateStatusMentions: error creating mention: %s", err)
 		}
-		mentionIDs = append(mentionIDs, m.ID)
-	}
-	status.MentionIDs = mentionIDs
 
-	// status has replyToURI but we don't have an ID yet for the status it replies to
+		mentionIDs = append(mentionIDs, newMention.ID)
+		newMentions = append(newMentions, newMention)
+	}
+
+	status.MentionIDs = mentionIDs
+	status.Mentions = newMentions
+
+	return nil
+}
+
+func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error {
+	l := d.log
+
+	// At this point we should know:
+	// * the media type of the file we're looking for (a.File.ContentType)
+	// * the file type (a.Type)
+	// * the remote URL (a.RemoteURL)
+	// This should be enough to dereference the piece of media.
+
+	attachmentIDs := []string{}
+	attachments := []*gtsmodel.MediaAttachment{}
+
+	for _, a := range status.Attachments {
+
+		aURL, err := url.Parse(a.RemoteURL)
+		if err != nil {
+			l.Errorf("populateStatusAttachments: couldn't parse attachment url %s: %s", a.RemoteURL, err)
+			continue
+		}
+
+		attachment, err := d.GetRemoteAttachment(ctx, requestingUsername, aURL, status.AccountID, status.ID, a.File.ContentType)
+		if err != nil {
+			l.Errorf("populateStatusAttachments: couldn't get remote attachment %s: %s", a.RemoteURL, err)
+		}
+
+		attachmentIDs = append(attachmentIDs, attachment.ID)
+		attachments = append(attachments, attachment)
+	}
+
+	status.AttachmentIDs = attachmentIDs
+	status.Attachments = attachments
+
+	return nil
+}
+
+func (d *deref) populateStatusRepliedTo(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error {
 	if status.InReplyToURI != "" && status.InReplyToID == "" {
 		statusURI, err := url.Parse(status.InReplyToURI)
 		if err != nil {
 			return err
 		}
-		if replyToStatus, err := d.db.GetStatusByURI(ctx, status.InReplyToURI); err == nil {
+
-			// we have the status
+		var replyToStatus *gtsmodel.Status
-			status.InReplyToID = replyToStatus.ID
+		errs := []string{}
-			status.InReplyTo = replyToStatus
+
-			status.InReplyToAccountID = replyToStatus.AccountID
+		// see if we have the status in our db already
-			status.InReplyToAccount = replyToStatus.Account
+		if s, err := d.db.GetStatusByURI(ctx, status.InReplyToURI); err != nil {
-		} else if replyToStatus, _, _, err := d.GetRemoteStatus(ctx, requestingUsername, statusURI, false); err == nil {
+			errs = append(errs, err.Error())
-			// we got the status
+		} else {
-			status.InReplyToID = replyToStatus.ID
+			replyToStatus = s
-			status.InReplyTo = replyToStatus
-			status.InReplyToAccountID = replyToStatus.AccountID
-			status.InReplyToAccount = replyToStatus.Account
 		}
+
+		if replyToStatus == nil {
+			// didn't find the status in our db, try to get it remotely
+			if s, _, _, err := d.GetRemoteStatus(ctx, requestingUsername, statusURI, false); err != nil {
+				errs = append(errs, err.Error())
+			} else {
+				replyToStatus = s
+			}
+		}
+
+		if replyToStatus == nil {
+			return fmt.Errorf("populateStatusRepliedTo: couldn't get reply to status with uri %s: %s", statusURI, strings.Join(errs, " : "))
+		}
+
+		// we have the status
+		status.InReplyToID = replyToStatus.ID
+		status.InReplyTo = replyToStatus
+		status.InReplyToAccountID = replyToStatus.AccountID
+		status.InReplyToAccount = replyToStatus.Account
 	}
+
 	return nil
 }

internal/federation/dereferencing/status_test.go

@@ -0,0 +1,213 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/go-fed/activity/streams"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type StatusTestSuite struct {
+	DereferencerStandardTestSuite
+}
+
+// mockTransportController returns basically a miniature muxer, which returns a different
+// value based on the request URL. It can be used to return remote statuses, profiles, etc,
+// as though they were actually being dereferenced. If the URL doesn't correspond to any person
+// or note or attachment that we have stored, then just a 200 code will be returned, with an empty body.
+func (suite *StatusTestSuite) mockTransportController() transport.Controller {
+	do := func(req *http.Request) (*http.Response, error) {
+		suite.log.Debugf("received request for %s", req.URL)
+
+		responseBytes := []byte{}
+
+		note, ok := suite.testRemoteStatuses[req.URL.String()]
+		if ok {
+			// the request is for a note that we have stored
+			noteI, err := streams.Serialize(note)
+			if err != nil {
+				panic(err)
+			}
+			noteJson, err := json.Marshal(noteI)
+			if err != nil {
+				panic(err)
+			}
+			responseBytes = noteJson
+		}
+
+		person, ok := suite.testRemoteAccounts[req.URL.String()]
+		if ok {
+			// the request is for a person that we have stored
+			personI, err := streams.Serialize(person)
+			if err != nil {
+				panic(err)
+			}
+			personJson, err := json.Marshal(personI)
+			if err != nil {
+				panic(err)
+			}
+			responseBytes = personJson
+		}
+
+		if len(responseBytes) != 0 {
+			// we found something, so print what we're going to return
+			suite.log.Debugf("returning response %s", string(responseBytes))
+		}
+
+		reader := bytes.NewReader(responseBytes)
+		readCloser := io.NopCloser(reader)
+		response := &http.Response{
+			StatusCode: 200,
+			Body:       readCloser,
+		}
+
+		return response, nil
+	}
+	mockClient := testrig.NewMockHTTPClient(do)
+	return testrig.NewTestTransportController(mockClient, suite.db)
+}
+
+func (suite *StatusTestSuite) SetupSuite() {
+	suite.testAccounts = testrig.NewTestAccounts()
+	suite.testRemoteStatuses = testrig.NewTestFediStatuses()
+	suite.testRemoteAccounts = testrig.NewTestFediPeople()
+}
+
+func (suite *StatusTestSuite) SetupTest() {
+	suite.config = testrig.NewTestConfig()
+	suite.db = testrig.NewTestDB()
+	suite.log = testrig.NewTestLog()
+	suite.dereferencer = dereferencing.NewDereferencer(suite.config,
+		suite.db,
+		testrig.NewTestTypeConverter(suite.db),
+		suite.mockTransportController(),
+		testrig.NewTestMediaHandler(suite.db, testrig.NewTestStorage()),
+		suite.log)
+	testrig.StandardDBSetup(suite.db, nil)
+}
+
+func (suite *StatusTestSuite) TestDereferenceSimpleStatus() {
+	fetchingAccount := suite.testAccounts["local_account_1"]
+
+	statusURL := testrig.URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01FE4NTHKWW7THT67EF10EB839")
+	status, statusable, new, err := suite.dereferencer.GetRemoteStatus(context.Background(), fetchingAccount.Username, statusURL, false)
+	suite.NoError(err)
+	suite.NotNil(status)
+	suite.NotNil(statusable)
+	suite.True(new)
+
+	// status values should be set
+	suite.Equal("https://unknown-instance.com/users/brand_new_person/statuses/01FE4NTHKWW7THT67EF10EB839", status.URI)
+	suite.Equal("https://unknown-instance.com/users/@brand_new_person/01FE4NTHKWW7THT67EF10EB839", status.URL)
+	suite.Equal("Hello world!", status.Content)
+	suite.Equal("https://unknown-instance.com/users/brand_new_person", status.AccountURI)
+	suite.False(status.Local)
+	suite.Empty(status.ContentWarning)
+	suite.Equal(gtsmodel.VisibilityPublic, status.Visibility)
+	suite.Equal(gtsmodel.ActivityStreamsNote, status.ActivityStreamsType)
+
+	// status should be in the database
+	dbStatus, err := suite.db.GetStatusByURI(context.Background(), status.URI)
+	suite.NoError(err)
+	suite.Equal(status.ID, dbStatus.ID)
+
+	// account should be in the database now too
+	account, err := suite.db.GetAccountByURI(context.Background(), status.AccountURI)
+	suite.NoError(err)
+	suite.NotNil(account)
+	suite.True(account.Discoverable)
+	suite.Equal("https://unknown-instance.com/users/brand_new_person", account.URI)
+	suite.Equal("hey I'm a new person, your instance hasn't seen me yet uwu", account.Note)
+	suite.Equal("Geoff Brando New Personson", account.DisplayName)
+	suite.Equal("brand_new_person", account.Username)
+	suite.NotNil(account.PublicKey)
+	suite.Nil(account.PrivateKey)
+}
+
+func (suite *StatusTestSuite) TestDereferenceStatusWithMention() {
+	fetchingAccount := suite.testAccounts["local_account_1"]
+
+	statusURL := testrig.URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01FE5Y30E3W4P7TRE0R98KAYQV")
+	status, statusable, new, err := suite.dereferencer.GetRemoteStatus(context.Background(), fetchingAccount.Username, statusURL, false)
+	suite.NoError(err)
+	suite.NotNil(status)
+	suite.NotNil(statusable)
+	suite.True(new)
+
+	// status values should be set
+	suite.Equal("https://unknown-instance.com/users/brand_new_person/statuses/01FE5Y30E3W4P7TRE0R98KAYQV", status.URI)
+	suite.Equal("https://unknown-instance.com/users/@brand_new_person/01FE5Y30E3W4P7TRE0R98KAYQV", status.URL)
+	suite.Equal("Hey @the_mighty_zork@localhost:8080 how's it going?", status.Content)
+	suite.Equal("https://unknown-instance.com/users/brand_new_person", status.AccountURI)
+	suite.False(status.Local)
+	suite.Empty(status.ContentWarning)
+	suite.Equal(gtsmodel.VisibilityPublic, status.Visibility)
+	suite.Equal(gtsmodel.ActivityStreamsNote, status.ActivityStreamsType)
+
+	// status should be in the database
+	dbStatus, err := suite.db.GetStatusByURI(context.Background(), status.URI)
+	suite.NoError(err)
+	suite.Equal(status.ID, dbStatus.ID)
+
+	// account should be in the database now too
+	account, err := suite.db.GetAccountByURI(context.Background(), status.AccountURI)
+	suite.NoError(err)
+	suite.NotNil(account)
+	suite.True(account.Discoverable)
+	suite.Equal("https://unknown-instance.com/users/brand_new_person", account.URI)
+	suite.Equal("hey I'm a new person, your instance hasn't seen me yet uwu", account.Note)
+	suite.Equal("Geoff Brando New Personson", account.DisplayName)
+	suite.Equal("brand_new_person", account.Username)
+	suite.NotNil(account.PublicKey)
+	suite.Nil(account.PrivateKey)
+
+	// we should have a mention in the database
+	m := &gtsmodel.Mention{}
+	err = suite.db.GetWhere(context.Background(), []db.Where{{Key: "status_id", Value: status.ID}}, m)
+	suite.NoError(err)
+	suite.NotNil(m)
+	suite.Equal(status.ID, m.StatusID)
+	suite.Equal(account.ID, m.OriginAccountID)
+	suite.Equal(fetchingAccount.ID, m.TargetAccountID)
+	suite.Equal(account.URI, m.OriginAccountURI)
+	suite.WithinDuration(time.Now(), m.CreatedAt, 5*time.Minute)
+	suite.WithinDuration(time.Now(), m.UpdatedAt, 5*time.Minute)
+	suite.False(m.Silent)
+}
+
+func (suite *StatusTestSuite) TearDownTest() {
+	testrig.StandardDBTeardown(suite.db)
+}
+
+func TestStatusTestSuite(t *testing.T) {
+	suite.Run(t, new(StatusTestSuite))
+}

internal/federation/federatingdb/db.go

@@ -60,7 +60,7 @@ func New(db db.DB, config *config.Config, log *logrus.Logger) DB {
 		db:            db,
 		config:        config,
 		log:           log,
-		typeConverter: typeutils.NewConverter(config, db),
+		typeConverter: typeutils.NewConverter(config, db, log),
 	}
 	go fdb.cleanupLocks()
 	return &fdb

internal/gtsmodel/mention.go

@@ -25,19 +25,19 @@ type Mention struct {
 	// ID of this mention in the database
 	ID string `bun:"type:CHAR(26),pk,notnull,unique"`
 	// ID of the status this mention originates from
-	StatusID string  `bun:"type:CHAR(26),notnull"`
+	StatusID string  `bun:"type:CHAR(26),notnull,nullzero"`
 	Status   *Status `bun:"rel:belongs-to"`
 	// When was this mention created?
 	CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"`
 	// When was this mention last updated?
 	UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"`
 	// What's the internal account ID of the originator of the mention?
-	OriginAccountID string   `bun:"type:CHAR(26),notnull"`
+	OriginAccountID string   `bun:"type:CHAR(26),notnull,nullzero"`
 	OriginAccount   *Account `bun:"rel:belongs-to"`
 	// What's the AP URI of the originator of the mention?
 	OriginAccountURI string `bun:",notnull"`
 	// What's the internal account ID of the mention target?
-	TargetAccountID string   `bun:"type:CHAR(26),notnull"`
+	TargetAccountID string   `bun:"type:CHAR(26),notnull,nullzero"`
 	TargetAccount   *Account `bun:"rel:belongs-to"`
 	// Prevent this mention from generating a notification?
 	Silent bool

internal/media/handler.go

@@ -80,13 +80,6 @@ type Handler interface {
 	// in the database.
 	ProcessLocalEmoji(ctx context.Context, emojiBytes []byte, shortcode string) (*gtsmodel.Emoji, error)
 
-	// ProcessRemoteAttachment takes a transport, a bare-bones current attachment, and an accountID that the attachment belongs to.
-	// It then dereferences the attachment (ie., fetches the attachment bytes from the remote server), ensuring that the bytes are
-	// the correct content type. It stores the attachment in whatever storage backend the Handler has been initalized with, and returns
-	// information to the caller about the new attachment. It's the caller's responsibility to put the returned struct
-	// in the database.
-	ProcessRemoteAttachment(ctx context.Context, t transport.Transport, currentAttachment *gtsmodel.MediaAttachment, accountID string) (*gtsmodel.MediaAttachment, error)
-
 	ProcessRemoteHeaderOrAvatar(ctx context.Context, t transport.Transport, currentAttachment *gtsmodel.MediaAttachment, accountID string) (*gtsmodel.MediaAttachment, error)
 }
 
@@ -296,30 +289,6 @@ func (mh *mediaHandler) ProcessLocalEmoji(ctx context.Context, emojiBytes []byte
 	return e, nil
 }
 
-func (mh *mediaHandler) ProcessRemoteAttachment(ctx context.Context, t transport.Transport, currentAttachment *gtsmodel.MediaAttachment, accountID string) (*gtsmodel.MediaAttachment, error) {
-	if currentAttachment.RemoteURL == "" {
-		return nil, errors.New("no remote URL on media attachment to dereference")
-	}
-	remoteIRI, err := url.Parse(currentAttachment.RemoteURL)
-	if err != nil {
-		return nil, fmt.Errorf("error parsing attachment url %s: %s", currentAttachment.RemoteURL, err)
-	}
-
-	// for content type, we assume we don't know what to expect...
-	expectedContentType := "*/*"
-	if currentAttachment.File.ContentType != "" {
-		// ... and then narrow it down if we do
-		expectedContentType = currentAttachment.File.ContentType
-	}
-
-	attachmentBytes, err := t.DereferenceMedia(context.Background(), remoteIRI, expectedContentType)
-	if err != nil {
-		return nil, fmt.Errorf("dereferencing remote media with url %s: %s", remoteIRI.String(), err)
-	}
-
-	return mh.ProcessAttachment(ctx, attachmentBytes, accountID, currentAttachment.RemoteURL)
-}
-
 func (mh *mediaHandler) ProcessRemoteHeaderOrAvatar(ctx context.Context, t transport.Transport, currentAttachment *gtsmodel.MediaAttachment, accountID string) (*gtsmodel.MediaAttachment, error) {
 
 	if !currentAttachment.Header && !currentAttachment.Avatar {

internal/typeutils/astointernal.go

@@ -181,44 +181,62 @@ func (c *converter) ASStatusToStatus(ctx context.Context, statusable ap.Statusab
 	}
 	status.URI = uriProp.GetIRI().String()
 
+	l := c.log.WithField("statusURI", status.URI)
+
 	// web url for viewing this status
-	if statusURL, err := ap.ExtractURL(statusable); err == nil {
+	if statusURL, err := ap.ExtractURL(statusable); err != nil {
+		l.Infof("ASStatusToStatus: error extracting status URL: %s", err)
+	} else {
 		status.URL = statusURL.String()
 	}
 
 	// the html-formatted content of this status
-	if content, err := ap.ExtractContent(statusable); err == nil {
+	if content, err := ap.ExtractContent(statusable); err != nil {
+		l.Infof("ASStatusToStatus: error extracting status content: %s", err)
+	} else {
 		status.Content = content
 	}
 
 	// attachments to dereference and fetch later on (we don't do that here)
-	if attachments, err := ap.ExtractAttachments(statusable); err == nil {
+	if attachments, err := ap.ExtractAttachments(statusable); err != nil {
+		l.Infof("ASStatusToStatus: error extracting status attachments: %s", err)
+	} else {
 		status.Attachments = attachments
 	}
 
 	// hashtags to dereference later on
-	if hashtags, err := ap.ExtractHashtags(statusable); err == nil {
+	if hashtags, err := ap.ExtractHashtags(statusable); err != nil {
+		l.Infof("ASStatusToStatus: error extracting status hashtags: %s", err)
+	} else {
 		status.Tags = hashtags
 	}
 
 	// emojis to dereference and fetch later on
-	if emojis, err := ap.ExtractEmojis(statusable); err == nil {
+	if emojis, err := ap.ExtractEmojis(statusable); err != nil {
+		l.Infof("ASStatusToStatus: error extracting status emojis: %s", err)
+	} else {
 		status.Emojis = emojis
 	}
 
 	// mentions to dereference later on
-	if mentions, err := ap.ExtractMentions(statusable); err == nil {
+	if mentions, err := ap.ExtractMentions(statusable); err != nil {
+		l.Infof("ASStatusToStatus: error extracting status mentions: %s", err)
+	} else {
 		status.Mentions = mentions
 	}
 
 	// cw string for this status
-	if cw, err := ap.ExtractSummary(statusable); err == nil {
+	if cw, err := ap.ExtractSummary(statusable); err != nil {
+		l.Infof("ASStatusToStatus: error extracting status summary: %s", err)
+	} else {
 		status.ContentWarning = cw
 	}
 
 	// when was this status created?
 	published, err := ap.ExtractPublished(statusable)
-	if err == nil {
+	if err != nil {
+		l.Infof("ASStatusToStatus: error extracting status published: %s", err)
+	} else {
 		status.CreatedAt = published
 		status.UpdatedAt = published
 	}

internal/typeutils/astointernal_test.go

@@ -338,7 +338,7 @@ func (suite *ASToInternalTestSuite) SetupSuite() {
 	suite.log = testrig.NewTestLog()
 	suite.accounts = testrig.NewTestAccounts()
 	suite.people = testrig.NewTestFediPeople()
-	suite.typeconverter = typeutils.NewConverter(suite.config, suite.db)
+	suite.typeconverter = typeutils.NewConverter(suite.config, suite.db, suite.log)
 }
 
 func (suite *ASToInternalTestSuite) SetupTest() {
@@ -346,7 +346,7 @@ func (suite *ASToInternalTestSuite) SetupTest() {
 }
 
 func (suite *ASToInternalTestSuite) TestParsePerson() {
-	testPerson := suite.people["new_person_1"]
+	testPerson := suite.people["https://unknown-instance.com/users/brand_new_person"]
 
 	acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), testPerson, false)
 	assert.NoError(suite.T(), err)
@@ -363,8 +363,6 @@ func (suite *ASToInternalTestSuite) TestParsePerson() {
 	suite.Equal("https://unknown-instance.com/@brand_new_person", acct.URL)
 	suite.True(acct.Discoverable)
 	suite.Equal("https://unknown-instance.com/users/brand_new_person#main-key", acct.PublicKeyURI)
-	suite.Equal("https://unknown-instance.com/media/some_avatar_filename.jpeg", acct.AvatarRemoteURL)
-	suite.Equal("https://unknown-instance.com/media/some_header_filename.jpeg", acct.HeaderRemoteURL)
 	suite.False(acct.Locked)
 }
 

internal/typeutils/converter.go

@@ -23,6 +23,7 @@
 	"net/url"
 
 	"github.com/go-fed/activity/streams/vocab"
+	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/cache"
@@ -179,15 +180,17 @@ type TypeConverter interface {
 type converter struct {
 	config        *config.Config
 	db            db.DB
+	log           *logrus.Logger
 	frontendCache cache.Cache
 	asCache       cache.Cache
 }
 
 // NewConverter returns a new Converter
-func NewConverter(config *config.Config, db db.DB) TypeConverter {
+func NewConverter(config *config.Config, db db.DB, log *logrus.Logger) TypeConverter {
 	return &converter{
 		config:        config,
 		db:            db,
+		log:           log,
 		frontendCache: cache.New(),
 		asCache:       cache.New(),
 	}

internal/typeutils/converter_test.go

@@ -19,9 +19,9 @@
 package typeutils_test
 
 import (
+	"github.com/go-fed/activity/streams/vocab"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/suite"
-	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -35,7 +35,7 @@ type ConverterStandardTestSuite struct {
 	db       db.DB
 	log      *logrus.Logger
 	accounts map[string]*gtsmodel.Account
-	people   map[string]ap.Accountable
+	people   map[string]vocab.ActivityStreamsPerson
 
 	typeconverter typeutils.TypeConverter
 }

internal/typeutils/internaltoas.go

@@ -27,7 +27,6 @@
 
 	"github.com/go-fed/activity/streams"
 	"github.com/go-fed/activity/streams/vocab"
-	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
@@ -600,12 +599,12 @@ func (c *converter) FollowToAS(ctx context.Context, f *gtsmodel.Follow, originAc
 }
 
 func (c *converter) MentionToAS(ctx context.Context, m *gtsmodel.Mention) (vocab.ActivityStreamsMention, error) {
-	if m.OriginAccount == nil {
+	if m.TargetAccount == nil {
-		a := &gtsmodel.Account{}
+		a, err := c.db.GetAccountByID(ctx, m.TargetAccountID)
-		if err := c.db.GetWhere(ctx, []db.Where{{Key: "target_account_id", Value: m.TargetAccountID}}, a); err != nil {
+		if err != nil {
 			return nil, fmt.Errorf("MentionToAS: error getting target account from db: %s", err)
 		}
-		m.OriginAccount = a
+		m.TargetAccount = a
 	}
 
 	// create the mention
@@ -613,21 +612,21 @@ func (c *converter) MentionToAS(ctx context.Context, m *gtsmodel.Mention) (vocab
 
 	// href -- this should be the URI of the mentioned user
 	hrefProp := streams.NewActivityStreamsHrefProperty()
-	hrefURI, err := url.Parse(m.OriginAccount.URI)
+	hrefURI, err := url.Parse(m.TargetAccount.URI)
 	if err != nil {
-		return nil, fmt.Errorf("MentionToAS: error parsing uri %s: %s", m.OriginAccount.URI, err)
+		return nil, fmt.Errorf("MentionToAS: error parsing uri %s: %s", m.TargetAccount.URI, err)
 	}
 	hrefProp.SetIRI(hrefURI)
 	mention.SetActivityStreamsHref(hrefProp)
 
 	// name -- this should be the namestring of the mentioned user, something like @whatever@example.org
 	var domain string
-	if m.OriginAccount.Domain == "" {
+	if m.TargetAccount.Domain == "" {
 		domain = c.config.AccountDomain
 	} else {
-		domain = m.OriginAccount.Domain
+		domain = m.TargetAccount.Domain
 	}
-	username := m.OriginAccount.Username
+	username := m.TargetAccount.Username
 	nameString := fmt.Sprintf("@%s@%s", username, domain)
 	nameProp := streams.NewActivityStreamsNameProperty()
 	nameProp.AppendXMLSchemaString(nameString)
@@ -684,8 +683,8 @@ func (c *converter) AttachmentToAS(ctx context.Context, a *gtsmodel.MediaAttachm
 func (c *converter) FaveToAS(ctx context.Context, f *gtsmodel.StatusFave) (vocab.ActivityStreamsLike, error) {
 	// check if targetStatus is already pinned to this fave, and fetch it if not
 	if f.Status == nil {
-		s := &gtsmodel.Status{}
+		s, err := c.db.GetStatusByID(ctx, f.StatusID)
-		if err := c.db.GetByID(ctx, f.StatusID, s); err != nil {
+		if err != nil {
 			return nil, fmt.Errorf("FaveToAS: error fetching target status from database: %s", err)
 		}
 		f.Status = s
@@ -693,8 +692,8 @@ func (c *converter) FaveToAS(ctx context.Context, f *gtsmodel.StatusFave) (vocab
 
 	// check if the targetAccount is already pinned to this fave, and fetch it if not
 	if f.TargetAccount == nil {
-		a := &gtsmodel.Account{}
+		a, err := c.db.GetAccountByID(ctx, f.TargetAccountID)
-		if err := c.db.GetByID(ctx, f.TargetAccountID, a); err != nil {
+		if err != nil {
 			return nil, fmt.Errorf("FaveToAS: error fetching target account from database: %s", err)
 		}
 		f.TargetAccount = a
@@ -702,8 +701,8 @@ func (c *converter) FaveToAS(ctx context.Context, f *gtsmodel.StatusFave) (vocab
 
 	// check if the faving account is already pinned to this fave, and fetch it if not
 	if f.Account == nil {
-		a := &gtsmodel.Account{}
+		a, err := c.db.GetAccountByID(ctx, f.AccountID)
-		if err := c.db.GetByID(ctx, f.AccountID, a); err != nil {
+		if err != nil {
 			return nil, fmt.Errorf("FaveToAS: error fetching faving account from database: %s", err)
 		}
 		f.Account = a
@@ -754,8 +753,8 @@ func (c *converter) FaveToAS(ctx context.Context, f *gtsmodel.StatusFave) (vocab
 func (c *converter) BoostToAS(ctx context.Context, boostWrapperStatus *gtsmodel.Status, boostingAccount *gtsmodel.Account, boostedAccount *gtsmodel.Account) (vocab.ActivityStreamsAnnounce, error) {
 	// the boosted status is probably pinned to the boostWrapperStatus but double check to make sure
 	if boostWrapperStatus.BoostOf == nil {
-		b := &gtsmodel.Status{}
+		b, err := c.db.GetStatusByID(ctx, boostWrapperStatus.BoostOfID)
-		if err := c.db.GetByID(ctx, boostWrapperStatus.BoostOfID, b); err != nil {
+		if err != nil {
 			return nil, fmt.Errorf("BoostToAS: error getting status with ID %s from the db: %s", boostWrapperStatus.BoostOfID, err)
 		}
 		boostWrapperStatus.BoostOf = b
@@ -837,16 +836,16 @@ func (c *converter) BoostToAS(ctx context.Context, boostWrapperStatus *gtsmodel.
 */
 func (c *converter) BlockToAS(ctx context.Context, b *gtsmodel.Block) (vocab.ActivityStreamsBlock, error) {
 	if b.Account == nil {
-		a := &gtsmodel.Account{}
+		a, err := c.db.GetAccountByID(ctx, b.AccountID)
-		if err := c.db.GetByID(ctx, b.AccountID, a); err != nil {
+		if err != nil {
-			return nil, fmt.Errorf("BlockToAS: error getting block account from database: %s", err)
+			return nil, fmt.Errorf("BlockToAS: error getting block owner account from database: %s", err)
 		}
 		b.Account = a
 	}
 
 	if b.TargetAccount == nil {
-		a := &gtsmodel.Account{}
+		a, err := c.db.GetAccountByID(ctx, b.TargetAccountID)
-		if err := c.db.GetByID(ctx, b.TargetAccountID, a); err != nil {
+		if err != nil {
 			return nil, fmt.Errorf("BlockToAS: error getting block target account from database: %s", err)
 		}
 		b.TargetAccount = a

internal/typeutils/internaltoas_test.go

@@ -44,7 +44,7 @@ func (suite *InternalToASTestSuite) SetupSuite() {
 	suite.log = testrig.NewTestLog()
 	suite.accounts = testrig.NewTestAccounts()
 	suite.people = testrig.NewTestFediPeople()
-	suite.typeconverter = typeutils.NewConverter(suite.config, suite.db)
+	suite.typeconverter = typeutils.NewConverter(suite.config, suite.db, suite.log)
 }
 
 func (suite *InternalToASTestSuite) SetupTest() {

internal/typeutils/internaltofrontend.go

@@ -316,6 +316,8 @@ func (c *converter) TagToMasto(ctx context.Context, t *gtsmodel.Tag) (model.Tag,
 }
 
 func (c *converter) StatusToMasto(ctx context.Context, s *gtsmodel.Status, requestingAccount *gtsmodel.Account) (*model.Status, error) {
+	l := c.log
+
 	repliesCount, err := c.db.CountStatusReplies(ctx, s)
 	if err != nil {
 		return nil, fmt.Errorf("error counting replies: %s", err)
@@ -392,7 +394,8 @@ func (c *converter) StatusToMasto(ctx context.Context, s *gtsmodel.Status, reque
 		for _, gtsAttachment := range s.Attachments {
 			mastoAttachment, err := c.AttachmentToMasto(ctx, gtsAttachment)
 			if err != nil {
-				return nil, fmt.Errorf("error converting attachment with id %s: %s", gtsAttachment.ID, err)
+				l.Errorf("error converting attachment with id %s: %s", gtsAttachment.ID, err)
+				continue
 			}
 			mastoAttachments = append(mastoAttachments, mastoAttachment)
 		}
@@ -402,11 +405,13 @@ func (c *converter) StatusToMasto(ctx context.Context, s *gtsmodel.Status, reque
 		for _, aID := range s.AttachmentIDs {
 			gtsAttachment, err := c.db.GetAttachmentByID(ctx, aID)
 			if err != nil {
-				return nil, fmt.Errorf("error getting attachment with id %s: %s", aID, err)
+				l.Errorf("error getting attachment with id %s: %s", aID, err)
+				continue
 			}
 			mastoAttachment, err := c.AttachmentToMasto(ctx, gtsAttachment)
 			if err != nil {
-				return nil, fmt.Errorf("error converting attachment with id %s: %s", aID, err)
+				l.Errorf("error converting attachment with id %s: %s", aID, err)
+				continue
 			}
 			mastoAttachments = append(mastoAttachments, mastoAttachment)
 		}
@@ -419,7 +424,8 @@ func (c *converter) StatusToMasto(ctx context.Context, s *gtsmodel.Status, reque
 		for _, gtsMention := range s.Mentions {
 			mastoMention, err := c.MentionToMasto(ctx, gtsMention)
 			if err != nil {
-				return nil, fmt.Errorf("error converting mention with id %s: %s", gtsMention.ID, err)
+				l.Errorf("error converting mention with id %s: %s", gtsMention.ID, err)
+				continue
 			}
 			mastoMentions = append(mastoMentions, mastoMention)
 		}
@@ -429,11 +435,13 @@ func (c *converter) StatusToMasto(ctx context.Context, s *gtsmodel.Status, reque
 		for _, mID := range s.MentionIDs {
 			gtsMention, err := c.db.GetMention(ctx, mID)
 			if err != nil {
-				return nil, fmt.Errorf("error getting mention with id %s: %s", mID, err)
+				l.Errorf("error getting mention with id %s: %s", mID, err)
+				continue
 			}
 			mastoMention, err := c.MentionToMasto(ctx, gtsMention)
 			if err != nil {
-				return nil, fmt.Errorf("error converting mention with id %s: %s", gtsMention.ID, err)
+				l.Errorf("error converting mention with id %s: %s", gtsMention.ID, err)
+				continue
 			}
 			mastoMentions = append(mastoMentions, mastoMention)
 		}
@@ -446,7 +454,8 @@ func (c *converter) StatusToMasto(ctx context.Context, s *gtsmodel.Status, reque
 		for _, gtsTag := range s.Tags {
 			mastoTag, err := c.TagToMasto(ctx, gtsTag)
 			if err != nil {
-				return nil, fmt.Errorf("error converting tag with id %s: %s", gtsTag.ID, err)
+				l.Errorf("error converting tag with id %s: %s", gtsTag.ID, err)
+				continue
 			}
 			mastoTags = append(mastoTags, mastoTag)
 		}
@@ -456,11 +465,13 @@ func (c *converter) StatusToMasto(ctx context.Context, s *gtsmodel.Status, reque
 		for _, t := range s.TagIDs {
 			gtsTag := &gtsmodel.Tag{}
 			if err := c.db.GetByID(ctx, t, gtsTag); err != nil {
-				return nil, fmt.Errorf("error getting tag with id %s: %s", t, err)
+				l.Errorf("error getting tag with id %s: %s", t, err)
+				continue
 			}
 			mastoTag, err := c.TagToMasto(ctx, gtsTag)
 			if err != nil {
-				return nil, fmt.Errorf("error converting tag with id %s: %s", gtsTag.ID, err)
+				l.Errorf("error converting tag with id %s: %s", gtsTag.ID, err)
+				continue
 			}
 			mastoTags = append(mastoTags, mastoTag)
 		}
@@ -473,7 +484,8 @@ func (c *converter) StatusToMasto(ctx context.Context, s *gtsmodel.Status, reque
 		for _, gtsEmoji := range s.Emojis {
 			mastoEmoji, err := c.EmojiToMasto(ctx, gtsEmoji)
 			if err != nil {
-				return nil, fmt.Errorf("error converting emoji with id %s: %s", gtsEmoji.ID, err)
+				l.Errorf("error converting emoji with id %s: %s", gtsEmoji.ID, err)
+				continue
 			}
 			mastoEmojis = append(mastoEmojis, mastoEmoji)
 		}
@@ -483,11 +495,13 @@ func (c *converter) StatusToMasto(ctx context.Context, s *gtsmodel.Status, reque
 		for _, e := range s.EmojiIDs {
 			gtsEmoji := &gtsmodel.Emoji{}
 			if err := c.db.GetByID(ctx, e, gtsEmoji); err != nil {
-				return nil, fmt.Errorf("error getting emoji with id %s: %s", e, err)
+				l.Errorf("error getting emoji with id %s: %s", e, err)
+				continue
 			}
 			mastoEmoji, err := c.EmojiToMasto(ctx, gtsEmoji)
 			if err != nil {
-				return nil, fmt.Errorf("error converting emoji with id %s: %s", gtsEmoji.ID, err)
+				l.Errorf("error converting emoji with id %s: %s", gtsEmoji.ID, err)
+				continue
 			}
 			mastoEmojis = append(mastoEmojis, mastoEmoji)
 		}

internal/util/regexes.go

@@ -30,7 +30,7 @@
 )
 
 var (
-	mentionNameRegexString = `^@(\w+)(?:@([a-zA-Z0-9_\-\.]+)?)$`
+	mentionNameRegexString = `^@(\w+)(?:@([a-zA-Z0-9_\-\.:]+)?)$`
 	// mention name regex captures the username and domain part from a mention string
 	// such as @whatever_user@example.org, returning whatever_user and example.org (without the @ symbols)
 	mentionNameRegex = regexp.MustCompile(mentionNameRegexString)

testrig/testmodels.go

@@ -36,7 +36,6 @@
 	"github.com/go-fed/activity/pub"
 	"github.com/go-fed/activity/streams"
 	"github.com/go-fed/activity/streams/vocab"
-	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 )
@@ -1224,12 +1223,14 @@ func NewTestActivities(accounts map[string]*gtsmodel.Account) map[string]Activit
 	dmForZork := newNote(
 		URLMustParse("https://fossbros-anonymous.io/users/foss_satan/statuses/5424b153-4553-4f30-9358-7b92f7cd42f6"),
 		URLMustParse("https://fossbros-anonymous.io/@foss_satan/5424b153-4553-4f30-9358-7b92f7cd42f6"),
+		time.Now(),
 		"hey zork here's a new private note for you",
 		"new note for zork",
 		URLMustParse("https://fossbros-anonymous.io/users/foss_satan"),
 		[]*url.URL{URLMustParse("http://localhost:8080/users/the_mighty_zork")},
 		nil,
-		true)
+		true,
+		[]vocab.ActivityStreamsMention{})
 	createDmForZork := wrapNoteInCreate(
 		URLMustParse("https://fossbros-anonymous.io/users/foss_satan/statuses/5424b153-4553-4f30-9358-7b92f7cd42f6/activity"),
 		URLMustParse("https://fossbros-anonymous.io/users/foss_satan"),
@@ -1248,15 +1249,15 @@ func NewTestActivities(accounts map[string]*gtsmodel.Account) map[string]Activit
 }
 
 // NewTestFediPeople returns a bunch of activity pub Person representations for testing converters and so on.
-func NewTestFediPeople() map[string]ap.Accountable {
+func NewTestFediPeople() map[string]vocab.ActivityStreamsPerson {
 	newPerson1Priv, err := rsa.GenerateKey(rand.Reader, 2048)
 	if err != nil {
 		panic(err)
 	}
 	newPerson1Pub := &newPerson1Priv.PublicKey
 
-	return map[string]ap.Accountable{
+	return map[string]vocab.ActivityStreamsPerson{
-		"new_person_1": newPerson(
+		"https://unknown-instance.com/users/brand_new_person": newPerson(
 			URLMustParse("https://unknown-instance.com/users/brand_new_person"),
 			URLMustParse("https://unknown-instance.com/users/brand_new_person/following"),
 			URLMustParse("https://unknown-instance.com/users/brand_new_person/followers"),
@@ -1270,15 +1271,53 @@ func NewTestFediPeople() map[string]ap.Accountable {
 			true,
 			URLMustParse("https://unknown-instance.com/users/brand_new_person#main-key"),
 			newPerson1Pub,
-			URLMustParse("https://unknown-instance.com/media/some_avatar_filename.jpeg"),
+			nil,
 			"image/jpeg",
-			URLMustParse("https://unknown-instance.com/media/some_header_filename.jpeg"),
+			nil,
 			"image/png",
 			false,
 		),
 	}
 }
 
+func NewTestFediStatuses() map[string]vocab.ActivityStreamsNote {
+	return map[string]vocab.ActivityStreamsNote{
+		"https://unknown-instance.com/users/brand_new_person/statuses/01FE4NTHKWW7THT67EF10EB839": newNote(
+			URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01FE4NTHKWW7THT67EF10EB839"),
+			URLMustParse("https://unknown-instance.com/users/@brand_new_person/01FE4NTHKWW7THT67EF10EB839"),
+			time.Now(),
+			"Hello world!",
+			"",
+			URLMustParse("https://unknown-instance.com/users/brand_new_person"),
+			[]*url.URL{
+				URLMustParse("https://www.w3.org/ns/activitystreams#Public"),
+			},
+			[]*url.URL{},
+			false,
+			[]vocab.ActivityStreamsMention{},
+		),
+		"https://unknown-instance.com/users/brand_new_person/statuses/01FE5Y30E3W4P7TRE0R98KAYQV": newNote(
+			URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01FE5Y30E3W4P7TRE0R98KAYQV"),
+			URLMustParse("https://unknown-instance.com/users/@brand_new_person/01FE5Y30E3W4P7TRE0R98KAYQV"),
+			time.Now(),
+			"Hey @the_mighty_zork@localhost:8080 how's it going?",
+			"",
+			URLMustParse("https://unknown-instance.com/users/brand_new_person"),
+			[]*url.URL{
+				URLMustParse("https://www.w3.org/ns/activitystreams#Public"),
+			},
+			[]*url.URL{},
+			false,
+			[]vocab.ActivityStreamsMention{
+				newMention(
+					URLMustParse("http://localhost:8080/users/the_mighty_zork"),
+					"@the_mighty_zork@localhost:8080",
+				),
+			},
+		),
+	}
+}
+
 // NewTestDereferenceRequests returns a map of incoming dereference requests, with their signatures.
 func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[string]ActivityWithSignature {
 	var sig, digest, date string
@@ -1418,7 +1457,7 @@ func newPerson(
 	avatarContentType string,
 	headerURL *url.URL,
 	headerContentType string,
-	manuallyApprovesFollowers bool) ap.Accountable {
+	manuallyApprovesFollowers bool) vocab.ActivityStreamsPerson {
 	person := streams.NewActivityStreamsPerson()
 
 	// id should be the activitypub URI of this user
@@ -1583,16 +1622,32 @@ func newPerson(
 	return person
 }
 
+func newMention(uri *url.URL, namestring string) vocab.ActivityStreamsMention {
+	mention := streams.NewActivityStreamsMention()
+
+	hrefProp := streams.NewActivityStreamsHrefProperty()
+	hrefProp.SetIRI(uri)
+	mention.SetActivityStreamsHref(hrefProp)
+
+	nameProp := streams.NewActivityStreamsNameProperty()
+	nameProp.AppendXMLSchemaString(namestring)
+	mention.SetActivityStreamsName(nameProp)
+
+	return mention
+}
+
 // newNote returns a new activity streams note for the given parameters
 func newNote(
 	noteID *url.URL,
 	noteURL *url.URL,
+	noteCreatedAt time.Time,
 	noteContent string,
 	noteSummary string,
 	noteAttributedTo *url.URL,
 	noteTo []*url.URL,
 	noteCC []*url.URL,
-	noteSensitive bool) vocab.ActivityStreamsNote {
+	noteSensitive bool,
+	noteMentions []vocab.ActivityStreamsMention) vocab.ActivityStreamsNote {
 
 	// create the note itself
 	note := streams.NewActivityStreamsNote()
@@ -1611,6 +1666,13 @@ func newNote(
 		note.SetActivityStreamsUrl(url)
 	}
 
+	if noteCreatedAt.IsZero() {
+		noteCreatedAt = time.Now()
+	}
+	published := streams.NewActivityStreamsPublishedProperty()
+	published.Set(noteCreatedAt)
+	note.SetActivityStreamsPublished(published)
+
 	// set noteContent
 	if noteContent != "" {
 		content := streams.NewActivityStreamsContentProperty()
@@ -1632,6 +1694,34 @@ func newNote(
 		note.SetActivityStreamsAttributedTo(attributedTo)
 	}
 
+	// set noteTO
+	if noteTo != nil {
+		to := streams.NewActivityStreamsToProperty()
+		for _, r := range noteTo {
+			to.AppendIRI(r)
+		}
+		note.SetActivityStreamsTo(to)
+	}
+
+	// set noteCC
+	if noteCC != nil {
+		cc := streams.NewActivityStreamsCcProperty()
+		for _, r := range noteCC {
+			cc.AppendIRI(r)
+		}
+		note.SetActivityStreamsCc(cc)
+	}
+
+	// set note tags
+	tag := streams.NewActivityStreamsTagProperty()
+
+	// mentions
+	for _, m := range noteMentions {
+		tag.AppendActivityStreamsMention(m)
+	}
+
+	note.SetActivityStreamsTag(tag)
+
 	return note
 }
 

testrig/typeconverter.go

@@ -25,5 +25,5 @@
 
 // NewTestTypeConverter returned a type converter with the given db and the default test config
 func NewTestTypeConverter(db db.DB) typeutils.TypeConverter {
-	return typeutils.NewConverter(NewTestConfig(), db)
+	return typeutils.NewConverter(NewTestConfig(), db, NewTestLog())
 }