Refine statuses (#26)

Remote media is now dereferenced and attached properly to incoming federated statuses.
    Mentions are now dereferenced and attached properly to incoming federated statuses.
    Small fixes to status visibility.
    Allow URL params for filtering statuses:

	// ExcludeRepliesKey is for specifying whether to exclude replies in a list of returned statuses by an account.
      	// PinnedKey is for specifying whether to include pinned statuses in a list of returned statuses by an account.
      	// MaxIDKey is for specifying the maximum ID of the status to retrieve.
      	// MediaOnlyKey is for specifying that only statuses with media should be returned in a list of returned statuses by an account.

    Add endpoint for fetching an account's statuses.

internal/api/client/account/account.go

@@ -32,6 +32,17 @@
 )
 
 const (
+	// LimitKey is for setting the return amount limit for eg., requesting an account's statuses
+	LimitKey = "limit"
+	// ExcludeRepliesKey is for specifying whether to exclude replies in a list of returned statuses by an account.
+	ExcludeRepliesKey = "exclude_replies"
+	// PinnedKey is for specifying whether to include pinned statuses in a list of returned statuses by an account.
+	PinnedKey = "pinned"
+	// MaxIDKey is for specifying the maximum ID of the status to retrieve.
+	MaxIDKey = "max_id"
+	// MediaOnlyKey is for specifying that only statuses with media should be returned in a list of returned statuses by an account.
+	MediaOnlyKey = "only_media"
+
 	// IDKey is the key to use for retrieving account ID in requests
 	IDKey = "id"
 	// BasePath is the base API path for this module
@@ -42,6 +53,10 @@
 	VerifyPath = BasePath + "/verify_credentials"
 	// UpdateCredentialsPath is for updating account credentials
 	UpdateCredentialsPath = BasePath + "/update_credentials"
+	// GetStatusesPath is for showing an account's statuses
+	GetStatusesPath = BasePathWithID + "/statuses"
+	// GetFollowersPath is for showing an account's followers
+	GetFollowersPath = BasePathWithID + "/followers"
 )
 
 // Module implements the ClientAPIModule interface for account-related actions
@@ -65,6 +80,8 @@ func (m *Module) Route(r router.Router) error {
 	r.AttachHandler(http.MethodPost, BasePath, m.AccountCreatePOSTHandler)
 	r.AttachHandler(http.MethodGet, BasePathWithID, m.muxHandler)
 	r.AttachHandler(http.MethodPatch, BasePathWithID, m.muxHandler)
+	r.AttachHandler(http.MethodGet, GetStatusesPath, m.AccountStatusesGETHandler)
+	r.AttachHandler(http.MethodGet, GetFollowersPath, m.AccountFollowersGETHandler)
 	return nil
 }
 

internal/api/client/account/followers.go

@@ -0,0 +1,49 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package account
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+)
+
+// AccountFollowersGETHandler serves the followers of the requested account, if they're visible to the requester.
+func (m *Module) AccountFollowersGETHandler(c *gin.Context) {
+	authed, err := oauth.Authed(c, true, true, true, true)
+	if err != nil {
+		c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"})
+		return
+	}
+
+	targetAcctID := c.Param(IDKey)
+	if targetAcctID == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "no account id specified"})
+		return
+	}
+
+	followers, errWithCode := m.processor.AccountFollowersGet(authed, targetAcctID)
+	if errWithCode != nil {
+		c.JSON(errWithCode.Code(), gin.H{"error": errWithCode.Safe()})
+		return
+	}
+
+	c.JSON(http.StatusOK, followers)
+}

internal/api/client/account/statuses.go

@@ -0,0 +1,117 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package account
+
+import (
+	"net/http"
+	"strconv"
+
+	"github.com/gin-gonic/gin"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+)
+
+// AccountStatusesGETHandler serves the statuses of the requested account, if they're visible to the requester.
+//
+// Several different filters might be passed into this function in the query:
+//
+// 	limit -- show only limit number of statuses
+// 	exclude_replies -- exclude statuses that are a reply to another status
+// 	max_id -- the maximum ID of the status to show
+// 	pinned -- show only pinned statuses
+// 	media_only -- show only statuses that have media attachments
+func (m *Module) AccountStatusesGETHandler(c *gin.Context) {
+	l := m.log.WithField("func", "AccountStatusesGETHandler")
+
+	authed, err := oauth.Authed(c, false, false, false, false)
+	if err != nil {
+		l.Debugf("error authing: %s", err)
+		c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"})
+		return
+	}
+
+	targetAcctID := c.Param(IDKey)
+	if targetAcctID == "" {
+		l.Debug("no account id specified in query")
+		c.JSON(http.StatusBadRequest, gin.H{"error": "no account id specified"})
+		return
+	}
+
+	limit := 30
+	limitString := c.Query(LimitKey)
+	if limitString != "" {
+		i, err := strconv.ParseInt(limitString, 10, 64)
+		if err != nil {
+			l.Debugf("error parsing limit string: %s", err)
+			c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse limit query param"})
+			return
+		}
+		limit = int(i)
+	}
+
+	excludeReplies := false
+	excludeRepliesString := c.Query(ExcludeRepliesKey)
+	if excludeRepliesString != "" {
+		i, err := strconv.ParseBool(excludeRepliesString)
+		if err != nil {
+			l.Debugf("error parsing replies string: %s", err)
+			c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse exclude replies query param"})
+			return
+		}
+		excludeReplies = i
+	}
+
+	maxID := ""
+	maxIDString := c.Query(MaxIDKey)
+	if maxIDString != "" {
+		maxID = maxIDString
+	}
+
+	pinned := false
+	pinnedString := c.Query(PinnedKey)
+	if pinnedString != "" {
+		i, err := strconv.ParseBool(pinnedString)
+		if err != nil {
+			l.Debugf("error parsing pinned string: %s", err)
+			c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse pinned query param"})
+			return
+		}
+		pinned = i
+	}
+
+	mediaOnly := false
+	mediaOnlyString := c.Query(MediaOnlyKey)
+	if mediaOnlyString != "" {
+		i, err := strconv.ParseBool(mediaOnlyString)
+		if err != nil {
+			l.Debugf("error parsing media only string: %s", err)
+			c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse media only query param"})
+			return
+		}
+		mediaOnly = i
+	}
+
+	statuses, errWithCode := m.processor.AccountStatusesGet(authed, targetAcctID, limit, excludeReplies, maxID, pinned, mediaOnly)
+	if errWithCode != nil {
+		l.Debugf("error from processor account statuses get: %s", errWithCode)
+		c.JSON(errWithCode.Code(), gin.H{"error": errWithCode.Safe()})
+		return
+	}
+
+	c.JSON(http.StatusOK, statuses)
+}

internal/api/model/status.go

@@ -55,7 +55,7 @@ type Status struct {
 	// Have you bookmarked this status?
 	Bookmarked bool `json:"bookmarked"`
 	// Have you pinned this status? Only appears if the status is pinnable.
-	Pinned bool `json:"pinned"`
+	Pinned bool `json:"pinned,omitempty"`
 	// HTML-encoded status content.
 	Content string `json:"content"`
 	// The status being reblogged.
@@ -86,23 +86,23 @@ type Status struct {
 // It should be used at the path https://mastodon.example/api/v1/statuses
 type StatusCreateRequest struct {
 	// Text content of the status. If media_ids is provided, this becomes optional. Attaching a poll is optional while status is provided.
-	Status string `form:"status"`
+	Status string `form:"status" json:"status" xml:"status"`
 	// Array of Attachment ids to be attached as media. If provided, status becomes optional, and poll cannot be used.
 	MediaIDs []string `form:"media_ids" json:"media_ids" xml:"media_ids"`
 	// Poll to include with this status.
-	Poll *PollRequest `form:"poll"`
+	Poll *PollRequest `form:"poll" json:"poll" xml:"poll"`
 	// ID of the status being replied to, if status is a reply
-	InReplyToID string `form:"in_reply_to_id"`
+	InReplyToID string `form:"in_reply_to_id" json:"in_reply_to_id" xml:"in_reply_to_id"`
 	// Mark status and attached media as sensitive?
-	Sensitive bool `form:"sensitive"`
+	Sensitive bool `form:"sensitive" json:"sensitive" xml:"sensitive"`
 	// Text to be shown as a warning or subject before the actual content. Statuses are generally collapsed behind this field.
-	SpoilerText string `form:"spoiler_text"`
+	SpoilerText string `form:"spoiler_text" json:"spoiler_text" xml:"spoiler_text"`
 	// Visibility of the posted status. Enumerable oneOf public, unlisted, private, direct.
-	Visibility Visibility `form:"visibility"`
+	Visibility Visibility `form:"visibility" json:"visibility" xml:"visibility"`
 	// ISO 8601 Datetime at which to schedule a status. Providing this paramter will cause ScheduledStatus to be returned instead of Status. Must be at least 5 minutes in the future.
-	ScheduledAt string `form:"scheduled_at"`
+	ScheduledAt string `form:"scheduled_at" json:"scheduled_at" xml:"scheduled_at"`
 	// ISO 639 language code for this status.
-	Language string `form:"language"`
+	Language string `form:"language" json:"language" xml:"language"`
 }
 
 // Visibility denotes the visibility of this status to other users
@@ -130,13 +130,13 @@ type AdvancedStatusCreateForm struct {
 // to the standard mastodon-compatible ones.
 type AdvancedVisibilityFlagsForm struct {
 	// The gotosocial visibility model
-	VisibilityAdvanced *string `form:"visibility_advanced"`
+	VisibilityAdvanced *string `form:"visibility_advanced" json:"visibility_advanced" xml:"visibility_advanced"`
 	// This status will be federated beyond the local timeline(s)
-	Federated *bool `form:"federated"`
+	Federated *bool `form:"federated" json:"federated" xml:"federated"`
 	// This status can be boosted/reblogged
-	Boostable *bool `form:"boostable"`
+	Boostable *bool `form:"boostable" json:"boostable" xml:"boostable"`
 	// This status can be replied to
-	Replyable *bool `form:"replyable"`
+	Replyable *bool `form:"replyable" json:"replyable" xml:"replyable"`
 	// This status can be liked/faved
-	Likeable *bool `form:"likeable"`
+	Likeable *bool `form:"likeable" json:"likeable" xml:"likeable"`
 }

internal/api/s2s/user/userget.go

@@ -56,7 +56,7 @@ func (m *Module) UsersGETHandler(c *gin.Context) {
 
 	// make a copy of the context to pass along so we don't break anything
 	cp := c.Copy()
-	user, err := m.processor.GetFediUser(requestedUsername, cp.Request) // GetAPUser handles auth as well
+	user, err := m.processor.GetFediUser(requestedUsername, cp.Request) // GetFediUser handles auth as well
 	if err != nil {
 		l.Info(err.Error())
 		c.JSON(err.Code(), gin.H{"error": err.Safe()})

internal/db/db.go

@@ -160,16 +160,14 @@ type DB interface {
 	// In case of no entries, a 'no entries' error will be returned
 	GetFavesByAccountID(accountID string, faves *[]gtsmodel.StatusFave) error
 
-	// GetStatusesByAccountID is a shortcut for the common action of fetching a list of statuses produced by accountID.
+	// CountStatusesByAccountID is a shortcut for the common action of counting statuses produced by accountID.
-	// The given slice 'statuses' will be set to the result of the query, whatever it is.
+	CountStatusesByAccountID(accountID string) (int, error)
-	// In case of no entries, a 'no entries' error will be returned
-	GetStatusesByAccountID(accountID string, statuses *[]gtsmodel.Status) error
 
 	// GetStatusesByTimeDescending is a shortcut for getting the most recent statuses. accountID is optional, if not provided
 	// then all statuses will be returned. If limit is set to 0, the size of the returned slice will not be limited. This can
 	// be very memory intensive so you probably shouldn't do this!
 	// In case of no entries, a 'no entries' error will be returned
-	GetStatusesByTimeDescending(accountID string, statuses *[]gtsmodel.Status, limit int) error
+	GetStatusesByTimeDescending(accountID string, statuses *[]gtsmodel.Status, limit int, excludeReplies bool, maxID string, pinned bool, mediaOnly bool) error
 
 	// GetLastStatusForAccountID simply gets the most recent status by the given account.
 	// The given slice 'status' pointer will be set to the result of the query, whatever it is.
@@ -251,9 +249,6 @@ type DB interface {
 	// StatusBookmarkedBy checks if a given status has been bookmarked by a given account ID
 	StatusBookmarkedBy(status *gtsmodel.Status, accountID string) (bool, error)
 
-	// StatusPinnedBy checks if a given status has been pinned by a given account ID
-	StatusPinnedBy(status *gtsmodel.Status, accountID string) (bool, error)
-
 	// FaveStatus faves the given status, using accountID as the faver.
 	// The returned fave will be nil if the status was already faved.
 	FaveStatus(status *gtsmodel.Status, accountID string) (*gtsmodel.StatusFave, error)

internal/db/pg/pg.go

@@ -456,23 +456,35 @@ func (ps *postgresService) GetFavesByAccountID(accountID string, faves *[]gtsmod
 	return nil
 }
 
-func (ps *postgresService) GetStatusesByAccountID(accountID string, statuses *[]gtsmodel.Status) error {
+func (ps *postgresService) CountStatusesByAccountID(accountID string) (int, error) {
-	if err := ps.conn.Model(statuses).Where("account_id = ?", accountID).Select(); err != nil {
+	count, err := ps.conn.Model(&gtsmodel.Status{}).Where("account_id = ?", accountID).Count()
+	if err != nil {
 		if err == pg.ErrNoRows {
-			return db.ErrNoEntries{}
+			return 0, nil
 		}
-		return err
+		return 0, err
 	}
-	return nil
+	return count, nil
 }
 
-func (ps *postgresService) GetStatusesByTimeDescending(accountID string, statuses *[]gtsmodel.Status, limit int) error {
+func (ps *postgresService) GetStatusesByTimeDescending(accountID string, statuses *[]gtsmodel.Status, limit int, excludeReplies bool, maxID string, pinned bool, mediaOnly bool) error {
 	q := ps.conn.Model(statuses).Order("created_at DESC")
+	if accountID != "" {
+		q = q.Where("account_id = ?", accountID)
+	}
 	if limit != 0 {
 		q = q.Limit(limit)
 	}
-	if accountID != "" {
+	if excludeReplies {
-		q = q.Where("account_id = ?", accountID)
+		q = q.Where("? IS NULL", pg.Ident("in_reply_to_id"))
+	}
+	if pinned {
+		q = q.Where("pinned = ?", true)
+	}
+	if mediaOnly {
+		q = q.WhereGroup(func(q *pg.Query) (*pg.Query, error) {
+			return q.Where("? IS NOT NULL", pg.Ident("attachments")).Where("attachments != '{}'"), nil
+		})
 	}
 	if err := q.Select(); err != nil {
 		if err == pg.ErrNoRows {
@@ -679,20 +691,23 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, targetAc
 	}
 
 	// if the target user doesn't exist (anymore) then the status also shouldn't be visible
-	targetUser := &gtsmodel.User{}
+	// note: we only do this for local users
-	if err := ps.conn.Model(targetUser).Where("account_id = ?", targetAccount.ID).Select(); err != nil {
+	if targetAccount.Domain == "" {
-		l.Debug("target user could not be selected")
+		targetUser := &gtsmodel.User{}
-		if err == pg.ErrNoRows {
+		if err := ps.conn.Model(targetUser).Where("account_id = ?", targetAccount.ID).Select(); err != nil {
-			return false, db.ErrNoEntries{}
+			l.Debug("target user could not be selected")
+			if err == pg.ErrNoRows {
+				return false, db.ErrNoEntries{}
+			}
+			return false, err
 		}
-		return false, err
-	}
 
-	// if target user is disabled, not yet approved, or not confirmed then don't show the status
+		// if target user is disabled, not yet approved, or not confirmed then don't show the status
-	// (although in the latter two cases it's unlikely they posted a status yet anyway, but you never know!)
+		// (although in the latter two cases it's unlikely they posted a status yet anyway, but you never know!)
-	if targetUser.Disabled || !targetUser.Approved || targetUser.ConfirmedAt.IsZero() {
+		if targetUser.Disabled || !targetUser.Approved || targetUser.ConfirmedAt.IsZero() {
-		l.Debug("target user is disabled, not approved, or not confirmed")
+			l.Debug("target user is disabled, not approved, or not confirmed")
-		return false, nil
+			return false, nil
+		}
 	}
 
 	// If requesting account is nil, that means whoever requested the status didn't auth, or their auth failed.
@@ -755,6 +770,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, targetAc
 			if blocked, err := ps.Blocked(relevantAccounts.ReplyToAccount.ID, requestingAccount.ID); err != nil {
 				return false, err
 			} else if blocked {
+				l.Debug("a block exists between requesting account and reply to account")
 				return false, nil
 			}
 		}
@@ -764,6 +780,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, targetAc
 			if blocked, err := ps.Blocked(relevantAccounts.BoostedAccount.ID, requestingAccount.ID); err != nil {
 				return false, err
 			} else if blocked {
+				l.Debug("a block exists between requesting account and boosted account")
 				return false, nil
 			}
 		}
@@ -773,6 +790,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, targetAc
 			if blocked, err := ps.Blocked(relevantAccounts.BoostedReplyToAccount.ID, requestingAccount.ID); err != nil {
 				return false, err
 			} else if blocked {
+				l.Debug("a block exists between requesting account and boosted reply to account")
 				return false, nil
 			}
 		}
@@ -782,9 +800,17 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, targetAc
 			if blocked, err := ps.Blocked(a.ID, requestingAccount.ID); err != nil {
 				return false, err
 			} else if blocked {
+				l.Debug("a block exists between requesting account and a mentioned account")
 				return false, nil
 			}
 		}
+
+		// if the requesting account is mentioned in the status it should always be visible
+		for _, acct := range relevantAccounts.MentionedAccounts {
+			if acct.ID == requestingAccount.ID {
+				return true, nil // yep it's mentioned!
+			}
+		}
 	}
 
 	// at this point we know neither account blocks the other, or another account mentioned or otherwise referred to in the status
@@ -800,6 +826,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, targetAc
 			return false, err
 		}
 		if !follows {
+			l.Debug("requested status is followers only but requesting account is not a follower")
 			return false, nil
 		}
 		return true, nil
@@ -810,16 +837,12 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, targetAc
 			return false, err
 		}
 		if !mutuals {
+			l.Debug("requested status is mutuals only but accounts aren't mufos")
 			return false, nil
 		}
 		return true, nil
 	case gtsmodel.VisibilityDirect:
-		// make sure the requesting account is mentioned in the status
+		l.Debug("requesting account requests a status it's not mentioned in")
-		for _, menchie := range targetStatus.Mentions {
-			if menchie == requestingAccount.ID {
-				return true, nil // yep it's mentioned!
-			}
-		}
 		return false, nil // it's not mentioned -_-
 	}
 
@@ -890,10 +913,16 @@ func (ps *postgresService) PullRelevantAccountsFromStatus(targetStatus *gtsmodel
 	}
 
 	// now get all accounts with IDs that are mentioned in the status
-	for _, mentionedAccountID := range targetStatus.Mentions {
+	for _, mentionID := range targetStatus.Mentions {
+
+		mention := &gtsmodel.Mention{}
+		if err := ps.conn.Model(mention).Where("id = ?", mentionID).Select(); err != nil {
+			return accounts, fmt.Errorf("error getting mention with id %s: %s", mentionID, err)
+		}
+
 		mentionedAccount := &gtsmodel.Account{}
-		if err := ps.conn.Model(mentionedAccount).Where("id = ?", mentionedAccountID).Select(); err != nil {
+		if err := ps.conn.Model(mentionedAccount).Where("id = ?", mention.TargetAccountID).Select(); err != nil {
-			return accounts, err
+			return accounts, fmt.Errorf("error getting mentioned account: %s", err)
 		}
 		accounts.MentionedAccounts = append(accounts.MentionedAccounts, mentionedAccount)
 	}
@@ -929,10 +958,6 @@ func (ps *postgresService) StatusBookmarkedBy(status *gtsmodel.Status, accountID
 	return ps.conn.Model(&gtsmodel.StatusBookmark{}).Where("status_id = ?", status.ID).Where("account_id = ?", accountID).Exists()
 }
 
-func (ps *postgresService) StatusPinnedBy(status *gtsmodel.Status, accountID string) (bool, error) {
-	return ps.conn.Model(&gtsmodel.StatusPin{}).Where("status_id = ?", status.ID).Where("account_id = ?", accountID).Exists()
-}
-
 func (ps *postgresService) FaveStatus(status *gtsmodel.Status, accountID string) (*gtsmodel.StatusFave, error) {
 	// first check if a fave already exists, we can just return if so
 	existingFave := &gtsmodel.StatusFave{}

internal/federation/federating_db.go

@@ -364,7 +364,7 @@ func (f *federatingDB) Get(c context.Context, id *url.URL) (value vocab.Type, er
 //
 // Under certain conditions and network activities, Create may be called
 // multiple times for the same ActivityStreams object.
-func (f *federatingDB) Create(c context.Context, asType vocab.Type) error {
+func (f *federatingDB) Create(ctx context.Context, asType vocab.Type) error {
 	l := f.log.WithFields(
 		logrus.Fields{
 			"func":   "Create",
@@ -373,6 +373,24 @@ func (f *federatingDB) Create(c context.Context, asType vocab.Type) error {
 	)
 	l.Debugf("received CREATE asType %+v", asType)
 
+	targetAcctI := ctx.Value(util.APAccount)
+	if targetAcctI == nil {
+		l.Error("target account wasn't set on context")
+	}
+	targetAcct, ok := targetAcctI.(*gtsmodel.Account)
+	if !ok {
+		l.Error("target account was set on context but couldn't be parsed")
+	}
+
+	fromFederatorChanI := ctx.Value(util.APFromFederatorChanKey)
+	if fromFederatorChanI == nil {
+		l.Error("from federator channel wasn't set on context")
+	}
+	fromFederatorChan, ok := fromFederatorChanI.(chan gtsmodel.FromFederator)
+	if !ok {
+		l.Error("from federator channel was set on context but couldn't be parsed")
+	}
+
 	switch gtsmodel.ActivityStreamsActivity(asType.GetTypeName()) {
 	case gtsmodel.ActivityStreamsCreate:
 		create, ok := asType.(vocab.ActivityStreamsCreate)
@@ -391,6 +409,12 @@ func (f *federatingDB) Create(c context.Context, asType vocab.Type) error {
 				if err := f.db.Put(status); err != nil {
 					return fmt.Errorf("database error inserting status: %s", err)
 				}
+
+				fromFederatorChan <- gtsmodel.FromFederator{
+					APObjectType:   gtsmodel.ActivityStreamsNote,
+					APActivityType: gtsmodel.ActivityStreamsCreate,
+					GTSModel:       status,
+				}
 			}
 		}
 	case gtsmodel.ActivityStreamsFollow:
@@ -407,6 +431,12 @@ func (f *federatingDB) Create(c context.Context, asType vocab.Type) error {
 		if err := f.db.Put(followRequest); err != nil {
 			return fmt.Errorf("database error inserting follow request: %s", err)
 		}
+
+		if !targetAcct.Locked {
+			if err := f.db.AcceptFollowRequest(followRequest.AccountID, followRequest.TargetAccountID); err != nil {
+				return fmt.Errorf("database error accepting follow request: %s", err)
+			}
+		}
 	}
 	return nil
 }

internal/federation/federatingprotocol.go

@@ -71,49 +71,7 @@ func (f *federator) PostInboxRequestBodyHook(ctx context.Context, r *http.Reques
 		l.Debug(err)
 		return nil, err
 	}
-
-	// derefence the actor of the activity already
-	// var requestingActorIRI *url.URL
-	// actorProp := activity.GetActivityStreamsActor()
-	// if actorProp != nil {
-	// 	for i := actorProp.Begin(); i != actorProp.End(); i = i.Next() {
-	// 		if i.IsIRI() {
-	// 			requestingActorIRI = i.GetIRI()
-	// 			break
-	// 		}
-	// 	}
-	// }
-	// if requestingActorIRI != nil {
-
-	// 	requestedAccountI := ctx.Value(util.APAccount)
-	// 	requestedAccount, ok := requestedAccountI.(*gtsmodel.Account)
-	// 	if !ok {
-	// 		return nil, errors.New("requested account was not set on request context")
-	// 	}
-
-	// 	requestingActor := &gtsmodel.Account{}
-	// 	if err := f.db.GetWhere("uri", requestingActorIRI.String(), requestingActor); err != nil {
-	// 		// there's been a proper error so return it
-	// 		if _, ok := err.(db.ErrNoEntries); !ok {
-	// 			return nil, fmt.Errorf("error getting requesting actor with id %s: %s", requestingActorIRI.String(), err)
-	// 		}
-
-	// 		// we don't know this account (yet) so let's dereference it right now
-	// 		person, err := f.DereferenceRemoteAccount(requestedAccount.Username, publicKeyOwnerURI)
-	// 		if err != nil {
-	// 			return ctx, false, fmt.Errorf("error dereferencing account with public key id %s: %s", publicKeyOwnerURI.String(), err)
-	// 		}
-
-	// 		a, err := f.typeConverter.ASRepresentationToAccount(person)
-	// 		if err != nil {
-	// 			return ctx, false, fmt.Errorf("error converting person with public key id %s to account: %s", publicKeyOwnerURI.String(), err)
-	// 		}
-	// 		requestingAccount = a
-	// 	}
-	// }
-
 	// set the activity on the context for use later on
-
 	return context.WithValue(ctx, util.APActivity, activity), nil
 }
 
@@ -285,14 +243,6 @@ func (f *federator) FederatingCallbacks(ctx context.Context) (wrapped pub.Federa
 	}
 
 	wrapped = pub.FederatingWrappedCallbacks{
-		// Follow handles additional side effects for the Follow ActivityStreams
-		// type, specific to the application using go-fed.
-		//
-		// The wrapping function can have one of several default behaviors,
-		// depending on the value of the OnFollow setting.
-		Follow: func(context.Context, vocab.ActivityStreamsFollow) error {
-			return nil
-		},
 		// OnFollow determines what action to take for this particular callback
 		// if a Follow Activity is handled.
 		OnFollow: onFollow,

internal/federation/federator.go

@@ -42,7 +42,9 @@ type Federator interface {
 	DereferenceRemoteAccount(username string, remoteAccountID *url.URL) (typeutils.Accountable, error)
 	// GetTransportForUser returns a new transport initialized with the key credentials belonging to the given username.
 	// This can be used for making signed http requests.
-	GetTransportForUser(username string) (pub.Transport, error)
+	//
+	// If username is an empty string, our instance user's credentials will be used instead.
+	GetTransportForUser(username string) (transport.Transport, error)
 	pub.CommonBehavior
 	pub.FederatingProtocol
 }

internal/federation/util.go

@@ -33,6 +33,7 @@
 	"github.com/go-fed/activity/streams/vocab"
 	"github.com/go-fed/httpsig"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
 
@@ -221,7 +222,7 @@ func (f *federator) DereferenceRemoteAccount(username string, remoteAccountID *u
 	return nil, fmt.Errorf("type name %s not supported", t.GetTypeName())
 }
 
-func (f *federator) GetTransportForUser(username string) (pub.Transport, error) {
+func (f *federator) GetTransportForUser(username string) (transport.Transport, error) {
 	// We need an account to use to create a transport for dereferecing the signature.
 	// If a username has been given, we can fetch the account with that username and use it.
 	// Otherwise, we can take the instance account and use those credentials to make the request.

internal/gotosocial/actions.go

@@ -68,7 +68,6 @@
 	&gtsmodel.StatusFave{},
 	&gtsmodel.StatusBookmark{},
 	&gtsmodel.StatusMute{},
-	&gtsmodel.StatusPin{},
 	&gtsmodel.Tag{},
 	&gtsmodel.User{},
 	&gtsmodel.Emoji{},

internal/gtsmodel/messages.go

@@ -0,0 +1,29 @@
+package gtsmodel
+
+// // ToClientAPI wraps a message that travels from the processor into the client API
+// type ToClientAPI struct {
+// 	APObjectType   ActivityStreamsObject
+// 	APActivityType ActivityStreamsActivity
+// 	Activity       interface{}
+// }
+
+// FromClientAPI wraps a message that travels from client API into the processor
+type FromClientAPI struct {
+	APObjectType   ActivityStreamsObject
+	APActivityType ActivityStreamsActivity
+	GTSModel       interface{}
+}
+
+// // ToFederator wraps a message that travels from the processor into the federator
+// type ToFederator struct {
+// 	APObjectType   ActivityStreamsObject
+// 	APActivityType ActivityStreamsActivity
+// 	GTSModel       interface{}
+// }
+
+// FromFederator wraps a message that travels from the federator into the processor
+type FromFederator struct {
+	APObjectType   ActivityStreamsObject
+	APActivityType ActivityStreamsActivity
+	GTSModel       interface{}
+}

internal/gtsmodel/status.go

@@ -34,7 +34,7 @@ type Status struct {
 	Attachments []string `pg:",array"`
 	// Database IDs of any tags used in this status
 	Tags []string `pg:",array"`
-	// Database IDs of any accounts mentioned in this status
+	// Database IDs of any mentions in this status
 	Mentions []string `pg:",array"`
 	// Database IDs of any emojis used in this status
 	Emojis []string `pg:",array"`
@@ -69,6 +69,8 @@ type Status struct {
 	ActivityStreamsType ActivityStreamsObject
 	// Original text of the status without formatting
 	Text string
+	// Has this status been pinned by its owner?
+	Pinned bool
 
 	/*
 		INTERNAL MODEL NON-DATABASE FIELDS

internal/media/handler.go

@@ -19,8 +19,10 @@
 package media
 
 import (
+	"context"
 	"errors"
 	"fmt"
+	"net/url"
 	"strings"
 	"time"
 
@@ -30,6 +32,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
 )
 
 // Size describes the *size* of a piece of media
@@ -68,13 +71,21 @@ type Handler interface {
 
 	// ProcessLocalAttachment takes a new attachment and the requesting account, checks it out, removes exif data from it,
 	// puts it in whatever storage backend we're using, sets the relevant fields in the database for the new media,
-	// and then returns information to the caller about the attachment.
+	// and then returns information to the caller about the attachment. It's the caller's responsibility to put the returned struct
-	ProcessLocalAttachment(attachment []byte, accountID string) (*gtsmodel.MediaAttachment, error)
+	// in the database.
+	ProcessAttachment(attachment []byte, accountID string, remoteURL string) (*gtsmodel.MediaAttachment, error)
 
 	// ProcessLocalEmoji takes a new emoji and a shortcode, cleans it up, puts it in storage, and creates a new
 	// *gts.Emoji for it, then returns it to the caller. It's the caller's responsibility to put the returned struct
 	// in the database.
 	ProcessLocalEmoji(emojiBytes []byte, shortcode string) (*gtsmodel.Emoji, error)
+
+	// ProcessRemoteAttachment takes a transport, a bare-bones current attachment, and an accountID that the attachment belongs to.
+	// It then dereferences the attachment (ie., fetches the attachment bytes from the remote server), ensuring that the bytes are
+	// the correct content type. It stores the attachment in whatever storage backend the Handler has been initalized with, and returns
+	// information to the caller about the new attachment. It's the caller's responsibility to put the returned struct
+	// in the database.
+	ProcessRemoteAttachment(t transport.Transport, currentAttachment *gtsmodel.MediaAttachment, accountID string) (*gtsmodel.MediaAttachment, error)
 }
 
 type mediaHandler struct {
@@ -136,27 +147,24 @@ func (mh *mediaHandler) ProcessHeaderOrAvatar(attachment []byte, accountID strin
 	return ma, nil
 }
 
-// ProcessLocalAttachment takes a new attachment and the requesting account, checks it out, removes exif data from it,
+// ProcessAttachment takes a new attachment and the owning account, checks it out, removes exif data from it,
 // puts it in whatever storage backend we're using, sets the relevant fields in the database for the new media,
 // and then returns information to the caller about the attachment.
-func (mh *mediaHandler) ProcessLocalAttachment(attachment []byte, accountID string) (*gtsmodel.MediaAttachment, error) {
+func (mh *mediaHandler) ProcessAttachment(attachment []byte, accountID string, remoteURL string) (*gtsmodel.MediaAttachment, error) {
 	contentType, err := parseContentType(attachment)
 	if err != nil {
 		return nil, err
 	}
 	mainType := strings.Split(contentType, "/")[0]
 	switch mainType {
-	case MIMEVideo:
+	// case MIMEVideo:
-		if !SupportedVideoType(contentType) {
+	// 	if !SupportedVideoType(contentType) {
-			return nil, fmt.Errorf("video type %s not supported", contentType)
+	// 		return nil, fmt.Errorf("video type %s not supported", contentType)
-		}
+	// 	}
-		if len(attachment) == 0 {
+	// 	if len(attachment) == 0 {
-			return nil, errors.New("video was of size 0")
+	// 		return nil, errors.New("video was of size 0")
-		}
+	// 	}
-		if len(attachment) > mh.config.MediaConfig.MaxVideoSize {
+	// 	return mh.processVideoAttachment(attachment, accountID, contentType, remoteURL)
-			return nil, fmt.Errorf("video size %d bytes exceeded max video size of %d bytes", len(attachment), mh.config.MediaConfig.MaxVideoSize)
-		}
-		return mh.processVideoAttachment(attachment, accountID, contentType)
 	case MIMEImage:
 		if !SupportedImageType(contentType) {
 			return nil, fmt.Errorf("image type %s not supported", contentType)
@@ -164,10 +172,7 @@ func (mh *mediaHandler) ProcessLocalAttachment(attachment []byte, accountID stri
 		if len(attachment) == 0 {
 			return nil, errors.New("image was of size 0")
 		}
-		if len(attachment) > mh.config.MediaConfig.MaxImageSize {
+		return mh.processImageAttachment(attachment, accountID, contentType, remoteURL)
-			return nil, fmt.Errorf("image size %d bytes exceeded max image size of %d bytes", len(attachment), mh.config.MediaConfig.MaxImageSize)
-		}
-		return mh.processImageAttachment(attachment, accountID, contentType)
 	default:
 		break
 	}
@@ -287,221 +292,26 @@ func (mh *mediaHandler) ProcessLocalEmoji(emojiBytes []byte, shortcode string) (
 	return e, nil
 }
 
-/*
+func (mh *mediaHandler) ProcessRemoteAttachment(t transport.Transport, currentAttachment *gtsmodel.MediaAttachment, accountID string) (*gtsmodel.MediaAttachment, error) {
-	HELPER FUNCTIONS
+	if currentAttachment.RemoteURL == "" {
-*/
+		return nil, errors.New("no remote URL on media attachment to dereference")
-
-func (mh *mediaHandler) processVideoAttachment(data []byte, accountID string, contentType string) (*gtsmodel.MediaAttachment, error) {
-	return nil, nil
-}
-
-func (mh *mediaHandler) processImageAttachment(data []byte, accountID string, contentType string) (*gtsmodel.MediaAttachment, error) {
-	var clean []byte
-	var err error
-	var original *imageAndMeta
-	var small *imageAndMeta
-
-	switch contentType {
-	case MIMEJpeg, MIMEPng:
-		if clean, err = purgeExif(data); err != nil {
-			return nil, fmt.Errorf("error cleaning exif data: %s", err)
-		}
-		original, err = deriveImage(clean, contentType)
-		if err != nil {
-			return nil, fmt.Errorf("error parsing image: %s", err)
-		}
-	case MIMEGif:
-		clean = data
-		original, err = deriveGif(clean, contentType)
-		if err != nil {
-			return nil, fmt.Errorf("error parsing gif: %s", err)
-		}
-	default:
-		return nil, errors.New("media type unrecognized")
 	}
-
+	remoteIRI, err := url.Parse(currentAttachment.RemoteURL)
-	small, err = deriveThumbnail(clean, contentType, 256, 256)
 	if err != nil {
-		return nil, fmt.Errorf("error deriving thumbnail: %s", err)
+		return nil, fmt.Errorf("error parsing attachment url %s: %s", currentAttachment.RemoteURL, err)
 	}
 
-	// now put it in storage, take a new uuid for the name of the file so we don't store any unnecessary info about it
+	// for content type, we assume we don't know what to expect...
-	extension := strings.Split(contentType, "/")[1]
+	expectedContentType := "*/*"
-	newMediaID := uuid.NewString()
+	if currentAttachment.File.ContentType != "" {
-
+		// ... and then narrow it down if we do
-	URLbase := fmt.Sprintf("%s://%s%s", mh.config.StorageConfig.ServeProtocol, mh.config.StorageConfig.ServeHost, mh.config.StorageConfig.ServeBasePath)
+		expectedContentType = currentAttachment.File.ContentType
-	originalURL := fmt.Sprintf("%s/%s/attachment/original/%s.%s", URLbase, accountID, newMediaID, extension)
-	smallURL := fmt.Sprintf("%s/%s/attachment/small/%s.jpeg", URLbase, accountID, newMediaID) // all thumbnails/smalls are encoded as jpeg
-
-	// we store the original...
-	originalPath := fmt.Sprintf("%s/%s/%s/%s/%s.%s", mh.config.StorageConfig.BasePath, accountID, Attachment, Original, newMediaID, extension)
-	if err := mh.storage.StoreFileAt(originalPath, original.image); err != nil {
-		return nil, fmt.Errorf("storage error: %s", err)
-	}
-
-	// and a thumbnail...
-	smallPath := fmt.Sprintf("%s/%s/%s/%s/%s.jpeg", mh.config.StorageConfig.BasePath, accountID, Attachment, Small, newMediaID) // all thumbnails/smalls are encoded as jpeg
-	if err := mh.storage.StoreFileAt(smallPath, small.image); err != nil {
-		return nil, fmt.Errorf("storage error: %s", err)
-	}
-
-	ma := &gtsmodel.MediaAttachment{
-		ID:        newMediaID,
-		StatusID:  "",
-		URL:       originalURL,
-		RemoteURL: "",
-		CreatedAt: time.Now(),
-		UpdatedAt: time.Now(),
-		Type:      gtsmodel.FileTypeImage,
-		FileMeta: gtsmodel.FileMeta{
-			Original: gtsmodel.Original{
-				Width:  original.width,
-				Height: original.height,
-				Size:   original.size,
-				Aspect: original.aspect,
-			},
-			Small: gtsmodel.Small{
-				Width:  small.width,
-				Height: small.height,
-				Size:   small.size,
-				Aspect: small.aspect,
-			},
-		},
-		AccountID:         accountID,
-		Description:       "",
-		ScheduledStatusID: "",
-		Blurhash:          original.blurhash,
-		Processing:        2,
-		File: gtsmodel.File{
-			Path:        originalPath,
-			ContentType: contentType,
-			FileSize:    len(original.image),
-			UpdatedAt:   time.Now(),
-		},
-		Thumbnail: gtsmodel.Thumbnail{
-			Path:        smallPath,
-			ContentType: MIMEJpeg, // all thumbnails/smalls are encoded as jpeg
-			FileSize:    len(small.image),
-			UpdatedAt:   time.Now(),
-			URL:         smallURL,
-			RemoteURL:   "",
-		},
-		Avatar: false,
-		Header: false,
-	}
-
-	return ma, nil
-
-}
-
-func (mh *mediaHandler) processHeaderOrAvi(imageBytes []byte, contentType string, mediaType Type, accountID string) (*gtsmodel.MediaAttachment, error) {
-	var isHeader bool
-	var isAvatar bool
-
-	switch mediaType {
-	case Header:
-		isHeader = true
-	case Avatar:
-		isAvatar = true
-	default:
-		return nil, errors.New("header or avatar not selected")
-	}
-
-	var clean []byte
-	var err error
-
-	var original *imageAndMeta
-	switch contentType {
-	case MIMEJpeg:
-		if clean, err = purgeExif(imageBytes); err != nil {
-			return nil, fmt.Errorf("error cleaning exif data: %s", err)
-		}
-		original, err = deriveImage(clean, contentType)
-	case MIMEPng:
-		if clean, err = purgeExif(imageBytes); err != nil {
-			return nil, fmt.Errorf("error cleaning exif data: %s", err)
-		}
-		original, err = deriveImage(clean, contentType)
-	case MIMEGif:
-		clean = imageBytes
-		original, err = deriveGif(clean, contentType)
-	default:
-		return nil, errors.New("media type unrecognized")
 	}
 
+	attachmentBytes, err := t.DereferenceMedia(context.Background(), remoteIRI, expectedContentType)
 	if err != nil {
-		return nil, fmt.Errorf("error parsing image: %s", err)
+		return nil, fmt.Errorf("dereferencing remote media with url %s: %s", remoteIRI.String(), err)
 	}
 
-	small, err := deriveThumbnail(clean, contentType, 256, 256)
+	return mh.ProcessAttachment(attachmentBytes, accountID, currentAttachment.RemoteURL)
-	if err != nil {
-		return nil, fmt.Errorf("error deriving thumbnail: %s", err)
-	}
-
-	// now put it in storage, take a new uuid for the name of the file so we don't store any unnecessary info about it
-	extension := strings.Split(contentType, "/")[1]
-	newMediaID := uuid.NewString()
-
-	URLbase := fmt.Sprintf("%s://%s%s", mh.config.StorageConfig.ServeProtocol, mh.config.StorageConfig.ServeHost, mh.config.StorageConfig.ServeBasePath)
-	originalURL := fmt.Sprintf("%s/%s/%s/original/%s.%s", URLbase, accountID, mediaType, newMediaID, extension)
-	smallURL := fmt.Sprintf("%s/%s/%s/small/%s.%s", URLbase, accountID, mediaType, newMediaID, extension)
-
-	// we store the original...
-	originalPath := fmt.Sprintf("%s/%s/%s/%s/%s.%s", mh.config.StorageConfig.BasePath, accountID, mediaType, Original, newMediaID, extension)
-	if err := mh.storage.StoreFileAt(originalPath, original.image); err != nil {
-		return nil, fmt.Errorf("storage error: %s", err)
-	}
-
-	// and a thumbnail...
-	smallPath := fmt.Sprintf("%s/%s/%s/%s/%s.%s", mh.config.StorageConfig.BasePath, accountID, mediaType, Small, newMediaID, extension)
-	if err := mh.storage.StoreFileAt(smallPath, small.image); err != nil {
-		return nil, fmt.Errorf("storage error: %s", err)
-	}
-
-	ma := &gtsmodel.MediaAttachment{
-		ID:        newMediaID,
-		StatusID:  "",
-		URL:       originalURL,
-		RemoteURL: "",
-		CreatedAt: time.Now(),
-		UpdatedAt: time.Now(),
-		Type:      gtsmodel.FileTypeImage,
-		FileMeta: gtsmodel.FileMeta{
-			Original: gtsmodel.Original{
-				Width:  original.width,
-				Height: original.height,
-				Size:   original.size,
-				Aspect: original.aspect,
-			},
-			Small: gtsmodel.Small{
-				Width:  small.width,
-				Height: small.height,
-				Size:   small.size,
-				Aspect: small.aspect,
-			},
-		},
-		AccountID:         accountID,
-		Description:       "",
-		ScheduledStatusID: "",
-		Blurhash:          original.blurhash,
-		Processing:        2,
-		File: gtsmodel.File{
-			Path:        originalPath,
-			ContentType: contentType,
-			FileSize:    len(original.image),
-			UpdatedAt:   time.Now(),
-		},
-		Thumbnail: gtsmodel.Thumbnail{
-			Path:        smallPath,
-			ContentType: contentType,
-			FileSize:    len(small.image),
-			UpdatedAt:   time.Now(),
-			URL:         smallURL,
-			RemoteURL:   "",
-		},
-		Avatar: isAvatar,
-		Header: isHeader,
-	}
-
-	return ma, nil
 }

internal/media/mock_MediaHandler.go

@@ -1,59 +0,0 @@
-// Code generated by mockery v2.7.4. DO NOT EDIT.
-
-package media
-
-import (
-	mock "github.com/stretchr/testify/mock"
-	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-)
-
-// MockMediaHandler is an autogenerated mock type for the MediaHandler type
-type MockMediaHandler struct {
-	mock.Mock
-}
-
-// ProcessAttachment provides a mock function with given fields: img, accountID
-func (_m *MockMediaHandler) ProcessAttachment(img []byte, accountID string) (*gtsmodel.MediaAttachment, error) {
-	ret := _m.Called(img, accountID)
-
-	var r0 *gtsmodel.MediaAttachment
-	if rf, ok := ret.Get(0).(func([]byte, string) *gtsmodel.MediaAttachment); ok {
-		r0 = rf(img, accountID)
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).(*gtsmodel.MediaAttachment)
-		}
-	}
-
-	var r1 error
-	if rf, ok := ret.Get(1).(func([]byte, string) error); ok {
-		r1 = rf(img, accountID)
-	} else {
-		r1 = ret.Error(1)
-	}
-
-	return r0, r1
-}
-
-// SetHeaderOrAvatarForAccountID provides a mock function with given fields: img, accountID, headerOrAvi
-func (_m *MockMediaHandler) SetHeaderOrAvatarForAccountID(img []byte, accountID string, headerOrAvi string) (*gtsmodel.MediaAttachment, error) {
-	ret := _m.Called(img, accountID, headerOrAvi)
-
-	var r0 *gtsmodel.MediaAttachment
-	if rf, ok := ret.Get(0).(func([]byte, string, string) *gtsmodel.MediaAttachment); ok {
-		r0 = rf(img, accountID, headerOrAvi)
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).(*gtsmodel.MediaAttachment)
-		}
-	}
-
-	var r1 error
-	if rf, ok := ret.Get(1).(func([]byte, string, string) error); ok {
-		r1 = rf(img, accountID, headerOrAvi)
-	} else {
-		r1 = ret.Error(1)
-	}
-
-	return r0, r1
-}

internal/media/processicon.go

@@ -0,0 +1,141 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package media
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (mh *mediaHandler) processHeaderOrAvi(imageBytes []byte, contentType string, mediaType Type, accountID string) (*gtsmodel.MediaAttachment, error) {
+	var isHeader bool
+	var isAvatar bool
+
+	switch mediaType {
+	case Header:
+		isHeader = true
+	case Avatar:
+		isAvatar = true
+	default:
+		return nil, errors.New("header or avatar not selected")
+	}
+
+	var clean []byte
+	var err error
+
+	var original *imageAndMeta
+	switch contentType {
+	case MIMEJpeg:
+		if clean, err = purgeExif(imageBytes); err != nil {
+			return nil, fmt.Errorf("error cleaning exif data: %s", err)
+		}
+		original, err = deriveImage(clean, contentType)
+	case MIMEPng:
+		if clean, err = purgeExif(imageBytes); err != nil {
+			return nil, fmt.Errorf("error cleaning exif data: %s", err)
+		}
+		original, err = deriveImage(clean, contentType)
+	case MIMEGif:
+		clean = imageBytes
+		original, err = deriveGif(clean, contentType)
+	default:
+		return nil, errors.New("media type unrecognized")
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("error parsing image: %s", err)
+	}
+
+	small, err := deriveThumbnail(clean, contentType, 256, 256)
+	if err != nil {
+		return nil, fmt.Errorf("error deriving thumbnail: %s", err)
+	}
+
+	// now put it in storage, take a new uuid for the name of the file so we don't store any unnecessary info about it
+	extension := strings.Split(contentType, "/")[1]
+	newMediaID := uuid.NewString()
+
+	URLbase := fmt.Sprintf("%s://%s%s", mh.config.StorageConfig.ServeProtocol, mh.config.StorageConfig.ServeHost, mh.config.StorageConfig.ServeBasePath)
+	originalURL := fmt.Sprintf("%s/%s/%s/original/%s.%s", URLbase, accountID, mediaType, newMediaID, extension)
+	smallURL := fmt.Sprintf("%s/%s/%s/small/%s.%s", URLbase, accountID, mediaType, newMediaID, extension)
+
+	// we store the original...
+	originalPath := fmt.Sprintf("%s/%s/%s/%s/%s.%s", mh.config.StorageConfig.BasePath, accountID, mediaType, Original, newMediaID, extension)
+	if err := mh.storage.StoreFileAt(originalPath, original.image); err != nil {
+		return nil, fmt.Errorf("storage error: %s", err)
+	}
+
+	// and a thumbnail...
+	smallPath := fmt.Sprintf("%s/%s/%s/%s/%s.%s", mh.config.StorageConfig.BasePath, accountID, mediaType, Small, newMediaID, extension)
+	if err := mh.storage.StoreFileAt(smallPath, small.image); err != nil {
+		return nil, fmt.Errorf("storage error: %s", err)
+	}
+
+	ma := &gtsmodel.MediaAttachment{
+		ID:        newMediaID,
+		StatusID:  "",
+		URL:       originalURL,
+		RemoteURL: "",
+		CreatedAt: time.Now(),
+		UpdatedAt: time.Now(),
+		Type:      gtsmodel.FileTypeImage,
+		FileMeta: gtsmodel.FileMeta{
+			Original: gtsmodel.Original{
+				Width:  original.width,
+				Height: original.height,
+				Size:   original.size,
+				Aspect: original.aspect,
+			},
+			Small: gtsmodel.Small{
+				Width:  small.width,
+				Height: small.height,
+				Size:   small.size,
+				Aspect: small.aspect,
+			},
+		},
+		AccountID:         accountID,
+		Description:       "",
+		ScheduledStatusID: "",
+		Blurhash:          original.blurhash,
+		Processing:        2,
+		File: gtsmodel.File{
+			Path:        originalPath,
+			ContentType: contentType,
+			FileSize:    len(original.image),
+			UpdatedAt:   time.Now(),
+		},
+		Thumbnail: gtsmodel.Thumbnail{
+			Path:        smallPath,
+			ContentType: contentType,
+			FileSize:    len(small.image),
+			UpdatedAt:   time.Now(),
+			URL:         smallURL,
+			RemoteURL:   "",
+		},
+		Avatar: isAvatar,
+		Header: isHeader,
+	}
+
+	return ma, nil
+}

internal/media/processimage.go

@@ -0,0 +1,128 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package media
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (mh *mediaHandler) processImageAttachment(data []byte, accountID string, contentType string, remoteURL string) (*gtsmodel.MediaAttachment, error) {
+	var clean []byte
+	var err error
+	var original *imageAndMeta
+	var small *imageAndMeta
+
+	switch contentType {
+	case MIMEJpeg, MIMEPng:
+		if clean, err = purgeExif(data); err != nil {
+			return nil, fmt.Errorf("error cleaning exif data: %s", err)
+		}
+		original, err = deriveImage(clean, contentType)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing image: %s", err)
+		}
+	case MIMEGif:
+		clean = data
+		original, err = deriveGif(clean, contentType)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing gif: %s", err)
+		}
+	default:
+		return nil, errors.New("media type unrecognized")
+	}
+
+	small, err = deriveThumbnail(clean, contentType, 256, 256)
+	if err != nil {
+		return nil, fmt.Errorf("error deriving thumbnail: %s", err)
+	}
+
+	// now put it in storage, take a new uuid for the name of the file so we don't store any unnecessary info about it
+	extension := strings.Split(contentType, "/")[1]
+	newMediaID := uuid.NewString()
+
+	URLbase := fmt.Sprintf("%s://%s%s", mh.config.StorageConfig.ServeProtocol, mh.config.StorageConfig.ServeHost, mh.config.StorageConfig.ServeBasePath)
+	originalURL := fmt.Sprintf("%s/%s/attachment/original/%s.%s", URLbase, accountID, newMediaID, extension)
+	smallURL := fmt.Sprintf("%s/%s/attachment/small/%s.jpeg", URLbase, accountID, newMediaID) // all thumbnails/smalls are encoded as jpeg
+
+	// we store the original...
+	originalPath := fmt.Sprintf("%s/%s/%s/%s/%s.%s", mh.config.StorageConfig.BasePath, accountID, Attachment, Original, newMediaID, extension)
+	if err := mh.storage.StoreFileAt(originalPath, original.image); err != nil {
+		return nil, fmt.Errorf("storage error: %s", err)
+	}
+
+	// and a thumbnail...
+	smallPath := fmt.Sprintf("%s/%s/%s/%s/%s.jpeg", mh.config.StorageConfig.BasePath, accountID, Attachment, Small, newMediaID) // all thumbnails/smalls are encoded as jpeg
+	if err := mh.storage.StoreFileAt(smallPath, small.image); err != nil {
+		return nil, fmt.Errorf("storage error: %s", err)
+	}
+
+	ma := &gtsmodel.MediaAttachment{
+		ID:        newMediaID,
+		StatusID:  "",
+		URL:       originalURL,
+		RemoteURL: remoteURL,
+		CreatedAt: time.Now(),
+		UpdatedAt: time.Now(),
+		Type:      gtsmodel.FileTypeImage,
+		FileMeta: gtsmodel.FileMeta{
+			Original: gtsmodel.Original{
+				Width:  original.width,
+				Height: original.height,
+				Size:   original.size,
+				Aspect: original.aspect,
+			},
+			Small: gtsmodel.Small{
+				Width:  small.width,
+				Height: small.height,
+				Size:   small.size,
+				Aspect: small.aspect,
+			},
+		},
+		AccountID:         accountID,
+		Description:       "",
+		ScheduledStatusID: "",
+		Blurhash:          original.blurhash,
+		Processing:        2,
+		File: gtsmodel.File{
+			Path:        originalPath,
+			ContentType: contentType,
+			FileSize:    len(original.image),
+			UpdatedAt:   time.Now(),
+		},
+		Thumbnail: gtsmodel.Thumbnail{
+			Path:        smallPath,
+			ContentType: MIMEJpeg, // all thumbnails/smalls are encoded as jpeg
+			FileSize:    len(small.image),
+			UpdatedAt:   time.Now(),
+			URL:         smallURL,
+			RemoteURL:   "",
+		},
+		Avatar: false,
+		Header: false,
+	}
+
+	return ma, nil
+
+}

internal/media/processvideo.go

@@ -0,0 +1,23 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package media
+
+// func (mh *mediaHandler) processVideoAttachment(data []byte, accountID string, contentType string, remoteURL string) (*gtsmodel.MediaAttachment, error) {
+// 	return nil, nil
+// }

internal/media/util.go

@@ -206,7 +206,9 @@ func deriveImage(b []byte, contentType string) (*imageAndMeta, error) {
 	}
 
 	out := &bytes.Buffer{}
-	if err := jpeg.Encode(out, i, nil); err != nil {
+	if err := jpeg.Encode(out, i, &jpeg.Options{
+		Quality: 100,
+	}); err != nil {
 		return nil, err
 	}
 
@@ -256,7 +258,9 @@ func deriveThumbnail(b []byte, contentType string, x uint, y uint) (*imageAndMet
 	aspect := float64(width) / float64(height)
 
 	out := &bytes.Buffer{}
-	if err := jpeg.Encode(out, thumb, nil); err != nil {
+	if err := jpeg.Encode(out, thumb, &jpeg.Options{
+		Quality: 100,
+	}); err != nil {
 		return nil, err
 	}
 	return &imageAndMeta{

internal/message/accountprocess.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package message
 
 import (
@@ -166,3 +184,112 @@ func (p *processor) AccountUpdate(authed *oauth.Auth, form *apimodel.UpdateCrede
 	}
 	return acctSensitive, nil
 }
+
+func (p *processor) AccountStatusesGet(authed *oauth.Auth, targetAccountID string, limit int, excludeReplies bool, maxID string, pinned bool, mediaOnly bool) ([]apimodel.Status, ErrorWithCode) {
+	targetAccount := &gtsmodel.Account{}
+	if err := p.db.GetByID(targetAccountID, targetAccount); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return nil, NewErrorNotFound(fmt.Errorf("no entry found for account id %s", targetAccountID))
+		}
+		return nil, NewErrorInternalError(err)
+	}
+
+	statuses := []gtsmodel.Status{}
+	apiStatuses := []apimodel.Status{}
+	if err := p.db.GetStatusesByTimeDescending(targetAccountID, &statuses, limit, excludeReplies, maxID, pinned, mediaOnly); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return apiStatuses, nil
+		}
+		return nil, NewErrorInternalError(err)
+	}
+
+	for _, s := range statuses {
+		relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(&s)
+		if err != nil {
+			return nil, NewErrorInternalError(fmt.Errorf("error getting relevant statuses: %s", err))
+		}
+
+		visible, err := p.db.StatusVisible(&s, targetAccount, authed.Account, relevantAccounts)
+		if err != nil {
+			return nil, NewErrorInternalError(fmt.Errorf("error checking status visibility: %s", err))
+		}
+		if !visible {
+			continue
+		}
+
+		var boostedStatus *gtsmodel.Status
+		if s.BoostOfID != "" {
+			bs := &gtsmodel.Status{}
+			if err := p.db.GetByID(s.BoostOfID, bs); err != nil {
+				return nil, NewErrorInternalError(fmt.Errorf("error getting boosted status: %s", err))
+			}
+			boostedRelevantAccounts, err := p.db.PullRelevantAccountsFromStatus(bs)
+			if err != nil {
+				return nil, NewErrorInternalError(fmt.Errorf("error getting relevant accounts from boosted status: %s", err))
+			}
+
+			boostedVisible, err := p.db.StatusVisible(bs, relevantAccounts.BoostedAccount, authed.Account, boostedRelevantAccounts)
+			if err != nil {
+				return nil, NewErrorInternalError(fmt.Errorf("error checking boosted status visibility: %s", err))
+			}
+
+			if boostedVisible {
+				boostedStatus = bs
+			}
+		}
+
+		apiStatus, err := p.tc.StatusToMasto(&s, targetAccount, authed.Account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, boostedStatus)
+		if err != nil {
+			return nil, NewErrorInternalError(fmt.Errorf("error converting status to masto: %s", err))
+		}
+
+		apiStatuses = append(apiStatuses, *apiStatus)
+	}
+
+	return apiStatuses, nil
+}
+
+func (p *processor) AccountFollowersGet(authed *oauth.Auth, targetAccountID string) ([]apimodel.Account, ErrorWithCode) {
+	blocked, err := p.db.Blocked(authed.Account.ID, targetAccountID)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	if blocked {
+		return nil, NewErrorNotFound(fmt.Errorf("block exists between accounts"))
+	}
+
+	followers := []gtsmodel.Follow{}
+	accounts := []apimodel.Account{}
+	if err := p.db.GetFollowersByAccountID(targetAccountID, &followers); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return accounts, nil
+		}
+		return nil, NewErrorInternalError(err)
+	}
+
+	for _, f := range followers {
+		blocked, err := p.db.Blocked(authed.Account.ID, f.AccountID)
+		if err != nil {
+			return nil, NewErrorInternalError(err)
+		}
+		if blocked {
+			continue
+		}
+
+		a := &gtsmodel.Account{}
+		if err := p.db.GetByID(f.AccountID, a); err != nil {
+			if _, ok := err.(db.ErrNoEntries); ok {
+				continue
+			}
+			return nil, NewErrorInternalError(err)
+		}
+
+		account, err := p.tc.AccountToMastoPublic(a)
+		if err != nil {
+			return nil, NewErrorInternalError(err)
+		}
+		accounts = append(accounts, *account)
+	}
+	return accounts, nil
+}

internal/message/adminprocess.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package message
 
 import (

internal/message/appprocess.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package message
 
 import (

internal/message/error.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package message
 
 import (

internal/message/fediprocess.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package message
 
 import (
@@ -60,10 +78,10 @@ func (p *processor) authenticateAndDereferenceFediRequest(username string, r *ht
 	}
 
 	// put it in our channel to queue it for async processing
-	p.FromFederator() <- FromFederator{
+	p.FromFederator() <- gtsmodel.FromFederator{
 		APObjectType:   gtsmodel.ActivityStreamsProfile,
 		APActivityType: gtsmodel.ActivityStreamsCreate,
-		Activity:       requestingAccount,
+		GTSModel:       requestingAccount,
 	}
 
 	return requestingAccount, nil

internal/message/fromclientapiprocess.go

@@ -0,0 +1,73 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package message
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (p *processor) processFromClientAPI(clientMsg gtsmodel.FromClientAPI) error {
+	switch clientMsg.APObjectType {
+	case gtsmodel.ActivityStreamsNote:
+		status, ok := clientMsg.GTSModel.(*gtsmodel.Status)
+		if !ok {
+			return errors.New("note was not parseable as *gtsmodel.Status")
+		}
+
+		if err := p.notifyStatus(status); err != nil {
+			return err
+		}
+
+		if status.VisibilityAdvanced.Federated {
+			return p.federateStatus(status)
+		}
+		return nil
+	}
+	return fmt.Errorf("message type unprocessable: %+v", clientMsg)
+}
+
+func (p *processor) federateStatus(status *gtsmodel.Status) error {
+	// // derive the sending account -- it might be attached to the status already
+	// sendingAcct := &gtsmodel.Account{}
+	// if status.GTSAccount != nil {
+	// 	sendingAcct = status.GTSAccount
+	// } else {
+	// 	// it wasn't attached so get it from the db instead
+	// 	if err := p.db.GetByID(status.AccountID, sendingAcct); err != nil {
+	// 		return err
+	// 	}
+	// }
+
+	// outboxURI, err := url.Parse(sendingAcct.OutboxURI)
+	// if err != nil {
+	// 	return err
+	// }
+
+	// // convert the status to AS format Note
+	// note, err := p.tc.StatusToAS(status)
+	// if err != nil {
+	// 	return err
+	// }
+
+	// _, err = p.federator.FederatingActor().Send(context.Background(), outboxURI, note)
+	return nil
+}

internal/message/fromcommonprocess.go

@@ -16,18 +16,10 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-package gtsmodel
+package message
 
-import "time"
+import "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 
-// StatusPin refers to a status 'pinned' to the top of an account
+func (p *processor) notifyStatus(status *gtsmodel.Status) error {
-type StatusPin struct {
+	return nil
-	// id of this pin in the database
-	ID string `pg:"type:uuid,default:gen_random_uuid(),pk,notnull,unique"`
-	// when was this pin created
-	CreatedAt time.Time `pg:"type:timestamp,notnull,default:now()"`
-	// id of the account that created ('did') the pinning (this should always be the same as the author of the status)
-	AccountID string `pg:",notnull"`
-	// database id of the status that has been pinned
-	StatusID string `pg:",notnull"`
 }

internal/message/fromfederatorprocess.go

@@ -0,0 +1,208 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package message
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+
+	"github.com/google/uuid"
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
+)
+
+func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) error {
+	l := p.log.WithFields(logrus.Fields{
+		"func":         "processFromFederator",
+		"federatorMsg": fmt.Sprintf("%+v", federatorMsg),
+	})
+
+	l.Debug("entering function PROCESS FROM FEDERATOR")
+
+	switch federatorMsg.APObjectType {
+	case gtsmodel.ActivityStreamsNote:
+
+		incomingStatus, ok := federatorMsg.GTSModel.(*gtsmodel.Status)
+		if !ok {
+			return errors.New("note was not parseable as *gtsmodel.Status")
+		}
+
+		l.Debug("will now derefence incoming status")
+		if err := p.dereferenceStatusFields(incomingStatus); err != nil {
+			return fmt.Errorf("error dereferencing status from federator: %s", err)
+		}
+		if err := p.db.UpdateByID(incomingStatus.ID, incomingStatus); err != nil {
+			return fmt.Errorf("error updating dereferenced status in the db: %s", err)
+		}
+
+		if err := p.notifyStatus(incomingStatus); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// dereferenceStatusFields fetches all the information we temporarily pinned to an incoming
+// federated status, back in the federating db's Create function.
+//
+// When a status comes in from the federation API, there are certain fields that
+// haven't been dereferenced yet, because we needed to provide a snappy synchronous
+// response to the caller. By the time it reaches this function though, it's being
+// processed asynchronously, so we have all the time in the world to fetch the various
+// bits and bobs that are attached to the status, and properly flesh it out, before we
+// send the status to any timelines and notify people.
+//
+// Things to dereference and fetch here:
+//
+// 1. Media attachments.
+// 2. Hashtags.
+// 3. Emojis.
+// 4. Mentions.
+// 5. Posting account.
+// 6. Replied-to-status.
+//
+// SIDE EFFECTS:
+// This function will deference all of the above, insert them in the database as necessary,
+// and attach them to the status. The status itself will not be added to the database yet,
+// that's up the caller to do.
+func (p *processor) dereferenceStatusFields(status *gtsmodel.Status) error {
+	l := p.log.WithFields(logrus.Fields{
+		"func":   "dereferenceStatusFields",
+		"status": fmt.Sprintf("%+v", status),
+	})
+	l.Debug("entering function")
+
+	var t transport.Transport
+	var err error
+	var username string
+	// TODO: dereference with a user that's addressed by the status
+	t, err = p.federator.GetTransportForUser(username)
+	if err != nil {
+		return fmt.Errorf("error creating transport: %s", err)
+	}
+
+	// the status should have an ID by now, but just in case it doesn't let's generate one here
+	// because we'll need it further down
+	if status.ID == "" {
+		status.ID = uuid.NewString()
+	}
+
+	// 1. Media attachments.
+	//
+	// At this point we should know:
+	// * the media type of the file we're looking for (a.File.ContentType)
+	// * the blurhash (a.Blurhash)
+	// * the file type (a.Type)
+	// * the remote URL (a.RemoteURL)
+	// This should be enough to pass along to the media processor.
+	attachmentIDs := []string{}
+	for _, a := range status.GTSMediaAttachments {
+		l.Debugf("dereferencing attachment: %+v", a)
+
+		// it might have been processed elsewhere so check first if it's already in the database or not
+		maybeAttachment := &gtsmodel.MediaAttachment{}
+		err := p.db.GetWhere("remote_url", a.RemoteURL, maybeAttachment)
+		if err == nil {
+			// we already have it in the db, dereferenced, no need to do it again
+			l.Debugf("attachment already exists with id %s", maybeAttachment.ID)
+			attachmentIDs = append(attachmentIDs, maybeAttachment.ID)
+			continue
+		}
+		if _, ok := err.(db.ErrNoEntries); !ok {
+			// we have a real error
+			return fmt.Errorf("error checking db for existence of attachment with remote url %s: %s", a.RemoteURL, err)
+		}
+		// it just doesn't exist yet so carry on
+		l.Debug("attachment doesn't exist yet, calling ProcessRemoteAttachment", a)
+		deferencedAttachment, err := p.mediaHandler.ProcessRemoteAttachment(t, a, status.AccountID)
+		if err != nil {
+			p.log.Errorf("error dereferencing status attachment: %s", err)
+			continue
+		}
+		l.Debugf("dereferenced attachment: %+v", deferencedAttachment)
+		deferencedAttachment.StatusID = status.ID
+		if err := p.db.Put(deferencedAttachment); err != nil {
+			return fmt.Errorf("error inserting dereferenced attachment with remote url %s: %s", a.RemoteURL, err)
+		}
+		deferencedAttachment.Description = a.Description
+		attachmentIDs = append(attachmentIDs, deferencedAttachment.ID)
+	}
+	status.Attachments = attachmentIDs
+
+	// 2. Hashtags
+
+	// 3. Emojis
+
+	// 4. Mentions
+	// At this point, mentions should have the namestring and mentionedAccountURI set on them.
+	//
+	// We should dereference any accounts mentioned here which we don't have in our db yet, by their URI.
+	mentions := []string{}
+	for _, m := range status.GTSMentions {
+		uri, err := url.Parse(m.MentionedAccountURI)
+		if err != nil {
+			l.Debugf("error parsing mentioned account uri %s: %s", m.MentionedAccountURI, err)
+			continue
+		}
+
+		m.StatusID = status.ID
+		m.OriginAccountID = status.GTSAccount.ID
+		m.OriginAccountURI = status.GTSAccount.URI
+
+		targetAccount := &gtsmodel.Account{}
+		if err := p.db.GetWhere("uri", uri.String(), targetAccount); err != nil {
+			// proper error
+			if _, ok := err.(db.ErrNoEntries); !ok {
+				return fmt.Errorf("db error checking for account with uri %s", uri.String())
+			}
+
+			// we just don't have it yet, so we should go get it....
+			accountable, err := p.federator.DereferenceRemoteAccount(username, uri)
+			if err != nil {
+				// we can't dereference it so just skip it
+				l.Debugf("error dereferencing remote account with uri %s: %s", uri.String(), err)
+				continue
+			}
+
+			targetAccount, err = p.tc.ASRepresentationToAccount(accountable)
+			if err != nil {
+				l.Debugf("error converting remote account with uri %s into gts model: %s", uri.String(), err)
+				continue
+			}
+
+			if err := p.db.Put(targetAccount); err != nil {
+				return fmt.Errorf("db error inserting account with uri %s", uri.String())
+			}
+		}
+
+		// by this point, we know the targetAccount exists in our database with an ID :)
+		m.TargetAccountID = targetAccount.ID
+		if err := p.db.Put(m); err != nil {
+			return fmt.Errorf("error creating mention: %s", err)
+		}
+		mentions = append(mentions, m.ID)
+	}
+	status.Mentions = mentions
+
+	return nil
+}

internal/message/frprocess.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package message
 
 import (

internal/message/instanceprocess.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package message
 
 import (

internal/message/mediaprocess.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package message
 
 import (
@@ -40,7 +58,7 @@ func (p *processor) MediaCreate(authed *oauth.Auth, form *apimodel.AttachmentReq
 	}
 
 	// allow the mediaHandler to work its magic of processing the attachment bytes, and putting them in whatever storage backend we're using
-	attachment, err := p.mediaHandler.ProcessLocalAttachment(buf.Bytes(), authed.Account.ID)
+	attachment, err := p.mediaHandler.ProcessAttachment(buf.Bytes(), authed.Account.ID, "")
 	if err != nil {
 		return nil, fmt.Errorf("error reading attachment: %s", err)
 	}

internal/message/processor.go

@@ -20,10 +20,7 @@
 
 import (
 	"context"
-	"errors"
-	"fmt"
 	"net/http"
-	"net/url"
 
 	"github.com/sirupsen/logrus"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
@@ -45,13 +42,13 @@
 // for clean distribution of messages without slowing down the client API and harming the user experience.
 type Processor interface {
 	// ToClientAPI returns a channel for putting in messages that need to go to the gts client API.
-	ToClientAPI() chan ToClientAPI
+	// ToClientAPI() chan gtsmodel.ToClientAPI
 	// FromClientAPI returns a channel for putting messages in that come from the client api going to the processor
-	FromClientAPI() chan FromClientAPI
+	FromClientAPI() chan gtsmodel.FromClientAPI
 	// ToFederator returns a channel for putting in messages that need to go to the federator (activitypub).
-	ToFederator() chan ToFederator
+	// ToFederator() chan gtsmodel.ToFederator
 	// FromFederator returns a channel for putting messages in that come from the federator (activitypub) going into the processor
-	FromFederator() chan FromFederator
+	FromFederator() chan gtsmodel.FromFederator
 	// Start starts the Processor, reading from its channels and passing messages back and forth.
 	Start() error
 	// Stop stops the processor cleanly, finishing handling any remaining messages before closing down.
@@ -71,6 +68,11 @@ type Processor interface {
 	AccountGet(authed *oauth.Auth, targetAccountID string) (*apimodel.Account, error)
 	// AccountUpdate processes the update of an account with the given form
 	AccountUpdate(authed *oauth.Auth, form *apimodel.UpdateCredentialsRequest) (*apimodel.Account, error)
+	// AccountStatusesGet fetches a number of statuses (in time descending order) from the given account, filtered by visibility for
+	// the account given in authed.
+	AccountStatusesGet(authed *oauth.Auth, targetAccountID string, limit int, excludeReplies bool, maxID string, pinned bool, mediaOnly bool) ([]apimodel.Status, ErrorWithCode)
+	// AccountFollowersGet
+	AccountFollowersGet(authed *oauth.Auth, targetAccountID string) ([]apimodel.Account, ErrorWithCode)
 
 	// AdminEmojiCreate handles the creation of a new instance emoji by an admin, using the given form.
 	AdminEmojiCreate(authed *oauth.Auth, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, error)
@@ -142,10 +144,10 @@ type Processor interface {
 // processor just implements the Processor interface
 type processor struct {
 	// federator     pub.FederatingActor
-	toClientAPI   chan ToClientAPI
+	// toClientAPI   chan gtsmodel.ToClientAPI
-	fromClientAPI chan FromClientAPI
+	fromClientAPI chan gtsmodel.FromClientAPI
-	toFederator   chan ToFederator
+	// toFederator   chan gtsmodel.ToFederator
-	fromFederator chan FromFederator
+	fromFederator chan gtsmodel.FromFederator
 	federator     federation.Federator
 	stop          chan interface{}
 	log           *logrus.Logger
@@ -160,10 +162,10 @@ type processor struct {
 // NewProcessor returns a new Processor that uses the given federator and logger
 func NewProcessor(config *config.Config, tc typeutils.TypeConverter, federator federation.Federator, oauthServer oauth.Server, mediaHandler media.Handler, storage storage.Storage, db db.DB, log *logrus.Logger) Processor {
 	return &processor{
-		toClientAPI:   make(chan ToClientAPI, 100),
+		// toClientAPI:   make(chan gtsmodel.ToClientAPI, 100),
-		fromClientAPI: make(chan FromClientAPI, 100),
+		fromClientAPI: make(chan gtsmodel.FromClientAPI, 100),
-		toFederator:   make(chan ToFederator, 100),
+		// toFederator:   make(chan gtsmodel.ToFederator, 100),
-		fromFederator: make(chan FromFederator, 100),
+		fromFederator: make(chan gtsmodel.FromFederator, 100),
 		federator:     federator,
 		stop:          make(chan interface{}),
 		log:           log,
@@ -176,19 +178,19 @@ func NewProcessor(config *config.Config, tc typeutils.TypeConverter, federator f
 	}
 }
 
-func (p *processor) ToClientAPI() chan ToClientAPI {
+// func (p *processor) ToClientAPI() chan gtsmodel.ToClientAPI {
-	return p.toClientAPI
+// 	return p.toClientAPI
-}
+// }
 
-func (p *processor) FromClientAPI() chan FromClientAPI {
+func (p *processor) FromClientAPI() chan gtsmodel.FromClientAPI {
 	return p.fromClientAPI
 }
 
-func (p *processor) ToFederator() chan ToFederator {
+// func (p *processor) ToFederator() chan gtsmodel.ToFederator {
-	return p.toFederator
+// 	return p.toFederator
-}
+// }
 
-func (p *processor) FromFederator() chan FromFederator {
+func (p *processor) FromFederator() chan gtsmodel.FromFederator {
 	return p.fromFederator
 }
 
@@ -198,17 +200,20 @@ func (p *processor) Start() error {
 	DistLoop:
 		for {
 			select {
-			case clientMsg := <-p.toClientAPI:
+			// case clientMsg := <-p.toClientAPI:
-				p.log.Infof("received message TO client API: %+v", clientMsg)
+			// 	p.log.Infof("received message TO client API: %+v", clientMsg)
 			case clientMsg := <-p.fromClientAPI:
 				p.log.Infof("received message FROM client API: %+v", clientMsg)
 				if err := p.processFromClientAPI(clientMsg); err != nil {
 					p.log.Error(err)
 				}
-			case federatorMsg := <-p.toFederator:
+			// case federatorMsg := <-p.toFederator:
-				p.log.Infof("received message TO federator: %+v", federatorMsg)
+			// 	p.log.Infof("received message TO federator: %+v", federatorMsg)
 			case federatorMsg := <-p.fromFederator:
 				p.log.Infof("received message FROM federator: %+v", federatorMsg)
+				if err := p.processFromFederator(federatorMsg); err != nil {
+					p.log.Error(err)
+				}
 			case <-p.stop:
 				break DistLoop
 			}
@@ -223,82 +228,3 @@ func (p *processor) Stop() error {
 	close(p.stop)
 	return nil
 }
-
-// ToClientAPI wraps a message that travels from the processor into the client API
-type ToClientAPI struct {
-	APObjectType   gtsmodel.ActivityStreamsObject
-	APActivityType gtsmodel.ActivityStreamsActivity
-	Activity       interface{}
-}
-
-// FromClientAPI wraps a message that travels from client API into the processor
-type FromClientAPI struct {
-	APObjectType   gtsmodel.ActivityStreamsObject
-	APActivityType gtsmodel.ActivityStreamsActivity
-	Activity       interface{}
-}
-
-// ToFederator wraps a message that travels from the processor into the federator
-type ToFederator struct {
-	APObjectType   gtsmodel.ActivityStreamsObject
-	APActivityType gtsmodel.ActivityStreamsActivity
-	Activity       interface{}
-}
-
-// FromFederator wraps a message that travels from the federator into the processor
-type FromFederator struct {
-	APObjectType   gtsmodel.ActivityStreamsObject
-	APActivityType gtsmodel.ActivityStreamsActivity
-	Activity       interface{}
-}
-
-func (p *processor) processFromClientAPI(clientMsg FromClientAPI) error {
-	switch clientMsg.APObjectType {
-	case gtsmodel.ActivityStreamsNote:
-		status, ok := clientMsg.Activity.(*gtsmodel.Status)
-		if !ok {
-			return errors.New("note was not parseable as *gtsmodel.Status")
-		}
-
-		if err := p.notifyStatus(status); err != nil {
-			return err
-		}
-
-		if status.VisibilityAdvanced.Federated {
-			return p.federateStatus(status)
-		}
-		return nil
-	}
-	return fmt.Errorf("message type unprocessable: %+v", clientMsg)
-}
-
-func (p *processor) federateStatus(status *gtsmodel.Status) error {
-	// derive the sending account -- it might be attached to the status already
-	sendingAcct := &gtsmodel.Account{}
-	if status.GTSAccount != nil {
-		sendingAcct = status.GTSAccount
-	} else {
-		// it wasn't attached so get it from the db instead
-		if err := p.db.GetByID(status.AccountID, sendingAcct); err != nil {
-			return err
-		}
-	}
-
-	outboxURI, err := url.Parse(sendingAcct.OutboxURI)
-	if err != nil {
-		return err
-	}
-
-	// convert the status to AS format Note
-	note, err := p.tc.StatusToAS(status)
-	if err != nil {
-		return err
-	}
-
-	_, err = p.federator.FederatingActor().Send(context.Background(), outboxURI, note)
-	return err
-}
-
-func (p *processor) notifyStatus(status *gtsmodel.Status) error {
-	return nil
-}

internal/message/processorutil.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package message
 
 import (

internal/message/statusprocess.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package message
 
 import (
@@ -82,10 +100,10 @@ func (p *processor) StatusCreate(auth *oauth.Auth, form *apimodel.AdvancedStatus
 	}
 
 	// put the new status in the appropriate channel for async processing
-	p.fromClientAPI <- FromClientAPI{
+	p.fromClientAPI <- gtsmodel.FromClientAPI{
 		APObjectType:   newStatus.ActivityStreamsType,
 		APActivityType: gtsmodel.ActivityStreamsCreate,
-		Activity:       newStatus,
+		GTSModel:       newStatus,
 	}
 
 	// return the frontend representation of the new status to the submitter
@@ -161,8 +179,10 @@ func (p *processor) StatusFave(authed *oauth.Auth, targetStatusID string) (*apim
 	}
 
 	// is the status faveable?
-	if !targetStatus.VisibilityAdvanced.Likeable {
+	if targetStatus.VisibilityAdvanced != nil {
-		return nil, errors.New("status is not faveable")
+		if !targetStatus.VisibilityAdvanced.Likeable {
+			return nil, errors.New("status is not faveable")
+		}
 	}
 
 	// it's visible! it's faveable! so let's fave the FUCK out of it
@@ -218,8 +238,10 @@ func (p *processor) StatusBoost(authed *oauth.Auth, targetStatusID string) (*api
 		return nil, NewErrorNotFound(errors.New("status is not visible"))
 	}
 
-	if !targetStatus.VisibilityAdvanced.Boostable {
+	if targetStatus.VisibilityAdvanced != nil {
-		return nil, NewErrorForbidden(errors.New("status is not boostable"))
+		if !targetStatus.VisibilityAdvanced.Boostable {
+			return nil, NewErrorForbidden(errors.New("status is not boostable"))
+		}
 	}
 
 	// it's visible! it's boostable! so let's boost the FUCK out of it
@@ -428,8 +450,10 @@ func (p *processor) StatusUnfave(authed *oauth.Auth, targetStatusID string) (*ap
 	}
 
 	// is the status faveable?
-	if !targetStatus.VisibilityAdvanced.Likeable {
+	if targetStatus.VisibilityAdvanced != nil {
-		return nil, errors.New("status is not faveable")
+		if !targetStatus.VisibilityAdvanced.Likeable {
+			return nil, errors.New("status is not faveable")
+		}
 	}
 
 	// it's visible! it's faveable! so let's unfave the FUCK out of it

internal/router/router.go

@@ -153,7 +153,7 @@ func New(config *config.Config, logger *logrus.Logger) (Router, error) {
 	s := &http.Server{
 		Handler:           engine,
 		ReadTimeout:       60 * time.Second,
-		WriteTimeout:      5 * time.Second,
+		WriteTimeout:      30 * time.Second,
 		IdleTimeout:       30 * time.Second,
 		ReadHeaderTimeout: 30 * time.Second,
 	}

internal/transport/controller.go

@@ -21,6 +21,7 @@
 import (
 	"crypto"
 	"fmt"
+	"sync"
 
 	"github.com/go-fed/activity/pub"
 	"github.com/go-fed/httpsig"
@@ -30,7 +31,7 @@
 
 // Controller generates transports for use in making federation requests to other servers.
 type Controller interface {
-	NewTransport(pubKeyID string, privkey crypto.PrivateKey) (pub.Transport, error)
+	NewTransport(pubKeyID string, privkey crypto.PrivateKey) (Transport, error)
 }
 
 type controller struct {
@@ -51,7 +52,7 @@ func NewController(config *config.Config, clock pub.Clock, client pub.HttpClient
 }
 
 // NewTransport returns a new http signature transport with the given public key id (a URL), and the given private key.
-func (c *controller) NewTransport(pubKeyID string, privkey crypto.PrivateKey) (pub.Transport, error) {
+func (c *controller) NewTransport(pubKeyID string, privkey crypto.PrivateKey) (Transport, error) {
 	prefs := []httpsig.Algorithm{httpsig.RSA_SHA256, httpsig.RSA_SHA512}
 	digestAlgo := httpsig.DigestSha256
 	getHeaders := []string{"(request-target)", "host", "date"}
@@ -67,5 +68,17 @@ func (c *controller) NewTransport(pubKeyID string, privkey crypto.PrivateKey) (p
 		return nil, fmt.Errorf("error creating post signer: %s", err)
 	}
 
-	return pub.NewHttpSigTransport(c.client, c.appAgent, c.clock, getSigner, postSigner, pubKeyID, privkey), nil
+	sigTransport := pub.NewHttpSigTransport(c.client, c.appAgent, c.clock, getSigner, postSigner, pubKeyID, privkey)
+
+	return &transport{
+		client:       c.client,
+		appAgent:     c.appAgent,
+		gofedAgent:   "(go-fed/activity v1.0.0)",
+		clock:        c.clock,
+		pubKeyID:     pubKeyID,
+		privkey:      privkey,
+		sigTransport: sigTransport,
+		getSigner:    getSigner,
+		getSignerMu:  &sync.Mutex{},
+	}, nil
 }

internal/transport/transport.go

@@ -0,0 +1,77 @@
+package transport
+
+import (
+	"context"
+	"crypto"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"sync"
+
+	"github.com/go-fed/activity/pub"
+	"github.com/go-fed/httpsig"
+)
+
+// Transport wraps the pub.Transport interface with some additional
+// functionality for fetching remote media.
+type Transport interface {
+	pub.Transport
+	DereferenceMedia(c context.Context, iri *url.URL, expectedContentType string) ([]byte, error)
+}
+
+// transport implements the Transport interface
+type transport struct {
+	client       pub.HttpClient
+	appAgent     string
+	gofedAgent   string
+	clock        pub.Clock
+	pubKeyID     string
+	privkey      crypto.PrivateKey
+	sigTransport *pub.HttpSigTransport
+	getSigner    httpsig.Signer
+	getSignerMu  *sync.Mutex
+}
+
+func (t *transport) BatchDeliver(c context.Context, b []byte, recipients []*url.URL) error {
+	return t.sigTransport.BatchDeliver(c, b, recipients)
+}
+
+func (t *transport) Deliver(c context.Context, b []byte, to *url.URL) error {
+	return t.sigTransport.Deliver(c, b, to)
+}
+
+func (t *transport) Dereference(c context.Context, iri *url.URL) ([]byte, error) {
+	return t.sigTransport.Dereference(c, iri)
+}
+
+func (t *transport) DereferenceMedia(c context.Context, iri *url.URL, expectedContentType string) ([]byte, error) {
+	req, err := http.NewRequest("GET", iri.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	req = req.WithContext(c)
+	if expectedContentType == "" {
+		req.Header.Add("Accept", "*/*")
+	} else {
+		req.Header.Add("Accept", expectedContentType)
+	}
+	req.Header.Add("Date", t.clock.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
+	req.Header.Add("User-Agent", fmt.Sprintf("%s %s", t.appAgent, t.gofedAgent))
+	req.Header.Set("Host", iri.Host)
+	t.getSignerMu.Lock()
+	err = t.getSigner.SignRequest(t.privkey, t.pubKeyID, req, nil)
+	t.getSignerMu.Unlock()
+	if err != nil {
+		return nil, err
+	}
+	resp, err := t.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status)
+	}
+	return ioutil.ReadAll(resp.Body)
+}

internal/typeutils/asextractionutil.go

@@ -29,6 +29,7 @@
 	"time"
 
 	"github.com/go-fed/activity/pub"
+	"github.com/go-fed/activity/streams"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
@@ -304,12 +305,24 @@ func extractAttachments(i withAttachment) ([]*gtsmodel.MediaAttachment, error) {
 
 	attachmentProp := i.GetActivityStreamsAttachment()
 	for iter := attachmentProp.Begin(); iter != attachmentProp.End(); iter = iter.Next() {
-		attachmentable, ok := iter.(Attachmentable)
+
+		t := iter.GetType()
+		if t == nil {
+			fmt.Printf("\n\n\nGetType() nil\n\n\n")
+			continue
+		}
+
+		m, _ := streams.Serialize(t)
+		fmt.Printf("\n\n\n%s\n\n\n", m)
+
+		attachmentable, ok := t.(Attachmentable)
 		if !ok {
+			fmt.Printf("\n\n\nnot attachmentable\n\n\n")
 			continue
 		}
 		attachment, err := extractAttachment(attachmentable)
 		if err != nil {
+			fmt.Printf("\n\n\n%s\n\n\n", err)
 			continue
 		}
 		attachments = append(attachments, attachment)
@@ -343,23 +356,20 @@ func extractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) {
 		attachment.Description = name
 	}
 
-	blurhash, err := extractBlurhash(i)
+	attachment.Processing = gtsmodel.ProcessingStatusReceived
-	if err == nil {
-		attachment.Blurhash = blurhash
-	}
 
 	return attachment, nil
 }
 
-func extractBlurhash(i withBlurhash) (string, error) {
+// func extractBlurhash(i withBlurhash) (string, error) {
-	if i.GetTootBlurhashProperty() == nil {
+// 	if i.GetTootBlurhashProperty() == nil {
-		return "", errors.New("blurhash property was nil")
+// 		return "", errors.New("blurhash property was nil")
-	}
+// 	}
-	if i.GetTootBlurhashProperty().Get() == "" {
+// 	if i.GetTootBlurhashProperty().Get() == "" {
-		return "", errors.New("empty blurhash string")
+// 		return "", errors.New("empty blurhash string")
-	}
+// 	}
-	return i.GetTootBlurhashProperty().Get(), nil
+// 	return i.GetTootBlurhashProperty().Get(), nil
-}
+// }
 
 func extractHashtags(i withTag) ([]*gtsmodel.Tag, error) {
 	tags := []*gtsmodel.Tag{}

internal/typeutils/asinterfaces.go

@@ -69,8 +69,6 @@ type Attachmentable interface {
 	withMediaType
 	withURL
 	withName
-	withBlurhash
-	withFocalPoint
 }
 
 // Hashtaggable represents the minimum activitypub interface for representing a 'hashtag' tag.
@@ -212,13 +210,13 @@ type withMediaType interface {
 	GetActivityStreamsMediaType() vocab.ActivityStreamsMediaTypeProperty
 }
 
-type withBlurhash interface {
+// type withBlurhash interface {
-	GetTootBlurhashProperty() vocab.TootBlurhashProperty
+// 	GetTootBlurhashProperty() vocab.TootBlurhashProperty
-}
+// }
 
-type withFocalPoint interface {
+// type withFocalPoint interface {
-	// TODO
+// 	// TODO
-}
+// }
 
 type withHref interface {
 	GetActivityStreamsHref() vocab.ActivityStreamsHrefProperty

internal/typeutils/astointernal.go

@@ -281,7 +281,7 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e
 
 	// if it's CC'ed to public, it's public or unlocked
 	// mentioned SPECIFIC ACCOUNTS also get added to CC'es if it's not a direct message
-	if isPublic(to) {
+	if isPublic(cc) || isPublic(to) {
 		visibility = gtsmodel.VisibilityPublic
 	}
 

internal/typeutils/internaltofrontend.go

@@ -20,6 +20,7 @@
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/superseriousbusiness/gotosocial/internal/api/model"
@@ -86,16 +87,12 @@ func (c *converter) AccountToMastoPublic(a *gtsmodel.Account) (*model.Account, e
 	}
 
 	// count statuses
-	statuses := []gtsmodel.Status{}
+	statusesCount, err := c.db.CountStatusesByAccountID(a.ID)
-	if err := c.db.GetStatusesByAccountID(a.ID, &statuses); err != nil {
+	if err != nil {
 		if _, ok := err.(db.ErrNoEntries); !ok {
 			return nil, fmt.Errorf("error getting last statuses: %s", err)
 		}
 	}
-	var statusesCount int
-	if statuses != nil {
-		statusesCount = len(statuses)
-	}
 
 	// check when the last status was
 	lastStatus := &gtsmodel.Status{}
@@ -195,7 +192,7 @@ func (c *converter) AppToMastoPublic(a *gtsmodel.Application) (*model.Applicatio
 func (c *converter) AttachmentToMasto(a *gtsmodel.MediaAttachment) (model.Attachment, error) {
 	return model.Attachment{
 		ID:               a.ID,
-		Type:             string(a.Type),
+		Type:             strings.ToLower(string(a.Type)),
 		URL:              a.URL,
 		PreviewURL:       a.Thumbnail.URL,
 		RemoteURL:        a.RemoteURL,
@@ -294,7 +291,6 @@ func (c *converter) StatusToMasto(
 	var faved bool
 	var reblogged bool
 	var bookmarked bool
-	var pinned bool
 	var muted bool
 
 	// requestingAccount will be nil for public requests without auth
@@ -319,11 +315,6 @@ func (c *converter) StatusToMasto(
 		if err != nil {
 			return nil, fmt.Errorf("error checking if requesting account has bookmarked status: %s", err)
 		}
-
-		pinned, err = c.db.StatusPinnedBy(s, requestingAccount.ID)
-		if err != nil {
-			return nil, fmt.Errorf("error checking if requesting account has pinned status: %s", err)
-		}
 	}
 
 	var mastoRebloggedStatus *model.Status
@@ -522,7 +513,7 @@ func (c *converter) StatusToMasto(
 		Reblogged:          reblogged,
 		Muted:              muted,
 		Bookmarked:         bookmarked,
-		Pinned:             pinned,
+		Pinned:             s.Pinned,
 		Content:            s.Content,
 		Reblog:             mastoRebloggedStatus,
 		Application:        mastoApplication,

testrig/db.go

@@ -42,7 +42,6 @@
 	&gtsmodel.StatusFave{},
 	&gtsmodel.StatusBookmark{},
 	&gtsmodel.StatusMute{},
-	&gtsmodel.StatusPin{},
 	&gtsmodel.Tag{},
 	&gtsmodel.User{},
 	&gtsmodel.Emoji{},