Text duplication fix (#137)

* start testing text duplication

* tests

* fixes + tests

internal/processing/status/create.go

@@ -39,39 +39,39 @@ func (p *processor) Create(account *gtsmodel.Account, application *gtsmodel.Appl
 	}
 
 	// check if replyToID is ok
-	if err := p.processReplyToID(form, account.ID, newStatus); err != nil {
+	if err := p.ProcessReplyToID(form, account.ID, newStatus); err != nil {
 		return nil, gtserror.NewErrorInternalError(err)
 	}
 
 	// check if mediaIDs are ok
-	if err := p.processMediaIDs(form, account.ID, newStatus); err != nil {
+	if err := p.ProcessMediaIDs(form, account.ID, newStatus); err != nil {
 		return nil, gtserror.NewErrorInternalError(err)
 	}
 
 	// check if visibility settings are ok
-	if err := p.processVisibility(form, account.Privacy, newStatus); err != nil {
+	if err := p.ProcessVisibility(form, account.Privacy, newStatus); err != nil {
 		return nil, gtserror.NewErrorInternalError(err)
 	}
 
 	// handle language settings
-	if err := p.processLanguage(form, account.Language, newStatus); err != nil {
+	if err := p.ProcessLanguage(form, account.Language, newStatus); err != nil {
 		return nil, gtserror.NewErrorInternalError(err)
 	}
 
 	// handle mentions
-	if err := p.processMentions(form, account.ID, newStatus); err != nil {
+	if err := p.ProcessMentions(form, account.ID, newStatus); err != nil {
 		return nil, gtserror.NewErrorInternalError(err)
 	}
 
-	if err := p.processTags(form, account.ID, newStatus); err != nil {
+	if err := p.ProcessTags(form, account.ID, newStatus); err != nil {
 		return nil, gtserror.NewErrorInternalError(err)
 	}
 
-	if err := p.processEmojis(form, account.ID, newStatus); err != nil {
+	if err := p.ProcessEmojis(form, account.ID, newStatus); err != nil {
 		return nil, gtserror.NewErrorInternalError(err)
 	}
 
-	if err := p.processContent(form, account.ID, newStatus); err != nil {
+	if err := p.ProcessContent(form, account.ID, newStatus); err != nil {
 		return nil, gtserror.NewErrorInternalError(err)
 	}
 

internal/processing/status/status.go

@@ -34,6 +34,19 @@ type Processor interface {
 	Unfave(account *gtsmodel.Account, targetStatusID string) (*apimodel.Status, gtserror.WithCode)
 	// Context returns the context (previous and following posts) from the given status ID
 	Context(account *gtsmodel.Account, targetStatusID string) (*apimodel.Context, gtserror.WithCode)
+
+	/*
+		PROCESSING UTILS
+	*/
+
+	ProcessVisibility(form *apimodel.AdvancedStatusCreateForm, accountDefaultVis gtsmodel.Visibility, status *gtsmodel.Status) error
+	ProcessReplyToID(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error
+	ProcessMediaIDs(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error
+	ProcessLanguage(form *apimodel.AdvancedStatusCreateForm, accountDefaultLanguage string, status *gtsmodel.Status) error
+	ProcessMentions(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error
+	ProcessTags(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error
+	ProcessEmojis(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error
+	ProcessContent(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error
 }
 
 type processor struct {

internal/processing/status/status_test.go

@@ -0,0 +1,54 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package status_test
+
+import (
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+	"github.com/superseriousbusiness/gotosocial/internal/processing/status"
+	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+)
+
+// nolint
+type StatusStandardTestSuite struct {
+	suite.Suite
+	config            *config.Config
+	db                db.DB
+	log               *logrus.Logger
+	typeConverter     typeutils.TypeConverter
+	fromClientAPIChan chan gtsmodel.FromClientAPI
+
+	// standard suite models
+	testTokens       map[string]*oauth.Token
+	testClients      map[string]*oauth.Client
+	testApplications map[string]*gtsmodel.Application
+	testUsers        map[string]*gtsmodel.User
+	testAccounts     map[string]*gtsmodel.Account
+	testAttachments  map[string]*gtsmodel.MediaAttachment
+	testStatuses     map[string]*gtsmodel.Status
+	testTags         map[string]*gtsmodel.Tag
+	testMentions     map[string]*gtsmodel.Mention
+
+	// module being tested
+	status status.Processor
+}

internal/processing/status/util.go

@@ -12,7 +12,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
-func (p *processor) processVisibility(form *apimodel.AdvancedStatusCreateForm, accountDefaultVis gtsmodel.Visibility, status *gtsmodel.Status) error {
+func (p *processor) ProcessVisibility(form *apimodel.AdvancedStatusCreateForm, accountDefaultVis gtsmodel.Visibility, status *gtsmodel.Status) error {
 	// by default all flags are set to true
 	gtsAdvancedVis := &gtsmodel.VisibilityAdvanced{
 		Federated: true,
@@ -83,7 +83,7 @@ func (p *processor) processVisibility(form *apimodel.AdvancedStatusCreateForm, a
 	return nil
 }
 
-func (p *processor) processReplyToID(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error {
+func (p *processor) ProcessReplyToID(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error {
 	if form.InReplyToID == "" {
 		return nil
 	}
@@ -132,7 +132,7 @@ func (p *processor) processReplyToID(form *apimodel.AdvancedStatusCreateForm, th
 	return nil
 }
 
-func (p *processor) processMediaIDs(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error {
+func (p *processor) ProcessMediaIDs(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error {
 	if form.MediaIDs == nil {
 		return nil
 	}
@@ -161,7 +161,7 @@ func (p *processor) processMediaIDs(form *apimodel.AdvancedStatusCreateForm, thi
 	return nil
 }
 
-func (p *processor) processLanguage(form *apimodel.AdvancedStatusCreateForm, accountDefaultLanguage string, status *gtsmodel.Status) error {
+func (p *processor) ProcessLanguage(form *apimodel.AdvancedStatusCreateForm, accountDefaultLanguage string, status *gtsmodel.Status) error {
 	if form.Language != "" {
 		status.Language = form.Language
 	} else {
@@ -173,7 +173,7 @@ func (p *processor) processLanguage(form *apimodel.AdvancedStatusCreateForm, acc
 	return nil
 }
 
-func (p *processor) processMentions(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
+func (p *processor) ProcessMentions(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
 	menchies := []string{}
 	gtsMenchies, err := p.db.MentionStringsToMentions(util.DeriveMentionsFromStatus(form.Status), accountID, status.ID)
 	if err != nil {
@@ -198,7 +198,7 @@ func (p *processor) processMentions(form *apimodel.AdvancedStatusCreateForm, acc
 	return nil
 }
 
-func (p *processor) processTags(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
+func (p *processor) ProcessTags(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
 	tags := []string{}
 	gtsTags, err := p.db.TagStringsToTags(util.DeriveHashtagsFromStatus(form.Status), accountID, status.ID)
 	if err != nil {
@@ -217,7 +217,7 @@ func (p *processor) processTags(form *apimodel.AdvancedStatusCreateForm, account
 	return nil
 }
 
-func (p *processor) processEmojis(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
+func (p *processor) ProcessEmojis(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
 	emojis := []string{}
 	gtsEmojis, err := p.db.EmojiStringsToEmojis(util.DeriveEmojisFromStatus(form.Status), accountID, status.ID)
 	if err != nil {
@@ -233,7 +233,7 @@ func (p *processor) processEmojis(form *apimodel.AdvancedStatusCreateForm, accou
 	return nil
 }
 
-func (p *processor) processContent(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
+func (p *processor) ProcessContent(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
 	// if there's nothing in the status at all we can just return early
 	if form.Status == "" {
 		status.Content = ""
@@ -249,15 +249,16 @@ func (p *processor) processContent(form *apimodel.AdvancedStatusCreateForm, acco
 	content := text.RemoveHTML(form.Status)
 
 	// parse content out of the status depending on what format has been submitted
+	var formatted string
 	switch form.Format {
 	case apimodel.StatusFormatPlain:
-		content = p.formatter.FromPlain(content, status.GTSMentions, status.GTSTags)
+		formatted = p.formatter.FromPlain(content, status.GTSMentions, status.GTSTags)
 	case apimodel.StatusFormatMarkdown:
-		content = p.formatter.FromMarkdown(content, status.GTSMentions, status.GTSTags)
+		formatted = p.formatter.FromMarkdown(content, status.GTSMentions, status.GTSTags)
 	default:
 		return fmt.Errorf("format %s not recognised as a valid status format", form.Format)
 	}
 
-	status.Content = content
+	status.Content = formatted
 	return nil
 }

internal/processing/status/util_test.go

@@ -0,0 +1,349 @@
+package status_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/processing/status"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+const statusText1 = `Another test @foss_satan@fossbros-anonymous.io
+
+#Hashtag
+
+Text`
+const statusText1ExpectedFull = `<p>Another test <span class="h-card"><a href="http://fossbros-anonymous.io/@foss_satan" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>foss_satan</span></a></span><br/><br/><a href="http://localhost:8080/tags/Hashtag" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>Hashtag</span></a><br/><br/>Text</p>`
+const statusText1ExpectedPartial = `<p>Another test <span class="h-card"><a href="http://fossbros-anonymous.io/@foss_satan" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>foss_satan</span></a></span><br/><br/>#Hashtag<br/><br/>Text</p>`
+
+const statusText2 = `Another test @foss_satan@fossbros-anonymous.io
+
+#Hashtag
+
+#hashTAG`
+
+const status2TextExpectedFull = `<p>Another test <span class="h-card"><a href="http://fossbros-anonymous.io/@foss_satan" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>foss_satan</span></a></span><br/><br/><a href="http://localhost:8080/tags/Hashtag" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>Hashtag</span></a><br/><br/><a href="http://localhost:8080/tags/Hashtag" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>hashTAG</span></a></p>`
+
+type UtilTestSuite struct {
+	StatusStandardTestSuite
+}
+
+func (suite *UtilTestSuite) SetupSuite() {
+	suite.testTokens = testrig.NewTestTokens()
+	suite.testClients = testrig.NewTestClients()
+	suite.testApplications = testrig.NewTestApplications()
+	suite.testUsers = testrig.NewTestUsers()
+	suite.testAccounts = testrig.NewTestAccounts()
+	suite.testAttachments = testrig.NewTestAttachments()
+	suite.testStatuses = testrig.NewTestStatuses()
+	suite.testTags = testrig.NewTestTags()
+	suite.testMentions = testrig.NewTestMentions()
+}
+
+func (suite *UtilTestSuite) SetupTest() {
+	suite.config = testrig.NewTestConfig()
+	suite.db = testrig.NewTestDB()
+	suite.log = testrig.NewTestLog()
+	suite.typeConverter = testrig.NewTestTypeConverter(suite.db)
+	suite.fromClientAPIChan = make(chan gtsmodel.FromClientAPI, 100)
+	suite.status = status.New(suite.db, suite.typeConverter, suite.config, suite.fromClientAPIChan, suite.log)
+
+	testrig.StandardDBSetup(suite.db, nil)
+}
+
+func (suite *UtilTestSuite) TearDownTest() {
+	testrig.StandardDBTeardown(suite.db)
+}
+
+func (suite *UtilTestSuite) TestProcessMentions1() {
+	creatingAccount := suite.testAccounts["local_account_1"]
+	mentionedAccount := suite.testAccounts["remote_account_1"]
+
+	form := &model.AdvancedStatusCreateForm{
+		StatusCreateRequest: model.StatusCreateRequest{
+			Status:      statusText1,
+			MediaIDs:    []string{},
+			Poll:        nil,
+			InReplyToID: "",
+			Sensitive:   false,
+			SpoilerText: "",
+			Visibility:  model.VisibilityPublic,
+			ScheduledAt: "",
+			Language:    "en",
+			Format:      model.StatusFormatPlain,
+		},
+		AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
+			Federated: nil,
+			Boostable: nil,
+			Replyable: nil,
+			Likeable:  nil,
+		},
+	}
+
+	status := &gtsmodel.Status{
+		ID: "01FCTDD78JJMX3K9KPXQ7ZQ8BJ",
+	}
+
+	err := suite.status.ProcessMentions(form, creatingAccount.ID, status)
+	assert.NoError(suite.T(), err)
+
+	assert.Len(suite.T(), status.GTSMentions, 1)
+	newMention := status.GTSMentions[0]
+	assert.Equal(suite.T(), mentionedAccount.ID, newMention.TargetAccountID)
+	assert.Equal(suite.T(), creatingAccount.ID, newMention.OriginAccountID)
+	assert.Equal(suite.T(), creatingAccount.URI, newMention.OriginAccountURI)
+	assert.Equal(suite.T(), status.ID, newMention.StatusID)
+	assert.Equal(suite.T(), fmt.Sprintf("@%s@%s", mentionedAccount.Username, mentionedAccount.Domain), newMention.NameString)
+	assert.Equal(suite.T(), mentionedAccount.URI, newMention.MentionedAccountURI)
+	assert.Equal(suite.T(), mentionedAccount.URL, newMention.MentionedAccountURL)
+	assert.NotNil(suite.T(), newMention.GTSAccount)
+
+	assert.Len(suite.T(), status.Mentions, 1)
+	assert.Equal(suite.T(), newMention.ID, status.Mentions[0])
+}
+
+func (suite *UtilTestSuite) TestProcessContentFull1() {
+
+	/*
+		TEST PREPARATION
+	*/
+	// we need to partially process the status first since processContent expects a status with some stuff already set on it
+	creatingAccount := suite.testAccounts["local_account_1"]
+	form := &model.AdvancedStatusCreateForm{
+		StatusCreateRequest: model.StatusCreateRequest{
+			Status:      statusText1,
+			MediaIDs:    []string{},
+			Poll:        nil,
+			InReplyToID: "",
+			Sensitive:   false,
+			SpoilerText: "",
+			Visibility:  model.VisibilityPublic,
+			ScheduledAt: "",
+			Language:    "en",
+			Format:      model.StatusFormatPlain,
+		},
+		AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
+			Federated: nil,
+			Boostable: nil,
+			Replyable: nil,
+			Likeable:  nil,
+		},
+	}
+
+	status := &gtsmodel.Status{
+		ID: "01FCTDD78JJMX3K9KPXQ7ZQ8BJ",
+	}
+
+	err := suite.status.ProcessMentions(form, creatingAccount.ID, status)
+	assert.NoError(suite.T(), err)
+	assert.Empty(suite.T(), status.Content) // shouldn't be set yet
+
+	err = suite.status.ProcessTags(form, creatingAccount.ID, status)
+	assert.NoError(suite.T(), err)
+	assert.Empty(suite.T(), status.Content) // shouldn't be set yet
+
+	/*
+		ACTUAL TEST
+	*/
+
+	err = suite.status.ProcessContent(form, creatingAccount.ID, status)
+	assert.NoError(suite.T(), err)
+	assert.Equal(suite.T(), statusText1ExpectedFull, status.Content)
+}
+
+func (suite *UtilTestSuite) TestProcessContentPartial1() {
+
+	/*
+		TEST PREPARATION
+	*/
+	// we need to partially process the status first since processContent expects a status with some stuff already set on it
+	creatingAccount := suite.testAccounts["local_account_1"]
+	form := &model.AdvancedStatusCreateForm{
+		StatusCreateRequest: model.StatusCreateRequest{
+			Status:      statusText1,
+			MediaIDs:    []string{},
+			Poll:        nil,
+			InReplyToID: "",
+			Sensitive:   false,
+			SpoilerText: "",
+			Visibility:  model.VisibilityPublic,
+			ScheduledAt: "",
+			Language:    "en",
+			Format:      model.StatusFormatPlain,
+		},
+		AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
+			Federated: nil,
+			Boostable: nil,
+			Replyable: nil,
+			Likeable:  nil,
+		},
+	}
+
+	status := &gtsmodel.Status{
+		ID: "01FCTDD78JJMX3K9KPXQ7ZQ8BJ",
+	}
+
+	err := suite.status.ProcessMentions(form, creatingAccount.ID, status)
+	assert.NoError(suite.T(), err)
+	assert.Empty(suite.T(), status.Content) // shouldn't be set yet
+
+	/*
+		ACTUAL TEST
+	*/
+
+	err = suite.status.ProcessContent(form, creatingAccount.ID, status)
+	assert.NoError(suite.T(), err)
+	assert.Equal(suite.T(), statusText1ExpectedPartial, status.Content)
+}
+
+func (suite *UtilTestSuite) TestProcessMentions2() {
+	creatingAccount := suite.testAccounts["local_account_1"]
+	mentionedAccount := suite.testAccounts["remote_account_1"]
+
+	form := &model.AdvancedStatusCreateForm{
+		StatusCreateRequest: model.StatusCreateRequest{
+			Status:      statusText2,
+			MediaIDs:    []string{},
+			Poll:        nil,
+			InReplyToID: "",
+			Sensitive:   false,
+			SpoilerText: "",
+			Visibility:  model.VisibilityPublic,
+			ScheduledAt: "",
+			Language:    "en",
+			Format:      model.StatusFormatPlain,
+		},
+		AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
+			Federated: nil,
+			Boostable: nil,
+			Replyable: nil,
+			Likeable:  nil,
+		},
+	}
+
+	status := &gtsmodel.Status{
+		ID: "01FCTDD78JJMX3K9KPXQ7ZQ8BJ",
+	}
+
+	err := suite.status.ProcessMentions(form, creatingAccount.ID, status)
+	assert.NoError(suite.T(), err)
+
+	assert.Len(suite.T(), status.GTSMentions, 1)
+	newMention := status.GTSMentions[0]
+	assert.Equal(suite.T(), mentionedAccount.ID, newMention.TargetAccountID)
+	assert.Equal(suite.T(), creatingAccount.ID, newMention.OriginAccountID)
+	assert.Equal(suite.T(), creatingAccount.URI, newMention.OriginAccountURI)
+	assert.Equal(suite.T(), status.ID, newMention.StatusID)
+	assert.Equal(suite.T(), fmt.Sprintf("@%s@%s", mentionedAccount.Username, mentionedAccount.Domain), newMention.NameString)
+	assert.Equal(suite.T(), mentionedAccount.URI, newMention.MentionedAccountURI)
+	assert.Equal(suite.T(), mentionedAccount.URL, newMention.MentionedAccountURL)
+	assert.NotNil(suite.T(), newMention.GTSAccount)
+
+	assert.Len(suite.T(), status.Mentions, 1)
+	assert.Equal(suite.T(), newMention.ID, status.Mentions[0])
+}
+
+func (suite *UtilTestSuite) TestProcessContentFull2() {
+
+	/*
+		TEST PREPARATION
+	*/
+	// we need to partially process the status first since processContent expects a status with some stuff already set on it
+	creatingAccount := suite.testAccounts["local_account_1"]
+	form := &model.AdvancedStatusCreateForm{
+		StatusCreateRequest: model.StatusCreateRequest{
+			Status:      statusText2,
+			MediaIDs:    []string{},
+			Poll:        nil,
+			InReplyToID: "",
+			Sensitive:   false,
+			SpoilerText: "",
+			Visibility:  model.VisibilityPublic,
+			ScheduledAt: "",
+			Language:    "en",
+			Format:      model.StatusFormatPlain,
+		},
+		AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
+			Federated: nil,
+			Boostable: nil,
+			Replyable: nil,
+			Likeable:  nil,
+		},
+	}
+
+	status := &gtsmodel.Status{
+		ID: "01FCTDD78JJMX3K9KPXQ7ZQ8BJ",
+	}
+
+	err := suite.status.ProcessMentions(form, creatingAccount.ID, status)
+	assert.NoError(suite.T(), err)
+	assert.Empty(suite.T(), status.Content) // shouldn't be set yet
+
+	err = suite.status.ProcessTags(form, creatingAccount.ID, status)
+	assert.NoError(suite.T(), err)
+	assert.Empty(suite.T(), status.Content) // shouldn't be set yet
+
+	/*
+		ACTUAL TEST
+	*/
+
+	err = suite.status.ProcessContent(form, creatingAccount.ID, status)
+	assert.NoError(suite.T(), err)
+
+	assert.Equal(suite.T(), status2TextExpectedFull, status.Content)
+}
+
+func (suite *UtilTestSuite) TestProcessContentPartial2() {
+
+	/*
+		TEST PREPARATION
+	*/
+	// we need to partially process the status first since processContent expects a status with some stuff already set on it
+	creatingAccount := suite.testAccounts["local_account_1"]
+	form := &model.AdvancedStatusCreateForm{
+		StatusCreateRequest: model.StatusCreateRequest{
+			Status:      statusText2,
+			MediaIDs:    []string{},
+			Poll:        nil,
+			InReplyToID: "",
+			Sensitive:   false,
+			SpoilerText: "",
+			Visibility:  model.VisibilityPublic,
+			ScheduledAt: "",
+			Language:    "en",
+			Format:      model.StatusFormatPlain,
+		},
+		AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
+			Federated: nil,
+			Boostable: nil,
+			Replyable: nil,
+			Likeable:  nil,
+		},
+	}
+
+	status := &gtsmodel.Status{
+		ID: "01FCTDD78JJMX3K9KPXQ7ZQ8BJ",
+	}
+
+	err := suite.status.ProcessMentions(form, creatingAccount.ID, status)
+	assert.NoError(suite.T(), err)
+	assert.Empty(suite.T(), status.Content) // shouldn't be set yet
+
+	/*
+		ACTUAL TEST
+	*/
+
+	err = suite.status.ProcessContent(form, creatingAccount.ID, status)
+	assert.NoError(suite.T(), err)
+
+	fmt.Println(status.Content)
+	// assert.Equal(suite.T(), statusText2ExpectedPartial, status.Content)
+}
+
+func TestUtilTestSuite(t *testing.T) {
+	suite.Run(t, new(UtilTestSuite))
+}

internal/text/common.go

@@ -50,26 +50,54 @@ func postformat(in string) string {
 
 func (f *formatter) ReplaceTags(in string, tags []*gtsmodel.Tag) string {
 	return util.HashtagFinderRegex.ReplaceAllStringFunc(in, func(match string) string {
+		// we have a match
+		matchTrimmed := strings.TrimSpace(match)
+		tagAsEntered := strings.Split(matchTrimmed, "#")[1]
+
+		// check through the tags to find what we're matching
 		for _, tag := range tags {
-			if strings.TrimSpace(match) == fmt.Sprintf("#%s", tag.Name) {
-				tagContent := fmt.Sprintf(`<a href="%s" class="mention hashtag" rel="tag">#<span>%s</span></a>`, tag.URL, tag.Name)
+
+			if strings.EqualFold(matchTrimmed, fmt.Sprintf("#%s", tag.Name)) {
+				// replace the #tag with the formatted tag content
+				tagContent := fmt.Sprintf(`<a href="%s" class="mention hashtag" rel="tag">#<span>%s</span></a>`, tag.URL, tagAsEntered)
+
+				// in case the match picked up any previous space or newlines (thanks to the regex), include them as well
 				if strings.HasPrefix(match, " ") {
 					tagContent = " " + tagContent
+				} else if strings.HasPrefix(match, "\n") {
+					tagContent = "\n" + tagContent
 				}
+
+				// done
 				return tagContent
 			}
 		}
-		return in
+		// the match wasn't in the list of tags for whatever reason, so just return the match as we found it so nothing changes
+		return match
 	})
 }
 
 func (f *formatter) ReplaceMentions(in string, mentions []*gtsmodel.Mention) string {
 	for _, menchie := range mentions {
-		targetAccount := &gtsmodel.Account{}
-		if err := f.db.GetByID(menchie.TargetAccountID, targetAccount); err == nil {
-			mentionContent := fmt.Sprintf(`<span class="h-card"><a href="%s" class="u-url mention">@<span>%s</span></a></span>`, targetAccount.URL, targetAccount.Username)
-			in = strings.ReplaceAll(in, menchie.NameString, mentionContent)
+		// make sure we have a target account, either by getting one pinned on the mention,
+		// or by pulling it from the database
+		var targetAccount *gtsmodel.Account
+		if menchie.GTSAccount != nil {
+			// got it from the mention
+			targetAccount = menchie.GTSAccount
+		} else {
+			a := &gtsmodel.Account{}
+			if err := f.db.GetByID(menchie.TargetAccountID, a); err == nil {
+				// got it from the db
+				targetAccount = a
+			} else {
+				// couldn't get it so we can't do replacement
+				return in
+			}
 		}
+
+		mentionContent := fmt.Sprintf(`<span class="h-card"><a href="%s" class="u-url mention">@<span>%s</span></a></span>`, targetAccount.URL, targetAccount.Username)
+		in = strings.ReplaceAll(in, menchie.NameString, mentionContent)
 	}
 	return in
 }

internal/text/common_test.go

@@ -0,0 +1,116 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package text_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/text"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+const (
+	replaceMentionsString = `Another test @foss_satan@fossbros-anonymous.io
+
+#Hashtag
+
+Text`
+	replaceMentionsExpected = `Another test <span class="h-card"><a href="http://fossbros-anonymous.io/@foss_satan" class="u-url mention">@<span>foss_satan</span></a></span>
+
+#Hashtag
+
+Text`
+
+	replaceHashtagsExpected = `Another test @foss_satan@fossbros-anonymous.io
+
+<a href="http://localhost:8080/tags/Hashtag" class="mention hashtag" rel="tag">#<span>Hashtag</span></a>
+
+Text`
+
+	replaceHashtagsAfterMentionsExpected = `Another test <span class="h-card"><a href="http://fossbros-anonymous.io/@foss_satan" class="u-url mention">@<span>foss_satan</span></a></span>
+
+<a href="http://localhost:8080/tags/Hashtag" class="mention hashtag" rel="tag">#<span>Hashtag</span></a>
+
+Text`
+)
+
+type CommonTestSuite struct {
+	TextStandardTestSuite
+}
+
+func (suite *CommonTestSuite) SetupSuite() {
+	suite.testTokens = testrig.NewTestTokens()
+	suite.testClients = testrig.NewTestClients()
+	suite.testApplications = testrig.NewTestApplications()
+	suite.testUsers = testrig.NewTestUsers()
+	suite.testAccounts = testrig.NewTestAccounts()
+	suite.testAttachments = testrig.NewTestAttachments()
+	suite.testStatuses = testrig.NewTestStatuses()
+	suite.testTags = testrig.NewTestTags()
+	suite.testMentions = testrig.NewTestMentions()
+}
+
+func (suite *CommonTestSuite) SetupTest() {
+	suite.config = testrig.NewTestConfig()
+	suite.db = testrig.NewTestDB()
+	suite.log = testrig.NewTestLog()
+	suite.formatter = text.NewFormatter(suite.config, suite.db, suite.log)
+
+	testrig.StandardDBSetup(suite.db, nil)
+}
+
+func (suite *CommonTestSuite) TearDownTest() {
+	testrig.StandardDBTeardown(suite.db)
+}
+
+func (suite *CommonTestSuite) TestReplaceMentions() {
+	foundMentions := []*gtsmodel.Mention{
+		suite.testMentions["zork_mention_foss_satan"],
+	}
+
+	f := suite.formatter.ReplaceMentions(replaceMentionsString, foundMentions)
+	assert.Equal(suite.T(), replaceMentionsExpected, f)
+}
+
+func (suite *CommonTestSuite) TestReplaceHashtags() {
+	foundTags := []*gtsmodel.Tag{
+		suite.testTags["Hashtag"],
+	}
+
+	f := suite.formatter.ReplaceTags(replaceMentionsString, foundTags)
+
+	assert.Equal(suite.T(), replaceHashtagsExpected, f)
+}
+
+func (suite *CommonTestSuite) TestReplaceHashtagsAfterReplaceMentions() {
+	foundTags := []*gtsmodel.Tag{
+		suite.testTags["Hashtag"],
+	}
+
+	f := suite.formatter.ReplaceTags(replaceMentionsExpected, foundTags)
+
+	assert.Equal(suite.T(), replaceHashtagsAfterMentionsExpected, f)
+}
+
+func TestCommonTestSuite(t *testing.T) {
+	suite.Run(t, new(CommonTestSuite))
+}

internal/text/formatter_test.go

@@ -45,6 +45,7 @@ type TextStandardTestSuite struct {
 	testAttachments  map[string]*gtsmodel.MediaAttachment
 	testStatuses     map[string]*gtsmodel.Status
 	testTags         map[string]*gtsmodel.Tag
+	testMentions     map[string]*gtsmodel.Mention
 
 	// module being tested
 	formatter text.Formatter

internal/text/plain_test.go

@@ -19,6 +19,7 @@
 package text_test
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -34,6 +35,13 @@
 
 	withTag         = "this is a simple status that uses hashtag #welcome!"
 	withTagExpected = "<p>this is a simple status that uses hashtag <a href=\"http://localhost:8080/tags/welcome\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>welcome</span></a>!</p>"
+
+	moreComplex = `Another test @foss_satan@fossbros-anonymous.io
+
+#Hashtag
+
+Text`
+	moreComplexExpected = `<p>Another test <span class="h-card"><a href="http://fossbros-anonymous.io/@foss_satan" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>foss_satan</span></a></span><br/><br/><a href="http://localhost:8080/tags/Hashtag" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>Hashtag</span></a><br/><br/>Text</p>`
 )
 
 type PlainTestSuite struct {
@@ -49,6 +57,7 @@ func (suite *PlainTestSuite) SetupSuite() {
 	suite.testAttachments = testrig.NewTestAttachments()
 	suite.testStatuses = testrig.NewTestStatuses()
 	suite.testTags = testrig.NewTestTags()
+	suite.testMentions = testrig.NewTestMentions()
 }
 
 func (suite *PlainTestSuite) SetupTest() {
@@ -79,6 +88,23 @@ func (suite *PlainTestSuite) TestParseWithTag() {
 	assert.Equal(suite.T(), withTagExpected, f)
 }
 
+func (suite *PlainTestSuite) TestParseMoreComplex() {
+
+	foundTags := []*gtsmodel.Tag{
+		suite.testTags["Hashtag"],
+	}
+
+	foundMentions := []*gtsmodel.Mention{
+		suite.testMentions["zork_mention_foss_satan"],
+	}
+
+	f := suite.formatter.FromPlain(moreComplex, foundMentions, foundTags)
+
+	fmt.Println(f)
+
+	assert.Equal(suite.T(), moreComplexExpected, f)
+}
+
 func TestPlainTestSuite(t *testing.T) {
 	suite.Run(t, new(PlainTestSuite))
 }

internal/util/statustools.go

@@ -46,7 +46,7 @@ func DeriveHashtagsFromStatus(status string) []string {
 	for _, m := range HashtagFinderRegex.FindAllStringSubmatch(status, -1) {
 		tags = append(tags, strings.TrimPrefix(m[1], "#"))
 	}
-	return uniqueLower(tags)
+	return unique(tags)
 }
 
 // DeriveEmojisFromStatus takes a plaintext (ie., not html-formatted) status,
@@ -92,17 +92,3 @@ func unique(s []string) []string {
 	}
 	return list
 }
-
-// uniqueLower returns a deduplicated version of a given string slice, with all entries converted to lowercase
-func uniqueLower(s []string) []string {
-	keys := make(map[string]bool)
-	list := []string{}
-	for _, entry := range s {
-		eLower := strings.ToLower(entry)
-		if _, value := keys[eLower]; !value {
-			keys[eLower] = true
-			list = append(list, eLower)
-		}
-	}
-	return list
-}

internal/util/statustools_test.go

@@ -79,7 +79,7 @@ func (suite *StatusTestSuite) TestDeriveHashtagsOK() {
 	assert.Equal(suite.T(), "testing123", tags[0])
 	assert.Equal(suite.T(), "also", tags[1])
 	assert.Equal(suite.T(), "thisshouldwork", tags[2])
-	assert.Equal(suite.T(), "thisshouldalsowork", tags[3])
+	assert.Equal(suite.T(), "ThisShouldAlsoWork", tags[3])
 	assert.Equal(suite.T(), "111111", tags[4])
 }
 
@@ -108,6 +108,26 @@ func (suite *StatusTestSuite) TestDeriveEmojiOK() {
 	assert.Equal(suite.T(), "underscores_ok_too", tags[6])
 }
 
+func (suite *StatusTestSuite) TestDeriveMultiple() {
+	statusText := `Another test @foss_satan@fossbros-anonymous.io
+
+	#Hashtag
+
+	Text`
+
+	ms := util.DeriveMentionsFromStatus(statusText)
+	hs := util.DeriveHashtagsFromStatus(statusText)
+	es := util.DeriveEmojisFromStatus(statusText)
+
+	assert.Len(suite.T(), ms, 1)
+	assert.Equal(suite.T(), "@foss_satan@fossbros-anonymous.io", ms[0])
+
+	assert.Len(suite.T(), hs, 1)
+	assert.Equal(suite.T(), "Hashtag", hs[0])
+
+	assert.Len(suite.T(), es, 0)
+}
+
 func TestStatusTestSuite(t *testing.T) {
 	suite.Run(t, new(StatusTestSuite))
 }

testrig/db.go

@@ -141,6 +141,12 @@ func StandardDBSetup(db db.DB, accounts map[string]*gtsmodel.Account) {
 		}
 	}
 
+	for _, v := range NewTestMentions() {
+		if err := db.Put(v); err != nil {
+			panic(err)
+		}
+	}
+
 	for _, v := range NewTestFaves() {
 		if err := db.Put(v); err != nil {
 			panic(err)

testrig/testmodels.go

@@ -937,6 +937,31 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			},
 			ActivityStreamsType: gtsmodel.ActivityStreamsNote,
 		},
+		"local_account_1_status_5": {
+			ID:                       "01FCTA44PW9H1TB328S9AQXKDS",
+			URI:                      "http://localhost:8080/users/the_mighty_zork/statuses/01FCTA44PW9H1TB328S9AQXKDS",
+			URL:                      "http://localhost:8080/@the_mighty_zork/statuses/01FCTA44PW9H1TB328S9AQXKDS",
+			Content:                  "hi!",
+			Attachments:              []string{},
+			CreatedAt:                time.Now().Add(-1 * time.Minute),
+			UpdatedAt:                time.Now().Add(-1 * time.Minute),
+			Local:                    true,
+			AccountID:                "01F8MH1H7YV1Z7D2C8K2730QBF",
+			InReplyToID:              "",
+			BoostOfID:                "",
+			ContentWarning:           "",
+			Visibility:               gtsmodel.VisibilityMutualsOnly,
+			Sensitive:                false,
+			Language:                 "en",
+			CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG",
+			VisibilityAdvanced: &gtsmodel.VisibilityAdvanced{
+				Federated: true,
+				Boostable: true,
+				Replyable: true,
+				Likeable:  true,
+			},
+			ActivityStreamsType: gtsmodel.ActivityStreamsNote,
+		},
 		"local_account_2_status_1": {
 			ID:                       "01F8MHBQCBTDKN6X5VHGMMN4MA",
 			URI:                      "http://localhost:8080/users/1happyturtle/statuses/01F8MHBQCBTDKN6X5VHGMMN4MA",
@@ -1076,6 +1101,35 @@ func NewTestTags() map[string]*gtsmodel.Tag {
 			Listable:               true,
 			LastStatusAt:           time.Now().Add(-71 * time.Hour),
 		},
+		"Hashtag": {
+			ID:                     "01FCT9SGYA71487N8D0S1M638G",
+			URL:                    "http://localhost:8080/tags/Hashtag",
+			Name:                   "Hashtag",
+			FirstSeenFromAccountID: "",
+			CreatedAt:              time.Now().Add(-71 * time.Hour),
+			UpdatedAt:              time.Now().Add(-71 * time.Hour),
+			Useable:                true,
+			Listable:               true,
+			LastStatusAt:           time.Now().Add(-71 * time.Hour),
+		},
+	}
+}
+
+// NewTestMentions returns a map of gts model mentions keyed by their name.
+func NewTestMentions() map[string]*gtsmodel.Mention {
+	return map[string]*gtsmodel.Mention{
+		"zork_mention_foss_satan": {
+			ID:                  "01FCTA2Y6FGHXQA4ZE6N5NMNEX",
+			StatusID:            "01FCTA44PW9H1TB328S9AQXKDS",
+			CreatedAt:           time.Now().Add(-1 * time.Minute),
+			UpdatedAt:           time.Now().Add(-1 * time.Minute),
+			OriginAccountID:     "01F8MH1H7YV1Z7D2C8K2730QBF",
+			OriginAccountURI:    "http://localhost:8080/users/the_mighty_zork",
+			TargetAccountID:     "01F8MH5ZK5VRH73AKHQM6Y9VNX",
+			NameString:          "@foss_satan@fossbros-anonymous.io",
+			MentionedAccountURI: "http://fossbros-anonymous.io/users/foss_satan",
+			MentionedAccountURL: "http://fossbros-anonymous.io/@foss_satan",
+		},
 	}
 }