From 53507ac2a32a785b5467b1e58d033780d8e02693 Mon Sep 17 00:00:00 2001 From: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Sun, 29 Aug 2021 12:03:08 +0200 Subject: [PATCH] Mention fixup (#167) * rework mention creation a bit * rework mention creation a bit * tidy up status dereferencing * start adding tests for dereferencing * fixups * fix * review changes --- internal/ap/extract.go | 9 +- internal/cliactions/server/server.go | 2 +- internal/db/bundb/instance.go | 12 +- .../federation/dereferencing/attachment.go | 84 +++++++ .../federation/dereferencing/dereferencer.go | 3 + .../dereferencing/dereferencer_test.go | 42 ++++ internal/federation/dereferencing/status.go | 222 +++++++++++------- .../federation/dereferencing/status_test.go | 213 +++++++++++++++++ internal/federation/federatingdb/db.go | 2 +- internal/gtsmodel/mention.go | 6 +- internal/media/handler.go | 31 --- internal/typeutils/astointernal.go | 34 ++- internal/typeutils/astointernal_test.go | 6 +- internal/typeutils/converter.go | 5 +- internal/typeutils/converter_test.go | 4 +- internal/typeutils/internaltoas.go | 45 ++-- internal/typeutils/internaltoas_test.go | 2 +- internal/typeutils/internaltofrontend.go | 38 ++- internal/util/regexes.go | 2 +- testrig/testmodels.go | 108 ++++++++- testrig/typeconverter.go | 2 +- 21 files changed, 680 insertions(+), 192 deletions(-) create mode 100644 internal/federation/dereferencing/attachment.go create mode 100644 internal/federation/dereferencing/dereferencer_test.go create mode 100644 internal/federation/dereferencing/status_test.go diff --git a/internal/ap/extract.go b/internal/ap/extract.go index 1ee0e008e..f8453c9c0 100644 --- a/internal/ap/extract.go +++ b/internal/ap/extract.go @@ -242,8 +242,9 @@ func ExtractImageURL(i WithImage) (*url.URL, error) { // ExtractSummary extracts the summary/content warning of an interface. func ExtractSummary(i WithSummary) (string, error) { summaryProp := i.GetActivityStreamsSummary() - if summaryProp == nil { - return "", errors.New("summary property was nil") + if summaryProp == nil || summaryProp.Len() == 0 { + // no summary to speak of + return "", nil } for iter := summaryProp.Begin(); iter != summaryProp.End(); iter = iter.Next() { @@ -544,12 +545,12 @@ func ExtractMentions(i WithTag) ([]*gtsmodel.Mention, error) { mentionable, ok := t.(Mentionable) if !ok { - continue + return nil, errors.New("mention was not convertable to ap.Mentionable") } mention, err := ExtractMention(mentionable) if err != nil { - continue + return nil, err } mentions = append(mentions, mention) diff --git a/internal/cliactions/server/server.go b/internal/cliactions/server/server.go index 877a9d397..504a4c3c7 100644 --- a/internal/cliactions/server/server.go +++ b/internal/cliactions/server/server.go @@ -111,7 +111,7 @@ } // build converters and util - typeConverter := typeutils.NewConverter(c, dbService) + typeConverter := typeutils.NewConverter(c, dbService, log) timelineManager := timelineprocessing.NewManager(dbService, typeConverter, c, log) // build backend handlers diff --git a/internal/db/bundb/instance.go b/internal/db/bundb/instance.go index ea7ae194b..2813e7e1d 100644 --- a/internal/db/bundb/instance.go +++ b/internal/db/bundb/instance.go @@ -41,12 +41,12 @@ func (i *instanceDB) CountInstanceUsers(ctx context.Context, domain string) (int Where("username != ?", domain). Where("? IS NULL", bun.Ident("suspended_at")) - if domain == i.config.Host { - // if the domain is *this* domain, just count where the domain field is null - q = q.WhereGroup(" AND ", whereEmptyOrNull("domain")) - } else { - q = q.Where("domain = ?", domain) - } + if domain == i.config.Host { + // if the domain is *this* domain, just count where the domain field is null + q = q.WhereGroup(" AND ", whereEmptyOrNull("domain")) + } else { + q = q.Where("domain = ?", domain) + } count, err := q.Count(ctx) diff --git a/internal/federation/dereferencing/attachment.go b/internal/federation/dereferencing/attachment.go new file mode 100644 index 000000000..fd2e3cb8f --- /dev/null +++ b/internal/federation/dereferencing/attachment.go @@ -0,0 +1,84 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "context" + "fmt" + "net/url" + + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +func (d *deref) GetRemoteAttachment(ctx context.Context, requestingUsername string, remoteAttachmentURI *url.URL, ownerAccountID string, statusID string, expectedContentType string) (*gtsmodel.MediaAttachment, error) { + l := d.log.WithFields(logrus.Fields{ + "username": requestingUsername, + "remoteAttachmentURI": remoteAttachmentURI, + }) + + maybeAttachment := >smodel.MediaAttachment{} + where := []db.Where{ + { + Key: "remote_url", + Value: remoteAttachmentURI.String(), + }, + } + + if err := d.db.GetWhere(ctx, where, maybeAttachment); err == nil { + // we already the attachment in the database + l.Debugf("GetRemoteAttachment: attachment already exists with id %s", maybeAttachment.ID) + return maybeAttachment, nil + } + + a, err := d.RefreshAttachment(ctx, requestingUsername, remoteAttachmentURI, ownerAccountID, expectedContentType) + if err != nil { + return nil, fmt.Errorf("GetRemoteAttachment: error refreshing attachment: %s", err) + } + + a.StatusID = statusID + if err := d.db.Put(ctx, a); err != nil { + if err != db.ErrAlreadyExists { + return nil, fmt.Errorf("GetRemoteAttachment: error inserting attachment: %s", err) + } + } + + return a, nil +} + +func (d *deref) RefreshAttachment(ctx context.Context, requestingUsername string, remoteAttachmentURI *url.URL, ownerAccountID string, expectedContentType string) (*gtsmodel.MediaAttachment, error) { + // it just doesn't exist or we have to refresh + t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername) + if err != nil { + return nil, fmt.Errorf("RefreshAttachment: error creating transport: %s", err) + } + + attachmentBytes, err := t.DereferenceMedia(ctx, remoteAttachmentURI, expectedContentType) + if err != nil { + return nil, fmt.Errorf("RefreshAttachment: error dereferencing media: %s", err) + } + + a, err := d.mediaHandler.ProcessAttachment(ctx, attachmentBytes, ownerAccountID, remoteAttachmentURI.String()) + if err != nil { + return nil, fmt.Errorf("RefreshAttachment: error processing attachment: %s", err) + } + + return a, nil +} diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go index 71625ed88..4191bd283 100644 --- a/internal/federation/dereferencing/dereferencer.go +++ b/internal/federation/dereferencing/dereferencer.go @@ -43,6 +43,9 @@ type Dereferencer interface { GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) + GetRemoteAttachment(ctx context.Context, username string, remoteAttachmentURI *url.URL, ownerAccountID string, statusID string, expectedContentType string) (*gtsmodel.MediaAttachment, error) + RefreshAttachment(ctx context.Context, requestingUsername string, remoteAttachmentURI *url.URL, ownerAccountID string, expectedContentType string) (*gtsmodel.MediaAttachment, error) + DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error DereferenceThread(ctx context.Context, username string, statusIRI *url.URL) error diff --git a/internal/federation/dereferencing/dereferencer_test.go b/internal/federation/dereferencing/dereferencer_test.go new file mode 100644 index 000000000..299aba10a --- /dev/null +++ b/internal/federation/dereferencing/dereferencer_test.go @@ -0,0 +1,42 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing_test + +import ( + "github.com/go-fed/activity/streams/vocab" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +type DereferencerStandardTestSuite struct { + suite.Suite + config *config.Config + db db.DB + log *logrus.Logger + + testRemoteStatuses map[string]vocab.ActivityStreamsNote + testRemoteAccounts map[string]vocab.ActivityStreamsPerson + testAccounts map[string]*gtsmodel.Account + + dereferencer dereferencing.Dereferencer +} diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go index 93ead6523..3fa1e4133 100644 --- a/internal/federation/dereferencing/status.go +++ b/internal/federation/dereferencing/status.go @@ -24,12 +24,12 @@ "errors" "fmt" "net/url" + "strings" "github.com/go-fed/activity/streams" "github.com/go-fed/activity/streams/vocab" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/ap" - "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" ) @@ -229,8 +229,7 @@ func (d *deref) dereferenceStatusable(ctx context.Context, username string, remo // 2. Hashtags. // 3. Emojis. // 4. Mentions. -// 5. Posting account. -// 6. Replied-to-status. +// 5. Replied-to-status. // // SIDE EFFECTS: // This function will deference all of the above, insert them in the database as necessary, @@ -243,117 +242,113 @@ func (d *deref) populateStatusFields(ctx context.Context, status *gtsmodel.Statu }) l.Debug("entering function") - // make sure we have a status URI and that the domain in question isn't blocked - statusURI, err := url.Parse(status.URI) + statusIRI, err := url.Parse(status.URI) if err != nil { - return fmt.Errorf("DereferenceStatusFields: couldn't parse status URI %s: %s", status.URI, err) - } - if blocked, err := d.db.IsDomainBlocked(ctx, statusURI.Host); blocked || err != nil { - return fmt.Errorf("DereferenceStatusFields: domain %s is blocked", statusURI.Host) + return fmt.Errorf("populateStatusFields: couldn't parse status URI %s: %s", status.URI, err) } - // we can continue -- create a new transport here because we'll probably need it - t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername) + blocked, err := d.db.IsURIBlocked(ctx, statusIRI) if err != nil { - return fmt.Errorf("error creating transport: %s", err) + return fmt.Errorf("populateStatusFields: error checking blocked status of %s: %s", statusIRI, err) + } + if blocked { + return fmt.Errorf("populateStatusFields: domain %s is blocked", statusIRI) } // in case the status doesn't have an id yet (ie., it hasn't entered the database yet), then create one if status.ID == "" { newID, err := id.NewULIDFromTime(status.CreatedAt) if err != nil { - return err + return fmt.Errorf("populateStatusFields: error creating ulid for status: %s", err) } status.ID = newID } // 1. Media attachments. - // - // At this point we should know: - // * the media type of the file we're looking for (a.File.ContentType) - // * the blurhash (a.Blurhash) - // * the file type (a.Type) - // * the remote URL (a.RemoteURL) - // This should be enough to pass along to the media processor. - attachmentIDs := []string{} - for _, a := range status.Attachments { - l.Tracef("dereferencing attachment: %+v", a) - - // it might have been processed elsewhere so check first if it's already in the database or not - maybeAttachment := >smodel.MediaAttachment{} - err := d.db.GetWhere(ctx, []db.Where{{Key: "remote_url", Value: a.RemoteURL}}, maybeAttachment) - if err == nil { - // we already have it in the db, dereferenced, no need to do it again - l.Tracef("attachment already exists with id %s", maybeAttachment.ID) - attachmentIDs = append(attachmentIDs, maybeAttachment.ID) - continue - } - if err != db.ErrNoEntries { - // we have a real error - return fmt.Errorf("error checking db for existence of attachment with remote url %s: %s", a.RemoteURL, err) - } - // it just doesn't exist yet so carry on - l.Debug("attachment doesn't exist yet, calling ProcessRemoteAttachment", a) - deferencedAttachment, err := d.mediaHandler.ProcessRemoteAttachment(ctx, t, a, status.AccountID) - if err != nil { - l.Errorf("error dereferencing status attachment: %s", err) - continue - } - l.Debugf("dereferenced attachment: %+v", deferencedAttachment) - deferencedAttachment.StatusID = status.ID - deferencedAttachment.Description = a.Description - if err := d.db.Put(ctx, deferencedAttachment); err != nil { - return fmt.Errorf("error inserting dereferenced attachment with remote url %s: %s", a.RemoteURL, err) - } - attachmentIDs = append(attachmentIDs, deferencedAttachment.ID) + if err := d.populateStatusAttachments(ctx, status, requestingUsername); err != nil { + return fmt.Errorf("populateStatusFields: error populating status attachments: %s", err) } - status.AttachmentIDs = attachmentIDs // 2. Hashtags + // TODO // 3. Emojis + // TODO // 4. Mentions + if err := d.populateStatusMentions(ctx, status, requestingUsername); err != nil { + return fmt.Errorf("populateStatusFields: error populating status mentions: %s", err) + } + + // 5. Replied-to-status. + if err := d.populateStatusRepliedTo(ctx, status, requestingUsername); err != nil { + return fmt.Errorf("populateStatusFields: error populating status repliedTo: %s", err) + } + + return nil +} + +func (d *deref) populateStatusMentions(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error { + l := d.log + // At this point, mentions should have the namestring and mentionedAccountURI set on them. - // - // We should dereference any accounts mentioned here which we don't have in our db yet, by their URI. + // We can use these to find the accounts. + mentionIDs := []string{} + newMentions := []*gtsmodel.Mention{} for _, m := range status.Mentions { if m.ID != "" { // we've already populated this mention, since it has an ID - l.Debug("mention already populated") + l.Debug("populateStatusMentions: mention already populated") + mentionIDs = append(mentionIDs, m.ID) + newMentions = append(newMentions, m) continue } if m.TargetAccountURI == "" { - // can't do anything with this mention - l.Debug("target URI not set on mention") + l.Debug("populateStatusMentions: target URI not set on mention") continue } targetAccountURI, err := url.Parse(m.TargetAccountURI) if err != nil { - l.Debugf("error parsing mentioned account uri %s: %s", m.TargetAccountURI, err) + l.Debugf("populateStatusMentions: error parsing mentioned account uri %s: %s", m.TargetAccountURI, err) continue } var targetAccount *gtsmodel.Account - if a, err := d.db.GetAccountByURL(ctx, targetAccountURI.String()); err == nil { - targetAccount = a - } else if a, _, err := d.GetRemoteAccount(ctx, requestingUsername, targetAccountURI, false); err == nil { - targetAccount = a + errs := []string{} + + // check if account is in the db already + if a, err := d.db.GetAccountByURL(ctx, targetAccountURI.String()); err != nil { + errs = append(errs, err.Error()) } else { - // we can't find the target account so bail - l.Debug("can't retrieve account targeted by mention") + l.Debugf("populateStatusMentions: got target account %s with id %s through GetAccountByURL", targetAccountURI, a.ID) + targetAccount = a + } + + if targetAccount == nil { + // we didn't find the account in our database already + // check if we can get the account remotely (dereference it) + if a, _, err := d.GetRemoteAccount(ctx, requestingUsername, targetAccountURI, false); err != nil { + errs = append(errs, err.Error()) + } else { + l.Debugf("populateStatusMentions: got target account %s with id %s through GetRemoteAccount", targetAccountURI, a.ID) + targetAccount = a + } + } + + if targetAccount == nil { + l.Debugf("populateStatusMentions: couldn't get target account %s: %s", m.TargetAccountURI, strings.Join(errs, " : ")) continue } mID, err := id.NewRandomULID() if err != nil { - return err + return fmt.Errorf("populateStatusMentions: error generating ulid: %s", err) } - m = >smodel.Mention{ + newMention := >smodel.Mention{ ID: mID, StatusID: status.ID, Status: m.Status, @@ -369,32 +364,91 @@ func (d *deref) populateStatusFields(ctx context.Context, status *gtsmodel.Statu TargetAccountURL: targetAccount.URL, } - if err := d.db.Put(ctx, m); err != nil { - return fmt.Errorf("error creating mention: %s", err) + if err := d.db.Put(ctx, newMention); err != nil { + return fmt.Errorf("populateStatusMentions: error creating mention: %s", err) } - mentionIDs = append(mentionIDs, m.ID) - } - status.MentionIDs = mentionIDs - // status has replyToURI but we don't have an ID yet for the status it replies to + mentionIDs = append(mentionIDs, newMention.ID) + newMentions = append(newMentions, newMention) + } + + status.MentionIDs = mentionIDs + status.Mentions = newMentions + + return nil +} + +func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error { + l := d.log + + // At this point we should know: + // * the media type of the file we're looking for (a.File.ContentType) + // * the file type (a.Type) + // * the remote URL (a.RemoteURL) + // This should be enough to dereference the piece of media. + + attachmentIDs := []string{} + attachments := []*gtsmodel.MediaAttachment{} + + for _, a := range status.Attachments { + + aURL, err := url.Parse(a.RemoteURL) + if err != nil { + l.Errorf("populateStatusAttachments: couldn't parse attachment url %s: %s", a.RemoteURL, err) + continue + } + + attachment, err := d.GetRemoteAttachment(ctx, requestingUsername, aURL, status.AccountID, status.ID, a.File.ContentType) + if err != nil { + l.Errorf("populateStatusAttachments: couldn't get remote attachment %s: %s", a.RemoteURL, err) + } + + attachmentIDs = append(attachmentIDs, attachment.ID) + attachments = append(attachments, attachment) + } + + status.AttachmentIDs = attachmentIDs + status.Attachments = attachments + + return nil +} + +func (d *deref) populateStatusRepliedTo(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error { if status.InReplyToURI != "" && status.InReplyToID == "" { statusURI, err := url.Parse(status.InReplyToURI) if err != nil { return err } - if replyToStatus, err := d.db.GetStatusByURI(ctx, status.InReplyToURI); err == nil { - // we have the status - status.InReplyToID = replyToStatus.ID - status.InReplyTo = replyToStatus - status.InReplyToAccountID = replyToStatus.AccountID - status.InReplyToAccount = replyToStatus.Account - } else if replyToStatus, _, _, err := d.GetRemoteStatus(ctx, requestingUsername, statusURI, false); err == nil { - // we got the status - status.InReplyToID = replyToStatus.ID - status.InReplyTo = replyToStatus - status.InReplyToAccountID = replyToStatus.AccountID - status.InReplyToAccount = replyToStatus.Account + + var replyToStatus *gtsmodel.Status + errs := []string{} + + // see if we have the status in our db already + if s, err := d.db.GetStatusByURI(ctx, status.InReplyToURI); err != nil { + errs = append(errs, err.Error()) + } else { + replyToStatus = s } + + if replyToStatus == nil { + // didn't find the status in our db, try to get it remotely + if s, _, _, err := d.GetRemoteStatus(ctx, requestingUsername, statusURI, false); err != nil { + errs = append(errs, err.Error()) + } else { + replyToStatus = s + } + } + + if replyToStatus == nil { + return fmt.Errorf("populateStatusRepliedTo: couldn't get reply to status with uri %s: %s", statusURI, strings.Join(errs, " : ")) + } + + // we have the status + status.InReplyToID = replyToStatus.ID + status.InReplyTo = replyToStatus + status.InReplyToAccountID = replyToStatus.AccountID + status.InReplyToAccount = replyToStatus.Account } + return nil } diff --git a/internal/federation/dereferencing/status_test.go b/internal/federation/dereferencing/status_test.go new file mode 100644 index 000000000..2d259682b --- /dev/null +++ b/internal/federation/dereferencing/status_test.go @@ -0,0 +1,213 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing_test + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "testing" + "time" + + "github.com/go-fed/activity/streams" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/transport" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type StatusTestSuite struct { + DereferencerStandardTestSuite +} + +// mockTransportController returns basically a miniature muxer, which returns a different +// value based on the request URL. It can be used to return remote statuses, profiles, etc, +// as though they were actually being dereferenced. If the URL doesn't correspond to any person +// or note or attachment that we have stored, then just a 200 code will be returned, with an empty body. +func (suite *StatusTestSuite) mockTransportController() transport.Controller { + do := func(req *http.Request) (*http.Response, error) { + suite.log.Debugf("received request for %s", req.URL) + + responseBytes := []byte{} + + note, ok := suite.testRemoteStatuses[req.URL.String()] + if ok { + // the request is for a note that we have stored + noteI, err := streams.Serialize(note) + if err != nil { + panic(err) + } + noteJson, err := json.Marshal(noteI) + if err != nil { + panic(err) + } + responseBytes = noteJson + } + + person, ok := suite.testRemoteAccounts[req.URL.String()] + if ok { + // the request is for a person that we have stored + personI, err := streams.Serialize(person) + if err != nil { + panic(err) + } + personJson, err := json.Marshal(personI) + if err != nil { + panic(err) + } + responseBytes = personJson + } + + if len(responseBytes) != 0 { + // we found something, so print what we're going to return + suite.log.Debugf("returning response %s", string(responseBytes)) + } + + reader := bytes.NewReader(responseBytes) + readCloser := io.NopCloser(reader) + response := &http.Response{ + StatusCode: 200, + Body: readCloser, + } + + return response, nil + } + mockClient := testrig.NewMockHTTPClient(do) + return testrig.NewTestTransportController(mockClient, suite.db) +} + +func (suite *StatusTestSuite) SetupSuite() { + suite.testAccounts = testrig.NewTestAccounts() + suite.testRemoteStatuses = testrig.NewTestFediStatuses() + suite.testRemoteAccounts = testrig.NewTestFediPeople() +} + +func (suite *StatusTestSuite) SetupTest() { + suite.config = testrig.NewTestConfig() + suite.db = testrig.NewTestDB() + suite.log = testrig.NewTestLog() + suite.dereferencer = dereferencing.NewDereferencer(suite.config, + suite.db, + testrig.NewTestTypeConverter(suite.db), + suite.mockTransportController(), + testrig.NewTestMediaHandler(suite.db, testrig.NewTestStorage()), + suite.log) + testrig.StandardDBSetup(suite.db, nil) +} + +func (suite *StatusTestSuite) TestDereferenceSimpleStatus() { + fetchingAccount := suite.testAccounts["local_account_1"] + + statusURL := testrig.URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01FE4NTHKWW7THT67EF10EB839") + status, statusable, new, err := suite.dereferencer.GetRemoteStatus(context.Background(), fetchingAccount.Username, statusURL, false) + suite.NoError(err) + suite.NotNil(status) + suite.NotNil(statusable) + suite.True(new) + + // status values should be set + suite.Equal("https://unknown-instance.com/users/brand_new_person/statuses/01FE4NTHKWW7THT67EF10EB839", status.URI) + suite.Equal("https://unknown-instance.com/users/@brand_new_person/01FE4NTHKWW7THT67EF10EB839", status.URL) + suite.Equal("Hello world!", status.Content) + suite.Equal("https://unknown-instance.com/users/brand_new_person", status.AccountURI) + suite.False(status.Local) + suite.Empty(status.ContentWarning) + suite.Equal(gtsmodel.VisibilityPublic, status.Visibility) + suite.Equal(gtsmodel.ActivityStreamsNote, status.ActivityStreamsType) + + // status should be in the database + dbStatus, err := suite.db.GetStatusByURI(context.Background(), status.URI) + suite.NoError(err) + suite.Equal(status.ID, dbStatus.ID) + + // account should be in the database now too + account, err := suite.db.GetAccountByURI(context.Background(), status.AccountURI) + suite.NoError(err) + suite.NotNil(account) + suite.True(account.Discoverable) + suite.Equal("https://unknown-instance.com/users/brand_new_person", account.URI) + suite.Equal("hey I'm a new person, your instance hasn't seen me yet uwu", account.Note) + suite.Equal("Geoff Brando New Personson", account.DisplayName) + suite.Equal("brand_new_person", account.Username) + suite.NotNil(account.PublicKey) + suite.Nil(account.PrivateKey) +} + +func (suite *StatusTestSuite) TestDereferenceStatusWithMention() { + fetchingAccount := suite.testAccounts["local_account_1"] + + statusURL := testrig.URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01FE5Y30E3W4P7TRE0R98KAYQV") + status, statusable, new, err := suite.dereferencer.GetRemoteStatus(context.Background(), fetchingAccount.Username, statusURL, false) + suite.NoError(err) + suite.NotNil(status) + suite.NotNil(statusable) + suite.True(new) + + // status values should be set + suite.Equal("https://unknown-instance.com/users/brand_new_person/statuses/01FE5Y30E3W4P7TRE0R98KAYQV", status.URI) + suite.Equal("https://unknown-instance.com/users/@brand_new_person/01FE5Y30E3W4P7TRE0R98KAYQV", status.URL) + suite.Equal("Hey @the_mighty_zork@localhost:8080 how's it going?", status.Content) + suite.Equal("https://unknown-instance.com/users/brand_new_person", status.AccountURI) + suite.False(status.Local) + suite.Empty(status.ContentWarning) + suite.Equal(gtsmodel.VisibilityPublic, status.Visibility) + suite.Equal(gtsmodel.ActivityStreamsNote, status.ActivityStreamsType) + + // status should be in the database + dbStatus, err := suite.db.GetStatusByURI(context.Background(), status.URI) + suite.NoError(err) + suite.Equal(status.ID, dbStatus.ID) + + // account should be in the database now too + account, err := suite.db.GetAccountByURI(context.Background(), status.AccountURI) + suite.NoError(err) + suite.NotNil(account) + suite.True(account.Discoverable) + suite.Equal("https://unknown-instance.com/users/brand_new_person", account.URI) + suite.Equal("hey I'm a new person, your instance hasn't seen me yet uwu", account.Note) + suite.Equal("Geoff Brando New Personson", account.DisplayName) + suite.Equal("brand_new_person", account.Username) + suite.NotNil(account.PublicKey) + suite.Nil(account.PrivateKey) + + // we should have a mention in the database + m := >smodel.Mention{} + err = suite.db.GetWhere(context.Background(), []db.Where{{Key: "status_id", Value: status.ID}}, m) + suite.NoError(err) + suite.NotNil(m) + suite.Equal(status.ID, m.StatusID) + suite.Equal(account.ID, m.OriginAccountID) + suite.Equal(fetchingAccount.ID, m.TargetAccountID) + suite.Equal(account.URI, m.OriginAccountURI) + suite.WithinDuration(time.Now(), m.CreatedAt, 5*time.Minute) + suite.WithinDuration(time.Now(), m.UpdatedAt, 5*time.Minute) + suite.False(m.Silent) +} + +func (suite *StatusTestSuite) TearDownTest() { + testrig.StandardDBTeardown(suite.db) +} + +func TestStatusTestSuite(t *testing.T) { + suite.Run(t, new(StatusTestSuite)) +} diff --git a/internal/federation/federatingdb/db.go b/internal/federation/federatingdb/db.go index 5f8b9ad90..8d41d082e 100644 --- a/internal/federation/federatingdb/db.go +++ b/internal/federation/federatingdb/db.go @@ -60,7 +60,7 @@ func New(db db.DB, config *config.Config, log *logrus.Logger) DB { db: db, config: config, log: log, - typeConverter: typeutils.NewConverter(config, db), + typeConverter: typeutils.NewConverter(config, db, log), } go fdb.cleanupLocks() return &fdb diff --git a/internal/gtsmodel/mention.go b/internal/gtsmodel/mention.go index ce5977659..79556500f 100644 --- a/internal/gtsmodel/mention.go +++ b/internal/gtsmodel/mention.go @@ -25,19 +25,19 @@ type Mention struct { // ID of this mention in the database ID string `bun:"type:CHAR(26),pk,notnull,unique"` // ID of the status this mention originates from - StatusID string `bun:"type:CHAR(26),notnull"` + StatusID string `bun:"type:CHAR(26),notnull,nullzero"` Status *Status `bun:"rel:belongs-to"` // When was this mention created? CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` // When was this mention last updated? UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` // What's the internal account ID of the originator of the mention? - OriginAccountID string `bun:"type:CHAR(26),notnull"` + OriginAccountID string `bun:"type:CHAR(26),notnull,nullzero"` OriginAccount *Account `bun:"rel:belongs-to"` // What's the AP URI of the originator of the mention? OriginAccountURI string `bun:",notnull"` // What's the internal account ID of the mention target? - TargetAccountID string `bun:"type:CHAR(26),notnull"` + TargetAccountID string `bun:"type:CHAR(26),notnull,nullzero"` TargetAccount *Account `bun:"rel:belongs-to"` // Prevent this mention from generating a notification? Silent bool diff --git a/internal/media/handler.go b/internal/media/handler.go index 1150f7e87..b467100b0 100644 --- a/internal/media/handler.go +++ b/internal/media/handler.go @@ -80,13 +80,6 @@ type Handler interface { // in the database. ProcessLocalEmoji(ctx context.Context, emojiBytes []byte, shortcode string) (*gtsmodel.Emoji, error) - // ProcessRemoteAttachment takes a transport, a bare-bones current attachment, and an accountID that the attachment belongs to. - // It then dereferences the attachment (ie., fetches the attachment bytes from the remote server), ensuring that the bytes are - // the correct content type. It stores the attachment in whatever storage backend the Handler has been initalized with, and returns - // information to the caller about the new attachment. It's the caller's responsibility to put the returned struct - // in the database. - ProcessRemoteAttachment(ctx context.Context, t transport.Transport, currentAttachment *gtsmodel.MediaAttachment, accountID string) (*gtsmodel.MediaAttachment, error) - ProcessRemoteHeaderOrAvatar(ctx context.Context, t transport.Transport, currentAttachment *gtsmodel.MediaAttachment, accountID string) (*gtsmodel.MediaAttachment, error) } @@ -296,30 +289,6 @@ func (mh *mediaHandler) ProcessLocalEmoji(ctx context.Context, emojiBytes []byte return e, nil } -func (mh *mediaHandler) ProcessRemoteAttachment(ctx context.Context, t transport.Transport, currentAttachment *gtsmodel.MediaAttachment, accountID string) (*gtsmodel.MediaAttachment, error) { - if currentAttachment.RemoteURL == "" { - return nil, errors.New("no remote URL on media attachment to dereference") - } - remoteIRI, err := url.Parse(currentAttachment.RemoteURL) - if err != nil { - return nil, fmt.Errorf("error parsing attachment url %s: %s", currentAttachment.RemoteURL, err) - } - - // for content type, we assume we don't know what to expect... - expectedContentType := "*/*" - if currentAttachment.File.ContentType != "" { - // ... and then narrow it down if we do - expectedContentType = currentAttachment.File.ContentType - } - - attachmentBytes, err := t.DereferenceMedia(context.Background(), remoteIRI, expectedContentType) - if err != nil { - return nil, fmt.Errorf("dereferencing remote media with url %s: %s", remoteIRI.String(), err) - } - - return mh.ProcessAttachment(ctx, attachmentBytes, accountID, currentAttachment.RemoteURL) -} - func (mh *mediaHandler) ProcessRemoteHeaderOrAvatar(ctx context.Context, t transport.Transport, currentAttachment *gtsmodel.MediaAttachment, accountID string) (*gtsmodel.MediaAttachment, error) { if !currentAttachment.Header && !currentAttachment.Avatar { diff --git a/internal/typeutils/astointernal.go b/internal/typeutils/astointernal.go index 46132233b..04d9cd824 100644 --- a/internal/typeutils/astointernal.go +++ b/internal/typeutils/astointernal.go @@ -181,44 +181,62 @@ func (c *converter) ASStatusToStatus(ctx context.Context, statusable ap.Statusab } status.URI = uriProp.GetIRI().String() + l := c.log.WithField("statusURI", status.URI) + // web url for viewing this status - if statusURL, err := ap.ExtractURL(statusable); err == nil { + if statusURL, err := ap.ExtractURL(statusable); err != nil { + l.Infof("ASStatusToStatus: error extracting status URL: %s", err) + } else { status.URL = statusURL.String() } // the html-formatted content of this status - if content, err := ap.ExtractContent(statusable); err == nil { + if content, err := ap.ExtractContent(statusable); err != nil { + l.Infof("ASStatusToStatus: error extracting status content: %s", err) + } else { status.Content = content } // attachments to dereference and fetch later on (we don't do that here) - if attachments, err := ap.ExtractAttachments(statusable); err == nil { + if attachments, err := ap.ExtractAttachments(statusable); err != nil { + l.Infof("ASStatusToStatus: error extracting status attachments: %s", err) + } else { status.Attachments = attachments } // hashtags to dereference later on - if hashtags, err := ap.ExtractHashtags(statusable); err == nil { + if hashtags, err := ap.ExtractHashtags(statusable); err != nil { + l.Infof("ASStatusToStatus: error extracting status hashtags: %s", err) + } else { status.Tags = hashtags } // emojis to dereference and fetch later on - if emojis, err := ap.ExtractEmojis(statusable); err == nil { + if emojis, err := ap.ExtractEmojis(statusable); err != nil { + l.Infof("ASStatusToStatus: error extracting status emojis: %s", err) + } else { status.Emojis = emojis } // mentions to dereference later on - if mentions, err := ap.ExtractMentions(statusable); err == nil { + if mentions, err := ap.ExtractMentions(statusable); err != nil { + l.Infof("ASStatusToStatus: error extracting status mentions: %s", err) + } else { status.Mentions = mentions } // cw string for this status - if cw, err := ap.ExtractSummary(statusable); err == nil { + if cw, err := ap.ExtractSummary(statusable); err != nil { + l.Infof("ASStatusToStatus: error extracting status summary: %s", err) + } else { status.ContentWarning = cw } // when was this status created? published, err := ap.ExtractPublished(statusable) - if err == nil { + if err != nil { + l.Infof("ASStatusToStatus: error extracting status published: %s", err) + } else { status.CreatedAt = published status.UpdatedAt = published } diff --git a/internal/typeutils/astointernal_test.go b/internal/typeutils/astointernal_test.go index a01e79202..21b36a5c4 100644 --- a/internal/typeutils/astointernal_test.go +++ b/internal/typeutils/astointernal_test.go @@ -338,7 +338,7 @@ func (suite *ASToInternalTestSuite) SetupSuite() { suite.log = testrig.NewTestLog() suite.accounts = testrig.NewTestAccounts() suite.people = testrig.NewTestFediPeople() - suite.typeconverter = typeutils.NewConverter(suite.config, suite.db) + suite.typeconverter = typeutils.NewConverter(suite.config, suite.db, suite.log) } func (suite *ASToInternalTestSuite) SetupTest() { @@ -346,7 +346,7 @@ func (suite *ASToInternalTestSuite) SetupTest() { } func (suite *ASToInternalTestSuite) TestParsePerson() { - testPerson := suite.people["new_person_1"] + testPerson := suite.people["https://unknown-instance.com/users/brand_new_person"] acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), testPerson, false) assert.NoError(suite.T(), err) @@ -363,8 +363,6 @@ func (suite *ASToInternalTestSuite) TestParsePerson() { suite.Equal("https://unknown-instance.com/@brand_new_person", acct.URL) suite.True(acct.Discoverable) suite.Equal("https://unknown-instance.com/users/brand_new_person#main-key", acct.PublicKeyURI) - suite.Equal("https://unknown-instance.com/media/some_avatar_filename.jpeg", acct.AvatarRemoteURL) - suite.Equal("https://unknown-instance.com/media/some_header_filename.jpeg", acct.HeaderRemoteURL) suite.False(acct.Locked) } diff --git a/internal/typeutils/converter.go b/internal/typeutils/converter.go index 4af9767bc..630e48300 100644 --- a/internal/typeutils/converter.go +++ b/internal/typeutils/converter.go @@ -23,6 +23,7 @@ "net/url" "github.com/go-fed/activity/streams/vocab" + "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/cache" @@ -179,15 +180,17 @@ type TypeConverter interface { type converter struct { config *config.Config db db.DB + log *logrus.Logger frontendCache cache.Cache asCache cache.Cache } // NewConverter returns a new Converter -func NewConverter(config *config.Config, db db.DB) TypeConverter { +func NewConverter(config *config.Config, db db.DB, log *logrus.Logger) TypeConverter { return &converter{ config: config, db: db, + log: log, frontendCache: cache.New(), asCache: cache.New(), } diff --git a/internal/typeutils/converter_test.go b/internal/typeutils/converter_test.go index c104ab06c..63f3fe705 100644 --- a/internal/typeutils/converter_test.go +++ b/internal/typeutils/converter_test.go @@ -19,9 +19,9 @@ package typeutils_test import ( + "github.com/go-fed/activity/streams/vocab" "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -35,7 +35,7 @@ type ConverterStandardTestSuite struct { db db.DB log *logrus.Logger accounts map[string]*gtsmodel.Account - people map[string]ap.Accountable + people map[string]vocab.ActivityStreamsPerson typeconverter typeutils.TypeConverter } diff --git a/internal/typeutils/internaltoas.go b/internal/typeutils/internaltoas.go index 14ed094c5..776cff24f 100644 --- a/internal/typeutils/internaltoas.go +++ b/internal/typeutils/internaltoas.go @@ -27,7 +27,6 @@ "github.com/go-fed/activity/streams" "github.com/go-fed/activity/streams/vocab" - "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) @@ -600,12 +599,12 @@ func (c *converter) FollowToAS(ctx context.Context, f *gtsmodel.Follow, originAc } func (c *converter) MentionToAS(ctx context.Context, m *gtsmodel.Mention) (vocab.ActivityStreamsMention, error) { - if m.OriginAccount == nil { - a := >smodel.Account{} - if err := c.db.GetWhere(ctx, []db.Where{{Key: "target_account_id", Value: m.TargetAccountID}}, a); err != nil { + if m.TargetAccount == nil { + a, err := c.db.GetAccountByID(ctx, m.TargetAccountID) + if err != nil { return nil, fmt.Errorf("MentionToAS: error getting target account from db: %s", err) } - m.OriginAccount = a + m.TargetAccount = a } // create the mention @@ -613,21 +612,21 @@ func (c *converter) MentionToAS(ctx context.Context, m *gtsmodel.Mention) (vocab // href -- this should be the URI of the mentioned user hrefProp := streams.NewActivityStreamsHrefProperty() - hrefURI, err := url.Parse(m.OriginAccount.URI) + hrefURI, err := url.Parse(m.TargetAccount.URI) if err != nil { - return nil, fmt.Errorf("MentionToAS: error parsing uri %s: %s", m.OriginAccount.URI, err) + return nil, fmt.Errorf("MentionToAS: error parsing uri %s: %s", m.TargetAccount.URI, err) } hrefProp.SetIRI(hrefURI) mention.SetActivityStreamsHref(hrefProp) // name -- this should be the namestring of the mentioned user, something like @whatever@example.org var domain string - if m.OriginAccount.Domain == "" { + if m.TargetAccount.Domain == "" { domain = c.config.AccountDomain } else { - domain = m.OriginAccount.Domain + domain = m.TargetAccount.Domain } - username := m.OriginAccount.Username + username := m.TargetAccount.Username nameString := fmt.Sprintf("@%s@%s", username, domain) nameProp := streams.NewActivityStreamsNameProperty() nameProp.AppendXMLSchemaString(nameString) @@ -684,8 +683,8 @@ func (c *converter) AttachmentToAS(ctx context.Context, a *gtsmodel.MediaAttachm func (c *converter) FaveToAS(ctx context.Context, f *gtsmodel.StatusFave) (vocab.ActivityStreamsLike, error) { // check if targetStatus is already pinned to this fave, and fetch it if not if f.Status == nil { - s := >smodel.Status{} - if err := c.db.GetByID(ctx, f.StatusID, s); err != nil { + s, err := c.db.GetStatusByID(ctx, f.StatusID) + if err != nil { return nil, fmt.Errorf("FaveToAS: error fetching target status from database: %s", err) } f.Status = s @@ -693,8 +692,8 @@ func (c *converter) FaveToAS(ctx context.Context, f *gtsmodel.StatusFave) (vocab // check if the targetAccount is already pinned to this fave, and fetch it if not if f.TargetAccount == nil { - a := >smodel.Account{} - if err := c.db.GetByID(ctx, f.TargetAccountID, a); err != nil { + a, err := c.db.GetAccountByID(ctx, f.TargetAccountID) + if err != nil { return nil, fmt.Errorf("FaveToAS: error fetching target account from database: %s", err) } f.TargetAccount = a @@ -702,8 +701,8 @@ func (c *converter) FaveToAS(ctx context.Context, f *gtsmodel.StatusFave) (vocab // check if the faving account is already pinned to this fave, and fetch it if not if f.Account == nil { - a := >smodel.Account{} - if err := c.db.GetByID(ctx, f.AccountID, a); err != nil { + a, err := c.db.GetAccountByID(ctx, f.AccountID) + if err != nil { return nil, fmt.Errorf("FaveToAS: error fetching faving account from database: %s", err) } f.Account = a @@ -754,8 +753,8 @@ func (c *converter) FaveToAS(ctx context.Context, f *gtsmodel.StatusFave) (vocab func (c *converter) BoostToAS(ctx context.Context, boostWrapperStatus *gtsmodel.Status, boostingAccount *gtsmodel.Account, boostedAccount *gtsmodel.Account) (vocab.ActivityStreamsAnnounce, error) { // the boosted status is probably pinned to the boostWrapperStatus but double check to make sure if boostWrapperStatus.BoostOf == nil { - b := >smodel.Status{} - if err := c.db.GetByID(ctx, boostWrapperStatus.BoostOfID, b); err != nil { + b, err := c.db.GetStatusByID(ctx, boostWrapperStatus.BoostOfID) + if err != nil { return nil, fmt.Errorf("BoostToAS: error getting status with ID %s from the db: %s", boostWrapperStatus.BoostOfID, err) } boostWrapperStatus.BoostOf = b @@ -837,16 +836,16 @@ func (c *converter) BoostToAS(ctx context.Context, boostWrapperStatus *gtsmodel. */ func (c *converter) BlockToAS(ctx context.Context, b *gtsmodel.Block) (vocab.ActivityStreamsBlock, error) { if b.Account == nil { - a := >smodel.Account{} - if err := c.db.GetByID(ctx, b.AccountID, a); err != nil { - return nil, fmt.Errorf("BlockToAS: error getting block account from database: %s", err) + a, err := c.db.GetAccountByID(ctx, b.AccountID) + if err != nil { + return nil, fmt.Errorf("BlockToAS: error getting block owner account from database: %s", err) } b.Account = a } if b.TargetAccount == nil { - a := >smodel.Account{} - if err := c.db.GetByID(ctx, b.TargetAccountID, a); err != nil { + a, err := c.db.GetAccountByID(ctx, b.TargetAccountID) + if err != nil { return nil, fmt.Errorf("BlockToAS: error getting block target account from database: %s", err) } b.TargetAccount = a diff --git a/internal/typeutils/internaltoas_test.go b/internal/typeutils/internaltoas_test.go index 46f04df2f..de046fe0a 100644 --- a/internal/typeutils/internaltoas_test.go +++ b/internal/typeutils/internaltoas_test.go @@ -44,7 +44,7 @@ func (suite *InternalToASTestSuite) SetupSuite() { suite.log = testrig.NewTestLog() suite.accounts = testrig.NewTestAccounts() suite.people = testrig.NewTestFediPeople() - suite.typeconverter = typeutils.NewConverter(suite.config, suite.db) + suite.typeconverter = typeutils.NewConverter(suite.config, suite.db, suite.log) } func (suite *InternalToASTestSuite) SetupTest() { diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index 89da9eb01..03aa0c77b 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -316,6 +316,8 @@ func (c *converter) TagToMasto(ctx context.Context, t *gtsmodel.Tag) (model.Tag, } func (c *converter) StatusToMasto(ctx context.Context, s *gtsmodel.Status, requestingAccount *gtsmodel.Account) (*model.Status, error) { + l := c.log + repliesCount, err := c.db.CountStatusReplies(ctx, s) if err != nil { return nil, fmt.Errorf("error counting replies: %s", err) @@ -392,7 +394,8 @@ func (c *converter) StatusToMasto(ctx context.Context, s *gtsmodel.Status, reque for _, gtsAttachment := range s.Attachments { mastoAttachment, err := c.AttachmentToMasto(ctx, gtsAttachment) if err != nil { - return nil, fmt.Errorf("error converting attachment with id %s: %s", gtsAttachment.ID, err) + l.Errorf("error converting attachment with id %s: %s", gtsAttachment.ID, err) + continue } mastoAttachments = append(mastoAttachments, mastoAttachment) } @@ -402,11 +405,13 @@ func (c *converter) StatusToMasto(ctx context.Context, s *gtsmodel.Status, reque for _, aID := range s.AttachmentIDs { gtsAttachment, err := c.db.GetAttachmentByID(ctx, aID) if err != nil { - return nil, fmt.Errorf("error getting attachment with id %s: %s", aID, err) + l.Errorf("error getting attachment with id %s: %s", aID, err) + continue } mastoAttachment, err := c.AttachmentToMasto(ctx, gtsAttachment) if err != nil { - return nil, fmt.Errorf("error converting attachment with id %s: %s", aID, err) + l.Errorf("error converting attachment with id %s: %s", aID, err) + continue } mastoAttachments = append(mastoAttachments, mastoAttachment) } @@ -419,7 +424,8 @@ func (c *converter) StatusToMasto(ctx context.Context, s *gtsmodel.Status, reque for _, gtsMention := range s.Mentions { mastoMention, err := c.MentionToMasto(ctx, gtsMention) if err != nil { - return nil, fmt.Errorf("error converting mention with id %s: %s", gtsMention.ID, err) + l.Errorf("error converting mention with id %s: %s", gtsMention.ID, err) + continue } mastoMentions = append(mastoMentions, mastoMention) } @@ -429,11 +435,13 @@ func (c *converter) StatusToMasto(ctx context.Context, s *gtsmodel.Status, reque for _, mID := range s.MentionIDs { gtsMention, err := c.db.GetMention(ctx, mID) if err != nil { - return nil, fmt.Errorf("error getting mention with id %s: %s", mID, err) + l.Errorf("error getting mention with id %s: %s", mID, err) + continue } mastoMention, err := c.MentionToMasto(ctx, gtsMention) if err != nil { - return nil, fmt.Errorf("error converting mention with id %s: %s", gtsMention.ID, err) + l.Errorf("error converting mention with id %s: %s", gtsMention.ID, err) + continue } mastoMentions = append(mastoMentions, mastoMention) } @@ -446,7 +454,8 @@ func (c *converter) StatusToMasto(ctx context.Context, s *gtsmodel.Status, reque for _, gtsTag := range s.Tags { mastoTag, err := c.TagToMasto(ctx, gtsTag) if err != nil { - return nil, fmt.Errorf("error converting tag with id %s: %s", gtsTag.ID, err) + l.Errorf("error converting tag with id %s: %s", gtsTag.ID, err) + continue } mastoTags = append(mastoTags, mastoTag) } @@ -456,11 +465,13 @@ func (c *converter) StatusToMasto(ctx context.Context, s *gtsmodel.Status, reque for _, t := range s.TagIDs { gtsTag := >smodel.Tag{} if err := c.db.GetByID(ctx, t, gtsTag); err != nil { - return nil, fmt.Errorf("error getting tag with id %s: %s", t, err) + l.Errorf("error getting tag with id %s: %s", t, err) + continue } mastoTag, err := c.TagToMasto(ctx, gtsTag) if err != nil { - return nil, fmt.Errorf("error converting tag with id %s: %s", gtsTag.ID, err) + l.Errorf("error converting tag with id %s: %s", gtsTag.ID, err) + continue } mastoTags = append(mastoTags, mastoTag) } @@ -473,7 +484,8 @@ func (c *converter) StatusToMasto(ctx context.Context, s *gtsmodel.Status, reque for _, gtsEmoji := range s.Emojis { mastoEmoji, err := c.EmojiToMasto(ctx, gtsEmoji) if err != nil { - return nil, fmt.Errorf("error converting emoji with id %s: %s", gtsEmoji.ID, err) + l.Errorf("error converting emoji with id %s: %s", gtsEmoji.ID, err) + continue } mastoEmojis = append(mastoEmojis, mastoEmoji) } @@ -483,11 +495,13 @@ func (c *converter) StatusToMasto(ctx context.Context, s *gtsmodel.Status, reque for _, e := range s.EmojiIDs { gtsEmoji := >smodel.Emoji{} if err := c.db.GetByID(ctx, e, gtsEmoji); err != nil { - return nil, fmt.Errorf("error getting emoji with id %s: %s", e, err) + l.Errorf("error getting emoji with id %s: %s", e, err) + continue } mastoEmoji, err := c.EmojiToMasto(ctx, gtsEmoji) if err != nil { - return nil, fmt.Errorf("error converting emoji with id %s: %s", gtsEmoji.ID, err) + l.Errorf("error converting emoji with id %s: %s", gtsEmoji.ID, err) + continue } mastoEmojis = append(mastoEmojis, mastoEmoji) } diff --git a/internal/util/regexes.go b/internal/util/regexes.go index c03fd878c..88212fc43 100644 --- a/internal/util/regexes.go +++ b/internal/util/regexes.go @@ -30,7 +30,7 @@ ) var ( - mentionNameRegexString = `^@(\w+)(?:@([a-zA-Z0-9_\-\.]+)?)$` + mentionNameRegexString = `^@(\w+)(?:@([a-zA-Z0-9_\-\.:]+)?)$` // mention name regex captures the username and domain part from a mention string // such as @whatever_user@example.org, returning whatever_user and example.org (without the @ symbols) mentionNameRegex = regexp.MustCompile(mentionNameRegexString) diff --git a/testrig/testmodels.go b/testrig/testmodels.go index 29a164c50..c3ff25e57 100644 --- a/testrig/testmodels.go +++ b/testrig/testmodels.go @@ -36,7 +36,6 @@ "github.com/go-fed/activity/pub" "github.com/go-fed/activity/streams" "github.com/go-fed/activity/streams/vocab" - "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/oauth" ) @@ -1224,12 +1223,14 @@ func NewTestActivities(accounts map[string]*gtsmodel.Account) map[string]Activit dmForZork := newNote( URLMustParse("https://fossbros-anonymous.io/users/foss_satan/statuses/5424b153-4553-4f30-9358-7b92f7cd42f6"), URLMustParse("https://fossbros-anonymous.io/@foss_satan/5424b153-4553-4f30-9358-7b92f7cd42f6"), + time.Now(), "hey zork here's a new private note for you", "new note for zork", URLMustParse("https://fossbros-anonymous.io/users/foss_satan"), []*url.URL{URLMustParse("http://localhost:8080/users/the_mighty_zork")}, nil, - true) + true, + []vocab.ActivityStreamsMention{}) createDmForZork := wrapNoteInCreate( URLMustParse("https://fossbros-anonymous.io/users/foss_satan/statuses/5424b153-4553-4f30-9358-7b92f7cd42f6/activity"), URLMustParse("https://fossbros-anonymous.io/users/foss_satan"), @@ -1248,15 +1249,15 @@ func NewTestActivities(accounts map[string]*gtsmodel.Account) map[string]Activit } // NewTestFediPeople returns a bunch of activity pub Person representations for testing converters and so on. -func NewTestFediPeople() map[string]ap.Accountable { +func NewTestFediPeople() map[string]vocab.ActivityStreamsPerson { newPerson1Priv, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { panic(err) } newPerson1Pub := &newPerson1Priv.PublicKey - return map[string]ap.Accountable{ - "new_person_1": newPerson( + return map[string]vocab.ActivityStreamsPerson{ + "https://unknown-instance.com/users/brand_new_person": newPerson( URLMustParse("https://unknown-instance.com/users/brand_new_person"), URLMustParse("https://unknown-instance.com/users/brand_new_person/following"), URLMustParse("https://unknown-instance.com/users/brand_new_person/followers"), @@ -1270,15 +1271,53 @@ func NewTestFediPeople() map[string]ap.Accountable { true, URLMustParse("https://unknown-instance.com/users/brand_new_person#main-key"), newPerson1Pub, - URLMustParse("https://unknown-instance.com/media/some_avatar_filename.jpeg"), + nil, "image/jpeg", - URLMustParse("https://unknown-instance.com/media/some_header_filename.jpeg"), + nil, "image/png", false, ), } } +func NewTestFediStatuses() map[string]vocab.ActivityStreamsNote { + return map[string]vocab.ActivityStreamsNote{ + "https://unknown-instance.com/users/brand_new_person/statuses/01FE4NTHKWW7THT67EF10EB839": newNote( + URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01FE4NTHKWW7THT67EF10EB839"), + URLMustParse("https://unknown-instance.com/users/@brand_new_person/01FE4NTHKWW7THT67EF10EB839"), + time.Now(), + "Hello world!", + "", + URLMustParse("https://unknown-instance.com/users/brand_new_person"), + []*url.URL{ + URLMustParse("https://www.w3.org/ns/activitystreams#Public"), + }, + []*url.URL{}, + false, + []vocab.ActivityStreamsMention{}, + ), + "https://unknown-instance.com/users/brand_new_person/statuses/01FE5Y30E3W4P7TRE0R98KAYQV": newNote( + URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01FE5Y30E3W4P7TRE0R98KAYQV"), + URLMustParse("https://unknown-instance.com/users/@brand_new_person/01FE5Y30E3W4P7TRE0R98KAYQV"), + time.Now(), + "Hey @the_mighty_zork@localhost:8080 how's it going?", + "", + URLMustParse("https://unknown-instance.com/users/brand_new_person"), + []*url.URL{ + URLMustParse("https://www.w3.org/ns/activitystreams#Public"), + }, + []*url.URL{}, + false, + []vocab.ActivityStreamsMention{ + newMention( + URLMustParse("http://localhost:8080/users/the_mighty_zork"), + "@the_mighty_zork@localhost:8080", + ), + }, + ), + } +} + // NewTestDereferenceRequests returns a map of incoming dereference requests, with their signatures. func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[string]ActivityWithSignature { var sig, digest, date string @@ -1418,7 +1457,7 @@ func newPerson( avatarContentType string, headerURL *url.URL, headerContentType string, - manuallyApprovesFollowers bool) ap.Accountable { + manuallyApprovesFollowers bool) vocab.ActivityStreamsPerson { person := streams.NewActivityStreamsPerson() // id should be the activitypub URI of this user @@ -1583,16 +1622,32 @@ func newPerson( return person } +func newMention(uri *url.URL, namestring string) vocab.ActivityStreamsMention { + mention := streams.NewActivityStreamsMention() + + hrefProp := streams.NewActivityStreamsHrefProperty() + hrefProp.SetIRI(uri) + mention.SetActivityStreamsHref(hrefProp) + + nameProp := streams.NewActivityStreamsNameProperty() + nameProp.AppendXMLSchemaString(namestring) + mention.SetActivityStreamsName(nameProp) + + return mention +} + // newNote returns a new activity streams note for the given parameters func newNote( noteID *url.URL, noteURL *url.URL, + noteCreatedAt time.Time, noteContent string, noteSummary string, noteAttributedTo *url.URL, noteTo []*url.URL, noteCC []*url.URL, - noteSensitive bool) vocab.ActivityStreamsNote { + noteSensitive bool, + noteMentions []vocab.ActivityStreamsMention) vocab.ActivityStreamsNote { // create the note itself note := streams.NewActivityStreamsNote() @@ -1611,6 +1666,13 @@ func newNote( note.SetActivityStreamsUrl(url) } + if noteCreatedAt.IsZero() { + noteCreatedAt = time.Now() + } + published := streams.NewActivityStreamsPublishedProperty() + published.Set(noteCreatedAt) + note.SetActivityStreamsPublished(published) + // set noteContent if noteContent != "" { content := streams.NewActivityStreamsContentProperty() @@ -1632,6 +1694,34 @@ func newNote( note.SetActivityStreamsAttributedTo(attributedTo) } + // set noteTO + if noteTo != nil { + to := streams.NewActivityStreamsToProperty() + for _, r := range noteTo { + to.AppendIRI(r) + } + note.SetActivityStreamsTo(to) + } + + // set noteCC + if noteCC != nil { + cc := streams.NewActivityStreamsCcProperty() + for _, r := range noteCC { + cc.AppendIRI(r) + } + note.SetActivityStreamsCc(cc) + } + + // set note tags + tag := streams.NewActivityStreamsTagProperty() + + // mentions + for _, m := range noteMentions { + tag.AppendActivityStreamsMention(m) + } + + note.SetActivityStreamsTag(tag) + return note } diff --git a/testrig/typeconverter.go b/testrig/typeconverter.go index 9d49e6c99..c1561a350 100644 --- a/testrig/typeconverter.go +++ b/testrig/typeconverter.go @@ -25,5 +25,5 @@ // NewTestTypeConverter returned a type converter with the given db and the default test config func NewTestTypeConverter(db db.DB) typeutils.TypeConverter { - return typeutils.NewConverter(NewTestConfig(), db) + return typeutils.NewConverter(NewTestConfig(), db, NewTestLog()) }