gotosocial/internal/federation/federatingdb/create.go

424 lines
13 KiB
Go
Raw Normal View History

// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package federatingdb
import (
"context"
"errors"
"fmt"
[chore] use our own logging implementation (#716) * first commit Signed-off-by: kim <grufwub@gmail.com> * replace logging with our own log library Signed-off-by: kim <grufwub@gmail.com> * fix imports Signed-off-by: kim <grufwub@gmail.com> * fix log imports Signed-off-by: kim <grufwub@gmail.com> * add license text Signed-off-by: kim <grufwub@gmail.com> * fix package import cycle between config and log package Signed-off-by: kim <grufwub@gmail.com> * fix empty kv.Fields{} being passed to WithFields() Signed-off-by: kim <grufwub@gmail.com> * fix uses of log.WithFields() with whitespace issues and empty slices Signed-off-by: kim <grufwub@gmail.com> * *linter related grumbling* Signed-off-by: kim <grufwub@gmail.com> * gofmt the codebase! also fix more log.WithFields() formatting issues Signed-off-by: kim <grufwub@gmail.com> * update testrig code to match new changes Signed-off-by: kim <grufwub@gmail.com> * fix error wrapping in non fmt.Errorf function Signed-off-by: kim <grufwub@gmail.com> * add benchmarking of log.Caller() vs non-cached Signed-off-by: kim <grufwub@gmail.com> * fix syslog tests, add standard build tags to test runner to ensure consistency Signed-off-by: kim <grufwub@gmail.com> * make syslog tests more robust Signed-off-by: kim <grufwub@gmail.com> * fix caller depth arithmatic (is that how you spell it?) Signed-off-by: kim <grufwub@gmail.com> * update to use unkeyed fields in kv.Field{} instances Signed-off-by: kim <grufwub@gmail.com> * update go-kv library Signed-off-by: kim <grufwub@gmail.com> * update libraries list Signed-off-by: kim <grufwub@gmail.com> * fuck you linter get nerfed Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tobi <31960611+tsmethurst@users.noreply.github.com>
2022-07-19 08:47:55 +00:00
"codeberg.org/gruf/go-logger/v2/level"
"github.com/superseriousbusiness/activity/pub"
2021-11-13 16:29:43 +00:00
"github.com/superseriousbusiness/activity/streams/vocab"
2021-08-31 13:59:12 +00:00
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
[chore] use our own logging implementation (#716) * first commit Signed-off-by: kim <grufwub@gmail.com> * replace logging with our own log library Signed-off-by: kim <grufwub@gmail.com> * fix imports Signed-off-by: kim <grufwub@gmail.com> * fix log imports Signed-off-by: kim <grufwub@gmail.com> * add license text Signed-off-by: kim <grufwub@gmail.com> * fix package import cycle between config and log package Signed-off-by: kim <grufwub@gmail.com> * fix empty kv.Fields{} being passed to WithFields() Signed-off-by: kim <grufwub@gmail.com> * fix uses of log.WithFields() with whitespace issues and empty slices Signed-off-by: kim <grufwub@gmail.com> * *linter related grumbling* Signed-off-by: kim <grufwub@gmail.com> * gofmt the codebase! also fix more log.WithFields() formatting issues Signed-off-by: kim <grufwub@gmail.com> * update testrig code to match new changes Signed-off-by: kim <grufwub@gmail.com> * fix error wrapping in non fmt.Errorf function Signed-off-by: kim <grufwub@gmail.com> * add benchmarking of log.Caller() vs non-cached Signed-off-by: kim <grufwub@gmail.com> * fix syslog tests, add standard build tags to test runner to ensure consistency Signed-off-by: kim <grufwub@gmail.com> * make syslog tests more robust Signed-off-by: kim <grufwub@gmail.com> * fix caller depth arithmatic (is that how you spell it?) Signed-off-by: kim <grufwub@gmail.com> * update to use unkeyed fields in kv.Field{} instances Signed-off-by: kim <grufwub@gmail.com> * update go-kv library Signed-off-by: kim <grufwub@gmail.com> * update libraries list Signed-off-by: kim <grufwub@gmail.com> * fuck you linter get nerfed Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tobi <31960611+tsmethurst@users.noreply.github.com>
2022-07-19 08:47:55 +00:00
"github.com/superseriousbusiness/gotosocial/internal/log"
2021-08-31 13:59:12 +00:00
"github.com/superseriousbusiness/gotosocial/internal/messages"
)
// Create adds a new entry to the database which must be able to be
// keyed by its id.
//
// Note that Activity values received from federated peers may also be
// created in the database this way if the Federating Protocol is
// enabled. The client may freely decide to store only the id instead of
// the entire value.
//
// The library makes this call only after acquiring a lock first.
//
// Under certain conditions and network activities, Create may be called
// multiple times for the same ActivityStreams object.
func (f *federatingDB) Create(ctx context.Context, asType vocab.Type) error {
if log.Level() >= level.TRACE {
i, err := marshalItem(asType)
if err != nil {
return err
}
log.
WithContext(ctx).
WithField("create", i).
Trace("entering Create")
}
receivingAccount, requestingAccount, internal := extractFromCtx(ctx)
if internal {
return nil // Already processed.
}
switch asType.GetTypeName() {
case ap.ActivityBlock:
// BLOCK SOMETHING
return f.activityBlock(ctx, asType, receivingAccount, requestingAccount)
2021-08-31 13:59:12 +00:00
case ap.ActivityCreate:
// CREATE SOMETHING
return f.activityCreate(ctx, asType, receivingAccount, requestingAccount)
2021-08-31 13:59:12 +00:00
case ap.ActivityFollow:
// FOLLOW SOMETHING
return f.activityFollow(ctx, asType, receivingAccount, requestingAccount)
case ap.ActivityLike:
// LIKE SOMETHING
return f.activityLike(ctx, asType, receivingAccount, requestingAccount)
case ap.ActivityFlag:
// FLAG / REPORT SOMETHING
return f.activityFlag(ctx, asType, receivingAccount, requestingAccount)
}
return nil
}
/*
BLOCK HANDLERS
*/
func (f *federatingDB) activityBlock(ctx context.Context, asType vocab.Type, receiving *gtsmodel.Account, requestingAccount *gtsmodel.Account) error {
blockable, ok := asType.(vocab.ActivityStreamsBlock)
if !ok {
return errors.New("activityBlock: could not convert type to block")
}
block, err := f.typeConverter.ASBlockToBlock(ctx, blockable)
if err != nil {
return fmt.Errorf("activityBlock: could not convert Block to gts model block")
}
block.ID = id.NewULID()
if err := f.state.DB.PutBlock(ctx, block); err != nil {
return fmt.Errorf("activityBlock: database error inserting block: %s", err)
}
f.state.Workers.EnqueueFederator(ctx, messages.FromFederator{
APObjectType: ap.ActivityBlock,
APActivityType: ap.ActivityCreate,
GTSModel: block,
ReceivingAccount: receiving,
})
return nil
}
/*
CREATE HANDLERS
*/
// activityCreate handles asType Create by checking
// the Object entries of the Create and calling other
// handlers as appropriate.
func (f *federatingDB) activityCreate(
ctx context.Context,
asType vocab.Type,
receivingAccount *gtsmodel.Account,
requestingAccount *gtsmodel.Account,
) error {
create, ok := asType.(vocab.ActivityStreamsCreate)
if !ok {
return gtserror.Newf("could not convert asType %T to ActivityStreamsCreate", asType)
}
// Create must have an Object.
objectProp := create.GetActivityStreamsObject()
if objectProp == nil {
return gtserror.New("create had no Object")
}
// Iterate through the Object property and process FIRST provided statusable.
// todo: https://github.com/superseriousbusiness/gotosocial/issues/1905
for iter := objectProp.Begin(); iter != objectProp.End(); iter = iter.Next() {
object := iter.GetType()
if object == nil {
// Can't do Create with Object that's just a URI.
// Warn log this because it's an AP error.
log.Warn(ctx, "object entry was not a type: %[1]T%[1]+v", iter)
continue
}
// Ensure given object type is a statusable.
statusable, ok := object.(ap.Statusable)
if !ok {
// Can't (currently) Create anything other than a Statusable. ([1] is a format arg index)
log.Debugf(ctx, "object entry type (currently) unsupported: %[1]T%[1]+v", object)
continue
}
// Handle creation of statusable.
return f.createStatusable(ctx,
statusable,
receivingAccount,
requestingAccount,
)
}
return nil
}
// createStatusable handles a Create activity for a Statusable.
// This function won't insert anything in the database yet,
// but will pass the Statusable (if appropriate) through to
// the processor for further asynchronous processing.
func (f *federatingDB) createStatusable(
ctx context.Context,
statusable ap.Statusable,
receivingAccount *gtsmodel.Account,
requestingAccount *gtsmodel.Account,
) error {
// Statusable must have an attributedTo.
attrToProp := statusable.GetActivityStreamsAttributedTo()
if attrToProp == nil {
return gtserror.Newf("statusable had no attributedTo")
}
// Statusable must have an ID.
idProp := statusable.GetJSONLDId()
if idProp == nil || !idProp.IsIRI() {
return gtserror.Newf("statusable had no id, or id was not a URI")
}
statusableURI := idProp.GetIRI()
// Check if we have a forward. In other words, was the
// statusable posted to our inbox by at least one actor
// who actually created it, or are they forwarding it?
forward := true
for iter := attrToProp.Begin(); iter != attrToProp.End(); iter = iter.Next() {
actorURI, err := pub.ToId(iter)
if err != nil {
return gtserror.Newf("error extracting id from attributedTo entry: %w", err)
}
if requestingAccount.URI == actorURI.String() {
// The actor who posted this statusable to our inbox is
// (one of) its creator(s), so this is not a forward.
forward = false
break
}
}
// Check if we already have a status entry
// for this statusable, based on the ID/URI.
statusableURIStr := statusableURI.String()
status, err := f.state.DB.GetStatusByURI(ctx, statusableURIStr)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return gtserror.Newf("db error checking existence of status %s: %w", statusableURIStr, err)
}
if status != nil {
// We already had this status in the db, no need for further action.
log.Trace(ctx, "status already exists: %s", statusableURIStr)
return nil
}
// If we do have a forward, we should ignore the content
// and instead deref based on the URI of the statusable.
//
// In other words, don't automatically trust whoever sent
// this status to us, but fetch the authentic article from
// the server it originated from.
if forward {
// Pass the statusable URI (APIri) into the processor worker
// and do the rest of the processing asynchronously.
f.state.Workers.EnqueueFederator(ctx, messages.FromFederator{
APObjectType: ap.ObjectNote,
2021-08-31 13:59:12 +00:00
APActivityType: ap.ActivityCreate,
APIri: statusableURI,
[feature] status refetch support (#1690) * revamp http client to not limit requests, instead use sender worker Signed-off-by: kim <grufwub@gmail.com> * remove separate sender worker pool, spawn 2*GOMAXPROCS batch senders each time, no need for transport cache sweeping Signed-off-by: kim <grufwub@gmail.com> * improve batch senders to keep popping recipients until remote URL found Signed-off-by: kim <grufwub@gmail.com> * fix recipient looping issue Signed-off-by: kim <grufwub@gmail.com> * move request id ctx key to gtscontext, finish filling out more code comments, add basic support for not logging client IP Signed-off-by: kim <grufwub@gmail.com> * first draft of status refetching logic Signed-off-by: kim <grufwub@gmail.com> * fix testrig to use new federation alloc func signature Signed-off-by: kim <grufwub@gmail.com> * fix log format directive Signed-off-by: kim <grufwub@gmail.com> * add status fetched_at migration Signed-off-by: kim <grufwub@gmail.com> * remove unused / unchecked for error types Signed-off-by: kim <grufwub@gmail.com> * add back the used type... Signed-off-by: kim <grufwub@gmail.com> * add separate internal getStatus() function for derefThread() that doesn't recurse Signed-off-by: kim <grufwub@gmail.com> * improved mention and media attachment error handling Signed-off-by: kim <grufwub@gmail.com> * fix log and error format directives Signed-off-by: kim <grufwub@gmail.com> * update account deref to match status deref changes Signed-off-by: kim <grufwub@gmail.com> * very small code formatting change to make things clearer Signed-off-by: kim <grufwub@gmail.com> * add more code comments Signed-off-by: kim <grufwub@gmail.com> * improved code commenting Signed-off-by: kim <grufwub@gmail.com> * only check for required further derefs if needed Signed-off-by: kim <grufwub@gmail.com> * improved cache invalidation Signed-off-by: kim <grufwub@gmail.com> * tweak cache restarting to use a (very small) backoff Signed-off-by: kim <grufwub@gmail.com> * small readability changes and fixes Signed-off-by: kim <grufwub@gmail.com> * fix account sync issues Signed-off-by: kim <grufwub@gmail.com> * fix merge conflicts + update account enrichment to accept already-passed accountable Signed-off-by: kim <grufwub@gmail.com> * remove secondary function declaration Signed-off-by: kim <grufwub@gmail.com> * normalise dereferencer get status / account behaviour, fix remaining tests Signed-off-by: kim <grufwub@gmail.com> * fix remaining rebase conflicts, finish commenting code Signed-off-by: kim <grufwub@gmail.com> * appease the linter Signed-off-by: kim <grufwub@gmail.com> * add source file header Signed-off-by: kim <grufwub@gmail.com> * update to use TIMESTAMPTZ column type instead of just TIMESTAMP Signed-off-by: kim <grufwub@gmail.com> * don't pass in 'updated_at' to UpdateEmoji() Signed-off-by: kim <grufwub@gmail.com> * use new ap.Resolve{Account,Status}able() functions Signed-off-by: kim <grufwub@gmail.com> * remove the somewhat confusing rescoping of the same variable names Signed-off-by: kim <grufwub@gmail.com> * update migration file name, improved database delete error returns Signed-off-by: kim <grufwub@gmail.com> * formatting Signed-off-by: kim <grufwub@gmail.com> * improved multi-delete database functions to minimise DB calls Signed-off-by: kim <grufwub@gmail.com> * remove unused type Signed-off-by: kim <grufwub@gmail.com> * fix delete statements Signed-off-by: kim <grufwub@gmail.com> --------- Signed-off-by: kim <grufwub@gmail.com>
2023-05-12 09:15:54 +00:00
APObjectModel: nil,
GTSModel: nil,
ReceivingAccount: receivingAccount,
})
return nil
}
// This is a non-forwarded status we can trust the requester on,
// convert this provided statusable data to a useable gtsmodel status.
status, err = f.typeConverter.ASStatusToStatus(ctx, statusable)
if err != nil {
return gtserror.Newf("error converting statusable to status: %w", err)
}
// Check whether we should accept this new status.
accept, err := f.shouldAcceptStatusable(ctx,
receivingAccount,
requestingAccount,
status,
)
if err != nil {
return gtserror.Newf("error checking status acceptibility: %w", err)
}
if !accept {
// This is a status sent with no relation to receiver, i.e.
// - receiving account does not follow requesting account
// - received status does not mention receiving account
//
// We just pretend that all is fine (dog with cuppa, flames everywhere)
log.Trace(ctx, "status failed acceptability check")
return nil
}
// ID the new status based on the time it was created.
status.ID, err = id.NewULIDFromTime(status.CreatedAt)
if err != nil {
return err
}
// Put this newly parsed status in the database.
if err := f.state.DB.PutStatus(ctx, status); err != nil {
if errors.Is(err, db.ErrAlreadyExists) {
// The status already exists in the database, which
// means we've already processed it and some race
// condition means we didn't catch it yet. We can
// just return nil here and be done with it.
return nil
}
return gtserror.Newf("db error inserting status: %w", err)
}
// Do the rest of the processing asynchronously. The processor
// will handle inserting/updating + further dereferencing the status.
f.state.Workers.EnqueueFederator(ctx, messages.FromFederator{
APObjectType: ap.ObjectNote,
APActivityType: ap.ActivityCreate,
APIri: nil,
APObjectModel: statusable,
GTSModel: status,
ReceivingAccount: receivingAccount,
})
return nil
}
func (f *federatingDB) shouldAcceptStatusable(ctx context.Context, receiver *gtsmodel.Account, requester *gtsmodel.Account, status *gtsmodel.Status) (bool, error) {
// Check whether status mentions the receiver,
// this is the quickest check so perform it first.
for _, mention := range status.Mentions {
if mention.TargetAccountURI == receiver.URI {
return true, nil
}
}
// Check whether receiving account follows the requesting account.
follows, err := f.state.DB.IsFollowing(ctx, receiver.ID, requester.ID)
if err != nil {
return false, gtserror.Newf("error checking follow status: %w", err)
}
// Status will only be acceptable
// if receiver follows requester.
return follows, nil
}
/*
FOLLOW HANDLERS
*/
func (f *federatingDB) activityFollow(ctx context.Context, asType vocab.Type, receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account) error {
follow, ok := asType.(vocab.ActivityStreamsFollow)
if !ok {
return errors.New("activityFollow: could not convert type to follow")
}
followRequest, err := f.typeConverter.ASFollowToFollowRequest(ctx, follow)
if err != nil {
return fmt.Errorf("activityFollow: could not convert Follow to follow request: %s", err)
}
followRequest.ID = id.NewULID()
[performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
if err := f.state.DB.PutFollowRequest(ctx, followRequest); err != nil {
return fmt.Errorf("activityFollow: database error inserting follow request: %s", err)
}
f.state.Workers.EnqueueFederator(ctx, messages.FromFederator{
APObjectType: ap.ActivityFollow,
APActivityType: ap.ActivityCreate,
GTSModel: followRequest,
ReceivingAccount: receivingAccount,
})
return nil
}
/*
LIKE HANDLERS
*/
func (f *federatingDB) activityLike(ctx context.Context, asType vocab.Type, receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account) error {
like, ok := asType.(vocab.ActivityStreamsLike)
if !ok {
return errors.New("activityLike: could not convert type to like")
}
fave, err := f.typeConverter.ASLikeToFave(ctx, like)
if err != nil {
return fmt.Errorf("activityLike: could not convert Like to fave: %w", err)
}
fave.ID = id.NewULID()
if err := f.state.DB.PutStatusFave(ctx, fave); err != nil {
if errors.Is(err, db.ErrAlreadyExists) {
// The Like already exists in the database, which
// means we've already handled side effects. We can
// just return nil here and be done with it.
return nil
}
return fmt.Errorf("activityLike: database error inserting fave: %w", err)
}
f.state.Workers.EnqueueFederator(ctx, messages.FromFederator{
APObjectType: ap.ActivityLike,
APActivityType: ap.ActivityCreate,
GTSModel: fave,
ReceivingAccount: receivingAccount,
})
return nil
}
/*
FLAG HANDLERS
*/
func (f *federatingDB) activityFlag(ctx context.Context, asType vocab.Type, receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account) error {
flag, ok := asType.(vocab.ActivityStreamsFlag)
if !ok {
return errors.New("activityFlag: could not convert type to flag")
}
report, err := f.typeConverter.ASFlagToReport(ctx, flag)
if err != nil {
return fmt.Errorf("activityFlag: could not convert Flag to report: %w", err)
}
report.ID = id.NewULID()
if err := f.state.DB.PutReport(ctx, report); err != nil {
return fmt.Errorf("activityFlag: database error inserting report: %w", err)
}
f.state.Workers.EnqueueFederator(ctx, messages.FromFederator{
APObjectType: ap.ActivityFlag,
APActivityType: ap.ActivityCreate,
GTSModel: report,
ReceivingAccount: receivingAccount,
})
return nil
}