[performance] Add dereference shortcuts to avoid making http calls to self (#430)

* update transport (controller) to allow shortcuts

* go fmt

* expose underlying sig transport to allow test sigs

cmd/gotosocial/action/server/server.go

@@ -117,7 +117,7 @@
 		return fmt.Errorf("error creating media manager: %s", err)
 	}
 	oauthServer := oauth.New(ctx, dbService)
-	transportController := transport.NewController(dbService, &federation.Clock{}, http.DefaultClient)
+	transportController := transport.NewController(dbService, federatingDB, &federation.Clock{}, http.DefaultClient)
 	federator := federation.NewFederator(dbService, federatingDB, transportController, typeConverter, mediaManager)
 
 	// decide whether to create a noop email sender (won't send emails) or a real one

internal/transport/controller.go

@@ -21,14 +21,18 @@
 import (
 	"context"
 	"crypto"
+	"encoding/json"
 	"fmt"
+	"net/url"
 	"sync"
 
 	"github.com/go-fed/httpsig"
 	"github.com/spf13/viper"
 	"github.com/superseriousbusiness/activity/pub"
+	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
 )
 
 // Controller generates transports for use in making federation requests to other servers.
@@ -42,19 +46,65 @@ type controller struct {
 	clock    pub.Clock
 	client   pub.HttpClient
 	appAgent string
+
+	// dereferenceFollowersShortcut is a shortcut to dereference followers of an
+	// account on this instance, without making any external api/http calls.
+	//
+	// It is passed to new transports, and should only be invoked when the iri.Host == this host.
+	dereferenceFollowersShortcut func(ctx context.Context, iri *url.URL) ([]byte, error)
+
+	// dereferenceUserShortcut is a shortcut to dereference followers an account on
+	// this instance, without making any external api/http calls.
+	//
+	// It is passed to new transports, and should only be invoked when the iri.Host == this host.
+	dereferenceUserShortcut func(ctx context.Context, iri *url.URL) ([]byte, error)
+}
+
+func dereferenceFollowersShortcut(federatingDB federatingdb.DB) func(context.Context, *url.URL) ([]byte, error) {
+	return func(ctx context.Context, iri *url.URL) ([]byte, error) {
+		followers, err := federatingDB.Followers(ctx, iri)
+		if err != nil {
+			return nil, err
+		}
+
+		i, err := streams.Serialize(followers)
+		if err != nil {
+			return nil, err
+		}
+
+		return json.Marshal(i)
+	}
+}
+
+func dereferenceUserShortcut(federatingDB federatingdb.DB) func(context.Context, *url.URL) ([]byte, error) {
+	return func(ctx context.Context, iri *url.URL) ([]byte, error) {
+		user, err := federatingDB.Get(ctx, iri)
+		if err != nil {
+			return nil, err
+		}
+
+		i, err := streams.Serialize(user)
+		if err != nil {
+			return nil, err
+		}
+
+		return json.Marshal(i)
+	}
 }
 
 // NewController returns an implementation of the Controller interface for creating new transports
-func NewController(db db.DB, clock pub.Clock, client pub.HttpClient) Controller {
+func NewController(db db.DB, federatingDB federatingdb.DB, clock pub.Clock, client pub.HttpClient) Controller {
 	applicationName := viper.GetString(config.Keys.ApplicationName)
 	host := viper.GetString(config.Keys.Host)
 	appAgent := fmt.Sprintf("%s %s", applicationName, host)
 
 	return &controller{
-		db:       db,
+		db:                           db,
-		clock:    clock,
+		clock:                        clock,
-		client:   client,
+		client:                       client,
-		appAgent: appAgent,
+		appAgent:                     appAgent,
+		dereferenceFollowersShortcut: dereferenceFollowersShortcut(federatingDB),
+		dereferenceUserShortcut:      dereferenceUserShortcut(federatingDB),
 	}
 }
 
@@ -78,15 +128,17 @@ func (c *controller) NewTransport(pubKeyID string, privkey crypto.PrivateKey) (T
 	sigTransport := pub.NewHttpSigTransport(c.client, c.appAgent, c.clock, getSigner, postSigner, pubKeyID, privkey)
 
 	return &transport{
-		client:       c.client,
+		client:                       c.client,
-		appAgent:     c.appAgent,
+		appAgent:                     c.appAgent,
-		gofedAgent:   "(go-fed/activity v1.0.0)",
+		gofedAgent:                   "(go-fed/activity v1.0.0)",
-		clock:        c.clock,
+		clock:                        c.clock,
-		pubKeyID:     pubKeyID,
+		pubKeyID:                     pubKeyID,
-		privkey:      privkey,
+		privkey:                      privkey,
-		sigTransport: sigTransport,
+		sigTransport:                 sigTransport,
-		getSigner:    getSigner,
+		getSigner:                    getSigner,
-		getSignerMu:  &sync.Mutex{},
+		getSignerMu:                  &sync.Mutex{},
+		dereferenceFollowersShortcut: c.dereferenceFollowersShortcut,
+		dereferenceUserShortcut:      c.dereferenceUserShortcut,
 	}, nil
 }
 

internal/transport/deliver.go

@@ -23,6 +23,8 @@
 	"net/url"
 
 	"github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
 )
 
 func (t *transport) BatchDeliver(ctx context.Context, b []byte, recipients []*url.URL) error {
@@ -30,6 +32,11 @@ func (t *transport) BatchDeliver(ctx context.Context, b []byte, recipients []*ur
 }
 
 func (t *transport) Deliver(ctx context.Context, b []byte, to *url.URL) error {
+	// if the 'to' host is our own, just skip this delivery since we by definition already have the message!
+	if to.Host == viper.GetString(config.Keys.Host) {
+		return nil
+	}
+
 	l := logrus.WithField("func", "Deliver")
 	l.Debugf("performing POST to %s", to.String())
 	return t.sigTransport.Deliver(ctx, b, to)

internal/transport/dereference.go

@@ -23,10 +23,29 @@
 	"net/url"
 
 	"github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/uris"
 )
 
 func (t *transport) Dereference(ctx context.Context, iri *url.URL) ([]byte, error) {
 	l := logrus.WithField("func", "Dereference")
+
+	// if the request is to us, we can shortcut for certain URIs rather than going through
+	// the normal request flow, thereby saving time and energy
+	if iri.Host == viper.GetString(config.Keys.Host) {
+		if uris.IsFollowersPath(iri) {
+			// the request is for followers of one of our accounts, which we can shortcut
+			return t.dereferenceFollowersShortcut(ctx, iri)
+		}
+
+		if uris.IsUserPath(iri) {
+			// the request is for one of our accounts, which we can shortcut
+			return t.dereferenceUserShortcut(ctx, iri)
+		}
+	}
+
+	// the request is either for a remote host or for us but we don't have a shortcut, so continue as normal
 	l.Debugf("performing GET to %s", iri.String())
 	return t.sigTransport.Dereference(ctx, iri)
 }

internal/transport/transport.go

@@ -30,8 +30,11 @@
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
-// Transport wraps the pub.Transport interface with some additional
+// Transport wraps the pub.Transport interface with some additional functionality for fetching remote media.
-// functionality for fetching remote media.
+//
+// Since the transport has the concept of 'shortcuts' for fetching data locally rather than remotely, it is
+// not *always* the case that calling a Transport function does an http call, but it usually will for remote
+// hosts or resources for which a shortcut isn't provided by the transport controller (also in this package).
 type Transport interface {
 	pub.Transport
 	// DereferenceMedia fetches the given media attachment IRI, returning the reader and filesize.
@@ -40,6 +43,8 @@ type Transport interface {
 	DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error)
 	// Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body.
 	Finger(ctx context.Context, targetUsername string, targetDomains string) ([]byte, error)
+	// SigTransport returns the underlying http signature transport wrapped by the GoToSocial transport.
+	SigTransport() pub.Transport
 }
 
 // transport implements the Transport interface
@@ -53,4 +58,13 @@ type transport struct {
 	sigTransport *pub.HttpSigTransport
 	getSigner    httpsig.Signer
 	getSignerMu  *sync.Mutex
+
+	// shortcuts for dereferencing things that exist on our instance without making an http call to ourself
+
+	dereferenceFollowersShortcut func(ctx context.Context, iri *url.URL) ([]byte, error)
+	dereferenceUserShortcut      func(ctx context.Context, iri *url.URL) ([]byte, error)
+}
+
+func (t *transport) SigTransport() pub.Transport {
+	return t.sigTransport
 }

testrig/testmodels.go

@@ -1749,8 +1749,8 @@ func GetSignatureForActivity(activity pub.Activity, pubKeyID string, privkey cry
 		panic(err)
 	}
 
-	// trigger the delivery function, which will trigger the 'do' function of the recorder above
+	// trigger the delivery function for the underlying signature transport, which will trigger the 'do' function of the recorder above
-	if err := tp.Deliver(context.Background(), bytes, destination); err != nil {
+	if err := tp.SigTransport().Deliver(context.Background(), bytes, destination); err != nil {
 		panic(err)
 	}
 
@@ -1781,8 +1781,8 @@ func GetSignatureForDereference(pubKeyID string, privkey crypto.PrivateKey, dest
 		panic(err)
 	}
 
-	// trigger the delivery function, which will trigger the 'do' function of the recorder above
+	// trigger the dereference function for the underlying signature transport, which will trigger the 'do' function of the recorder above
-	if _, err := tp.Dereference(context.Background(), destination); err != nil {
+	if _, err := tp.SigTransport().Dereference(context.Background(), destination); err != nil {
 		panic(err)
 	}
 

testrig/transportcontroller.go

@@ -39,7 +39,7 @@
 // PER TEST rather than per suite, so that the do function can be set on a test by test (or even more granular)
 // basis.
 func NewTestTransportController(client pub.HttpClient, db db.DB) transport.Controller {
-	return transport.NewController(db, &federation.Clock{}, client)
+	return transport.NewController(db, NewTestFederatingDB(db), &federation.Clock{}, client)
 }
 
 // NewMockHTTPClient returns a client that conforms to the pub.HttpClient interface,