replace async client API / federator msg processing with worker pools (#497)

* replace async client API / federator msg processing with worker pools
* appease our lord-and-saviour, the linter

cmd/gotosocial/action/server/server.go

@@ -63,6 +63,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
 	"github.com/superseriousbusiness/gotosocial/internal/gotosocial"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/oidc"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
@@ -70,6 +71,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 	"github.com/superseriousbusiness/gotosocial/internal/web"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 )
 
 // Start creates and starts a gotosocial server
@@ -87,7 +89,14 @@
 		return fmt.Errorf("error creating instance instance: %s", err)
 	}
 
-	federatingDB := federatingdb.New(dbService)
+	// Create the client API and federator worker pools
+	// NOTE: these MUST NOT be used until they are passed to the
+	// processor and it is started. The reason being that the processor
+	// sets the Worker process functions and start the underlying pools
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
+	federatingDB := federatingdb.New(dbService, fedWorker)
 
 	router, err := router.New(ctx, dbService)
 	if err != nil {
@@ -138,8 +147,8 @@
 	}
 
 	// create and start the message processor using the other services we've created so far
-	processor := processing.NewProcessor(typeConverter, federator, oauthServer, mediaManager, storage, dbService, emailSender)
+	processor := processing.NewProcessor(typeConverter, federator, oauthServer, mediaManager, storage, dbService, emailSender, clientWorker, fedWorker)
-	if err := processor.Start(ctx); err != nil {
+	if err := processor.Start(); err != nil {
 		return fmt.Errorf("error starting processor: %s", err)
 	}
 

cmd/gotosocial/action/testrig/testrig.go

@@ -55,8 +55,10 @@
 	"github.com/superseriousbusiness/gotosocial/internal/api/s2s/webfinger"
 	"github.com/superseriousbusiness/gotosocial/internal/api/security"
 	"github.com/superseriousbusiness/gotosocial/internal/gotosocial"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/oidc"
 	"github.com/superseriousbusiness/gotosocial/internal/web"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -71,6 +73,10 @@
 	storageBackend := testrig.NewTestStorage()
 	testrig.StandardStorageSetup(storageBackend, "./testrig/media")
 
+	// Create client API and federator worker pools
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
 	// build backend handlers
 	oauthServer := testrig.NewTestOauthServer(dbService)
 	transportController := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
@@ -79,14 +85,14 @@
 			StatusCode: 200,
 			Body:       r,
 		}, nil
-	}), dbService)
+	}), dbService, fedWorker)
 	mediaManager := testrig.NewTestMediaManager(dbService, storageBackend)
-	federator := testrig.NewTestFederator(dbService, transportController, storageBackend, mediaManager)
+	federator := testrig.NewTestFederator(dbService, transportController, storageBackend, mediaManager, fedWorker)
 
 	emailSender := testrig.NewEmailSender("./web/template/", nil)
 
-	processor := testrig.NewTestProcessor(dbService, storageBackend, federator, emailSender, mediaManager)
+	processor := testrig.NewTestProcessor(dbService, storageBackend, federator, emailSender, mediaManager, clientWorker, fedWorker)
-	if err := processor.Start(ctx); err != nil {
+	if err := processor.Start(); err != nil {
 		return fmt.Errorf("error starting processor: %s", err)
 	}
 

internal/api/client/account/account_test.go

@@ -17,8 +17,10 @@
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -60,13 +62,16 @@ func (suite *AccountStandardTestSuite) SetupTest() {
 	testrig.InitTestConfig()
 	testrig.InitTestLog()
 
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
+
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.sentEmails = make(map[string]string)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails)
-	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
+	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
 	suite.accountModule = account.New(suite.processor).(*account.Module)
 	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")

internal/api/client/admin/admin_test.go

@@ -35,8 +35,10 @@
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -78,13 +80,16 @@ func (suite *AdminStandardTestSuite) SetupTest() {
 	testrig.InitTestConfig()
 	testrig.InitTestLog()
 
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
+
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.sentEmails = make(map[string]string)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails)
-	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
+	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
 	suite.adminModule = admin.New(suite.processor).(*admin.Module)
 	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")

internal/api/client/fileserver/servefile_test.go

@@ -36,9 +36,11 @@
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -74,12 +76,16 @@ func (suite *ServeFileTestSuite) SetupSuite() {
 	// setup standard items
 	testrig.InitTestConfig()
 	testrig.InitTestLog()
+
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
+
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, testrig.NewTestMediaManager(suite.db, suite.storage))
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, testrig.NewTestMediaManager(suite.db, suite.storage), fedWorker)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
 
-	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, testrig.NewTestMediaManager(suite.db, suite.storage))
+	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, testrig.NewTestMediaManager(suite.db, suite.storage), clientWorker, fedWorker)
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
 	suite.oauthServer = testrig.NewTestOauthServer(suite.db)

internal/api/client/followrequest/followrequest_test.go

@@ -34,8 +34,10 @@
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -74,12 +76,16 @@ func (suite *FollowRequestStandardTestSuite) SetupSuite() {
 func (suite *FollowRequestStandardTestSuite) SetupTest() {
 	testrig.InitTestConfig()
 	testrig.InitTestLog()
+
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
+
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
-	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
+	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
 	suite.followRequestModule = followrequest.New(suite.processor).(*followrequest.Module)
 	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")

internal/api/client/media/mediacreate_test.go

@@ -43,9 +43,11 @@
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -81,14 +83,18 @@ func (suite *MediaCreateTestSuite) SetupSuite() {
 	// setup standard items
 	testrig.InitTestConfig()
 	testrig.InitTestLog()
+
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
+
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
 	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
-	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
+	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
 
 	// setup module being tested
 	suite.mediaModule = mediamodule.New(suite.processor).(*mediamodule.Module)

internal/api/client/media/mediaupdate_test.go

@@ -41,9 +41,11 @@
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -79,14 +81,18 @@ func (suite *MediaUpdateTestSuite) SetupSuite() {
 	// setup standard items
 	testrig.InitTestConfig()
 	testrig.InitTestLog()
+
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
+
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
 	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
-	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
+	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
 
 	// setup module being tested
 	suite.mediaModule = mediamodule.New(suite.processor).(*mediamodule.Module)

internal/api/client/status/status_test.go

@@ -37,8 +37,10 @@
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -79,13 +81,17 @@ func (suite *StatusStandardTestSuite) SetupSuite() {
 func (suite *StatusStandardTestSuite) SetupTest() {
 	testrig.InitTestConfig()
 	testrig.InitTestLog()
+
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
+
 	suite.db = testrig.NewTestDB()
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.storage = testrig.NewTestStorage()
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(suite.testHttpClient(), suite.db), suite.storage, suite.mediaManager)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(suite.testHttpClient(), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
-	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
+	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
 	suite.statusModule = status.New(suite.processor).(*status.Module)
 	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
@@ -104,7 +110,6 @@ func (suite *StatusStandardTestSuite) testHttpClient() pub.HttpClient {
 	fmt.Println(remoteAccountWebfingerURI)
 
 	httpClient := testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
-
 		// respond correctly to a webfinger lookup
 		if req.URL.String() == remoteAccountWebfingerURI {
 			responseJson := fmt.Sprintf(`

internal/api/client/user/user_test.go

@@ -27,8 +27,10 @@
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -56,6 +58,8 @@ type UserStandardTestSuite struct {
 func (suite *UserStandardTestSuite) SetupTest() {
 	testrig.InitTestLog()
 	testrig.InitTestConfig()
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
 	suite.testTokens = testrig.NewTestTokens()
 	suite.testClients = testrig.NewTestClients()
 	suite.testApplications = testrig.NewTestApplications()
@@ -65,10 +69,10 @@ func (suite *UserStandardTestSuite) SetupTest() {
 	suite.storage = testrig.NewTestStorage()
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.sentEmails = make(map[string]string)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails)
-	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
+	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
 	suite.userModule = user.New(suite.processor).(*user.Module)
 	testrig.StandardDBSetup(suite.db, suite.testAccounts)
 	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")

internal/api/s2s/user/inboxpost_test.go

@@ -36,6 +36,8 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -83,10 +85,13 @@ func (suite *InboxPostTestSuite) TestPostBlock() {
 	suite.NoError(err)
 	body := bytes.NewReader(bodyJson)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
-	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
-	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
 	userModule := user.New(processor).(*user.Module)
 
 	// setup request
@@ -183,10 +188,13 @@ func (suite *InboxPostTestSuite) TestPostUnblock() {
 	suite.NoError(err)
 	body := bytes.NewReader(bodyJson)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
-	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
-	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
 	userModule := user.New(processor).(*user.Module)
 
 	// setup request
@@ -273,10 +281,13 @@ func (suite *InboxPostTestSuite) TestPostUpdate() {
 	suite.NoError(err)
 	body := bytes.NewReader(bodyJson)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
-	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
-	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
 	userModule := user.New(processor).(*user.Module)
 
 	// setup request
@@ -392,11 +403,14 @@ func (suite *InboxPostTestSuite) TestPostDelete() {
 	suite.NoError(err)
 	body := bytes.NewReader(bodyJson)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
-	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
-	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
-	err = processor.Start(context.Background())
+	err = processor.Start()
 	suite.NoError(err)
 	userModule := user.New(processor).(*user.Module)
 

internal/api/s2s/user/outboxget_test.go

@@ -31,6 +31,8 @@
 	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/activity/streams/vocab"
 	"github.com/superseriousbusiness/gotosocial/internal/api/s2s/user"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -44,10 +46,13 @@ func (suite *OutboxGetTestSuite) TestGetOutbox() {
 	signedRequest := derefRequests["foss_satan_dereference_zork_outbox"]
 	targetAccount := suite.testAccounts["local_account_1"]
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
-	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
-	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
 	userModule := user.New(processor).(*user.Module)
 
 	// setup request
@@ -99,10 +104,13 @@ func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() {
 	signedRequest := derefRequests["foss_satan_dereference_zork_outbox_first"]
 	targetAccount := suite.testAccounts["local_account_1"]
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
-	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
-	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
 	userModule := user.New(processor).(*user.Module)
 
 	// setup request
@@ -154,10 +162,13 @@ func (suite *OutboxGetTestSuite) TestGetOutboxNextPage() {
 	signedRequest := derefRequests["foss_satan_dereference_zork_outbox_next"]
 	targetAccount := suite.testAccounts["local_account_1"]
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
-	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
-	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
 	userModule := user.New(processor).(*user.Module)
 
 	// setup request

internal/api/s2s/user/repliesget_test.go

@@ -33,6 +33,8 @@
 	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/activity/streams/vocab"
 	"github.com/superseriousbusiness/gotosocial/internal/api/s2s/user"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -47,10 +49,13 @@ func (suite *RepliesGetTestSuite) TestGetReplies() {
 	targetAccount := suite.testAccounts["local_account_1"]
 	targetStatus := suite.testStatuses["local_account_1_status_1"]
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
-	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
-	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
 	userModule := user.New(processor).(*user.Module)
 
 	// setup request
@@ -108,10 +113,13 @@ func (suite *RepliesGetTestSuite) TestGetRepliesNext() {
 	targetAccount := suite.testAccounts["local_account_1"]
 	targetStatus := suite.testStatuses["local_account_1_status_1"]
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
-	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
-	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
 	userModule := user.New(processor).(*user.Module)
 
 	// setup request
@@ -172,10 +180,13 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() {
 	targetAccount := suite.testAccounts["local_account_1"]
 	targetStatus := suite.testStatuses["local_account_1_status_1"]
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
-	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
-	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
 	userModule := user.New(processor).(*user.Module)
 
 	// setup request

internal/api/s2s/user/statusget_test.go

@@ -32,6 +32,8 @@
 	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/activity/streams/vocab"
 	"github.com/superseriousbusiness/gotosocial/internal/api/s2s/user"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -46,10 +48,13 @@ func (suite *StatusGetTestSuite) TestGetStatus() {
 	targetAccount := suite.testAccounts["local_account_1"]
 	targetStatus := suite.testStatuses["local_account_1_status_1"]
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
-	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
-	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
 	userModule := user.New(processor).(*user.Module)
 
 	// setup request
@@ -111,10 +116,13 @@ func (suite *StatusGetTestSuite) TestGetStatusLowercase() {
 	targetAccount := suite.testAccounts["local_account_1"]
 	targetStatus := suite.testStatuses["local_account_1_status_1"]
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
-	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
-	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
 	userModule := user.New(processor).(*user.Module)
 
 	// setup request

internal/api/s2s/user/user_test.go

@@ -28,9 +28,11 @@
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -76,13 +78,16 @@ func (suite *UserStandardTestSuite) SetupTest() {
 	testrig.InitTestConfig()
 	testrig.InitTestLog()
 
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
 	suite.db = testrig.NewTestDB()
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.storage = testrig.NewTestStorage()
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
-	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
+	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
 	suite.userModule = user.New(suite.processor).(*user.Module)
 	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
 	suite.securityModule = security.New(suite.db, suite.oauthServer).(*security.Module)

internal/api/s2s/user/userget_test.go

@@ -33,7 +33,9 @@
 	"github.com/superseriousbusiness/activity/streams/vocab"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/api/s2s/user"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -47,10 +49,13 @@ func (suite *UserGetTestSuite) TestGetUser() {
 	signedRequest := derefRequests["foss_satan_dereference_zork"]
 	targetAccount := suite.testAccounts["local_account_1"]
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
-	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
-	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
 	userModule := user.New(processor).(*user.Module)
 
 	// setup request
@@ -125,10 +130,13 @@ func (suite *UserGetTestSuite) TestGetUserPublicKeyDeleted() {
 	derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts)
 	signedRequest := derefRequests["foss_satan_dereference_zork_public_key"]
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
-	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
-	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
 	userModule := user.New(processor).(*user.Module)
 
 	// setup request

internal/api/s2s/webfinger/webfinger_test.go

@@ -33,9 +33,11 @@
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -79,13 +81,16 @@ func (suite *WebfingerStandardTestSuite) SetupTest() {
 	testrig.InitTestLog()
 	testrig.InitTestConfig()
 
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
 	suite.db = testrig.NewTestDB()
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.storage = testrig.NewTestStorage()
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
-	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
+	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
 	suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module)
 	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
 	suite.securityModule = security.New(suite.db, suite.oauthServer).(*security.Module)

internal/api/s2s/webfinger/webfingerget_test.go

@@ -32,7 +32,9 @@
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/api/s2s/webfinger"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -69,7 +71,9 @@ func (suite *WebfingerGetTestSuite) TestFingerUser() {
 func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByHost() {
 	viper.Set(config.Keys.Host, "gts.example.org")
 	viper.Set(config.Keys.AccountDomain, "example.org")
-	suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+	suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender, clientWorker, fedWorker)
 	suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module)
 
 	targetAccount := accountDomainAccount()
@@ -103,7 +107,9 @@ func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByHo
 func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByAccountDomain() {
 	viper.Set(config.Keys.Host, "gts.example.org")
 	viper.Set(config.Keys.AccountDomain, "example.org")
-	suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+	suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender, clientWorker, fedWorker)
 	suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module)
 
 	targetAccount := accountDomainAccount()

internal/federation/dereferencing/dereferencer_test.go

@@ -32,7 +32,9 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -148,6 +150,7 @@ func (suite *DereferencerStandardTestSuite) mockTransportController() transport.
 
 		return response, nil
 	}
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
 	mockClient := testrig.NewMockHTTPClient(do)
-	return testrig.NewTestTransportController(mockClient, suite.db)
+	return testrig.NewTestTransportController(mockClient, suite.db, fedWorker)
 }

internal/federation/federatingdb/accept.go

@@ -48,9 +48,9 @@ func (f *federatingDB) Accept(ctx context.Context, accept vocab.ActivityStreamsA
 		l.Debug("entering Accept")
 	}
 
-	receivingAccount, _, fromFederatorChan := extractFromCtx(ctx)
+	receivingAccount, _ := extractFromCtx(ctx)
-	if receivingAccount == nil || fromFederatorChan == nil {
+	if receivingAccount == nil {
-		// If the receiving account or federator channel wasn't set on the context, that means this request didn't pass
+		// If the receiving account  wasn't set on the context, that means this request didn't pass
 		// through the API, but came from inside GtS as the result of another activity on this instance. That being so,
 		// we can safely just ignore this activity, since we know we've already processed it elsewhere.
 		return nil
@@ -82,12 +82,12 @@ func (f *federatingDB) Accept(ctx context.Context, accept vocab.ActivityStreamsA
 					return err
 				}
 
-				fromFederatorChan <- messages.FromFederator{
+				f.fedWorker.Queue(messages.FromFederator{
 					APObjectType:     ap.ActivityFollow,
 					APActivityType:   ap.ActivityAccept,
 					GTSModel:         follow,
 					ReceivingAccount: receivingAccount,
-				}
+				})
 
 				return nil
 			}
@@ -117,12 +117,12 @@ func (f *federatingDB) Accept(ctx context.Context, accept vocab.ActivityStreamsA
 				return err
 			}
 
-			fromFederatorChan <- messages.FromFederator{
+			f.fedWorker.Queue(messages.FromFederator{
 				APObjectType:     ap.ActivityFollow,
 				APActivityType:   ap.ActivityAccept,
 				GTSModel:         follow,
 				ReceivingAccount: receivingAccount,
-			}
+			})
 
 			return nil
 		}

internal/federation/federatingdb/announce.go

@@ -44,9 +44,9 @@ func (f *federatingDB) Announce(ctx context.Context, announce vocab.ActivityStre
 		l.Debug("entering Announce")
 	}
 
-	receivingAccount, _, fromFederatorChan := extractFromCtx(ctx)
+	receivingAccount, _ := extractFromCtx(ctx)
-	if receivingAccount == nil || fromFederatorChan == nil {
+	if receivingAccount == nil {
-		// If the receiving account or federator channel wasn't set on the context, that means this request didn't pass
+		// If the receiving account wasn't set on the context, that means this request didn't pass
 		// through the API, but came from inside GtS as the result of another activity on this instance. That being so,
 		// we can safely just ignore this activity, since we know we've already processed it elsewhere.
 		return nil
@@ -63,12 +63,12 @@ func (f *federatingDB) Announce(ctx context.Context, announce vocab.ActivityStre
 	}
 
 	// it's a new announce so pass it back to the processor async for dereferencing etc
-	fromFederatorChan <- messages.FromFederator{
+	f.fedWorker.Queue(messages.FromFederator{
 		APObjectType:     ap.ActivityAnnounce,
 		APActivityType:   ap.ActivityCreate,
 		GTSModel:         boost,
 		ReceivingAccount: receivingAccount,
-	}
+	})
 
 	return nil
 }

internal/federation/federatingdb/create.go

@@ -61,9 +61,9 @@ func (f *federatingDB) Create(ctx context.Context, asType vocab.Type) error {
 		l.Debug("entering Create")
 	}
 
-	receivingAccount, requestingAccount, fromFederatorChan := extractFromCtx(ctx)
+	receivingAccount, requestingAccount := extractFromCtx(ctx)
-	if receivingAccount == nil || fromFederatorChan == nil {
+	if receivingAccount == nil {
-		// If the receiving account or federator channel wasn't set on the context, that means this request didn't pass
+		// If the receiving account wasn't set on the context, that means this request didn't pass
 		// through the API, but came from inside GtS as the result of another activity on this instance. That being so,
 		// we can safely just ignore this activity, since we know we've already processed it elsewhere.
 		return nil
@@ -72,16 +72,16 @@ func (f *federatingDB) Create(ctx context.Context, asType vocab.Type) error {
 	switch asType.GetTypeName() {
 	case ap.ActivityBlock:
 		// BLOCK SOMETHING
-		return f.activityBlock(ctx, asType, receivingAccount, requestingAccount, fromFederatorChan)
+		return f.activityBlock(ctx, asType, receivingAccount, requestingAccount)
 	case ap.ActivityCreate:
 		// CREATE SOMETHING
-		return f.activityCreate(ctx, asType, receivingAccount, requestingAccount, fromFederatorChan)
+		return f.activityCreate(ctx, asType, receivingAccount, requestingAccount)
 	case ap.ActivityFollow:
 		// FOLLOW SOMETHING
-		return f.activityFollow(ctx, asType, receivingAccount, requestingAccount, fromFederatorChan)
+		return f.activityFollow(ctx, asType, receivingAccount, requestingAccount)
 	case ap.ActivityLike:
 		// LIKE SOMETHING
-		return f.activityLike(ctx, asType, receivingAccount, requestingAccount, fromFederatorChan)
+		return f.activityLike(ctx, asType, receivingAccount, requestingAccount)
 	}
 	return nil
 }
@@ -90,7 +90,7 @@ func (f *federatingDB) Create(ctx context.Context, asType vocab.Type) error {
 	BLOCK HANDLERS
 */
 
-func (f *federatingDB) activityBlock(ctx context.Context, asType vocab.Type, receiving *gtsmodel.Account, requestingAccount *gtsmodel.Account, fromFederatorChan chan messages.FromFederator) error {
+func (f *federatingDB) activityBlock(ctx context.Context, asType vocab.Type, receiving *gtsmodel.Account, requestingAccount *gtsmodel.Account) error {
 	blockable, ok := asType.(vocab.ActivityStreamsBlock)
 	if !ok {
 		return errors.New("activityBlock: could not convert type to block")
@@ -111,12 +111,12 @@ func (f *federatingDB) activityBlock(ctx context.Context, asType vocab.Type, rec
 		return fmt.Errorf("activityBlock: database error inserting block: %s", err)
 	}
 
-	fromFederatorChan <- messages.FromFederator{
+	f.fedWorker.Queue(messages.FromFederator{
 		APObjectType:     ap.ActivityBlock,
 		APActivityType:   ap.ActivityCreate,
 		GTSModel:         block,
 		ReceivingAccount: receiving,
-	}
+	})
 	return nil
 }
 
@@ -124,7 +124,7 @@ func (f *federatingDB) activityBlock(ctx context.Context, asType vocab.Type, rec
 	CREATE HANDLERS
 */
 
-func (f *federatingDB) activityCreate(ctx context.Context, asType vocab.Type, receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account, fromFederatorChan chan messages.FromFederator) error {
+func (f *federatingDB) activityCreate(ctx context.Context, asType vocab.Type, receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account) error {
 	create, ok := asType.(vocab.ActivityStreamsCreate)
 	if !ok {
 		return errors.New("activityCreate: could not convert type to create")
@@ -152,7 +152,7 @@ func (f *federatingDB) activityCreate(ctx context.Context, asType vocab.Type, re
 		switch asObjectTypeName {
 		case ap.ObjectNote:
 			// CREATE A NOTE
-			if err := f.createNote(ctx, objectIter.GetActivityStreamsNote(), receivingAccount, requestingAccount, fromFederatorChan); err != nil {
+			if err := f.createNote(ctx, objectIter.GetActivityStreamsNote(), receivingAccount, requestingAccount); err != nil {
 				errs = append(errs, err.Error())
 			}
 		default:
@@ -168,7 +168,7 @@ func (f *federatingDB) activityCreate(ctx context.Context, asType vocab.Type, re
 }
 
 // createNote handles a Create activity with a Note type.
-func (f *federatingDB) createNote(ctx context.Context, note vocab.ActivityStreamsNote, receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account, fromFederatorChan chan messages.FromFederator) error {
+func (f *federatingDB) createNote(ctx context.Context, note vocab.ActivityStreamsNote, receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account) error {
 	l := logrus.WithFields(logrus.Fields{
 		"func":              "createNote",
 		"receivingAccount":  receivingAccount.URI,
@@ -206,13 +206,13 @@ func (f *federatingDB) createNote(ctx context.Context, note vocab.ActivityStream
 			return nil
 		}
 		// pass the note iri into the processor and have it do the dereferencing instead of doing it here
-		fromFederatorChan <- messages.FromFederator{
+		f.fedWorker.Queue(messages.FromFederator{
 			APObjectType:     ap.ObjectNote,
 			APActivityType:   ap.ActivityCreate,
 			APIri:            id.GetIRI(),
 			GTSModel:         nil,
 			ReceivingAccount: receivingAccount,
-		}
+		})
 		return nil
 	}
 
@@ -241,12 +241,12 @@ func (f *federatingDB) createNote(ctx context.Context, note vocab.ActivityStream
 		return fmt.Errorf("createNote: database error inserting status: %s", err)
 	}
 
-	fromFederatorChan <- messages.FromFederator{
+	f.fedWorker.Queue(messages.FromFederator{
 		APObjectType:     ap.ObjectNote,
 		APActivityType:   ap.ActivityCreate,
 		GTSModel:         status,
 		ReceivingAccount: receivingAccount,
-	}
+	})
 
 	return nil
 }
@@ -255,7 +255,7 @@ func (f *federatingDB) createNote(ctx context.Context, note vocab.ActivityStream
 	FOLLOW HANDLERS
 */
 
-func (f *federatingDB) activityFollow(ctx context.Context, asType vocab.Type, receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account, fromFederatorChan chan messages.FromFederator) error {
+func (f *federatingDB) activityFollow(ctx context.Context, asType vocab.Type, receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account) error {
 	follow, ok := asType.(vocab.ActivityStreamsFollow)
 	if !ok {
 		return errors.New("activityFollow: could not convert type to follow")
@@ -276,12 +276,12 @@ func (f *federatingDB) activityFollow(ctx context.Context, asType vocab.Type, re
 		return fmt.Errorf("activityFollow: database error inserting follow request: %s", err)
 	}
 
-	fromFederatorChan <- messages.FromFederator{
+	f.fedWorker.Queue(messages.FromFederator{
 		APObjectType:     ap.ActivityFollow,
 		APActivityType:   ap.ActivityCreate,
 		GTSModel:         followRequest,
 		ReceivingAccount: receivingAccount,
-	}
+	})
 
 	return nil
 }
@@ -290,7 +290,7 @@ func (f *federatingDB) activityFollow(ctx context.Context, asType vocab.Type, re
 	LIKE HANDLERS
 */
 
-func (f *federatingDB) activityLike(ctx context.Context, asType vocab.Type, receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account, fromFederatorChan chan messages.FromFederator) error {
+func (f *federatingDB) activityLike(ctx context.Context, asType vocab.Type, receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account) error {
 	like, ok := asType.(vocab.ActivityStreamsLike)
 	if !ok {
 		return errors.New("activityLike: could not convert type to like")
@@ -311,12 +311,12 @@ func (f *federatingDB) activityLike(ctx context.Context, asType vocab.Type, rece
 		return fmt.Errorf("activityLike: database error inserting fave: %s", err)
 	}
 
-	fromFederatorChan <- messages.FromFederator{
+	f.fedWorker.Queue(messages.FromFederator{
 		APObjectType:     ap.ActivityLike,
 		APActivityType:   ap.ActivityCreate,
 		GTSModel:         fave,
 		ReceivingAccount: receivingAccount,
-	}
+	})
 
 	return nil
 }

internal/federation/federatingdb/create_test.go

@@ -25,7 +25,6 @@
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/messages"
 )
 
 type CreateTestSuite struct {
@@ -35,9 +34,8 @@ type CreateTestSuite struct {
 func (suite *CreateTestSuite) TestCreateNote() {
 	receivingAccount := suite.testAccounts["local_account_1"]
 	requestingAccount := suite.testAccounts["remote_account_1"]
-	fromFederatorChan := make(chan messages.FromFederator, 10)
 
-	ctx := createTestContext(receivingAccount, requestingAccount, fromFederatorChan)
+	ctx := createTestContext(receivingAccount, requestingAccount)
 
 	create := suite.testActivities["dm_for_zork"].Activity
 
@@ -45,7 +43,7 @@ func (suite *CreateTestSuite) TestCreateNote() {
 	suite.NoError(err)
 
 	// should be a message heading to the processor now, which we can intercept here
-	msg := <-fromFederatorChan
+	msg := <-suite.fromFederator
 	suite.Equal(ap.ObjectNote, msg.APObjectType)
 	suite.Equal(ap.ActivityCreate, msg.APActivityType)
 
@@ -65,9 +63,8 @@ func (suite *CreateTestSuite) TestCreateNote() {
 func (suite *CreateTestSuite) TestCreateNoteForward() {
 	receivingAccount := suite.testAccounts["local_account_1"]
 	requestingAccount := suite.testAccounts["remote_account_1"]
-	fromFederatorChan := make(chan messages.FromFederator, 10)
 
-	ctx := createTestContext(receivingAccount, requestingAccount, fromFederatorChan)
+	ctx := createTestContext(receivingAccount, requestingAccount)
 
 	create := suite.testActivities["forwarded_message"].Activity
 
@@ -75,7 +72,7 @@ func (suite *CreateTestSuite) TestCreateNoteForward() {
 	suite.NoError(err)
 
 	// should be a message heading to the processor now, which we can intercept here
-	msg := <-fromFederatorChan
+	msg := <-suite.fromFederator
 	suite.Equal(ap.ObjectNote, msg.APObjectType)
 	suite.Equal(ap.ActivityCreate, msg.APActivityType)
 

internal/federation/federatingdb/db.go

@@ -25,7 +25,9 @@
 	"github.com/superseriousbusiness/activity/pub"
 	"github.com/superseriousbusiness/activity/streams/vocab"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 )
 
 // DB wraps the pub.Database interface with a couple of custom functions for GoToSocial.
@@ -42,14 +44,16 @@ type DB interface {
 type federatingDB struct {
 	locks         mutexes.MutexMap
 	db            db.DB
+	fedWorker     *worker.Worker[messages.FromFederator]
 	typeConverter typeutils.TypeConverter
 }
 
 // New returns a DB interface using the given database and config
-func New(db db.DB) DB {
+func New(db db.DB, fedWorker *worker.Worker[messages.FromFederator]) DB {
 	fdb := federatingDB{
 		locks:         mutexes.NewMap(-1, -1), // use defaults
 		db:            db,
+		fedWorker:     fedWorker,
 		typeConverter: typeutils.NewConverter(db),
 	}
 	return &fdb

internal/federation/federatingdb/delete.go

@@ -44,9 +44,9 @@ func (f *federatingDB) Delete(ctx context.Context, id *url.URL) error {
 	)
 	l.Debug("entering Delete")
 
-	receivingAccount, _, fromFederatorChan := extractFromCtx(ctx)
+	receivingAccount, _ := extractFromCtx(ctx)
-	if receivingAccount == nil || fromFederatorChan == nil {
+	if receivingAccount == nil {
-		// If the receiving account or federator channel wasn't set on the context, that means this request didn't pass
+		// If the receiving account wasn't set on the context, that means this request didn't pass
 		// through the API, but came from inside GtS as the result of another activity on this instance. That being so,
 		// we can safely just ignore this activity, since we know we've already processed it elsewhere.
 		return nil
@@ -61,24 +61,24 @@ func (f *federatingDB) Delete(ctx context.Context, id *url.URL) error {
 		if err := f.db.DeleteByID(ctx, s.ID, &gtsmodel.Status{}); err != nil {
 			return fmt.Errorf("DELETE: err deleting status: %s", err)
 		}
-		fromFederatorChan <- messages.FromFederator{
+		f.fedWorker.Queue(messages.FromFederator{
 			APObjectType:     ap.ObjectNote,
 			APActivityType:   ap.ActivityDelete,
 			GTSModel:         s,
 			ReceivingAccount: receivingAccount,
-		}
+		})
 	}
 
 	a, err := f.db.GetAccountByURI(ctx, id.String())
 	if err == nil {
 		// it's an account
 		l.Debugf("uri is for an account with id %s, passing delete message to the processor", a.ID)
-		fromFederatorChan <- messages.FromFederator{
+		f.fedWorker.Queue(messages.FromFederator{
 			APObjectType:     ap.ObjectProfile,
 			APActivityType:   ap.ActivityDelete,
 			GTSModel:         a,
 			ReceivingAccount: receivingAccount,
-		}
+		})
 	}
 
 	return nil

internal/federation/federatingdb/federatingdb_test.go

@@ -28,14 +28,17 @@
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
 type FederatingDBTestSuite struct {
 	suite.Suite
-	db           db.DB
+	db            db.DB
-	tc           typeutils.TypeConverter
+	tc            typeutils.TypeConverter
-	federatingDB federatingdb.DB
+	fedWorker     *worker.Worker[messages.FromFederator]
+	fromFederator chan messages.FromFederator
+	federatingDB  federatingdb.DB
 
 	testTokens       map[string]*gtsmodel.Token
 	testClients      map[string]*gtsmodel.Client
@@ -62,10 +65,17 @@ func (suite *FederatingDBTestSuite) SetupSuite() {
 func (suite *FederatingDBTestSuite) SetupTest() {
 	testrig.InitTestLog()
 	testrig.InitTestConfig()
+	suite.fedWorker = worker.New[messages.FromFederator](-1, -1)
+	suite.fromFederator = make(chan messages.FromFederator, 10)
+	suite.fedWorker.SetProcessor(func(ctx context.Context, msg messages.FromFederator) error {
+		suite.fromFederator <- msg
+		return nil
+	})
+	_ = suite.fedWorker.Start()
 	suite.db = testrig.NewTestDB()
 	suite.testActivities = testrig.NewTestActivities(suite.testAccounts)
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
-	suite.federatingDB = testrig.NewTestFederatingDB(suite.db)
+	suite.federatingDB = testrig.NewTestFederatingDB(suite.db, suite.fedWorker)
 	testrig.StandardDBSetup(suite.db, suite.testAccounts)
 }
 
@@ -73,10 +83,9 @@ func (suite *FederatingDBTestSuite) TearDownTest() {
 	testrig.StandardDBTeardown(suite.db)
 }
 
-func createTestContext(receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account, fromFederatorChan chan messages.FromFederator) context.Context {
+func createTestContext(receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account) context.Context {
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, ap.ContextReceivingAccount, receivingAccount)
 	ctx = context.WithValue(ctx, ap.ContextRequestingAccount, requestingAccount)
-	ctx = context.WithValue(ctx, ap.ContextFromFederatorChan, fromFederatorChan)
 	return ctx
 }

internal/federation/federatingdb/reject.go

@@ -47,8 +47,8 @@ func (f *federatingDB) Reject(ctx context.Context, reject vocab.ActivityStreamsR
 		l.Debug("entering Reject")
 	}
 
-	receivingAccount, _, fromFederatorChan := extractFromCtx(ctx)
+	receivingAccount, _ := extractFromCtx(ctx)
-	if receivingAccount == nil || fromFederatorChan == nil {
+	if receivingAccount == nil {
 		// If the receiving account or federator channel wasn't set on the context, that means this request didn't pass
 		// through the API, but came from inside GtS as the result of another activity on this instance. That being so,
 		// we can safely just ignore this activity, since we know we've already processed it elsewhere.

internal/federation/federatingdb/reject_test.go

@@ -26,7 +26,6 @@
 	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/uris"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
@@ -40,8 +39,7 @@ func (suite *RejectTestSuite) TestRejectFollowRequest() {
 	// remote_account_2 rejects the follow request
 	followingAccount := suite.testAccounts["local_account_1"]
 	followedAccount := suite.testAccounts["remote_account_2"]
-	fromFederatorChan := make(chan messages.FromFederator, 10)
+	ctx := createTestContext(followingAccount, followedAccount)
-	ctx := createTestContext(followingAccount, followedAccount, fromFederatorChan)
 
 	// put the follow request in the database
 	fr := &gtsmodel.FollowRequest{
@@ -84,7 +82,7 @@ func (suite *RejectTestSuite) TestRejectFollowRequest() {
 	suite.NoError(err)
 
 	// there should be nothing in the federator channel since nothing needs to be passed
-	suite.Empty(fromFederatorChan)
+	suite.Empty(suite.fromFederator)
 
 	// the follow request should not be in the database anymore -- it's been rejected
 	err = suite.db.GetByID(ctx, fr.ID, &gtsmodel.FollowRequest{})

internal/federation/federatingdb/undo.go

@@ -46,9 +46,9 @@ func (f *federatingDB) Undo(ctx context.Context, undo vocab.ActivityStreamsUndo)
 		l.Debug("entering Undo")
 	}
 
-	receivingAccount, _, fromFederatorChan := extractFromCtx(ctx)
+	receivingAccount, _ := extractFromCtx(ctx)
-	if receivingAccount == nil || fromFederatorChan == nil {
+	if receivingAccount == nil {
-		// If the receiving account or federator channel wasn't set on the context, that means this request didn't pass
+		// If the receiving account wasn't set on the context, that means this request didn't pass
 		// through the API, but came from inside GtS as the result of another activity on this instance. That being so,
 		// we can safely just ignore this activity, since we know we've already processed it elsewhere.
 		return nil

internal/federation/federatingdb/update.go

@@ -57,9 +57,9 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error {
 		l.Debug("entering Update")
 	}
 
-	receivingAccount, _, fromFederatorChan := extractFromCtx(ctx)
+	receivingAccount, _ := extractFromCtx(ctx)
-	if receivingAccount == nil || fromFederatorChan == nil {
+	if receivingAccount == nil {
-		// If the receiving account or federator channel wasn't set on the context, that means this request didn't pass
+		// If the receiving account wasn't set on the context, that means this request didn't pass
 		// through the API, but came from inside GtS as the result of another activity on this instance. That being so,
 		// we can safely just ignore this activity, since we know we've already processed it elsewhere.
 		return nil
@@ -148,12 +148,12 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error {
 		}
 
 		// pass to the processor for further processing of eg., avatar/header
-		fromFederatorChan <- messages.FromFederator{
+		f.fedWorker.Queue(messages.FromFederator{
 			APObjectType:     ap.ObjectProfile,
 			APActivityType:   ap.ActivityUpdate,
 			GTSModel:         updatedAcct,
 			ReceivingAccount: receivingAccount,
-		}
+		})
 	}
 
 	return nil

internal/federation/federatingdb/util.go

@@ -34,7 +34,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
-	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/uris"
 )
 
@@ -310,7 +309,7 @@ func (f *federatingDB) collectIRIs(ctx context.Context, iris []*url.URL) (vocab.
 //   - The requesting account that posted to the inbox.
 //   - A channel that messages for the processor can be placed into.
 // If a value is not present, nil will be returned for it. It's up to the caller to check this and respond appropriately.
-func extractFromCtx(ctx context.Context) (receivingAccount, requestingAccount *gtsmodel.Account, fromFederatorChan chan messages.FromFederator) {
+func extractFromCtx(ctx context.Context) (receivingAccount, requestingAccount *gtsmodel.Account) {
 	receivingAccountI := ctx.Value(ap.ContextReceivingAccount)
 	if receivingAccountI != nil {
 		var ok bool
@@ -329,15 +328,6 @@ func extractFromCtx(ctx context.Context) (receivingAccount, requestingAccount *g
 		}
 	}
 
-	fromFederatorChanI := ctx.Value(ap.ContextFromFederatorChan)
-	if fromFederatorChanI != nil {
-		var ok bool
-		fromFederatorChan, ok = fromFederatorChanI.(chan messages.FromFederator)
-		if !ok {
-			logrus.Panicf("extractFromCtx: context entry with key %s could not be asserted to chan messages.FromFederator", ap.ContextFromFederatorChan)
-		}
-	}
-
 	return
 }
 

internal/federation/federator_test.go

@@ -34,7 +34,9 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -73,12 +75,14 @@ func (suite *ProtocolTestSuite) TestPostInboxRequestBodyHook() {
 	// the activity we're gonna use
 	activity := suite.activities["dm_for_zork"]
 
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
 	// setup transport controller with a no-op client so we don't make external calls
 	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
 		return nil, nil
-	}), suite.db)
+	}), suite.db, fedWorker)
 	// setup module being tested
-	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.typeConverter, testrig.NewTestMediaManager(suite.db, suite.storage))
+	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.typeConverter, testrig.NewTestMediaManager(suite.db, suite.storage))
 
 	// setup request
 	ctx := context.Background()
@@ -105,9 +109,11 @@ func (suite *ProtocolTestSuite) TestAuthenticatePostInbox() {
 	sendingAccount := suite.accounts["remote_account_1"]
 	inboxAccount := suite.accounts["local_account_1"]
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
 	// now setup module being tested, with the mock transport controller
-	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.typeConverter, testrig.NewTestMediaManager(suite.db, suite.storage))
+	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.typeConverter, testrig.NewTestMediaManager(suite.db, suite.storage))
 
 	request := httptest.NewRequest(http.MethodPost, "http://localhost:8080/users/the_mighty_zork/inbox", nil)
 	// we need these headers for the request to be validated

internal/processing/account/account.go

@@ -33,6 +33,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/text"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 	"github.com/superseriousbusiness/gotosocial/internal/visibility"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/oauth2/v4"
 )
 
@@ -81,28 +82,28 @@ type Processor interface {
 }
 
 type processor struct {
-	tc            typeutils.TypeConverter
+	tc           typeutils.TypeConverter
-	mediaManager  media.Manager
+	mediaManager media.Manager
-	fromClientAPI chan messages.FromClientAPI
+	clientWorker *worker.Worker[messages.FromClientAPI]
-	oauthServer   oauth.Server
+	oauthServer  oauth.Server
-	filter        visibility.Filter
+	filter       visibility.Filter
-	formatter     text.Formatter
+	formatter    text.Formatter
-	db            db.DB
+	db           db.DB
-	federator     federation.Federator
+	federator    federation.Federator
-	parseMention  gtsmodel.ParseMentionFunc
+	parseMention gtsmodel.ParseMentionFunc
 }
 
 // New returns a new account processor.
-func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, oauthServer oauth.Server, fromClientAPI chan messages.FromClientAPI, federator federation.Federator, parseMention gtsmodel.ParseMentionFunc) Processor {
+func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, oauthServer oauth.Server, clientWorker *worker.Worker[messages.FromClientAPI], federator federation.Federator, parseMention gtsmodel.ParseMentionFunc) Processor {
 	return &processor{
-		tc:            tc,
+		tc:           tc,
-		mediaManager:  mediaManager,
+		mediaManager: mediaManager,
-		fromClientAPI: fromClientAPI,
+		clientWorker: clientWorker,
-		oauthServer:   oauthServer,
+		oauthServer:  oauthServer,
-		filter:        visibility.NewFilter(db),
+		filter:       visibility.NewFilter(db),
-		formatter:     text.NewFormatter(db),
+		formatter:    text.NewFormatter(db),
-		db:            db,
+		db:           db,
-		federator:     federator,
+		federator:    federator,
-		parseMention:  parseMention,
+		parseMention: parseMention,
 	}
 }

internal/processing/account/account_test.go

@@ -19,6 +19,8 @@
 package account_test
 
 import (
+	"context"
+
 	"codeberg.org/gruf/go-store/kv"
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/activity/pub"
@@ -33,6 +35,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/processing/account"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -78,6 +81,16 @@ func (suite *AccountStandardTestSuite) SetupTest() {
 	testrig.InitTestLog()
 	testrig.InitTestConfig()
 
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
+	clientWorker.SetProcessor(func(_ context.Context, msg messages.FromClientAPI) error {
+		suite.fromClientAPIChan <- msg
+		return nil
+	})
+
+	_ = fedWorker.Start()
+	_ = clientWorker.Start()
+
 	suite.db = testrig.NewTestDB()
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.storage = testrig.NewTestStorage()
@@ -85,11 +98,11 @@ func (suite *AccountStandardTestSuite) SetupTest() {
 	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
 	suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100)
 	suite.httpClient = testrig.NewMockHTTPClient(nil)
-	suite.transportController = testrig.NewTestTransportController(suite.httpClient, suite.db)
+	suite.transportController = testrig.NewTestTransportController(suite.httpClient, suite.db, fedWorker)
-	suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager)
+	suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager, fedWorker)
 	suite.sentEmails = make(map[string]string)
 	suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails)
-	suite.accountProcessor = account.New(suite.db, suite.tc, suite.mediaManager, suite.oauthServer, suite.fromClientAPIChan, suite.federator, processing.GetParseMentionFunc(suite.db, suite.federator))
+	suite.accountProcessor = account.New(suite.db, suite.tc, suite.mediaManager, suite.oauthServer, clientWorker, suite.federator, processing.GetParseMentionFunc(suite.db, suite.federator))
 	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")
 }

internal/processing/account/create.go

@@ -85,12 +85,12 @@ func (p *processor) Create(ctx context.Context, applicationToken oauth2.TokenInf
 
 	// there are side effects for creating a new account (sending confirmation emails etc)
 	// so pass a message to the processor so that it can do it asynchronously
-	p.fromClientAPI <- messages.FromClientAPI{
+	p.clientWorker.Queue(messages.FromClientAPI{
 		APObjectType:   ap.ObjectProfile,
 		APActivityType: ap.ActivityCreate,
 		GTSModel:       user.Account,
 		OriginAccount:  user.Account,
-	}
+	})
 
 	return &apimodel.Token{
 		AccessToken: accessToken.GetAccess(),

internal/processing/account/createblock.go

@@ -113,7 +113,7 @@ func (p *processor) BlockCreate(ctx context.Context, requestingAccount *gtsmodel
 
 	// follow request status changed so send the UNDO activity to the channel for async processing
 	if frChanged {
-		p.fromClientAPI <- messages.FromClientAPI{
+		p.clientWorker.Queue(messages.FromClientAPI{
 			APObjectType:   ap.ActivityFollow,
 			APActivityType: ap.ActivityUndo,
 			GTSModel: &gtsmodel.Follow{
@@ -123,12 +123,12 @@ func (p *processor) BlockCreate(ctx context.Context, requestingAccount *gtsmodel
 			},
 			OriginAccount: requestingAccount,
 			TargetAccount: targetAccount,
-		}
+		})
 	}
 
 	// follow status changed so send the UNDO activity to the channel for async processing
 	if fChanged {
-		p.fromClientAPI <- messages.FromClientAPI{
+		p.clientWorker.Queue(messages.FromClientAPI{
 			APObjectType:   ap.ActivityFollow,
 			APActivityType: ap.ActivityUndo,
 			GTSModel: &gtsmodel.Follow{
@@ -138,17 +138,17 @@ func (p *processor) BlockCreate(ctx context.Context, requestingAccount *gtsmodel
 			},
 			OriginAccount: requestingAccount,
 			TargetAccount: targetAccount,
-		}
+		})
 	}
 
 	// handle the rest of the block process asynchronously
-	p.fromClientAPI <- messages.FromClientAPI{
+	p.clientWorker.Queue(messages.FromClientAPI{
 		APObjectType:   ap.ActivityBlock,
 		APActivityType: ap.ActivityCreate,
 		GTSModel:       block,
 		OriginAccount:  requestingAccount,
 		TargetAccount:  targetAccount,
-	}
+	})
 
 	return p.RelationshipGet(ctx, requestingAccount, targetAccountID)
 }

internal/processing/account/createfollow.go

@@ -101,13 +101,13 @@ func (p *processor) FollowCreate(ctx context.Context, requestingAccount *gtsmode
 	}
 
 	// otherwise we leave the follow request as it is and we handle the rest of the process asynchronously
-	p.fromClientAPI <- messages.FromClientAPI{
+	p.clientWorker.Queue(messages.FromClientAPI{
 		APObjectType:   ap.ActivityFollow,
 		APActivityType: ap.ActivityCreate,
 		GTSModel:       fr,
 		OriginAccount:  requestingAccount,
 		TargetAccount:  targetAcct,
-	}
+	})
 
 	// return whatever relationship results from this
 	return p.RelationshipGet(ctx, requestingAccount, form.ID)

internal/processing/account/delete.go

@@ -159,13 +159,13 @@ func (p *processor) Delete(ctx context.Context, account *gtsmodel.Account, origi
 			// pass the status delete through the client api channel for processing
 			s.Account = account
 			l.Debug("putting status in the client api channel")
-			p.fromClientAPI <- messages.FromClientAPI{
+			p.clientWorker.Queue(messages.FromClientAPI{
 				APObjectType:   ap.ObjectNote,
 				APActivityType: ap.ActivityDelete,
 				GTSModel:       s,
 				OriginAccount:  account,
 				TargetAccount:  account,
-			}
+			})
 
 			if err := p.db.DeleteByID(ctx, s.ID, s); err != nil {
 				if err != db.ErrNoEntries {
@@ -195,13 +195,13 @@ func (p *processor) Delete(ctx context.Context, account *gtsmodel.Account, origi
 				}
 
 				l.Debug("putting boost undo in the client api channel")
-				p.fromClientAPI <- messages.FromClientAPI{
+				p.clientWorker.Queue(messages.FromClientAPI{
 					APObjectType:   ap.ActivityAnnounce,
 					APActivityType: ap.ActivityUndo,
 					GTSModel:       s,
 					OriginAccount:  b.Account,
 					TargetAccount:  account,
-				}
+				})
 
 				if err := p.db.DeleteByID(ctx, b.ID, b); err != nil {
 					if err != db.ErrNoEntries {
@@ -331,7 +331,7 @@ func (p *processor) DeleteLocal(ctx context.Context, account *gtsmodel.Account,
 	}
 
 	// put the delete in the processor queue to handle the rest of it asynchronously
-	p.fromClientAPI <- fromClientAPIMessage
+	p.clientWorker.Queue(fromClientAPIMessage)
 
 	return nil
 }

internal/processing/account/removeblock.go

@@ -54,13 +54,13 @@ func (p *processor) BlockRemove(ctx context.Context, requestingAccount *gtsmodel
 
 	// block status changed so send the UNDO activity to the channel for async processing
 	if blockChanged {
-		p.fromClientAPI <- messages.FromClientAPI{
+		p.clientWorker.Queue(messages.FromClientAPI{
 			APObjectType:   ap.ActivityBlock,
 			APActivityType: ap.ActivityUndo,
 			GTSModel:       block,
 			OriginAccount:  requestingAccount,
 			TargetAccount:  targetAccount,
-		}
+		})
 	}
 
 	// return whatever relationship results from all this

internal/processing/account/removefollow.go

@@ -80,7 +80,7 @@ func (p *processor) FollowRemove(ctx context.Context, requestingAccount *gtsmode
 
 	// follow request status changed so send the UNDO activity to the channel for async processing
 	if frChanged {
-		p.fromClientAPI <- messages.FromClientAPI{
+		p.clientWorker.Queue(messages.FromClientAPI{
 			APObjectType:   ap.ActivityFollow,
 			APActivityType: ap.ActivityUndo,
 			GTSModel: &gtsmodel.Follow{
@@ -90,12 +90,12 @@ func (p *processor) FollowRemove(ctx context.Context, requestingAccount *gtsmode
 			},
 			OriginAccount: requestingAccount,
 			TargetAccount: targetAcct,
-		}
+		})
 	}
 
 	// follow status changed so send the UNDO activity to the channel for async processing
 	if fChanged {
-		p.fromClientAPI <- messages.FromClientAPI{
+		p.clientWorker.Queue(messages.FromClientAPI{
 			APObjectType:   ap.ActivityFollow,
 			APActivityType: ap.ActivityUndo,
 			GTSModel: &gtsmodel.Follow{
@@ -105,7 +105,7 @@ func (p *processor) FollowRemove(ctx context.Context, requestingAccount *gtsmode
 			},
 			OriginAccount: requestingAccount,
 			TargetAccount: targetAcct,
-		}
+		})
 	}
 
 	// return whatever relationship results from all this

internal/processing/account/update.go

@@ -117,12 +117,12 @@ func (p *processor) Update(ctx context.Context, account *gtsmodel.Account, form
 		return nil, fmt.Errorf("could not update account %s: %s", account.ID, err)
 	}
 
-	p.fromClientAPI <- messages.FromClientAPI{
+	p.clientWorker.Queue(messages.FromClientAPI{
 		APObjectType:   ap.ObjectProfile,
 		APActivityType: ap.ActivityUpdate,
 		GTSModel:       updatedAccount,
 		OriginAccount:  updatedAccount,
-	}
+	})
 
 	acctSensitive, err := p.tc.AccountToAPIAccountSensitive(ctx, updatedAccount)
 	if err != nil {

internal/processing/admin/accountaction.go

@@ -34,12 +34,12 @@ func (p *processor) AccountAction(ctx context.Context, account *gtsmodel.Account
 	case string(gtsmodel.AdminActionSuspend):
 		adminAction.Type = gtsmodel.AdminActionSuspend
 		// pass the account delete through the client api channel for processing
-		p.fromClientAPI <- messages.FromClientAPI{
+		p.clientWorker.Queue(messages.FromClientAPI{
 			APObjectType:   ap.ActorPerson,
 			APActivityType: ap.ActivityDelete,
 			OriginAccount:  account,
 			TargetAccount:  targetAccount,
-		}
+		})
 	default:
 		return gtserror.NewErrorBadRequest(fmt.Errorf("admin action type %s is not supported for this endpoint", form.Type))
 	}

internal/processing/admin/admin.go

@@ -29,6 +29,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 )
 
 // Processor wraps a bunch of functions for processing admin actions.
@@ -43,18 +44,18 @@ type Processor interface {
 }
 
 type processor struct {
-	tc            typeutils.TypeConverter
+	tc           typeutils.TypeConverter
-	mediaManager  media.Manager
+	mediaManager media.Manager
-	fromClientAPI chan messages.FromClientAPI
+	clientWorker *worker.Worker[messages.FromClientAPI]
-	db            db.DB
+	db           db.DB
 }
 
 // New returns a new admin processor.
-func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, fromClientAPI chan messages.FromClientAPI) Processor {
+func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, clientWorker *worker.Worker[messages.FromClientAPI]) Processor {
 	return &processor{
-		tc:            tc,
+		tc:           tc,
-		mediaManager:  mediaManager,
+		mediaManager: mediaManager,
-		fromClientAPI: fromClientAPI,
+		clientWorker: clientWorker,
-		db:            db,
+		db:           db,
 	}
 }

internal/processing/admin/createdomainblock.go

@@ -141,13 +141,13 @@ func (p *processor) initiateDomainBlockSideEffects(ctx context.Context, account
 			l.Debugf("putting delete for account %s in the clientAPI channel", a.Username)
 
 			// pass the account delete through the client api channel for processing
-			p.fromClientAPI <- messages.FromClientAPI{
+			p.clientWorker.Queue(messages.FromClientAPI{
 				APObjectType:   ap.ActorPerson,
 				APActivityType: ap.ActivityDelete,
 				GTSModel:       block,
 				OriginAccount:  account,
 				TargetAccount:  a,
-			}
+			})
 
 			// if this is the last account in the slice, set the maxID appropriately for the next query
 			if i == len(accounts)-1 {

internal/processing/federation/federation.go

@@ -27,7 +27,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
-	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 )
@@ -81,20 +80,18 @@ type Processor interface {
 }
 
 type processor struct {
-	db            db.DB
+	db        db.DB
-	federator     federation.Federator
+	federator federation.Federator
-	tc            typeutils.TypeConverter
+	tc        typeutils.TypeConverter
-	filter        visibility.Filter
+	filter    visibility.Filter
-	fromFederator chan messages.FromFederator
 }
 
 // New returns a new federation processor.
-func New(db db.DB, tc typeutils.TypeConverter, federator federation.Federator, fromFederator chan messages.FromFederator) Processor {
+func New(db db.DB, tc typeutils.TypeConverter, federator federation.Federator) Processor {
 	return &processor{
-		db:            db,
+		db:        db,
-		federator:     federator,
+		federator: federator,
-		tc:            tc,
+		tc:        tc,
-		filter:        visibility.NewFilter(db),
+		filter:    visibility.NewFilter(db),
-		fromFederator: fromFederator,
 	}
 }

internal/processing/federation/postinbox.go

@@ -21,12 +21,8 @@
 import (
 	"context"
 	"net/http"
-
-	"github.com/superseriousbusiness/gotosocial/internal/ap"
 )
 
 func (p *processor) PostInbox(ctx context.Context, w http.ResponseWriter, r *http.Request) (bool, error) {
-	// pass the fromFederator channel through to postInbox, since it'll be needed later
+	return p.federator.FederatingActor().PostInbox(ctx, w, r)
-	contextWithChannel := context.WithValue(ctx, ap.ContextFromFederatorChan, p.fromFederator)
-	return p.federator.FederatingActor().PostInbox(contextWithChannel, w, r)
 }

internal/processing/followrequest.go

@@ -78,13 +78,13 @@ func (p *processor) FollowRequestAccept(ctx context.Context, auth *oauth.Auth, a
 		follow.TargetAccount = followTargetAccount
 	}
 
-	p.fromClientAPI <- messages.FromClientAPI{
+	p.clientWorker.Queue(messages.FromClientAPI{
 		APObjectType:   ap.ActivityFollow,
 		APActivityType: ap.ActivityAccept,
 		GTSModel:       follow,
 		OriginAccount:  follow.Account,
 		TargetAccount:  follow.TargetAccount,
-	}
+	})
 
 	gtsR, err := p.db.GetRelationship(ctx, auth.Account.ID, accountID)
 	if err != nil {
@@ -121,13 +121,13 @@ func (p *processor) FollowRequestReject(ctx context.Context, auth *oauth.Auth, a
 		followRequest.TargetAccount = a
 	}
 
-	p.fromClientAPI <- messages.FromClientAPI{
+	p.clientWorker.Queue(messages.FromClientAPI{
 		APObjectType:   ap.ActivityFollow,
 		APActivityType: ap.ActivityReject,
 		GTSModel:       followRequest,
 		OriginAccount:  followRequest.Account,
 		TargetAccount:  followRequest.TargetAccount,
-	}
+	})
 
 	gtsR, err := p.db.GetRelationship(ctx, auth.Account.ID, accountID)
 	if err != nil {

internal/processing/media/media_test.go

@@ -29,9 +29,11 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	mediaprocessing "github.com/superseriousbusiness/gotosocial/internal/processing/media"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -120,6 +122,7 @@ func (suite *MediaStandardTestSuite) mockTransportController() transport.Control
 
 		return response, nil
 	}
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
 	mockClient := testrig.NewMockHTTPClient(do)
-	return testrig.NewTestTransportController(mockClient, suite.db)
+	return testrig.NewTestTransportController(mockClient, suite.db, fedWorker)
 }

internal/processing/processor.go

@@ -24,7 +24,6 @@
 	"net/url"
 
 	"codeberg.org/gruf/go-store/kv"
-	"github.com/sirupsen/logrus"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -45,6 +44,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/timeline"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 	"github.com/superseriousbusiness/gotosocial/internal/visibility"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 )
 
 // Processor should be passed to api modules (see internal/apimodule/...). It is used for
@@ -55,7 +55,7 @@
 // for clean distribution of messages without slowing down the client API and harming the user experience.
 type Processor interface {
 	// Start starts the Processor, reading from its channels and passing messages back and forth.
-	Start(ctx context.Context) error
+	Start() error
 	// Stop stops the processor cleanly, finishing handling any remaining messages before closing down.
 	Stop() error
 	// ProcessFromClientAPI processes one message coming from the clientAPI channel, and triggers appropriate side effects.
@@ -235,10 +235,10 @@ type Processor interface {
 
 // processor just implements the Processor interface
 type processor struct {
-	fromClientAPI   chan messages.FromClientAPI
+	clientWorker *worker.Worker[messages.FromClientAPI]
-	fromFederator   chan messages.FromFederator
+	fedWorker    *worker.Worker[messages.FromFederator]
+
 	federator       federation.Federator
-	stop            chan interface{}
 	tc              typeutils.TypeConverter
 	oauthServer     oauth.Server
 	mediaManager    media.Manager
@@ -268,26 +268,26 @@ func NewProcessor(
 	mediaManager media.Manager,
 	storage *kv.KVStore,
 	db db.DB,
-	emailSender email.Sender) Processor {
+	emailSender email.Sender,
-
+	clientWorker *worker.Worker[messages.FromClientAPI],
-	fromClientAPI := make(chan messages.FromClientAPI, 1000)
+	fedWorker *worker.Worker[messages.FromFederator],
-	fromFederator := make(chan messages.FromFederator, 1000)
+) Processor {
 	parseMentionFunc := GetParseMentionFunc(db, federator)
 
-	statusProcessor := status.New(db, tc, fromClientAPI, parseMentionFunc)
+	statusProcessor := status.New(db, tc, clientWorker, parseMentionFunc)
 	streamingProcessor := streaming.New(db, oauthServer)
-	accountProcessor := account.New(db, tc, mediaManager, oauthServer, fromClientAPI, federator, parseMentionFunc)
+	accountProcessor := account.New(db, tc, mediaManager, oauthServer, clientWorker, federator, parseMentionFunc)
-	adminProcessor := admin.New(db, tc, mediaManager, fromClientAPI)
+	adminProcessor := admin.New(db, tc, mediaManager, clientWorker)
 	mediaProcessor := mediaProcessor.New(db, tc, mediaManager, federator.TransportController(), storage)
 	userProcessor := user.New(db, emailSender)
-	federationProcessor := federationProcessor.New(db, tc, federator, fromFederator)
+	federationProcessor := federationProcessor.New(db, tc, federator)
 	filter := visibility.NewFilter(db)
 
 	return &processor{
-		fromClientAPI:   fromClientAPI,
+		clientWorker: clientWorker,
-		fromFederator:   fromFederator,
+		fedWorker:    fedWorker,
+
 		federator:       federator,
-		stop:            make(chan interface{}),
 		tc:              tc,
 		oauthServer:     oauthServer,
 		mediaManager:    mediaManager,
@@ -307,36 +307,29 @@ func NewProcessor(
 }
 
 // Start starts the Processor, reading from its channels and passing messages back and forth.
-func (p *processor) Start(ctx context.Context) error {
+func (p *processor) Start() error {
-	go func() {
+	// Setup and start the client API worker pool
-	DistLoop:
+	p.clientWorker.SetProcessor(p.ProcessFromClientAPI)
-		for {
+	if err := p.clientWorker.Start(); err != nil {
-			select {
+		return err
-			case clientMsg := <-p.fromClientAPI:
+	}
-				logrus.Tracef("received message FROM client API: %+v", clientMsg)
+
-				go func() {
+	// Setup and start the federator worker pool
-					if err := p.ProcessFromClientAPI(ctx, clientMsg); err != nil {
+	p.fedWorker.SetProcessor(p.ProcessFromFederator)
-						logrus.Error(err)
+	if err := p.fedWorker.Start(); err != nil {
-					}
+		return err
-				}()
+	}
-			case federatorMsg := <-p.fromFederator:
+
-				logrus.Tracef("received message FROM federator: %+v", federatorMsg)
-				go func() {
-					if err := p.ProcessFromFederator(ctx, federatorMsg); err != nil {
-						logrus.Error(err)
-					}
-				}()
-			case <-p.stop:
-				break DistLoop
-			}
-		}
-	}()
 	return nil
 }
 
 // Stop stops the processor cleanly, finishing handling any remaining messages before closing down.
-// TODO: empty message buffer properly before stopping otherwise we'll lose federating messages.
 func (p *processor) Stop() error {
-	close(p.stop)
+	if err := p.clientWorker.Stop(); err != nil {
+		return err
+	}
+	if err := p.fedWorker.Stop(); err != nil {
+		return err
+	}
 	return nil
 }

internal/processing/processor_test.go

@@ -34,11 +34,13 @@
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/timeline"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -215,17 +217,20 @@ func (suite *ProcessingStandardTestSuite) SetupTest() {
 		}, nil
 	})
 
-	suite.transportController = testrig.NewTestTransportController(httpClient, suite.db)
+	clientWorker := worker.New[messages.FromClientAPI](-1, -1)
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
+	suite.transportController = testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager)
+	suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager, fedWorker)
 	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
 	suite.emailSender = testrig.NewEmailSender("../../web/template/", nil)
 
-	suite.processor = processing.NewProcessor(suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaManager, suite.storage, suite.db, suite.emailSender)
+	suite.processor = processing.NewProcessor(suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaManager, suite.storage, suite.db, suite.emailSender, clientWorker, fedWorker)
 
 	testrig.StandardDBSetup(suite.db, suite.testAccounts)
 	testrig.StandardStorageSetup(suite.storage, "../../testrig/media")
-	if err := suite.processor.Start(context.Background()); err != nil {
+	if err := suite.processor.Start(); err != nil {
 		panic(err)
 	}
 }

internal/processing/status/boost.go

@@ -65,13 +65,13 @@ func (p *processor) Boost(ctx context.Context, requestingAccount *gtsmodel.Accou
 	}
 
 	// send it back to the processor for async processing
-	p.fromClientAPI <- messages.FromClientAPI{
+	p.clientWorker.Queue(messages.FromClientAPI{
 		APObjectType:   ap.ActivityAnnounce,
 		APActivityType: ap.ActivityCreate,
 		GTSModel:       boostWrapperStatus,
 		OriginAccount:  requestingAccount,
 		TargetAccount:  targetStatus.Account,
-	}
+	})
 
 	// return the frontend representation of the new status to the submitter
 	apiStatus, err := p.tc.StatusToAPIStatus(ctx, boostWrapperStatus, requestingAccount)

internal/processing/status/create.go

@@ -97,12 +97,12 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, appli
 	}
 
 	// send it back to the processor for async processing
-	p.fromClientAPI <- messages.FromClientAPI{
+	p.clientWorker.Queue(messages.FromClientAPI{
 		APObjectType:   ap.ObjectNote,
 		APActivityType: ap.ActivityCreate,
 		GTSModel:       newStatus,
 		OriginAccount:  account,
-	}
+	})
 
 	// return the frontend representation of the new status to the submitter
 	apiStatus, err := p.tc.StatusToAPIStatus(ctx, newStatus, account)

internal/processing/status/delete.go

@@ -53,13 +53,13 @@ func (p *processor) Delete(ctx context.Context, requestingAccount *gtsmodel.Acco
 	}
 
 	// send it back to the processor for async processing
-	p.fromClientAPI <- messages.FromClientAPI{
+	p.clientWorker.Queue(messages.FromClientAPI{
 		APObjectType:   ap.ObjectNote,
 		APActivityType: ap.ActivityDelete,
 		GTSModel:       targetStatus,
 		OriginAccount:  requestingAccount,
 		TargetAccount:  requestingAccount,
-	}
+	})
 
 	return apiStatus, nil
 }

internal/processing/status/fave.go

@@ -84,13 +84,13 @@ func (p *processor) Fave(ctx context.Context, requestingAccount *gtsmodel.Accoun
 		}
 
 		// send it back to the processor for async processing
-		p.fromClientAPI <- messages.FromClientAPI{
+		p.clientWorker.Queue(messages.FromClientAPI{
 			APObjectType:   ap.ActivityLike,
 			APActivityType: ap.ActivityCreate,
 			GTSModel:       gtsFave,
 			OriginAccount:  requestingAccount,
 			TargetAccount:  targetStatus.Account,
-		}
+		})
 	}
 
 	// return the apidon representation of the target status

internal/processing/status/status.go

@@ -29,6 +29,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/text"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 	"github.com/superseriousbusiness/gotosocial/internal/visibility"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 )
 
 // Processor wraps a bunch of functions for processing statuses.
@@ -69,22 +70,22 @@ type Processor interface {
 }
 
 type processor struct {
-	tc            typeutils.TypeConverter
+	tc           typeutils.TypeConverter
-	db            db.DB
+	db           db.DB
-	filter        visibility.Filter
+	filter       visibility.Filter
-	formatter     text.Formatter
+	formatter    text.Formatter
-	fromClientAPI chan messages.FromClientAPI
+	clientWorker *worker.Worker[messages.FromClientAPI]
-	parseMention  gtsmodel.ParseMentionFunc
+	parseMention gtsmodel.ParseMentionFunc
 }
 
 // New returns a new status processor.
-func New(db db.DB, tc typeutils.TypeConverter, fromClientAPI chan messages.FromClientAPI, parseMention gtsmodel.ParseMentionFunc) Processor {
+func New(db db.DB, tc typeutils.TypeConverter, clientWorker *worker.Worker[messages.FromClientAPI], parseMention gtsmodel.ParseMentionFunc) Processor {
 	return &processor{
-		tc:            tc,
+		tc:           tc,
-		db:            db,
+		db:           db,
-		filter:        visibility.NewFilter(db),
+		filter:       visibility.NewFilter(db),
-		formatter:     text.NewFormatter(db),
+		formatter:    text.NewFormatter(db),
-		fromClientAPI: fromClientAPI,
+		clientWorker: clientWorker,
-		parseMention:  parseMention,
+		parseMention: parseMention,
 	}
 }

internal/processing/status/status_test.go

@@ -30,18 +30,19 @@
 	"github.com/superseriousbusiness/gotosocial/internal/processing/status"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
 type StatusStandardTestSuite struct {
 	suite.Suite
-	db                db.DB
+	db            db.DB
-	typeConverter     typeutils.TypeConverter
+	typeConverter typeutils.TypeConverter
-	tc                transport.Controller
+	tc            transport.Controller
-	storage           *kv.KVStore
+	storage       *kv.KVStore
-	mediaManager      media.Manager
+	mediaManager  media.Manager
-	federator         federation.Federator
+	federator     federation.Federator
-	fromClientAPIChan chan messages.FromClientAPI
+	clientWorker  *worker.Worker[messages.FromClientAPI]
 
 	// standard suite models
 	testTokens       map[string]*gtsmodel.Token
@@ -74,14 +75,16 @@ func (suite *StatusStandardTestSuite) SetupTest() {
 	testrig.InitTestConfig()
 	testrig.InitTestLog()
 
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+
 	suite.db = testrig.NewTestDB()
 	suite.typeConverter = testrig.NewTestTypeConverter(suite.db)
-	suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100)
+	suite.clientWorker = worker.New[messages.FromClientAPI](-1, -1)
-	suite.tc = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	suite.tc = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
 	suite.storage = testrig.NewTestStorage()
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.federator = testrig.NewTestFederator(suite.db, suite.tc, suite.storage, suite.mediaManager)
+	suite.federator = testrig.NewTestFederator(suite.db, suite.tc, suite.storage, suite.mediaManager, fedWorker)
-	suite.status = status.New(suite.db, suite.typeConverter, suite.fromClientAPIChan, processing.GetParseMentionFunc(suite.db, suite.federator))
+	suite.status = status.New(suite.db, suite.typeConverter, suite.clientWorker, processing.GetParseMentionFunc(suite.db, suite.federator))
 
 	testrig.StandardDBSetup(suite.db, suite.testAccounts)
 	testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")

internal/processing/status/unboost.go

@@ -91,13 +91,13 @@ func (p *processor) Unboost(ctx context.Context, requestingAccount *gtsmodel.Acc
 		gtsBoost.BoostOf.Account = targetStatus.Account
 
 		// send it back to the processor for async processing
-		p.fromClientAPI <- messages.FromClientAPI{
+		p.clientWorker.Queue(messages.FromClientAPI{
 			APObjectType:   ap.ActivityAnnounce,
 			APActivityType: ap.ActivityUndo,
 			GTSModel:       gtsBoost,
 			OriginAccount:  requestingAccount,
 			TargetAccount:  targetStatus.Account,
-		}
+		})
 	}
 
 	apiStatus, err := p.tc.StatusToAPIStatus(ctx, targetStatus, requestingAccount)

internal/processing/status/unfave.go

@@ -73,13 +73,13 @@ func (p *processor) Unfave(ctx context.Context, requestingAccount *gtsmodel.Acco
 		}
 
 		// send it back to the processor for async processing
-		p.fromClientAPI <- messages.FromClientAPI{
+		p.clientWorker.Queue(messages.FromClientAPI{
 			APObjectType:   ap.ActivityLike,
 			APActivityType: ap.ActivityUndo,
 			GTSModel:       gtsFave,
 			OriginAccount:  requestingAccount,
 			TargetAccount:  targetStatus.Account,
-		}
+		})
 	}
 
 	apiStatus, err := p.tc.StatusToAPIStatus(ctx, targetStatus, requestingAccount)

internal/worker/workers.go

@@ -0,0 +1,69 @@
+package worker
+
+import (
+	"context"
+	"errors"
+	"runtime"
+
+	"codeberg.org/gruf/go-runners"
+	"github.com/sirupsen/logrus"
+)
+
+// Worker represents a proccessor for MsgType objects, using a worker pool to allocate resources.
+type Worker[MsgType any] struct {
+	workers runners.WorkerPool
+	process func(context.Context, MsgType) error
+}
+
+// New returns a new Worker[MsgType] with given number of workers and queue size
+// (see runners.WorkerPool for more information on args). If args < 1 then suitable
+// defaults are determined from the runtime's GOMAXPROCS variable.
+func New[MsgType any](workers int, queue int) *Worker[MsgType] {
+	if workers < 1 {
+		workers = runtime.GOMAXPROCS(0)
+	}
+	if queue < 1 {
+		queue = workers * 100
+	}
+	return &Worker[MsgType]{
+		workers: runners.NewWorkerPool(workers, queue),
+		process: nil,
+	}
+}
+
+// Start will attempt to start the underlying worker pool, or return error.
+func (w *Worker[MsgType]) Start() error {
+	if w.process == nil {
+		return errors.New("nil Worker.process function")
+	}
+	if !w.workers.Start() {
+		return errors.New("failed to start Worker pool")
+	}
+	return nil
+}
+
+// Stop will attempt to stop the underlying worker pool, or return error.
+func (w *Worker[MsgType]) Stop() error {
+	if !w.workers.Stop() {
+		return errors.New("failed to stop Worker pool")
+	}
+	return nil
+}
+
+// SetProcessor will set the Worker's processor function, which is called for each queued message.
+func (w *Worker[MsgType]) SetProcessor(fn func(context.Context, MsgType) error) {
+	if w.process != nil {
+		logrus.Panic("Worker.process is already set")
+	}
+	w.process = fn
+}
+
+// Queue will queue provided message to be processed with there's a free worker.
+func (w *Worker[MsgType]) Queue(msg MsgType) {
+	logrus.Tracef("queueing %[1]T message; %+[1]v", msg)
+	w.workers.Enqueue(func(ctx context.Context) {
+		if err := w.process(ctx, msg); err != nil {
+			logrus.Error(err)
+		}
+	})
+}

testrig/federatingdb.go

@@ -3,9 +3,11 @@
 import (
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 )
 
 // NewTestFederatingDB returns a federating DB with the underlying db
-func NewTestFederatingDB(db db.DB) federatingdb.DB {
+func NewTestFederatingDB(db db.DB, fedWorker *worker.Worker[messages.FromFederator]) federatingdb.DB {
-	return federatingdb.New(db)
+	return federatingdb.New(db, fedWorker)
 }

testrig/federator.go

@@ -23,10 +23,12 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 )
 
 // NewTestFederator returns a federator with the given database and (mock!!) transport controller.
-func NewTestFederator(db db.DB, tc transport.Controller, storage *kv.KVStore, mediaManager media.Manager) federation.Federator {
+func NewTestFederator(db db.DB, tc transport.Controller, storage *kv.KVStore, mediaManager media.Manager, fedWorker *worker.Worker[messages.FromFederator]) federation.Federator {
-	return federation.NewFederator(db, NewTestFederatingDB(db), tc, NewTestTypeConverter(db), mediaManager)
+	return federation.NewFederator(db, NewTestFederatingDB(db, fedWorker), tc, NewTestTypeConverter(db), mediaManager)
 }

testrig/processor.go

@@ -24,10 +24,12 @@
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 )
 
 // NewTestProcessor returns a Processor suitable for testing purposes
-func NewTestProcessor(db db.DB, storage *kv.KVStore, federator federation.Federator, emailSender email.Sender, mediaManager media.Manager) processing.Processor {
+func NewTestProcessor(db db.DB, storage *kv.KVStore, federator federation.Federator, emailSender email.Sender, mediaManager media.Manager, clientWorker *worker.Worker[messages.FromClientAPI], fedWorker *worker.Worker[messages.FromFederator]) processing.Processor {
-	return processing.NewProcessor(NewTestTypeConverter(db), federator, NewTestOauthServer(db), mediaManager, storage, db, emailSender)
+	return processing.NewProcessor(NewTestTypeConverter(db), federator, NewTestOauthServer(db), mediaManager, storage, db, emailSender, clientWorker, fedWorker)
 }

testrig/testmodels.go

@@ -42,6 +42,8 @@
 	"github.com/superseriousbusiness/activity/streams/vocab"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 )
 
 // NewTestTokens returns a map of tokens keyed according to which account the token belongs to.
@@ -505,7 +507,7 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
 	}
 
 	if diff := len(accounts) - len(preserializedKeys); diff > 0 {
-		var keyStrings = make([]string, diff)
+		keyStrings := make([]string, diff)
 		for i := 0; i < diff; i++ {
 			priv, _ := rsa.GenerateKey(rand.Reader, 2048)
 			key, _ := x509.MarshalPKCS8PrivateKey(priv)
@@ -1823,8 +1825,13 @@ func GetSignatureForActivity(activity pub.Activity, pubKeyID string, privkey cry
 		},
 	}
 
+	// Create temporary federator worker for transport controller
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+	_ = fedWorker.Start()
+	defer func() { _ = fedWorker.Stop() }()
+
 	// use the client to create a new transport
-	c := NewTestTransportController(client, NewTestDB())
+	c := NewTestTransportController(client, NewTestDB(), fedWorker)
 	tp, err := c.NewTransport(pubKeyID, privkey)
 	if err != nil {
 		panic(err)
@@ -1865,8 +1872,13 @@ func GetSignatureForDereference(pubKeyID string, privkey crypto.PrivateKey, dest
 		},
 	}
 
+	// Create temporary federator worker for transport controller
+	fedWorker := worker.New[messages.FromFederator](-1, -1)
+	_ = fedWorker.Start()
+	defer func() { _ = fedWorker.Stop() }()
+
 	// use the client to create a new transport
-	c := NewTestTransportController(client, NewTestDB())
+	c := NewTestTransportController(client, NewTestDB(), fedWorker)
 	tp, err := c.NewTransport(pubKeyID, privkey)
 	if err != nil {
 		panic(err)
@@ -1899,7 +1911,8 @@ func newAPPerson(
 	avatarContentType string,
 	headerURL *url.URL,
 	headerContentType string,
-	manuallyApprovesFollowers bool) vocab.ActivityStreamsPerson {
+	manuallyApprovesFollowers bool,
+) vocab.ActivityStreamsPerson {
 	person := streams.NewActivityStreamsPerson()
 
 	// id should be the activitypub URI of this user
@@ -2082,7 +2095,8 @@ func newAPGroup(
 	avatarContentType string,
 	headerURL *url.URL,
 	headerContentType string,
-	manuallyApprovesFollowers bool) vocab.ActivityStreamsGroup {
+	manuallyApprovesFollowers bool,
+) vocab.ActivityStreamsGroup {
 	group := streams.NewActivityStreamsGroup()
 
 	// id should be the activitypub URI of this group
@@ -2303,8 +2317,8 @@ func newAPNote(
 	noteCC []*url.URL,
 	noteSensitive bool,
 	noteMentions []vocab.ActivityStreamsMention,
-	noteAttachments []vocab.ActivityStreamsImage) vocab.ActivityStreamsNote {
+	noteAttachments []vocab.ActivityStreamsImage,
-
+) vocab.ActivityStreamsNote {
 	// create the note itself
 	note := streams.NewActivityStreamsNote()
 

testrig/transportcontroller.go

@@ -26,7 +26,9 @@
 	"github.com/superseriousbusiness/activity/pub"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
+	"github.com/superseriousbusiness/gotosocial/internal/worker"
 )
 
 // NewTestTransportController returns a test transport controller with the given http client.
@@ -38,8 +40,8 @@
 // Unlike the other test interfaces provided in this package, you'll probably want to call this function
 // PER TEST rather than per suite, so that the do function can be set on a test by test (or even more granular)
 // basis.
-func NewTestTransportController(client pub.HttpClient, db db.DB) transport.Controller {
+func NewTestTransportController(client pub.HttpClient, db db.DB, fedWorker *worker.Worker[messages.FromFederator]) transport.Controller {
-	return transport.NewController(db, NewTestFederatingDB(db), &federation.Clock{}, client)
+	return transport.NewController(db, NewTestFederatingDB(db, fedWorker), &federation.Clock{}, client)
 }
 
 // NewMockHTTPClient returns a client that conforms to the pub.HttpClient interface,