[feature] Fetch + create domain permissions from subscriptions nightly (#3635)

* peepeepoopoo

* test domain perm subs

* swagger

* envparsing

* dries your wets

* start on docs

* finish up docs

* copy paste errors

* rename actions package

* rename force -> skipCache

* move obfuscate parse nearer to where err is checked

* make higherPrios a simple slice

* don't use receiver for permsFrom funcs

* add more context to error logs

* defer finished log

* use switch for permType instead of if/else

* thanks linter, love you <3

* validate csv headers before full read

* use bufio scanner

README.md

@@ -225,10 +225,11 @@ Simply download the binary + assets (or Docker container), tweak your configurat
 
 ### Safety + security features
 
-- Built-in, automatic support for secure HTTPS with [Let's Encrypt](https://letsencrypt.org/).
+- Strict privacy enforcement for posts, and strict blocking logic.
-- Strict privacy enforcement for posts and strict blocking logic.
+- [Choose the visibility of posts on the web view of your profile](https://docs.gotosocial.org/en/latest/user_guide/settings/#visibility-level-of-posts-to-show-on-your-profile).
-- Import and export allow lists and deny lists. Subscribe to community-created block lists (think Ad blocker, but for federation!) (feature still in progress).
+- [Import, export](https://docs.gotosocial.org/en/latest/admin/settings/#importexport), and [subscribe](https://docs.gotosocial.org/en/latest/admin/domain_permission_subscriptions) to community-created domain allow and domain block lists.
 - HTTP signature authentication: GoToSocial requires [HTTP Signatures](https://datatracker.ietf.org/doc/html/draft-cavage-http-signatures-12) when sending and receiving messages, to ensure that your messages can't be tampered with and your identity can't be forged.
+- Built-in, automatic support for secure HTTPS with [Let's Encrypt](https://letsencrypt.org/).
 
 ### Various federation modes
 

ROADMAP.md

@@ -71,7 +71,7 @@ These are provided in no specific order.
 - [x] **Filters v2** -- implement v2 of the filters API.
 - [x] **Mute accounts** -- mute accounts to prevent their posts showing up in your home timeline (optional: for limited period of time).
 - [x] **Non-replyable posts** -- design a non-replyable post path for GoToSocial based on https://github.com/mastodon/mastodon/issues/14762#issuecomment-1196889788; allow users to create non-replyable posts.
-- [ ] **Block + allow list subscriptions** -- allow instance admins to subscribe their instance to plaintext domain block/allow lists (much of the work for this is already in place).
+- [x] **Block + allow list subscriptions** -- allow instance admins to subscribe their instance to domain block/allow lists.
 - [x] **Direct conversation view** -- allow users to easily page through all direct-message conversations they're a part of.
 - [ ] **Oauth token management** -- create / view / invalidate OAuth tokens via the settings panel.
 - [ ] **Status EDIT support** -- edit statuses that you've created, without having to delete + redraft. Federate edits out properly.

cmd/gotosocial/action/admin/account/account.go

@@ -40,7 +40,8 @@ func initState(ctx context.Context) (*state.State, error) {
 	state.Caches.Init()
 	state.Caches.Start()
 
-	// Set the state DB connection
+	// Only set state DB connection.
+	// Don't need Actions or Workers for this (yet).
 	dbConn, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
 		return nil, fmt.Errorf("error creating dbConn: %w", err)

cmd/gotosocial/action/admin/media/list.go

@@ -127,6 +127,8 @@ func setupList(ctx context.Context) (*list, error) {
 	state.Caches.Init()
 	state.Caches.Start()
 
+	// Only set state DB connection.
+	// Don't need Actions or Workers for this.
 	dbService, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
 		return nil, fmt.Errorf("error creating dbservice: %w", err)

cmd/gotosocial/action/admin/media/prune/common.go

@@ -45,10 +45,12 @@ func setupPrune(ctx context.Context) (*prune, error) {
 	state.Caches.Start()
 
 	// Scheduler is required for the
-	// claner, but no other workers
+	// cleaner, but no other workers
 	// are needed for this CLI action.
 	state.Workers.StartScheduler()
 
+	// Set state DB connection.
+	// Don't need Actions for this.
 	dbService, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
 		return nil, fmt.Errorf("error creating dbservice: %w", err)

cmd/gotosocial/action/admin/trans/export.go

@@ -33,12 +33,12 @@
 var Export action.GTSAction = func(ctx context.Context) error {
 	var state state.State
 
+	// Only set state DB connection.
+	// Don't need Actions or Workers for this.
 	dbConn, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
 		return fmt.Errorf("error creating dbservice: %s", err)
 	}
-
-	// Set the state DB connection
 	state.DB = dbConn
 
 	exporter := trans.NewExporter(dbConn)

cmd/gotosocial/action/admin/trans/import.go

@@ -33,12 +33,12 @@
 var Import action.GTSAction = func(ctx context.Context) error {
 	var state state.State
 
+	// Only set state DB connection.
+	// Don't need Actions or Workers for this.
 	dbConn, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
 		return fmt.Errorf("error creating dbservice: %s", err)
 	}
-
-	// Set the state DB connection
 	state.DB = dbConn
 
 	importer := trans.NewImporter(dbConn)

cmd/gotosocial/action/server/server.go

@@ -32,6 +32,7 @@
 	"github.com/KimMachineGun/automemlimit/memlimit"
 	"github.com/gin-gonic/gin"
 	"github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api"
 	apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
 	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
@@ -44,6 +45,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/metrics"
 	"github.com/superseriousbusiness/gotosocial/internal/middleware"
 	tlprocessor "github.com/superseriousbusiness/gotosocial/internal/processing/timeline"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
 	"github.com/superseriousbusiness/gotosocial/internal/timeline"
 	"github.com/superseriousbusiness/gotosocial/internal/tracing"
 	"go.uber.org/automaxprocs/maxprocs"
@@ -164,6 +166,10 @@
 	// Set DB on state.
 	state.DB = dbService
 
+	// Set Actions on state, providing workers to
+	// Actions as well for triggering side effects.
+	state.AdminActions = admin.New(dbService, &state.Workers)
+
 	// Ensure necessary database instance prerequisites exist.
 	if err := dbService.CreateInstanceAccount(ctx); err != nil {
 		return fmt.Errorf("error creating instance account: %s", err)
@@ -283,15 +289,18 @@ func(context.Context, time.Time) {
 	// Create background cleaner.
 	cleaner := cleaner.New(state)
 
-	// Now schedule background cleaning tasks.
+	// Create subscriptions fetcher.
-	if err := cleaner.ScheduleJobs(); err != nil {
+	subscriptions := subscriptions.New(
-		return fmt.Errorf("error scheduling cleaner jobs: %w", err)
+		state,
-	}
+		transportController,
+		typeConverter,
+	)
 
 	// Create the processor using all the
 	// other services we've created so far.
 	process = processing.NewProcessor(
 		cleaner,
+		subscriptions,
 		typeConverter,
 		federator,
 		oauthServer,
@@ -302,6 +311,16 @@ func(context.Context, time.Time) {
 		intFilter,
 	)
 
+	// Schedule background cleaning tasks.
+	if err := cleaner.ScheduleJobs(); err != nil {
+		return fmt.Errorf("error scheduling cleaner jobs: %w", err)
+	}
+
+	// Schedule background subscriptions updating.
+	if err := subscriptions.ScheduleJobs(); err != nil {
+		return fmt.Errorf("error scheduling subscriptions jobs: %w", err)
+	}
+
 	// Initialize the specialized workers pools.
 	state.Workers.Client.Init(messages.ClientMsgIndices())
 	state.Workers.Federator.Init(messages.FederatorMsgIndices())

cmd/gotosocial/action/testrig/testrig.go

@@ -20,11 +20,9 @@
 package testrig
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
-	"io"
 	"net/http"
 	"os"
 	"os/signal"
@@ -47,6 +45,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/router"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
 	"github.com/superseriousbusiness/gotosocial/internal/timeline"
 	"github.com/superseriousbusiness/gotosocial/internal/tracing"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
@@ -159,16 +158,8 @@
 	testrig.StandardStorageSetup(state.Storage, "./testrig/media")
 
 	// build backend handlers
-	transportController := testrig.NewTestTransportController(state, testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
+	httpClient := testrig.NewMockHTTPClient(nil, "./testrig/media")
-		r := io.NopCloser(bytes.NewReader([]byte{}))
+	transportController := testrig.NewTestTransportController(state, httpClient)
-		return &http.Response{
-			StatusCode: 200,
-			Body:       r,
-			Header: http.Header{
-				"Content-Type": req.Header.Values("Accept"),
-			},
-		}, nil
-	}, ""))
 	mediaManager := testrig.NewTestMediaManager(state)
 	federator := testrig.NewTestFederator(state, transportController, mediaManager)
 
@@ -314,11 +305,23 @@
 	// Create background cleaner.
 	cleaner := cleaner.New(state)
 
-	// Now schedule background cleaning tasks.
+	// Schedule background cleaning tasks.
 	if err := cleaner.ScheduleJobs(); err != nil {
 		return fmt.Errorf("error scheduling cleaner jobs: %w", err)
 	}
 
+	// Create subscriptions fetcher.
+	subscriptions := subscriptions.New(
+		state,
+		transportController,
+		typeConverter,
+	)
+
+	// Schedule background subscriptions updating.
+	if err := subscriptions.ScheduleJobs(); err != nil {
+		return fmt.Errorf("error scheduling subscriptions jobs: %w", err)
+	}
+
 	// Finally start the main http server!
 	if err := route.Start(); err != nil {
 		return fmt.Errorf("error starting router: %w", err)

docs/admin/domain_blocks.md

@@ -2,7 +2,7 @@
 
 GoToSocial supports 'blocking'/'suspending' domains that you don't want your instance to federate with. In our documentation, the two terms 'block' and 'suspend' are used interchangeably with regard to domains, because they mean the same thing: preventing your instance and the instance running on the target domain from communicating with one another, effectively cutting off federation between the two instances.
 
-You can view, create, and remove domain blocks and domain allows using the [instance admin panel](./settings.md#federation).
+You can view, create, and remove domain blocks and domain allows using the [instance admin panel](./settings.md#domain-permissions).
 
 This document focuses on what domain blocks actually *do* and what side effects are processed when you create a new domain block.
 

docs/admin/domain_permission_subscriptions.md

@@ -0,0 +1,145 @@
+# Domain Permission Subscriptions
+
+Via the [admin settings panel](./settings.md#subscriptions), you can create and manage domain permission subscriptions.
+
+Domain permission subscriptions allow you to specify a URL at which a permission list is hosted. Every 24hrs at 11pm (by default), your instance will fetch and parse each list you're subscribed to, in order of priority (highest to lowest), and create domain permissions (or domain permission drafts) based on entries discovered in the lists.
+
+Each domain permission subscription can be used to create domain allow or domain block entries.
+
+!!! warning
+    Currently, via blocklist subscriptions it is only possible to create "suspend" level domain blocks; other severities are not yet supported. Entries of severity "silence" or "limit" etc. on subscribed blocklists will be skipped.
+
+## Priority
+
+When you specify multiple domain permission subscriptions, they will be fetched and parsed in order of priority, from highest priority (255) to lowest priority (0).
+
+Permissions discovered on lists higher up in the priority ranking will override permissions on lists lower down in the priority ranking.
+
+For example, an instance admin subscribes to two allow lists, "Important List" at priority 255, and "Less Important List" at priority 128. Each of these subscribed lists contain an entry for `good-eggs.example.org`.
+
+The subscription with the higher priority is the one that now creates and manages the domain allow entry for `good-eggs.example.org`.
+
+If the subscription with the higher priority is removed, then the next time all the subscriptions are fetched, "Less Important List" will create (or take ownership of) the domain allow instead.
+
+## Orphan Permissions
+
+Domain permissions (blocks or allows) that are not currently managed by a domain permission subscription are considered "orphan" permissions. This includes permissions that an admin created in the settings panel by hand, or which were imported manually via the import/export page.
+
+If you wish, when creating a domain permission subscription, you can set ["adopt orphans"](./settings.md#adopt-orphan-permissions) to true for that subscription. If a domain permission subscription that is set to adopt orphans encounters an orphan permission which is *also present on the list at the subscription's URI*, then it will "adopt" the orphan by setting the orphan's subscription ID to its own ID.
+
+For example, an instance admin manually creates a domain block for the domain `horrid-trolls.example.org`. Later, they create a domain permission subscription for a block list that contains an entry for `horrid-trolls.example.org`, and they set "adopt orphans" to true. When their instance fetches and parses the list, and creates domain permission entries from it, then the orphan domain block for `horrid-trolls.example.org` gets adopted by the domain permission subscription. Now, if the domain permission subscription is removed, and the option to remove all permissions owned by the subscription is checked, then the domain block for `horrid-trolls.example.org` will also be removed.
+
+## Fun Stuff To Do With Domain Permission Subscriptions
+
+### 1. Create an allowlist-federation cluster.
+
+Domain permission subscriptions make it possible to easily create allowlist-federation clusters, ie., a group of instances can essentially form their own mini-fediverse, wherein each instance runs in [allowlist federation mode](./federation_modes.md#allowlist-federation-mode), and subscribes to a cooperatively-managed allowlist hosted somewhere.
+
+For example, instances `instance-a.example.org`, `instance-b.example.org`, and `instance-c.example.org` decide that they only want to federate with each other.
+
+Using some version management platform like GitHub, they host a plaintext-formatted allowlist at something like `https://raw.githubusercontent.com/our-cluster/allowlist/refs/heads/main/allows.txt`.
+
+The contents of the plaintext-formatted allowlist are as follows:
+
+```text
+instance-a.example.org
+instance-b.example.org
+instance-c.example.org
+```
+
+Each instance admin sets their federation mode to `allowlist`, and creates a subscription to create allows from `https://raw.githubusercontent.com/our-cluster/allowlist/refs/heads/main/allows.txt`, which results in domain allow entries being created for their own domain, and for each other domain in the cluster.
+
+At some point, someone from `instance-d.example.org` asks (out of band) whether they can be added to the cluster. The existing admins agree, and update their plaintext-formatted allowlist to read:
+
+```text
+instance-a.example.org
+instance-b.example.org
+instance-c.example.org
+instance-d.example.org
+```
+
+The next time each instance fetches the list, a new domain allow entry will be created for `instance-d.example.org`, and it will be able to federate with the other domains on the list.
+
+### 2. Cooperatively manage a blocklist.
+
+Domain permission subscriptions make it easy to collaborate on and subscribe to shared blocklists of domains that host illegal / fashy / otherwise undesired accounts and content.
+
+For example, the admins of instances `instance-e.example.org`, `instance-f.example.org`, and `instance-g.example.org` decide that they are tired of duplicating work by playing whack-a-mole with bad actors. To make their lives easier, they decide to collaborate on a shared blocklist.
+
+Using some version management platform like GitHub, they host a blocklist at something like `https://raw.githubusercontent.com/baddies/blocklist/refs/heads/main/blocks.csv`.
+
+When someone discovers a new domain hosting an instance they don't like, they can open a pull request or similar against the list, to add the questionable instance to the domain.
+
+For example, someone gets an unpleasant reply from a new instance `fashy-arseholes.example.org`. Using their collaboration tools, they propose adding `fashy-arseholes.example.org` to the blocklist. After some deliberation and discussion, the domain is added to the list.
+
+The next time each of `instance-e.example.org`, `instance-f.example.org`, and `instance-g.example.org` fetch the block list, a block entry will be created for ``fashy-arseholes.example.org``.
+
+### 3. Subscribe to a blocklist, but ignore some of it.
+
+Say that `instance-g.example.org` in the previous section decides that they agree with most of the collaboratively-curated blocklist, but they actually would like to keep federating with ``fashy-arseholes.example.org`` for some godforsaken reason.
+
+This can be done in one of three ways:
+
+1. The admin of `instance-g.example.org` subscribes to the shared blocklist, but they do so with the ["create as drafts"](./settings.md#create-permissions-as-drafts) option set to true. When their instance fetches the blocklist, a draft block is created for `fashy-arseholes.example.org`. The admin of `instance-g` just leaves the permission as a draft, or rejects it, so it never comes into force.
+2. Before the blocklist is re-fetched, the admin of `instance-g.example.org` creates a [domain permission exclude](./settings.md#excludes) entry for ``instance-g.example.org``. The domain ``instance-g.example.org`` then becomes exempt/excluded from automatic permission creation, and so the block for ``instance-g.example.org`` on the shared blocklist does not get created in the database of ``instance-g.example.org`` the next time the list is fetched.
+3. The admin of `instance-g.example.org` creates an explicit domain allow entry for `fashy-arseholes.example.org` on their own instance. Because their instance is running in `blocklist` federation mode, [the explicit allow overrides the domain block entry](./federation_modes.md#in-blocklist-mode), and so the domain remains unblocked.
+
+### 4. Subscribe directly to another instance's blocklist.
+
+Because GoToSocial is able to fetch and parse JSON-formatted lists of domain permissions, it is possible to subscribe directly to another instance's list of blocked domains via their `/api/v1/instance/domain_blocks` (Mastodon) or `/api/v1/instance/peers?filter=suspended` (GoToSocial) endpoint (if exposed).
+
+For example, the Mastodon instance `peepee.poopoo.example.org` exposes their block list publicly, and the owner of the GoToSocial instance `instance-h.example.org` decides they quite like the cut of the Mastodon moderator's jib. They create a domain permission subscription of type JSON, and set the URI to `https://peepee.poopoo.example.org/api/v1/instance/domain_blocks`. Every 24 hours, their instance will go fetch the blocklist JSON from the Mastodon instance, and create permissions based on entries discovered therein.
+
+## Example lists per content type
+
+Shown below are examples of the different permission list formats that GoToSocial is able to understand and parse.
+
+Each list contains three domains, `bumfaces.net`, `peepee.poopoo`, and `nothanks.com`.
+
+### CSV
+
+CSV lists use content type `text/csv`.
+
+Mastodon domain permission exports generally use this format.
+
+```csv
+#domain,#severity,#reject_media,#reject_reports,#public_comment,#obfuscate
+bumfaces.net,suspend,false,false,big jerks,false
+peepee.poopoo,suspend,false,false,harassment,false
+nothanks.com,suspend,false,false,,false
+```
+
+### JSON (application/json)
+
+JSON lists use content type `application/json`.
+
+```json
+[
+  {
+    "domain": "bumfaces.net",
+    "suspended_at": "2020-05-13T13:29:12.000Z",
+    "public_comment": "big jerks"
+  },
+  {
+    "domain": "peepee.poopoo",
+    "suspended_at": "2020-05-13T13:29:12.000Z",
+    "public_comment": "harassment"
+  },
+  {
+    "domain": "nothanks.com",
+    "suspended_at": "2020-05-13T13:29:12.000Z"
+  }
+]
+```
+
+### Plaintext (text/plain)
+
+Plaintext lists use content type `text/plain`.
+
+Note that it is not possible to include any fields like "obfuscate" or "public comment" in plaintext lists, as they are simply a newline-separated list of domains.
+
+```text
+bumfaces.net
+peepee.poopoo
+nothanks.com
+```

docs/admin/settings.md

@@ -34,11 +34,11 @@ Clicking on the username of the reported account opens that account in the 'Acco
 
 You can use this section to search for an account and perform moderation actions on it.
 
-### Federation
+### Domain Permissions
 
 ![List of suspended instances, with a field to filter/add new blocks. Below is a link to the bulk import/export interface](../public/admin-settings-federation.png)
 
-In the federation section you can create, delete, and review explicit domain blocks and domain allows.
+In the domain permissions section you can create, delete, and review domain blocks, domain allows, drafts, excludes, and subscriptions.
 
 For more detail on federation settings, and specifically how domain allows and domain blocks work in combination, please see [the federation modes section](./federation_modes.md), and [the domain blocks section](./domain_blocks.md).
 
@@ -46,20 +46,99 @@ For more detail on federation settings, and specifically how domain allows and d
 
 You can enter a domain to suspend in the search field, which will filter the list to show you if you already have a block for it.
 
-Clicking 'suspend' gives you a form to add a public and/or private comment, and submit to add the block. Adding a suspension will suspend all the currently known accounts on the instance, and prevent any new interactions with any user on the blocked instance.
+Clicking 'suspend' gives you a form to add a public and/or private comment, and submit to add the block.
+
+Adding a domain block will suspend all currently known accounts from that domain, and prevent any new interactions with the blocked domain.
 
 #### Domain Allows
 
 The domain allows section works much like the domain blocks section, described above, only for explicit domain allows rather than domain blocks.
 
-#### Bulk import/export
+#### Import/export
 
-Through the link at the bottom of the Federation section (or going to `/settings/admin/federation/import-export`) you can do bulk import/export of blocklists and allowlists.
+In this section you can do bulk import/export of domain permissions in JSON, CSV, or plaintext formats.
 
 ![List of domains included in an import, providing ways to select some or all of them, change their domains, and update the use of subdomains.](../public/admin-settings-federation-import-export.png)
 
 Upon importing a list, either through the input field or from a file, you can review the entries in the list before importing a subset. You'll also be warned for entries that use subdomains, providing an easy way to change them to the main domain.
 
+#### Drafts
+
+In this section you can create, search through, accept, and reject domain permission drafts.
+
+Domain permission drafts are domain permissions that have been proposed (either via manual creation or as an entry from a subscribed block / allow list), but have not yet come into force.
+
+Until it is accepted, a domain permission draft will not have any effect on federation with the domain it targets. Upon acceptance, it will be converted into either a domain block or a domain allow, and start being enforced.
+
+#### Excludes
+
+In this section, you can create, search through, and remove domain permission excludes.
+
+Domain permission excludes prevent permissions for a domain (and all subdomains) from being automatically managed by domain permission subscriptions.
+
+For example, if you create an exclude entry for the domain `example.org`, then a blocklist or allowlist subscription will exclude entries for `example.org` and any of its subdomains (`sub.example.org`, `another.sub.example.org` etc.) when creating domain permission drafts and domain blocks/allows.
+
+This functionality allows you to manually manage permissions for excluded domains, in cases where you know you definitely do or don't want to federate with a given domain, no matter what entries are contained in a domain permission subscription.
+
+Note that by itself, creation of an exclude entry for a given domain does not affect federation with that domain at all, it is only useful in combination with permission subscriptions.
+
+#### Subscriptions
+
+In this section, you can create, search through, edit, test, and remove domain permission subscriptions.
+
+Domain permission subscriptions allow you to specify a URL at which a permission list is hosted. Every 24hrs at 11pm (by default), your instance will fetch and parse each subscribed list, and create domain permissions (or domain permission drafts) based on entries in the lists.
+
+##### Title
+
+You can optionally use the title field to set a title for the subscription, as a reminder for yourself and other admins.
+
+For example, you might subscribe to a list at `https://lists.example.org/baddies.csv` and set the title of the subscription to something that reflects the contents of that list, such as "Basic block list (worst of the worst)", or similar.
+
+##### Subscription Priority
+
+When you specify multiple domain permission subscriptions, they will be fetched and parsed in order of priority, from highest priority (255) to lowest priority (0).
+
+Permissions discovered on lists higher up in the priority ranking will override permissions on lists lower down in the priority ranking.
+
+For more information on priority, please see the separate [domain permission subscriptions](./domain_permission_subscriptions.md) document.
+
+##### Permission Type
+
+You can use this dropdown to select whether permissions discovered at the list URL should be created as domain blocks, or domain allows.
+
+##### Content Type
+
+You can use this dropdown to select the content type of the list at the subscribed URL.
+
+Use CSV for Mastodon-style permission lists, plain for plaintext lists of domain names, or JSON for json-exported lists.
+
+##### Basic Auth
+
+Check this box to provide a basic auth username and/or password credential for the subscribed list, which will be sent along with each request to fetch the list.
+
+##### Adopt Orphan Permissions
+
+If you check this box, then any existing domain permissions will become managed by this subscription in the following circumstances:
+
+1. They don't already have a subscription ID (ie., they're not managed by any domain permission subscription).
+2. They match a domain permission included in the list at the URL of this subscription.
+
+For more information on orphan permissions, please see the separate [domain permission subscriptions](./domain_permission_subscriptions.md) document.
+
+##### Create Permissions as Drafts
+
+With this box checked (default), any permissions created by this subscription will be created as **drafts** which require manual approval to come into force.
+
+It is recommended to leave this box checked unless you absolutely trust the subscribed list, to avoid inadvertent blocking or allowing of domains you'd rather not block or allow.
+
+##### Test a Subscription
+
+To test whether a subscription can be successfully parsed, first create the subscription, then in the detailed view for that subscription, click on the "Test" button.
+
+If your instance is able to fetch and parse permissions at the subscription URI, then you will see a list of these after clicking "Test". Otherwise, you will see an error message.
+
+![Screenshot of the detailed view of a subscription, with arrows pointing to the test section near the bottom.](../public/admin-settings-federation-subscription-test.png)
+
 ## Administration
 
 Instance administration settings.

docs/api/swagger.yaml

@@ -6436,6 +6436,46 @@ paths:
             summary: Remove a domain permission subscription.
             tags:
                 - admin
+    /api/v1/admin/domain_permission_subscriptions/{id}/test:
+        post:
+            description: |-
+                The response body will be a list of domain permissions that *would* be created by this subscription, OR an error message.
+
+                This is useful in cases where you want to check that your instance can actually fetch + parse a list.
+            operationId: domainPermissionSubscriptionTest
+            parameters:
+                - description: ID of the domain permission draft.
+                  in: path
+                  name: id
+                  required: true
+                  type: string
+            produces:
+                - application/json
+            responses:
+                "200":
+                    description: Either an array of domain permissions, OR an error message of the form `{"error":"[ERROR MESSAGE HERE]"}` indicating why the list could not be fetched.
+                    schema:
+                        items:
+                            $ref: '#/definitions/domain'
+                        type: array
+                "400":
+                    description: bad request
+                "401":
+                    description: unauthorized
+                "403":
+                    description: forbidden
+                "406":
+                    description: not acceptable
+                "409":
+                    description: conflict
+                "500":
+                    description: internal server error
+            security:
+                - OAuth2 Bearer:
+                    - admin
+            summary: Test one domain permission subscription by making your instance fetch and parse it *without creating permissions*.
+            tags:
+                - admin
     /api/v1/admin/domain_permission_subscriptions/preview:
         get:
             description: This view allows you to see the order in which domain permissions will actually be fetched and created.

internal/admin/actions.go

@@ -23,11 +23,12 @@
 	"sync"
 	"time"
 
+	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
-	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/workers"
 )
 
 func errActionConflict(action *gtsmodel.AdminAction) gtserror.WithCode {
@@ -42,15 +43,34 @@ func errActionConflict(action *gtsmodel.AdminAction) gtserror.WithCode {
 }
 
 type Actions struct {
-	r     map[string]*gtsmodel.AdminAction
+	// Map of running actions.
-	state *state.State
+	running map[string]*gtsmodel.AdminAction
 
-	// Not embedded struct,
+	// Lock for running admin actions.
-	// to shield from access
+	//
-	// by outside packages.
+	// Not embedded struct, to shield
+	// from access by outside packages.
 	m sync.Mutex
+
+	// DB for storing, updating,
+	// deleting admin actions etc.
+	db db.DB
+
+	// Workers for queuing
+	// admin action side effects.
+	workers *workers.Workers
 }
 
+func New(db db.DB, workers *workers.Workers) *Actions {
+	return &Actions{
+		running: make(map[string]*gtsmodel.AdminAction),
+		db:      db,
+		workers: workers,
+	}
+}
+
+type ActionF func(context.Context) gtserror.MultiError
+
 // Run runs the given admin action by executing the supplied function.
 //
 // Run handles locking, action insertion and updating, so you don't have to!
@@ -62,10 +82,10 @@ type Actions struct {
 // will be updated on the provided admin action in the database.
 func (a *Actions) Run(
 	ctx context.Context,
-	action *gtsmodel.AdminAction,
+	adminAction *gtsmodel.AdminAction,
-	f func(context.Context) gtserror.MultiError,
+	f ActionF,
 ) gtserror.WithCode {
-	actionKey := action.Key()
+	actionKey := adminAction.Key()
 
 	// LOCK THE MAP HERE, since we're
 	// going to do some operations on it.
@@ -73,7 +93,7 @@ func (a *Actions) Run(
 
 	// Bail if an action with
 	// this key is already running.
-	running, ok := a.r[actionKey]
+	running, ok := a.running[actionKey]
 	if ok {
 		a.m.Unlock()
 		return errActionConflict(running)
@@ -81,7 +101,7 @@ func (a *Actions) Run(
 
 	// Action with this key not
 	// yet running, create it.
-	if err := a.state.DB.PutAdminAction(ctx, action); err != nil {
+	if err := a.db.PutAdminAction(ctx, adminAction); err != nil {
 		err = gtserror.Newf("db error putting admin action %s: %w", actionKey, err)
 
 		// Don't store in map
@@ -92,7 +112,7 @@ func (a *Actions) Run(
 
 	// Action was inserted,
 	// store in map.
-	a.r[actionKey] = action
+	a.running[actionKey] = adminAction
 
 	// UNLOCK THE MAP HERE, since
 	// we're done modifying it for now.
@@ -104,22 +124,22 @@ func (a *Actions) Run(
 
 		// Run the thing and collect errors.
 		if errs := f(ctx); errs != nil {
-			action.Errors = make([]string, 0, len(errs))
+			adminAction.Errors = make([]string, 0, len(errs))
 			for _, err := range errs {
-				action.Errors = append(action.Errors, err.Error())
+				adminAction.Errors = append(adminAction.Errors, err.Error())
 			}
 		}
 
 		// Action is no longer running:
 		// remove from running map.
 		a.m.Lock()
-		delete(a.r, actionKey)
+		delete(a.running, actionKey)
 		a.m.Unlock()
 
 		// Mark as completed in the db,
 		// storing errors for later review.
-		action.CompletedAt = time.Now()
+		adminAction.CompletedAt = time.Now()
-		if err := a.state.DB.UpdateAdminAction(ctx, action, "completed_at", "errors"); err != nil {
+		if err := a.db.UpdateAdminAction(ctx, adminAction, "completed_at", "errors"); err != nil {
 			log.Errorf(ctx, "db error marking action %s as completed: %q", actionKey, err)
 		}
 	}()
@@ -135,8 +155,8 @@ func (a *Actions) GetRunning() []*gtsmodel.AdminAction {
 	defer a.m.Unlock()
 
 	// Assemble all currently running actions.
-	running := make([]*gtsmodel.AdminAction, 0, len(a.r))
+	running := make([]*gtsmodel.AdminAction, 0, len(a.running))
-	for _, action := range a.r {
+	for _, action := range a.running {
 		running = append(running, action)
 	}
 
@@ -166,5 +186,5 @@ func (a *Actions) TotalRunning() int {
 	a.m.Lock()
 	defer a.m.Unlock()
 
-	return len(a.r)
+	return len(a.running)
 }

internal/admin/actions_test.go

@@ -32,12 +32,26 @@
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
+const (
+	rMediaPath    = "../../testrig/media"
+	rTemplatePath = "../../web/template"
+)
+
 type ActionsTestSuite struct {
-	AdminStandardTestSuite
+	suite.Suite
+}
+
+func (suite *ActionsTestSuite) SetupSuite() {
+	testrig.InitTestConfig()
+	testrig.InitTestLog()
 }
 
 func (suite *ActionsTestSuite) TestActionOverlap() {
-	ctx := context.Background()
+	var (
+		testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		ctx         = context.Background()
+	)
+	defer testrig.TearDownTestStructs(testStructs)
 
 	// Suspend account.
 	action1 := &gtsmodel.AdminAction{
@@ -61,7 +75,7 @@ func (suite *ActionsTestSuite) TestActionOverlap() {
 	key2 := action2.Key()
 	suite.Equal("account/01H90S1CXQ97J9625C5YBXZWGT", key2)
 
-	errWithCode := suite.adminProcessor.Actions().Run(
+	errWithCode := testStructs.State.AdminActions.Run(
 		ctx,
 		action1,
 		func(ctx context.Context) gtserror.MultiError {
@@ -74,7 +88,7 @@ func(ctx context.Context) gtserror.MultiError {
 
 	// While first action is sleeping, try to
 	// process another with the same key.
-	errWithCode = suite.adminProcessor.Actions().Run(
+	errWithCode = testStructs.State.AdminActions.Run(
 		ctx,
 		action2,
 		func(ctx context.Context) gtserror.MultiError {
@@ -90,13 +104,13 @@ func(ctx context.Context) gtserror.MultiError {
 
 	// Wait for action to finish.
 	if !testrig.WaitFor(func() bool {
-		return suite.adminProcessor.Actions().TotalRunning() == 0
+		return testStructs.State.AdminActions.TotalRunning() == 0
 	}) {
 		suite.FailNow("timed out waiting for admin action(s) to finish")
 	}
 
 	// Try again.
-	errWithCode = suite.adminProcessor.Actions().Run(
+	errWithCode = testStructs.State.AdminActions.Run(
 		ctx,
 		action2,
 		func(ctx context.Context) gtserror.MultiError {
@@ -107,14 +121,18 @@ func(ctx context.Context) gtserror.MultiError {
 
 	// Wait for action to finish.
 	if !testrig.WaitFor(func() bool {
-		return suite.adminProcessor.Actions().TotalRunning() == 0
+		return testStructs.State.AdminActions.TotalRunning() == 0
 	}) {
 		suite.FailNow("timed out waiting for admin action(s) to finish")
 	}
 }
 
 func (suite *ActionsTestSuite) TestActionWithErrors() {
-	ctx := context.Background()
+	var (
+		testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		ctx         = context.Background()
+	)
+	defer testrig.TearDownTestStructs(testStructs)
 
 	// Suspend a domain.
 	action := &gtsmodel.AdminAction{
@@ -125,7 +143,7 @@ func (suite *ActionsTestSuite) TestActionWithErrors() {
 		AccountID:      "01H90S1ZZXP4N74H4A9RVW1MRP",
 	}
 
-	errWithCode := suite.adminProcessor.Actions().Run(
+	errWithCode := testStructs.State.AdminActions.Run(
 		ctx,
 		action,
 		func(ctx context.Context) gtserror.MultiError {
@@ -140,13 +158,13 @@ func(ctx context.Context) gtserror.MultiError {
 
 	// Wait for action to finish.
 	if !testrig.WaitFor(func() bool {
-		return suite.adminProcessor.Actions().TotalRunning() == 0
+		return testStructs.State.AdminActions.TotalRunning() == 0
 	}) {
 		suite.FailNow("timed out waiting for admin action(s) to finish")
 	}
 
 	// Get action from the db.
-	dbAction, err := suite.db.GetAdminAction(ctx, action.ID)
+	dbAction, err := testStructs.State.DB.GetAdminAction(ctx, action.ID)
 	if err != nil {
 		suite.FailNow(err.Error())
 	}

internal/admin/domainkeys.go

@@ -0,0 +1,51 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package admin
+
+import (
+	"context"
+	"time"
+
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (a *Actions) DomainKeysExpireF(domain string) ActionF {
+	return func(ctx context.Context) gtserror.MultiError {
+		var (
+			expiresAt = time.Now()
+			errs      gtserror.MultiError
+		)
+
+		// For each account on this domain, expire
+		// the public key and update the account.
+		if err := a.rangeDomainAccounts(ctx, domain, func(account *gtsmodel.Account) {
+			account.PublicKeyExpiresAt = expiresAt
+			if err := a.db.UpdateAccount(ctx,
+				account,
+				"public_key_expires_at",
+			); err != nil {
+				errs.Appendf("db error updating account: %w", err)
+			}
+		}); err != nil {
+			errs.Appendf("db error ranging through accounts: %w", err)
+		}
+
+		return errs
+	}
+}

internal/admin/domainperms.go

@@ -0,0 +1,387 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package admin
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"codeberg.org/gruf/go-kv"
+
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/log"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
+)
+
+// Returns an AdminActionF for
+// domain allow side effects.
+func (a *Actions) DomainAllowF(
+	actionID string,
+	domainAllow *gtsmodel.DomainAllow,
+) ActionF {
+	return func(ctx context.Context) gtserror.MultiError {
+		l := log.
+			WithContext(ctx).
+			WithFields(kv.Fields{
+				{"action", "allow"},
+				{"actionID", actionID},
+				{"domain", domainAllow.Domain},
+			}...)
+
+		// Log start + finish.
+		l.Info("processing side effects")
+		errs := a.domainAllowSideEffects(ctx, domainAllow)
+		l.Info("finished processing side effects")
+
+		return errs
+	}
+}
+
+func (a *Actions) domainAllowSideEffects(
+	ctx context.Context,
+	allow *gtsmodel.DomainAllow,
+) gtserror.MultiError {
+	if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist {
+		// We're running in allowlist mode,
+		// so there are no side effects to
+		// process here.
+		return nil
+	}
+
+	// We're running in blocklist mode or
+	// some similar mode which necessitates
+	// domain allow side effects if a block
+	// was in place when the allow was created.
+	//
+	// So, check if there's a block.
+	block, err := a.db.GetDomainBlock(ctx, allow.Domain)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		errs := gtserror.NewMultiError(1)
+		errs.Appendf("db error getting domain block %s: %w", allow.Domain, err)
+		return errs
+	}
+
+	if block == nil {
+		// No block?
+		// No problem!
+		return nil
+	}
+
+	// There was a block, over which the new
+	// allow ought to take precedence. To account
+	// for this, just run side effects as though
+	// the domain was being unblocked, while
+	// leaving the existing block in place.
+	//
+	// Any accounts that were suspended by
+	// the block will be unsuspended and be
+	// able to interact with the instance again.
+	return a.domainUnblockSideEffects(ctx, block)
+}
+
+// Returns an AdminActionF for
+// domain unallow side effects.
+func (a *Actions) DomainUnallowF(
+	actionID string,
+	domainAllow *gtsmodel.DomainAllow,
+) ActionF {
+	return func(ctx context.Context) gtserror.MultiError {
+		l := log.
+			WithContext(ctx).
+			WithFields(kv.Fields{
+				{"action", "unallow"},
+				{"actionID", actionID},
+				{"domain", domainAllow.Domain},
+			}...)
+
+		// Log start + finish.
+		l.Info("processing side effects")
+		errs := a.domainUnallowSideEffects(ctx, domainAllow)
+		l.Info("finished processing side effects")
+
+		return errs
+	}
+}
+
+func (a *Actions) domainUnallowSideEffects(
+	ctx context.Context,
+	allow *gtsmodel.DomainAllow,
+) gtserror.MultiError {
+	if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist {
+		// We're running in allowlist mode,
+		// so there are no side effects to
+		// process here.
+		return nil
+	}
+
+	// We're running in blocklist mode or
+	// some similar mode which necessitates
+	// domain allow side effects if a block
+	// was in place when the allow was removed.
+	//
+	// So, check if there's a block.
+	block, err := a.db.GetDomainBlock(ctx, allow.Domain)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		errs := gtserror.NewMultiError(1)
+		errs.Appendf("db error getting domain block %s: %w", allow.Domain, err)
+		return errs
+	}
+
+	if block == nil {
+		// No block?
+		// No problem!
+		return nil
+	}
+
+	// There was a block, over which the previous
+	// allow was taking precedence. Now that the
+	// allow has been removed, we should put the
+	// side effects of the block back in place.
+	//
+	// To do this, process the block side effects
+	// again as though the block were freshly
+	// created. This will mark all accounts from
+	// the blocked domain as suspended, and clean
+	// up their follows/following, media, etc.
+	return a.domainBlockSideEffects(ctx, block)
+}
+
+func (a *Actions) DomainBlockF(
+	actionID string,
+	domainBlock *gtsmodel.DomainBlock,
+) ActionF {
+	return func(ctx context.Context) gtserror.MultiError {
+		l := log.
+			WithContext(ctx).
+			WithFields(kv.Fields{
+				{"action", "block"},
+				{"actionID", actionID},
+				{"domain", domainBlock.Domain},
+			}...)
+
+		skip, err := a.skipBlockSideEffects(ctx, domainBlock.Domain)
+		if err != nil {
+			return err
+		}
+
+		if skip != "" {
+			l.Infof("skipping side effects: %s", skip)
+			return nil
+		}
+
+		l.Info("processing side effects")
+		errs := a.domainBlockSideEffects(ctx, domainBlock)
+		l.Info("finished processing side effects")
+
+		return errs
+	}
+}
+
+// domainBlockSideEffects processes the side effects of a domain block:
+//
+//  1. Strip most info away from the instance entry for the domain.
+//  2. Pass each account from the domain to the processor for deletion.
+//
+// It should be called asynchronously, since it can take a while when
+// there are many accounts present on the given domain.
+func (a *Actions) domainBlockSideEffects(
+	ctx context.Context,
+	block *gtsmodel.DomainBlock,
+) gtserror.MultiError {
+	var errs gtserror.MultiError
+
+	// If we have an instance entry for this domain,
+	// update it with the new block ID and clear all fields
+	instance, err := a.db.GetInstance(ctx, block.Domain)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		errs.Appendf("db error getting instance %s: %w", block.Domain, err)
+		return errs
+	}
+
+	if instance != nil {
+		// We had an entry for this domain.
+		columns := stubbifyInstance(instance, block.ID)
+		if err := a.db.UpdateInstance(ctx, instance, columns...); err != nil {
+			errs.Appendf("db error updating instance: %w", err)
+			return errs
+		}
+	}
+
+	// For each account that belongs to this domain,
+	// process an account delete message to remove
+	// that account's posts, media, etc.
+	if err := a.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) {
+		if err := a.workers.Client.Process(ctx, &messages.FromClientAPI{
+			APObjectType:   ap.ActorPerson,
+			APActivityType: ap.ActivityDelete,
+			GTSModel:       block,
+			Origin:         account,
+			Target:         account,
+		}); err != nil {
+			errs.Append(err)
+		}
+	}); err != nil {
+		errs.Appendf("db error ranging through accounts: %w", err)
+	}
+
+	return errs
+}
+
+func (a *Actions) DomainUnblockF(
+	actionID string,
+	domainBlock *gtsmodel.DomainBlock,
+) ActionF {
+	return func(ctx context.Context) gtserror.MultiError {
+		l := log.
+			WithContext(ctx).
+			WithFields(kv.Fields{
+				{"action", "unblock"},
+				{"actionID", actionID},
+				{"domain", domainBlock.Domain},
+			}...)
+
+		l.Info("processing side effects")
+		errs := a.domainUnblockSideEffects(ctx, domainBlock)
+		l.Info("finished processing side effects")
+
+		return errs
+	}
+}
+
+// domainUnblockSideEffects processes the side effects of undoing a
+// domain block:
+//
+//  1. Mark instance entry as no longer suspended.
+//  2. Mark each account from the domain as no longer suspended, if the
+//     suspension origin corresponds to the ID of the provided domain block.
+//
+// It should be called asynchronously, since it can take a while when
+// there are many accounts present on the given domain.
+func (a *Actions) domainUnblockSideEffects(
+	ctx context.Context,
+	block *gtsmodel.DomainBlock,
+) gtserror.MultiError {
+	var errs gtserror.MultiError
+
+	// Update instance entry for this domain, if we have it.
+	instance, err := a.db.GetInstance(ctx, block.Domain)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		errs.Appendf("db error getting instance %s: %w", block.Domain, err)
+	}
+
+	if instance != nil {
+		// We had an entry, update it to signal
+		// that it's no longer suspended.
+		instance.SuspendedAt = time.Time{}
+		instance.DomainBlockID = ""
+		if err := a.db.UpdateInstance(
+			ctx,
+			instance,
+			"suspended_at",
+			"domain_block_id",
+		); err != nil {
+			errs.Appendf("db error updating instance: %w", err)
+			return errs
+		}
+	}
+
+	// Unsuspend all accounts whose suspension origin was this domain block.
+	if err := a.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) {
+		if account.SuspensionOrigin == "" || account.SuspendedAt.IsZero() {
+			// Account wasn't suspended, nothing to do.
+			return
+		}
+
+		if account.SuspensionOrigin != block.ID {
+			// Account was suspended, but not by
+			// this domain block, leave it alone.
+			return
+		}
+
+		// Account was suspended by this domain
+		// block, mark it as unsuspended.
+		account.SuspendedAt = time.Time{}
+		account.SuspensionOrigin = ""
+
+		if err := a.db.UpdateAccount(
+			ctx,
+			account,
+			"suspended_at",
+			"suspension_origin",
+		); err != nil {
+			errs.Appendf("db error updating account %s: %w", account.Username, err)
+		}
+	}); err != nil {
+		errs.Appendf("db error ranging through accounts: %w", err)
+	}
+
+	return errs
+}
+
+// skipBlockSideEffects checks if side effects of block creation
+// should be skipped for the given domain, taking account of
+// instance federation mode, and existence of any allows
+// which ought to "shield" this domain from being blocked.
+//
+// If the caller should skip, the returned string will be non-zero
+// and will be set to a reason why side effects should be skipped.
+//
+//   - blocklist mode + allow exists: "..." (skip)
+//   - blocklist mode + no allow:     ""    (don't skip)
+//   - allowlist mode + allow exists: ""    (don't skip)
+//   - allowlist mode + no allow:     ""    (don't skip)
+func (a *Actions) skipBlockSideEffects(
+	ctx context.Context,
+	domain string,
+) (string, gtserror.MultiError) {
+	var (
+		skip string // Assume "" (don't skip).
+		errs gtserror.MultiError
+	)
+
+	// Never skip block side effects in allowlist mode.
+	fediMode := config.GetInstanceFederationMode()
+	if fediMode == config.InstanceFederationModeAllowlist {
+		return skip, errs
+	}
+
+	// We know we're in blocklist mode.
+	//
+	// We want to skip domain block side
+	// effects if an allow is already
+	// in place which overrides the block.
+
+	// Check if an explicit allow exists for this domain.
+	domainAllow, err := a.db.GetDomainAllow(ctx, domain)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		errs.Appendf("error getting domain allow: %w", err)
+		return skip, errs
+	}
+
+	if domainAllow != nil {
+		skip = "running in blocklist mode, and an explicit allow exists for this domain"
+		return skip, errs
+	}
+
+	return skip, errs
+}

internal/admin/util.go

@@ -0,0 +1,99 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package admin
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+// stubbifyInstance renders the given instance as a stub,
+// removing most information from it and marking it as
+// suspended.
+//
+// For caller's convenience, this function returns the db
+// names of all columns that are updated by it.
+func stubbifyInstance(instance *gtsmodel.Instance, domainBlockID string) []string {
+	instance.Title = ""
+	instance.SuspendedAt = time.Now()
+	instance.DomainBlockID = domainBlockID
+	instance.ShortDescription = ""
+	instance.Description = ""
+	instance.Terms = ""
+	instance.ContactEmail = ""
+	instance.ContactAccountUsername = ""
+	instance.ContactAccountID = ""
+	instance.Version = ""
+
+	return []string{
+		"title",
+		"suspended_at",
+		"domain_block_id",
+		"short_description",
+		"description",
+		"terms",
+		"contact_email",
+		"contact_account_username",
+		"contact_account_id",
+		"version",
+	}
+}
+
+// rangeDomainAccounts iterates through all accounts
+// originating from the given domain, and calls the
+// provided range function on each account.
+//
+// If an error is returned while selecting accounts,
+// the loop will stop and return the error.
+func (a *Actions) rangeDomainAccounts(
+	ctx context.Context,
+	domain string,
+	rangeF func(*gtsmodel.Account),
+) error {
+	var (
+		limit = 50   // Limit selection to avoid spiking mem/cpu.
+		maxID string // Start with empty string to select from top.
+	)
+
+	for {
+		// Get (next) page of accounts.
+		accounts, err := a.db.GetInstanceAccounts(ctx, domain, maxID, limit)
+		if err != nil && !errors.Is(err, db.ErrNoEntries) {
+			// Real db error.
+			return gtserror.Newf("db error getting instance accounts: %w", err)
+		}
+
+		if len(accounts) == 0 {
+			// No accounts left, we're done.
+			return nil
+		}
+
+		// Set next max ID for paging down.
+		maxID = accounts[len(accounts)-1].ID
+
+		// Call provided range function.
+		for _, account := range accounts {
+			rangeF(account)
+		}
+	}
+}

internal/api/activitypub/emoji/emojiget_test.go

@@ -25,6 +25,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/activitypub/emoji"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -73,6 +74,7 @@ func (suite *EmojiGetTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 	suite.tc = typeutils.NewConverter(&suite.state)

internal/api/activitypub/users/user_test.go

@@ -20,6 +20,7 @@
 import (
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/activitypub/users"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -84,6 +85,7 @@ func (suite *UserStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.tc = typeutils.NewConverter(&suite.state)
 
 	testrig.StartTimelines(

internal/api/auth/auth_test.go

@@ -26,6 +26,7 @@
 	"github.com/gin-contrib/sessions/memstore"
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/auth"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -84,6 +85,7 @@ func (suite *AuthStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 	suite.mediaManager = testrig.NewTestMediaManager(&suite.state)

internal/api/client/accounts/account_test.go

@@ -25,6 +25,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/accounts"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -85,6 +86,7 @@ func (suite *AccountStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/admin/admin.go

@@ -46,6 +46,7 @@
 	DomainPermissionSubscriptionsPathWithID  = DomainPermissionSubscriptionsPath + "/:" + apiutil.IDKey
 	DomainPermissionSubscriptionsPreviewPath = DomainPermissionSubscriptionsPath + "/preview"
 	DomainPermissionSubscriptionRemovePath   = DomainPermissionSubscriptionsPathWithID + "/remove"
+	DomainPermissionSubscriptionTestPath     = DomainPermissionSubscriptionsPathWithID + "/test"
 	DomainKeysExpirePath                     = BasePath + "/domain_keys_expire"
 	HeaderAllowsPath                         = BasePath + "/header_allows"
 	HeaderAllowsPathWithID                   = HeaderAllowsPath + "/:" + apiutil.IDKey
@@ -129,6 +130,7 @@ func (m *Module) Route(attachHandler func(method string, path string, f ...gin.H
 	attachHandler(http.MethodGet, DomainPermissionSubscriptionsPathWithID, m.DomainPermissionSubscriptionGETHandler)
 	attachHandler(http.MethodPatch, DomainPermissionSubscriptionsPathWithID, m.DomainPermissionSubscriptionPATCHHandler)
 	attachHandler(http.MethodPost, DomainPermissionSubscriptionRemovePath, m.DomainPermissionSubscriptionRemovePOSTHandler)
+	attachHandler(http.MethodPost, DomainPermissionSubscriptionTestPath, m.DomainPermissionSubscriptionTestPOSTHandler)
 
 	// header filtering administration routes
 	attachHandler(http.MethodGet, HeaderAllowsPathWithID, m.HeaderFilterAllowGET)

internal/api/client/admin/admin_test.go

@@ -25,6 +25,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	adminactions "github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -91,6 +92,7 @@ func (suite *AdminStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = adminactions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/admin/domainpermissionsubscriptiontest.go

@@ -0,0 +1,118 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package admin
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+)
+
+// DomainPermissionSubscriptionTestPOSTHandler swagger:operation POST /api/v1/admin/domain_permission_subscriptions/{id}/test domainPermissionSubscriptionTest
+//
+// Test one domain permission subscription by making your instance fetch and parse it *without creating permissions*.
+//
+// The response body will be a list of domain permissions that *would* be created by this subscription, OR an error message.
+//
+// This is useful in cases where you want to check that your instance can actually fetch + parse a list.
+//
+//	---
+//	tags:
+//	- admin
+//
+//	produces:
+//	- application/json
+//
+//	parameters:
+//	-
+//		name: id
+//		required: true
+//		in: path
+//		description: ID of the domain permission draft.
+//		type: string
+//
+//	security:
+//	- OAuth2 Bearer:
+//		- admin
+//
+//	responses:
+//		'200':
+//			description: >-
+//				Either an array of domain permissions, OR an error message of the form
+//				`{"error":"[ERROR MESSAGE HERE]"}` indicating why the list could not be fetched.
+//			schema:
+//				type: array
+//				items:
+//					"$ref": "#/definitions/domain"
+//		'400':
+//			description: bad request
+//		'401':
+//			description: unauthorized
+//		'403':
+//			description: forbidden
+//		'406':
+//			description: not acceptable
+//		'409':
+//			description: conflict
+//		'500':
+//			description: internal server error
+func (m *Module) DomainPermissionSubscriptionTestPOSTHandler(c *gin.Context) {
+	authed, err := oauth.Authed(c, true, true, true, true)
+	if err != nil {
+		apiutil.ErrorHandler(c, gtserror.NewErrorUnauthorized(err, err.Error()), m.processor.InstanceGetV1)
+		return
+	}
+
+	if !*authed.User.Admin {
+		err := fmt.Errorf("user %s not an admin", authed.User.ID)
+		apiutil.ErrorHandler(c, gtserror.NewErrorForbidden(err, err.Error()), m.processor.InstanceGetV1)
+		return
+	}
+
+	if authed.Account.IsMoving() {
+		apiutil.ForbiddenAfterMove(c)
+		return
+	}
+
+	if _, err := apiutil.NegotiateAccept(c, apiutil.JSONAcceptHeaders...); err != nil {
+		apiutil.ErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGetV1)
+		return
+	}
+
+	id, errWithCode := apiutil.ParseID(c.Param(apiutil.IDKey))
+	if errWithCode != nil {
+		apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
+		return
+	}
+
+	resp, errWithCode := m.processor.Admin().DomainPermissionSubscriptionTest(
+		c.Request.Context(),
+		authed.Account,
+		id,
+	)
+	if errWithCode != nil {
+		apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
+		return
+	}
+
+	apiutil.JSON(c, http.StatusOK, resp)
+}

internal/api/client/admin/domainpermissionsubscruptiontest_test.go

@@ -0,0 +1,125 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package admin_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/admin"
+	apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+type DomainPermissionSubscriptionTestTestSuite struct {
+	AdminStandardTestSuite
+}
+
+func (suite *DomainPermissionSubscriptionTestTestSuite) TestDomainPermissionSubscriptionTest() {
+	var (
+		ctx         = context.Background()
+		testAccount = suite.testAccounts["admin_account"]
+		permSub     = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.csv",
+			ContentType:        gtsmodel.DomainPermSubContentTypeCSV,
+		}
+	)
+
+	// Create a subscription for a CSV list of baddies.
+	err := suite.state.DB.PutDomainPermissionSubscription(ctx, permSub)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Prepare the request to the /test endpoint.
+	subPath := strings.ReplaceAll(
+		admin.DomainPermissionSubscriptionTestPath,
+		":id", permSub.ID,
+	)
+	path := "/api" + subPath
+	recorder := httptest.NewRecorder()
+	ginCtx := suite.newContext(recorder, http.MethodPost, nil, path, "application/json")
+	ginCtx.Params = gin.Params{
+		gin.Param{
+			Key:   apiutil.IDKey,
+			Value: permSub.ID,
+		},
+	}
+
+	// Trigger the handler.
+	suite.adminModule.DomainPermissionSubscriptionTestPOSTHandler(ginCtx)
+	suite.Equal(http.StatusOK, recorder.Code)
+
+	// Read the body back.
+	b, err := io.ReadAll(recorder.Body)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	dst := new(bytes.Buffer)
+	if err := json.Indent(dst, b, "", "  "); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Ensure expected.
+	suite.Equal(`[
+  {
+    "domain": "bumfaces.net",
+    "public_comment": "big jerks"
+  },
+  {
+    "domain": "peepee.poopoo",
+    "public_comment": "harassment"
+  },
+  {
+    "domain": "nothanks.com"
+  }
+]`, dst.String())
+
+	// No permissions should be created
+	// since this is a dry run / test.
+	blocked, err := suite.state.DB.AreDomainsBlocked(
+		ctx,
+		[]string{"bumfaces.net", "peepee.poopoo", "nothanks.com"},
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+	suite.False(blocked)
+}
+
+func TestDomainPermissionSubscriptionTestTestSuite(t *testing.T) {
+	suite.Run(t, &DomainPermissionSubscriptionTestTestSuite{})
+}

internal/api/client/bookmarks/bookmarks_test.go

@@ -28,6 +28,7 @@
 	"testing"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/bookmarks"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/statuses"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
@@ -95,6 +96,7 @@ func (suite *BookmarkTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/favourites/favourites_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/favourites"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -79,6 +80,7 @@ func (suite *FavouritesStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/filters/v1/filter_test.go

@@ -23,6 +23,7 @@
 	"time"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	filtersV1 "github.com/superseriousbusiness/gotosocial/internal/api/client/filters/v1"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -90,6 +91,7 @@ func (suite *FiltersTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/filters/v2/filter_test.go

@@ -23,6 +23,7 @@
 	"time"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	filtersV2 "github.com/superseriousbusiness/gotosocial/internal/api/client/filters/v2"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -90,6 +91,7 @@ func (suite *FiltersTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/followedtags/followedtags_test.go

@@ -21,6 +21,7 @@
 	"testing"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/followedtags"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -79,6 +80,7 @@ func (suite *FollowedTagsTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/followrequests/followrequest_test.go

@@ -24,6 +24,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/followrequests"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -82,6 +83,7 @@ func (suite *FollowRequestStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/instance/instance_test.go

@@ -24,6 +24,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/instance"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -84,6 +85,7 @@ func (suite *InstanceStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/lists/lists_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/lists"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -85,6 +86,7 @@ func (suite *ListsStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/mutes/mutes_test.go

@@ -25,6 +25,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/mutes"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -81,6 +82,7 @@ func (suite *MutesTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/notifications/notifications_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/notifications"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -81,6 +82,7 @@ func (suite *NotificationsTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/polls/polls_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/polls"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -76,6 +77,7 @@ func (suite *PollsStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/reports/reports_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/reports"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -76,6 +77,7 @@ func (suite *ReportsStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/search/search_test.go

@@ -24,6 +24,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/search"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -80,6 +81,7 @@ func (suite *SearchStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/statuses/status_test.go

@@ -25,6 +25,7 @@
 	"strings"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/statuses"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -192,6 +193,7 @@ func (suite *StatusStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/streaming/streaming_test.go

@@ -31,6 +31,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/streaming"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -92,6 +93,7 @@ func (suite *StreamingTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/tags/tags_test.go

@@ -26,6 +26,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/tags"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
@@ -87,6 +88,7 @@ func (suite *TagsTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/user/user_test.go

@@ -24,6 +24,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/user"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -72,6 +73,7 @@ func (suite *UserStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/fileserver/fileserver_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/fileserver"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -98,6 +99,7 @@ func (suite *FileserverTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 
 	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")

internal/api/wellknown/webfinger/webfinger_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/wellknown/webfinger"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -79,6 +80,7 @@ func (suite *WebfingerStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.tc = typeutils.NewConverter(&suite.state)
 
 	testrig.StartTimelines(

internal/api/wellknown/webfinger/webfingerget_test.go

@@ -39,6 +39,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -90,6 +91,7 @@ func (suite *WebfingerGetTestSuite) funkifyAccountDomain(host string, accountDom
 
 	suite.processor = processing.NewProcessor(
 		cleaner.New(&suite.state),
+		subscriptions.New(&suite.state, suite.federator.TransportController(), suite.tc),
 		suite.tc,
 		suite.federator,
 		testrig.NewTestOauthServer(suite.db),

internal/cleaner/media_test.go

@@ -26,6 +26,7 @@
 	"time"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
@@ -67,6 +68,7 @@ func (suite *MediaTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.state.Storage = suite.storage
 
 	testrig.StandardStorageSetup(suite.storage, "../../testrig/media")

internal/config/config.go

@@ -78,15 +78,17 @@ type Configuration struct {
 	WebTemplateBaseDir string `name:"web-template-base-dir" usage:"Basedir for html templating files for rendering pages and composing emails."`
 	WebAssetBaseDir    string `name:"web-asset-base-dir" usage:"Directory to serve static assets from, accessible at example.org/assets/"`
 
-	InstanceFederationMode         string             `name:"instance-federation-mode" usage:"Set instance federation mode."`
+	InstanceFederationMode            string             `name:"instance-federation-mode" usage:"Set instance federation mode."`
-	InstanceFederationSpamFilter   bool               `name:"instance-federation-spam-filter" usage:"Enable basic spam filter heuristics for messages coming from other instances, and drop messages identified as spam"`
+	InstanceFederationSpamFilter      bool               `name:"instance-federation-spam-filter" usage:"Enable basic spam filter heuristics for messages coming from other instances, and drop messages identified as spam"`
-	InstanceExposePeers            bool               `name:"instance-expose-peers" usage:"Allow unauthenticated users to query /api/v1/instance/peers?filter=open"`
+	InstanceExposePeers               bool               `name:"instance-expose-peers" usage:"Allow unauthenticated users to query /api/v1/instance/peers?filter=open"`
-	InstanceExposeSuspended        bool               `name:"instance-expose-suspended" usage:"Expose suspended instances via web UI, and allow unauthenticated users to query /api/v1/instance/peers?filter=suspended"`
+	InstanceExposeSuspended           bool               `name:"instance-expose-suspended" usage:"Expose suspended instances via web UI, and allow unauthenticated users to query /api/v1/instance/peers?filter=suspended"`
-	InstanceExposeSuspendedWeb     bool               `name:"instance-expose-suspended-web" usage:"Expose list of suspended instances as webpage on /about/suspended"`
+	InstanceExposeSuspendedWeb        bool               `name:"instance-expose-suspended-web" usage:"Expose list of suspended instances as webpage on /about/suspended"`
-	InstanceExposePublicTimeline   bool               `name:"instance-expose-public-timeline" usage:"Allow unauthenticated users to query /api/v1/timelines/public"`
+	InstanceExposePublicTimeline      bool               `name:"instance-expose-public-timeline" usage:"Allow unauthenticated users to query /api/v1/timelines/public"`
-	InstanceDeliverToSharedInboxes bool               `name:"instance-deliver-to-shared-inboxes" usage:"Deliver federated messages to shared inboxes, if they're available."`
+	InstanceDeliverToSharedInboxes    bool               `name:"instance-deliver-to-shared-inboxes" usage:"Deliver federated messages to shared inboxes, if they're available."`
-	InstanceInjectMastodonVersion  bool               `name:"instance-inject-mastodon-version" usage:"This injects a Mastodon compatible version in /api/v1/instance to help Mastodon clients that use that version for feature detection"`
+	InstanceInjectMastodonVersion     bool               `name:"instance-inject-mastodon-version" usage:"This injects a Mastodon compatible version in /api/v1/instance to help Mastodon clients that use that version for feature detection"`
-	InstanceLanguages              language.Languages `name:"instance-languages" usage:"BCP47 language tags for the instance. Used to indicate the preferred languages of instance residents (in order from most-preferred to least-preferred)."`
+	InstanceLanguages                 language.Languages `name:"instance-languages" usage:"BCP47 language tags for the instance. Used to indicate the preferred languages of instance residents (in order from most-preferred to least-preferred)."`
+	InstanceSubscriptionsProcessFrom  string             `name:"instance-subscriptions-process-from" usage:"Time of day from which to start running instance subscriptions processing jobs. Should be in the format 'hh:mm:ss', eg., '15:04:05'."`
+	InstanceSubscriptionsProcessEvery time.Duration      `name:"instance-subscriptions-process-every" usage:"Period to elapse between instance subscriptions processing jobs, starting from instance-subscriptions-process-from."`
 
 	AccountsRegistrationOpen bool `name:"accounts-registration-open" usage:"Allow anyone to submit an account signup request. If false, server will be invite-only."`
 	AccountsReasonRequired   bool `name:"accounts-reason-required" usage:"Do new account signups require a reason to be submitted on registration?"`

internal/config/defaults.go

@@ -58,13 +58,15 @@
 	WebTemplateBaseDir: "./web/template/",
 	WebAssetBaseDir:    "./web/assets/",
 
-	InstanceFederationMode:         InstanceFederationModeDefault,
+	InstanceFederationMode:            InstanceFederationModeDefault,
-	InstanceFederationSpamFilter:   false,
+	InstanceFederationSpamFilter:      false,
-	InstanceExposePeers:            false,
+	InstanceExposePeers:               false,
-	InstanceExposeSuspended:        false,
+	InstanceExposeSuspended:           false,
-	InstanceExposeSuspendedWeb:     false,
+	InstanceExposeSuspendedWeb:        false,
-	InstanceDeliverToSharedInboxes: true,
+	InstanceDeliverToSharedInboxes:    true,
-	InstanceLanguages:              make(language.Languages, 0),
+	InstanceLanguages:                 make(language.Languages, 0),
+	InstanceSubscriptionsProcessFrom:  "23:00",        // 11pm,
+	InstanceSubscriptionsProcessEvery: 24 * time.Hour, // 1/day.
 
 	AccountsRegistrationOpen: false,
 	AccountsReasonRequired:   true,

internal/config/flags.go

@@ -90,6 +90,8 @@ func (s *ConfigState) AddServerFlags(cmd *cobra.Command) {
 		cmd.Flags().Bool(InstanceExposeSuspendedWebFlag(), cfg.InstanceExposeSuspendedWeb, fieldtag("InstanceExposeSuspendedWeb", "usage"))
 		cmd.Flags().Bool(InstanceDeliverToSharedInboxesFlag(), cfg.InstanceDeliverToSharedInboxes, fieldtag("InstanceDeliverToSharedInboxes", "usage"))
 		cmd.Flags().StringSlice(InstanceLanguagesFlag(), cfg.InstanceLanguages.TagStrs(), fieldtag("InstanceLanguages", "usage"))
+		cmd.Flags().String(InstanceSubscriptionsProcessFromFlag(), cfg.InstanceSubscriptionsProcessFrom, fieldtag("InstanceSubscriptionsProcessFrom", "usage"))
+		cmd.Flags().Duration(InstanceSubscriptionsProcessEveryFlag(), cfg.InstanceSubscriptionsProcessEvery, fieldtag("InstanceSubscriptionsProcessEvery", "usage"))
 
 		// Accounts
 		cmd.Flags().Bool(AccountsRegistrationOpenFlag(), cfg.AccountsRegistrationOpen, fieldtag("AccountsRegistrationOpen", "usage"))

internal/config/helpers.gen.go

@@ -1000,6 +1000,62 @@ func GetInstanceLanguages() language.Languages { return global.GetInstanceLangua
 // SetInstanceLanguages safely sets the value for global configuration 'InstanceLanguages' field
 func SetInstanceLanguages(v language.Languages) { global.SetInstanceLanguages(v) }
 
+// GetInstanceSubscriptionsProcessFrom safely fetches the Configuration value for state's 'InstanceSubscriptionsProcessFrom' field
+func (st *ConfigState) GetInstanceSubscriptionsProcessFrom() (v string) {
+	st.mutex.RLock()
+	v = st.config.InstanceSubscriptionsProcessFrom
+	st.mutex.RUnlock()
+	return
+}
+
+// SetInstanceSubscriptionsProcessFrom safely sets the Configuration value for state's 'InstanceSubscriptionsProcessFrom' field
+func (st *ConfigState) SetInstanceSubscriptionsProcessFrom(v string) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.InstanceSubscriptionsProcessFrom = v
+	st.reloadToViper()
+}
+
+// InstanceSubscriptionsProcessFromFlag returns the flag name for the 'InstanceSubscriptionsProcessFrom' field
+func InstanceSubscriptionsProcessFromFlag() string { return "instance-subscriptions-process-from" }
+
+// GetInstanceSubscriptionsProcessFrom safely fetches the value for global configuration 'InstanceSubscriptionsProcessFrom' field
+func GetInstanceSubscriptionsProcessFrom() string {
+	return global.GetInstanceSubscriptionsProcessFrom()
+}
+
+// SetInstanceSubscriptionsProcessFrom safely sets the value for global configuration 'InstanceSubscriptionsProcessFrom' field
+func SetInstanceSubscriptionsProcessFrom(v string) { global.SetInstanceSubscriptionsProcessFrom(v) }
+
+// GetInstanceSubscriptionsProcessEvery safely fetches the Configuration value for state's 'InstanceSubscriptionsProcessEvery' field
+func (st *ConfigState) GetInstanceSubscriptionsProcessEvery() (v time.Duration) {
+	st.mutex.RLock()
+	v = st.config.InstanceSubscriptionsProcessEvery
+	st.mutex.RUnlock()
+	return
+}
+
+// SetInstanceSubscriptionsProcessEvery safely sets the Configuration value for state's 'InstanceSubscriptionsProcessEvery' field
+func (st *ConfigState) SetInstanceSubscriptionsProcessEvery(v time.Duration) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.InstanceSubscriptionsProcessEvery = v
+	st.reloadToViper()
+}
+
+// InstanceSubscriptionsProcessEveryFlag returns the flag name for the 'InstanceSubscriptionsProcessEvery' field
+func InstanceSubscriptionsProcessEveryFlag() string { return "instance-subscriptions-process-every" }
+
+// GetInstanceSubscriptionsProcessEvery safely fetches the value for global configuration 'InstanceSubscriptionsProcessEvery' field
+func GetInstanceSubscriptionsProcessEvery() time.Duration {
+	return global.GetInstanceSubscriptionsProcessEvery()
+}
+
+// SetInstanceSubscriptionsProcessEvery safely sets the value for global configuration 'InstanceSubscriptionsProcessEvery' field
+func SetInstanceSubscriptionsProcessEvery(v time.Duration) {
+	global.SetInstanceSubscriptionsProcessEvery(v)
+}
+
 // GetAccountsRegistrationOpen safely fetches the Configuration value for state's 'AccountsRegistrationOpen' field
 func (st *ConfigState) GetAccountsRegistrationOpen() (v bool) {
 	st.mutex.RLock()

internal/federation/dereferencing/dereferencer_test.go

@@ -20,6 +20,7 @@
 import (
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/activity/streams/vocab"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/interaction"
@@ -77,6 +78,7 @@ func (suite *DereferencerStandardTestSuite) SetupTest() {
 	suite.client = testrig.NewMockHTTPClient(nil, "../../../testrig/media")
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.state.Storage = suite.storage
 
 	visFilter := visibility.NewFilter(&suite.state)

internal/federation/federatingdb/federatingdb_test.go

@@ -22,6 +22,7 @@
 	"time"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
@@ -91,6 +92,7 @@ func (suite *FederatingDBTestSuite) SetupTest() {
 	testrig.StandardDBSetup(suite.db, suite.testAccounts)
 
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 }
 
 func (suite *FederatingDBTestSuite) TearDownTest() {

internal/media/media_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -53,6 +54,7 @@ func (suite *MediaStandardTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.state.Storage = suite.storage
 
 	testrig.StandardStorageSetup(suite.storage, "../../testrig/media")

internal/oauth/clientstore_test.go

@@ -22,6 +22,7 @@
 	"testing"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
@@ -54,6 +55,7 @@ func (suite *PgClientStoreTestSuite) SetupTest() {
 	testrig.InitTestConfig()
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	testrig.StandardDBSetup(suite.db, nil)
 }
 

internal/processing/account/account_test.go

@@ -22,6 +22,7 @@
 	"time"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -93,6 +94,7 @@ func (suite *AccountStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.tc = typeutils.NewConverter(&suite.state)
 
 	testrig.StartTimelines(

internal/processing/admin/account_test.go

@@ -53,7 +53,7 @@ func (suite *AccountTestSuite) TestAccountActionSuspend() {
 
 	// Wait for action to finish.
 	if !testrig.WaitFor(func() bool {
-		return suite.adminProcessor.Actions().TotalRunning() == 0
+		return suite.state.AdminActions.TotalRunning() == 0
 	}) {
 		suite.FailNow("timed out waiting for admin action(s) to finish")
 	}

internal/processing/admin/accountaction.go

@@ -68,7 +68,7 @@ func (p *Processor) accountActionSuspend(
 ) (string, gtserror.WithCode) {
 	actionID := id.NewULID()
 
-	errWithCode := p.actions.Run(
+	errWithCode := p.state.AdminActions.Run(
 		ctx,
 		&gtsmodel.AdminAction{
 			ID:             actionID,

internal/processing/admin/admin.go

@@ -21,10 +21,10 @@
 	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
-	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/processing/common"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
@@ -33,21 +33,14 @@ type Processor struct {
 	// common processor logic
 	c *common.Processor
 
-	state     *state.State
+	state         *state.State
-	cleaner   *cleaner.Cleaner
+	cleaner       *cleaner.Cleaner
-	converter *typeutils.Converter
+	subscriptions *subscriptions.Subscriptions
-	federator *federation.Federator
+	converter     *typeutils.Converter
-	media     *media.Manager
+	federator     *federation.Federator
-	transport transport.Controller
+	media         *media.Manager
-	email     email.Sender
+	transport     transport.Controller
-
+	email         email.Sender
-	// admin Actions currently
-	// undergoing processing
-	actions *Actions
-}
-
-func (p *Processor) Actions() *Actions {
-	return p.actions
 }
 
 // New returns a new admin processor.
@@ -55,6 +48,7 @@ func New(
 	common *common.Processor,
 	state *state.State,
 	cleaner *cleaner.Cleaner,
+	subscriptions *subscriptions.Subscriptions,
 	federator *federation.Federator,
 	converter *typeutils.Converter,
 	mediaManager *media.Manager,
@@ -62,17 +56,14 @@ func New(
 	emailSender email.Sender,
 ) Processor {
 	return Processor{
-		c:         common,
+		c:             common,
-		state:     state,
+		state:         state,
-		cleaner:   cleaner,
+		cleaner:       cleaner,
-		converter: converter,
+		subscriptions: subscriptions,
-		federator: federator,
+		converter:     converter,
-		media:     mediaManager,
+		federator:     federator,
-		transport: transportController,
+		media:         mediaManager,
-		email:     emailSender,
+		transport:     transportController,
-		actions: &Actions{
+		email:         emailSender,
-			r:     make(map[string]*gtsmodel.AdminAction),
-			state: state,
-		},
 	}
 }

internal/processing/admin/admin_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	adminactions "github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -33,6 +34,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/processing/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 	"github.com/superseriousbusiness/gotosocial/testrig"
@@ -89,6 +91,7 @@ func (suite *AdminStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = adminactions.New(suite.state.DB, &suite.state.Workers)
 	suite.tc = typeutils.NewConverter(&suite.state)
 
 	testrig.StartTimelines(
@@ -109,6 +112,7 @@ func (suite *AdminStandardTestSuite) SetupTest() {
 
 	suite.processor = processing.NewProcessor(
 		cleaner.New(&suite.state),
+		subscriptions.New(&suite.state, suite.transportController, suite.tc),
 		suite.tc,
 		suite.federator,
 		suite.oauthServer,

internal/processing/admin/domainallow.go

@@ -22,14 +22,11 @@
 	"errors"
 	"fmt"
 
-	"codeberg.org/gruf/go-kv"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
-	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
-	"github.com/superseriousbusiness/gotosocial/internal/log"
 	"github.com/superseriousbusiness/gotosocial/internal/text"
 )
 
@@ -69,84 +66,30 @@ func (p *Processor) createDomainAllow(
 		}
 	}
 
-	actionID := id.NewULID()
+	// Run admin action to process
+	// side effects of allow.
+	action := &gtsmodel.AdminAction{
+		ID:             id.NewULID(),
+		TargetCategory: gtsmodel.AdminActionCategoryDomain,
+		TargetID:       domainAllow.Domain,
+		Type:           gtsmodel.AdminActionUnsuspend,
+		AccountID:      adminAcct.ID,
+	}
 
-	// Process domain allow side
+	if errWithCode := p.state.AdminActions.Run(
-	// effects asynchronously.
-	if errWithCode := p.actions.Run(
 		ctx,
-		&gtsmodel.AdminAction{
+		action,
-			ID:             actionID,
+		p.state.AdminActions.DomainAllowF(action.ID, domainAllow),
-			TargetCategory: gtsmodel.AdminActionCategoryDomain,
-			TargetID:       domain,
-			Type:           gtsmodel.AdminActionSuspend,
-			AccountID:      adminAcct.ID,
-			Text:           domainAllow.PrivateComment,
-		},
-		func(ctx context.Context) gtserror.MultiError {
-			// Log start + finish.
-			l := log.WithFields(kv.Fields{
-				{"domain", domain},
-				{"actionID", actionID},
-			}...).WithContext(ctx)
-
-			l.Info("processing domain allow side effects")
-			defer func() { l.Info("finished processing domain allow side effects") }()
-
-			return p.domainAllowSideEffects(ctx, domainAllow)
-		},
 	); errWithCode != nil {
-		return nil, actionID, errWithCode
+		return nil, action.ID, errWithCode
 	}
 
 	apiDomainAllow, errWithCode := p.apiDomainPerm(ctx, domainAllow, false)
 	if errWithCode != nil {
-		return nil, actionID, errWithCode
+		return nil, action.ID, errWithCode
 	}
 
-	return apiDomainAllow, actionID, nil
+	return apiDomainAllow, action.ID, nil
-}
-
-func (p *Processor) domainAllowSideEffects(
-	ctx context.Context,
-	allow *gtsmodel.DomainAllow,
-) gtserror.MultiError {
-	if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist {
-		// We're running in allowlist mode,
-		// so there are no side effects to
-		// process here.
-		return nil
-	}
-
-	// We're running in blocklist mode or
-	// some similar mode which necessitates
-	// domain allow side effects if a block
-	// was in place when the allow was created.
-	//
-	// So, check if there's a block.
-	block, err := p.state.DB.GetDomainBlock(ctx, allow.Domain)
-	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		errs := gtserror.NewMultiError(1)
-		errs.Appendf("db error getting domain block %s: %w", allow.Domain, err)
-		return errs
-	}
-
-	if block == nil {
-		// No block?
-		// No problem!
-		return nil
-	}
-
-	// There was a block, over which the new
-	// allow ought to take precedence. To account
-	// for this, just run side effects as though
-	// the domain was being unblocked, while
-	// leaving the existing block in place.
-	//
-	// Any accounts that were suspended by
-	// the block will be unsuspended and be
-	// able to interact with the instance again.
-	return p.domainUnblockSideEffects(ctx, block)
 }
 
 func (p *Processor) deleteDomainAllow(
@@ -179,77 +122,23 @@ func (p *Processor) deleteDomainAllow(
 		return nil, "", gtserror.NewErrorInternalError(err)
 	}
 
-	actionID := id.NewULID()
+	// Run admin action to process
+	// side effects of unallow.
+	action := &gtsmodel.AdminAction{
+		ID:             id.NewULID(),
+		TargetCategory: gtsmodel.AdminActionCategoryDomain,
+		TargetID:       domainAllow.Domain,
+		Type:           gtsmodel.AdminActionUnsuspend,
+		AccountID:      adminAcct.ID,
+	}
 
-	// Process domain unallow side
+	if errWithCode := p.state.AdminActions.Run(
-	// effects asynchronously.
-	if errWithCode := p.actions.Run(
 		ctx,
-		&gtsmodel.AdminAction{
+		action,
-			ID:             actionID,
+		p.state.AdminActions.DomainUnallowF(action.ID, domainAllow),
-			TargetCategory: gtsmodel.AdminActionCategoryDomain,
-			TargetID:       domainAllow.Domain,
-			Type:           gtsmodel.AdminActionUnsuspend,
-			AccountID:      adminAcct.ID,
-		},
-		func(ctx context.Context) gtserror.MultiError {
-			// Log start + finish.
-			l := log.WithFields(kv.Fields{
-				{"domain", domainAllow.Domain},
-				{"actionID", actionID},
-			}...).WithContext(ctx)
-
-			l.Info("processing domain unallow side effects")
-			defer func() { l.Info("finished processing domain unallow side effects") }()
-
-			return p.domainUnallowSideEffects(ctx, domainAllow)
-		},
 	); errWithCode != nil {
-		return nil, actionID, errWithCode
+		return nil, action.ID, errWithCode
 	}
 
-	return apiDomainAllow, actionID, nil
+	return apiDomainAllow, action.ID, nil
-}
-
-func (p *Processor) domainUnallowSideEffects(
-	ctx context.Context,
-	allow *gtsmodel.DomainAllow,
-) gtserror.MultiError {
-	if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist {
-		// We're running in allowlist mode,
-		// so there are no side effects to
-		// process here.
-		return nil
-	}
-
-	// We're running in blocklist mode or
-	// some similar mode which necessitates
-	// domain allow side effects if a block
-	// was in place when the allow was removed.
-	//
-	// So, check if there's a block.
-	block, err := p.state.DB.GetDomainBlock(ctx, allow.Domain)
-	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		errs := gtserror.NewMultiError(1)
-		errs.Appendf("db error getting domain block %s: %w", allow.Domain, err)
-		return errs
-	}
-
-	if block == nil {
-		// No block?
-		// No problem!
-		return nil
-	}
-
-	// There was a block, over which the previous
-	// allow was taking precedence. Now that the
-	// allow has been removed, we should put the
-	// side effects of the block back in place.
-	//
-	// To do this, process the block side effects
-	// again as though the block were freshly
-	// created. This will mark all accounts from
-	// the blocked domain as suspended, and clean
-	// up their follows/following, media, etc.
-	return p.domainBlockSideEffects(ctx, block)
 }

internal/processing/admin/domainblock.go

@@ -21,18 +21,12 @@
 	"context"
 	"errors"
 	"fmt"
-	"time"
 
-	"codeberg.org/gruf/go-kv"
-	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
-	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
-	"github.com/superseriousbusiness/gotosocial/internal/log"
-	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/text"
 )
 
@@ -72,149 +66,31 @@ func (p *Processor) createDomainBlock(
 		}
 	}
 
-	actionID := id.NewULID()
+	// Run admin action to process
+	// side effects of block.
+	action := &gtsmodel.AdminAction{
+		ID:             id.NewULID(),
+		TargetCategory: gtsmodel.AdminActionCategoryDomain,
+		TargetID:       domain,
+		Type:           gtsmodel.AdminActionSuspend,
+		AccountID:      adminAcct.ID,
+		Text:           domainBlock.PrivateComment,
+	}
 
-	// Process domain block side
+	if errWithCode := p.state.AdminActions.Run(
-	// effects asynchronously.
-	if errWithCode := p.actions.Run(
 		ctx,
-		&gtsmodel.AdminAction{
+		action,
-			ID:             actionID,
+		p.state.AdminActions.DomainBlockF(action.ID, domainBlock),
-			TargetCategory: gtsmodel.AdminActionCategoryDomain,
-			TargetID:       domain,
-			Type:           gtsmodel.AdminActionSuspend,
-			AccountID:      adminAcct.ID,
-			Text:           domainBlock.PrivateComment,
-		},
-		func(ctx context.Context) gtserror.MultiError {
-			// Log start + finish.
-			l := log.WithFields(kv.Fields{
-				{"domain", domain},
-				{"actionID", actionID},
-			}...).WithContext(ctx)
-
-			skip, err := p.skipBlockSideEffects(ctx, domain)
-			if err != nil {
-				return err
-			}
-			if skip != "" {
-				l.Infof("skipping domain block side effects: %s", skip)
-				return nil
-			}
-
-			l.Info("processing domain block side effects")
-			defer func() { l.Info("finished processing domain block side effects") }()
-
-			return p.domainBlockSideEffects(ctx, domainBlock)
-		},
 	); errWithCode != nil {
-		return nil, actionID, errWithCode
+		return nil, action.ID, errWithCode
 	}
 
 	apiDomainBlock, errWithCode := p.apiDomainPerm(ctx, domainBlock, false)
 	if errWithCode != nil {
-		return nil, actionID, errWithCode
+		return nil, action.ID, errWithCode
 	}
 
-	return apiDomainBlock, actionID, nil
+	return apiDomainBlock, action.ID, nil
-}
-
-// skipBlockSideEffects checks if side effects of block creation
-// should be skipped for the given domain, taking account of
-// instance federation mode, and existence of any allows
-// which ought to "shield" this domain from being blocked.
-//
-// If the caller should skip, the returned string will be non-zero
-// and will be set to a reason why side effects should be skipped.
-//
-//   - blocklist mode + allow exists: "..." (skip)
-//   - blocklist mode + no allow:     ""    (don't skip)
-//   - allowlist mode + allow exists: ""    (don't skip)
-//   - allowlist mode + no allow:     ""    (don't skip)
-func (p *Processor) skipBlockSideEffects(
-	ctx context.Context,
-	domain string,
-) (string, gtserror.MultiError) {
-	var (
-		skip string // Assume "" (don't skip).
-		errs gtserror.MultiError
-	)
-
-	// Never skip block side effects in allowlist mode.
-	fediMode := config.GetInstanceFederationMode()
-	if fediMode == config.InstanceFederationModeAllowlist {
-		return skip, errs
-	}
-
-	// We know we're in blocklist mode.
-	//
-	// We want to skip domain block side
-	// effects if an allow is already
-	// in place which overrides the block.
-
-	// Check if an explicit allow exists for this domain.
-	domainAllow, err := p.state.DB.GetDomainAllow(ctx, domain)
-	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		errs.Appendf("error getting domain allow: %w", err)
-		return skip, errs
-	}
-
-	if domainAllow != nil {
-		skip = "running in blocklist mode, and an explicit allow exists for this domain"
-		return skip, errs
-	}
-
-	return skip, errs
-}
-
-// domainBlockSideEffects processes the side effects of a domain block:
-//
-//  1. Strip most info away from the instance entry for the domain.
-//  2. Pass each account from the domain to the processor for deletion.
-//
-// It should be called asynchronously, since it can take a while when
-// there are many accounts present on the given domain.
-func (p *Processor) domainBlockSideEffects(
-	ctx context.Context,
-	block *gtsmodel.DomainBlock,
-) gtserror.MultiError {
-	var errs gtserror.MultiError
-
-	// If we have an instance entry for this domain,
-	// update it with the new block ID and clear all fields
-	instance, err := p.state.DB.GetInstance(ctx, block.Domain)
-	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		errs.Appendf("db error getting instance %s: %w", block.Domain, err)
-		return errs
-	}
-
-	if instance != nil {
-		// We had an entry for this domain.
-		columns := stubbifyInstance(instance, block.ID)
-		if err := p.state.DB.UpdateInstance(ctx, instance, columns...); err != nil {
-			errs.Appendf("db error updating instance: %w", err)
-			return errs
-		}
-	}
-
-	// For each account that belongs to this domain,
-	// process an account delete message to remove
-	// that account's posts, media, etc.
-	if err := p.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) {
-		if err := p.state.Workers.Client.Process(ctx, &messages.FromClientAPI{
-			APObjectType:   ap.ActorPerson,
-			APActivityType: ap.ActivityDelete,
-			GTSModel:       block,
-			Origin:         account,
-			Target:         account,
-		}); err != nil {
-			errs.Append(err)
-		}
-	}); err != nil {
-		errs.Appendf("db error ranging through accounts: %w", err)
-	}
-
-	return errs
 }
 
 func (p *Processor) deleteDomainBlock(
@@ -247,104 +123,23 @@ func (p *Processor) deleteDomainBlock(
 		return nil, "", gtserror.NewErrorInternalError(err)
 	}
 
-	actionID := id.NewULID()
+	// Run admin action to process
+	// side effects of unblock.
+	action := &gtsmodel.AdminAction{
+		ID:             id.NewULID(),
+		TargetCategory: gtsmodel.AdminActionCategoryDomain,
+		TargetID:       domainBlock.Domain,
+		Type:           gtsmodel.AdminActionUnsuspend,
+		AccountID:      adminAcct.ID,
+	}
 
-	// Process domain unblock side
+	if errWithCode := p.state.AdminActions.Run(
-	// effects asynchronously.
-	if errWithCode := p.actions.Run(
 		ctx,
-		&gtsmodel.AdminAction{
+		action,
-			ID:             actionID,
+		p.state.AdminActions.DomainUnblockF(action.ID, domainBlock),
-			TargetCategory: gtsmodel.AdminActionCategoryDomain,
-			TargetID:       domainBlock.Domain,
-			Type:           gtsmodel.AdminActionUnsuspend,
-			AccountID:      adminAcct.ID,
-		},
-		func(ctx context.Context) gtserror.MultiError {
-			// Log start + finish.
-			l := log.WithFields(kv.Fields{
-				{"domain", domainBlock.Domain},
-				{"actionID", actionID},
-			}...).WithContext(ctx)
-
-			l.Info("processing domain unblock side effects")
-			defer func() { l.Info("finished processing domain unblock side effects") }()
-
-			return p.domainUnblockSideEffects(ctx, domainBlock)
-		},
 	); errWithCode != nil {
-		return nil, actionID, errWithCode
+		return nil, action.ID, errWithCode
 	}
 
-	return apiDomainBlock, actionID, nil
+	return apiDomainBlock, action.ID, nil
-}
-
-// domainUnblockSideEffects processes the side effects of undoing a
-// domain block:
-//
-//  1. Mark instance entry as no longer suspended.
-//  2. Mark each account from the domain as no longer suspended, if the
-//     suspension origin corresponds to the ID of the provided domain block.
-//
-// It should be called asynchronously, since it can take a while when
-// there are many accounts present on the given domain.
-func (p *Processor) domainUnblockSideEffects(
-	ctx context.Context,
-	block *gtsmodel.DomainBlock,
-) gtserror.MultiError {
-	var errs gtserror.MultiError
-
-	// Update instance entry for this domain, if we have it.
-	instance, err := p.state.DB.GetInstance(ctx, block.Domain)
-	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		errs.Appendf("db error getting instance %s: %w", block.Domain, err)
-	}
-
-	if instance != nil {
-		// We had an entry, update it to signal
-		// that it's no longer suspended.
-		instance.SuspendedAt = time.Time{}
-		instance.DomainBlockID = ""
-		if err := p.state.DB.UpdateInstance(
-			ctx,
-			instance,
-			"suspended_at",
-			"domain_block_id",
-		); err != nil {
-			errs.Appendf("db error updating instance: %w", err)
-			return errs
-		}
-	}
-
-	// Unsuspend all accounts whose suspension origin was this domain block.
-	if err := p.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) {
-		if account.SuspensionOrigin == "" || account.SuspendedAt.IsZero() {
-			// Account wasn't suspended, nothing to do.
-			return
-		}
-
-		if account.SuspensionOrigin != block.ID {
-			// Account was suspended, but not by
-			// this domain block, leave it alone.
-			return
-		}
-
-		// Account was suspended by this domain
-		// block, mark it as unsuspended.
-		account.SuspendedAt = time.Time{}
-		account.SuspensionOrigin = ""
-
-		if err := p.state.DB.UpdateAccount(
-			ctx,
-			account,
-			"suspended_at",
-			"suspension_origin",
-		); err != nil {
-			errs.Appendf("db error updating account %s: %w", account.Username, err)
-		}
-	}); err != nil {
-		errs.Appendf("db error ranging through accounts: %w", err)
-	}
-
-	return errs
 }

internal/processing/admin/domainkeysexpire.go

@@ -19,7 +19,6 @@
 
 import (
 	"context"
-	"time"
 
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -39,47 +38,23 @@ func (p *Processor) DomainKeysExpire(
 	adminAcct *gtsmodel.Account,
 	domain string,
 ) (string, gtserror.WithCode) {
-	actionID := id.NewULID()
+	// Run admin action to process
+	// side effects of key expiry.
+	action := &gtsmodel.AdminAction{
+		ID:             id.NewULID(),
+		TargetCategory: gtsmodel.AdminActionCategoryDomain,
+		TargetID:       domain,
+		Type:           gtsmodel.AdminActionExpireKeys,
+		AccountID:      adminAcct.ID,
+	}
 
-	// Process key expiration asynchronously.
+	if errWithCode := p.state.AdminActions.Run(
-	if errWithCode := p.actions.Run(
 		ctx,
-		&gtsmodel.AdminAction{
+		action,
-			ID:             actionID,
+		p.state.AdminActions.DomainKeysExpireF(domain),
-			TargetCategory: gtsmodel.AdminActionCategoryDomain,
-			TargetID:       domain,
-			Type:           gtsmodel.AdminActionExpireKeys,
-			AccountID:      adminAcct.ID,
-		},
-		func(ctx context.Context) gtserror.MultiError {
-			return p.domainKeysExpireSideEffects(ctx, domain)
-		},
 	); errWithCode != nil {
-		return actionID, errWithCode
+		return action.ID, errWithCode
 	}
 
-	return actionID, nil
+	return action.ID, nil
-}
-
-func (p *Processor) domainKeysExpireSideEffects(ctx context.Context, domain string) gtserror.MultiError {
-	var (
-		expiresAt = time.Now()
-		errs      gtserror.MultiError
-	)
-
-	// For each account on this domain, expire
-	// the public key and update the account.
-	if err := p.rangeDomainAccounts(ctx, domain, func(account *gtsmodel.Account) {
-		account.PublicKeyExpiresAt = expiresAt
-		if err := p.state.DB.UpdateAccount(ctx,
-			account,
-			"public_key_expires_at",
-		); err != nil {
-			errs.Appendf("db error updating account: %w", err)
-		}
-	}); err != nil {
-		errs.Appendf("db error ranging through accounts: %w", err)
-	}
-
-	return errs
 }

internal/processing/admin/domainpermission_test.go

@@ -186,7 +186,7 @@ func (suite *DomainBlockTestSuite) awaitAction(actionID string) {
 	ctx := context.Background()
 
 	if !testrig.WaitFor(func() bool {
-		return suite.adminProcessor.Actions().TotalRunning() == 0
+		return suite.state.AdminActions.TotalRunning() == 0
 	}) {
 		suite.FailNow("timed out waiting for admin action(s) to finish")
 	}

internal/processing/admin/domainpermissionsubscription.go

@@ -22,6 +22,7 @@
 	"errors"
 	"fmt"
 	"net/url"
+	"slices"
 
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
@@ -283,3 +284,89 @@ func (p *Processor) DomainPermissionSubscriptionRemove(
 
 	return p.apiDomainPermSub(ctx, permSub)
 }
+
+func (p *Processor) DomainPermissionSubscriptionTest(
+	ctx context.Context,
+	acct *gtsmodel.Account,
+	id string,
+) (any, gtserror.WithCode) {
+	permSub, err := p.state.DB.GetDomainPermissionSubscriptionByID(ctx, id)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		err := gtserror.Newf("db error getting domain permission subscription %s: %w", id, err)
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	if permSub == nil {
+		err := fmt.Errorf("domain permission subscription %s not found", id)
+		return nil, gtserror.NewErrorNotFound(err, err.Error())
+	}
+
+	// To process the test/dry-run correctly, we need to get
+	// all domain perm subs of this type with a *higher* priority,
+	// to know whether we ought to create permissions or not.
+	permSubs, err := p.state.DB.GetDomainPermissionSubscriptionsByPriority(
+		ctx,
+		permSub.PermissionType,
+	)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		err := gtserror.Newf("db error: %w", err)
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	// Find the index of the targeted
+	// subscription in the slice.
+	index := slices.IndexFunc(
+		permSubs,
+		func(ps *gtsmodel.DomainPermissionSubscription) bool {
+			return ps.ID == permSub.ID
+		},
+	)
+
+	// Get a transport for calling permSub.URI.
+	tsport, err := p.transport.NewTransportForUsername(ctx, acct.Username)
+	if err != nil {
+		err := gtserror.Newf("error getting transport: %w", err)
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	// Everything *before* the targeted
+	// subscription has a higher priority.
+	higherPrios := permSubs[:index]
+
+	// Call the permSub.URI and parse a list of perms from it.
+	// Any error returned here is a "real" one, not an error
+	// from fetching / parsing the list.
+	createdPerms, err := p.subscriptions.ProcessDomainPermissionSubscription(
+		ctx,
+		permSub,
+		tsport,
+		higherPrios,
+		true, // Dry run.
+	)
+	if err != nil {
+		err := gtserror.Newf("error doing dry-run: %w", err)
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	// If permSub has an error set on it now,
+	// we should return it to the caller.
+	if permSub.Error != "" {
+		return map[string]string{
+			"error": permSub.Error,
+		}, nil
+	}
+
+	// No error, so return the list of
+	// perms that would have been created.
+	apiPerms := make([]*apimodel.DomainPermission, 0, len(createdPerms))
+	for _, perm := range createdPerms {
+		apiPerm, errWithCode := p.apiDomainPerm(ctx, perm, false)
+		if errWithCode != nil {
+			return nil, errWithCode
+		}
+
+		apiPerms = append(apiPerms, apiPerm)
+	}
+
+	return apiPerms, nil
+}

internal/processing/admin/util.go

@@ -19,86 +19,12 @@
 
 import (
 	"context"
-	"errors"
-	"time"
 
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
-	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
-// stubbifyInstance renders the given instance as a stub,
-// removing most information from it and marking it as
-// suspended.
-//
-// For caller's convenience, this function returns the db
-// names of all columns that are updated by it.
-func stubbifyInstance(instance *gtsmodel.Instance, domainBlockID string) []string {
-	instance.Title = ""
-	instance.SuspendedAt = time.Now()
-	instance.DomainBlockID = domainBlockID
-	instance.ShortDescription = ""
-	instance.Description = ""
-	instance.Terms = ""
-	instance.ContactEmail = ""
-	instance.ContactAccountUsername = ""
-	instance.ContactAccountID = ""
-	instance.Version = ""
-
-	return []string{
-		"title",
-		"suspended_at",
-		"domain_block_id",
-		"short_description",
-		"description",
-		"terms",
-		"contact_email",
-		"contact_account_username",
-		"contact_account_id",
-		"version",
-	}
-}
-
-// rangeDomainAccounts iterates through all accounts
-// originating from the given domain, and calls the
-// provided range function on each account.
-//
-// If an error is returned while selecting accounts,
-// the loop will stop and return the error.
-func (p *Processor) rangeDomainAccounts(
-	ctx context.Context,
-	domain string,
-	rangeF func(*gtsmodel.Account),
-) error {
-	var (
-		limit = 50   // Limit selection to avoid spiking mem/cpu.
-		maxID string // Start with empty string to select from top.
-	)
-
-	for {
-		// Get (next) page of accounts.
-		accounts, err := p.state.DB.GetInstanceAccounts(ctx, domain, maxID, limit)
-		if err != nil && !errors.Is(err, db.ErrNoEntries) {
-			// Real db error.
-			return gtserror.Newf("db error getting instance accounts: %w", err)
-		}
-
-		if len(accounts) == 0 {
-			// No accounts left, we're done.
-			return nil
-		}
-
-		// Set next max ID for paging down.
-		maxID = accounts[len(accounts)-1].ID
-
-		// Call provided range function.
-		for _, account := range accounts {
-			rangeF(account)
-		}
-	}
-}
-
 // apiDomainPerm is a cheeky shortcut for returning
 // the API version of the given domain permission,
 // or an appropriate error if something goes wrong.

internal/processing/conversations/conversations_test.go

@@ -23,6 +23,7 @@
 	"time"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	dbtest "github.com/superseriousbusiness/gotosocial/internal/db/test"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -103,6 +104,7 @@ func (suite *ConversationsTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.tc = typeutils.NewConverter(&suite.state)
 	suite.filter = visibility.NewFilter(&suite.state)
 

internal/processing/media/media_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -75,6 +76,7 @@ func (suite *MediaStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.tc = typeutils.NewConverter(&suite.state)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage

internal/processing/processor.go

@@ -48,6 +48,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/processing/user"
 	"github.com/superseriousbusiness/gotosocial/internal/processing/workers"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
 	"github.com/superseriousbusiness/gotosocial/internal/text"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
@@ -180,6 +181,7 @@ func (p *Processor) Workers() *workers.Processor {
 // NewProcessor returns a new Processor.
 func NewProcessor(
 	cleaner *cleaner.Cleaner,
+	subscriptions *subscriptions.Subscriptions,
 	converter *typeutils.Converter,
 	federator *federation.Federator,
 	oauthServer oauth.Server,
@@ -210,7 +212,7 @@ func NewProcessor(
 	// Instantiate the rest of the sub
 	// processors + pin them to this struct.
 	processor.account = account.New(&common, state, converter, mediaManager, federator, visFilter, parseMentionFunc)
-	processor.admin = admin.New(&common, state, cleaner, federator, converter, mediaManager, federator.TransportController(), emailSender)
+	processor.admin = admin.New(&common, state, cleaner, subscriptions, federator, converter, mediaManager, federator.TransportController(), emailSender)
 	processor.conversations = conversations.New(state, converter, visFilter)
 	processor.fedi = fedi.New(state, &common, converter, federator, visFilter)
 	processor.filtersv1 = filtersv1.New(state, converter, &processor.stream)

internal/processing/processor_test.go

@@ -21,6 +21,7 @@
 	"context"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -34,6 +35,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/stream"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 	"github.com/superseriousbusiness/gotosocial/testrig"
@@ -102,6 +104,7 @@ func (suite *ProcessingStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.testActivities = testrig.NewTestActivities(suite.testAccounts)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
@@ -125,6 +128,7 @@ func (suite *ProcessingStandardTestSuite) SetupTest() {
 
 	suite.processor = processing.NewProcessor(
 		cleaner.New(&suite.state),
+		subscriptions.New(&suite.state, suite.transportController, suite.typeconverter),
 		suite.typeconverter,
 		suite.federator,
 		suite.oauthServer,

internal/processing/status/status_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/interaction"
@@ -84,6 +85,7 @@ func (suite *StatusStandardTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.typeConverter = typeutils.NewConverter(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 
 	suite.tc = testrig.NewTestTransportController(&suite.state, testrig.NewMockHTTPClient(nil, "../../../testrig/media"))
 	suite.storage = testrig.NewInMemoryStorage()

internal/processing/stream/stream_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
@@ -50,6 +51,7 @@ func (suite *StreamTestSuite) SetupTest() {
 	suite.testTokens = testrig.NewTestTokens()
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
 	suite.streamProcessor = stream.New(&suite.state, suite.oauthServer)
 

internal/processing/timeline/timeline_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -55,6 +56,7 @@ func (suite *TimelineStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 
 	suite.timeline = timeline.New(
 		&suite.state,

internal/processing/user/user_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -49,6 +50,7 @@ func (suite *UserStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 
 	suite.sentEmails = make(map[string]string)
 	suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails)

internal/state/state.go

@@ -19,6 +19,7 @@
 
 import (
 	"codeberg.org/gruf/go-mutexes"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/cache"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
@@ -61,9 +62,14 @@ type State struct {
 	// Storage provides access to the storage driver.
 	Storage *storage.Driver
 
-	// Workers provides access to this state's collection of worker pools.
+	// Workers provides access to this
+	// state's collection of worker pools.
 	Workers workers.Workers
 
+	// Struct to manage running admin
+	// actions (and locks thereupon).
+	AdminActions *admin.Actions
+
 	// prevent pass-by-value.
 	_ nocopy
 }

internal/subscriptions/domainperms.go

@@ -0,0 +1,811 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package subscriptions
+
+import (
+	"bufio"
+	"context"
+	"encoding/csv"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+
+	"codeberg.org/gruf/go-kv"
+
+	"github.com/miekg/dns"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
+	"github.com/superseriousbusiness/gotosocial/internal/log"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+// ScheduleJobs schedules domain permission subscription
+// fetching + updating using configured parameters.
+//
+// Returns an error if `MediaCleanupFrom`
+// is not a valid format (hh:mm:ss).
+func (s *Subscriptions) ScheduleJobs() error {
+	const hourMinute = "15:04"
+
+	var (
+		now            = time.Now()
+		processEvery   = config.GetInstanceSubscriptionsProcessEvery()
+		processFromStr = config.GetInstanceSubscriptionsProcessFrom()
+	)
+
+	// Parse processFromStr as hh:mm.
+	// Resulting time will be on 1 Jan year zero.
+	processFrom, err := time.Parse(hourMinute, processFromStr)
+	if err != nil {
+		return gtserror.Newf(
+			"error parsing '%s' in time format 'hh:mm': %w",
+			processFromStr, err,
+		)
+	}
+
+	// Time travel from
+	// year zero, groovy.
+	firstProcessAt := time.Date(
+		now.Year(),
+		now.Month(),
+		now.Day(),
+		processFrom.Hour(),
+		processFrom.Minute(),
+		0,
+		0,
+		now.Location(),
+	)
+
+	// Ensure first processing is in the future.
+	for firstProcessAt.Before(now) {
+		firstProcessAt = firstProcessAt.Add(processEvery)
+	}
+
+	fn := func(ctx context.Context, start time.Time) {
+		log.Info(ctx, "starting instance subscriptions processing")
+
+		// In blocklist (default) mode, process allows
+		// first to provide immunity to block side effects.
+		//
+		// In allowlist mode, process blocks first to
+		// ensure allowlist doesn't override blocks.
+		var order [2]gtsmodel.DomainPermissionType
+		if config.GetInstanceFederationMode() == config.InstanceFederationModeBlocklist {
+			order = [2]gtsmodel.DomainPermissionType{
+				gtsmodel.DomainPermissionAllow,
+				gtsmodel.DomainPermissionBlock,
+			}
+		} else {
+			order = [2]gtsmodel.DomainPermissionType{
+				gtsmodel.DomainPermissionBlock,
+				gtsmodel.DomainPermissionAllow,
+			}
+		}
+
+		// Fetch + process subscribed perms in order.
+		for _, permType := range order {
+			s.ProcessDomainPermissionSubscriptions(ctx, permType)
+		}
+
+		log.Infof(ctx, "finished instance subscriptions processing after %s", time.Since(start))
+	}
+
+	log.Infof(nil,
+		"scheduling instance subscriptions processing to run every %s, starting from %s; next processing will run at %s",
+		processEvery, processFromStr, firstProcessAt,
+	)
+
+	// Schedule processing to execute according to schedule.
+	if !s.state.Workers.Scheduler.AddRecurring(
+		"@subsprocessing",
+		firstProcessAt,
+		processEvery,
+		fn,
+	) {
+		panic("failed to schedule @subsprocessing")
+	}
+
+	return nil
+}
+
+// ProcessDomainPermissionSubscriptions processes all domain permission
+// subscriptions of the given permission type by, in turn, calling the
+// URI of each subscription, parsing the result into a list of domain
+// permissions, and creating (or skipping) each permission as appropriate.
+func (s *Subscriptions) ProcessDomainPermissionSubscriptions(
+	ctx context.Context,
+	permType gtsmodel.DomainPermissionType,
+) {
+	log.Info(ctx, "start")
+	defer log.Info(ctx, "finished")
+
+	// Get permission subscriptions in priority order (highest -> lowest).
+	permSubs, err := s.state.DB.GetDomainPermissionSubscriptionsByPriority(ctx, permType)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		// Real db error.
+		log.Errorf(ctx, "db error getting domain perm subs by priority: %v", err)
+		return
+	}
+
+	if len(permSubs) == 0 {
+		// No subscriptions of this
+		// type, so nothing to do.
+		return
+	}
+
+	// Get a transport using the instance account,
+	// we can reuse this for each HTTP call.
+	tsport, err := s.transportController.NewTransportForUsername(ctx, "")
+	if err != nil {
+		log.Errorf(ctx, "error getting transport for instance account: %v", err)
+		return
+	}
+
+	for i, permSub := range permSubs {
+		// Higher priority permission subs = everything
+		// above this permission sub in the slice.
+		higherPrios := permSubs[:i]
+
+		_, err := s.ProcessDomainPermissionSubscription(
+			ctx,
+			permSub,
+			tsport,
+			higherPrios,
+			false, // Not dry. Wet, if you will.
+		)
+		if err != nil {
+			// Real db error.
+			log.Errorf(ctx,
+				"error processing domain permission subscription %s: %v",
+				permSub.URI, err,
+			)
+			return
+		}
+
+		// Update this perm sub.
+		err = s.state.DB.UpdateDomainPermissionSubscription(ctx, permSub)
+		if err != nil {
+			// Real db error.
+			log.Errorf(ctx, "db error updating domain perm sub: %v", err)
+			return
+		}
+	}
+}
+
+// ProcessDomainPermissionSubscription processes one domain permission
+// subscription by dereferencing the URI, parsing the response into a list
+// of permissions, and for each discovered permission either creating an
+// entry in the database, or ignoring it if it's excluded or already
+// covered by a higher-priority subscription.
+//
+// On success, the slice of discovered DomainPermissions will be returned.
+// In case of parsing error, or error on the remote side, permSub.Error
+// will be updated with the calling/parsing error, and `nil, nil` will be
+// returned. In case of an actual db error, `nil, err` will be returned and
+// the caller should handle it.
+//
+// getHigherPrios should be a function for returning a slice of domain
+// permission subscriptions with a higher priority than the given permSub.
+//
+// If dry == true, then the URI will still be called, and permissions
+// will be parsed, but they will not actually be created.
+//
+// Note that while this function modifies fields on the given permSub,
+// it's up to the caller to update it in the database (if desired).
+func (s *Subscriptions) ProcessDomainPermissionSubscription(
+	ctx context.Context,
+	permSub *gtsmodel.DomainPermissionSubscription,
+	tsport transport.Transport,
+	higherPrios []*gtsmodel.DomainPermissionSubscription,
+	dry bool,
+) ([]gtsmodel.DomainPermission, error) {
+	l := log.
+		WithContext(ctx).
+		WithFields(kv.Fields{
+			{"permType", permSub.PermissionType.String()},
+			{"permSubURI", permSub.URI},
+		}...)
+
+	// Set FetchedAt as we're
+	// going to attempt this now.
+	permSub.FetchedAt = time.Now()
+
+	// Call the URI, and only skip
+	// cache if we're doing a dry run.
+	resp, err := tsport.DereferenceDomainPermissions(
+		ctx, permSub, dry,
+	)
+	if err != nil {
+		// Couldn't get this one,
+		// set error + return.
+		errStr := err.Error()
+		l.Warnf("couldn't dereference permSubURI: %+v", err)
+		permSub.Error = errStr
+		return nil, nil
+	}
+
+	// If the permissions at URI weren't modified
+	// since last time, just update some metadata
+	// to indicate a successful fetch, and return.
+	if resp.Unmodified {
+		l.Debug("received 304 Not Modified from remote")
+		permSub.SuccessfullyFetchedAt = permSub.FetchedAt
+		if permSub.ETag == "" && resp.ETag != "" {
+			// We didn't have an ETag before but
+			// we have one now: probably the remote
+			// added ETag support in the meantime.
+			permSub.ETag = resp.ETag
+		}
+		return nil, nil
+	}
+
+	// At this point we know we got a 200 OK
+	// from the URI, so we've got a live body!
+	// Try to parse the body as a list of wantedPerms
+	// that the subscription wants to create.
+	var wantedPerms []gtsmodel.DomainPermission
+
+	switch permSub.ContentType {
+
+	// text/csv
+	case gtsmodel.DomainPermSubContentTypeCSV:
+		wantedPerms, err = permsFromCSV(l, permSub.PermissionType, resp.Body)
+
+	// application/json
+	case gtsmodel.DomainPermSubContentTypeJSON:
+		wantedPerms, err = permsFromJSON(l, permSub.PermissionType, resp.Body)
+
+	// text/plain
+	case gtsmodel.DomainPermSubContentTypePlain:
+		wantedPerms, err = permsFromPlain(l, permSub.PermissionType, resp.Body)
+	}
+
+	if err != nil {
+		// We retrieved the permissions from remote but
+		// the connection died halfway through transfer,
+		// or we couldn't parse the results, or something.
+		// Just set error and return.
+		errStr := err.Error()
+		l.Warnf("couldn't parse results: %+v", err)
+		permSub.Error = errStr
+		return nil, nil
+	}
+
+	if len(wantedPerms) == 0 {
+		// Fetch was OK, and parsing was, on the surface at
+		// least, OK, but we didn't get any perms. Consider
+		// this an error as users will probably want to know.
+		const errStr = "fetch successful but parsed zero usable results"
+		l.Warn(errStr)
+		permSub.Error = errStr
+		return nil, nil
+	}
+
+	// This can now be considered a successful fetch.
+	permSub.SuccessfullyFetchedAt = permSub.FetchedAt
+	permSub.ETag = resp.ETag
+	permSub.Error = ""
+
+	// Keep track of which domain perms are
+	// created (or would be, if dry == true).
+	createdPerms := make([]gtsmodel.DomainPermission, 0, len(wantedPerms))
+
+	// Iterate through wantedPerms and
+	// create (or dry create) each one.
+	for _, wantedPerm := range wantedPerms {
+		l = l.WithField("domain", wantedPerm.GetDomain())
+		created, err := s.processDomainPermission(
+			ctx, l,
+			wantedPerm,
+			permSub,
+			higherPrios,
+			dry,
+		)
+		if err != nil {
+			// Proper db error.
+			return nil, err
+		}
+
+		if !created {
+			continue
+		}
+
+		createdPerms = append(createdPerms, wantedPerm)
+	}
+
+	return createdPerms, nil
+}
+
+// processDomainPermission processes one wanted domain
+// permission discovered via a domain permission sub's URI.
+//
+// Error will only be returned in case of an actual database
+// error, else the error will be logged and nil returned.
+func (s *Subscriptions) processDomainPermission(
+	ctx context.Context,
+	l log.Entry,
+	wantedPerm gtsmodel.DomainPermission,
+	permSub *gtsmodel.DomainPermissionSubscription,
+	higherPrios []*gtsmodel.DomainPermissionSubscription,
+	dry bool,
+) (bool, error) {
+	// Set to true if domain permission
+	// actually (would be) created.
+	var created bool
+
+	// If domain is excluded from automatic
+	// permission creation, don't process it.
+	domain := wantedPerm.GetDomain()
+	excluded, err := s.state.DB.IsDomainPermissionExcluded(ctx, domain)
+	if err != nil {
+		// Proper db error.
+		return created, err
+	}
+
+	if excluded {
+		l.Debug("domain is excluded, skipping")
+		return created, nil
+	}
+
+	// Check if a permission already exists for
+	// this domain, and if it's covered already
+	// by a higher-priority subscription.
+	existingPerm, covered, err := s.existingCovered(
+		ctx, permSub.PermissionType, domain, higherPrios,
+	)
+	if err != nil {
+		// Proper db error.
+		return created, err
+	}
+
+	if covered {
+		l.Debug("domain is covered by a higher-priority subscription, skipping")
+		return created, nil
+	}
+
+	// At this point we know we
+	// should create the perm.
+	created = true
+
+	if dry {
+		// Don't do creation or side
+		// effects if we're dry running.
+		return created, nil
+	}
+
+	// Handle perm creation differently depending
+	// on whether or not a perm already existed.
+	existing := !util.IsNil(existingPerm)
+	switch {
+
+	case !existing && *permSub.AsDraft:
+		// No existing perm, create as draft.
+		err = s.state.DB.PutDomainPermissionDraft(
+			ctx,
+			&gtsmodel.DomainPermissionDraft{
+				ID:                 id.NewULID(),
+				PermissionType:     permSub.PermissionType,
+				Domain:             domain,
+				CreatedByAccountID: permSub.CreatedByAccount.ID,
+				CreatedByAccount:   permSub.CreatedByAccount,
+				PrivateComment:     permSub.URI,
+				PublicComment:      wantedPerm.GetPublicComment(),
+				Obfuscate:          wantedPerm.GetObfuscate(),
+				SubscriptionID:     permSub.ID,
+			},
+		)
+
+	case !existing && !*permSub.AsDraft:
+		// No existing perm, create a new one of the
+		// appropriate type, and process side effects.
+		var (
+			insertF func() error
+			action  *gtsmodel.AdminAction
+			actionF admin.ActionF
+		)
+
+		if permSub.PermissionType == gtsmodel.DomainPermissionBlock {
+			// Prepare to insert + process a block.
+			domainBlock := &gtsmodel.DomainBlock{
+				ID:                 id.NewULID(),
+				Domain:             domain,
+				CreatedByAccountID: permSub.CreatedByAccount.ID,
+				CreatedByAccount:   permSub.CreatedByAccount,
+				PrivateComment:     permSub.URI,
+				PublicComment:      wantedPerm.GetPublicComment(),
+				Obfuscate:          wantedPerm.GetObfuscate(),
+				SubscriptionID:     permSub.ID,
+			}
+			insertF = func() error { return s.state.DB.CreateDomainBlock(ctx, domainBlock) }
+
+			action = &gtsmodel.AdminAction{
+				ID:             id.NewULID(),
+				TargetCategory: gtsmodel.AdminActionCategoryDomain,
+				TargetID:       domain,
+				Type:           gtsmodel.AdminActionSuspend,
+				AccountID:      permSub.CreatedByAccountID,
+			}
+			actionF = s.state.AdminActions.DomainBlockF(action.ID, domainBlock)
+
+		} else {
+			// Prepare to insert + process an allow.
+			domainAllow := &gtsmodel.DomainAllow{
+				ID:                 id.NewULID(),
+				Domain:             domain,
+				CreatedByAccountID: permSub.CreatedByAccount.ID,
+				CreatedByAccount:   permSub.CreatedByAccount,
+				PrivateComment:     permSub.URI,
+				PublicComment:      wantedPerm.GetPublicComment(),
+				Obfuscate:          wantedPerm.GetObfuscate(),
+				SubscriptionID:     permSub.ID,
+			}
+			insertF = func() error { return s.state.DB.CreateDomainAllow(ctx, domainAllow) }
+
+			action = &gtsmodel.AdminAction{
+				ID:             id.NewULID(),
+				TargetCategory: gtsmodel.AdminActionCategoryDomain,
+				TargetID:       domain,
+				Type:           gtsmodel.AdminActionUnsuspend,
+				AccountID:      permSub.CreatedByAccountID,
+			}
+			actionF = s.state.AdminActions.DomainAllowF(action.ID, domainAllow)
+		}
+
+		// Insert the new perm in the db.
+		if err = insertF(); err != nil {
+			// Couldn't insert wanted perm,
+			// don't process side effects.
+			break
+		}
+
+		// Run admin action to process
+		// side effects of permission.
+		err = s.state.AdminActions.Run(ctx, action, actionF)
+
+	case existingPerm.GetSubscriptionID() != "" || *permSub.AdoptOrphans:
+		// Perm exists but we should adopt/take
+		// it by copying over desired fields.
+		existingPerm.SetCreatedByAccountID(wantedPerm.GetCreatedByAccountID())
+		existingPerm.SetCreatedByAccount(wantedPerm.GetCreatedByAccount())
+		existingPerm.SetSubscriptionID(permSub.ID)
+		existingPerm.SetObfuscate(wantedPerm.GetObfuscate())
+		existingPerm.SetPrivateComment(wantedPerm.GetPrivateComment())
+		existingPerm.SetPublicComment(wantedPerm.GetPublicComment())
+
+		switch p := existingPerm.(type) {
+		case *gtsmodel.DomainBlock:
+			err = s.state.DB.UpdateDomainBlock(ctx, p)
+		case *gtsmodel.DomainAllow:
+			err = s.state.DB.UpdateDomainAllow(ctx, p)
+		}
+
+	default:
+		// Perm exists but we should leave it alone.
+		l.Debug("domain is covered by a higher-priority subscription, skipping")
+	}
+
+	if err != nil && !errors.Is(err, db.ErrAlreadyExists) {
+		// Proper db error.
+		return created, err
+	}
+
+	created = true
+	return created, nil
+}
+
+func permsFromCSV(
+	l log.Entry,
+	permType gtsmodel.DomainPermissionType,
+	body io.ReadCloser,
+) ([]gtsmodel.DomainPermission, error) {
+	csvReader := csv.NewReader(body)
+
+	// Read and validate column headers.
+	columnHeaders, err := csvReader.Read()
+	if err != nil {
+		body.Close()
+		return nil, gtserror.NewfAt(3, "error decoding csv column headers: %w", err)
+	}
+
+	if !slices.Equal(
+		columnHeaders,
+		[]string{
+			"#domain",
+			"#severity",
+			"#reject_media",
+			"#reject_reports",
+			"#public_comment",
+			"#obfuscate",
+		},
+	) {
+		body.Close()
+		err := gtserror.NewfAt(3, "unexpected column headers in csv: %+v", columnHeaders)
+		return nil, err
+	}
+
+	// Read remaining CSV records.
+	records, err := csvReader.ReadAll()
+
+	// Totally done
+	// with body now.
+	body.Close()
+
+	// Check for decode error.
+	if err != nil {
+		err := gtserror.NewfAt(3, "error decoding body into csv: %w", err)
+		return nil, err
+	}
+
+	// Make sure we actually
+	// have some records.
+	if len(records) == 0 {
+		return nil, nil
+	}
+
+	// Convert records to permissions slice.
+	perms := make([]gtsmodel.DomainPermission, 0, len(records))
+	for _, record := range records {
+		if len(record) != 6 {
+			l.Warnf("skipping invalid-length record: %+v", record)
+			continue
+		}
+
+		var (
+			domainRaw     = record[0]
+			severity      = record[1]
+			publicComment = record[4]
+			obfuscateStr  = record[5]
+		)
+
+		if severity != "suspend" {
+			l.Warnf("skipping non-suspend record: %+v", record)
+			continue
+		}
+
+		obfuscate, err := strconv.ParseBool(obfuscateStr)
+		if err != nil {
+			l.Warnf("couldn't parse obfuscate field of record: %+v", record)
+			continue
+		}
+
+		// Normalize + validate domain.
+		domain, err := validateDomain(domainRaw)
+		if err != nil {
+			l.Warnf("skipping invalid domain %s: %+v", domainRaw, err)
+			continue
+		}
+
+		// Instantiate the permission
+		// as either block or allow.
+		var perm gtsmodel.DomainPermission
+		switch permType {
+		case gtsmodel.DomainPermissionBlock:
+			perm = &gtsmodel.DomainBlock{Domain: domain}
+		case gtsmodel.DomainPermissionAllow:
+			perm = &gtsmodel.DomainAllow{Domain: domain}
+		}
+
+		// Set remaining fields.
+		perm.SetPublicComment(publicComment)
+		perm.SetObfuscate(&obfuscate)
+
+		// We're done.
+		perms = append(perms, perm)
+	}
+
+	return perms, nil
+}
+
+func permsFromJSON(
+	l log.Entry,
+	permType gtsmodel.DomainPermissionType,
+	body io.ReadCloser,
+) ([]gtsmodel.DomainPermission, error) {
+	var (
+		dec      = json.NewDecoder(body)
+		apiPerms = make([]*apimodel.DomainPermission, 0)
+	)
+
+	// Read body into memory as
+	// slice of domain permissions.
+	if err := dec.Decode(&apiPerms); err != nil {
+		_ = body.Close() // ensure closed.
+		return nil, gtserror.NewfAt(3, "error decoding into json: %w", err)
+	}
+
+	// Perform a secondary decode just to ensure we drained the
+	// entirety of the data source. Error indicates either extra
+	// trailing garbage, or multiple JSON values (invalid data).
+	if err := dec.Decode(&struct{}{}); err != io.EOF {
+		_ = body.Close() // ensure closed.
+		return nil, gtserror.NewfAt(3, "data remaining after json")
+	}
+
+	// Done with body.
+	_ = body.Close()
+
+	// Convert apimodel perms to barebones internal perms.
+	perms := make([]gtsmodel.DomainPermission, 0, len(apiPerms))
+	for _, apiPerm := range apiPerms {
+
+		// Normalize + validate domain.
+		domainRaw := apiPerm.Domain.Domain
+		domain, err := validateDomain(domainRaw)
+		if err != nil {
+			l.Warnf("skipping invalid domain %s: %+v", domainRaw, err)
+			continue
+		}
+
+		// Instantiate the permission
+		// as either block or allow.
+		var perm gtsmodel.DomainPermission
+		switch permType {
+		case gtsmodel.DomainPermissionBlock:
+			perm = &gtsmodel.DomainBlock{Domain: domain}
+		case gtsmodel.DomainPermissionAllow:
+			perm = &gtsmodel.DomainAllow{Domain: domain}
+		}
+
+		// Set remaining fields.
+		perm.SetPublicComment(apiPerm.PublicComment)
+		perm.SetObfuscate(&apiPerm.Obfuscate)
+
+		// We're done.
+		perms = append(perms, perm)
+	}
+
+	return perms, nil
+}
+
+func permsFromPlain(
+	l log.Entry,
+	permType gtsmodel.DomainPermissionType,
+	body io.ReadCloser,
+) ([]gtsmodel.DomainPermission, error) {
+	// Scan + split by line.
+	sc := bufio.NewScanner(body)
+
+	// Read into domains
+	// line by line.
+	var domains []string
+	for sc.Scan() {
+		domains = append(domains, sc.Text())
+	}
+
+	// Whatever happened, we're
+	// done with the body now.
+	body.Close()
+
+	// Check if error reading body.
+	if err := sc.Err(); err != nil {
+		return nil, gtserror.NewfAt(3, "error decoding into plain: %w", err)
+	}
+
+	// Convert raw domains to permissions.
+	perms := make([]gtsmodel.DomainPermission, 0, len(domains))
+	for _, domainRaw := range domains {
+
+		// Normalize + validate domain.
+		domain, err := validateDomain(domainRaw)
+		if err != nil {
+			l.Warnf("skipping invalid domain %s: %+v", domainRaw, err)
+			continue
+		}
+
+		// Instantiate the permission
+		// as either block or allow.
+		var perm gtsmodel.DomainPermission
+		switch permType {
+		case gtsmodel.DomainPermissionBlock:
+			perm = &gtsmodel.DomainBlock{Domain: domain}
+		case gtsmodel.DomainPermissionAllow:
+			perm = &gtsmodel.DomainAllow{Domain: domain}
+		}
+
+		// We're done.
+		perms = append(perms, perm)
+	}
+
+	return perms, nil
+}
+
+func validateDomain(domain string) (string, error) {
+	// Basic validation.
+	if _, ok := dns.IsDomainName(domain); !ok {
+		err := fmt.Errorf("invalid domain name")
+		return "", err
+	}
+
+	// Convert to punycode.
+	domain, err := util.Punify(domain)
+	if err != nil {
+		err := fmt.Errorf("could not punify domain: %w", err)
+		return "", err
+	}
+
+	// Check for invalid characters
+	// after the punification process.
+	if strings.ContainsAny(domain, "*, \n") {
+		err := fmt.Errorf("invalid char(s) in domain")
+		return "", err
+	}
+
+	return domain, nil
+}
+
+func (s *Subscriptions) existingCovered(
+	ctx context.Context,
+	permType gtsmodel.DomainPermissionType,
+	domain string,
+	higherPrios []*gtsmodel.DomainPermissionSubscription,
+) (
+	existingPerm gtsmodel.DomainPermission,
+	covered bool,
+	err error,
+) {
+	// Check for existing perm
+	// of appropriate type.
+	var dbErr error
+	switch permType {
+	case gtsmodel.DomainPermissionBlock:
+		existingPerm, dbErr = s.state.DB.GetDomainBlock(ctx, domain)
+	case gtsmodel.DomainPermissionAllow:
+		existingPerm, dbErr = s.state.DB.GetDomainAllow(ctx, domain)
+	}
+
+	if dbErr != nil && !errors.Is(dbErr, db.ErrNoEntries) {
+		// Real db error.
+		err = dbErr
+		return
+	}
+
+	if util.IsNil(existingPerm) {
+		// Can't be covered if
+		// no existing perm.
+		return
+	}
+
+	subscriptionID := existingPerm.GetSubscriptionID()
+	if subscriptionID == "" {
+		// Can't be covered if
+		// no subscription ID.
+		return
+	}
+
+	// Covered if subscription ID is in the slice
+	// of higher-priority permission subscriptions.
+	covered = slices.ContainsFunc(
+		higherPrios,
+		func(permSub *gtsmodel.DomainPermissionSubscription) bool {
+			return permSub.ID == subscriptionID
+		},
+	)
+
+	return
+}

internal/subscriptions/subscriptions.go

@@ -0,0 +1,42 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package subscriptions
+
+import (
+	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
+	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+)
+
+type Subscriptions struct {
+	state               *state.State
+	transportController transport.Controller
+	tc                  *typeutils.Converter
+}
+
+func New(
+	state *state.State,
+	transportController transport.Controller,
+	tc *typeutils.Converter,
+) *Subscriptions {
+	return &Subscriptions{
+		state:               state,
+		transportController: transportController,
+		tc:                  tc,
+	}
+}

internal/subscriptions/subscriptions_test.go

@@ -0,0 +1,538 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package subscriptions_test
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+const (
+	rMediaPath    = "../../testrig/media"
+	rTemplatePath = "../../web/template"
+)
+
+type SubscriptionsTestSuite struct {
+	suite.Suite
+
+	testAccounts map[string]*gtsmodel.Account
+}
+
+func (suite *SubscriptionsTestSuite) SetupSuite() {
+	testrig.InitTestConfig()
+	testrig.InitTestLog()
+	suite.testAccounts = testrig.NewTestAccounts()
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocksCSV() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a CSV list of baddies.
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.csv",
+			ContentType:        gtsmodel.DomainPermSubContentTypeCSV,
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// We should now have blocks for
+	// each domain on the subscribed list.
+	for _, domain := range []string{
+		"bumfaces.net",
+		"peepee.poopoo",
+		"nothanks.com",
+	} {
+		var (
+			perm gtsmodel.DomainPermission
+			err  error
+		)
+		if !testrig.WaitFor(func() bool {
+			perm, err = testStructs.State.DB.GetDomainBlock(ctx, domain)
+			return err == nil
+		}) {
+			suite.FailNowf("", "timed out waiting for domain %s", domain)
+		}
+
+		suite.Equal(testSubscription.ID, perm.GetSubscriptionID())
+	}
+
+	// The just-fetched perm sub should
+	// have ETag and count etc set now.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Should have some perms now.
+	count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Equal("bigbums6969", permSub.ETag)
+	suite.EqualValues(3, count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute)
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocksJSON() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a JSON list of baddies.
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.json",
+			ContentType:        gtsmodel.DomainPermSubContentTypeJSON,
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// We should now have blocks for
+	// each domain on the subscribed list.
+	for _, domain := range []string{
+		"bumfaces.net",
+		"peepee.poopoo",
+		"nothanks.com",
+	} {
+		var (
+			perm gtsmodel.DomainPermission
+			err  error
+		)
+		if !testrig.WaitFor(func() bool {
+			perm, err = testStructs.State.DB.GetDomainBlock(ctx, domain)
+			return err == nil
+		}) {
+			suite.FailNowf("", "timed out waiting for domain %s", domain)
+		}
+
+		suite.Equal(testSubscription.ID, perm.GetSubscriptionID())
+	}
+
+	// The just-fetched perm sub should
+	// have ETag and count etc set now.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Should have some perms now.
+	count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Equal("don't modify me daddy", permSub.ETag)
+	suite.EqualValues(3, count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute)
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocksPlain() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a plain list of baddies.
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.txt",
+			ContentType:        gtsmodel.DomainPermSubContentTypePlain,
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// We should now have blocks for
+	// each domain on the subscribed list.
+	for _, domain := range []string{
+		"bumfaces.net",
+		"peepee.poopoo",
+		"nothanks.com",
+	} {
+		var (
+			perm gtsmodel.DomainPermission
+			err  error
+		)
+		if !testrig.WaitFor(func() bool {
+			perm, err = testStructs.State.DB.GetDomainBlock(ctx, domain)
+			return err == nil
+		}) {
+			suite.FailNowf("", "timed out waiting for domain %s", domain)
+		}
+
+		suite.Equal(testSubscription.ID, perm.GetSubscriptionID())
+	}
+
+	// The just-fetched perm sub should
+	// have ETag and count etc set now.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Should have some perms now.
+	count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Equal("this is a legit etag i swear", permSub.ETag)
+	suite.EqualValues(3, count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute)
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocksCSVETag() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a CSV list of baddies.
+		// Include the ETag so it gets sent with the request.
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.csv",
+			ContentType:        gtsmodel.DomainPermSubContentTypeCSV,
+			ETag:               "bigbums6969",
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// We should now NOT have blocks for the domains
+	// on the list, as the remote will have returned
+	// 304, indicating we should do nothing.
+	for _, domain := range []string{
+		"bumfaces.net",
+		"peepee.poopoo",
+		"nothanks.com",
+	} {
+		_, err := testStructs.State.DB.GetDomainBlock(ctx, domain)
+		if !errors.Is(err, db.ErrNoEntries) {
+			suite.FailNowf("", "domain perm %s created when it shouldn't be")
+		}
+	}
+
+	// The just-fetched perm sub should
+	// have ETag and count etc set now.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Should have no perms.
+	count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Equal("bigbums6969", permSub.ETag)
+	suite.Zero(count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute)
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocks404() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a CSV list of baddies.
+		// The endpoint will return a 404 so we can test erroring.
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/does_not_exist.csv",
+			ContentType:        gtsmodel.DomainPermSubContentTypeCSV,
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// The just-fetched perm sub should have an error set on it.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Should have no perms.
+	count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Zero(count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.Zero(permSub.SuccessfullyFetchedAt)
+	suite.Equal(`DereferenceDomainPermissions: GET request to https://lists.example.org/does_not_exist.csv failed: status="" body="{"error":"not found"}"`, permSub.Error)
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocksWrongContentTypeCSV() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a plaintext list of baddies,
+		// but try to parse as CSV content type (shouldn't work).
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.txt",
+			ContentType:        gtsmodel.DomainPermSubContentTypeCSV,
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// The just-fetched perm sub should have an error set on it.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Should have no perms.
+	count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Zero(count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.Zero(permSub.SuccessfullyFetchedAt)
+	suite.Equal(`ProcessDomainPermissionSubscription: unexpected column headers in csv: [bumfaces.net]`, permSub.Error)
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocksWrongContentTypePlain() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a plaintext list of baddies,
+		// but try to parse as CSV content type (shouldn't work).
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.csv",
+			ContentType:        gtsmodel.DomainPermSubContentTypePlain,
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// The just-fetched perm sub should have an error set on it.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Should have no perms.
+	count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Zero(count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.Zero(permSub.SuccessfullyFetchedAt)
+	suite.Equal(`fetch successful but parsed zero usable results`, permSub.Error)
+}
+
+func TestSubscriptionTestSuite(t *testing.T) {
+	suite.Run(t, new(SubscriptionsTestSuite))
+}

internal/transport/derefdomainpermlist.go

@@ -0,0 +1,121 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package transport
+
+import (
+	"context"
+	"io"
+	"net/http"
+
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+type DereferenceDomainPermissionsResp struct {
+	// Set only if response was 200 OK.
+	// It's up to the caller to close
+	// this when they're done with it.
+	Body io.ReadCloser
+
+	// True if response
+	// was 304 Not Modified.
+	Unmodified bool
+
+	// May be set
+	// if 200 or 304.
+	ETag string
+}
+
+func (t *transport) DereferenceDomainPermissions(
+	ctx context.Context,
+	permSub *gtsmodel.DomainPermissionSubscription,
+	skipCache bool,
+) (*DereferenceDomainPermissionsResp, error) {
+	// Prepare new HTTP request to endpoint
+	req, err := http.NewRequestWithContext(ctx, "GET", permSub.URI, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// Set basic auth header if necessary.
+	if permSub.FetchUsername != "" || permSub.FetchPassword != "" {
+		req.SetBasicAuth(permSub.FetchUsername, permSub.FetchPassword)
+	}
+
+	// Set relevant Accept headers.
+	// Allow fallback in case target doesn't
+	// negotiate content type correctly.
+	req.Header.Add("Accept-Charset", "utf-8")
+	req.Header.Add("Accept", permSub.ContentType.String()+","+"*/*")
+
+	// If skipCache is true, we want to skip setting Cache
+	// headers so that we definitely don't get a 304 back.
+	if !skipCache {
+		// If we've successfully fetched this list
+		// before, set If-Modified-Since to last
+		// success to make the request conditional.
+		//
+		// See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Modified-Since
+		if !permSub.SuccessfullyFetchedAt.IsZero() {
+			timeStr := permSub.SuccessfullyFetchedAt.Format(http.TimeFormat)
+			req.Header.Add("If-Modified-Since", timeStr)
+		}
+
+		// If we've got an ETag stored for this list, set
+		// If-None-Match to make the request conditional.
+		// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag#caching_of_unchanged_resources.
+		if len(permSub.ETag) != 0 {
+			req.Header.Add("If-None-Match", permSub.ETag)
+		}
+	}
+
+	// Perform the HTTP request
+	rsp, err := t.GET(req)
+	if err != nil {
+		return nil, err
+	}
+
+	// If we have an unexpected / error response,
+	// wrap + return as error. This will also drain
+	// and close the response body for us.
+	if rsp.StatusCode != http.StatusOK &&
+		rsp.StatusCode != http.StatusNotModified {
+		err := gtserror.NewFromResponse(rsp)
+		return nil, err
+	}
+
+	// Check already if we were given an ETag
+	// we can use, as ETag is often returned
+	// even on 304 Not Modified responses.
+	permsResp := &DereferenceDomainPermissionsResp{
+		ETag: rsp.Header.Get("Etag"),
+	}
+
+	if rsp.StatusCode == http.StatusNotModified {
+		// Nothing has changed on the remote side
+		// since we last fetched, so there's nothing
+		// to do and we don't need to read the body.
+		rsp.Body.Close()
+		permsResp.Unmodified = true
+	} else {
+		// Return the live body to the caller.
+		permsResp.Body = rsp.Body
+	}
+
+	return permsResp, nil
+}

internal/transport/transport.go

@@ -78,6 +78,20 @@ type Transport interface {
 	// DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo.
 	DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error)
 
+	// DereferenceDomainPermissions dereferences the
+	// permissions list present at the given permSub's URI.
+	//
+	// If "force", then If-Modified-Since and If-None-Match
+	// headers will *NOT* be sent with the outgoing request.
+	//
+	// If err == nil and Unmodified == false, then it's up
+	// to the caller to close the returned io.ReadCloser.
+	DereferenceDomainPermissions(
+		ctx context.Context,
+		permSub *gtsmodel.DomainPermissionSubscription,
+		force bool,
+	) (*DereferenceDomainPermissionsResp, error)
+
 	// Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body.
 	Finger(ctx context.Context, targetUsername string, targetDomain string) ([]byte, error)
 }

internal/transport/transport_test.go

@@ -21,6 +21,7 @@
 	"context"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -74,6 +75,7 @@ func (suite *TransportTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/typeutils/converter_test.go

@@ -20,6 +20,7 @@
 import (
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/activity/streams/vocab"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -499,6 +500,7 @@ func (suite *TypeUtilsTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	storage := testrig.NewInMemoryStorage()
 	suite.state.Storage = storage
 

internal/typeutils/internaltofrontend.go

@@ -2119,7 +2119,9 @@ func (c *Converter) DomainPermToAPIDomainPerm(
 	domainPerm.PrivateComment = d.GetPrivateComment()
 	domainPerm.SubscriptionID = d.GetSubscriptionID()
 	domainPerm.CreatedBy = d.GetCreatedByAccountID()
-	domainPerm.CreatedAt = util.FormatISO8601(d.GetCreatedAt())
+	if createdAt := d.GetCreatedAt(); !createdAt.IsZero() {
+		domainPerm.CreatedAt = util.FormatISO8601(createdAt)
+	}
 
 	// If this is a draft, also add the permission type.
 	if _, ok := d.(*gtsmodel.DomainPermissionDraft); ok {

mkdocs.yml

@@ -138,6 +138,7 @@ nav:
       - "admin/signups.md"
       - "admin/federation_modes.md"
       - "admin/domain_blocks.md"
+      - "admin/domain_permission_subscriptions.md"
       - "admin/request_filtering_modes.md"
       - "admin/robots.md"
       - "admin/cli.md"

test/envparsing.sh

@@ -115,6 +115,8 @@ EXPECT=$(cat << "EOF"
         "nl",
         "en-GB"
     ],
+    "instance-subscriptions-process-every": 86400000000000,
+    "instance-subscriptions-process-from": "23:00",
     "landing-page-user": "admin",
     "letsencrypt-cert-dir": "/gotosocial/storage/certs",
     "letsencrypt-email-address": "",

testrig/config.go

@@ -99,6 +99,8 @@ func testDefaults() config.Configuration {
 				TagStr: "en-gb",
 			},
 		},
+		InstanceSubscriptionsProcessFrom:  "23:00",        // 11pm,
+		InstanceSubscriptionsProcessEvery: 24 * time.Hour, // 1/day.
 
 		AccountsRegistrationOpen: true,
 		AccountsReasonRequired:   true,

testrig/processor.go

@@ -26,15 +26,27 @@
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
 
 // NewTestProcessor returns a Processor suitable for testing purposes.
 // The passed in state will have its worker functions set appropriately,
 // but the state will not be initialized.
-func NewTestProcessor(state *state.State, federator *federation.Federator, emailSender email.Sender, mediaManager *media.Manager) *processing.Processor {
+func NewTestProcessor(
+	state *state.State,
+	federator *federation.Federator,
+	emailSender email.Sender,
+	mediaManager *media.Manager,
+) *processing.Processor {
+
 	return processing.NewProcessor(
 		cleaner.New(state),
+		subscriptions.New(
+			state,
+			federator.TransportController(),
+			typeutils.NewConverter(state),
+		),
 		typeutils.NewConverter(state),
 		federator,
 		NewTestOauthServer(state.DB),

testrig/teststructs.go

@@ -18,6 +18,7 @@
 package testrig
 
 import (
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/interaction"
@@ -25,6 +26,8 @@
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/processing/common"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
 
@@ -38,12 +41,13 @@
 // and worker queues, which was causing issues
 // when running all tests at once.
 type TestStructs struct {
-	State         *state.State
+	State               *state.State
-	Common        *common.Processor
+	Common              *common.Processor
-	Processor     *processing.Processor
+	Processor           *processing.Processor
-	HTTPClient    *MockHTTPClient
+	HTTPClient          *MockHTTPClient
-	TypeConverter *typeutils.Converter
+	TypeConverter       *typeutils.Converter
-	EmailSender   email.Sender
+	EmailSender         email.Sender
+	TransportController transport.Controller
 }
 
 func SetupTestStructs(
@@ -56,6 +60,7 @@ func SetupTestStructs(
 
 	db := NewTestDB(&state)
 	state.DB = db
+	state.AdminActions = admin.New(db, &state.Workers)
 
 	storage := NewInMemoryStorage()
 	state.Storage = storage
@@ -89,6 +94,7 @@ func SetupTestStructs(
 
 	processor := processing.NewProcessor(
 		cleaner.New(&state),
+		subscriptions.New(&state, transportController, typeconverter),
 		typeconverter,
 		federator,
 		oauthServer,
@@ -105,12 +111,13 @@ func SetupTestStructs(
 	StandardStorageSetup(storage, rMediaPath)
 
 	return &TestStructs{
-		State:         &state,
+		State:               &state,
-		Common:        &common,
+		Common:              &common,
-		Processor:     processor,
+		Processor:           processor,
-		HTTPClient:    httpClient,
+		HTTPClient:          httpClient,
-		TypeConverter: typeconverter,
+		TypeConverter:       typeconverter,
-		EmailSender:   emailSender,
+		EmailSender:         emailSender,
+		TransportController: transportController,
 	}
 }
 

testrig/transportcontroller.go

@@ -41,6 +41,8 @@
 const (
 	applicationJSON         = "application/json"
 	applicationActivityJSON = "application/activity+json"
+	textCSV                 = "text/csv"
+	textPlain               = "text/plain"
 )
 
 // NewTestTransportController returns a test transport controller with the given http client.
@@ -101,6 +103,7 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
 			responseBytes         = []byte(`{"error":"404 not found"}`)
 			responseContentType   = applicationJSON
 			responseContentLength = len(responseBytes)
+			extraHeaders          = make(map[string]string, 0)
 			reqURLString          = req.URL.String()
 		)
 
@@ -124,11 +127,13 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
 			responseContentType = applicationJSON
 			responseContentLength = len(responseBytes)
 		} else if strings.Contains(reqURLString, ".well-known/webfinger") {
-			responseCode, responseBytes, responseContentType, responseContentLength = WebfingerResponse(req)
+			responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = WebfingerResponse(req)
 		} else if strings.Contains(reqURLString, ".weird-webfinger-location/webfinger") {
-			responseCode, responseBytes, responseContentType, responseContentLength = WebfingerResponse(req)
+			responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = WebfingerResponse(req)
 		} else if strings.Contains(reqURLString, ".well-known/host-meta") {
-			responseCode, responseBytes, responseContentType, responseContentLength = HostMetaResponse(req)
+			responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = HostMetaResponse(req)
+		} else if strings.Contains(reqURLString, "lists.example.org") {
+			responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = DomainPermissionSubscriptionResponse(req)
 		} else if note, ok := mockHTTPClient.TestRemoteStatuses[reqURLString]; ok {
 			// the request is for a note that we have stored
 			noteI, err := streams.Serialize(note)
@@ -239,14 +244,23 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
 		}
 
 		log.Debugf(nil, "returning response %s", string(responseBytes))
+
 		reader := bytes.NewReader(responseBytes)
 		readCloser := io.NopCloser(reader)
+
+		header := http.Header{
+			"Content-Type": {responseContentType},
+		}
+		for k, v := range extraHeaders {
+			header.Add(k, v)
+		}
+
 		return &http.Response{
 			Request:       req,
 			StatusCode:    responseCode,
 			Body:          readCloser,
 			ContentLength: int64(responseContentLength),
-			Header:        http.Header{"Content-Type": {responseContentType}},
+			Header:        header,
 		}, nil
 	}
 
@@ -261,7 +275,13 @@ func (m *MockHTTPClient) DoSigned(req *http.Request, sign httpclient.SignFunc) (
 	return m.do(req)
 }
 
-func HostMetaResponse(req *http.Request) (responseCode int, responseBytes []byte, responseContentType string, responseContentLength int) {
+func HostMetaResponse(req *http.Request) (
+	responseCode int,
+	responseBytes []byte,
+	responseContentType string,
+	responseContentLength int,
+	extraHeaders map[string]string,
+) {
 	var hm *apimodel.HostMeta
 
 	if req.URL.String() == "https://misconfigured-instance.com/.well-known/host-meta" {
@@ -297,7 +317,13 @@ func HostMetaResponse(req *http.Request) (responseCode int, responseBytes []byte
 	return
 }
 
-func WebfingerResponse(req *http.Request) (responseCode int, responseBytes []byte, responseContentType string, responseContentLength int) {
+func WebfingerResponse(req *http.Request) (
+	responseCode int,
+	responseBytes []byte,
+	responseContentType string,
+	responseContentLength int,
+	extraHeaders map[string]string,
+) {
 	var wfr *apimodel.WellKnownResponse
 
 	switch req.URL.String() {
@@ -410,3 +436,89 @@ func WebfingerResponse(req *http.Request) (responseCode int, responseBytes []byt
 	responseContentLength = len(wfrJSON)
 	return
 }
+
+func DomainPermissionSubscriptionResponse(req *http.Request) (
+	responseCode int,
+	responseBytes []byte,
+	responseContentType string,
+	responseContentLength int,
+	extraHeaders map[string]string,
+) {
+
+	const (
+		csvResp = `#domain,#severity,#reject_media,#reject_reports,#public_comment,#obfuscate
+bumfaces.net,suspend,false,false,big jerks,false
+peepee.poopoo,suspend,false,false,harassment,false
+nothanks.com,suspend,false,false,,false`
+		csvRespETag = "bigbums6969"
+
+		textResp = `bumfaces.net
+peepee.poopoo
+nothanks.com`
+		textRespETag = "this is a legit etag i swear"
+
+		jsonResp = `[
+  {
+    "domain": "bumfaces.net",
+    "suspended_at": "2020-05-13T13:29:12.000Z",
+    "public_comment": "big jerks"
+  },
+    {
+    "domain": "peepee.poopoo",
+    "suspended_at": "2020-05-13T13:29:12.000Z",
+    "public_comment": "harassment"
+  },
+    {
+    "domain": "nothanks.com",
+    "suspended_at": "2020-05-13T13:29:12.000Z"
+  }
+]`
+		jsonRespETag = "don't modify me daddy"
+	)
+
+	switch req.URL.String() {
+	case "https://lists.example.org/baddies.csv":
+		extraHeaders = map[string]string{"ETag": csvRespETag}
+		if req.Header.Get("If-None-Match") == csvRespETag {
+			// Cached.
+			responseCode = http.StatusNotModified
+		} else {
+			responseBytes = []byte(csvResp)
+			responseContentType = textCSV
+			responseCode = http.StatusOK
+		}
+		responseContentLength = len(responseBytes)
+
+	case "https://lists.example.org/baddies.txt":
+		extraHeaders = map[string]string{"ETag": textRespETag}
+		if req.Header.Get("If-None-Match") == textRespETag {
+			// Cached.
+			responseCode = http.StatusNotModified
+		} else {
+			responseBytes = []byte(textResp)
+			responseContentType = textPlain
+			responseCode = http.StatusOK
+		}
+		responseContentLength = len(responseBytes)
+
+	case "https://lists.example.org/baddies.json":
+		extraHeaders = map[string]string{"ETag": jsonRespETag}
+		if req.Header.Get("If-None-Match") == jsonRespETag {
+			// Cached.
+			responseCode = http.StatusNotModified
+		} else {
+			responseBytes = []byte(jsonResp)
+			responseContentType = applicationJSON
+			responseCode = http.StatusOK
+		}
+		responseContentLength = len(responseBytes)
+
+	default:
+		responseCode = http.StatusNotFound
+		responseBytes = []byte(`{"error":"not found"}`)
+		responseContentType = applicationJSON
+		responseContentLength = len(responseBytes)
+	}
+
+	return
+}

web/source/settings/components/pageable-list.tsx

@@ -30,7 +30,7 @@ export interface PageableListProps<T> {
 	items?: T[];
 	itemToEntry: (_item: T) => ReactNode;
 	isLoading: boolean;
-	isFetching: boolean;
+	isFetching?: boolean;
 	isError: boolean;
 	error: FetchBaseQueryError | SerializedError | undefined;
 	emptyMessage: ReactNode;

web/source/settings/lib/query/admin/domain-permissions/subscriptions.ts

@@ -20,6 +20,7 @@
 import { gtsApi } from "../../gts-api";
 
 import type {
+	DomainPerm,
 	DomainPermSub,
 	DomainPermSubCreateUpdateParams,
 	DomainPermSubSearchParams,
@@ -120,6 +121,13 @@ const extended = gtsApi.injectEndpoints({
 				asForm: true,
 				body: { remove_children: remove_children },
 			}),
+		}),
+
+		testDomainPermissionSubscription: build.mutation<{ error: string } | DomainPerm[], string>({
+			query: (id) => ({
+				method: "POST",
+				url: `/api/v1/admin/domain_permission_subscriptions/${id}/test`,
+			}),
 		})
 	}),
 });
@@ -154,6 +162,11 @@ const useUpdateDomainPermissionSubscriptionMutation = extended.useUpdateDomainPe
  */
 const useRemoveDomainPermissionSubscriptionMutation = extended.useRemoveDomainPermissionSubscriptionMutation;
 
+/**
+ * Test a domain permission subscription to see if data can be fetched + parsed.
+ */
+const useTestDomainPermissionSubscriptionMutation = extended.useTestDomainPermissionSubscriptionMutation;
+
 export {
 	useLazySearchDomainPermissionSubscriptionsQuery,
 	useGetDomainPermissionSubscriptionQuery,
@@ -161,4 +174,5 @@ export {
 	useGetDomainPermissionSubscriptionsPreviewQuery,
 	useUpdateDomainPermissionSubscriptionMutation,
 	useRemoveDomainPermissionSubscriptionMutation,
+	useTestDomainPermissionSubscriptionMutation,
 };

web/source/settings/style.css

@@ -1426,6 +1426,18 @@ button.tab-button {
 	}
 }
 
+.domain-permission-subscription-details {
+	> .list > .entries > .perm-preview {
+		gap: 0.5rem;
+	}
+
+	> .perm-issue > b > code {
+		background: $info-bg;
+		padding: 0;
+	}
+}
+
+
 .domain-permission-subscription-title {
 	font-size: 1.2rem;
 	font-weight: bold;
@@ -1451,7 +1463,8 @@ button.tab-button {
 	}
 }
 
-.domain-permission-subscription-remove {
+.domain-permission-subscription-remove,
+.domain-permission-subscription-test {
 	gap: 1rem;
 }
 

web/source/settings/views/moderation/domain-permissions/subscriptions/detail.tsx

@@ -17,19 +17,20 @@
 	along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-import React, { useState } from "react";
+import React, { ReactNode, useState } from "react";
 import { useLocation, useParams } from "wouter";
 import { useBaseUrl } from "../../../../lib/navigation/util";
 import BackButton from "../../../../components/back-button";
-import { useGetDomainPermissionSubscriptionQuery, useRemoveDomainPermissionSubscriptionMutation, useUpdateDomainPermissionSubscriptionMutation } from "../../../../lib/query/admin/domain-permissions/subscriptions";
+import { useGetDomainPermissionSubscriptionQuery, useRemoveDomainPermissionSubscriptionMutation, useTestDomainPermissionSubscriptionMutation, useUpdateDomainPermissionSubscriptionMutation } from "../../../../lib/query/admin/domain-permissions/subscriptions";
 import { useBoolInput, useNumberInput, useTextInput } from "../../../../lib/form";
 import FormWithData from "../../../../lib/form/form-with-data";
-import { DomainPermSub } from "../../../../lib/types/domain-permission";
+import { DomainPerm, DomainPermSub } from "../../../../lib/types/domain-permission";
 import MutationButton from "../../../../components/form/mutation-button";
 import { Checkbox, NumberInput, Select, TextInput } from "../../../../components/form/inputs";
 import useFormSubmit from "../../../../lib/form/submit";
 import UsernameLozenge from "../../../../components/username-lozenge";
 import { urlValidator } from "../../../../lib/util/formvalidators";
+import { PageableList } from "../../../../components/pageable-list";
 
 export default function DomainPermissionSubscriptionDetail() {
 	const params = useParams();
@@ -56,6 +57,7 @@ function DomainPermSubForm({ data: permSub }: { data: DomainPermSub }) {
 			<h1><BackButton to={backLocation} /> Domain Permission Subscription Detail</h1>
 			<DomainPermSubDetails permSub={permSub} />
 			<UpdateDomainPermSub permSub={permSub} />
+			<TestDomainPermSub permSub={permSub} />
 			<DeleteDomainPermSub permSub={permSub} backLocation={backLocation} />
 		</div>
 	);
@@ -382,3 +384,73 @@ function DeleteDomainPermSub({ permSub, backLocation }: { permSub: DomainPermSub
 		</form>
 	);
 }
+
+function TestDomainPermSub({ permSub }: { permSub: DomainPermSub }) {
+	const permType = permSub.permission_type;
+	if (!permType) {
+		throw "permission_type was undefined";
+	}
+	
+	const [ testSub, testRes ] = useTestDomainPermissionSubscriptionMutation();
+	const onSubmit = (e) => {
+		e.preventDefault();
+		testSub(permSub.id);
+	};
+
+	// Function to map an item to a list entry.
+	function itemToEntry(perm: DomainPerm): ReactNode {
+		return (
+			<span className="text-cutoff entry perm-preview">
+				<strong>{ perm.domain }</strong>
+				{ perm.public_comment && <>({ perm.public_comment })</> }
+			</span>
+		);
+	}
+
+	return (
+		<>
+			<form
+				className="domain-permission-subscription-test"
+				onSubmit={onSubmit}
+			>
+				<h2>Test Subscription</h2>
+				Click the "test" button to instruct your instance to do a test
+				fetch and parse of the {permType} list at the subscription URI.
+				<br/>
+				If the fetch is successful, you will see a list of {permType}s
+				(or {permType} drafts) that *would* be created by this subscription,
+				along with the public comment for each {permType} (if applicable).
+				<br/>
+				The test does not actually create those {permType}s in your database.
+				<MutationButton
+					disabled={false}
+					label={"Test"}
+					result={testRes}
+				/>
+			</form>
+			{ testRes.data && "error" in testRes.data
+				? <div className="info perm-issue">
+					<i className="fa fa-fw fa-exclamation-circle" aria-hidden="true"></i>
+					<b>
+						The following issue was encountered when doing a fetch + parse:
+						<br/><code>{ testRes.data.error }</code>
+						<br/>This may be due to a temporary outage at the remote URL,
+						or you may wish to check your subscription settings and test again.
+					</b>
+				</div>
+				: <>
+					{ testRes.data && `${testRes.data?.length} ${permType}s would be created by this subscription:`}
+					<PageableList
+						isLoading={testRes.isLoading}
+						isSuccess={testRes.isSuccess}
+						items={testRes.data}
+						itemToEntry={itemToEntry}
+						isError={testRes.isError}
+						error={testRes.error}
+						emptyMessage={<b>No entries!</b>}
+					/>
+				</>
+			}
+		</>
+	);
+}