[chore/bugfix] Serve + throttle publickey separately from rest of ActivityPub API (#1461)

* serve publickey separately from AP, don't throttle it

* update nginx cache documentation, cache main-key too

* throttle public key, but separately from other endpoints

cmd/gotosocial/action/server/server.go

@@ -206,6 +206,7 @@
 	clThrottle := middleware.Throttle(cpuMultiplier)  // client api
 	s2sThrottle := middleware.Throttle(cpuMultiplier) // server-to-server (AP)
 	fsThrottle := middleware.Throttle(cpuMultiplier)  // fileserver / web templates
+	pkThrottle := middleware.Throttle(cpuMultiplier)  // throttle public key endpoint separately
 
 	gzip := middleware.Gzip() // applied to all except fileserver
 
@@ -217,6 +218,7 @@
 	wellKnownModule.Route(router, gzip, s2sLimit, s2sThrottle)
 	nodeInfoModule.Route(router, s2sLimit, s2sThrottle, gzip)
 	activityPubModule.Route(router, s2sLimit, s2sThrottle, gzip)
+	activityPubModule.RoutePublicKey(router, s2sLimit, pkThrottle, gzip)
 	webModule.Route(router, fsLimit, fsThrottle, gzip)
 
 	gts, err := gotosocial.NewServer(dbService, router, federator, mediaManager)

cmd/gotosocial/action/testrig/testrig.go

@@ -139,6 +139,7 @@
 	wellKnownModule.Route(router)
 	nodeInfoModule.Route(router)
 	activityPubModule.Route(router)
+	activityPubModule.RoutePublicKey(router)
 	webModule.Route(router)
 
 	gts, err := gotosocial.NewServer(dbService, router, federator, mediaManager)

docs/installation_guide/advanced.md

@@ -279,36 +279,44 @@ This section contains a number of additional things for configuring nginx.
 
 If you want to harden up your NGINX deployment with advanced configuration options, there are many guides online for doing so ([for example](https://beaglesecurity.com/blog/article/nginx-server-security.html)). Try to find one that's up to date. Mozilla also publishes best-practice ssl configuration [here](https://ssl-config.mozilla.org/).
 
-### Caching Webfinger
+### Caching Webfinger and Public Key responses
 
-It's possible to use nginx to cache the webfinger responses. This may be useful in order to ensure clients still get a response on the webfinger endpoint even if GTS is (temporarily) down.
+It's possible to use nginx to cache webfinger and public key responses. This may be useful in order to ensure clients still get a response on these endpoints even if your GoToSocial instance is (temporarily) down, or requests are being throttled.
 
 You'll need to configure two things:
-* A cache path
-* An additional `location` block for webfinger
 
-First, the cache path which needs to happen in the `http` section, usually inside your `nginx.conf`:
+- A cache path.
+- Additional `location` blocks for webfinger and public key requests.
+
+First, the cache path which needs to happen in the `http` section, usually inside your `nginx.conf` at `/etc/nginx/nginx.conf`:
 
 ```nginx.conf
 http {
   ... there will be other things here ...
-  proxy_cache_path /var/cache/nginx keys_zone=ap_webfinger:10m inactive=1w;
+  proxy_cache_path /var/cache/nginx keys_zone=gotosocial_ap_public_responses:10m inactive=1w;
 }
 ```
 
-This configures a cache of 10MB whose entries will be kept up to one week if they're not accessed. The zone is named `ap_webfinger` but you can name it whatever you want. 10MB is a lot of cache keys, you can probably use a much smaller value on small instances.
+This configures a cache of 10MB whose entries will be kept up to one week if they're not accessed.
 
-Second, actually use the cache for webfinger:
+The zone is named `gotosocial_ap_public_responses` but you can name it whatever you want. 10MB is a lot of cache keys; you can probably use a smaller value on small instances.
+
+Second, we need to update our GoToSocial nginx configuration to actually use the cache for the endpoints we want to cache.
+
+From the below configuration example, copy the entries between `### NEW STUFF STARTS HERE ###` and `### NEW STUFF ENDS HERE ###` and paste them into your GoToSocial nginx configuration.
 
 ```nginx.conf
 server {
   server_name example.org;
+  
+  ### NEW STUFF STARTS HERE ###
+  
   location /.well-known/webfinger {
     proxy_set_header Host $host;
     proxy_set_header X-Forwarded-For $remote_addr;
     proxy_set_header X-Forwarded-Proto $scheme;
 
-    proxy_cache ap_webfinger;
+    proxy_cache gotosocial_ap_public_responses;
     proxy_cache_background_update on;
     proxy_cache_key $scheme://$host$uri$is_args$query_string;
     proxy_cache_valid 200 10m;
@@ -319,6 +327,25 @@ server {
     proxy_pass http://localhost:8080;
   }
 
+  location ~ ^\/users\/(?:[a-z0-9_\.]+)\/main-key$ {
+    proxy_set_header Host $host;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    proxy_set_header X-Forwarded-Proto $scheme;
+
+    proxy_cache gotosocial_ap_public_responses;
+    proxy_cache_background_update on;
+    proxy_cache_key $scheme://$host$uri;
+    proxy_cache_valid 200 604800s;
+    proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504 http_429;
+    proxy_cache_lock on;
+    add_header X-Cache-Status $upstream_cache_status;
+
+    proxy_pass http://localhost:8080;
+  }
+
+  ### NEW STUFF ENDS HERE ###
+
+  ### EXISTING STUFF IS BELOW HERE, NOTHING TO CHANGE ###
   location / {
     proxy_pass http://localhost:8080/;
     proxy_set_header Host $host;
@@ -327,28 +354,21 @@ server {
     proxy_set_header X-Forwarded-For $remote_addr;
     proxy_set_header X-Forwarded-Proto $scheme;
   }
-  client_max_body_size 40M;
-
-  listen [::]:443 ssl ipv6only=on; # managed by Certbot
-  listen 443 ssl; # managed by Certbot
-  ssl_certificate /etc/letsencrypt/live/example.org/fullchain.pem; # managed by Certbot
-  ssl_certificate_key /etc/letsencrypt/live/example.org/privkey.pem; # managed by Certbot
-  include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
-  ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
+  # ....... etc
 }
 ```
 
 The `proxy_pass` and `proxy_set_header` are mostly the same, but the `proxy_cache*` entries warrant some explanation:
 
-* `proxy_cache ap_webfinger` tells it to use the `ap_webfinger` cache zone we previously created. If you named it something else, you should change this value
-* `proxy_cache_background_update on` means nginx will try and refresh a cached resource that's about to expire in the background, to ensure it has a current copy on disk
-* `proxy_cache_key` is configured in such a way that it takes the query string into account for caching. So a request for `.well-known/webfinger?acct=user1@example.org` and `.well-known/webfinger?acct=user2@example.org` are not seen as the same
-* `proxy_cache_valid 200 10m;` means we only cache 200 responses from GTS and for 10 minutes. You can add additional lines of these, like `proxy_cache_valid 404 1m;` to cache 404 responses for 1 minute
-* `proxy_cache_use_stale` tells nginx it's allowed to use a stale cache entry (so older than 10 minutes) in certain cases
-* `proxy_cache_lock on` means that if a resource is not cached and there's multiple concurrent requests for them, the queries will be queued up so that only one request goes through and the rest is then answered from cache
-* `add_header X-Cache-Status $upstream_cache_status` will add an `X-Cache-Status` header to the response so you can check if things are getting cached. You can remove this.
+- `proxy_cache gotosocial_ap_public_responses` tells nginx to use the `gotosocial_ap_public_responses` cache zone we previously created. If you named it something else, you should change this value
+- `proxy_cache_background_update on` means nginx will try and refresh a cached resource that's about to expire in the background, to ensure it has a current copy on disk
+- `proxy_cache_key` is configured in such a way that it takes the query string into account for caching. So a request for `.well-known/webfinger?acct=user1@example.org` and `.well-known/webfinger?acct=user2@example.org` are not seen as the same.
+- `proxy_cache_valid 200 10m;` means we only cache 200 responses from GTS and for 10 minutes. You can add additional lines of these, like `proxy_cache_valid 404 1m;` to cache 404 responses for 1 minute
+- `proxy_cache_use_stale` tells nginx it's allowed to use a stale cache entry (so older than 10 minutes) in certain cases
+- `proxy_cache_lock on` means that if a resource is not cached and there's multiple concurrent requests for them, the queries will be queued up so that only one request goes through and the rest is then answered from cache
+- `add_header X-Cache-Status $upstream_cache_status` will add an `X-Cache-Status` header to the response so you can check if things are getting cached. You can remove this.
 
-Tweaking `proxy_cache_use_stale` is how you can ensure webfinger responses are still answered even if GTS itself is down. The provided configuration will serve a stale response in case there's an error proxying to GTS, if our connection to GTS times out, if GTS returns a 5xx status code or if GTS returns 429 (Too Many Requests). The `updating` value says that we're allowed to serve a stale entry if nginx is currently in the process of refreshing its cache. Because we configured `inactive=1w` in the `proxy_cache_path` directive, nginx may serve a response up to one week old if the conditions in `proxy_cache_use_stale` are met.
+The provided configuration will serve a stale response in case there's an error proxying to GoToSocial, if our connection to GoToSocial times out, if GoToSocial returns a `5xx` status code or if GoToSocial returns 429 (Too Many Requests). The `updating` value says that we're allowed to serve a stale entry if nginx is currently in the process of refreshing its cache. Because we configured `inactive=1w` in the `proxy_cache_path` directive, nginx may serve a response up to one week old if the conditions in `proxy_cache_use_stale` are met.
 
 ### Serving static assets
 

internal/api/activitypub.go

@@ -19,11 +19,9 @@
 package api
 
 import (
-	"context"
-	"net/url"
-
 	"github.com/gin-gonic/gin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/activitypub/emoji"
+	"github.com/superseriousbusiness/gotosocial/internal/api/activitypub/publickey"
 	"github.com/superseriousbusiness/gotosocial/internal/api/activitypub/users"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/middleware"
@@ -32,10 +30,10 @@
 )
 
 type ActivityPub struct {
-	emoji *emoji.Module
-	users *users.Module
-
-	isURIBlocked func(context.Context, *url.URL) (bool, db.Error)
+	emoji                    *emoji.Module
+	users                    *users.Module
+	publicKey                *publickey.Module
+	signatureCheckMiddleware gin.HandlerFunc
 }
 
 func (a *ActivityPub) Route(r router.Router, m ...gin.HandlerFunc) {
@@ -43,25 +41,29 @@ func (a *ActivityPub) Route(r router.Router, m ...gin.HandlerFunc) {
 	emojiGroup := r.AttachGroup("emoji")
 	usersGroup := r.AttachGroup("users")
 
-	// instantiate + attach shared, non-global middlewares to both of these groups
-	var (
-		signatureCheckMiddleware = middleware.SignatureCheck(a.isURIBlocked)
-		cacheControlMiddleware   = middleware.CacheControl("no-store")
-	)
+	// attach shared, non-global middlewares to both of these groups
+	cacheControlMiddleware := middleware.CacheControl("no-store")
 	emojiGroup.Use(m...)
 	usersGroup.Use(m...)
-	emojiGroup.Use(signatureCheckMiddleware, cacheControlMiddleware)
-	usersGroup.Use(signatureCheckMiddleware, cacheControlMiddleware)
+	emojiGroup.Use(a.signatureCheckMiddleware, cacheControlMiddleware)
+	usersGroup.Use(a.signatureCheckMiddleware, cacheControlMiddleware)
 
 	a.emoji.Route(emojiGroup.Handle)
 	a.users.Route(usersGroup.Handle)
 }
 
+// Public key endpoint requires different middleware + cache policies from other AP endpoints.
+func (a *ActivityPub) RoutePublicKey(r router.Router, m ...gin.HandlerFunc) {
+	publicKeyGroup := r.AttachGroup(publickey.PublicKeyPath)
+	publicKeyGroup.Use(a.signatureCheckMiddleware, middleware.CacheControl("public,max-age=604800"))
+	a.publicKey.Route(publicKeyGroup.Handle)
+}
+
 func NewActivityPub(db db.DB, p processing.Processor) *ActivityPub {
 	return &ActivityPub{
-		emoji: emoji.New(p),
-		users: users.New(p),
-
-		isURIBlocked: db.IsURIBlocked,
+		emoji:                    emoji.New(p),
+		users:                    users.New(p),
+		publicKey:                publickey.New(p),
+		signatureCheckMiddleware: middleware.SignatureCheck(db.IsURIBlocked),
 	}
 }

internal/api/activitypub/publickey/publickey.go

@@ -0,0 +1,48 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package publickey
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/superseriousbusiness/gotosocial/internal/processing"
+	"github.com/superseriousbusiness/gotosocial/internal/uris"
+)
+
+const (
+	// UsernameKey is for account usernames.
+	UsernameKey = "username"
+	// PublicKeyPath is a path to a user's public key, for serving bare minimum AP representations.
+	PublicKeyPath = "users/:" + UsernameKey + "/" + uris.PublicKeyPath
+)
+
+type Module struct {
+	processor processing.Processor
+}
+
+func New(processor processing.Processor) *Module {
+	return &Module{
+		processor: processor,
+	}
+}
+
+func (m *Module) Route(attachHandler func(method string, path string, f ...gin.HandlerFunc) gin.IRoutes) {
+	attachHandler(http.MethodGet, "", m.PublicKeyGETHandler)
+}

internal/api/activitypub/publickey/publickeyget.go

@@ -16,7 +16,7 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-package users
+package publickey
 
 import (
 	"encoding/json"

internal/api/activitypub/users/user.go

@@ -42,8 +42,6 @@
 
 	// BasePath is the base path for serving AP 'users' requests, minus the 'users' prefix.
 	BasePath = "/:" + UsernameKey
-	// PublicKeyPath is a path to a user's public key, for serving bare minimum AP representations.
-	PublicKeyPath = BasePath + "/" + uris.PublicKeyPath
 	// InboxPath is for serving POST requests to a user's inbox with the given username key.
 	InboxPath = BasePath + "/" + uris.InboxPath
 	// OutboxPath is for serving GET requests to a user's outbox with the given username key.
@@ -74,7 +72,6 @@ func (m *Module) Route(attachHandler func(method string, path string, f ...gin.H
 	attachHandler(http.MethodGet, FollowersPath, m.FollowersGETHandler)
 	attachHandler(http.MethodGet, FollowingPath, m.FollowingGETHandler)
 	attachHandler(http.MethodGet, StatusPath, m.StatusGETHandler)
-	attachHandler(http.MethodGet, PublicKeyPath, m.PublicKeyGETHandler)
 	attachHandler(http.MethodGet, StatusRepliesPath, m.StatusRepliesGETHandler)
 	attachHandler(http.MethodGet, OutboxPath, m.OutboxGETHandler)
 }