[feature] Ratelimit + serve emoji images on separate router group (#2548)

* [feature] Serve + rate limit emoji files separately from attachments

* add a wee little warning about uploading loads of emojis
This commit is contained in:
tobi 2024-01-22 16:17:04 +01:00 committed by GitHub
parent d9729e7d28
commit 138cbe4d60
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 115 additions and 30 deletions

View file

@ -104,6 +104,13 @@
return fmt.Errorf("error creating instance instance: %s", err)
}
// Get the instance account
// (we'll need this later).
instanceAccount, err := dbService.GetInstanceAccount(ctx, "")
if err != nil {
return fmt.Errorf("error retrieving instance account: %w", err)
}
// Open the storage backend
storage, err := gtsstorage.AutoConfig()
if err != nil {
@ -311,16 +318,17 @@ func(context.Context, time.Time) {
// rate limiting
rlLimit := config.GetAdvancedRateLimitRequests()
rlExceptions := config.GetAdvancedRateLimitExceptions()
clLimit := middleware.RateLimit(rlLimit, rlExceptions) // client api
s2sLimit := middleware.RateLimit(rlLimit, rlExceptions) // server-to-server (AP)
fsLimit := middleware.RateLimit(rlLimit, rlExceptions) // fileserver / web templates
clLimit := middleware.RateLimit(rlLimit, rlExceptions) // client api
s2sLimit := middleware.RateLimit(rlLimit, rlExceptions) // server-to-server (AP)
fsMainLimit := middleware.RateLimit(rlLimit, rlExceptions) // fileserver / web templates
fsEmojiLimit := middleware.RateLimit(rlLimit*2, rlExceptions) // fileserver (emojis only, use high limit)
// throttling
cpuMultiplier := config.GetAdvancedThrottlingMultiplier()
retryAfter := config.GetAdvancedThrottlingRetryAfter()
clThrottle := middleware.Throttle(cpuMultiplier, retryAfter) // client api
s2sThrottle := middleware.Throttle(cpuMultiplier, retryAfter) // server-to-server (AP)
fsThrottle := middleware.Throttle(cpuMultiplier, retryAfter) // fileserver / web templates
fsThrottle := middleware.Throttle(cpuMultiplier, retryAfter) // fileserver / web templates / emojis
pkThrottle := middleware.Throttle(cpuMultiplier, retryAfter) // throttle public key endpoint separately
gzip := middleware.Gzip() // applied to all except fileserver
@ -330,12 +338,13 @@ func(context.Context, time.Time) {
authModule.Route(router, clLimit, clThrottle, gzip)
clientModule.Route(router, clLimit, clThrottle, gzip)
metricsModule.Route(router, clLimit, clThrottle, gzip)
fileserverModule.Route(router, fsLimit, fsThrottle)
fileserverModule.Route(router, fsMainLimit, fsThrottle)
fileserverModule.RouteEmojis(router, instanceAccount.ID, fsEmojiLimit, fsThrottle)
wellKnownModule.Route(router, gzip, s2sLimit, s2sThrottle)
nodeInfoModule.Route(router, s2sLimit, s2sThrottle, gzip)
activityPubModule.Route(router, s2sLimit, s2sThrottle, gzip)
activityPubModule.RoutePublicKey(router, s2sLimit, pkThrottle, gzip)
webModule.Route(router, fsLimit, fsThrottle, gzip)
webModule.Route(router, fsMainLimit, fsThrottle, gzip)
// Start the GoToSocial server.
server := gotosocial.NewServer(dbService, router, cleaner)

View file

@ -83,6 +83,13 @@
testrig.StandardDBSetup(state.DB, nil)
// Get the instance account
// (we'll need this later).
instanceAccount, err := state.DB.GetInstanceAccount(ctx, "")
if err != nil {
return fmt.Errorf("error retrieving instance account: %w", err)
}
if os.Getenv("GTS_STORAGE_BACKEND") == "s3" {
var err error
state.Storage, err = storage.NewS3Storage()
@ -225,6 +232,7 @@
clientModule.Route(router)
metricsModule.Route(router)
fileserverModule.Route(router)
fileserverModule.RouteEmojis(router, instanceAccount.ID)
wellKnownModule.Route(router)
nodeInfoModule.Route(router)
activityPubModule.Route(router)

View file

@ -21,6 +21,7 @@
"github.com/gin-gonic/gin"
"github.com/superseriousbusiness/gotosocial/internal/api/fileserver"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/middleware"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/router"
@ -30,36 +31,98 @@ type Fileserver struct {
fileserver *fileserver.Module
}
func (f *Fileserver) Route(r *router.Router, m ...gin.HandlerFunc) {
fileserverGroup := r.AttachGroup("fileserver")
// Attach middlewares appropriate for this group.
fileserverGroup.Use(m...)
// If we're using local storage or proxying s3, we can set a
// long max-age + immutable on all file requests to reflect
// that we never host different files at the same URL (since
// ULIDs are generated per piece of media), so we can
// easily prevent clients having to fetch files repeatedly.
// Attach cache middleware appropriate for file serving.
func useFSCacheMiddleware(grp *gin.RouterGroup) {
// If we're using local storage or proxying s3 (ie., serving
// from here) we can set a long max-age + immutable on file
// requests to reflect that we never host different files at
// the same URL (since ULIDs are generated per piece of media),
// so we can prevent clients having to fetch files repeatedly.
//
// If we *are* using non-proxying s3, however, the max age
// must be set dynamically within the request handler,
// based on how long the signed URL has left to live before
// it expires. This ensures that clients won't cache expired
// links. This is done within fileserver/servefile.go, so we
// should not set the middleware here in that case.
// If we *are* using non-proxying s3, however (ie., not serving
// from here) the max age must be set dynamically within the
// request handler, based on how long the signed URL has left
// to live before it expires. This ensures that clients won't
// cache expired links. This is done within fileserver/servefile.go
// so we should not set the middleware here in that case.
//
// See:
//
// - https://developer.mozilla.org/en-US/docs/Web/HTTP/Caching#avoiding_revalidation
// - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#immutable
if config.GetStorageBackend() == "local" || config.GetStorageS3Proxy() {
fileserverGroup.Use(middleware.CacheControl(middleware.CacheControlConfig{
Directives: []string{"private", "max-age=604800", "immutable"},
Vary: []string{"Range"}, // Cache partial ranges separately.
}))
servingFromHere := config.GetStorageBackend() == "local" || config.GetStorageS3Proxy()
if !servingFromHere {
return
}
f.fileserver.Route(fileserverGroup.Handle)
grp.Use(middleware.CacheControl(middleware.CacheControlConfig{
Directives: []string{"private", "max-age=604800", "immutable"},
Vary: []string{"Range"}, // Cache partial ranges separately.
}))
}
// Route the "main" fileserver group
// that handles everything except emojis.
func (f *Fileserver) Route(
r *router.Router,
m ...gin.HandlerFunc,
) {
const fsGroupPath = "fileserver" +
"/:" + fileserver.AccountIDKey +
"/:" + fileserver.MediaTypeKey
fsGroup := r.AttachGroup(fsGroupPath)
// Attach provided +
// cache middlewares.
fsGroup.Use(m...)
useFSCacheMiddleware(fsGroup)
f.fileserver.Route(fsGroup.Handle)
}
// Route the "emojis" fileserver
// group to handle emojis specifically.
//
// instanceAccount ID is required because
// that is the ID under which all emoji
// files are stored, and from which all
// emoji file requests are therefore served.
func (f *Fileserver) RouteEmojis(
r *router.Router,
instanceAcctID string,
m ...gin.HandlerFunc,
) {
var fsEmojiGroupPath = "fileserver" +
"/" + instanceAcctID +
"/" + string(media.TypeEmoji)
fsEmojiGroup := r.AttachGroup(fsEmojiGroupPath)
// Inject the instance account and emoji media
// type params into the gin context manually,
// since we know we're only going to be serving
// emojis (stored under the instance account ID)
// from this group. This allows us to use the
// same handler functions for both the "main"
// fileserver handler and the emojis handler.
fsEmojiGroup.Use(func(c *gin.Context) {
c.Params = append(c.Params, []gin.Param{
{
Key: fileserver.AccountIDKey,
Value: instanceAcctID,
},
{
Key: fileserver.MediaTypeKey,
Value: string(media.TypeEmoji),
},
}...)
})
// Attach provided +
// cache middlewares.
fsEmojiGroup.Use(m...)
useFSCacheMiddleware(fsEmojiGroup)
f.fileserver.Route(fsEmojiGroup.Handle)
}
func NewFileserver(p *processing.Processor) *Fileserver {

View file

@ -33,8 +33,8 @@
MediaSizeKey = "media_size"
// FileNameKey is the actual filename being sought. Will usually be a UUID then something like .jpeg
FileNameKey = "file_name"
// FileServePath is the fileserve path minus the 'fileserver' prefix.
FileServePath = "/:" + AccountIDKey + "/:" + MediaTypeKey + "/:" + MediaSizeKey + "/:" + FileNameKey
// FileServePath is the fileserve path minus the 'fileserver/:account_id/:media_type' prefix.
FileServePath = "/:" + MediaSizeKey + "/:" + FileNameKey
)
type Module struct {

View file

@ -64,6 +64,11 @@ module.exports = function EmojiOverview({ }) {
You can either upload them here directly, or copy from those already
present on other (known) instances through the <Link to={`./remote`}>Remote Emoji</Link> page.
</p>
<p>
<strong>Be warned!</strong> If you upload more than about 300-400 custom emojis in
total on your instance, this may lead to rate-limiting issues for users and clients
if they try to load all the emoji images at once (which is what many clients do).
</p>
{content}
</>
);