mirror of
https://github.com/superseriousbusiness/gotosocial.git
synced 2024-12-23 02:32:12 +00:00
[performance] cache recently allowed/denied domains to cut down on db calls (#794)
* fetch creation and fetching domain blocks from db Signed-off-by: kim <grufwub@gmail.com> * add separate domainblock cache type, handle removing block from cache on delete Signed-off-by: kim <grufwub@gmail.com> * fix sentinel nil values being passed into cache Signed-off-by: kim <grufwub@gmail.com> Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
parent
bf9d146987
commit
d68c04a6c0
106
internal/cache/domain.go
vendored
Normal file
106
internal/cache/domain.go
vendored
Normal file
|
@ -0,0 +1,106 @@
|
|||
/*
|
||||
GoToSocial
|
||||
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package cache
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"codeberg.org/gruf/go-cache/v2"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
|
||||
)
|
||||
|
||||
// DomainCache is a cache wrapper to provide URL and URI lookups for gtsmodel.Status
|
||||
type DomainBlockCache struct {
|
||||
cache cache.LookupCache[string, string, *gtsmodel.DomainBlock]
|
||||
}
|
||||
|
||||
// NewStatusCache returns a new instantiated statusCache object
|
||||
func NewDomainBlockCache() *DomainBlockCache {
|
||||
c := &DomainBlockCache{}
|
||||
c.cache = cache.NewLookup(cache.LookupCfg[string, string, *gtsmodel.DomainBlock]{
|
||||
RegisterLookups: func(lm *cache.LookupMap[string, string]) {
|
||||
lm.RegisterLookup("id")
|
||||
},
|
||||
|
||||
AddLookups: func(lm *cache.LookupMap[string, string], block *gtsmodel.DomainBlock) {
|
||||
// Block can be equal to nil when sentinel
|
||||
if block != nil && block.ID != "" {
|
||||
lm.Set("id", block.ID, block.Domain)
|
||||
}
|
||||
},
|
||||
|
||||
DeleteLookups: func(lm *cache.LookupMap[string, string], block *gtsmodel.DomainBlock) {
|
||||
// Block can be equal to nil when sentinel
|
||||
if block != nil && block.ID != "" {
|
||||
lm.Delete("id", block.ID)
|
||||
}
|
||||
},
|
||||
})
|
||||
c.cache.SetTTL(time.Minute*5, false)
|
||||
c.cache.Start(time.Second * 10)
|
||||
return c
|
||||
}
|
||||
|
||||
// GetByID attempts to fetch a status from the cache by its ID, you will receive a copy for thread-safety
|
||||
func (c *DomainBlockCache) GetByID(id string) (*gtsmodel.DomainBlock, bool) {
|
||||
return c.cache.GetBy("id", id)
|
||||
}
|
||||
|
||||
// GetByURL attempts to fetch a status from the cache by its URL, you will receive a copy for thread-safety
|
||||
func (c *DomainBlockCache) GetByDomain(domain string) (*gtsmodel.DomainBlock, bool) {
|
||||
return c.cache.Get(domain)
|
||||
}
|
||||
|
||||
// Put places a status in the cache, ensuring that the object place is a copy for thread-safety
|
||||
func (c *DomainBlockCache) Put(domain string, block *gtsmodel.DomainBlock) {
|
||||
if domain == "" {
|
||||
panic("invalid domain")
|
||||
}
|
||||
|
||||
if block == nil {
|
||||
// This is a sentinel value for (no block)
|
||||
c.cache.Set(domain, nil)
|
||||
} else {
|
||||
// This is a valid domain block
|
||||
c.cache.Set(domain, copyDomainBlock(block))
|
||||
}
|
||||
}
|
||||
|
||||
// InvalidateByDomain will invalidate a domain block from the cache by domain name.
|
||||
func (c *DomainBlockCache) InvalidateByDomain(domain string) {
|
||||
c.cache.Invalidate(domain)
|
||||
}
|
||||
|
||||
// copyStatus performs a surface-level copy of status, only keeping attached IDs intact, not the objects.
|
||||
// due to all the data being copied being 99% primitive types or strings (which are immutable and passed by ptr)
|
||||
// this should be a relatively cheap process
|
||||
func copyDomainBlock(block *gtsmodel.DomainBlock) *gtsmodel.DomainBlock {
|
||||
return >smodel.DomainBlock{
|
||||
ID: block.ID,
|
||||
CreatedAt: block.CreatedAt,
|
||||
UpdatedAt: block.UpdatedAt,
|
||||
Domain: block.Domain,
|
||||
CreatedByAccountID: block.CreatedByAccountID,
|
||||
CreatedByAccount: nil,
|
||||
PrivateComment: block.PrivateComment,
|
||||
PublicComment: block.PublicComment,
|
||||
Obfuscate: block.Obfuscate,
|
||||
SubscriptionID: block.SubscriptionID,
|
||||
}
|
||||
}
|
|
@ -173,6 +173,9 @@ func NewBunDBService(ctx context.Context) (db.DB, error) {
|
|||
notifCache.SetTTL(time.Minute*5, false)
|
||||
notifCache.Start(time.Second * 10)
|
||||
|
||||
// Prepare domain block cache
|
||||
blockCache := cache.NewDomainBlockCache()
|
||||
|
||||
ps := &bunDBService{
|
||||
Account: accounts,
|
||||
Admin: &adminDB{
|
||||
|
@ -182,7 +185,8 @@ func NewBunDBService(ctx context.Context) (db.DB, error) {
|
|||
conn: conn,
|
||||
},
|
||||
Domain: &domainDB{
|
||||
conn: conn,
|
||||
conn: conn,
|
||||
cache: blockCache,
|
||||
},
|
||||
Emoji: &emojiDB{
|
||||
conn: conn,
|
||||
|
|
|
@ -20,59 +20,134 @@
|
|||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/superseriousbusiness/gotosocial/internal/cache"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/config"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/db"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/util"
|
||||
)
|
||||
|
||||
type domainDB struct {
|
||||
conn *DBConn
|
||||
conn *DBConn
|
||||
cache *cache.DomainBlockCache
|
||||
}
|
||||
|
||||
func (d *domainDB) IsDomainBlocked(ctx context.Context, domain string) (bool, db.Error) {
|
||||
if domain == "" || domain == config.GetHost() {
|
||||
return false, nil
|
||||
func (d *domainDB) CreateDomainBlock(ctx context.Context, block gtsmodel.DomainBlock) db.Error {
|
||||
// Normalize to lowercase
|
||||
block.Domain = strings.ToLower(block.Domain)
|
||||
|
||||
// Attempt to insert new domain block
|
||||
_, err := d.conn.NewInsert().
|
||||
Model(&block).
|
||||
Exec(ctx, &block)
|
||||
if err != nil {
|
||||
return d.conn.ProcessError(err)
|
||||
}
|
||||
|
||||
// Cache this domain block
|
||||
d.cache.Put(block.Domain, &block)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *domainDB) GetDomainBlock(ctx context.Context, domain string) (*gtsmodel.DomainBlock, db.Error) {
|
||||
// Normalize to lowercase
|
||||
domain = strings.ToLower(domain)
|
||||
|
||||
// Check for easy case, domain referencing *us*
|
||||
if domain == "" || domain == config.GetAccountDomain() {
|
||||
return nil, db.ErrNoEntries
|
||||
}
|
||||
|
||||
// Check for already cached rblock
|
||||
if block, ok := d.cache.GetByDomain(domain); ok {
|
||||
// A 'nil' return value is a sentinel value for no block
|
||||
if block == nil {
|
||||
return nil, db.ErrNoEntries
|
||||
}
|
||||
|
||||
// Else, this block exists
|
||||
return block, nil
|
||||
}
|
||||
|
||||
block := >smodel.DomainBlock{}
|
||||
|
||||
q := d.conn.
|
||||
NewSelect().
|
||||
Model(>smodel.DomainBlock{}).
|
||||
ExcludeColumn("id", "created_at", "updated_at", "created_by_account_id", "private_comment", "public_comment", "obfuscate", "subscription_id").
|
||||
Model(block).
|
||||
Where("domain = ?", domain).
|
||||
Limit(1)
|
||||
|
||||
return d.conn.Exists(ctx, q)
|
||||
// Query database for domain block
|
||||
switch err := q.Scan(ctx); err {
|
||||
// No error, block found
|
||||
case nil:
|
||||
d.cache.Put(domain, block)
|
||||
return block, nil
|
||||
|
||||
// No error, simply not found
|
||||
case sql.ErrNoRows:
|
||||
d.cache.Put(domain, nil)
|
||||
return nil, db.ErrNoEntries
|
||||
|
||||
// Any other db error
|
||||
default:
|
||||
return nil, d.conn.ProcessError(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *domainDB) DeleteDomainBlock(ctx context.Context, domain string) db.Error {
|
||||
// Normalize to lowercase
|
||||
domain = strings.ToLower(domain)
|
||||
|
||||
// Attempt to delete domain block
|
||||
_, err := d.conn.NewDelete().
|
||||
Model((*gtsmodel.DomainBlock)(nil)).
|
||||
Where("domain = ?", domain).
|
||||
Exec(ctx, nil)
|
||||
if err != nil {
|
||||
return d.conn.ProcessError(err)
|
||||
}
|
||||
|
||||
// Clear domain from cache
|
||||
d.cache.InvalidateByDomain(domain)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *domainDB) IsDomainBlocked(ctx context.Context, domain string) (bool, db.Error) {
|
||||
block, err := d.GetDomainBlock(ctx, domain)
|
||||
if err == nil || err == db.ErrNoEntries {
|
||||
return (block != nil), nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
func (d *domainDB) AreDomainsBlocked(ctx context.Context, domains []string) (bool, db.Error) {
|
||||
// filter out any doubles
|
||||
uniqueDomains := util.UniqueStrings(domains)
|
||||
|
||||
for _, domain := range uniqueDomains {
|
||||
if blocked, err := d.IsDomainBlocked(ctx, strings.ToLower(domain)); err != nil {
|
||||
for _, domain := range domains {
|
||||
if blocked, err := d.IsDomainBlocked(ctx, domain); err != nil {
|
||||
return false, err
|
||||
} else if blocked {
|
||||
return blocked, nil
|
||||
}
|
||||
}
|
||||
|
||||
// no blocks found
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (d *domainDB) IsURIBlocked(ctx context.Context, uri *url.URL) (bool, db.Error) {
|
||||
domain := uri.Hostname()
|
||||
return d.IsDomainBlocked(ctx, domain)
|
||||
return d.IsDomainBlocked(ctx, uri.Hostname())
|
||||
}
|
||||
|
||||
func (d *domainDB) AreURIsBlocked(ctx context.Context, uris []*url.URL) (bool, db.Error) {
|
||||
domains := []string{}
|
||||
for _, uri := range uris {
|
||||
domains = append(domains, uri.Hostname())
|
||||
if blocked, err := d.IsDomainBlocked(ctx, uri.Hostname()); err != nil {
|
||||
return false, err
|
||||
} else if blocked {
|
||||
return blocked, nil
|
||||
}
|
||||
}
|
||||
return d.AreDomainsBlocked(ctx, domains)
|
||||
return false, nil
|
||||
}
|
||||
|
|
|
@ -21,6 +21,7 @@
|
|||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/suite"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
|
||||
|
@ -33,10 +34,15 @@ type DomainTestSuite struct {
|
|||
func (suite *DomainTestSuite) TestIsDomainBlocked() {
|
||||
ctx := context.Background()
|
||||
|
||||
now := time.Now()
|
||||
|
||||
domainBlock := >smodel.DomainBlock{
|
||||
ID: "01G204214Y9TNJEBX39C7G88SW",
|
||||
Domain: "some.bad.apples",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
CreatedByAccountID: suite.testAccounts["admin_account"].ID,
|
||||
CreatedByAccount: suite.testAccounts["admin_account"],
|
||||
}
|
||||
|
||||
// no domain block exists for the given domain yet
|
||||
|
@ -44,7 +50,8 @@ func (suite *DomainTestSuite) TestIsDomainBlocked() {
|
|||
suite.NoError(err)
|
||||
suite.False(blocked)
|
||||
|
||||
suite.db.Put(ctx, domainBlock)
|
||||
err = suite.db.CreateDomainBlock(ctx, *domainBlock)
|
||||
suite.NoError(err)
|
||||
|
||||
// domain block now exists
|
||||
blocked, err = suite.db.IsDomainBlocked(ctx, domainBlock.Domain)
|
||||
|
|
|
@ -21,10 +21,21 @@
|
|||
import (
|
||||
"context"
|
||||
"net/url"
|
||||
|
||||
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
|
||||
)
|
||||
|
||||
// Domain contains DB functions related to domains and domain blocks.
|
||||
type Domain interface {
|
||||
// CreateDomainBlock ...
|
||||
CreateDomainBlock(ctx context.Context, block gtsmodel.DomainBlock) Error
|
||||
|
||||
// GetDomainBlock ...
|
||||
GetDomainBlock(ctx context.Context, domain string) (*gtsmodel.DomainBlock, Error)
|
||||
|
||||
// DeleteDomainBlock ...
|
||||
DeleteDomainBlock(ctx context.Context, domain string) Error
|
||||
|
||||
// IsDomainBlocked checks if an instance-level domain block exists for the given domain string (eg., `example.org`).
|
||||
IsDomainBlocked(ctx context.Context, domain string) (bool, Error)
|
||||
|
||||
|
|
|
@ -20,6 +20,7 @@
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
@ -41,22 +42,21 @@ func (p *processor) DomainBlockCreate(ctx context.Context, account *gtsmodel.Acc
|
|||
domain = strings.ToLower(domain)
|
||||
|
||||
// first check if we already have a block -- if err == nil we already had a block so we can skip a whole lot of work
|
||||
domainBlock := >smodel.DomainBlock{}
|
||||
err := p.db.GetWhere(ctx, []db.Where{{Key: "domain", Value: domain}}, domainBlock)
|
||||
block, err := p.db.GetDomainBlock(ctx, domain)
|
||||
if err != nil {
|
||||
if err != db.ErrNoEntries {
|
||||
if !errors.Is(err, db.ErrNoEntries) {
|
||||
// something went wrong in the DB
|
||||
return nil, gtserror.NewErrorInternalError(fmt.Errorf("DomainBlockCreate: db error checking for existence of domain block %s: %s", domain, err))
|
||||
return nil, gtserror.NewErrorInternalError(fmt.Errorf("db error checking for existence of domain block %s: %s", domain, err))
|
||||
}
|
||||
|
||||
// there's no block for this domain yet so create one
|
||||
// note: we take a new ulid from timestamp here in case we need to sort blocks
|
||||
blockID, err := id.NewULID()
|
||||
if err != nil {
|
||||
return nil, gtserror.NewErrorInternalError(fmt.Errorf("DomainBlockCreate: error creating id for new domain block %s: %s", domain, err))
|
||||
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error creating id for new domain block %s: %s", domain, err))
|
||||
}
|
||||
|
||||
domainBlock = >smodel.DomainBlock{
|
||||
newBlock := gtsmodel.DomainBlock{
|
||||
ID: blockID,
|
||||
Domain: domain,
|
||||
CreatedByAccountID: account.ID,
|
||||
|
@ -66,20 +66,22 @@ func (p *processor) DomainBlockCreate(ctx context.Context, account *gtsmodel.Acc
|
|||
SubscriptionID: subscriptionID,
|
||||
}
|
||||
|
||||
// put the new block in the database
|
||||
if err := p.db.Put(ctx, domainBlock); err != nil {
|
||||
if err != db.ErrNoEntries {
|
||||
// there's a real error creating the block
|
||||
return nil, gtserror.NewErrorInternalError(fmt.Errorf("DomainBlockCreate: db error putting new domain block %s: %s", domain, err))
|
||||
}
|
||||
// Insert the new block into the database
|
||||
if err := p.db.CreateDomainBlock(ctx, newBlock); err != nil {
|
||||
return nil, gtserror.NewErrorInternalError(fmt.Errorf("db error putting new domain block %s: %s", domain, err))
|
||||
}
|
||||
// process the side effects of the domain block asynchronously since it might take a while
|
||||
go p.initiateDomainBlockSideEffects(context.Background(), account, domainBlock) // TODO: add this to a queuing system so it can retry/resume
|
||||
|
||||
// Set the newly created block
|
||||
block = &newBlock
|
||||
|
||||
// Process the side effects of the domain block asynchronously since it might take a while
|
||||
go p.initiateDomainBlockSideEffects(ctx, account, block)
|
||||
}
|
||||
|
||||
apiDomainBlock, err := p.tc.DomainBlockToAPIDomainBlock(ctx, domainBlock, false)
|
||||
// Convert our gts model domain block into an API model
|
||||
apiDomainBlock, err := p.tc.DomainBlockToAPIDomainBlock(ctx, block, false)
|
||||
if err != nil {
|
||||
return nil, gtserror.NewErrorInternalError(fmt.Errorf("DomainBlockCreate: error converting domain block to frontend/api representation %s: %s", domain, err))
|
||||
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error converting domain block to frontend/api representation %s: %s", domain, err))
|
||||
}
|
||||
|
||||
return apiDomainBlock, nil
|
||||
|
@ -92,7 +94,6 @@ func (p *processor) DomainBlockCreate(ctx context.Context, account *gtsmodel.Acc
|
|||
// 3. Select all accounts from this instance and pass them through the delete functionality of the processor.
|
||||
func (p *processor) initiateDomainBlockSideEffects(ctx context.Context, account *gtsmodel.Account, block *gtsmodel.DomainBlock) {
|
||||
l := log.WithFields(kv.Fields{
|
||||
|
||||
{"domain", block.Domain},
|
||||
}...)
|
||||
|
||||
|
|
|
@ -47,8 +47,8 @@ func (p *processor) DomainBlockDelete(ctx context.Context, account *gtsmodel.Acc
|
|||
return nil, gtserror.NewErrorInternalError(err)
|
||||
}
|
||||
|
||||
// delete the domain block
|
||||
if err := p.db.DeleteByID(ctx, id, domainBlock); err != nil {
|
||||
// Delete the domain block
|
||||
if err := p.db.DeleteDomainBlock(ctx, domainBlock.Domain); err != nil {
|
||||
return nil, gtserror.NewErrorInternalError(err)
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue