2023-03-12 15:00:57 +00:00
|
|
|
// GoToSocial
|
|
|
|
// Copyright (C) GoToSocial Authors admin@gotosocial.org
|
|
|
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
//
|
|
|
|
// This program is free software: you can redistribute it and/or modify
|
|
|
|
// it under the terms of the GNU Affero General Public License as published by
|
|
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
|
|
// (at your option) any later version.
|
|
|
|
//
|
|
|
|
// This program is distributed in the hope that it will be useful,
|
|
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
// GNU Affero General Public License for more details.
|
|
|
|
//
|
|
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
|
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
2022-12-08 17:35:14 +00:00
|
|
|
|
|
|
|
package cache
|
|
|
|
|
[performance] refactoring + add fave / follow / request / visibility caching (#1607)
* refactor visibility checking, add caching for visibility
* invalidate visibility cache items on account / status deletes
* fix requester ID passed to visibility cache nil ptr
* de-interface caches, fix home / public timeline caching + visibility
* finish adding code comments for visibility filter
* fix angry goconst linter warnings
* actually finish adding filter visibility code comments for timeline functions
* move home timeline status author check to after visibility
* remove now-unused code
* add more code comments
* add TODO code comment, update printed cache start names
* update printed cache names on stop
* start adding separate follow(request) delete db functions, add specific visibility cache tests
* add relationship type caching
* fix getting local account follows / followed-bys, other small codebase improvements
* simplify invalidation using cache hooks, add more GetAccountBy___() functions
* fix boosting to return 404 if not boostable but no error (to not leak status ID)
* remove dead code
* improved placement of cache invalidation
* update license headers
* add example follow, follow-request config entries
* add example visibility cache configuration to config file
* use specific PutFollowRequest() instead of just Put()
* add tests for all GetAccountBy()
* add GetBlockBy() tests
* update block to check primitive fields
* update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests
* fix copy-pasted code
* update envparsing test
* whitespace
* fix bun struct tag
* add license header to gtscontext
* fix old license header
* improved error creation to not use fmt.Errorf() when not needed
* fix various rebase conflicts, fix account test
* remove commented-out code, fix-up mention caching
* fix mention select bun statement
* ensure mention target account populated, pass in context to customrenderer logging
* remove more uncommented code, fix typeutil test
* add statusfave database model caching
* add status fave cache configuration
* add status fave cache example config
* woops, catch missed error. nice catch linter!
* add back testrig panic on nil db
* update example configuration to match defaults, slight tweak to cache configuration defaults
* update envparsing test with new defaults
* fetch followingget to use the follow target account
* use accounnt.IsLocal() instead of empty domain check
* use constants for the cache visibility type check
* use bun.In() for notification type restriction in db query
* include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable())
* use bun query building for nested select statements to ensure working with postgres
* update public timeline future status checks to match visibility filter
* same as previous, for home timeline
* update public timeline tests to dynamically check for appropriate statuses
* migrate accounts to allow unique constraint on public_key
* provide minimal account with publicKey
---------
Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
|
|
|
import (
|
2024-01-19 12:57:29 +00:00
|
|
|
"time"
|
|
|
|
|
2024-07-24 08:41:43 +00:00
|
|
|
"codeberg.org/gruf/go-cache/v3/ttl"
|
2023-12-18 14:18:25 +00:00
|
|
|
"github.com/superseriousbusiness/gotosocial/internal/cache/headerfilter"
|
2024-07-24 08:41:43 +00:00
|
|
|
"github.com/superseriousbusiness/gotosocial/internal/config"
|
2023-05-12 09:15:54 +00:00
|
|
|
"github.com/superseriousbusiness/gotosocial/internal/log"
|
[performance] refactoring + add fave / follow / request / visibility caching (#1607)
* refactor visibility checking, add caching for visibility
* invalidate visibility cache items on account / status deletes
* fix requester ID passed to visibility cache nil ptr
* de-interface caches, fix home / public timeline caching + visibility
* finish adding code comments for visibility filter
* fix angry goconst linter warnings
* actually finish adding filter visibility code comments for timeline functions
* move home timeline status author check to after visibility
* remove now-unused code
* add more code comments
* add TODO code comment, update printed cache start names
* update printed cache names on stop
* start adding separate follow(request) delete db functions, add specific visibility cache tests
* add relationship type caching
* fix getting local account follows / followed-bys, other small codebase improvements
* simplify invalidation using cache hooks, add more GetAccountBy___() functions
* fix boosting to return 404 if not boostable but no error (to not leak status ID)
* remove dead code
* improved placement of cache invalidation
* update license headers
* add example follow, follow-request config entries
* add example visibility cache configuration to config file
* use specific PutFollowRequest() instead of just Put()
* add tests for all GetAccountBy()
* add GetBlockBy() tests
* update block to check primitive fields
* update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests
* fix copy-pasted code
* update envparsing test
* whitespace
* fix bun struct tag
* add license header to gtscontext
* fix old license header
* improved error creation to not use fmt.Errorf() when not needed
* fix various rebase conflicts, fix account test
* remove commented-out code, fix-up mention caching
* fix mention select bun statement
* ensure mention target account populated, pass in context to customrenderer logging
* remove more uncommented code, fix typeutil test
* add statusfave database model caching
* add status fave cache configuration
* add status fave cache example config
* woops, catch missed error. nice catch linter!
* add back testrig panic on nil db
* update example configuration to match defaults, slight tweak to cache configuration defaults
* update envparsing test with new defaults
* fetch followingget to use the follow target account
* use accounnt.IsLocal() instead of empty domain check
* use constants for the cache visibility type check
* use bun.In() for notification type restriction in db query
* include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable())
* use bun query building for nested select statements to ensure working with postgres
* update public timeline future status checks to match visibility filter
* same as previous, for home timeline
* update public timeline tests to dynamically check for appropriate statuses
* migrate accounts to allow unique constraint on public_key
* provide minimal account with publicKey
---------
Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
|
|
|
)
|
|
|
|
|
2022-12-08 17:35:14 +00:00
|
|
|
type Caches struct {
|
2024-04-11 09:45:35 +00:00
|
|
|
|
2024-07-24 08:41:43 +00:00
|
|
|
// DB provides access to the collection of
|
2023-12-18 14:18:25 +00:00
|
|
|
// gtsmodel object caches. (used by the database).
|
2024-07-24 08:41:43 +00:00
|
|
|
DB DBCaches
|
2022-12-08 17:35:14 +00:00
|
|
|
|
2023-12-18 14:18:25 +00:00
|
|
|
// AllowHeaderFilters provides access to
|
|
|
|
// the allow []headerfilter.Filter cache.
|
|
|
|
AllowHeaderFilters headerfilter.Cache
|
2022-12-08 17:35:14 +00:00
|
|
|
|
2023-12-18 14:18:25 +00:00
|
|
|
// BlockHeaderFilters provides access to
|
|
|
|
// the block []headerfilter.Filter cache.
|
|
|
|
BlockHeaderFilters headerfilter.Cache
|
|
|
|
|
|
|
|
// Visibility provides access to the item visibility
|
|
|
|
// cache. (used by the visibility filter).
|
[performance] refactoring + add fave / follow / request / visibility caching (#1607)
* refactor visibility checking, add caching for visibility
* invalidate visibility cache items on account / status deletes
* fix requester ID passed to visibility cache nil ptr
* de-interface caches, fix home / public timeline caching + visibility
* finish adding code comments for visibility filter
* fix angry goconst linter warnings
* actually finish adding filter visibility code comments for timeline functions
* move home timeline status author check to after visibility
* remove now-unused code
* add more code comments
* add TODO code comment, update printed cache start names
* update printed cache names on stop
* start adding separate follow(request) delete db functions, add specific visibility cache tests
* add relationship type caching
* fix getting local account follows / followed-bys, other small codebase improvements
* simplify invalidation using cache hooks, add more GetAccountBy___() functions
* fix boosting to return 404 if not boostable but no error (to not leak status ID)
* remove dead code
* improved placement of cache invalidation
* update license headers
* add example follow, follow-request config entries
* add example visibility cache configuration to config file
* use specific PutFollowRequest() instead of just Put()
* add tests for all GetAccountBy()
* add GetBlockBy() tests
* update block to check primitive fields
* update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests
* fix copy-pasted code
* update envparsing test
* whitespace
* fix bun struct tag
* add license header to gtscontext
* fix old license header
* improved error creation to not use fmt.Errorf() when not needed
* fix various rebase conflicts, fix account test
* remove commented-out code, fix-up mention caching
* fix mention select bun statement
* ensure mention target account populated, pass in context to customrenderer logging
* remove more uncommented code, fix typeutil test
* add statusfave database model caching
* add status fave cache configuration
* add status fave cache example config
* woops, catch missed error. nice catch linter!
* add back testrig panic on nil db
* update example configuration to match defaults, slight tweak to cache configuration defaults
* update envparsing test with new defaults
* fetch followingget to use the follow target account
* use accounnt.IsLocal() instead of empty domain check
* use constants for the cache visibility type check
* use bun.In() for notification type restriction in db query
* include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable())
* use bun query building for nested select statements to ensure working with postgres
* update public timeline future status checks to match visibility filter
* same as previous, for home timeline
* update public timeline tests to dynamically check for appropriate statuses
* migrate accounts to allow unique constraint on public_key
* provide minimal account with publicKey
---------
Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
|
|
|
Visibility VisibilityCache
|
|
|
|
|
2024-07-24 08:41:43 +00:00
|
|
|
// Webfinger provides access to the webfinger URL cache.
|
|
|
|
Webfinger *ttl.Cache[string, string] // TTL=24hr, sweep=5min
|
|
|
|
|
2022-12-08 17:35:14 +00:00
|
|
|
// prevent pass-by-value.
|
|
|
|
_ nocopy
|
|
|
|
}
|
|
|
|
|
|
|
|
// Init will (re)initialize both the GTS and AP cache collections.
|
|
|
|
// NOTE: the cache MUST NOT be in use anywhere, this is not thread-safe.
|
|
|
|
func (c *Caches) Init() {
|
2023-05-12 09:15:54 +00:00
|
|
|
log.Infof(nil, "init: %p", c)
|
|
|
|
|
2024-01-19 12:57:29 +00:00
|
|
|
c.initAccount()
|
2024-07-29 18:26:31 +00:00
|
|
|
c.initAccountIDsFollowingTag()
|
2024-01-19 12:57:29 +00:00
|
|
|
c.initAccountNote()
|
2024-03-22 13:03:46 +00:00
|
|
|
c.initAccountSettings()
|
2024-04-16 11:10:13 +00:00
|
|
|
c.initAccountStats()
|
2024-01-19 12:57:29 +00:00
|
|
|
c.initApplication()
|
|
|
|
c.initBlock()
|
|
|
|
c.initBlockIDs()
|
|
|
|
c.initBoostOfIDs()
|
2024-04-15 13:22:21 +00:00
|
|
|
c.initClient()
|
2024-07-23 19:44:31 +00:00
|
|
|
c.initConversation()
|
|
|
|
c.initConversationLastStatusIDs()
|
2024-01-19 12:57:29 +00:00
|
|
|
c.initDomainAllow()
|
|
|
|
c.initDomainBlock()
|
|
|
|
c.initEmoji()
|
|
|
|
c.initEmojiCategory()
|
2024-03-06 10:15:58 +00:00
|
|
|
c.initFilter()
|
|
|
|
c.initFilterKeyword()
|
|
|
|
c.initFilterStatus()
|
2024-01-19 12:57:29 +00:00
|
|
|
c.initFollow()
|
|
|
|
c.initFollowIDs()
|
|
|
|
c.initFollowRequest()
|
|
|
|
c.initFollowRequestIDs()
|
|
|
|
c.initInReplyToIDs()
|
|
|
|
c.initInstance()
|
2024-07-11 14:44:29 +00:00
|
|
|
c.initInteractionApproval()
|
2024-01-19 12:57:29 +00:00
|
|
|
c.initList()
|
|
|
|
c.initListEntry()
|
|
|
|
c.initMarker()
|
|
|
|
c.initMedia()
|
|
|
|
c.initMention()
|
2024-03-06 10:18:57 +00:00
|
|
|
c.initMove()
|
2024-01-19 12:57:29 +00:00
|
|
|
c.initNotification()
|
|
|
|
c.initPoll()
|
|
|
|
c.initPollVote()
|
|
|
|
c.initPollVoteIDs()
|
|
|
|
c.initReport()
|
|
|
|
c.initStatus()
|
2024-06-06 10:44:43 +00:00
|
|
|
c.initStatusBookmark()
|
|
|
|
c.initStatusBookmarkIDs()
|
2024-01-19 12:57:29 +00:00
|
|
|
c.initStatusFave()
|
2024-04-15 13:22:21 +00:00
|
|
|
c.initStatusFaveIDs()
|
2024-01-19 12:57:29 +00:00
|
|
|
c.initTag()
|
2024-07-29 18:26:31 +00:00
|
|
|
c.initTagIDsFollowedByAccount()
|
2024-01-19 12:57:29 +00:00
|
|
|
c.initThreadMute()
|
2024-04-15 13:22:21 +00:00
|
|
|
c.initToken()
|
2024-01-19 12:57:29 +00:00
|
|
|
c.initTombstone()
|
|
|
|
c.initUser()
|
2024-06-06 16:38:02 +00:00
|
|
|
c.initUserMute()
|
|
|
|
c.initUserMuteIDs()
|
2024-01-19 12:57:29 +00:00
|
|
|
c.initWebfinger()
|
|
|
|
c.initVisibility()
|
2022-12-08 17:35:14 +00:00
|
|
|
}
|
|
|
|
|
2024-01-19 12:57:29 +00:00
|
|
|
// Start will start any caches that require a background
|
|
|
|
// routine, which usually means any kind of TTL caches.
|
2022-12-08 17:35:14 +00:00
|
|
|
func (c *Caches) Start() {
|
2023-05-12 09:15:54 +00:00
|
|
|
log.Infof(nil, "start: %p", c)
|
|
|
|
|
2024-06-06 10:44:43 +00:00
|
|
|
tryUntil("starting webfinger cache", 5, func() bool {
|
2024-07-24 08:41:43 +00:00
|
|
|
return c.Webfinger.Start(5 * time.Minute)
|
2024-01-19 12:57:29 +00:00
|
|
|
})
|
2022-12-08 17:35:14 +00:00
|
|
|
}
|
|
|
|
|
2024-01-19 12:57:29 +00:00
|
|
|
// Stop will stop any caches that require a background
|
|
|
|
// routine, which usually means any kind of TTL caches.
|
2022-12-08 17:35:14 +00:00
|
|
|
func (c *Caches) Stop() {
|
2023-05-12 09:15:54 +00:00
|
|
|
log.Infof(nil, "stop: %p", c)
|
|
|
|
|
2024-07-24 08:41:43 +00:00
|
|
|
tryUntil("stopping webfinger cache", 5, c.Webfinger.Stop)
|
2022-12-08 17:35:14 +00:00
|
|
|
}
|
2023-08-03 09:34:35 +00:00
|
|
|
|
|
|
|
// Sweep will sweep all the available caches to ensure none
|
|
|
|
// are above threshold percent full to their total capacity.
|
|
|
|
//
|
|
|
|
// This helps with cache performance, as a full cache will
|
|
|
|
// require an eviction on every single write, which adds
|
|
|
|
// significant overhead to all cache writes.
|
|
|
|
func (c *Caches) Sweep(threshold float64) {
|
2024-07-24 08:41:43 +00:00
|
|
|
c.DB.Account.Trim(threshold)
|
2024-07-29 18:26:31 +00:00
|
|
|
c.DB.AccountIDsFollowingTag.Trim(threshold)
|
2024-07-24 08:41:43 +00:00
|
|
|
c.DB.AccountNote.Trim(threshold)
|
|
|
|
c.DB.AccountSettings.Trim(threshold)
|
|
|
|
c.DB.AccountStats.Trim(threshold)
|
|
|
|
c.DB.Application.Trim(threshold)
|
|
|
|
c.DB.Block.Trim(threshold)
|
|
|
|
c.DB.BlockIDs.Trim(threshold)
|
|
|
|
c.DB.BoostOfIDs.Trim(threshold)
|
|
|
|
c.DB.Client.Trim(threshold)
|
2024-07-29 18:26:31 +00:00
|
|
|
c.DB.Conversation.Trim(threshold)
|
|
|
|
c.DB.ConversationLastStatusIDs.Trim(threshold)
|
2024-07-24 08:41:43 +00:00
|
|
|
c.DB.Emoji.Trim(threshold)
|
|
|
|
c.DB.EmojiCategory.Trim(threshold)
|
|
|
|
c.DB.Filter.Trim(threshold)
|
|
|
|
c.DB.FilterKeyword.Trim(threshold)
|
|
|
|
c.DB.FilterStatus.Trim(threshold)
|
|
|
|
c.DB.Follow.Trim(threshold)
|
|
|
|
c.DB.FollowIDs.Trim(threshold)
|
|
|
|
c.DB.FollowRequest.Trim(threshold)
|
|
|
|
c.DB.FollowRequestIDs.Trim(threshold)
|
|
|
|
c.DB.InReplyToIDs.Trim(threshold)
|
|
|
|
c.DB.Instance.Trim(threshold)
|
|
|
|
c.DB.InteractionApproval.Trim(threshold)
|
|
|
|
c.DB.List.Trim(threshold)
|
|
|
|
c.DB.ListEntry.Trim(threshold)
|
|
|
|
c.DB.Marker.Trim(threshold)
|
|
|
|
c.DB.Media.Trim(threshold)
|
|
|
|
c.DB.Mention.Trim(threshold)
|
|
|
|
c.DB.Move.Trim(threshold)
|
|
|
|
c.DB.Notification.Trim(threshold)
|
|
|
|
c.DB.Poll.Trim(threshold)
|
|
|
|
c.DB.PollVote.Trim(threshold)
|
|
|
|
c.DB.PollVoteIDs.Trim(threshold)
|
|
|
|
c.DB.Report.Trim(threshold)
|
|
|
|
c.DB.Status.Trim(threshold)
|
|
|
|
c.DB.StatusBookmark.Trim(threshold)
|
|
|
|
c.DB.StatusBookmarkIDs.Trim(threshold)
|
|
|
|
c.DB.StatusFave.Trim(threshold)
|
|
|
|
c.DB.StatusFaveIDs.Trim(threshold)
|
|
|
|
c.DB.Tag.Trim(threshold)
|
2024-07-29 18:26:31 +00:00
|
|
|
c.DB.TagIDsFollowedByAccount.Trim(threshold)
|
2024-07-24 08:41:43 +00:00
|
|
|
c.DB.ThreadMute.Trim(threshold)
|
|
|
|
c.DB.Token.Trim(threshold)
|
|
|
|
c.DB.Tombstone.Trim(threshold)
|
|
|
|
c.DB.User.Trim(threshold)
|
|
|
|
c.DB.UserMute.Trim(threshold)
|
|
|
|
c.DB.UserMuteIDs.Trim(threshold)
|
2023-08-03 09:34:35 +00:00
|
|
|
c.Visibility.Trim(threshold)
|
|
|
|
}
|
2024-07-24 08:41:43 +00:00
|
|
|
|
|
|
|
func (c *Caches) initWebfinger() {
|
|
|
|
// Calculate maximum cache size.
|
|
|
|
cap := calculateCacheMax(
|
|
|
|
sizeofURIStr, sizeofURIStr,
|
|
|
|
config.GetCacheWebfingerMemRatio(),
|
|
|
|
)
|
|
|
|
|
|
|
|
log.Infof(nil, "cache size = %d", cap)
|
|
|
|
|
|
|
|
c.Webfinger = new(ttl.Cache[string, string])
|
|
|
|
c.Webfinger.Init(
|
|
|
|
0,
|
|
|
|
cap,
|
|
|
|
24*time.Hour,
|
|
|
|
)
|
|
|
|
}
|