[feature] Implement markers API (#1989)

* Implement markers API

Fixes #1856

* Correct import grouping in markers files

* Regenerate Swagger for markers API

* Shorten names for readability

* Cache markers for 6 hours

* Update DB ref

* Update envparsing.sh
This commit is contained in:
Vyr Cossont 2023-07-29 03:49:14 -07:00 committed by GitHub
parent cf4bd700fb
commit b874e9251e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 1083 additions and 3 deletions

View file

@ -4988,6 +4988,72 @@ paths:
summary: Add one or more accounts to the given list.
tags:
- lists
/api/v1/markers:
get:
description: Get timeline markers by name
operationId: markersGet
parameters:
- description: Timelines to retrieve.
in: query
items:
enum:
- home
- notifications
type: string
name: timeline
type: array
produces:
- application/json
responses:
"200":
description: Requested markers
schema:
$ref: '#/definitions/markers'
"400":
description: bad request
"401":
description: unauthorized
"500":
description: internal server error
security:
- OAuth2 Bearer:
- read:statuses
tags:
- markers
post:
consumes:
- multipart/form-data
description: Update timeline markers by name
operationId: markersPost
parameters:
- description: Last status ID read on the home timeline.
in: formData
name: home[last_read_id]
type: string
- description: Last notification ID read on the notifications timeline.
in: formData
name: notifications[last_read_id]
type: string
produces:
- application/json
responses:
"200":
description: Requested markers
schema:
$ref: '#/definitions/markers'
"400":
description: bad request
"401":
description: unauthorized
"409":
description: conflict (when two clients try to update the same timeline at the same time)
"500":
description: internal server error
security:
- OAuth2 Bearer:
- write:statuses
tags:
- markers
/api/v1/media/{id}:
get:
operationId: mediaGet

View file

@ -33,6 +33,7 @@
"github.com/superseriousbusiness/gotosocial/internal/api/client/followrequests"
"github.com/superseriousbusiness/gotosocial/internal/api/client/instance"
"github.com/superseriousbusiness/gotosocial/internal/api/client/lists"
"github.com/superseriousbusiness/gotosocial/internal/api/client/markers"
"github.com/superseriousbusiness/gotosocial/internal/api/client/media"
"github.com/superseriousbusiness/gotosocial/internal/api/client/notifications"
"github.com/superseriousbusiness/gotosocial/internal/api/client/preferences"
@ -64,6 +65,7 @@ type Client struct {
followRequests *followrequests.Module // api/v1/follow_requests
instance *instance.Module // api/v1/instance
lists *lists.Module // api/v1/lists
markers *markers.Module // api/v1/markers
media *media.Module // api/v1/media, api/v2/media
notifications *notifications.Module // api/v1/notifications
preferences *preferences.Module // api/v1/preferences
@ -104,6 +106,7 @@ func (c *Client) Route(r router.Router, m ...gin.HandlerFunc) {
c.followRequests.Route(h)
c.instance.Route(h)
c.lists.Route(h)
c.markers.Route(h)
c.media.Route(h)
c.notifications.Route(h)
c.preferences.Route(h)
@ -132,6 +135,7 @@ func NewClient(db db.DB, p *processing.Processor) *Client {
followRequests: followrequests.New(p),
instance: instance.New(p),
lists: lists.New(p),
markers: markers.New(p),
media: media.New(p),
notifications: notifications.New(p),
preferences: preferences.New(p),

View file

@ -0,0 +1,45 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package markers
import (
"net/http"
"github.com/gin-gonic/gin"
"github.com/superseriousbusiness/gotosocial/internal/processing"
)
const (
// BasePath is the base path for serving the markers API, minus the 'api' prefix
BasePath = "/v1/markers"
)
type Module struct {
processor *processing.Processor
}
func New(processor *processing.Processor) *Module {
return &Module{
processor: processor,
}
}
func (m *Module) Route(attachHandler func(method string, path string, f ...gin.HandlerFunc) gin.IRoutes) {
attachHandler(http.MethodGet, BasePath, m.MarkersGETHandler)
attachHandler(http.MethodPost, BasePath, m.MarkersPOSTHandler)
}

View file

@ -0,0 +1,108 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package markers
import (
"net/http"
"github.com/gin-gonic/gin"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/validate"
)
// MarkersGETHandler swagger:operation GET /api/v1/markers markersGet
//
// Get timeline markers by name
//
// ---
// tags:
// - markers
//
// produces:
// - application/json
//
// parameters:
// -
// name: timeline
// type: array
// items:
// type: string
// enum:
// - home
// - notifications
// description: Timelines to retrieve.
// in: query
//
// security:
// - OAuth2 Bearer:
// - read:statuses
//
// responses:
// '200':
// description: Requested markers
// schema:
// "$ref": "#/definitions/markers"
// '400':
// description: bad request
// '401':
// description: unauthorized
// '500':
// description: internal server error
func (m *Module) MarkersGETHandler(c *gin.Context) {
authed, err := oauth.Authed(c, true, true, true, true)
if err != nil {
apiutil.ErrorHandler(c, gtserror.NewErrorUnauthorized(err, err.Error()), m.processor.InstanceGetV1)
return
}
names, errWithCode := parseMarkerNames(c.QueryArray("timeline[]"))
if errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
}
marker, errWithCode := m.processor.Markers().Get(c.Request.Context(), authed.Account, names)
if errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
c.JSON(http.StatusOK, marker)
}
// parseMarkerNames turns a list of strings into a set of valid marker timeline names, or returns an error.
func parseMarkerNames(nameStrings []string) ([]apimodel.MarkerName, gtserror.WithCode) {
nameSet := make(map[apimodel.MarkerName]struct{}, apimodel.MarkerNameNumValues)
for _, timelineString := range nameStrings {
if err := validate.MarkerName(timelineString); err != nil {
return nil, gtserror.NewErrorBadRequest(err, err.Error())
}
nameSet[apimodel.MarkerName(timelineString)] = struct{}{}
}
i := 0
names := make([]apimodel.MarkerName, len(nameSet))
for name := range nameSet {
names[i] = name
i++
}
return names, nil
}

View file

@ -0,0 +1,110 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package markers
import (
"net/http"
"github.com/gin-gonic/gin"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
)
// MarkersPOSTHandler swagger:operation POST /api/v1/markers markersPost
//
// Update timeline markers by name
//
// ---
// tags:
// - markers
//
// consumes:
// - multipart/form-data
//
// produces:
// - application/json
//
// parameters:
// -
// name: home[last_read_id]
// type: string
// description: Last status ID read on the home timeline.
// in: formData
// -
// name: notifications[last_read_id]
// type: string
// description: Last notification ID read on the notifications timeline.
// in: formData
//
// security:
// - OAuth2 Bearer:
// - write:statuses
//
// responses:
// '200':
// description: Requested markers
// schema:
// "$ref": "#/definitions/markers"
// '400':
// description: bad request
// '401':
// description: unauthorized
// '409':
// description: conflict (when two clients try to update the same timeline at the same time)
// '500':
// description: internal server error
func (m *Module) MarkersPOSTHandler(c *gin.Context) {
authed, err := oauth.Authed(c, true, true, true, true)
if err != nil {
apiutil.ErrorHandler(c, gtserror.NewErrorUnauthorized(err, err.Error()), m.processor.InstanceGetV1)
return
}
form := &apimodel.MarkerPostRequest{}
if err := c.ShouldBind(form); err != nil {
apiutil.ErrorHandler(c, gtserror.NewErrorBadRequest(err, err.Error()), m.processor.InstanceGetV1)
return
}
markers := make([]*gtsmodel.Marker, 0, apimodel.MarkerNameNumValues)
if homeLastReadID := form.HomeLastReadID(); homeLastReadID != "" {
markers = append(markers, &gtsmodel.Marker{
AccountID: authed.Account.ID,
Name: gtsmodel.MarkerNameHome,
LastReadID: homeLastReadID,
})
}
if notificationsLastReadID := form.NotificationsLastReadID(); notificationsLastReadID != "" {
markers = append(markers, &gtsmodel.Marker{
AccountID: authed.Account.ID,
Name: gtsmodel.MarkerNameNotifications,
LastReadID: notificationsLastReadID,
})
}
marker, errWithCode := m.processor.Markers().Update(c.Request.Context(), markers)
if errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
c.JSON(http.StatusOK, marker)
}

View file

@ -20,9 +20,9 @@
// Marker represents the last read position within a user's timelines.
type Marker struct {
// Information about the user's position in the home timeline.
Home *TimelineMarker `json:"home"`
Home *TimelineMarker `json:"home,omitempty"`
// Information about the user's position in their notifications.
Notifications *TimelineMarker `json:"notifications"`
Notifications *TimelineMarker `json:"notifications,omitempty"`
}
// TimelineMarker contains information about a user's progress through a specific timeline.
@ -32,5 +32,46 @@ type TimelineMarker struct {
// The timestamp of when the marker was set (ISO 8601 Datetime)
UpdatedAt string `json:"updated_at"`
// Used for locking to prevent write conflicts.
Version string `json:"version"`
Version int `json:"version"`
}
// MarkerName is the name of one of the timelines we can store markers for.
type MarkerName string
const (
MarkerNameHome MarkerName = "home"
MarkerNameNotifications MarkerName = "notifications"
MarkerNameNumValues = 2
)
// MarkerPostRequest models a request to update one or more markers.
// This has two sets of fields to support a goofy nested map structure in both form data and JSON bodies.
//
// swagger:ignore
type MarkerPostRequest struct {
Home *MarkerPostRequestMarker `json:"home"`
FormHomeLastReadID string `form:"home[last_read_id]"`
Notifications *MarkerPostRequestMarker `json:"notifications"`
FormNotificationsLastReadID string `form:"notifications[last_read_id]"`
}
type MarkerPostRequestMarker struct {
// The ID of the most recently viewed entity.
LastReadID string `json:"last_read_id"`
}
// HomeLastReadID should be used instead of Home or FormHomeLastReadID.
func (r *MarkerPostRequest) HomeLastReadID() string {
if r.Home != nil {
return r.Home.LastReadID
}
return r.FormHomeLastReadID
}
// NotificationsLastReadID should be used instead of Notifications or FormNotificationsLastReadID.
func (r *MarkerPostRequest) NotificationsLastReadID() string {
if r.Notifications != nil {
return r.Notifications.LastReadID
}
return r.FormNotificationsLastReadID
}

21
internal/cache/gts.go vendored
View file

@ -39,6 +39,7 @@ type GTSCaches struct {
instance *result.Cache[*gtsmodel.Instance]
list *result.Cache[*gtsmodel.List]
listEntry *result.Cache[*gtsmodel.ListEntry]
marker *result.Cache[*gtsmodel.Marker]
media *result.Cache[*gtsmodel.MediaAttachment]
mention *result.Cache[*gtsmodel.Mention]
notification *result.Cache[*gtsmodel.Notification]
@ -65,6 +66,7 @@ func (c *GTSCaches) Init() {
c.initInstance()
c.initList()
c.initListEntry()
c.initMarker()
c.initMedia()
c.initMention()
c.initNotification()
@ -88,6 +90,7 @@ func (c *GTSCaches) Start() {
tryStart(c.instance, config.GetCacheGTSInstanceSweepFreq())
tryStart(c.list, config.GetCacheGTSListSweepFreq())
tryStart(c.listEntry, config.GetCacheGTSListEntrySweepFreq())
tryStart(c.marker, config.GetCacheGTSMarkerSweepFreq())
tryStart(c.media, config.GetCacheGTSMediaSweepFreq())
tryStart(c.mention, config.GetCacheGTSMentionSweepFreq())
tryStart(c.notification, config.GetCacheGTSNotificationSweepFreq())
@ -116,6 +119,7 @@ func (c *GTSCaches) Stop() {
tryStop(c.instance, config.GetCacheGTSInstanceSweepFreq())
tryStop(c.list, config.GetCacheGTSListSweepFreq())
tryStop(c.listEntry, config.GetCacheGTSListEntrySweepFreq())
tryStop(c.marker, config.GetCacheGTSMarkerSweepFreq())
tryStop(c.media, config.GetCacheGTSMediaSweepFreq())
tryStop(c.mention, config.GetCacheGTSNotificationSweepFreq())
tryStop(c.notification, config.GetCacheGTSNotificationSweepFreq())
@ -182,6 +186,11 @@ func (c *GTSCaches) ListEntry() *result.Cache[*gtsmodel.ListEntry] {
return c.listEntry
}
// Marker provides access to the gtsmodel Marker database cache.
func (c *GTSCaches) Marker() *result.Cache[*gtsmodel.Marker] {
return c.marker
}
// Media provides access to the gtsmodel Media database cache.
func (c *GTSCaches) Media() *result.Cache[*gtsmodel.MediaAttachment] {
return c.media
@ -372,6 +381,18 @@ func (c *GTSCaches) initListEntry() {
c.list.IgnoreErrors(ignoreErrors)
}
func (c *GTSCaches) initMarker() {
c.marker = result.New([]result.Lookup{
{Name: "AccountID.Name"},
}, func(m1 *gtsmodel.Marker) *gtsmodel.Marker {
m2 := new(gtsmodel.Marker)
*m2 = *m1
return m2
}, config.GetCacheGTSMarkerMaxSize())
c.marker.SetTTL(config.GetCacheGTSMarkerTTL(), true)
c.marker.IgnoreErrors(ignoreErrors)
}
func (c *GTSCaches) initMedia() {
c.media = result.New([]result.Lookup{
{Name: "ID"},

View file

@ -226,6 +226,10 @@ type GTSCacheConfiguration struct {
ListEntryTTL time.Duration `name:"list-entry-ttl"`
ListEntrySweepFreq time.Duration `name:"list-entry-sweep-freq"`
MarkerMaxSize int `name:"marker-max-size"`
MarkerTTL time.Duration `name:"marker-ttl"`
MarkerSweepFreq time.Duration `name:"marker-sweep-freq"`
MediaMaxSize int `name:"media-max-size"`
MediaTTL time.Duration `name:"media-ttl"`
MediaSweepFreq time.Duration `name:"media-sweep-freq"`

View file

@ -171,6 +171,10 @@
ListEntryTTL: time.Minute * 30,
ListEntrySweepFreq: time.Minute,
MarkerMaxSize: 2000,
MarkerTTL: time.Hour * 6,
MarkerSweepFreq: time.Minute,
MediaMaxSize: 1000,
MediaTTL: time.Minute * 30,
MediaSweepFreq: time.Minute,

View file

@ -3228,6 +3228,81 @@ func GetCacheGTSListEntrySweepFreq() time.Duration { return global.GetCacheGTSLi
// SetCacheGTSListEntrySweepFreq safely sets the value for global configuration 'Cache.GTS.ListEntrySweepFreq' field
func SetCacheGTSListEntrySweepFreq(v time.Duration) { global.SetCacheGTSListEntrySweepFreq(v) }
// GetCacheGTSMarkerMaxSize safely fetches the Configuration value for state's 'Cache.GTS.MarkerMaxSize' field
func (st *ConfigState) GetCacheGTSMarkerMaxSize() (v int) {
st.mutex.RLock()
v = st.config.Cache.GTS.MarkerMaxSize
st.mutex.RUnlock()
return
}
// SetCacheGTSMarkerMaxSize safely sets the Configuration value for state's 'Cache.GTS.MarkerMaxSize' field
func (st *ConfigState) SetCacheGTSMarkerMaxSize(v int) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.Cache.GTS.MarkerMaxSize = v
st.reloadToViper()
}
// CacheGTSMarkerMaxSizeFlag returns the flag name for the 'Cache.GTS.MarkerMaxSize' field
func CacheGTSMarkerMaxSizeFlag() string { return "cache-gts-marker-max-size" }
// GetCacheGTSMarkerMaxSize safely fetches the value for global configuration 'Cache.GTS.MarkerMaxSize' field
func GetCacheGTSMarkerMaxSize() int { return global.GetCacheGTSMarkerMaxSize() }
// SetCacheGTSMarkerMaxSize safely sets the value for global configuration 'Cache.GTS.MarkerMaxSize' field
func SetCacheGTSMarkerMaxSize(v int) { global.SetCacheGTSMarkerMaxSize(v) }
// GetCacheGTSMarkerTTL safely fetches the Configuration value for state's 'Cache.GTS.MarkerTTL' field
func (st *ConfigState) GetCacheGTSMarkerTTL() (v time.Duration) {
st.mutex.RLock()
v = st.config.Cache.GTS.MarkerTTL
st.mutex.RUnlock()
return
}
// SetCacheGTSMarkerTTL safely sets the Configuration value for state's 'Cache.GTS.MarkerTTL' field
func (st *ConfigState) SetCacheGTSMarkerTTL(v time.Duration) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.Cache.GTS.MarkerTTL = v
st.reloadToViper()
}
// CacheGTSMarkerTTLFlag returns the flag name for the 'Cache.GTS.MarkerTTL' field
func CacheGTSMarkerTTLFlag() string { return "cache-gts-marker-ttl" }
// GetCacheGTSMarkerTTL safely fetches the value for global configuration 'Cache.GTS.MarkerTTL' field
func GetCacheGTSMarkerTTL() time.Duration { return global.GetCacheGTSMarkerTTL() }
// SetCacheGTSMarkerTTL safely sets the value for global configuration 'Cache.GTS.MarkerTTL' field
func SetCacheGTSMarkerTTL(v time.Duration) { global.SetCacheGTSMarkerTTL(v) }
// GetCacheGTSMarkerSweepFreq safely fetches the Configuration value for state's 'Cache.GTS.MarkerSweepFreq' field
func (st *ConfigState) GetCacheGTSMarkerSweepFreq() (v time.Duration) {
st.mutex.RLock()
v = st.config.Cache.GTS.MarkerSweepFreq
st.mutex.RUnlock()
return
}
// SetCacheGTSMarkerSweepFreq safely sets the Configuration value for state's 'Cache.GTS.MarkerSweepFreq' field
func (st *ConfigState) SetCacheGTSMarkerSweepFreq(v time.Duration) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.Cache.GTS.MarkerSweepFreq = v
st.reloadToViper()
}
// CacheGTSMarkerSweepFreqFlag returns the flag name for the 'Cache.GTS.MarkerSweepFreq' field
func CacheGTSMarkerSweepFreqFlag() string { return "cache-gts-marker-sweep-freq" }
// GetCacheGTSMarkerSweepFreq safely fetches the value for global configuration 'Cache.GTS.MarkerSweepFreq' field
func GetCacheGTSMarkerSweepFreq() time.Duration { return global.GetCacheGTSMarkerSweepFreq() }
// SetCacheGTSMarkerSweepFreq safely sets the value for global configuration 'Cache.GTS.MarkerSweepFreq' field
func SetCacheGTSMarkerSweepFreq(v time.Duration) { global.SetCacheGTSMarkerSweepFreq(v) }
// GetCacheGTSMediaMaxSize safely fetches the Configuration value for state's 'Cache.GTS.MediaMaxSize' field
func (st *ConfigState) GetCacheGTSMediaMaxSize() (v int) {
st.mutex.RLock()

View file

@ -66,6 +66,7 @@ type DBService struct {
db.Emoji
db.Instance
db.List
db.Marker
db.Media
db.Mention
db.Notification
@ -186,6 +187,10 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) {
db: db,
state: state,
},
Marker: &markerDB{
db: db,
state: state,
},
Media: &mediaDB{
db: db,
state: state,

View file

@ -50,6 +50,7 @@ type BunDBStandardTestSuite struct {
testLists map[string]*gtsmodel.List
testListEntries map[string]*gtsmodel.ListEntry
testAccountNotes map[string]*gtsmodel.AccountNote
testMarkers map[string]*gtsmodel.Marker
}
func (suite *BunDBStandardTestSuite) SetupSuite() {
@ -70,6 +71,7 @@ func (suite *BunDBStandardTestSuite) SetupSuite() {
suite.testLists = testrig.NewTestLists()
suite.testListEntries = testrig.NewTestListEntries()
suite.testAccountNotes = testrig.NewTestAccountNotes()
suite.testMarkers = testrig.NewTestMarkers()
}
func (suite *BunDBStandardTestSuite) SetupTest() {

115
internal/db/bundb/marker.go Normal file
View file

@ -0,0 +1,115 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package bundb
import (
"context"
"errors"
"fmt"
"time"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/uptrace/bun"
)
type markerDB struct {
db *WrappedDB
state *state.State
}
/*
MARKER FUNCTIONS
*/
func (m *markerDB) GetMarker(ctx context.Context, accountID string, name gtsmodel.MarkerName) (*gtsmodel.Marker, error) {
marker, err := m.state.Caches.GTS.Marker().Load(
"AccountID.Name",
func() (*gtsmodel.Marker, error) {
var marker gtsmodel.Marker
if err := m.db.NewSelect().
Model(&marker).
Where("? = ? AND ? = ?", bun.Ident("account_id"), accountID, bun.Ident("name"), name).
Scan(ctx); err != nil {
return nil, m.db.ProcessError(err)
}
return &marker, nil
},
accountID,
name,
)
if err != nil {
return nil, err // already processed
}
return marker, nil
}
func (m *markerDB) UpdateMarker(ctx context.Context, marker *gtsmodel.Marker) error {
prevMarker, err := m.GetMarker(ctx, marker.AccountID, marker.Name)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return fmt.Errorf("UpdateMarker: error fetching previous version of marker: %w", err)
}
marker.UpdatedAt = time.Now()
if prevMarker != nil {
marker.Version = prevMarker.Version + 1
}
return m.state.Caches.GTS.Marker().Store(marker, func() error {
if prevMarker == nil {
if _, err := m.db.NewInsert().
Model(marker).
Exec(ctx); err != nil {
return m.db.ProcessError(err)
}
return nil
}
// Optimistic concurrency control: start a transaction, try to update a row with a previously retrieved version.
// If the update in the transaction fails to actually change anything, another update happened concurrently, and
// this update should be retried by the caller, which in this case involves sending HTTP 409 to the API client.
return m.db.RunInTx(ctx, func(tx bun.Tx) error {
result, err := tx.NewUpdate().
Model(marker).
WherePK().
Where("? = ?", bun.Ident("version"), prevMarker.Version).
Exec(ctx)
if err != nil {
return m.db.ProcessError(err)
}
rowsAffected, err := result.RowsAffected()
if err != nil {
return m.db.ProcessError(err)
}
if rowsAffected == 0 {
// Will trigger a rollback, although there should be no changes to roll back.
return db.ErrAlreadyExists
} else if rowsAffected > 1 {
// This shouldn't happen.
return db.ErrNoEntries
}
return nil
})
})
}

View file

@ -0,0 +1,127 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http:www.gnu.org/licenses/>.
package bundb_test
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
// MarkersTestSuite uses home timelines for Get tests
// and notifications timelines for Update tests
// so that multiple tests running at once can't step on each other.
type MarkersTestSuite struct {
BunDBStandardTestSuite
}
func (suite *MarkersTestSuite) TestGetExisting() {
ctx := context.Background()
// This account has home and notifications markers set.
localAccount1 := suite.testAccounts["local_account_1"]
marker, err := suite.db.GetMarker(ctx, localAccount1.ID, gtsmodel.MarkerNameHome)
suite.NoError(err)
if err != nil {
suite.FailNow(err.Error())
}
// Should match our fixture.
suite.Equal("01F8MH82FYRXD2RC6108DAJ5HB", marker.LastReadID)
}
func (suite *MarkersTestSuite) TestGetUnset() {
ctx := context.Background()
// This account has no markers set.
localAccount2 := suite.testAccounts["local_account_2"]
marker, err := suite.db.GetMarker(ctx, localAccount2.ID, gtsmodel.MarkerNameHome)
// Should not return anything.
suite.Nil(marker)
suite.ErrorIs(err, db.ErrNoEntries)
}
func (suite *MarkersTestSuite) TestUpdateExisting() {
ctx := context.Background()
now := time.Now()
// This account has home and notifications markers set.
localAccount1 := suite.testAccounts["local_account_1"]
prevMarker := suite.testMarkers["local_account_1_notification_marker"]
marker := &gtsmodel.Marker{
AccountID: localAccount1.ID,
Name: gtsmodel.MarkerNameNotifications,
LastReadID: "01H57YZECGJ2ZW39H8TJWAH0KY",
}
err := suite.db.UpdateMarker(ctx, marker)
suite.NoError(err)
if err != nil {
suite.FailNow(err.Error())
}
// Modifies the update and version fields of the marker as an intentional side effect.
suite.GreaterOrEqual(marker.UpdatedAt, now)
suite.Greater(marker.Version, prevMarker.Version)
// Re-fetch it from the DB and confirm that we got the updated version.
marker2, err := suite.db.GetMarker(ctx, localAccount1.ID, gtsmodel.MarkerNameNotifications)
suite.NoError(err)
if err != nil {
suite.FailNow(err.Error())
}
suite.GreaterOrEqual(marker2.UpdatedAt, now)
suite.GreaterOrEqual(marker2.Version, prevMarker.Version)
suite.Equal("01H57YZECGJ2ZW39H8TJWAH0KY", marker2.LastReadID)
}
func (suite *MarkersTestSuite) TestUpdateUnset() {
ctx := context.Background()
now := time.Now()
// This account has no markers set.
localAccount2 := suite.testAccounts["local_account_2"]
marker := &gtsmodel.Marker{
AccountID: localAccount2.ID,
Name: gtsmodel.MarkerNameNotifications,
LastReadID: "01H57ZVGMD348ZJD5WENDZDH9Z",
}
err := suite.db.UpdateMarker(ctx, marker)
suite.NoError(err)
if err != nil {
suite.FailNow(err.Error())
}
// Modifies the update and version fields of the marker as an intentional side effect.
suite.GreaterOrEqual(marker.UpdatedAt, now)
suite.GreaterOrEqual(marker.Version, 0)
// Re-fetch it from the DB and confirm that we got the updated version.
marker2, err := suite.db.GetMarker(ctx, localAccount2.ID, gtsmodel.MarkerNameNotifications)
suite.NoError(err)
if err != nil {
suite.FailNow(err.Error())
}
suite.GreaterOrEqual(marker2.UpdatedAt, now)
suite.GreaterOrEqual(marker2.Version, 0)
suite.Equal("01H57ZVGMD348ZJD5WENDZDH9Z", marker2.LastReadID)
}
func TestMarkersTestSuite(t *testing.T) {
suite.Run(t, new(MarkersTestSuite))
}

View file

@ -0,0 +1,66 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package migrations
import (
"context"
gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/uptrace/bun"
)
func init() {
up := func(ctx context.Context, db *bun.DB) error {
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
// Marker table.
if _, err := tx.
NewCreateTable().
Model(&gtsmodel.Marker{}).
IfNotExists().
Exec(ctx); err != nil {
return err
}
// Add indexes to the Marker table.
for index, columns := range map[string][]string{
"markers_account_id_name_idx": {"account_id", "name"},
} {
if _, err := tx.
NewCreateIndex().
Table("markers").
Index(index).
Column(columns...).
Exec(ctx); err != nil {
return err
}
}
return nil
})
}
down := func(ctx context.Context, db *bun.DB) error {
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
return nil
})
}
if err := Migrations.Register(up, down); err != nil {
panic(err)
}
}

View file

@ -37,6 +37,7 @@ type DB interface {
Emoji
Instance
List
Marker
Media
Mention
Notification

32
internal/db/marker.go Normal file
View file

@ -0,0 +1,32 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package db
import (
"context"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
type Marker interface {
// GetMarker gets one marker with the given timeline name.
GetMarker(ctx context.Context, accountID string, name gtsmodel.MarkerName) (*gtsmodel.Marker, error)
// UpdateMarker updates the given marker.
UpdateMarker(ctx context.Context, marker *gtsmodel.Marker) error
}

View file

@ -0,0 +1,37 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package gtsmodel
import "time"
// Marker stores a local account's read position on a given timeline.
type Marker struct {
AccountID string `validate:"required,ulid" bun:"type:CHAR(26),pk,unique:markers_account_id_timeline_uniq,notnull,nullzero"` // ID of the local account that owns the marker
Name MarkerName `validate:"oneof=home notifications" bun:",nullzero,notnull,pk,unique:markers_account_id_timeline_uniq"` // Name of the marked timeline
UpdatedAt time.Time `validate:"required" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // When marker was last updated
Version int `validate:"required,min=0" bun:",nullzero,notnull,default:0"` // For optimistic concurrency control
LastReadID string `validate:"required,ulid" bun:"type:CHAR(26),notnull,nullzero"` // Last ID read on this timeline (status ID for home, notification ID for notifications)
}
// MarkerName is the name of one of the timelines we can store markers for.
type MarkerName string
const (
MarkerNameHome MarkerName = "home"
MarkerNameNotifications MarkerName = "notifications"
)

View file

@ -0,0 +1,54 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package markers
import (
"context"
"errors"
"fmt"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
// Get returns an API model for the markers of the requested timelines.
// If a timeline marker hasn't been set yet, it's not included in the response.
func (p *Processor) Get(ctx context.Context, account *gtsmodel.Account, names []apimodel.MarkerName) (*apimodel.Marker, gtserror.WithCode) {
markers := make([]*gtsmodel.Marker, 0, len(names))
for _, name := range names {
marker, err := p.state.DB.GetMarker(ctx, account.ID, typeutils.APIMarkerNameToMarkerName(name))
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
continue
}
// Real database error.
return nil, gtserror.NewErrorInternalError(err)
}
markers = append(markers, marker)
}
apiMarker, err := p.tc.MarkersToAPIMarker(ctx, markers)
if err != nil {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error converting marker to api: %w", err))
}
return apiMarker, nil
}

View file

@ -0,0 +1,35 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package markers
import (
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
type Processor struct {
state *state.State
tc typeutils.TypeConverter
}
func New(state *state.State, tc typeutils.TypeConverter) Processor {
return Processor{
state: state,
tc: tc,
}
}

View file

@ -0,0 +1,48 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package markers
import (
"context"
"errors"
"fmt"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
// Update updates the given markers and returns an API model for them.
func (p *Processor) Update(ctx context.Context, markers []*gtsmodel.Marker) (*apimodel.Marker, gtserror.WithCode) {
for _, marker := range markers {
if err := p.state.DB.UpdateMarker(ctx, marker); err != nil {
if errors.Is(err, db.ErrAlreadyExists) {
return nil, gtserror.NewErrorConflict(err, "marker updated by another client")
}
return nil, gtserror.NewErrorInternalError(err)
}
}
apiMarker, err := p.tc.MarkersToAPIMarker(ctx, markers)
if err != nil {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error converting marker to api: %w", err))
}
return apiMarker, nil
}

View file

@ -30,6 +30,7 @@
"github.com/superseriousbusiness/gotosocial/internal/processing/admin"
"github.com/superseriousbusiness/gotosocial/internal/processing/fedi"
"github.com/superseriousbusiness/gotosocial/internal/processing/list"
"github.com/superseriousbusiness/gotosocial/internal/processing/markers"
"github.com/superseriousbusiness/gotosocial/internal/processing/media"
"github.com/superseriousbusiness/gotosocial/internal/processing/report"
"github.com/superseriousbusiness/gotosocial/internal/processing/search"
@ -59,6 +60,7 @@ type Processor struct {
admin admin.Processor
fedi fedi.Processor
list list.Processor
markers markers.Processor
media media.Processor
report report.Processor
search search.Processor
@ -84,6 +86,10 @@ func (p *Processor) List() *list.Processor {
return &p.list
}
func (p *Processor) Markers() *markers.Processor {
return &p.markers
}
func (p *Processor) Media() *media.Processor {
return &p.media
}
@ -140,6 +146,7 @@ func NewProcessor(
processor.admin = admin.New(state, tc, mediaManager, federator.TransportController(), emailSender)
processor.fedi = fedi.New(state, tc, federator, filter)
processor.list = list.New(state, tc)
processor.markers = markers.New(state, tc)
processor.media = media.New(state, tc, mediaManager, federator.TransportController())
processor.report = report.New(state, tc)
processor.timeline = timeline.New(state, tc, filter)

View file

@ -94,6 +94,8 @@ type TypeConverter interface {
ReportToAdminAPIReport(ctx context.Context, r *gtsmodel.Report, requestingAccount *gtsmodel.Account) (*apimodel.AdminReport, error)
// ListToAPIList converts one gts model list into an api model list, for serving at /api/v1/lists/{id}
ListToAPIList(ctx context.Context, l *gtsmodel.List) (*apimodel.List, error)
// MarkersToAPIMarker converts several gts model markers into an api marker, for serving at /api/v1/markers
MarkersToAPIMarker(ctx context.Context, markers []*gtsmodel.Marker) (*apimodel.Marker, error)
/*
INTERNAL (gts) MODEL TO FRONTEND (rss) MODEL

View file

@ -37,3 +37,13 @@ func APIVisToVis(m apimodel.Visibility) gtsmodel.Visibility {
}
return ""
}
func APIMarkerNameToMarkerName(m apimodel.MarkerName) gtsmodel.MarkerName {
switch m {
case apimodel.MarkerNameHome:
return gtsmodel.MarkerNameHome
case apimodel.MarkerNameNotifications:
return gtsmodel.MarkerNameNotifications
}
return ""
}

View file

@ -1166,6 +1166,26 @@ func (c *converter) ListToAPIList(ctx context.Context, l *gtsmodel.List) (*apimo
}, nil
}
func (c *converter) MarkersToAPIMarker(ctx context.Context, markers []*gtsmodel.Marker) (*apimodel.Marker, error) {
apiMarker := &apimodel.Marker{}
for _, marker := range markers {
apiTimelineMarker := &apimodel.TimelineMarker{
LastReadID: marker.LastReadID,
UpdatedAt: util.FormatISO8601(marker.UpdatedAt),
Version: marker.Version,
}
switch apimodel.MarkerName(marker.Name) {
case apimodel.MarkerNameHome:
apiMarker.Home = apiTimelineMarker
case apimodel.MarkerNameNotifications:
apiMarker.Notifications = apiTimelineMarker
default:
return nil, fmt.Errorf("unknown marker timeline name: %s", marker.Name)
}
}
return apiMarker, nil
}
// convertAttachmentsToAPIAttachments will convert a slice of GTS model attachments to frontend API model attachments, falling back to IDs if no GTS models supplied.
func (c *converter) convertAttachmentsToAPIAttachments(ctx context.Context, attachments []*gtsmodel.MediaAttachment, attachmentIDs []string) ([]apimodel.Attachment, error) {
var errs gtserror.MultiError

View file

@ -289,3 +289,15 @@ func ListRepliesPolicy(repliesPolicy gtsmodel.RepliesPolicy) error {
return fmt.Errorf("list replies_policy must be either empty or one of 'followed', 'list', 'none'")
}
}
// MarkerName checks that the desired marker timeline name is valid.
func MarkerName(name string) error {
if name == "" {
return fmt.Errorf("empty string for marker timeline name not allowed")
}
switch apimodel.MarkerName(name) {
case apimodel.MarkerNameHome, apimodel.MarkerNameNotifications:
return nil
}
return fmt.Errorf("marker timeline name '%s' was not recognized, valid options are '%s', '%s'", name, apimodel.MarkerNameHome, apimodel.MarkerNameNotifications)
}

View file

@ -52,6 +52,9 @@ EXPECT=$(cat <<"EOF"
"list-max-size": 2000,
"list-sweep-freq": 60000000000,
"list-ttl": 1800000000000,
"marker-max-size": 2000,
"marker-sweep-freq": 60000000000,
"marker-ttl": 21600000000000,
"media-max-size": 1000,
"media-sweep-freq": 60000000000,
"media-ttl": 1800000000000,

View file

@ -41,6 +41,7 @@
&gtsmodel.FollowRequest{},
&gtsmodel.List{},
&gtsmodel.ListEntry{},
&gtsmodel.Marker{},
&gtsmodel.MediaAttachment{},
&gtsmodel.Mention{},
&gtsmodel.Status{},
@ -287,6 +288,12 @@ func StandardDBSetup(db db.DB, accounts map[string]*gtsmodel.Account) {
}
}
for _, v := range NewTestMarkers() {
if err := db.Put(ctx, v); err != nil {
log.Panic(nil, err)
}
}
if err := db.CreateInstanceAccount(ctx); err != nil {
log.Panic(nil, err)
}

View file

@ -2007,6 +2007,25 @@ func NewTestListEntries() map[string]*gtsmodel.ListEntry {
}
}
func NewTestMarkers() map[string]*gtsmodel.Marker {
return map[string]*gtsmodel.Marker{
"local_account_1_home_marker": {
AccountID: "01F8MH1H7YV1Z7D2C8K2730QBF",
Name: gtsmodel.MarkerNameHome,
UpdatedAt: TimeMustParse("2022-05-14T13:21:09+02:00"),
Version: 0,
LastReadID: "01F8MH82FYRXD2RC6108DAJ5HB",
},
"local_account_1_notification_marker": {
AccountID: "01F8MH1H7YV1Z7D2C8K2730QBF",
Name: gtsmodel.MarkerNameNotifications,
UpdatedAt: TimeMustParse("2022-05-14T13:21:09+02:00"),
Version: 4,
LastReadID: "01F8Q0ANPTWW10DAKTX7BRPBJP",
},
}
}
func NewTestBlocks() map[string]*gtsmodel.Block {
return map[string]*gtsmodel.Block{
"local_account_2_block_remote_account_1": {