mirror of
https://github.com/superseriousbusiness/gotosocial.git
synced 2024-11-22 03:36:39 +00:00
[bugfix] fix emoji recaching operations (#3167)
* add test for emoji update image * update emoji recache to set the instance account id * don't refresh emoji if only not cached. in that case literally just recache * code comment * rename + move a few things * add some more code comments, and rename some functions to make logic a bit clearer * remove unnecessary nil check (the value can be nil) * comment wording * remove test data output * handle the case of caching an emoji which has been refreshed then uncached * allow overwriting on testrig storage as we do now on regular storage * fix emoji category ID not getting updated --------- Co-authored-by: tobi <tobi.smethurst@protonmail.com>
This commit is contained in:
parent
fa59c3713c
commit
b85a9983d0
|
@ -21,9 +21,9 @@
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"io"
|
"io"
|
||||||
"io/ioutil"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
|
@ -68,7 +68,7 @@ func (suite *EmojiUpdateTestSuite) TestEmojiUpdateNewCategory() {
|
||||||
defer result.Body.Close()
|
defer result.Body.Close()
|
||||||
|
|
||||||
// check the response
|
// check the response
|
||||||
b, err := ioutil.ReadAll(result.Body)
|
b, err := io.ReadAll(result.Body)
|
||||||
suite.NoError(err)
|
suite.NoError(err)
|
||||||
suite.NotEmpty(b)
|
suite.NotEmpty(b)
|
||||||
|
|
||||||
|
@ -145,7 +145,7 @@ func (suite *EmojiUpdateTestSuite) TestEmojiUpdateSwitchCategory() {
|
||||||
defer result.Body.Close()
|
defer result.Body.Close()
|
||||||
|
|
||||||
// check the response
|
// check the response
|
||||||
b, err := ioutil.ReadAll(result.Body)
|
b, err := io.ReadAll(result.Body)
|
||||||
suite.NoError(err)
|
suite.NoError(err)
|
||||||
suite.NotEmpty(b)
|
suite.NotEmpty(b)
|
||||||
|
|
||||||
|
@ -223,7 +223,7 @@ func (suite *EmojiUpdateTestSuite) TestEmojiUpdateCopyRemoteToLocal() {
|
||||||
defer result.Body.Close()
|
defer result.Body.Close()
|
||||||
|
|
||||||
// check the response
|
// check the response
|
||||||
b, err := ioutil.ReadAll(result.Body)
|
b, err := io.ReadAll(result.Body)
|
||||||
suite.NoError(err)
|
suite.NoError(err)
|
||||||
suite.NotEmpty(b)
|
suite.NotEmpty(b)
|
||||||
|
|
||||||
|
@ -299,7 +299,7 @@ func (suite *EmojiUpdateTestSuite) TestEmojiUpdateDisableEmoji() {
|
||||||
defer result.Body.Close()
|
defer result.Body.Close()
|
||||||
|
|
||||||
// check the response
|
// check the response
|
||||||
b, err := ioutil.ReadAll(result.Body)
|
b, err := io.ReadAll(result.Body)
|
||||||
suite.NoError(err)
|
suite.NoError(err)
|
||||||
suite.NotEmpty(b)
|
suite.NotEmpty(b)
|
||||||
|
|
||||||
|
@ -338,12 +338,97 @@ func (suite *EmojiUpdateTestSuite) TestEmojiUpdateDisableLocalEmoji() {
|
||||||
defer result.Body.Close()
|
defer result.Body.Close()
|
||||||
|
|
||||||
// check the response
|
// check the response
|
||||||
b, err := ioutil.ReadAll(result.Body)
|
b, err := io.ReadAll(result.Body)
|
||||||
suite.NoError(err)
|
suite.NoError(err)
|
||||||
|
|
||||||
suite.Equal(`{"error":"Bad Request: emoji 01F8MH9H8E4VG3KDYJR9EGPXCQ is not a remote emoji, cannot disable it via this endpoint"}`, string(b))
|
suite.Equal(`{"error":"Bad Request: emoji 01F8MH9H8E4VG3KDYJR9EGPXCQ is not a remote emoji, cannot disable it via this endpoint"}`, string(b))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (suite *EmojiUpdateTestSuite) TestEmojiUpdateModify() {
|
||||||
|
testEmoji := >smodel.Emoji{}
|
||||||
|
*testEmoji = *suite.testEmojis["rainbow"]
|
||||||
|
|
||||||
|
// set up the request
|
||||||
|
requestBody, w, err := testrig.CreateMultipartFormData(
|
||||||
|
testrig.FileToDataF("image", "../../../../testrig/media/kip-original.gif"),
|
||||||
|
map[string][]string{
|
||||||
|
"type": {"modify"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
bodyBytes := requestBody.Bytes()
|
||||||
|
recorder := httptest.NewRecorder()
|
||||||
|
ctx := suite.newContext(recorder, http.MethodPost, bodyBytes, admin.EmojiPathWithID, w.FormDataContentType())
|
||||||
|
ctx.AddParam(apiutil.IDKey, testEmoji.ID)
|
||||||
|
|
||||||
|
// call the handler
|
||||||
|
suite.adminModule.EmojiPATCHHandler(ctx)
|
||||||
|
suite.Equal(http.StatusOK, recorder.Code)
|
||||||
|
|
||||||
|
// 2. we should have no error message in the result body
|
||||||
|
result := recorder.Result()
|
||||||
|
defer result.Body.Close()
|
||||||
|
|
||||||
|
// check the response
|
||||||
|
b, err := io.ReadAll(result.Body)
|
||||||
|
suite.NoError(err)
|
||||||
|
suite.NotEmpty(b)
|
||||||
|
|
||||||
|
// response should be an admin model emoji
|
||||||
|
adminEmoji := &apimodel.AdminEmoji{}
|
||||||
|
err = json.Unmarshal(b, adminEmoji)
|
||||||
|
suite.NoError(err)
|
||||||
|
|
||||||
|
// appropriate fields should be set
|
||||||
|
suite.Equal("rainbow", adminEmoji.Shortcode)
|
||||||
|
suite.NotEmpty(adminEmoji.URL)
|
||||||
|
suite.NotEmpty(adminEmoji.StaticURL)
|
||||||
|
suite.True(adminEmoji.VisibleInPicker)
|
||||||
|
|
||||||
|
// emoji should be in the db
|
||||||
|
dbEmoji, err := suite.db.GetEmojiByShortcodeDomain(context.Background(), adminEmoji.Shortcode, "")
|
||||||
|
suite.NoError(err)
|
||||||
|
|
||||||
|
// check fields on the emoji
|
||||||
|
suite.NotEmpty(dbEmoji.ID)
|
||||||
|
suite.Equal("rainbow", dbEmoji.Shortcode)
|
||||||
|
suite.Empty(dbEmoji.Domain)
|
||||||
|
suite.Empty(dbEmoji.ImageRemoteURL)
|
||||||
|
suite.Empty(dbEmoji.ImageStaticRemoteURL)
|
||||||
|
suite.Equal(adminEmoji.URL, dbEmoji.ImageURL)
|
||||||
|
suite.Equal(adminEmoji.StaticURL, dbEmoji.ImageStaticURL)
|
||||||
|
|
||||||
|
// Ensure image path as expected.
|
||||||
|
suite.NotEmpty(dbEmoji.ImagePath)
|
||||||
|
if !strings.HasPrefix(dbEmoji.ImagePath, suite.testAccounts["instance_account"].ID+"/emoji/original") {
|
||||||
|
suite.FailNow("", "image path %s not valid", dbEmoji.ImagePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure static image path as expected.
|
||||||
|
suite.NotEmpty(dbEmoji.ImageStaticPath)
|
||||||
|
if !strings.HasPrefix(dbEmoji.ImageStaticPath, suite.testAccounts["instance_account"].ID+"/emoji/static") {
|
||||||
|
suite.FailNow("", "image path %s not valid", dbEmoji.ImageStaticPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
suite.Equal("image/gif", dbEmoji.ImageContentType)
|
||||||
|
suite.Equal("image/png", dbEmoji.ImageStaticContentType)
|
||||||
|
suite.Equal(1428, dbEmoji.ImageFileSize)
|
||||||
|
suite.Equal(1056, dbEmoji.ImageStaticFileSize)
|
||||||
|
suite.False(*dbEmoji.Disabled)
|
||||||
|
suite.NotEmpty(dbEmoji.URI)
|
||||||
|
suite.True(*dbEmoji.VisibleInPicker)
|
||||||
|
suite.NotEmpty(dbEmoji.CategoryID)
|
||||||
|
|
||||||
|
// emoji should be in storage
|
||||||
|
entry, err := suite.storage.Storage.Stat(ctx, dbEmoji.ImagePath)
|
||||||
|
suite.NoError(err)
|
||||||
|
suite.Equal(int64(dbEmoji.ImageFileSize), entry.Size)
|
||||||
|
entry, err = suite.storage.Storage.Stat(ctx, dbEmoji.ImageStaticPath)
|
||||||
|
suite.NoError(err)
|
||||||
|
suite.Equal(int64(dbEmoji.ImageStaticFileSize), entry.Size)
|
||||||
|
}
|
||||||
|
|
||||||
func (suite *EmojiUpdateTestSuite) TestEmojiUpdateModifyRemoteEmoji() {
|
func (suite *EmojiUpdateTestSuite) TestEmojiUpdateModifyRemoteEmoji() {
|
||||||
testEmoji := >smodel.Emoji{}
|
testEmoji := >smodel.Emoji{}
|
||||||
*testEmoji = *suite.testEmojis["yell"]
|
*testEmoji = *suite.testEmojis["yell"]
|
||||||
|
@ -404,7 +489,7 @@ func (suite *EmojiUpdateTestSuite) TestEmojiUpdateModifyNoParams() {
|
||||||
defer result.Body.Close()
|
defer result.Body.Close()
|
||||||
|
|
||||||
// check the response
|
// check the response
|
||||||
b, err := ioutil.ReadAll(result.Body)
|
b, err := io.ReadAll(result.Body)
|
||||||
suite.NoError(err)
|
suite.NoError(err)
|
||||||
|
|
||||||
suite.Equal(`{"error":"Bad Request: emoji action type was 'modify' but no image or category name was provided"}`, string(b))
|
suite.Equal(`{"error":"Bad Request: emoji action type was 'modify' but no image or category name was provided"}`, string(b))
|
||||||
|
@ -438,7 +523,7 @@ func (suite *EmojiUpdateTestSuite) TestEmojiUpdateCopyLocalToLocal() {
|
||||||
defer result.Body.Close()
|
defer result.Body.Close()
|
||||||
|
|
||||||
// check the response
|
// check the response
|
||||||
b, err := ioutil.ReadAll(result.Body)
|
b, err := io.ReadAll(result.Body)
|
||||||
suite.NoError(err)
|
suite.NoError(err)
|
||||||
|
|
||||||
suite.Equal(`{"error":"Bad Request: target emoji is not remote; cannot copy to local"}`, string(b))
|
suite.Equal(`{"error":"Bad Request: target emoji is not remote; cannot copy to local"}`, string(b))
|
||||||
|
@ -472,7 +557,7 @@ func (suite *EmojiUpdateTestSuite) TestEmojiUpdateCopyEmptyShortcode() {
|
||||||
defer result.Body.Close()
|
defer result.Body.Close()
|
||||||
|
|
||||||
// check the response
|
// check the response
|
||||||
b, err := ioutil.ReadAll(result.Body)
|
b, err := io.ReadAll(result.Body)
|
||||||
suite.NoError(err)
|
suite.NoError(err)
|
||||||
|
|
||||||
suite.Equal(`{"error":"Bad Request: shortcode did not pass validation, must be between 2 and 30 characters, letters, numbers, and underscores only"}`, string(b))
|
suite.Equal(`{"error":"Bad Request: shortcode did not pass validation, must be between 2 and 30 characters, letters, numbers, and underscores only"}`, string(b))
|
||||||
|
@ -505,7 +590,7 @@ func (suite *EmojiUpdateTestSuite) TestEmojiUpdateCopyNoShortcode() {
|
||||||
defer result.Body.Close()
|
defer result.Body.Close()
|
||||||
|
|
||||||
// check the response
|
// check the response
|
||||||
b, err := ioutil.ReadAll(result.Body)
|
b, err := io.ReadAll(result.Body)
|
||||||
suite.NoError(err)
|
suite.NoError(err)
|
||||||
|
|
||||||
suite.Equal(`{"error":"Bad Request: emoji action type was 'copy' but no shortcode was provided"}`, string(b))
|
suite.Equal(`{"error":"Bad Request: emoji action type was 'copy' but no shortcode was provided"}`, string(b))
|
||||||
|
@ -539,7 +624,7 @@ func (suite *EmojiUpdateTestSuite) TestEmojiUpdateCopyShortcodeAlreadyInUse() {
|
||||||
defer result.Body.Close()
|
defer result.Body.Close()
|
||||||
|
|
||||||
// check the response
|
// check the response
|
||||||
b, err := ioutil.ReadAll(result.Body)
|
b, err := io.ReadAll(result.Body)
|
||||||
suite.NoError(err)
|
suite.NoError(err)
|
||||||
|
|
||||||
suite.Equal(`{"error":"Conflict: emoji with shortcode already exists"}`, string(b))
|
suite.Equal(`{"error":"Conflict: emoji with shortcode already exists"}`, string(b))
|
||||||
|
|
|
@ -386,7 +386,7 @@ func (suite *MediaTestSuite) TestUncacheAndRecache() {
|
||||||
testStatusAttachment,
|
testStatusAttachment,
|
||||||
testHeader,
|
testHeader,
|
||||||
} {
|
} {
|
||||||
processing := suite.manager.RecacheMedia(original, data)
|
processing := suite.manager.CacheMedia(original, data)
|
||||||
|
|
||||||
// synchronously load the recached attachment
|
// synchronously load the recached attachment
|
||||||
recachedAttachment, err := processing.Load(ctx)
|
recachedAttachment, err := processing.Load(ctx)
|
||||||
|
|
|
@ -134,11 +134,6 @@ func (d *Dereferencer) RefreshEmoji(
|
||||||
*gtsmodel.Emoji,
|
*gtsmodel.Emoji,
|
||||||
error,
|
error,
|
||||||
) {
|
) {
|
||||||
// Can't refresh local.
|
|
||||||
if emoji.IsLocal() {
|
|
||||||
return emoji, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check emoji is up-to-date
|
// Check emoji is up-to-date
|
||||||
// with provided extra info.
|
// with provided extra info.
|
||||||
switch {
|
switch {
|
||||||
|
@ -156,8 +151,18 @@ func (d *Dereferencer) RefreshEmoji(
|
||||||
force = true
|
force = true
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if needs updating.
|
// Check if needs
|
||||||
if *emoji.Cached && !force {
|
// force refresh.
|
||||||
|
if !force {
|
||||||
|
|
||||||
|
// We still want to make sure
|
||||||
|
// the emoji is cached. Simply
|
||||||
|
// check whether emoji is cached.
|
||||||
|
return d.RecacheEmoji(ctx, emoji)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Can't refresh local.
|
||||||
|
if emoji.IsLocal() {
|
||||||
return emoji, nil
|
return emoji, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -191,8 +196,8 @@ func() (*media.ProcessingEmoji, error) {
|
||||||
return tsport.DereferenceMedia(ctx, url, int64(maxsz))
|
return tsport.DereferenceMedia(ctx, url, int64(maxsz))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Refresh emoji with prepared info.
|
// Update emoji with prepared info.
|
||||||
return d.mediaManager.RefreshEmoji(ctx,
|
return d.mediaManager.UpdateEmoji(ctx,
|
||||||
emoji,
|
emoji,
|
||||||
data,
|
data,
|
||||||
info,
|
info,
|
||||||
|
@ -201,6 +206,72 @@ func() (*media.ProcessingEmoji, error) {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RecacheEmoji handles the simplest case which is that
|
||||||
|
// of an existing emoji that only needs to be recached.
|
||||||
|
// It handles the case of both local emojis, and those
|
||||||
|
// already cached as no-ops.
|
||||||
|
//
|
||||||
|
// Please note that even if an error is returned,
|
||||||
|
// an emoji model may still be returned if the error
|
||||||
|
// was only encountered during actual dereferencing.
|
||||||
|
// In this case, it will act as a placeholder.
|
||||||
|
func (d *Dereferencer) RecacheEmoji(
|
||||||
|
ctx context.Context,
|
||||||
|
emoji *gtsmodel.Emoji,
|
||||||
|
) (
|
||||||
|
*gtsmodel.Emoji,
|
||||||
|
error,
|
||||||
|
) {
|
||||||
|
// Can't recache local.
|
||||||
|
if emoji.IsLocal() {
|
||||||
|
return emoji, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if *emoji.Cached {
|
||||||
|
// Already cached.
|
||||||
|
return emoji, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get shortcode domain for locks + logging.
|
||||||
|
shortcodeDomain := emoji.ShortcodeDomain()
|
||||||
|
|
||||||
|
// Ensure we have a valid image remote URL.
|
||||||
|
url, err := url.Parse(emoji.ImageRemoteURL)
|
||||||
|
if err != nil {
|
||||||
|
err := gtserror.Newf("invalid image remote url %s for emoji %s: %w", emoji.ImageRemoteURL, shortcodeDomain, err)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pass along for safe processing.
|
||||||
|
return d.processEmojiSafely(ctx,
|
||||||
|
shortcodeDomain,
|
||||||
|
func() (*media.ProcessingEmoji, error) {
|
||||||
|
|
||||||
|
// Acquire new instance account transport for emoji dereferencing.
|
||||||
|
tsport, err := d.transportController.NewTransportForUsername(ctx, "")
|
||||||
|
if err != nil {
|
||||||
|
err := gtserror.Newf("error getting instance transport: %w", err)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get maximum supported remote emoji size.
|
||||||
|
maxsz := config.GetMediaEmojiRemoteMaxSize()
|
||||||
|
|
||||||
|
// Prepare data function to dereference remote emoji media.
|
||||||
|
data := func(context.Context) (io.ReadCloser, error) {
|
||||||
|
return tsport.DereferenceMedia(ctx, url, int64(maxsz))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recache emoji with prepared info.
|
||||||
|
return d.mediaManager.CacheEmoji(ctx,
|
||||||
|
emoji,
|
||||||
|
data,
|
||||||
|
)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
// processingEmojiSafely provides concurrency-safe processing of
|
// processingEmojiSafely provides concurrency-safe processing of
|
||||||
// an emoji with given shortcode+domain. if a copy of the emoji is
|
// an emoji with given shortcode+domain. if a copy of the emoji is
|
||||||
// not already being processed, the given 'process' callback will
|
// not already being processed, the given 'process' callback will
|
||||||
|
|
|
@ -172,7 +172,7 @@ func() (*media.ProcessingMedia, error) {
|
||||||
|
|
||||||
// Recache media with prepared info,
|
// Recache media with prepared info,
|
||||||
// this will also update media in db.
|
// this will also update media in db.
|
||||||
return d.mediaManager.RecacheMedia(
|
return d.mediaManager.CacheMedia(
|
||||||
attach,
|
attach,
|
||||||
func(ctx context.Context) (io.ReadCloser, error) {
|
func(ctx context.Context) (io.ReadCloser, error) {
|
||||||
return tsport.DereferenceMedia(ctx, url, int64(maxsz))
|
return tsport.DereferenceMedia(ctx, url, int64(maxsz))
|
||||||
|
|
|
@ -20,6 +20,7 @@
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"codeberg.org/gruf/go-iotools"
|
"codeberg.org/gruf/go-iotools"
|
||||||
|
@ -161,14 +162,14 @@ func (m *Manager) CreateMedia(
|
||||||
}
|
}
|
||||||
|
|
||||||
// Pass prepared media as ready to be cached.
|
// Pass prepared media as ready to be cached.
|
||||||
return m.RecacheMedia(attachment, data), nil
|
return m.CacheMedia(attachment, data), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// RecacheMedia wraps a media model (assumed already
|
// CacheMedia wraps a media model (assumed already
|
||||||
// inserted in the database!) with given data function
|
// inserted in the database!) with given data function
|
||||||
// to perform a blocking dereference / decode operation
|
// to perform a blocking dereference / decode operation
|
||||||
// from the data stream returned.
|
// from the data stream returned.
|
||||||
func (m *Manager) RecacheMedia(
|
func (m *Manager) CacheMedia(
|
||||||
media *gtsmodel.MediaAttachment,
|
media *gtsmodel.MediaAttachment,
|
||||||
data DataFunc,
|
data DataFunc,
|
||||||
) *ProcessingMedia {
|
) *ProcessingMedia {
|
||||||
|
@ -220,7 +221,7 @@ func (m *Manager) CreateEmoji(
|
||||||
}
|
}
|
||||||
|
|
||||||
// Finally, create new emoji.
|
// Finally, create new emoji.
|
||||||
return m.createEmoji(ctx,
|
return m.createOrUpdateEmoji(ctx,
|
||||||
m.state.DB.PutEmoji,
|
m.state.DB.PutEmoji,
|
||||||
data,
|
data,
|
||||||
emoji,
|
emoji,
|
||||||
|
@ -228,12 +229,14 @@ func (m *Manager) CreateEmoji(
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// RefreshEmoji will prepare a recache operation
|
// UpdateEmoji prepares an update operation for the given emoji,
|
||||||
// for the given emoji, updating it with extra
|
// which is assumed to already exist in the database.
|
||||||
// information, and in particular using new storage
|
//
|
||||||
// paths for the dereferenced media files to skirt
|
// Calling load on the returned *ProcessingEmoji will update the
|
||||||
// around browser caching of the old files.
|
// db entry with provided extra information, ensure emoji images
|
||||||
func (m *Manager) RefreshEmoji(
|
// are cached, and use new storage paths for the dereferenced media
|
||||||
|
// files to skirt around browser caching of the old files.
|
||||||
|
func (m *Manager) UpdateEmoji(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
emoji *gtsmodel.Emoji,
|
emoji *gtsmodel.Emoji,
|
||||||
data DataFunc,
|
data DataFunc,
|
||||||
|
@ -289,8 +292,8 @@ func (m *Manager) RefreshEmoji(
|
||||||
return rct, nil
|
return rct, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Finally, create new emoji in database.
|
// Update existing emoji in database.
|
||||||
processingEmoji, err := m.createEmoji(ctx,
|
processingEmoji, err := m.createOrUpdateEmoji(ctx,
|
||||||
func(ctx context.Context, emoji *gtsmodel.Emoji) error {
|
func(ctx context.Context, emoji *gtsmodel.Emoji) error {
|
||||||
return m.state.DB.UpdateEmoji(ctx, emoji)
|
return m.state.DB.UpdateEmoji(ctx, emoji)
|
||||||
},
|
},
|
||||||
|
@ -308,9 +311,49 @@ func(ctx context.Context, emoji *gtsmodel.Emoji) error {
|
||||||
return processingEmoji, nil
|
return processingEmoji, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Manager) createEmoji(
|
// CacheEmoji wraps an emoji model (assumed already
|
||||||
|
// inserted in the database!) with given data function
|
||||||
|
// to perform a blocking dereference / decode operation
|
||||||
|
// from the data stream returned.
|
||||||
|
func (m *Manager) CacheEmoji(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
putDB func(context.Context, *gtsmodel.Emoji) error,
|
emoji *gtsmodel.Emoji,
|
||||||
|
data DataFunc,
|
||||||
|
) (
|
||||||
|
*ProcessingEmoji,
|
||||||
|
error,
|
||||||
|
) {
|
||||||
|
// Fetch the local instance account for emoji path generation.
|
||||||
|
instanceAcc, err := m.state.DB.GetInstanceAccount(ctx, "")
|
||||||
|
if err != nil {
|
||||||
|
return nil, gtserror.Newf("error fetching instance account: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var pathID string
|
||||||
|
|
||||||
|
// Look for an emoji path ID that differs from its actual ID, this indicates
|
||||||
|
// a previous 'refresh'. We need to be sure to set this on the ProcessingEmoji{}
|
||||||
|
// so it knows to store the emoji under this path, and not default to emoji.ID.
|
||||||
|
if id := extractEmojiPathID(emoji.ImagePath); id != emoji.ID {
|
||||||
|
pathID = id
|
||||||
|
}
|
||||||
|
|
||||||
|
return &ProcessingEmoji{
|
||||||
|
newPathID: pathID,
|
||||||
|
instAccID: instanceAcc.ID,
|
||||||
|
emoji: emoji,
|
||||||
|
dataFn: data,
|
||||||
|
mgr: m,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// createOrUpdateEmoji updates the emoji according to
|
||||||
|
// provided additional data, and performs the actual
|
||||||
|
// database write, finally returning an emoji ready
|
||||||
|
// for processing (i.e. caching to local storage).
|
||||||
|
func (m *Manager) createOrUpdateEmoji(
|
||||||
|
ctx context.Context,
|
||||||
|
storeDB func(context.Context, *gtsmodel.Emoji) error,
|
||||||
data DataFunc,
|
data DataFunc,
|
||||||
emoji *gtsmodel.Emoji,
|
emoji *gtsmodel.Emoji,
|
||||||
info AdditionalEmojiInfo,
|
info AdditionalEmojiInfo,
|
||||||
|
@ -351,8 +394,8 @@ func (m *Manager) createEmoji(
|
||||||
emoji.CategoryID = *info.CategoryID
|
emoji.CategoryID = *info.CategoryID
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store emoji in database in initial form.
|
// Put or update emoji in database.
|
||||||
if err := putDB(ctx, emoji); err != nil {
|
if err := storeDB(ctx, emoji); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -367,17 +410,26 @@ func (m *Manager) createEmoji(
|
||||||
return processingEmoji, nil
|
return processingEmoji, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// RecacheEmoji wraps an emoji model (assumed already
|
// extractEmojiPathID pulls the ID used in the final path segment of an emoji path (can be URL).
|
||||||
// inserted in the database!) with given data function
|
func extractEmojiPathID(path string) string {
|
||||||
// to perform a blocking dereference / decode operation
|
// Look for '.' indicating file ext.
|
||||||
// from the data stream returned.
|
i := strings.LastIndexByte(path, '.')
|
||||||
func (m *Manager) RecacheEmoji(
|
if i == -1 {
|
||||||
emoji *gtsmodel.Emoji,
|
return ""
|
||||||
data DataFunc,
|
|
||||||
) *ProcessingEmoji {
|
|
||||||
return &ProcessingEmoji{
|
|
||||||
emoji: emoji,
|
|
||||||
dataFn: data,
|
|
||||||
mgr: m,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Strip ext.
|
||||||
|
path = path[:i]
|
||||||
|
|
||||||
|
// Look for '/' of final path sep.
|
||||||
|
i = strings.LastIndexByte(path, '/')
|
||||||
|
if i == -1 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// Strip up to
|
||||||
|
// final segment.
|
||||||
|
path = path[i+1:]
|
||||||
|
|
||||||
|
return path
|
||||||
}
|
}
|
||||||
|
|
|
@ -105,7 +105,7 @@ func (suite *ManagerTestSuite) TestEmojiProcessRefresh() {
|
||||||
return io.NopCloser(bytes.NewBuffer(b)), nil
|
return io.NopCloser(bytes.NewBuffer(b)), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
processing, err := suite.manager.RefreshEmoji(ctx,
|
processing, err := suite.manager.UpdateEmoji(ctx,
|
||||||
emojiToUpdate,
|
emojiToUpdate,
|
||||||
data,
|
data,
|
||||||
media.AdditionalEmojiInfo{
|
media.AdditionalEmojiInfo{
|
||||||
|
|
|
@ -115,7 +115,7 @@ func (m *Manager) RefetchEmojis(ctx context.Context, domain string, dereferenceM
|
||||||
return dereferenceMedia(ctx, emojiImageIRI, int64(maxsz))
|
return dereferenceMedia(ctx, emojiImageIRI, int64(maxsz))
|
||||||
}
|
}
|
||||||
|
|
||||||
processingEmoji, err := m.RefreshEmoji(ctx, emoji, dataFunc, AdditionalEmojiInfo{
|
processingEmoji, err := m.UpdateEmoji(ctx, emoji, dataFunc, AdditionalEmojiInfo{
|
||||||
Domain: &emoji.Domain,
|
Domain: &emoji.Domain,
|
||||||
ImageRemoteURL: &emoji.ImageRemoteURL,
|
ImageRemoteURL: &emoji.ImageRemoteURL,
|
||||||
ImageStaticRemoteURL: &emoji.ImageStaticRemoteURL,
|
ImageStaticRemoteURL: &emoji.ImageStaticRemoteURL,
|
||||||
|
@ -123,16 +123,16 @@ func (m *Manager) RefetchEmojis(ctx context.Context, domain string, dereferenceM
|
||||||
VisibleInPicker: emoji.VisibleInPicker,
|
VisibleInPicker: emoji.VisibleInPicker,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf(ctx, "emoji %s could not be refreshed because of an error during processing: %s", shortcodeDomain, err)
|
log.Errorf(ctx, "emoji %s could not be updated because of an error during processing: %s", shortcodeDomain, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err := processingEmoji.Load(ctx); err != nil {
|
if _, err := processingEmoji.Load(ctx); err != nil {
|
||||||
log.Errorf(ctx, "emoji %s could not be refreshed because of an error during loading: %s", shortcodeDomain, err)
|
log.Errorf(ctx, "emoji %s could not be updated because of an error during loading: %s", shortcodeDomain, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Tracef(ctx, "refetched emoji %s successfully from remote", shortcodeDomain)
|
log.Tracef(ctx, "refetched + updated emoji %s successfully from remote", shortcodeDomain)
|
||||||
totalRefetched++
|
totalRefetched++
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -291,13 +291,8 @@ func (p *Processor) emojiUpdateCopy(
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ensure target emoji is locally cached.
|
// Ensure target emoji is locally cached.
|
||||||
target, err := p.federator.RefreshEmoji(
|
target, err := p.federator.RecacheEmoji(ctx,
|
||||||
ctx,
|
|
||||||
target,
|
target,
|
||||||
|
|
||||||
// no changes we want to make.
|
|
||||||
media.AdditionalEmojiInfo{},
|
|
||||||
false,
|
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err := gtserror.Newf("error recaching emoji %s: %w", target.ImageRemoteURL, err)
|
err := gtserror.Newf("error recaching emoji %s: %w", target.ImageRemoteURL, err)
|
||||||
|
@ -325,8 +320,8 @@ func (p *Processor) emojiUpdateCopy(
|
||||||
|
|
||||||
// Attempt to create the new local emoji.
|
// Attempt to create the new local emoji.
|
||||||
emoji, errWithCode := p.createEmoji(ctx,
|
emoji, errWithCode := p.createEmoji(ctx,
|
||||||
util.PtrOrValue(shortcode, ""),
|
util.PtrOrZero(shortcode),
|
||||||
util.PtrOrValue(categoryName, ""),
|
util.PtrOrZero(categoryName),
|
||||||
data,
|
data,
|
||||||
)
|
)
|
||||||
if errWithCode != nil {
|
if errWithCode != nil {
|
||||||
|
@ -401,35 +396,39 @@ func (p *Processor) emojiUpdateModify(
|
||||||
return nil, gtserror.NewErrorBadRequest(errors.New(text), text)
|
return nil, gtserror.NewErrorBadRequest(errors.New(text), text)
|
||||||
}
|
}
|
||||||
|
|
||||||
if categoryName != nil {
|
// Check if we need to
|
||||||
if *categoryName != "" {
|
// set a new category ID.
|
||||||
// A category was provided, get / create relevant emoji category.
|
var newCategoryID *string
|
||||||
|
switch {
|
||||||
|
case categoryName == nil:
|
||||||
|
// No changes.
|
||||||
|
|
||||||
|
case *categoryName == "":
|
||||||
|
// Emoji category was unset.
|
||||||
|
newCategoryID = util.Ptr("")
|
||||||
|
emoji.CategoryID = ""
|
||||||
|
emoji.Category = nil
|
||||||
|
|
||||||
|
case *categoryName != "":
|
||||||
|
// A category was provided, get or create relevant emoji category.
|
||||||
category, errWithCode := p.mustGetEmojiCategory(ctx, *categoryName)
|
category, errWithCode := p.mustGetEmojiCategory(ctx, *categoryName)
|
||||||
if errWithCode != nil {
|
if errWithCode != nil {
|
||||||
return nil, errWithCode
|
return nil, errWithCode
|
||||||
}
|
}
|
||||||
|
|
||||||
if category.ID == emoji.CategoryID {
|
// Update emoji category if
|
||||||
// There was no change,
|
// it's different from before.
|
||||||
// indicate this by unsetting
|
if category.ID != emoji.CategoryID {
|
||||||
// the category name pointer.
|
newCategoryID = &category.ID
|
||||||
categoryName = nil
|
|
||||||
} else {
|
|
||||||
// Update emoji category.
|
|
||||||
emoji.CategoryID = category.ID
|
emoji.CategoryID = category.ID
|
||||||
emoji.Category = category
|
emoji.Category = category
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
// Emoji category was unset.
|
|
||||||
emoji.CategoryID = ""
|
|
||||||
emoji.Category = nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check whether any image changes were requested.
|
// Check whether any image changes were requested.
|
||||||
imageUpdated := (image != nil && image.Size > 0)
|
imageUpdated := (image != nil && image.Size > 0)
|
||||||
|
|
||||||
if !imageUpdated && categoryName != nil {
|
if !imageUpdated && newCategoryID != nil {
|
||||||
// Only updating category; only a single database update required.
|
// Only updating category; only a single database update required.
|
||||||
if err := p.state.DB.UpdateEmoji(ctx, emoji, "category_id"); err != nil {
|
if err := p.state.DB.UpdateEmoji(ctx, emoji, "category_id"); err != nil {
|
||||||
err := gtserror.Newf("error updating emoji in db: %w", err)
|
err := gtserror.Newf("error updating emoji in db: %w", err)
|
||||||
|
@ -463,8 +462,17 @@ func (p *Processor) emojiUpdateModify(
|
||||||
return rc, nil
|
return rc, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prepare emoji model for recache from new data.
|
// Include category ID
|
||||||
processing := p.media.RecacheEmoji(emoji, data)
|
// update if necessary.
|
||||||
|
ai := media.AdditionalEmojiInfo{}
|
||||||
|
ai.CategoryID = newCategoryID
|
||||||
|
|
||||||
|
// Prepare emoji model for update+recache from new data.
|
||||||
|
processing, err := p.media.UpdateEmoji(ctx, emoji, data, ai)
|
||||||
|
if err != nil {
|
||||||
|
err := gtserror.Newf("error preparing recache: %w", err)
|
||||||
|
return nil, gtserror.NewErrorInternalError(err)
|
||||||
|
}
|
||||||
|
|
||||||
// Load to trigger update + write.
|
// Load to trigger update + write.
|
||||||
emoji, err = processing.Load(ctx)
|
emoji, err = processing.Load(ctx)
|
||||||
|
|
|
@ -246,11 +246,9 @@ func (p *Processor) getEmojiContent(
|
||||||
|
|
||||||
// Ensure that stored emoji is cached.
|
// Ensure that stored emoji is cached.
|
||||||
// (this handles local emoji / recaches).
|
// (this handles local emoji / recaches).
|
||||||
emoji, err = p.federator.RefreshEmoji(
|
emoji, err = p.federator.RecacheEmoji(
|
||||||
ctx,
|
ctx,
|
||||||
emoji,
|
emoji,
|
||||||
media.AdditionalEmojiInfo{},
|
|
||||||
false,
|
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err := gtserror.Newf("error recaching emoji: %w", err)
|
err := gtserror.Newf("error recaching emoji: %w", err)
|
||||||
|
|
|
@ -29,7 +29,7 @@
|
||||||
|
|
||||||
// NewInMemoryStorage returns a new in memory storage with the default test config
|
// NewInMemoryStorage returns a new in memory storage with the default test config
|
||||||
func NewInMemoryStorage() *gtsstorage.Driver {
|
func NewInMemoryStorage() *gtsstorage.Driver {
|
||||||
storage := memory.Open(200, false)
|
storage := memory.Open(200, true)
|
||||||
return >sstorage.Driver{
|
return >sstorage.Driver{
|
||||||
Storage: storage,
|
Storage: storage,
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue