[performance] media processing improvements (#1288)

* media processor consolidation and reformatting, reduce amount of required syscalls

Signed-off-by: kim <grufwub@gmail.com>

* update go-store library, stream jpeg/png encoding + use buffer pools, improved media processing AlreadyExists error handling

Signed-off-by: kim <grufwub@gmail.com>

* fix duration not being set, fix mp4 test expecting error

Signed-off-by: kim <grufwub@gmail.com>

* fix test expecting media files with different extension

Signed-off-by: kim <grufwub@gmail.com>

* remove unused code

Signed-off-by: kim <grufwub@gmail.com>

* fix expected storage paths in tests, update expected test thumbnails

Signed-off-by: kim <grufwub@gmail.com>

* remove dead code

Signed-off-by: kim <grufwub@gmail.com>

* fix cached presigned s3 url fetching

Signed-off-by: kim <grufwub@gmail.com>

* fix tests

Signed-off-by: kim <grufwub@gmail.com>

* fix test models

Signed-off-by: kim <grufwub@gmail.com>

* update media processing to use sync.Once{} for concurrency protection

Signed-off-by: kim <grufwub@gmail.com>

* shutup linter

Signed-off-by: kim <grufwub@gmail.com>

* fix passing in KVStore GetStream() as stream to PutStream()

Signed-off-by: kim <grufwub@gmail.com>

* fix unlocks of storage keys

Signed-off-by: kim <grufwub@gmail.com>

* whoops, return the error...

Signed-off-by: kim <grufwub@gmail.com>

* pour one out for tobi's code <3

Signed-off-by: kim <grufwub@gmail.com>

* add back the byte slurping code

Signed-off-by: kim <grufwub@gmail.com>

* check for both ErrUnexpectedEOF and EOF

Signed-off-by: kim <grufwub@gmail.com>

* add back links to file format header information

Signed-off-by: kim <grufwub@gmail.com>

Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim 2023-01-11 11:13:13 +00:00 committed by GitHub
parent 3512325e46
commit 5318054808
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
64 changed files with 1279 additions and 1405 deletions

7
go.mod
View file

@ -10,9 +10,9 @@ require (
codeberg.org/gruf/go-errors/v2 v2.0.2
codeberg.org/gruf/go-kv v1.5.2
codeberg.org/gruf/go-logger/v2 v2.2.1
codeberg.org/gruf/go-mutexes v1.1.4
codeberg.org/gruf/go-mutexes v1.1.5
codeberg.org/gruf/go-runners v1.4.0
codeberg.org/gruf/go-store/v2 v2.0.10
codeberg.org/gruf/go-store/v2 v2.2.1
github.com/abema/go-mp4 v0.9.0
github.com/buckket/go-blurhash v1.1.0
github.com/coreos/go-oidc/v3 v3.5.0
@ -65,10 +65,11 @@ require (
codeberg.org/gruf/go-atomics v1.1.0 // indirect
codeberg.org/gruf/go-bitutil v1.0.1 // indirect
codeberg.org/gruf/go-bytes v1.0.2 // indirect
codeberg.org/gruf/go-fastcopy v1.1.1 // indirect
codeberg.org/gruf/go-fastcopy v1.1.2 // indirect
codeberg.org/gruf/go-fastpath v1.0.3 // indirect
codeberg.org/gruf/go-fastpath/v2 v2.0.0 // indirect
codeberg.org/gruf/go-hashenc v1.0.2 // indirect
codeberg.org/gruf/go-iotools v0.0.0-20221224124424-3386841cb225 // indirect
codeberg.org/gruf/go-mangler v1.2.2 // indirect
codeberg.org/gruf/go-maps v1.0.3 // indirect
codeberg.org/gruf/go-pools v1.1.0 // indirect

15
go.sum
View file

@ -56,8 +56,8 @@ codeberg.org/gruf/go-debug v1.2.0/go.mod h1:N+vSy9uJBQgpQcJUqjctvqFz7tBHJf+S/PIj
codeberg.org/gruf/go-errors/v2 v2.0.0/go.mod h1:ZRhbdhvgoUA3Yw6e56kd9Ox984RrvbEFC2pOXyHDJP4=
codeberg.org/gruf/go-errors/v2 v2.0.2 h1:T9CqfC+ntSIQL5mdQxwHlUMod1htpgNe3P1tugxKlT4=
codeberg.org/gruf/go-errors/v2 v2.0.2/go.mod h1:6sI75OmvXE2AtRm4WUyGMEyqEOKTsfe+CA+aBXwbtJY=
codeberg.org/gruf/go-fastcopy v1.1.1 h1:HhPCeFdVR5pwiSVDnQEGJ+J2ny9b5QgfiESc0zrWQAY=
codeberg.org/gruf/go-fastcopy v1.1.1/go.mod h1:GDDYR0Cnb3U/AIfGM3983V/L+GN+vuwVMvrmVABo21s=
codeberg.org/gruf/go-fastcopy v1.1.2 h1:YwmYXPsyOcRBxKEE2+w1bGAZfclHVaPijFsOVOcnNcw=
codeberg.org/gruf/go-fastcopy v1.1.2/go.mod h1:GDDYR0Cnb3U/AIfGM3983V/L+GN+vuwVMvrmVABo21s=
codeberg.org/gruf/go-fastpath v1.0.1/go.mod h1:edveE/Kp3Eqi0JJm0lXYdkVrB28cNUkcb/bRGFTPqeI=
codeberg.org/gruf/go-fastpath v1.0.3 h1:3Iftz9Z2suCEgTLkQMucew+2+4Oe46JPbAM2JEhnjTU=
codeberg.org/gruf/go-fastpath v1.0.3/go.mod h1:edveE/Kp3Eqi0JJm0lXYdkVrB28cNUkcb/bRGFTPqeI=
@ -65,6 +65,8 @@ codeberg.org/gruf/go-fastpath/v2 v2.0.0 h1:iAS9GZahFhyWEH0KLhFEJR+txx1ZhMXxYzu2q
codeberg.org/gruf/go-fastpath/v2 v2.0.0/go.mod h1:3pPqu5nZjpbRrOqvLyAK7puS1OfEtQvjd6342Cwz56Q=
codeberg.org/gruf/go-hashenc v1.0.2 h1:U3jH6zMXZiL96czD/qaJd8OR2h7LlBzGv/2WxnMHI/g=
codeberg.org/gruf/go-hashenc v1.0.2/go.mod h1:eK+A8clLcEN/m1nftNsRId0kfYDQnETnuIfBGZ8Gvsg=
codeberg.org/gruf/go-iotools v0.0.0-20221224124424-3386841cb225 h1:tP9YvEBfADGG3mXkfrALLadlcbrZsFsWKZvFtUZtrt8=
codeberg.org/gruf/go-iotools v0.0.0-20221224124424-3386841cb225/go.mod h1:B8uq4yHtIcKXhBZT9C/SYisz25lldLHMVpwZPz4ADLQ=
codeberg.org/gruf/go-kv v1.5.2 h1:B0RkAXLUXYn3Za1NzTXOcUvAc+JUC2ZadTMkCUDa0mc=
codeberg.org/gruf/go-kv v1.5.2/go.mod h1:al6ASW/2CbGqz2YcM8B00tvWnVi1bU1CH3HYs5tZxo4=
codeberg.org/gruf/go-logger/v2 v2.2.1 h1:RP2u059EQKTBFV3cN8X6xDxNk2RkzqdgXGKflKqB7Oc=
@ -73,16 +75,16 @@ codeberg.org/gruf/go-mangler v1.2.2 h1:fisdWXa6dW4p1uYdbz5Of3R4lDDFPuRqKavGI9O03
codeberg.org/gruf/go-mangler v1.2.2/go.mod h1:X/7URkFhLBAVKkTxmqF11Oxw3A6pSSxgPeHssQaiq28=
codeberg.org/gruf/go-maps v1.0.3 h1:VDwhnnaVNUIy5O93CvkcE2IZXnMB1+IJjzfop9V12es=
codeberg.org/gruf/go-maps v1.0.3/go.mod h1:D5LNDxlC9rsDuVQVM6JObaVGAdHB6g2dTdOdkh1aXWA=
codeberg.org/gruf/go-mutexes v1.1.4 h1:HWaIZavPL92SBJxNOlIXAmAT5CB2hAs72/lBN31jnzM=
codeberg.org/gruf/go-mutexes v1.1.4/go.mod h1:1j/6/MBeBQUedAtAtysLLnBKogfOZAxdym0E3wlaBD8=
codeberg.org/gruf/go-mutexes v1.1.5 h1:8Y8DwCGf24MyzOSaPvLrtk/B4ecVx4z+fppL6dY+PG8=
codeberg.org/gruf/go-mutexes v1.1.5/go.mod h1:1j/6/MBeBQUedAtAtysLLnBKogfOZAxdym0E3wlaBD8=
codeberg.org/gruf/go-pools v1.1.0 h1:LbYP24eQLl/YI1fSU2pafiwhGol1Z1zPjRrMsXpF88s=
codeberg.org/gruf/go-pools v1.1.0/go.mod h1:ZMYpt/DjQWYC3zFD3T97QWSFKs62zAUGJ/tzvgB9D68=
codeberg.org/gruf/go-runners v1.4.0 h1:977nVjigAdH95+VAB/a6tyBJOKk99e60h+mfHzBs/n8=
codeberg.org/gruf/go-runners v1.4.0/go.mod h1:kUM6GYL7dC+f9Sc/XuwdvB/mB4FuI4fJFb150ADMsmw=
codeberg.org/gruf/go-sched v1.2.0 h1:utZl/7srVcbh30rFw42LC2/cMtak4UZRxtIOt/5riNA=
codeberg.org/gruf/go-sched v1.2.0/go.mod h1:v4ueWq+fAtAw9JYt4aFXvadI1YoOqofgHQgszRYuslA=
codeberg.org/gruf/go-store/v2 v2.0.10 h1:/2iZ4j29A//EhM3XziJP6SxtdIcaAyPmJEv31+6XD8g=
codeberg.org/gruf/go-store/v2 v2.0.10/go.mod h1:KMRE173S6W2sGhuIa4jY/OPIO65F9++7rmWTfZ4xTeY=
codeberg.org/gruf/go-store/v2 v2.2.1 h1:lbvMjhMLebefiaPNLtWvPySKSYM5xN1aztSxxz+vCzU=
codeberg.org/gruf/go-store/v2 v2.2.1/go.mod h1:pxdyfSzau8fFs1TfZlyRzhDYvZWLaj1sXpcjXpzBB6k=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@ -818,6 +820,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

View file

@ -300,8 +300,8 @@ func (suite *AccountUpdateTestSuite) TestAccountUpdateCredentialsPATCHHandlerWit
suite.NotEmpty(apimodelAccount.HeaderStatic)
// should be different from the values set before
suite.NotEqual("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg", apimodelAccount.Header)
suite.NotEqual("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg", apimodelAccount.HeaderStatic)
suite.NotEqual("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg", apimodelAccount.Header)
suite.NotEqual("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg", apimodelAccount.HeaderStatic)
}
func (suite *AccountUpdateTestSuite) TestAccountUpdateCredentialsPATCHHandlerEmptyForm() {

View file

@ -74,10 +74,10 @@ func (suite *AccountVerifyTestSuite) TestAccountVerifyGet() {
suite.Equal(*testAccount.Bot, apimodelAccount.Bot)
suite.WithinDuration(testAccount.CreatedAt, createdAt, 30*time.Second) // we lose a bit of accuracy serializing so fuzz this a bit
suite.Equal(testAccount.URL, apimodelAccount.URL)
suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg", apimodelAccount.Avatar)
suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg", apimodelAccount.AvatarStatic)
suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg", apimodelAccount.Header)
suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg", apimodelAccount.HeaderStatic)
suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg", apimodelAccount.Avatar)
suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg", apimodelAccount.AvatarStatic)
suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg", apimodelAccount.Header)
suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg", apimodelAccount.HeaderStatic)
suite.Equal(2, apimodelAccount.FollowersCount)
suite.Equal(2, apimodelAccount.FollowingCount)
suite.Equal(5, apimodelAccount.StatusesCount)

View file

@ -117,14 +117,19 @@ func (m *Module) ServeFile(c *gin.Context) {
return
}
// try to slurp the first few bytes to make sure we have something
b := bytes.NewBuffer(make([]byte, 0, 64))
if _, err := io.CopyN(b, content.Content, 64); err != nil {
// create a "slurp" buffer ;)
b := make([]byte, 64)
// Try read the first 64 bytes into memory, to try return a more useful "not found" error.
if _, err := io.ReadFull(content.Content, b); err != nil &&
(err != io.ErrUnexpectedEOF && err != io.EOF) {
err = fmt.Errorf("ServeFile: error reading from content: %w", err)
apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err, err.Error()), m.processor.InstanceGet)
return
}
// we're good, return the slurped bytes + the rest of the content
c.DataFromReader(http.StatusOK, content.ContentLength, format, io.MultiReader(b, content.Content), nil)
c.DataFromReader(http.StatusOK, content.ContentLength, format, io.MultiReader(
bytes.NewReader(b), content.Content,
), nil)
}

View file

@ -99,7 +99,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalLocalFileOK() {
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeOriginal,
targetAttachment.ID+".jpeg",
targetAttachment.ID+".jpg",
)
suite.Equal(http.StatusOK, code)
@ -119,7 +119,7 @@ func (suite *ServeFileTestSuite) TestServeSmallLocalFileOK() {
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeSmall,
targetAttachment.ID+".jpeg",
targetAttachment.ID+".jpg",
)
suite.Equal(http.StatusOK, code)
@ -139,7 +139,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileOK() {
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeOriginal,
targetAttachment.ID+".jpeg",
targetAttachment.ID+".jpg",
)
suite.Equal(http.StatusOK, code)
@ -159,7 +159,7 @@ func (suite *ServeFileTestSuite) TestServeSmallRemoteFileOK() {
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeSmall,
targetAttachment.ID+".jpeg",
targetAttachment.ID+".jpg",
)
suite.Equal(http.StatusOK, code)
@ -182,7 +182,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileRecache() {
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeOriginal,
targetAttachment.ID+".jpeg",
targetAttachment.ID+".jpg",
)
suite.Equal(http.StatusOK, code)
@ -205,7 +205,7 @@ func (suite *ServeFileTestSuite) TestServeSmallRemoteFileRecache() {
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeSmall,
targetAttachment.ID+".jpeg",
targetAttachment.ID+".jpg",
)
suite.Equal(http.StatusOK, code)
@ -228,7 +228,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileRecacheNotFound() {
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeOriginal,
targetAttachment.ID+".jpeg",
targetAttachment.ID+".jpg",
)
suite.Equal(http.StatusNotFound, code)
@ -249,7 +249,7 @@ func (suite *ServeFileTestSuite) TestServeSmallRemoteFileRecacheNotFound() {
targetAttachment.AccountID,
media.TypeAttachment,
media.SizeSmall,
targetAttachment.ID+".jpeg",
targetAttachment.ID+".jpg",
)
suite.Equal(http.StatusNotFound, code)
@ -261,7 +261,7 @@ func (suite *ServeFileTestSuite) TestServeFileNotFound() {
"01GMMY4G9B0QEG0PQK5Q5JGJWZ",
media.TypeAttachment,
media.SizeOriginal,
"01GMMY68Y7E5DJ3CA3Y9SS8524.jpeg",
"01GMMY68Y7E5DJ3CA3Y9SS8524.jpg",
)
suite.Equal(http.StatusNotFound, code)

View file

@ -21,11 +21,11 @@
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type AttachmentTestSuite struct {
@ -42,7 +42,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentBlocking() {
attachmentContentType := "image/jpeg"
attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg"
attachmentDescription := "It's a cute plushie."
attachmentBlurhash := "LwP?p=aK_4%N%MRjWXt7%hozM_a}"
attachmentBlurhash := "LtQ9yKi__4%g%MRjWCt7%hozM_az"
media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalMediaInfo{
StatusID: &attachmentStatus,
@ -116,7 +116,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentAsync() {
attachmentContentType := "image/jpeg"
attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg"
attachmentDescription := "It's a cute plushie."
attachmentBlurhash := "LwP?p=aK_4%N%MRjWXt7%hozM_a}"
attachmentBlurhash := "LtQ9yKi__4%g%MRjWCt7%hozM_az"
processingMedia, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalMediaInfo{
StatusID: &attachmentStatus,
@ -127,11 +127,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentAsync() {
suite.NoError(err)
attachmentID := processingMedia.AttachmentID()
if !testrig.WaitFor(func() bool {
return processingMedia.Finished()
}) {
suite.FailNow("timed out waiting for media to be processed")
}
time.Sleep(time.Second * 3)
// now get the attachment from the database
attachment, err := suite.db.GetAttachmentByID(ctx, attachmentID)

View file

@ -119,3 +119,41 @@ func (w *SilentWriter) Write(b []byte) (int, error) {
func (w *SilentWriter) Error() error {
return w.err
}
func StreamReadFunc(read func(io.Reader) error) io.Writer {
// In-memory stream.
pr, pw := io.Pipe()
go func() {
var err error
defer func() {
// Always pass along error.
pr.CloseWithError(err)
}()
// Start reading.
err = read(pr)
}()
return pw
}
func StreamWriteFunc(write func(io.Writer) error) io.Reader {
// In-memory stream.
pr, pw := io.Pipe()
go func() {
var err error
defer func() {
// Always pass along error.
pw.CloseWithError(err)
}()
// Start writing.
err = write(pw)
}()
return pr
}

View file

@ -19,182 +19,167 @@
package media
import (
"bytes"
"errors"
"fmt"
"bufio"
"image"
"image/gif"
"image/color"
"image/draw"
"image/jpeg"
"image/png"
"io"
"sync"
"github.com/buckket/go-blurhash"
"github.com/disintegration/imaging"
_ "golang.org/x/image/webp" // blank import to support WebP decoding
"github.com/superseriousbusiness/gotosocial/internal/iotools"
// import to init webp encode/decoding.
_ "golang.org/x/image/webp"
)
const (
thumbnailMaxWidth = 512
thumbnailMaxHeight = 512
var (
// pngEncoder provides our global PNG encoding with
// specified compression level, and memory pooled buffers.
pngEncoder = png.Encoder{
CompressionLevel: png.DefaultCompression,
BufferPool: &pngEncoderBufferPool{},
}
// jpegBufferPool is a memory pool of byte buffers for JPEG encoding.
jpegBufferPool = sync.Pool{
New: func() any {
return bufio.NewWriter(nil)
},
}
)
func decodeGif(r io.Reader) (*mediaMeta, error) {
gif, err := gif.DecodeAll(r)
// gtsImage is a thin wrapper around the standard library image
// interface to provide our own useful helper functions for image
// size and aspect ratio calculations, streamed encoding to various
// types, and creating reduced size thumbnail images.
type gtsImage struct{ image image.Image }
// blankImage generates a blank image of given dimensions.
func blankImage(width int, height int) *gtsImage {
// create a rectangle with the same dimensions as the video
img := image.NewRGBA(image.Rect(0, 0, width, height))
// fill the rectangle with our desired fill color.
draw.Draw(img, img.Bounds(), &image.Uniform{
color.RGBA{42, 43, 47, 0},
}, image.Point{}, draw.Src)
return &gtsImage{image: img}
}
// decodeImage will decode image from reader stream and return image wrapped in our own gtsImage{} type.
func decodeImage(r io.Reader, opts ...imaging.DecodeOption) (*gtsImage, error) {
img, err := imaging.Decode(r, opts...)
if err != nil {
return nil, err
}
// use the first frame to get the static characteristics
width := gif.Config.Width
height := gif.Config.Height
size := width * height
aspect := float32(width) / float32(height)
return &mediaMeta{
width: width,
height: height,
size: size,
aspect: aspect,
}, nil
return &gtsImage{image: img}, nil
}
func decodeImage(r io.Reader, contentType string) (*mediaMeta, error) {
var i image.Image
var err error
switch contentType {
case mimeImageJpeg, mimeImageWebp:
i, err = imaging.Decode(r, imaging.AutoOrientation(true))
case mimeImagePng:
strippedPngReader := io.Reader(&PNGAncillaryChunkStripper{
Reader: r,
})
i, err = imaging.Decode(strippedPngReader, imaging.AutoOrientation(true))
default:
err = fmt.Errorf("content type %s not recognised", contentType)
}
if err != nil {
return nil, err
}
if i == nil {
return nil, errors.New("processed image was nil")
}
width := i.Bounds().Size().X
height := i.Bounds().Size().Y
size := width * height
aspect := float32(width) / float32(height)
return &mediaMeta{
width: width,
height: height,
size: size,
aspect: aspect,
}, nil
// Width returns the image width in pixels.
func (m *gtsImage) Width() uint32 {
return uint32(m.image.Bounds().Size().X)
}
// deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png.
func deriveStaticEmoji(r io.Reader, contentType string) (*mediaMeta, error) {
var i image.Image
var err error
switch contentType {
case mimeImagePng:
i, err = StrippedPngDecode(r)
if err != nil {
return nil, err
}
case mimeImageGif:
i, err = gif.Decode(r)
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("content type %s not allowed for emoji", contentType)
}
out := &bytes.Buffer{}
if err := png.Encode(out, i); err != nil {
return nil, err
}
return &mediaMeta{
small: out.Bytes(),
}, nil
// Height returns the image height in pixels.
func (m *gtsImage) Height() uint32 {
return uint32(m.image.Bounds().Size().Y)
}
// deriveThumbnailFromImage returns a byte slice and metadata for a thumbnail
// of a given piece of media, or an error if something goes wrong.
//
// If createBlurhash is true, then a blurhash will also be generated from a tiny
// version of the image. This costs precious CPU cycles, so only use it if you
// really need a blurhash and don't have one already.
//
// If createBlurhash is false, then the blurhash field on the returned ImageAndMeta
// will be an empty string.
func deriveThumbnailFromImage(r io.Reader, contentType string, createBlurhash bool) (*mediaMeta, error) {
var i image.Image
var err error
switch contentType {
case mimeImageJpeg, mimeImageGif, mimeImageWebp:
i, err = imaging.Decode(r, imaging.AutoOrientation(true))
case mimeImagePng:
strippedPngReader := io.Reader(&PNGAncillaryChunkStripper{
Reader: r,
})
i, err = imaging.Decode(strippedPngReader, imaging.AutoOrientation(true))
default:
err = fmt.Errorf("content type %s can't be thumbnailed as an image", contentType)
}
if err != nil {
return nil, fmt.Errorf("error decoding %s: %s", contentType, err)
}
originalX := i.Bounds().Size().X
originalY := i.Bounds().Size().Y
var thumb image.Image
if originalX <= thumbnailMaxWidth && originalY <= thumbnailMaxHeight {
// it's already small, no need to resize
thumb = i
} else {
thumb = imaging.Fit(i, thumbnailMaxWidth, thumbnailMaxHeight, imaging.Linear)
}
thumbX := thumb.Bounds().Size().X
thumbY := thumb.Bounds().Size().Y
size := thumbX * thumbY
aspect := float32(thumbX) / float32(thumbY)
im := &mediaMeta{
width: thumbX,
height: thumbY,
size: size,
aspect: aspect,
}
if createBlurhash {
// for generating blurhashes, it's more cost effective to lose detail rather than
// pass a big image into the blurhash algorithm, so make a teeny tiny version
tiny := imaging.Resize(thumb, 32, 0, imaging.NearestNeighbor)
bh, err := blurhash.Encode(4, 3, tiny)
if err != nil {
return nil, fmt.Errorf("error creating blurhash: %s", err)
}
im.blurhash = bh
}
out := &bytes.Buffer{}
if err := jpeg.Encode(out, thumb, &jpeg.Options{
// Quality isn't extremely important for thumbnails, so 75 is "good enough"
Quality: 75,
}); err != nil {
return nil, fmt.Errorf("error encoding thumbnail: %s", err)
}
im.small = out.Bytes()
return im, nil
// Size returns the total number of image pixels.
func (m *gtsImage) Size() uint64 {
return uint64(m.image.Bounds().Size().X) *
uint64(m.image.Bounds().Size().Y)
}
// AspectRatio returns the image ratio of width:height.
func (m *gtsImage) AspectRatio() float32 {
return float32(m.image.Bounds().Size().X) /
float32(m.image.Bounds().Size().Y)
}
// Thumbnail returns a small sized copy of gtsImage{}, limited to 512x512 if not small enough.
func (m *gtsImage) Thumbnail() *gtsImage {
const (
// max thumb
// dimensions.
maxWidth = 512
maxHeight = 512
)
// Check the receiving image is within max thumnail bounds.
if m.Width() <= maxWidth && m.Height() <= maxHeight {
return &gtsImage{image: imaging.Clone(m.image)}
}
// Image is too large, needs to be resized to thumbnail max.
img := imaging.Fit(m.image, maxWidth, maxHeight, imaging.Linear)
return &gtsImage{image: img}
}
// Blurhash calculates the blurhash for the receiving image data.
func (m *gtsImage) Blurhash() (string, error) {
// for generating blurhashes, it's more cost effective to
// lose detail since it's blurry, so make a tiny version.
tiny := imaging.Resize(m.image, 32, 0, imaging.NearestNeighbor)
// Encode blurhash from resized version
return blurhash.Encode(4, 3, tiny)
}
// ToJPEG creates a new streaming JPEG encoder from receiving image, and a size ptr
// which stores the number of bytes written during the image encoding process.
func (m *gtsImage) ToJPEG(opts *jpeg.Options) io.Reader {
return iotools.StreamWriteFunc(func(w io.Writer) error {
// Get encoding buffer
bw := getJPEGBuffer(w)
// Encode JPEG to buffered writer.
err := jpeg.Encode(bw, m.image, opts)
// Replace buffer.
//
// NOTE: jpeg.Encode() already
// performs a bufio.Writer.Flush().
putJPEGBuffer(bw)
return err
})
}
// ToPNG creates a new streaming PNG encoder from receiving image, and a size ptr
// which stores the number of bytes written during the image encoding process.
func (m *gtsImage) ToPNG() io.Reader {
return iotools.StreamWriteFunc(func(w io.Writer) error {
return pngEncoder.Encode(w, m.image)
})
}
// getJPEGBuffer fetches a reset JPEG encoding buffer from global JPEG buffer pool.
func getJPEGBuffer(w io.Writer) *bufio.Writer {
buf, _ := jpegBufferPool.Get().(*bufio.Writer)
buf.Reset(w)
return buf
}
// putJPEGBuffer resets the given bufio writer and places in global JPEG buffer pool.
func putJPEGBuffer(buf *bufio.Writer) {
buf.Reset(nil)
jpegBufferPool.Put(buf)
}
// pngEncoderBufferPool implements png.EncoderBufferPool.
type pngEncoderBufferPool sync.Pool
func (p *pngEncoderBufferPool) Get() *png.EncoderBuffer {
buf, _ := (*sync.Pool)(p).Get().(*png.EncoderBuffer)
return buf
}
func (p *pngEncoderBufferPool) Put(buf *png.EncoderBuffer) {
(*sync.Pool)(p).Put(buf)
}

View file

@ -148,9 +148,6 @@ func NewManager(database db.DB, storage *storage.Driver) (Manager, error) {
// Prepare the media worker pool
m.mediaWorker = concurrency.NewWorkerPool[*ProcessingMedia](-1, 10)
m.mediaWorker.SetProcessor(func(ctx context.Context, media *ProcessingMedia) error {
if err := ctx.Err(); err != nil {
return err
}
if _, err := media.LoadAttachment(ctx); err != nil {
return fmt.Errorf("error loading media %s: %v", media.AttachmentID(), err)
}
@ -160,9 +157,6 @@ func NewManager(database db.DB, storage *storage.Driver) (Manager, error) {
// Prepare the emoji worker pool
m.emojiWorker = concurrency.NewWorkerPool[*ProcessingEmoji](-1, 10)
m.emojiWorker.SetProcessor(func(ctx context.Context, emoji *ProcessingEmoji) error {
if err := ctx.Err(); err != nil {
return err
}
if _, err := emoji.LoadEmoji(ctx); err != nil {
return fmt.Errorf("error loading emoji %s: %v", emoji.EmojiID(), err)
}

View file

@ -26,6 +26,7 @@
"os"
"path"
"testing"
"time"
"codeberg.org/gruf/go-store/v2/kv"
"codeberg.org/gruf/go-store/v2/storage"
@ -33,7 +34,6 @@
gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
gtsstorage "github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type ManagerTestSuite struct {
@ -214,7 +214,7 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLarge() {
// do a blocking call to fetch the emoji
emoji, err := processingEmoji.LoadEmoji(ctx)
suite.EqualError(err, "store: given emoji fileSize (645688b) is larger than allowed size (51200b)")
suite.EqualError(err, "given emoji size 630kiB greater than max allowed 50.0kiB")
suite.Nil(emoji)
}
@ -227,7 +227,7 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLargeNoSizeGiven() {
if err != nil {
panic(err)
}
return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil
return io.NopCloser(bytes.NewBuffer(b)), -1, nil
}
emojiID := "01GDQ9G782X42BAMFASKP64343"
@ -238,7 +238,7 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLargeNoSizeGiven() {
// do a blocking call to fetch the emoji
emoji, err := processingEmoji.LoadEmoji(ctx)
suite.EqualError(err, "store: given emoji fileSize (645688b) is larger than allowed size (51200b)")
suite.EqualError(err, "calculated emoji size 630kiB greater than max allowed 50.0kiB")
suite.Nil(emoji)
}
@ -396,6 +396,9 @@ func (suite *ManagerTestSuite) TestSlothVineProcessBlocking() {
// fetch the attachment id from the processing media
attachmentID := processingMedia.AttachmentID()
// Give time for processing
time.Sleep(time.Second * 3)
// do a blocking call to fetch the attachment
attachment, err := processingMedia.LoadAttachment(ctx)
suite.NoError(err)
@ -420,7 +423,7 @@ func (suite *ManagerTestSuite) TestSlothVineProcessBlocking() {
suite.Equal("video/mp4", attachment.File.ContentType)
suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)
suite.Equal(312413, attachment.File.FileSize)
suite.Equal("", attachment.Blurhash)
suite.Equal("L00000fQfQfQfQfQfQfQfQfQfQfQ", attachment.Blurhash)
// now make sure the attachment is in the database
dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachmentID)
@ -491,12 +494,12 @@ func (suite *ManagerTestSuite) TestLongerMp4ProcessBlocking() {
suite.EqualValues(10, *attachment.FileMeta.Original.Framerate)
suite.EqualValues(0xc8fb, *attachment.FileMeta.Original.Bitrate)
suite.EqualValues(gtsmodel.Small{
Width: 600, Height: 330, Size: 198000, Aspect: 1.8181819,
Width: 512, Height: 281, Size: 143872, Aspect: 1.822064,
}, attachment.FileMeta.Small)
suite.Equal("video/mp4", attachment.File.ContentType)
suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)
suite.Equal(109549, attachment.File.FileSize)
suite.Equal("", attachment.Blurhash)
suite.Equal("L00000fQfQfQfQfQfQfQfQfQfQfQ", attachment.Blurhash)
// now make sure the attachment is in the database
dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachmentID)
@ -550,7 +553,7 @@ func (suite *ManagerTestSuite) TestNotAnMp4ProcessBlocking() {
// we should get an error while loading
attachment, err := processingMedia.LoadAttachment(ctx)
suite.EqualError(err, "\"video width could not be discovered\",\"video height could not be discovered\",\"video duration could not be discovered\",\"video framerate could not be discovered\",\"video bitrate could not be discovered\"")
suite.EqualError(err, "error decoding video: error determining video metadata: [width height duration framerate bitrate]")
suite.Nil(attachment)
}
@ -928,7 +931,8 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() {
}
func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() {
ctx := context.Background()
ctx, cncl := context.WithTimeout(context.Background(), time.Second*30)
defer cncl()
data := func(_ context.Context) (io.ReadCloser, int64, error) {
// load bytes from a test image
@ -944,15 +948,12 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() {
// process the media with no additional info provided
processingMedia, err := suite.manager.ProcessMedia(ctx, data, nil, accountID, nil)
suite.NoError(err)
// fetch the attachment id from the processing media
attachmentID := processingMedia.AttachmentID()
// wait for the media to finish processing
if !testrig.WaitFor(func() bool {
return processingMedia.Finished()
}) {
suite.FailNow("timed out waiting for media to be processed")
}
// Give time for processing to happen.
time.Sleep(time.Second * 3)
// fetch the attachment from the database
attachment, err := suite.db.GetAttachmentByID(ctx, attachmentID)

View file

@ -75,8 +75,6 @@
import (
"encoding/binary"
"image"
"image/png"
"io"
)
@ -192,13 +190,3 @@ func (r *PNGAncillaryChunkStripper) Read(p []byte) (int, error) {
}
}
}
// StrippedPngDecode strips ancillary data from png to allow more lenient decoding of pngs
// see: https://github.com/golang/go/issues/43382
// and: https://github.com/google/wuffs/blob/414a011491ff513b86d8694c5d71800f3cb5a715/script/strip-png-ancillary-chunks.go
func StrippedPngDecode(r io.Reader) (image.Image, error) {
strippedPngReader := io.Reader(&PNGAncillaryChunkStripper{
Reader: r,
})
return png.Decode(strippedPngReader)
}

View file

@ -24,84 +24,74 @@
"errors"
"fmt"
"io"
"strings"
"sync"
"sync/atomic"
"time"
"codeberg.org/gruf/go-bytesize"
gostore "codeberg.org/gruf/go-store/v2/storage"
"github.com/h2non/filetype"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
// ProcessingEmoji represents an emoji currently processing. It exposes
// various functions for retrieving data from the process.
type ProcessingEmoji struct {
mu sync.Mutex
// id of this instance's account -- pinned for convenience here so we only need to fetch it once
instanceAccountID string
/*
below fields should be set on newly created media;
emoji will be updated incrementally as media goes through processing
*/
emoji *gtsmodel.Emoji
data DataFunc
postData PostDataCallbackFunc
read bool // bool indicating that data function has been triggered already
/*
below fields represent the processing state of the static of the emoji
*/
staticState int32
/*
below pointers to database and storage are maintained so that
the media can store and update itself during processing steps
*/
database db.DB
storage *storage.Driver
err error // error created during processing, if any
// track whether this emoji has already been put in the databse
insertedInDB bool
// is this a refresh of an existing emoji?
refresh bool
// if it is a refresh, which alternate ID should we use in the storage and URL paths?
newPathID string
instAccID string // instance account ID
emoji *gtsmodel.Emoji // processing emoji details
refresh bool // whether this is an existing emoji being refreshed
newPathID string // new emoji path ID to use if refreshed
dataFn DataFunc // load-data function, returns media stream
postFn PostDataCallbackFunc // post data callback function
err error // error encountered during processing
manager *manager // manager instance (access to db / storage)
once sync.Once // once ensures processing only occurs once
}
// EmojiID returns the ID of the underlying emoji without blocking processing.
func (p *ProcessingEmoji) EmojiID() string {
return p.emoji.ID
return p.emoji.ID // immutable, safe outside mutex.
}
// LoadEmoji blocks until the static and fullsize image
// has been processed, and then returns the completed emoji.
func (p *ProcessingEmoji) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) {
p.mu.Lock()
defer p.mu.Unlock()
// only process once.
p.once.Do(func() {
var err error
if err := p.store(ctx); err != nil {
return nil, err
}
defer func() {
if r := recover(); r != nil {
if err != nil {
rOld := r // wrap the panic so we don't lose existing returned error
r = fmt.Errorf("panic occured after error %q: %v", err.Error(), rOld)
}
if err := p.loadStatic(ctx); err != nil {
return nil, err
}
// Catch any panics and wrap as error.
err = fmt.Errorf("caught panic: %v", r)
}
if err != nil {
// Store error.
p.err = err
}
}()
// Attempt to store media and calculate
// full-size media attachment details.
if err = p.store(ctx); err != nil {
return
}
// Finish processing by reloading media into
// memory to get dimension and generate a thumb.
if err = p.finish(ctx); err != nil {
return
}
// store the result in the database before returning it
if !p.insertedInDB {
if p.refresh {
columns := []string{
"updated_at",
@ -118,176 +108,195 @@ func (p *ProcessingEmoji) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error
"shortcode",
"uri",
}
if _, err := p.database.UpdateEmoji(ctx, p.emoji, columns...); err != nil {
return nil, err
}
} else {
if err := p.database.PutEmoji(ctx, p.emoji); err != nil {
return nil, err
}
// Existing emoji we're refreshing, so only need to update.
_, err = p.manager.db.UpdateEmoji(ctx, p.emoji, columns...)
return
}
p.insertedInDB = true
// New emoji media, first time caching.
err = p.manager.db.PutEmoji(ctx, p.emoji)
return //nolint shutup linter i like this here
})
if p.err != nil {
return nil, p.err
}
return p.emoji, nil
}
// Finished returns true if processing has finished for both the thumbnail
// and full fized version of this piece of media.
func (p *ProcessingEmoji) Finished() bool {
return atomic.LoadInt32(&p.staticState) == int32(complete)
}
func (p *ProcessingEmoji) loadStatic(ctx context.Context) error {
staticState := atomic.LoadInt32(&p.staticState)
switch processState(staticState) {
case received:
// stream the original file out of storage...
stored, err := p.storage.GetStream(ctx, p.emoji.ImagePath)
if err != nil {
p.err = fmt.Errorf("loadStatic: error fetching file from storage: %s", err)
atomic.StoreInt32(&p.staticState, int32(errored))
return p.err
}
defer stored.Close()
// we haven't processed a static version of this emoji yet so do it now
static, err := deriveStaticEmoji(stored, p.emoji.ImageContentType)
if err != nil {
p.err = fmt.Errorf("loadStatic: error deriving static: %s", err)
atomic.StoreInt32(&p.staticState, int32(errored))
return p.err
}
// Close stored emoji now we're done
if err := stored.Close(); err != nil {
log.Errorf("loadStatic: error closing stored full size: %s", err)
}
// put the static image in storage
if err := p.storage.Put(ctx, p.emoji.ImageStaticPath, static.small); err != nil && err != storage.ErrAlreadyExists {
p.err = fmt.Errorf("loadStatic: error storing static: %s", err)
atomic.StoreInt32(&p.staticState, int32(errored))
return p.err
}
p.emoji.ImageStaticFileSize = len(static.small)
// we're done processing the static version of the emoji!
atomic.StoreInt32(&p.staticState, int32(complete))
fallthrough
case complete:
return nil
case errored:
return p.err
}
return fmt.Errorf("static processing status %d unknown", p.staticState)
}
// store calls the data function attached to p if it hasn't been called yet,
// and updates the underlying attachment fields as necessary. It will then stream
// bytes from p's reader directly into storage so that it can be retrieved later.
func (p *ProcessingEmoji) store(ctx context.Context) error {
// check if we've already done this and bail early if we have
if p.read {
return nil
}
// execute the data function to get the readcloser out of it
rc, fileSize, err := p.data(ctx)
if err != nil {
return fmt.Errorf("store: error executing data function: %s", err)
}
// defer closing the reader when we're done with it
defer func() {
if p.postFn == nil {
return
}
// Ensure post callback gets called.
if err := p.postFn(ctx); err != nil {
log.Errorf("error executing postdata function: %v", err)
}
}()
// Load media from provided data fn.
rc, sz, err := p.dataFn(ctx)
if err != nil {
return fmt.Errorf("error executing data function: %w", err)
}
defer func() {
// Ensure data reader gets closed on return.
if err := rc.Close(); err != nil {
log.Errorf("store: error closing readcloser: %s", err)
log.Errorf("error closing data reader: %v", err)
}
}()
// execute the postData function no matter what happens
defer func() {
if p.postData != nil {
if err := p.postData(ctx); err != nil {
log.Errorf("store: error executing postData: %s", err)
}
}
}()
// Byte buffer to read file header into.
// See: https://en.wikipedia.org/wiki/File_format#File_header
// and https://github.com/h2non/filetype
hdrBuf := make([]byte, 261)
// extract no more than 261 bytes from the beginning of the file -- this is the header
firstBytes := make([]byte, maxFileHeaderBytes)
if _, err := rc.Read(firstBytes); err != nil {
return fmt.Errorf("store: error reading initial %d bytes: %s", maxFileHeaderBytes, err)
// Read the first 261 header bytes into buffer.
if _, err := io.ReadFull(rc, hdrBuf); err != nil {
return fmt.Errorf("error reading incoming media: %w", err)
}
// now we have the file header we can work out the content type from it
contentType, err := parseContentType(firstBytes)
// Parse file type info from header buffer.
info, err := filetype.Match(hdrBuf)
if err != nil {
return fmt.Errorf("store: error parsing content type: %s", err)
return fmt.Errorf("error parsing file type: %w", err)
}
// bail if this is a type we can't process
if !supportedEmoji(contentType) {
return fmt.Errorf("store: content type %s was not valid for an emoji", contentType)
switch info.Extension {
// only supported emoji types
case "gif", "png":
// unhandled
default:
return fmt.Errorf("unsupported emoji filetype: %s", info.Extension)
}
// extract the file extension
split := strings.Split(contentType, "/")
extension := split[1] // something like 'gif'
// Recombine header bytes with remaining stream
r := io.MultiReader(bytes.NewReader(hdrBuf), rc)
var maxSize bytesize.Size
if p.emoji.Domain == "" {
// this is a local emoji upload
maxSize = config.GetMediaEmojiLocalMaxSize()
} else {
// this is a remote incoming emoji
maxSize = config.GetMediaEmojiRemoteMaxSize()
}
// Check that provided size isn't beyond max. We check beforehand
// so that we don't attempt to stream the emoji into storage if not needed.
if size := bytesize.Size(sz); sz > 0 && size > maxSize {
return fmt.Errorf("given emoji size %s greater than max allowed %s", size, maxSize)
}
// set some additional fields on the emoji now that
// we know more about what the underlying image actually is
var pathID string
if p.refresh {
// This is a refreshed emoji with a new
// path ID that this will be stored under.
pathID = p.newPathID
} else {
// This is a new emoji, simply use provided ID.
pathID = p.emoji.ID
}
p.emoji.ImageURL = uris.GenerateURIForAttachment(p.instanceAccountID, string(TypeEmoji), string(SizeOriginal), pathID, extension)
p.emoji.ImagePath = fmt.Sprintf("%s/%s/%s/%s.%s", p.instanceAccountID, TypeEmoji, SizeOriginal, pathID, extension)
p.emoji.ImageContentType = contentType
// concatenate the first bytes with the existing bytes still in the reader (thanks Mara)
readerToStore := io.MultiReader(bytes.NewBuffer(firstBytes), rc)
// Calculate emoji file path.
p.emoji.ImagePath = fmt.Sprintf(
"%s/%s/%s/%s.%s",
p.instAccID,
TypeEmoji,
SizeOriginal,
pathID,
info.Extension,
)
var maxEmojiSize int64
if p.emoji.Domain == "" {
maxEmojiSize = int64(config.GetMediaEmojiLocalMaxSize())
} else {
maxEmojiSize = int64(config.GetMediaEmojiRemoteMaxSize())
}
// This shouldn't already exist, but we do a check as it's worth logging.
if have, _ := p.manager.storage.Has(ctx, p.emoji.ImagePath); have {
log.Warnf("emoji already exists at storage path: %s", p.emoji.ImagePath)
// if we know the fileSize already, make sure it's not bigger than our limit
var checkedSize bool
if fileSize > 0 {
checkedSize = true
if fileSize > maxEmojiSize {
return fmt.Errorf("store: given emoji fileSize (%db) is larger than allowed size (%db)", fileSize, maxEmojiSize)
// Attempt to remove existing emoji at storage path (might be broken / out-of-date)
if err := p.manager.storage.Delete(ctx, p.emoji.ImagePath); err != nil {
return fmt.Errorf("error removing emoji from storage: %v", err)
}
}
// store this for now -- other processes can pull it out of storage as they please
if fileSize, err = putStream(ctx, p.storage, p.emoji.ImagePath, readerToStore, fileSize); err != nil {
if !errors.Is(err, storage.ErrAlreadyExists) {
return fmt.Errorf("store: error storing stream: %s", err)
}
log.Warnf("emoji %s already exists at storage path: %s", p.emoji.ID, p.emoji.ImagePath)
// Write the final image reader stream to our storage.
sz, err = p.manager.storage.PutStream(ctx, p.emoji.ImagePath, r)
if err != nil {
return fmt.Errorf("error writing emoji to storage: %w", err)
}
// if we didn't know the fileSize yet, we do now, so check if we need to
if !checkedSize && fileSize > maxEmojiSize {
err = fmt.Errorf("store: discovered emoji fileSize (%db) is larger than allowed emojiRemoteMaxSize (%db), will delete from the store now", fileSize, maxEmojiSize)
log.Warn(err)
if deleteErr := p.storage.Delete(ctx, p.emoji.ImagePath); deleteErr != nil {
log.Errorf("store: error removing too-large emoji from the store: %s", deleteErr)
// Once again check size in case none was provided previously.
if size := bytesize.Size(sz); size > maxSize {
if err := p.manager.storage.Delete(ctx, p.emoji.ImagePath); err != nil {
log.Errorf("error removing too-large-emoji from storage: %v", err)
}
return err
return fmt.Errorf("calculated emoji size %s greater than max allowed %s", size, maxSize)
}
p.emoji.ImageFileSize = int(fileSize)
p.read = true
// Fill in remaining attachment data now it's stored.
p.emoji.ImageURL = uris.GenerateURIForAttachment(
p.instAccID,
string(TypeEmoji),
string(SizeOriginal),
pathID,
info.Extension,
)
p.emoji.ImageContentType = info.MIME.Value
p.emoji.ImageFileSize = int(sz)
return nil
}
func (p *ProcessingEmoji) finish(ctx context.Context) error {
// Fetch a stream to the original file in storage.
rc, err := p.manager.storage.GetStream(ctx, p.emoji.ImagePath)
if err != nil {
return fmt.Errorf("error loading file from storage: %w", err)
}
defer rc.Close()
// Decode the image from storage.
staticImg, err := decodeImage(rc)
if err != nil {
return fmt.Errorf("error decoding image: %w", err)
}
// The image should be in-memory by now.
if err := rc.Close(); err != nil {
return fmt.Errorf("error closing file: %w", err)
}
// This shouldn't already exist, but we do a check as it's worth logging.
if have, _ := p.manager.storage.Has(ctx, p.emoji.ImageStaticPath); have {
log.Warnf("static emoji already exists at storage path: %s", p.emoji.ImagePath)
// Attempt to remove static existing emoji at storage path (might be broken / out-of-date)
if err := p.manager.storage.Delete(ctx, p.emoji.ImageStaticPath); err != nil {
return fmt.Errorf("error removing static emoji from storage: %v", err)
}
}
// Create an emoji PNG encoder stream.
enc := staticImg.ToPNG()
// Stream-encode the PNG static image into storage.
sz, err := p.manager.storage.PutStream(ctx, p.emoji.ImageStaticPath, enc)
if err != nil {
return fmt.Errorf("error stream-encoding static emoji to storage: %w", err)
}
// Set written image size.
p.emoji.ImageStaticFileSize = int(sz)
return nil
}
@ -406,15 +415,13 @@ func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, postData P
}
processingEmoji := &ProcessingEmoji{
instanceAccountID: instanceAccount.ID,
emoji: emoji,
data: data,
postData: postData,
staticState: int32(received),
database: m.db,
storage: m.storage,
refresh: refresh,
newPathID: newPathID,
instAccID: instanceAccount.ID,
emoji: emoji,
refresh: refresh,
newPathID: newPathID,
dataFn: data,
postFn: postData,
manager: m,
}
return processingEmoji, nil

View file

@ -21,387 +21,329 @@
import (
"bytes"
"context"
"errors"
"fmt"
"image/jpeg"
"io"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/disintegration/imaging"
"github.com/h2non/filetype"
terminator "github.com/superseriousbusiness/exif-terminator"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
// ProcessingMedia represents a piece of media that is currently being processed. It exposes
// various functions for retrieving data from the process.
type ProcessingMedia struct {
mu sync.Mutex
/*
below fields should be set on newly created media;
attachment will be updated incrementally as media goes through processing
*/
attachment *gtsmodel.MediaAttachment
data DataFunc
postData PostDataCallbackFunc
read bool // bool indicating that data function has been triggered already
thumbState int32 // the processing state of the media thumbnail
fullSizeState int32 // the processing state of the full-sized media
/*
below pointers to database and storage are maintained so that
the media can store and update itself during processing steps
*/
database db.DB
storage *storage.Driver
err error // error created during processing, if any
// track whether this media has already been put in the databse
insertedInDB bool
// true if this is a recache, false if it's brand new media
recache bool
media *gtsmodel.MediaAttachment // processing media attachment details
recache bool // recaching existing (uncached) media
dataFn DataFunc // load-data function, returns media stream
postFn PostDataCallbackFunc // post data callback function
err error // error encountered during processing
manager *manager // manager instance (access to db / storage)
once sync.Once // once ensures processing only occurs once
}
// AttachmentID returns the ID of the underlying media attachment without blocking processing.
func (p *ProcessingMedia) AttachmentID() string {
return p.attachment.ID
return p.media.ID // immutable, safe outside mutex.
}
// LoadAttachment blocks until the thumbnail and fullsize content
// has been processed, and then returns the completed attachment.
func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAttachment, error) {
p.mu.Lock()
defer p.mu.Unlock()
if err := p.store(ctx); err != nil {
return nil, err
}
if err := p.loadFullSize(ctx); err != nil {
return nil, err
}
if err := p.loadThumb(ctx); err != nil {
return nil, err
}
if !p.insertedInDB {
if p.recache {
// This is an existing media attachment we're recaching, so only need to update it
if err := p.database.UpdateByID(ctx, p.attachment, p.attachment.ID); err != nil {
return nil, err
}
} else {
// This is a new media attachment we're caching for first time
if err := p.database.Put(ctx, p.attachment); err != nil {
return nil, err
}
}
// Mark this as stored in DB
p.insertedInDB = true
}
log.Tracef("finished loading attachment %s", p.attachment.URL)
return p.attachment, nil
}
// Finished returns true if processing has finished for both the thumbnail
// and full fized version of this piece of media.
func (p *ProcessingMedia) Finished() bool {
return atomic.LoadInt32(&p.thumbState) == int32(complete) && atomic.LoadInt32(&p.fullSizeState) == int32(complete)
}
func (p *ProcessingMedia) loadThumb(ctx context.Context) error {
thumbState := atomic.LoadInt32(&p.thumbState)
switch processState(thumbState) {
case received:
// we haven't processed a thumbnail for this media yet so do it now
// check if we need to create a blurhash or if there's already one set
var createBlurhash bool
if p.attachment.Blurhash == "" {
// no blurhash created yet
createBlurhash = true
}
var (
thumb *mediaMeta
err error
)
switch ct := p.attachment.File.ContentType; ct {
case mimeImageJpeg, mimeImagePng, mimeImageWebp, mimeImageGif:
// thumbnail the image from the original stored full size version
stored, err := p.storage.GetStream(ctx, p.attachment.File.Path)
if err != nil {
p.err = fmt.Errorf("loadThumb: error fetching file from storage: %s", err)
atomic.StoreInt32(&p.thumbState, int32(errored))
return p.err
}
thumb, err = deriveThumbnailFromImage(stored, ct, createBlurhash)
// try to close the stored stream we had open, no matter what
if closeErr := stored.Close(); closeErr != nil {
log.Errorf("error closing stream: %s", closeErr)
}
// now check if we managed to get a thumbnail
if err != nil {
p.err = fmt.Errorf("loadThumb: error deriving thumbnail: %s", err)
atomic.StoreInt32(&p.thumbState, int32(errored))
return p.err
}
case mimeVideoMp4:
// create a generic thumbnail based on video height + width
thumb, err = deriveThumbnailFromVideo(p.attachment.FileMeta.Original.Height, p.attachment.FileMeta.Original.Width)
if err != nil {
p.err = fmt.Errorf("loadThumb: error deriving thumbnail: %s", err)
atomic.StoreInt32(&p.thumbState, int32(errored))
return p.err
}
default:
p.err = fmt.Errorf("loadThumb: content type %s not a processible image type", ct)
atomic.StoreInt32(&p.thumbState, int32(errored))
return p.err
}
// put the thumbnail in storage
if err := p.storage.Put(ctx, p.attachment.Thumbnail.Path, thumb.small); err != nil && err != storage.ErrAlreadyExists {
p.err = fmt.Errorf("loadThumb: error storing thumbnail: %s", err)
atomic.StoreInt32(&p.thumbState, int32(errored))
return p.err
}
// set appropriate fields on the attachment based on the thumbnail we derived
if createBlurhash {
p.attachment.Blurhash = thumb.blurhash
}
p.attachment.FileMeta.Small = gtsmodel.Small{
Width: thumb.width,
Height: thumb.height,
Size: thumb.size,
Aspect: thumb.aspect,
}
p.attachment.Thumbnail.FileSize = len(thumb.small)
// we're done processing the thumbnail!
atomic.StoreInt32(&p.thumbState, int32(complete))
log.Tracef("finished processing thumbnail for attachment %s", p.attachment.URL)
fallthrough
case complete:
return nil
case errored:
return p.err
}
return fmt.Errorf("loadThumb: thumbnail processing status %d unknown", p.thumbState)
}
func (p *ProcessingMedia) loadFullSize(ctx context.Context) error {
fullSizeState := atomic.LoadInt32(&p.fullSizeState)
switch processState(fullSizeState) {
case received:
// only process once.
p.once.Do(func() {
var err error
var decoded *mediaMeta
// stream the original file out of storage...
stored, err := p.storage.GetStream(ctx, p.attachment.File.Path)
if err != nil {
p.err = fmt.Errorf("loadFullSize: error fetching file from storage: %s", err)
atomic.StoreInt32(&p.fullSizeState, int32(errored))
return p.err
}
defer func() {
if err := stored.Close(); err != nil {
log.Errorf("loadFullSize: error closing stored full size: %s", err)
if r := recover(); r != nil {
if err != nil {
rOld := r // wrap the panic so we don't lose existing returned error
r = fmt.Errorf("panic occured after error %q: %v", err.Error(), rOld)
}
// Catch any panics and wrap as error.
err = fmt.Errorf("caught panic: %v", r)
}
if err != nil {
// Store error.
p.err = err
}
}()
// decode the image
ct := p.attachment.File.ContentType
switch ct {
case mimeImageJpeg, mimeImagePng, mimeImageWebp:
decoded, err = decodeImage(stored, ct)
case mimeImageGif:
decoded, err = decodeGif(stored)
case mimeVideoMp4:
decoded, err = decodeVideo(stored, ct)
default:
err = fmt.Errorf("loadFullSize: content type %s not a processible image type", ct)
// Attempt to store media and calculate
// full-size media attachment details.
if err = p.store(ctx); err != nil {
return
}
if err != nil {
p.err = err
atomic.StoreInt32(&p.fullSizeState, int32(errored))
return p.err
// Finish processing by reloading media into
// memory to get dimension and generate a thumb.
if err = p.finish(ctx); err != nil {
return
}
// set appropriate fields on the attachment based on the image we derived
// generic fields
p.attachment.File.UpdatedAt = time.Now()
p.attachment.FileMeta.Original = gtsmodel.Original{
Width: decoded.width,
Height: decoded.height,
Size: decoded.size,
Aspect: decoded.aspect,
if p.recache {
// Existing attachment we're recaching, so only need to update.
err = p.manager.db.UpdateByID(ctx, p.media, p.media.ID)
return
}
// nullable fields
if decoded.duration != 0 {
i := decoded.duration
p.attachment.FileMeta.Original.Duration = &i
}
if decoded.framerate != 0 {
i := decoded.framerate
p.attachment.FileMeta.Original.Framerate = &i
}
if decoded.bitrate != 0 {
i := decoded.bitrate
p.attachment.FileMeta.Original.Bitrate = &i
}
// New attachment, first time caching.
err = p.manager.db.Put(ctx, p.media)
return //nolint shutup linter i like this here
})
// we're done processing the full-size image
p.attachment.Processing = gtsmodel.ProcessingStatusProcessed
atomic.StoreInt32(&p.fullSizeState, int32(complete))
log.Tracef("finished processing full size image for attachment %s", p.attachment.URL)
fallthrough
case complete:
return nil
case errored:
return p.err
if p.err != nil {
return nil, p.err
}
return fmt.Errorf("loadFullSize: full size processing status %d unknown", p.fullSizeState)
return p.media, nil
}
// store calls the data function attached to p if it hasn't been called yet,
// and updates the underlying attachment fields as necessary. It will then stream
// bytes from p's reader directly into storage so that it can be retrieved later.
func (p *ProcessingMedia) store(ctx context.Context) error {
// check if we've already done this and bail early if we have
if p.read {
return nil
}
// execute the data function to get the readcloser out of it
rc, fileSize, err := p.data(ctx)
if err != nil {
return fmt.Errorf("store: error executing data function: %s", err)
}
// defer closing the reader when we're done with it
defer func() {
if p.postFn == nil {
return
}
// ensure post callback gets called.
if err := p.postFn(ctx); err != nil {
log.Errorf("error executing postdata function: %v", err)
}
}()
// Load media from provided data fun
rc, sz, err := p.dataFn(ctx)
if err != nil {
return fmt.Errorf("error executing data function: %w", err)
}
defer func() {
// Ensure data reader gets closed on return.
if err := rc.Close(); err != nil {
log.Errorf("store: error closing readcloser: %s", err)
log.Errorf("error closing data reader: %v", err)
}
}()
// execute the postData function no matter what happens
defer func() {
if p.postData != nil {
if err := p.postData(ctx); err != nil {
log.Errorf("store: error executing postData: %s", err)
}
}
}()
// Byte buffer to read file header into.
// See: https://en.wikipedia.org/wiki/File_format#File_header
// and https://github.com/h2non/filetype
hdrBuf := make([]byte, 261)
// extract no more than 261 bytes from the beginning of the file -- this is the header
firstBytes := make([]byte, maxFileHeaderBytes)
if _, err := rc.Read(firstBytes); err != nil {
return fmt.Errorf("store: error reading initial %d bytes: %s", maxFileHeaderBytes, err)
// Read the first 261 header bytes into buffer.
if _, err := io.ReadFull(rc, hdrBuf); err != nil {
return fmt.Errorf("error reading incoming media: %w", err)
}
// now we have the file header we can work out the content type from it
contentType, err := parseContentType(firstBytes)
// Parse file type info from header buffer.
info, err := filetype.Match(hdrBuf)
if err != nil {
return fmt.Errorf("store: error parsing content type: %s", err)
return fmt.Errorf("error parsing file type: %w", err)
}
// bail if this is a type we can't process
if !supportedAttachment(contentType) {
return fmt.Errorf("store: media type %s not (yet) supported", contentType)
}
// Recombine header bytes with remaining stream
r := io.MultiReader(bytes.NewReader(hdrBuf), rc)
// extract the file extension
split := strings.Split(contentType, "/")
if len(split) != 2 {
return fmt.Errorf("store: content type %s was not valid", contentType)
}
extension := split[1] // something like 'jpeg'
switch info.Extension {
case "mp4":
p.media.Type = gtsmodel.FileTypeVideo
// concatenate the cleaned up first bytes with the existing bytes still in the reader (thanks Mara)
multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), rc)
case "gif":
p.media.Type = gtsmodel.FileTypeImage
// use the extension to derive the attachment type
// and, while we're in here, clean up exif data from
// the image if we already know the fileSize
var readerToStore io.Reader
switch extension {
case mimeGif:
p.attachment.Type = gtsmodel.FileTypeImage
// nothing to terminate, we can just store the multireader
readerToStore = multiReader
case mimeJpeg, mimePng, mimeWebp:
p.attachment.Type = gtsmodel.FileTypeImage
if fileSize > 0 {
terminated, err := terminator.Terminate(multiReader, int(fileSize), extension)
case "jpg", "jpeg", "png", "webp":
p.media.Type = gtsmodel.FileTypeImage
if sz > 0 {
// A file size was provided so we can clean exif data from image.
r, err = terminator.Terminate(r, int(sz), info.Extension)
if err != nil {
return fmt.Errorf("store: exif error: %s", err)
return fmt.Errorf("error cleaning exif data: %w", err)
}
defer func() {
if closer, ok := terminated.(io.Closer); ok {
if err := closer.Close(); err != nil {
log.Errorf("store: error closing terminator reader: %s", err)
}
}
}()
// store the exif-terminated version of what was in the multireader
readerToStore = terminated
} else {
// can't terminate if we don't know the file size, so just store the multiReader
readerToStore = multiReader
}
case mimeMp4:
p.attachment.Type = gtsmodel.FileTypeVideo
// nothing to terminate, we can just store the multireader
readerToStore = multiReader
default:
return fmt.Errorf("store: couldn't process %s", extension)
return fmt.Errorf("unsupported file type: %s", info.Extension)
}
// now set some additional fields on the attachment since
// we know more about what the underlying media actually is
p.attachment.URL = uris.GenerateURIForAttachment(p.attachment.AccountID, string(TypeAttachment), string(SizeOriginal), p.attachment.ID, extension)
p.attachment.File.ContentType = contentType
p.attachment.File.Path = fmt.Sprintf("%s/%s/%s/%s.%s", p.attachment.AccountID, TypeAttachment, SizeOriginal, p.attachment.ID, extension)
// Calculate attachment file path.
p.media.File.Path = fmt.Sprintf(
"%s/%s/%s/%s.%s",
p.media.AccountID,
TypeAttachment,
SizeOriginal,
p.media.ID,
info.Extension,
)
// store this for now -- other processes can pull it out of storage as they please
if fileSize, err = putStream(ctx, p.storage, p.attachment.File.Path, readerToStore, fileSize); err != nil {
if !errors.Is(err, storage.ErrAlreadyExists) {
return fmt.Errorf("store: error storing stream: %s", err)
// This shouldn't already exist, but we do a check as it's worth logging.
if have, _ := p.manager.storage.Has(ctx, p.media.File.Path); have {
log.Warnf("media already exists at storage path: %s", p.media.File.Path)
// Attempt to remove existing media at storage path (might be broken / out-of-date)
if err := p.manager.storage.Delete(ctx, p.media.File.Path); err != nil {
return fmt.Errorf("error removing media from storage: %v", err)
}
log.Warnf("attachment %s already exists at storage path: %s", p.attachment.ID, p.attachment.File.Path)
}
cached := true
p.attachment.Cached = &cached
p.attachment.File.FileSize = int(fileSize)
p.read = true
// Write the final image reader stream to our storage.
sz, err = p.manager.storage.PutStream(ctx, p.media.File.Path, r)
if err != nil {
return fmt.Errorf("error writing media to storage: %w", err)
}
// Set written image size.
p.media.File.FileSize = int(sz)
// Fill in remaining attachment data now it's stored.
p.media.URL = uris.GenerateURIForAttachment(
p.media.AccountID,
string(TypeAttachment),
string(SizeOriginal),
p.media.ID,
info.Extension,
)
p.media.File.ContentType = info.MIME.Value
cached := true
p.media.Cached = &cached
return nil
}
func (p *ProcessingMedia) finish(ctx context.Context) error {
// Fetch a stream to the original file in storage.
rc, err := p.manager.storage.GetStream(ctx, p.media.File.Path)
if err != nil {
return fmt.Errorf("error loading file from storage: %w", err)
}
defer rc.Close()
var fullImg *gtsImage
switch p.media.File.ContentType {
// .jpeg, .gif, .webp image type
case mimeImageJpeg, mimeImageGif, mimeImageWebp:
fullImg, err = decodeImage(rc, imaging.AutoOrientation(true))
if err != nil {
return fmt.Errorf("error decoding image: %w", err)
}
// .png image (requires ancillary chunk stripping)
case mimeImagePng:
fullImg, err = decodeImage(&PNGAncillaryChunkStripper{
Reader: rc,
}, imaging.AutoOrientation(true))
if err != nil {
return fmt.Errorf("error decoding image: %w", err)
}
// .mp4 video type
case mimeVideoMp4:
video, err := decodeVideoFrame(rc)
if err != nil {
return fmt.Errorf("error decoding video: %w", err)
}
// Set video frame as image.
fullImg = video.frame
// Set video metadata in attachment info.
p.media.FileMeta.Original.Duration = &video.duration
p.media.FileMeta.Original.Framerate = &video.framerate
p.media.FileMeta.Original.Bitrate = &video.bitrate
}
// The image should be in-memory by now.
if err := rc.Close(); err != nil {
return fmt.Errorf("error closing file: %w", err)
}
// Set full-size dimensions in attachment info.
p.media.FileMeta.Original.Width = int(fullImg.Width())
p.media.FileMeta.Original.Height = int(fullImg.Height())
p.media.FileMeta.Original.Size = int(fullImg.Size())
p.media.FileMeta.Original.Aspect = fullImg.AspectRatio()
// Calculate attachment thumbnail file path
p.media.Thumbnail.Path = fmt.Sprintf(
"%s/%s/%s/%s.jpg",
p.media.AccountID,
TypeAttachment,
SizeSmall,
p.media.ID,
)
// Get smaller thumbnail image
thumbImg := fullImg.Thumbnail()
// Garbage collector, you may
// now take our large son.
fullImg = nil
// Blurhash needs generating from thumb.
hash, err := thumbImg.Blurhash()
if err != nil {
return fmt.Errorf("error generating blurhash: %w", err)
}
// Set the attachment blurhash.
p.media.Blurhash = hash
// This shouldn't already exist, but we do a check as it's worth logging.
if have, _ := p.manager.storage.Has(ctx, p.media.Thumbnail.Path); have {
log.Warnf("thumbnail already exists at storage path: %s", p.media.Thumbnail.Path)
// Attempt to remove existing thumbnail at storage path (might be broken / out-of-date)
if err := p.manager.storage.Delete(ctx, p.media.Thumbnail.Path); err != nil {
return fmt.Errorf("error removing thumbnail from storage: %v", err)
}
}
// Create a thumbnail JPEG encoder stream.
enc := thumbImg.ToJPEG(&jpeg.Options{
Quality: 70, // enough for a thumbnail.
})
// Stream-encode the JPEG thumbnail image into storage.
sz, err := p.manager.storage.PutStream(ctx, p.media.Thumbnail.Path, enc)
if err != nil {
return fmt.Errorf("error stream-encoding thumbnail to storage: %w", err)
}
// Fill in remaining thumbnail now it's stored
p.media.Thumbnail.ContentType = mimeImageJpeg
p.media.Thumbnail.URL = uris.GenerateURIForAttachment(
p.media.AccountID,
string(TypeAttachment),
string(SizeSmall),
p.media.ID,
"jpg", // always jpeg
)
// Set thumbnail dimensions in attachment info.
p.media.FileMeta.Small = gtsmodel.Small{
Width: int(thumbImg.Width()),
Height: int(thumbImg.Height()),
Size: int(thumbImg.Size()),
Aspect: thumbImg.AspectRatio(),
}
// Set written image size.
p.media.Thumbnail.FileSize = int(sz)
// Finally set the attachment as processed and update time.
p.media.Processing = gtsmodel.ProcessingStatusProcessed
p.media.File.UpdatedAt = time.Now()
log.Tracef("finished storing initial data for attachment %s", p.attachment.URL)
return nil
}
@ -411,19 +353,6 @@ func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, postData P
return nil, err
}
file := gtsmodel.File{
Path: "", // we don't know yet because it depends on the uncalled DataFunc
ContentType: "", // we don't know yet because it depends on the uncalled DataFunc
UpdatedAt: time.Now(),
}
thumbnail := gtsmodel.Thumbnail{
URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeSmall), id, mimeJpeg), // all thumbnails are encoded as jpeg,
Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeSmall, id, mimeJpeg), // all thumbnails are encoded as jpeg,
ContentType: mimeImageJpeg,
UpdatedAt: time.Now(),
}
avatar := false
header := false
cached := false
@ -443,8 +372,8 @@ func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, postData P
ScheduledStatusID: "",
Blurhash: "",
Processing: gtsmodel.ProcessingStatusReceived,
File: file,
Thumbnail: thumbnail,
File: gtsmodel.File{UpdatedAt: time.Now()},
Thumbnail: gtsmodel.Thumbnail{UpdatedAt: time.Now()},
Avatar: &avatar,
Header: &header,
Cached: &cached,
@ -495,34 +424,28 @@ func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, postData P
}
processingMedia := &ProcessingMedia{
attachment: attachment,
data: data,
postData: postData,
thumbState: int32(received),
fullSizeState: int32(received),
database: m.db,
storage: m.storage,
media: attachment,
dataFn: data,
postFn: postData,
manager: m,
}
return processingMedia, nil
}
func (m *manager) preProcessRecache(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error) {
// get the existing attachment
attachment, err := m.db.GetAttachmentByID(ctx, attachmentID)
func (m *manager) preProcessRecache(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, id string) (*ProcessingMedia, error) {
// get the existing attachment from database.
attachment, err := m.db.GetAttachmentByID(ctx, id)
if err != nil {
return nil, err
}
processingMedia := &ProcessingMedia{
attachment: attachment,
data: data,
postData: postData,
thumbState: int32(received),
fullSizeState: int32(received),
database: m.db,
storage: m.storage,
recache: true, // indicate it's a recache
media: attachment,
dataFn: data,
postFn: postData,
manager: m,
recache: true, // indicate it's a recache
}
return processingMedia, nil

View file

@ -39,7 +39,7 @@ func (suite *PruneOrphanedTestSuite) TestPruneOrphanedDry() {
}
pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachments/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif"
if err := suite.storage.PutStream(context.Background(), pandaPath, bytes.NewBuffer(b)); err != nil {
if _, err := suite.storage.PutStream(context.Background(), pandaPath, bytes.NewBuffer(b)); err != nil {
panic(err)
}
@ -62,7 +62,7 @@ func (suite *PruneOrphanedTestSuite) TestPruneOrphanedMoist() {
}
pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachments/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif"
if err := suite.storage.PutStream(context.Background(), pandaPath, bytes.NewBuffer(b)); err != nil {
if _, err := suite.storage.PutStream(context.Background(), pandaPath, bytes.NewBuffer(b)); err != nil {
panic(err)
}

View file

@ -87,7 +87,7 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
// now recache the image....
data := func(_ context.Context) (io.ReadCloser, int64, error) {
// load bytes from a test image
b, err := os.ReadFile("../../testrig/media/thoughtsofdog-original.jpeg")
b, err := os.ReadFile("../../testrig/media/thoughtsofdog-original.jpg")
if err != nil {
panic(err)
}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.7 KiB

After

Width:  |  Height:  |  Size: 2.8 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 22 KiB

After

Width:  |  Height:  |  Size: 20 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.9 KiB

After

Width:  |  Height:  |  Size: 1.9 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.3 KiB

After

Width:  |  Height:  |  Size: 5.8 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.3 KiB

After

Width:  |  Height:  |  Size: 5.8 KiB

View file

@ -24,13 +24,6 @@
"time"
)
// maxFileHeaderBytes represents the maximum amount of bytes we want
// to examine from the beginning of a file to determine its type.
//
// See: https://en.wikipedia.org/wiki/File_format#File_header
// and https://github.com/h2non/filetype
const maxFileHeaderBytes = 261
// mime consts
const (
mimeImage = "image"
@ -52,14 +45,6 @@
mimeVideoMp4 = mimeVideo + "/" + mimeMp4
)
type processState int32
const (
received processState = iota // processing order has been received but not done yet
complete // processing order has been completed successfully
errored // processing order has been completed with an error
)
// EmojiMaxBytes is the maximum permitted bytes of an emoji upload (50kb)
// const EmojiMaxBytes = 51200
@ -132,17 +117,3 @@ type AdditionalEmojiInfo struct {
//
// This can be set to nil, and will then not be executed.
type PostDataCallbackFunc func(ctx context.Context) error
type mediaMeta struct {
width int
height int
size int
aspect float32
blurhash string
small []byte
// video-specific properties
duration float32
framerate float32
bitrate uint64
}

View file

@ -19,72 +19,22 @@
package media
import (
"context"
"errors"
"fmt"
"io"
"github.com/h2non/filetype"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/storage"
)
// AllSupportedMIMETypes just returns all media
// MIME types supported by this instance.
func AllSupportedMIMETypes() []string {
return []string{
mimeImageJpeg,
mimeImageGif,
mimeImagePng,
mimeImageWebp,
mimeVideoMp4,
}
var SupportedMIMETypes = []string{
mimeImageJpeg,
mimeImageGif,
mimeImagePng,
mimeImageWebp,
mimeVideoMp4,
}
// parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg").
// Returns an error if the content type is not something we can process.
//
// Fileheader should be no longer than 262 bytes; anything more than this is inefficient.
func parseContentType(fileHeader []byte) (string, error) {
if fhLength := len(fileHeader); fhLength > maxFileHeaderBytes {
return "", fmt.Errorf("parseContentType requires %d bytes max, we got %d", maxFileHeaderBytes, fhLength)
}
kind, err := filetype.Match(fileHeader)
if err != nil {
return "", err
}
if kind == filetype.Unknown {
return "", errors.New("filetype unknown")
}
return kind.MIME.Value, nil
}
// supportedAttachment checks mime type of an attachment against a
// slice of accepted types, and returns True if the mime type is accepted.
func supportedAttachment(mimeType string) bool {
for _, accepted := range AllSupportedMIMETypes() {
if mimeType == accepted {
return true
}
}
return false
}
// supportedEmoji checks that the content type is image/png or image/gif -- the only types supported for emoji.
func supportedEmoji(mimeType string) bool {
acceptedEmojiTypes := []string{
mimeImageGif,
mimeImagePng,
}
for _, accepted := range acceptedEmojiTypes {
if mimeType == accepted {
return true
}
}
return false
var SupportedEmojiMIMETypes = []string{
mimeImageGif,
mimeImagePng,
}
// ParseMediaType converts s to a recognized MediaType, or returns an error if unrecognized
@ -127,31 +77,3 @@ func (l *logrusWrapper) Info(msg string, keysAndValues ...interface{}) {
func (l *logrusWrapper) Error(err error, msg string, keysAndValues ...interface{}) {
log.Error("media manager cron logger: ", err, msg, keysAndValues)
}
// lengthReader wraps a reader and reads the length of total bytes written as it goes.
type lengthReader struct {
source io.Reader
length int64
}
func (r *lengthReader) Read(b []byte) (int, error) {
n, err := r.source.Read(b)
r.length += int64(n)
return n, err
}
// putStream either puts a file with a known fileSize into storage directly, and returns the
// fileSize unchanged, or it wraps the reader with a lengthReader and returns the discovered
// fileSize.
func putStream(ctx context.Context, storage *storage.Driver, key string, r io.Reader, fileSize int64) (int64, error) {
if fileSize > 0 {
return fileSize, storage.PutStream(ctx, key, r)
}
lr := &lengthReader{
source: r,
}
err := storage.PutStream(ctx, key, lr)
return lr.length, err
}

View file

@ -19,63 +19,55 @@
package media
import (
"bytes"
"fmt"
"image"
"image/color"
"image/draw"
"image/jpeg"
"io"
"os"
"github.com/abema/go-mp4"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/log"
)
var thumbFill = color.RGBA{42, 43, 47, 0} // the color to fill video thumbnails with
type gtsVideo struct {
frame *gtsImage
duration float32 // in seconds
bitrate uint64
framerate float32
}
func decodeVideo(r io.Reader, contentType string) (*mediaMeta, error) {
// decodeVideoFrame decodes and returns an image from a single frame in the given video stream.
// (note: currently this only returns a blank image resized to fit video dimensions).
func decodeVideoFrame(r io.Reader) (*gtsVideo, error) {
// We'll need a readseeker to decode the video. We can get a readseeker
// without burning too much mem by first copying the reader into a temp file.
// First create the file in the temporary directory...
tempFile, err := os.CreateTemp(os.TempDir(), "gotosocial-")
tmp, err := os.CreateTemp(os.TempDir(), "gotosocial-")
if err != nil {
return nil, fmt.Errorf("could not create temporary file while decoding video: %w", err)
return nil, err
}
tempFileName := tempFile.Name()
// Make sure to clean up the temporary file when we're done with it
defer func() {
if err := tempFile.Close(); err != nil {
log.Errorf("could not close file %s: %s", tempFileName, err)
}
if err := os.Remove(tempFileName); err != nil {
log.Errorf("could not remove file %s: %s", tempFileName, err)
}
tmp.Close()
os.Remove(tmp.Name())
}()
// Now copy the entire reader we've been provided into the
// temporary file; we won't use the reader again after this.
if _, err := io.Copy(tempFile, r); err != nil {
return nil, fmt.Errorf("could not copy video reader into temporary file %s: %w", tempFileName, err)
if _, err := io.Copy(tmp, r); err != nil {
return nil, err
}
var (
width int
height int
duration float32
framerate float32
bitrate uint64
)
// probe the video file to extract useful metadata from it; for methodology, see:
// https://github.com/abema/go-mp4/blob/7d8e5a7c5e644e0394261b0cf72fef79ce246d31/mp4tool/probe/probe.go#L85-L154
info, err := mp4.Probe(tempFile)
info, err := mp4.Probe(tmp)
if err != nil {
return nil, fmt.Errorf("could not probe temporary video file %s: %w", tempFileName, err)
return nil, fmt.Errorf("error probing tmp file %s: %w", tmp.Name(), err)
}
var (
width int
height int
video gtsVideo
)
for _, tr := range info.Tracks {
if tr.AVC == nil {
continue
@ -89,72 +81,42 @@ func decodeVideo(r io.Reader, contentType string) (*mediaMeta, error) {
height = h
}
if br := tr.Samples.GetBitrate(tr.Timescale); br > bitrate {
bitrate = br
} else if br := info.Segments.GetBitrate(tr.TrackID, tr.Timescale); br > bitrate {
bitrate = br
if br := tr.Samples.GetBitrate(tr.Timescale); br > video.bitrate {
video.bitrate = br
} else if br := info.Segments.GetBitrate(tr.TrackID, tr.Timescale); br > video.bitrate {
video.bitrate = br
}
if d := float32(tr.Duration) / float32(tr.Timescale); d > duration {
duration = d
framerate = float32(len(tr.Samples)) / duration
if d := float64(tr.Duration) / float64(tr.Timescale); d > float64(video.duration) {
video.framerate = float32(len(tr.Samples)) / float32(d)
video.duration = float32(d)
}
}
var errs gtserror.MultiError
// Check for empty video metadata.
var empty []string
if width == 0 {
errs = append(errs, "video width could not be discovered")
empty = append(empty, "width")
}
if height == 0 {
errs = append(errs, "video height could not be discovered")
empty = append(empty, "height")
}
if video.duration == 0 {
empty = append(empty, "duration")
}
if video.framerate == 0 {
empty = append(empty, "framerate")
}
if video.bitrate == 0 {
empty = append(empty, "bitrate")
}
if len(empty) > 0 {
return nil, fmt.Errorf("error determining video metadata: %v", empty)
}
if duration == 0 {
errs = append(errs, "video duration could not be discovered")
}
// Create new empty "frame" image.
// TODO: decode frame from video file.
video.frame = blankImage(width, height)
if framerate == 0 {
errs = append(errs, "video framerate could not be discovered")
}
if bitrate == 0 {
errs = append(errs, "video bitrate could not be discovered")
}
if errs != nil {
return nil, errs.Combine()
}
return &mediaMeta{
width: width,
height: height,
duration: duration,
framerate: framerate,
bitrate: bitrate,
size: height * width,
aspect: float32(width) / float32(height),
}, nil
}
func deriveThumbnailFromVideo(height int, width int) (*mediaMeta, error) {
// create a rectangle with the same dimensions as the video
img := image.NewRGBA(image.Rect(0, 0, width, height))
// fill the rectangle with our desired fill color
draw.Draw(img, img.Bounds(), &image.Uniform{thumbFill}, image.Point{}, draw.Src)
// we can get away with using extremely poor quality for this monocolor thumbnail
out := &bytes.Buffer{}
if err := jpeg.Encode(out, img, &jpeg.Options{Quality: 1}); err != nil {
return nil, fmt.Errorf("error encoding video thumbnail: %w", err)
}
return &mediaMeta{
width: width,
height: height,
size: width * height,
aspect: float32(width) / float32(height),
small: out.Bytes(),
}, nil
return &video, nil
}

View file

@ -40,7 +40,7 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSAdmin() {
fmt.Println(feed)
suite.Equal("<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss version=\"2.0\" xmlns:content=\"http://purl.org/rss/1.0/modules/content/\">\n <channel>\n <title>Posts from @admin@localhost:8080</title>\n <link>http://localhost:8080/@admin</link>\n <description>Posts from @admin@localhost:8080</description>\n <pubDate>Wed, 20 Oct 2021 12:36:45 +0000</pubDate>\n <lastBuildDate>Wed, 20 Oct 2021 12:36:45 +0000</lastBuildDate>\n <item>\n <title>open to see some puppies</title>\n <link>http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37</link>\n <description>@admin@localhost:8080 made a new post: &#34;🐕🐕🐕🐕🐕&#34;</description>\n <content:encoded><![CDATA[🐕🐕🐕🐕🐕]]></content:encoded>\n <author>@admin@localhost:8080</author>\n <guid>http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37</guid>\n <pubDate>Wed, 20 Oct 2021 12:36:45 +0000</pubDate>\n <source>http://localhost:8080/@admin/feed.rss</source>\n </item>\n <item>\n <title>hello world! #welcome ! first post on the instance :rainbow: !</title>\n <link>http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R</link>\n <description>@admin@localhost:8080 posted 1 attachment: &#34;hello world! #welcome ! first post on the instance :rainbow: !&#34;</description>\n <content:encoded><![CDATA[hello world! #welcome ! first post on the instance <img src=\"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png\" title=\":rainbow:\" alt=\":rainbow:\" class=\"emoji\"/> !]]></content:encoded>\n <author>@admin@localhost:8080</author>\n <enclosure url=\"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg\" length=\"62529\" type=\"image/jpeg\"></enclosure>\n <guid>http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R</guid>\n <pubDate>Wed, 20 Oct 2021 11:36:45 +0000</pubDate>\n <source>http://localhost:8080/@admin/feed.rss</source>\n </item>\n </channel>\n</rss>", feed)
suite.Equal("<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss version=\"2.0\" xmlns:content=\"http://purl.org/rss/1.0/modules/content/\">\n <channel>\n <title>Posts from @admin@localhost:8080</title>\n <link>http://localhost:8080/@admin</link>\n <description>Posts from @admin@localhost:8080</description>\n <pubDate>Wed, 20 Oct 2021 12:36:45 +0000</pubDate>\n <lastBuildDate>Wed, 20 Oct 2021 12:36:45 +0000</lastBuildDate>\n <item>\n <title>open to see some puppies</title>\n <link>http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37</link>\n <description>@admin@localhost:8080 made a new post: &#34;🐕🐕🐕🐕🐕&#34;</description>\n <content:encoded><![CDATA[🐕🐕🐕🐕🐕]]></content:encoded>\n <author>@admin@localhost:8080</author>\n <guid>http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37</guid>\n <pubDate>Wed, 20 Oct 2021 12:36:45 +0000</pubDate>\n <source>http://localhost:8080/@admin/feed.rss</source>\n </item>\n <item>\n <title>hello world! #welcome ! first post on the instance :rainbow: !</title>\n <link>http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R</link>\n <description>@admin@localhost:8080 posted 1 attachment: &#34;hello world! #welcome ! first post on the instance :rainbow: !&#34;</description>\n <content:encoded><![CDATA[hello world! #welcome ! first post on the instance <img src=\"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png\" title=\":rainbow:\" alt=\":rainbow:\" class=\"emoji\"/> !]]></content:encoded>\n <author>@admin@localhost:8080</author>\n <enclosure url=\"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg\" length=\"62529\" type=\"image/jpeg\"></enclosure>\n <guid>http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R</guid>\n <pubDate>Wed, 20 Oct 2021 11:36:45 +0000</pubDate>\n <source>http://localhost:8080/@admin/feed.rss</source>\n </item>\n </channel>\n</rss>", feed)
}
func (suite *GetRSSTestSuite) TestGetAccountRSSZork() {
@ -53,7 +53,7 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSZork() {
fmt.Println(feed)
suite.Equal("<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss version=\"2.0\" xmlns:content=\"http://purl.org/rss/1.0/modules/content/\">\n <channel>\n <title>Posts from @the_mighty_zork@localhost:8080</title>\n <link>http://localhost:8080/@the_mighty_zork</link>\n <description>Posts from @the_mighty_zork@localhost:8080</description>\n <pubDate>Wed, 20 Oct 2021 10:40:37 +0000</pubDate>\n <lastBuildDate>Wed, 20 Oct 2021 10:40:37 +0000</lastBuildDate>\n <image>\n <url>http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg</url>\n <title>Avatar for @the_mighty_zork@localhost:8080</title>\n <link>http://localhost:8080/@the_mighty_zork</link>\n </image>\n <item>\n <title>introduction post</title>\n <link>http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY</link>\n <description>@the_mighty_zork@localhost:8080 made a new post: &#34;hello everyone!&#34;</description>\n <content:encoded><![CDATA[hello everyone!]]></content:encoded>\n <author>@the_mighty_zork@localhost:8080</author>\n <guid>http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY</guid>\n <pubDate>Wed, 20 Oct 2021 10:40:37 +0000</pubDate>\n <source>http://localhost:8080/@the_mighty_zork/feed.rss</source>\n </item>\n </channel>\n</rss>", feed)
suite.Equal("<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss version=\"2.0\" xmlns:content=\"http://purl.org/rss/1.0/modules/content/\">\n <channel>\n <title>Posts from @the_mighty_zork@localhost:8080</title>\n <link>http://localhost:8080/@the_mighty_zork</link>\n <description>Posts from @the_mighty_zork@localhost:8080</description>\n <pubDate>Wed, 20 Oct 2021 10:40:37 +0000</pubDate>\n <lastBuildDate>Wed, 20 Oct 2021 10:40:37 +0000</lastBuildDate>\n <image>\n <url>http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg</url>\n <title>Avatar for @the_mighty_zork@localhost:8080</title>\n <link>http://localhost:8080/@the_mighty_zork</link>\n </image>\n <item>\n <title>introduction post</title>\n <link>http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY</link>\n <description>@the_mighty_zork@localhost:8080 made a new post: &#34;hello everyone!&#34;</description>\n <content:encoded><![CDATA[hello everyone!]]></content:encoded>\n <author>@the_mighty_zork@localhost:8080</author>\n <guid>http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY</guid>\n <pubDate>Wed, 20 Oct 2021 10:40:37 +0000</pubDate>\n <source>http://localhost:8080/@the_mighty_zork/feed.rss</source>\n </item>\n </channel>\n</rss>", feed)
}
func TestGetRSSTestSuite(t *testing.T) {

View file

@ -90,9 +90,8 @@ func (p *processor) emojiUpdateCopy(ctx context.Context, emoji *gtsmodel.Emoji,
newEmojiURI := uris.GenerateURIForEmoji(newEmojiID)
data := func(ctx context.Context) (reader io.ReadCloser, fileSize int64, err error) {
// 'copy' the emoji by pulling the existing one out of storage
i, err := p.storage.GetStream(ctx, emoji.ImagePath)
return i, int64(emoji.ImageFileSize), err
rc, err := p.storage.GetStream(ctx, emoji.ImagePath)
return rc, int64(emoji.ImageFileSize), err
}
var ai *media.AdditionalEmojiInfo

View file

@ -28,7 +28,6 @@
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/iotools"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/uris"
@ -99,6 +98,54 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount
return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s is not owned by %s", wantedMediaID, owningAccountID))
}
if !*a.Cached {
// if we don't have it cached, then we can assume two things:
// 1. this is remote media, since local media should never be uncached
// 2. we need to fetch it again using a transport and the media manager
remoteMediaIRI, err := url.Parse(a.RemoteURL)
if err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error parsing remote media iri %s: %s", a.RemoteURL, err))
}
// use an empty string as requestingUsername to use the instance account, unless the request for this
// media has been http signed, then use the requesting account to make the request to remote server
var requestingUsername string
if requestingAccount != nil {
requestingUsername = requestingAccount.Username
}
// Pour one out for tobi's original streamed recache
// (streaming data both to the client and storage).
// Gone and forever missed <3
//
// [
// the reason it was removed was because a slow
// client connection could hold open a storage
// recache operation, and so holding open a media
// worker worker.
// ]
dataFn := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
t, err := p.transportController.NewTransportForUsername(innerCtx, requestingUsername)
if err != nil {
return nil, 0, err
}
return t.DereferenceMedia(transport.WithFastfail(innerCtx), remoteMediaIRI)
}
// Start recaching this media with the prepared data function.
processingMedia, err := p.mediaManager.RecacheMedia(ctx, dataFn, nil, wantedMediaID)
if err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error recaching media: %s", err))
}
// Load attachment and block until complete
a, err = processingMedia.LoadAttachment(ctx)
if err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error loading recached attachment: %s", err))
}
}
// get file information from the attachment depending on the requested media size
switch mediaSize {
case media.SizeOriginal:
@ -113,121 +160,8 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount
return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for attachment", mediaSize))
}
// if we have the media cached on our server already, we can now simply return it from storage
if *a.Cached {
return p.retrieveFromStorage(ctx, storagePath, attachmentContent)
}
// if we don't have it cached, then we can assume two things:
// 1. this is remote media, since local media should never be uncached
// 2. we need to fetch it again using a transport and the media manager
remoteMediaIRI, err := url.Parse(a.RemoteURL)
if err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error parsing remote media iri %s: %s", a.RemoteURL, err))
}
// use an empty string as requestingUsername to use the instance account, unless the request for this
// media has been http signed, then use the requesting account to make the request to remote server
var requestingUsername string
if requestingAccount != nil {
requestingUsername = requestingAccount.Username
}
var data media.DataFunc
if mediaSize == media.SizeSmall {
// if it's the thumbnail that's requested then the user will have to wait a bit while we process the
// large version and derive a thumbnail from it, so use the normal recaching procedure: fetch the media,
// process it, then return the thumbnail data
data = func(innerCtx context.Context) (io.ReadCloser, int64, error) {
t, err := p.transportController.NewTransportForUsername(innerCtx, requestingUsername)
if err != nil {
return nil, 0, err
}
return t.DereferenceMedia(transport.WithFastfail(innerCtx), remoteMediaIRI)
}
} else {
// if it's the full-sized version being requested, we can cheat a bit by streaming data to the user as
// it's retrieved from the remote server, using tee; this saves the user from having to wait while
// we process the media on our side
//
// this looks a bit like this:
//
// http fetch pipe
// remote server ------------> data function ----------------> api caller
// |
// | tee
// |
// ▼
// instance storage
// This pipe will connect the caller to the in-process media retrieval...
pipeReader, pipeWriter := io.Pipe()
// Wrap the output pipe to silence any errors during the actual media
// streaming process. We catch the error later but they must be silenced
// during stream to prevent interruptions to storage of the actual media.
silencedWriter := iotools.SilenceWriter(pipeWriter)
// Pass the reader side of the pipe to the caller to slurp from.
attachmentContent.Content = pipeReader
// Create a data function which injects the writer end of the pipe
// into the data retrieval process. If something goes wrong while
// doing the data retrieval, we hang up the underlying pipeReader
// to indicate to the caller that no data is available. It's up to
// the caller of this processor function to handle that gracefully.
data = func(innerCtx context.Context) (io.ReadCloser, int64, error) {
t, err := p.transportController.NewTransportForUsername(innerCtx, requestingUsername)
if err != nil {
// propagate the transport error to read end of pipe.
_ = pipeWriter.CloseWithError(fmt.Errorf("error getting transport for user: %w", err))
return nil, 0, err
}
readCloser, fileSize, err := t.DereferenceMedia(transport.WithFastfail(innerCtx), remoteMediaIRI)
if err != nil {
// propagate the dereference error to read end of pipe.
_ = pipeWriter.CloseWithError(fmt.Errorf("error dereferencing media: %w", err))
return nil, 0, err
}
// Make a TeeReader so that everything read from the readCloser,
// aka the remote instance, will also be written into the pipe.
teeReader := io.TeeReader(readCloser, silencedWriter)
// Wrap teereader to implement original readcloser's close,
// and also ensuring that we close the pipe from write end.
return iotools.ReadFnCloser(teeReader, func() error {
defer func() {
// We use the error (if any) encountered by the
// silenced writer to close connection to make sure it
// gets propagated to the attachment.Content reader.
_ = pipeWriter.CloseWithError(silencedWriter.Error())
}()
return readCloser.Close()
}), fileSize, nil
}
}
// put the media recached in the queue
processingMedia, err := p.mediaManager.RecacheMedia(ctx, data, nil, wantedMediaID)
if err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error recaching media: %s", err))
}
// if it's the thumbnail, stream the processed thumbnail from storage, after waiting for processing to finish
if mediaSize == media.SizeSmall {
// below function call blocks until all processing on the attachment has finished...
if _, err := processingMedia.LoadAttachment(ctx); err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error loading recached attachment: %s", err))
}
// ... so now we can safely return it
return p.retrieveFromStorage(ctx, storagePath, attachmentContent)
}
return attachmentContent, nil
// ... so now we can safely return it
return p.retrieveFromStorage(ctx, storagePath, attachmentContent)
}
func (p *processor) getEmojiContent(ctx context.Context, fileName string, owningAccountID string, emojiSize media.Size) (*apimodel.Content, gtserror.WithCode) {

View file

@ -26,12 +26,14 @@
"path"
"time"
"codeberg.org/gruf/go-bytesize"
"codeberg.org/gruf/go-cache/v3/ttl"
"codeberg.org/gruf/go-store/v2/kv"
"codeberg.org/gruf/go-store/v2/storage"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/log"
)
const (
@ -63,9 +65,14 @@ func (d *Driver) URL(ctx context.Context, key string) *url.URL {
return nil
}
// access the cache member directly to avoid extending the TTL
if u, ok := d.PresignedCache.Cache.Get(key); ok {
return u.Value
// Check cache underlying cache map directly to
// avoid extending the TTL (which cache.Get() does).
d.PresignedCache.Lock()
e, ok := d.PresignedCache.Cache.Get(key)
d.PresignedCache.Unlock()
if ok {
return e.Value
}
u, err := s3.Client().PresignedGetObject(ctx, d.Bucket, key, urlCacheTTL, url.Values{
@ -88,7 +95,6 @@ func AutoConfig() (*Driver, error) {
default:
return nil, fmt.Errorf("invalid storage backend: %s", backend)
}
}
func NewFileStorage() (*Driver, error) {
@ -102,12 +108,17 @@ func NewFileStorage() (*Driver, error) {
// overwriting the lockfile if we store a file called 'store.lock'.
// However, in this case it's OK because the keys are set by
// GtS and not the user, so we know we're never going to overwrite it.
LockFile: path.Join(basePath, "store.lock"),
LockFile: path.Join(basePath, "store.lock"),
WriteBufSize: int(16 * bytesize.KiB),
})
if err != nil {
return nil, fmt.Errorf("error opening disk storage: %w", err)
}
if err := disk.Clean(context.Background()); err != nil {
log.Errorf("error performing storage cleanup: %v", err)
}
return &Driver{
KVStore: kv.New(disk),
Storage: disk,

View file

@ -51,7 +51,7 @@ func (suite *InternalToASTestSuite) TestAccountToAS() {
// this is necessary because the order of multiple 'context' entries is not determinate
trimmed := strings.Split(string(bytes), "\"discoverable\"")[1]
suite.Equal(`:true,"featured":"http://localhost:8080/users/the_mighty_zork/collections/featured","followers":"http://localhost:8080/users/the_mighty_zork/followers","following":"http://localhost:8080/users/the_mighty_zork/following","icon":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg"},"id":"http://localhost:8080/users/the_mighty_zork","image":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg"},"inbox":"http://localhost:8080/users/the_mighty_zork/inbox","manuallyApprovesFollowers":false,"name":"original zork (he/they)","outbox":"http://localhost:8080/users/the_mighty_zork/outbox","preferredUsername":"the_mighty_zork","publicKey":{"id":"http://localhost:8080/users/the_mighty_zork/main-key","owner":"http://localhost:8080/users/the_mighty_zork","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXTcOAvM1Jiw5Ffpk0qn\nr0cwbNvFe/5zQ+Tp7tumK/ZnT37o7X0FUEXrxNi+dkhmeJ0gsaiN+JQGNUewvpSk\nPIAXKvi908aSfCGjs7bGlJCJCuDuL5d6m7hZnP9rt9fJc70GElPpG0jc9fXwlz7T\nlsPb2ecatmG05Y4jPwdC+oN4MNCv9yQzEvCVMzl76EJaM602kIHC1CISn0rDFmYd\n9rSN7XPlNJw1F6PbpJ/BWQ+pXHKw3OEwNTETAUNYiVGnZU+B7a7bZC9f6/aPbJuV\nt8Qmg+UnDvW1Y8gmfHnxaWG2f5TDBvCHmcYtucIZPLQD4trAozC4ryqlmCWQNKbt\n0wIDAQAB\n-----END PUBLIC KEY-----\n"},"summary":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","tag":[],"type":"Person","url":"http://localhost:8080/@the_mighty_zork"}`, trimmed)
suite.Equal(`:true,"featured":"http://localhost:8080/users/the_mighty_zork/collections/featured","followers":"http://localhost:8080/users/the_mighty_zork/followers","following":"http://localhost:8080/users/the_mighty_zork/following","icon":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg"},"id":"http://localhost:8080/users/the_mighty_zork","image":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg"},"inbox":"http://localhost:8080/users/the_mighty_zork/inbox","manuallyApprovesFollowers":false,"name":"original zork (he/they)","outbox":"http://localhost:8080/users/the_mighty_zork/outbox","preferredUsername":"the_mighty_zork","publicKey":{"id":"http://localhost:8080/users/the_mighty_zork/main-key","owner":"http://localhost:8080/users/the_mighty_zork","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXTcOAvM1Jiw5Ffpk0qn\nr0cwbNvFe/5zQ+Tp7tumK/ZnT37o7X0FUEXrxNi+dkhmeJ0gsaiN+JQGNUewvpSk\nPIAXKvi908aSfCGjs7bGlJCJCuDuL5d6m7hZnP9rt9fJc70GElPpG0jc9fXwlz7T\nlsPb2ecatmG05Y4jPwdC+oN4MNCv9yQzEvCVMzl76EJaM602kIHC1CISn0rDFmYd\n9rSN7XPlNJw1F6PbpJ/BWQ+pXHKw3OEwNTETAUNYiVGnZU+B7a7bZC9f6/aPbJuV\nt8Qmg+UnDvW1Y8gmfHnxaWG2f5TDBvCHmcYtucIZPLQD4trAozC4ryqlmCWQNKbt\n0wIDAQAB\n-----END PUBLIC KEY-----\n"},"summary":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","tag":[],"type":"Person","url":"http://localhost:8080/@the_mighty_zork"}`, trimmed)
}
func (suite *InternalToASTestSuite) TestAccountToASWithEmoji() {
@ -72,7 +72,7 @@ func (suite *InternalToASTestSuite) TestAccountToASWithEmoji() {
// this is necessary because the order of multiple 'context' entries is not determinate
trimmed := strings.Split(string(bytes), "\"discoverable\"")[1]
suite.Equal(`:true,"featured":"http://localhost:8080/users/the_mighty_zork/collections/featured","followers":"http://localhost:8080/users/the_mighty_zork/followers","following":"http://localhost:8080/users/the_mighty_zork/following","icon":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg"},"id":"http://localhost:8080/users/the_mighty_zork","image":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg"},"inbox":"http://localhost:8080/users/the_mighty_zork/inbox","manuallyApprovesFollowers":false,"name":"original zork (he/they)","outbox":"http://localhost:8080/users/the_mighty_zork/outbox","preferredUsername":"the_mighty_zork","publicKey":{"id":"http://localhost:8080/users/the_mighty_zork/main-key","owner":"http://localhost:8080/users/the_mighty_zork","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXTcOAvM1Jiw5Ffpk0qn\nr0cwbNvFe/5zQ+Tp7tumK/ZnT37o7X0FUEXrxNi+dkhmeJ0gsaiN+JQGNUewvpSk\nPIAXKvi908aSfCGjs7bGlJCJCuDuL5d6m7hZnP9rt9fJc70GElPpG0jc9fXwlz7T\nlsPb2ecatmG05Y4jPwdC+oN4MNCv9yQzEvCVMzl76EJaM602kIHC1CISn0rDFmYd\n9rSN7XPlNJw1F6PbpJ/BWQ+pXHKw3OEwNTETAUNYiVGnZU+B7a7bZC9f6/aPbJuV\nt8Qmg+UnDvW1Y8gmfHnxaWG2f5TDBvCHmcYtucIZPLQD4trAozC4ryqlmCWQNKbt\n0wIDAQAB\n-----END PUBLIC KEY-----\n"},"summary":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji","updated":"2021-09-20T12:40:37+02:00"},"type":"Person","url":"http://localhost:8080/@the_mighty_zork"}`, trimmed)
suite.Equal(`:true,"featured":"http://localhost:8080/users/the_mighty_zork/collections/featured","followers":"http://localhost:8080/users/the_mighty_zork/followers","following":"http://localhost:8080/users/the_mighty_zork/following","icon":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg"},"id":"http://localhost:8080/users/the_mighty_zork","image":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg"},"inbox":"http://localhost:8080/users/the_mighty_zork/inbox","manuallyApprovesFollowers":false,"name":"original zork (he/they)","outbox":"http://localhost:8080/users/the_mighty_zork/outbox","preferredUsername":"the_mighty_zork","publicKey":{"id":"http://localhost:8080/users/the_mighty_zork/main-key","owner":"http://localhost:8080/users/the_mighty_zork","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXTcOAvM1Jiw5Ffpk0qn\nr0cwbNvFe/5zQ+Tp7tumK/ZnT37o7X0FUEXrxNi+dkhmeJ0gsaiN+JQGNUewvpSk\nPIAXKvi908aSfCGjs7bGlJCJCuDuL5d6m7hZnP9rt9fJc70GElPpG0jc9fXwlz7T\nlsPb2ecatmG05Y4jPwdC+oN4MNCv9yQzEvCVMzl76EJaM602kIHC1CISn0rDFmYd\n9rSN7XPlNJw1F6PbpJ/BWQ+pXHKw3OEwNTETAUNYiVGnZU+B7a7bZC9f6/aPbJuV\nt8Qmg+UnDvW1Y8gmfHnxaWG2f5TDBvCHmcYtucIZPLQD4trAozC4ryqlmCWQNKbt\n0wIDAQAB\n-----END PUBLIC KEY-----\n"},"summary":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji","updated":"2021-09-20T12:40:37+02:00"},"type":"Person","url":"http://localhost:8080/@the_mighty_zork"}`, trimmed)
}
func (suite *InternalToASTestSuite) TestAccountToASWithSharedInbox() {
@ -94,7 +94,7 @@ func (suite *InternalToASTestSuite) TestAccountToASWithSharedInbox() {
// this is necessary because the order of multiple 'context' entries is not determinate
trimmed := strings.Split(string(bytes), "\"discoverable\"")[1]
suite.Equal(`:true,"endpoints":{"sharedInbox":"http://localhost:8080/sharedInbox"},"featured":"http://localhost:8080/users/the_mighty_zork/collections/featured","followers":"http://localhost:8080/users/the_mighty_zork/followers","following":"http://localhost:8080/users/the_mighty_zork/following","icon":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg"},"id":"http://localhost:8080/users/the_mighty_zork","image":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg"},"inbox":"http://localhost:8080/users/the_mighty_zork/inbox","manuallyApprovesFollowers":false,"name":"original zork (he/they)","outbox":"http://localhost:8080/users/the_mighty_zork/outbox","preferredUsername":"the_mighty_zork","publicKey":{"id":"http://localhost:8080/users/the_mighty_zork/main-key","owner":"http://localhost:8080/users/the_mighty_zork","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXTcOAvM1Jiw5Ffpk0qn\nr0cwbNvFe/5zQ+Tp7tumK/ZnT37o7X0FUEXrxNi+dkhmeJ0gsaiN+JQGNUewvpSk\nPIAXKvi908aSfCGjs7bGlJCJCuDuL5d6m7hZnP9rt9fJc70GElPpG0jc9fXwlz7T\nlsPb2ecatmG05Y4jPwdC+oN4MNCv9yQzEvCVMzl76EJaM602kIHC1CISn0rDFmYd\n9rSN7XPlNJw1F6PbpJ/BWQ+pXHKw3OEwNTETAUNYiVGnZU+B7a7bZC9f6/aPbJuV\nt8Qmg+UnDvW1Y8gmfHnxaWG2f5TDBvCHmcYtucIZPLQD4trAozC4ryqlmCWQNKbt\n0wIDAQAB\n-----END PUBLIC KEY-----\n"},"summary":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","tag":[],"type":"Person","url":"http://localhost:8080/@the_mighty_zork"}`, trimmed)
suite.Equal(`:true,"endpoints":{"sharedInbox":"http://localhost:8080/sharedInbox"},"featured":"http://localhost:8080/users/the_mighty_zork/collections/featured","followers":"http://localhost:8080/users/the_mighty_zork/followers","following":"http://localhost:8080/users/the_mighty_zork/following","icon":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg"},"id":"http://localhost:8080/users/the_mighty_zork","image":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg"},"inbox":"http://localhost:8080/users/the_mighty_zork/inbox","manuallyApprovesFollowers":false,"name":"original zork (he/they)","outbox":"http://localhost:8080/users/the_mighty_zork/outbox","preferredUsername":"the_mighty_zork","publicKey":{"id":"http://localhost:8080/users/the_mighty_zork/main-key","owner":"http://localhost:8080/users/the_mighty_zork","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXTcOAvM1Jiw5Ffpk0qn\nr0cwbNvFe/5zQ+Tp7tumK/ZnT37o7X0FUEXrxNi+dkhmeJ0gsaiN+JQGNUewvpSk\nPIAXKvi908aSfCGjs7bGlJCJCuDuL5d6m7hZnP9rt9fJc70GElPpG0jc9fXwlz7T\nlsPb2ecatmG05Y4jPwdC+oN4MNCv9yQzEvCVMzl76EJaM602kIHC1CISn0rDFmYd\n9rSN7XPlNJw1F6PbpJ/BWQ+pXHKw3OEwNTETAUNYiVGnZU+B7a7bZC9f6/aPbJuV\nt8Qmg+UnDvW1Y8gmfHnxaWG2f5TDBvCHmcYtucIZPLQD4trAozC4ryqlmCWQNKbt\n0wIDAQAB\n-----END PUBLIC KEY-----\n"},"summary":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","tag":[],"type":"Person","url":"http://localhost:8080/@the_mighty_zork"}`, trimmed)
}
func (suite *InternalToASTestSuite) TestOutboxToASCollection() {
@ -157,7 +157,7 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASWithIDs() {
// http://joinmastodon.org/ns, https://www.w3.org/ns/activitystreams --
// will appear, so trim them out of the string for consistency
trimmed := strings.SplitAfter(string(bytes), `"attachment":`)[1]
suite.Equal(`{"blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj","mediaType":"image/jpeg","name":"Black and white image of some 50's style text saying: Welcome On Board","type":"Document","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg"},"attributedTo":"http://localhost:8080/users/admin","cc":"http://localhost:8080/users/admin/followers","content":"hello world! #welcome ! first post on the instance :rainbow: !","id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","published":"2021-10-20T11:36:45Z","replies":{"first":{"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?page=true","next":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"Collection"},"sensitive":false,"summary":"","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji","updated":"2021-09-20T10:40:37Z"},"to":"https://www.w3.org/ns/activitystreams#Public","type":"Note","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"}`, trimmed)
suite.Equal(`{"blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj","mediaType":"image/jpeg","name":"Black and white image of some 50's style text saying: Welcome On Board","type":"Document","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg"},"attributedTo":"http://localhost:8080/users/admin","cc":"http://localhost:8080/users/admin/followers","content":"hello world! #welcome ! first post on the instance :rainbow: !","id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","published":"2021-10-20T11:36:45Z","replies":{"first":{"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?page=true","next":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"Collection"},"sensitive":false,"summary":"","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji","updated":"2021-09-20T10:40:37Z"},"to":"https://www.w3.org/ns/activitystreams#Public","type":"Note","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"}`, trimmed)
}
func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() {
@ -179,7 +179,7 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() {
// http://joinmastodon.org/ns, https://www.w3.org/ns/activitystreams --
// will appear, so trim them out of the string for consistency
trimmed := strings.SplitAfter(string(bytes), `"attachment":`)[1]
suite.Equal(`{"blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj","mediaType":"image/jpeg","name":"Black and white image of some 50's style text saying: Welcome On Board","type":"Document","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg"},"attributedTo":"http://localhost:8080/users/admin","cc":"http://localhost:8080/users/admin/followers","content":"hello world! #welcome ! first post on the instance :rainbow: !","id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","published":"2021-10-20T11:36:45Z","replies":{"first":{"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?page=true","next":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"Collection"},"sensitive":false,"summary":"","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji","updated":"2021-09-20T10:40:37Z"},"to":"https://www.w3.org/ns/activitystreams#Public","type":"Note","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"}`, trimmed)
suite.Equal(`{"blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj","mediaType":"image/jpeg","name":"Black and white image of some 50's style text saying: Welcome On Board","type":"Document","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg"},"attributedTo":"http://localhost:8080/users/admin","cc":"http://localhost:8080/users/admin/followers","content":"hello world! #welcome ! first post on the instance :rainbow: !","id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","published":"2021-10-20T11:36:45Z","replies":{"first":{"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?page=true","next":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"Collection"},"sensitive":false,"summary":"","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji","updated":"2021-09-20T10:40:37Z"},"to":"https://www.w3.org/ns/activitystreams#Public","type":"Note","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"}`, trimmed)
}
func (suite *InternalToASTestSuite) TestStatusToASWithMentions() {

View file

@ -663,7 +663,7 @@ func (c *converter) InstanceToAPIInstance(ctx context.Context, i *gtsmodel.Insta
CharactersReservedPerURL: instanceStatusesCharactersReservedPerURL,
},
MediaAttachments: &apimodel.InstanceConfigurationMediaAttachments{
SupportedMimeTypes: media.AllSupportedMIMETypes(),
SupportedMimeTypes: media.SupportedMIMETypes,
ImageSizeLimit: int(config.GetMediaImageMaxSize()), // bytes
ImageMatrixLimit: instanceMediaAttachmentsImageMatrixLimit, // height*width
VideoSizeLimit: int(config.GetMediaVideoMaxSize()), // bytes

View file

@ -40,7 +40,7 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontend() {
b, err := json.Marshal(apiAccount)
suite.NoError(err)
suite.Equal(`{"id":"01F8MH1H7YV1Z7D2C8K2730QBF","username":"the_mighty_zork","acct":"the_mighty_zork","display_name":"original zork (he/they)","locked":false,"bot":false,"created_at":"2022-05-20T11:09:18.000Z","note":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","url":"http://localhost:8080/@the_mighty_zork","avatar":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg","avatar_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg","header":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","header_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","followers_count":2,"following_count":2,"statuses_count":5,"last_status_at":"2022-05-20T11:37:55.000Z","emojis":[],"fields":[],"enable_rss":true,"role":"user"}`, string(b))
suite.Equal(`{"id":"01F8MH1H7YV1Z7D2C8K2730QBF","username":"the_mighty_zork","acct":"the_mighty_zork","display_name":"original zork (he/they)","locked":false,"bot":false,"created_at":"2022-05-20T11:09:18.000Z","note":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","url":"http://localhost:8080/@the_mighty_zork","avatar":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg","avatar_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg","header":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg","header_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg","followers_count":2,"following_count":2,"statuses_count":5,"last_status_at":"2022-05-20T11:37:55.000Z","emojis":[],"fields":[],"enable_rss":true,"role":"user"}`, string(b))
}
func (suite *InternalToFrontendTestSuite) TestAccountToFrontendWithEmojiStruct() {
@ -55,7 +55,7 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontendWithEmojiStruct()
b, err := json.Marshal(apiAccount)
suite.NoError(err)
suite.Equal(`{"id":"01F8MH1H7YV1Z7D2C8K2730QBF","username":"the_mighty_zork","acct":"the_mighty_zork","display_name":"original zork (he/they)","locked":false,"bot":false,"created_at":"2022-05-20T11:09:18.000Z","note":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","url":"http://localhost:8080/@the_mighty_zork","avatar":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg","avatar_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg","header":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","header_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","followers_count":2,"following_count":2,"statuses_count":5,"last_status_at":"2022-05-20T11:37:55.000Z","emojis":[{"shortcode":"rainbow","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","static_url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","visible_in_picker":true,"category":"reactions"}],"fields":[],"enable_rss":true,"role":"user"}`, string(b))
suite.Equal(`{"id":"01F8MH1H7YV1Z7D2C8K2730QBF","username":"the_mighty_zork","acct":"the_mighty_zork","display_name":"original zork (he/they)","locked":false,"bot":false,"created_at":"2022-05-20T11:09:18.000Z","note":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","url":"http://localhost:8080/@the_mighty_zork","avatar":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg","avatar_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg","header":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg","header_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg","followers_count":2,"following_count":2,"statuses_count":5,"last_status_at":"2022-05-20T11:37:55.000Z","emojis":[{"shortcode":"rainbow","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","static_url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","visible_in_picker":true,"category":"reactions"}],"fields":[],"enable_rss":true,"role":"user"}`, string(b))
}
func (suite *InternalToFrontendTestSuite) TestAccountToFrontendWithEmojiIDs() {
@ -70,7 +70,7 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontendWithEmojiIDs() {
b, err := json.Marshal(apiAccount)
suite.NoError(err)
suite.Equal(`{"id":"01F8MH1H7YV1Z7D2C8K2730QBF","username":"the_mighty_zork","acct":"the_mighty_zork","display_name":"original zork (he/they)","locked":false,"bot":false,"created_at":"2022-05-20T11:09:18.000Z","note":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","url":"http://localhost:8080/@the_mighty_zork","avatar":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg","avatar_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg","header":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","header_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","followers_count":2,"following_count":2,"statuses_count":5,"last_status_at":"2022-05-20T11:37:55.000Z","emojis":[{"shortcode":"rainbow","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","static_url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","visible_in_picker":true,"category":"reactions"}],"fields":[],"enable_rss":true,"role":"user"}`, string(b))
suite.Equal(`{"id":"01F8MH1H7YV1Z7D2C8K2730QBF","username":"the_mighty_zork","acct":"the_mighty_zork","display_name":"original zork (he/they)","locked":false,"bot":false,"created_at":"2022-05-20T11:09:18.000Z","note":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","url":"http://localhost:8080/@the_mighty_zork","avatar":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg","avatar_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg","header":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg","header_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg","followers_count":2,"following_count":2,"statuses_count":5,"last_status_at":"2022-05-20T11:37:55.000Z","emojis":[{"shortcode":"rainbow","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","static_url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","visible_in_picker":true,"category":"reactions"}],"fields":[],"enable_rss":true,"role":"user"}`, string(b))
}
func (suite *InternalToFrontendTestSuite) TestAccountToFrontendSensitive() {
@ -81,7 +81,7 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontendSensitive() {
b, err := json.Marshal(apiAccount)
suite.NoError(err)
suite.Equal(`{"id":"01F8MH1H7YV1Z7D2C8K2730QBF","username":"the_mighty_zork","acct":"the_mighty_zork","display_name":"original zork (he/they)","locked":false,"bot":false,"created_at":"2022-05-20T11:09:18.000Z","note":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","url":"http://localhost:8080/@the_mighty_zork","avatar":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg","avatar_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg","header":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","header_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","followers_count":2,"following_count":2,"statuses_count":5,"last_status_at":"2022-05-20T11:37:55.000Z","emojis":[],"fields":[],"source":{"privacy":"public","language":"en","status_format":"plain","note":"hey yo this is my profile!","fields":[]},"enable_rss":true,"role":"user"}`, string(b))
suite.Equal(`{"id":"01F8MH1H7YV1Z7D2C8K2730QBF","username":"the_mighty_zork","acct":"the_mighty_zork","display_name":"original zork (he/they)","locked":false,"bot":false,"created_at":"2022-05-20T11:09:18.000Z","note":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","url":"http://localhost:8080/@the_mighty_zork","avatar":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg","avatar_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg","header":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg","header_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg","followers_count":2,"following_count":2,"statuses_count":5,"last_status_at":"2022-05-20T11:37:55.000Z","emojis":[],"fields":[],"source":{"privacy":"public","language":"en","status_format":"plain","note":"hey yo this is my profile!","fields":[]},"enable_rss":true,"role":"user"}`, string(b))
}
func (suite *InternalToFrontendTestSuite) TestStatusToFrontend() {
@ -93,7 +93,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontend() {
b, err := json.Marshal(apiStatus)
suite.NoError(err)
suite.Equal(`{"id":"01F8MH75CBF9JFX4ZAD54N0W0R","created_at":"2021-10-20T11:36:45.000Z","in_reply_to_id":null,"in_reply_to_account_id":null,"sensitive":false,"spoiler_text":"","visibility":"public","language":"en","uri":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","replies_count":0,"reblogs_count":0,"favourites_count":1,"favourited":true,"reblogged":false,"muted":false,"bookmarked":true,"pinned":false,"content":"hello world! #welcome ! first post on the instance :rainbow: !","reblog":null,"application":{"name":"superseriousbusiness","website":"https://superserious.business"},"account":{"id":"01F8MH17FWEB39HZJ76B6VXSKF","username":"admin","acct":"admin","display_name":"","locked":false,"bot":false,"created_at":"2022-05-17T13:10:59.000Z","note":"","url":"http://localhost:8080/@admin","avatar":"","avatar_static":"","header":"http://localhost:8080/assets/default_header.png","header_static":"http://localhost:8080/assets/default_header.png","followers_count":1,"following_count":1,"statuses_count":4,"last_status_at":"2021-10-20T10:41:37.000Z","emojis":[],"fields":[],"enable_rss":true,"role":"admin"},"media_attachments":[{"id":"01F8MH6NEM8D7527KZAECTCR76","type":"image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg","text_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg","preview_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpeg","remote_url":null,"preview_remote_url":null,"meta":{"original":{"width":1200,"height":630,"size":"1200x630","aspect":1.9047619},"small":{"width":256,"height":134,"size":"256x134","aspect":1.9104477},"focus":{"x":0,"y":0}},"description":"Black and white image of some 50's style text saying: Welcome On Board","blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj"}],"mentions":[],"tags":[{"name":"welcome","url":"http://localhost:8080/tags/welcome"}],"emojis":[{"shortcode":"rainbow","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","static_url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","visible_in_picker":true,"category":"reactions"}],"card":null,"poll":null,"text":"hello world! #welcome ! first post on the instance :rainbow: !"}`, string(b))
suite.Equal(`{"id":"01F8MH75CBF9JFX4ZAD54N0W0R","created_at":"2021-10-20T11:36:45.000Z","in_reply_to_id":null,"in_reply_to_account_id":null,"sensitive":false,"spoiler_text":"","visibility":"public","language":"en","uri":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","replies_count":0,"reblogs_count":0,"favourites_count":1,"favourited":true,"reblogged":false,"muted":false,"bookmarked":true,"pinned":false,"content":"hello world! #welcome ! first post on the instance :rainbow: !","reblog":null,"application":{"name":"superseriousbusiness","website":"https://superserious.business"},"account":{"id":"01F8MH17FWEB39HZJ76B6VXSKF","username":"admin","acct":"admin","display_name":"","locked":false,"bot":false,"created_at":"2022-05-17T13:10:59.000Z","note":"","url":"http://localhost:8080/@admin","avatar":"","avatar_static":"","header":"http://localhost:8080/assets/default_header.png","header_static":"http://localhost:8080/assets/default_header.png","followers_count":1,"following_count":1,"statuses_count":4,"last_status_at":"2021-10-20T10:41:37.000Z","emojis":[],"fields":[],"enable_rss":true,"role":"admin"},"media_attachments":[{"id":"01F8MH6NEM8D7527KZAECTCR76","type":"image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg","text_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg","preview_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpg","remote_url":null,"preview_remote_url":null,"meta":{"original":{"width":1200,"height":630,"size":"1200x630","aspect":1.9047619},"small":{"width":256,"height":134,"size":"256x134","aspect":1.9104477},"focus":{"x":0,"y":0}},"description":"Black and white image of some 50's style text saying: Welcome On Board","blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj"}],"mentions":[],"tags":[{"name":"welcome","url":"http://localhost:8080/tags/welcome"}],"emojis":[{"shortcode":"rainbow","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","static_url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","visible_in_picker":true,"category":"reactions"}],"card":null,"poll":null,"text":"hello world! #welcome ! first post on the instance :rainbow: !"}`, string(b))
}
func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownLanguage() {
@ -107,7 +107,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownLanguage()
b, err := json.Marshal(apiStatus)
suite.NoError(err)
suite.Equal(`{"id":"01F8MH75CBF9JFX4ZAD54N0W0R","created_at":"2021-10-20T11:36:45.000Z","in_reply_to_id":null,"in_reply_to_account_id":null,"sensitive":false,"spoiler_text":"","visibility":"public","language":null,"uri":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","replies_count":0,"reblogs_count":0,"favourites_count":1,"favourited":true,"reblogged":false,"muted":false,"bookmarked":true,"pinned":false,"content":"hello world! #welcome ! first post on the instance :rainbow: !","reblog":null,"application":{"name":"superseriousbusiness","website":"https://superserious.business"},"account":{"id":"01F8MH17FWEB39HZJ76B6VXSKF","username":"admin","acct":"admin","display_name":"","locked":false,"bot":false,"created_at":"2022-05-17T13:10:59.000Z","note":"","url":"http://localhost:8080/@admin","avatar":"","avatar_static":"","header":"http://localhost:8080/assets/default_header.png","header_static":"http://localhost:8080/assets/default_header.png","followers_count":1,"following_count":1,"statuses_count":4,"last_status_at":"2021-10-20T10:41:37.000Z","emojis":[],"fields":[],"enable_rss":true,"role":"admin"},"media_attachments":[{"id":"01F8MH6NEM8D7527KZAECTCR76","type":"image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg","text_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg","preview_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpeg","remote_url":null,"preview_remote_url":null,"meta":{"original":{"width":1200,"height":630,"size":"1200x630","aspect":1.9047619},"small":{"width":256,"height":134,"size":"256x134","aspect":1.9104477},"focus":{"x":0,"y":0}},"description":"Black and white image of some 50's style text saying: Welcome On Board","blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj"}],"mentions":[],"tags":[{"name":"welcome","url":"http://localhost:8080/tags/welcome"}],"emojis":[{"shortcode":"rainbow","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","static_url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","visible_in_picker":true,"category":"reactions"}],"card":null,"poll":null,"text":"hello world! #welcome ! first post on the instance :rainbow: !"}`, string(b))
suite.Equal(`{"id":"01F8MH75CBF9JFX4ZAD54N0W0R","created_at":"2021-10-20T11:36:45.000Z","in_reply_to_id":null,"in_reply_to_account_id":null,"sensitive":false,"spoiler_text":"","visibility":"public","language":null,"uri":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","replies_count":0,"reblogs_count":0,"favourites_count":1,"favourited":true,"reblogged":false,"muted":false,"bookmarked":true,"pinned":false,"content":"hello world! #welcome ! first post on the instance :rainbow: !","reblog":null,"application":{"name":"superseriousbusiness","website":"https://superserious.business"},"account":{"id":"01F8MH17FWEB39HZJ76B6VXSKF","username":"admin","acct":"admin","display_name":"","locked":false,"bot":false,"created_at":"2022-05-17T13:10:59.000Z","note":"","url":"http://localhost:8080/@admin","avatar":"","avatar_static":"","header":"http://localhost:8080/assets/default_header.png","header_static":"http://localhost:8080/assets/default_header.png","followers_count":1,"following_count":1,"statuses_count":4,"last_status_at":"2021-10-20T10:41:37.000Z","emojis":[],"fields":[],"enable_rss":true,"role":"admin"},"media_attachments":[{"id":"01F8MH6NEM8D7527KZAECTCR76","type":"image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg","text_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg","preview_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpg","remote_url":null,"preview_remote_url":null,"meta":{"original":{"width":1200,"height":630,"size":"1200x630","aspect":1.9047619},"small":{"width":256,"height":134,"size":"256x134","aspect":1.9104477},"focus":{"x":0,"y":0}},"description":"Black and white image of some 50's style text saying: Welcome On Board","blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj"}],"mentions":[],"tags":[{"name":"welcome","url":"http://localhost:8080/tags/welcome"}],"emojis":[{"shortcode":"rainbow","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","static_url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","visible_in_picker":true,"category":"reactions"}],"card":null,"poll":null,"text":"hello world! #welcome ! first post on the instance :rainbow: !"}`, string(b))
}
func (suite *InternalToFrontendTestSuite) TestVideoAttachmentToFrontend() {
@ -118,7 +118,7 @@ func (suite *InternalToFrontendTestSuite) TestVideoAttachmentToFrontend() {
b, err := json.Marshal(apiAttachment)
suite.NoError(err)
suite.Equal(`{"id":"01CDR64G398ADCHXK08WWTHEZ5","type":"video","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.mp4","text_url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.mp4","preview_url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpeg","remote_url":null,"preview_remote_url":null,"meta":{"original":{"width":720,"height":404,"frame_rate":"30/1","duration":15.033334,"bitrate":1206522,"size":"720x404","aspect":1.7821782},"small":{"width":720,"height":404,"size":"720x404","aspect":1.7821782},"focus":{"x":0,"y":0}},"description":"A cow adorably licking another cow!"}`, string(b))
suite.Equal(`{"id":"01CDR64G398ADCHXK08WWTHEZ5","type":"video","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.mp4","text_url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.mp4","preview_url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpg","remote_url":null,"preview_remote_url":null,"meta":{"original":{"width":720,"height":404,"frame_rate":"30/1","duration":15.033334,"bitrate":1206522,"size":"720x404","aspect":1.7821782},"small":{"width":720,"height":404,"size":"720x404","aspect":1.7821782},"focus":{"x":0,"y":0}},"description":"A cow adorably licking another cow!"}`, string(b))
}
func (suite *InternalToFrontendTestSuite) TestInstanceToFrontend() {

View file

@ -77,7 +77,7 @@ func (suite *InternalToRSSTestSuite) TestStatusToRSSItem2() {
suite.EqualValues(1634729805, item.Created.Unix())
suite.Equal("62529", item.Enclosure.Length)
suite.Equal("image/jpeg", item.Enclosure.Type)
suite.Equal("http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg", item.Enclosure.Url)
suite.Equal("http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg", item.Enclosure.Url)
suite.Equal("hello world! #welcome ! first post on the instance <img src=\"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png\" title=\":rainbow:\" alt=\":rainbow:\" class=\"emoji\"/> !", item.Content)
}

View file

@ -62,7 +62,7 @@ func happyMediaAttachment() *gtsmodel.MediaAttachment {
CreatedAt: time.Now().Add(-71 * time.Hour),
UpdatedAt: time.Now().Add(-71 * time.Hour),
StatusID: "01F8MH75CBF9JFX4ZAD54N0W0R",
URL: "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg",
URL: "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg",
RemoteURL: "",
Type: gtsmodel.FileTypeImage,
FileMeta: gtsmodel.FileMeta{
@ -95,7 +95,7 @@ func happyMediaAttachment() *gtsmodel.MediaAttachment {
ContentType: "image/jpeg",
FileSize: 6872,
UpdatedAt: time.Now().Add(-71 * time.Hour),
URL: "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpeg",
URL: "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpg",
RemoteURL: "",
},
Avatar: testrig.FalseBool(),

View file

Before

Width:  |  Height:  |  Size: 27 KiB

After

Width:  |  Height:  |  Size: 27 KiB

View file

Before

Width:  |  Height:  |  Size: 6 KiB

After

Width:  |  Height:  |  Size: 6 KiB

View file

Before

Width:  |  Height:  |  Size: 505 KiB

After

Width:  |  Height:  |  Size: 505 KiB

View file

Before

Width:  |  Height:  |  Size: 41 KiB

After

Width:  |  Height:  |  Size: 41 KiB

View file

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 19 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 20 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 19 KiB

View file

Before

Width:  |  Height:  |  Size: 8.6 KiB

After

Width:  |  Height:  |  Size: 8.6 KiB

View file

Before

Width:  |  Height:  |  Size: 61 KiB

After

Width:  |  Height:  |  Size: 61 KiB

View file

Before

Width:  |  Height:  |  Size: 6.7 KiB

After

Width:  |  Height:  |  Size: 6.7 KiB

View file

Before

Width:  |  Height:  |  Size: 447 KiB

After

Width:  |  Height:  |  Size: 447 KiB

View file

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

View file

@ -55,14 +55,14 @@ func StandardStorageSetup(storage *gtsstorage.Driver, relativePath string) {
if err != nil {
panic(err)
}
if err := storage.Put(context.TODO(), pathOriginal, bOriginal); err != nil {
if _, err := storage.Put(context.TODO(), pathOriginal, bOriginal); err != nil {
panic(err)
}
bSmall, err := os.ReadFile(fmt.Sprintf("%s/%s", relativePath, filenameSmall))
if err != nil {
panic(err)
}
if err := storage.Put(context.TODO(), pathSmall, bSmall); err != nil {
if _, err := storage.Put(context.TODO(), pathSmall, bSmall); err != nil {
panic(err)
}
}
@ -82,14 +82,14 @@ func StandardStorageSetup(storage *gtsstorage.Driver, relativePath string) {
if err != nil {
panic(err)
}
if err := storage.Put(context.TODO(), pathOriginal, bOriginal); err != nil {
if _, err := storage.Put(context.TODO(), pathOriginal, bOriginal); err != nil {
panic(err)
}
bStatic, err := os.ReadFile(fmt.Sprintf("%s/%s", relativePath, filenameStatic))
if err != nil {
panic(err)
}
if err := storage.Put(context.TODO(), pathStatic, bStatic); err != nil {
if _, err := storage.Put(context.TODO(), pathStatic, bStatic); err != nil {
panic(err)
}
}

View file

@ -689,7 +689,7 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
"admin_account_status_1_attachment_1": {
ID: "01F8MH6NEM8D7527KZAECTCR76",
StatusID: "01F8MH75CBF9JFX4ZAD54N0W0R",
URL: "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg",
URL: "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg",
RemoteURL: "",
CreatedAt: TimeMustParse("2022-06-04T13:12:00Z"),
UpdatedAt: TimeMustParse("2022-06-04T13:12:00Z"),
@ -714,17 +714,17 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
Blurhash: "LNJRdVM{00Rj%Mayt7j[4nWBofRj",
Processing: 2,
File: gtsmodel.File{
Path: "01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg",
Path: "01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg",
ContentType: "image/jpeg",
FileSize: 62529,
UpdatedAt: TimeMustParse("2022-06-04T13:12:00Z"),
},
Thumbnail: gtsmodel.Thumbnail{
Path: "01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpeg",
Path: "01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpg",
ContentType: "image/jpeg",
FileSize: 6872,
UpdatedAt: TimeMustParse("2022-06-04T13:12:00Z"),
URL: "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpeg",
URL: "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpg",
RemoteURL: "",
},
Avatar: FalseBool(),
@ -769,11 +769,11 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
},
Thumbnail: gtsmodel.Thumbnail{
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH7TDVANYKWVE8VVKFPJTJ.jpeg",
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH7TDVANYKWVE8VVKFPJTJ.jpg",
ContentType: "image/jpeg",
FileSize: 8803,
UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH7TDVANYKWVE8VVKFPJTJ.jpeg",
URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH7TDVANYKWVE8VVKFPJTJ.jpg",
RemoteURL: "",
},
Avatar: FalseBool(),
@ -821,11 +821,11 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
},
Thumbnail: gtsmodel.Thumbnail{
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpeg",
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpg",
ContentType: "image/jpeg",
FileSize: 5272,
UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpeg",
URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpg",
RemoteURL: "",
},
Avatar: FalseBool(),
@ -835,7 +835,7 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
"local_account_1_unattached_1": {
ID: "01F8MH8RMYQ6MSNY3JM2XT1CQ5",
StatusID: "", // this attachment isn't connected to a status YET
URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpeg",
URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpg",
RemoteURL: "",
CreatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
@ -864,17 +864,17 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
Blurhash: "LSAd]9ogDge-R:M|j=xWIto0xXWX",
Processing: 2,
File: gtsmodel.File{
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpeg",
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpg",
ContentType: "image/jpeg",
FileSize: 27759,
UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
},
Thumbnail: gtsmodel.Thumbnail{
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpeg",
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpg",
ContentType: "image/jpeg",
FileSize: 6177,
UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpeg",
URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpg",
RemoteURL: "",
},
Avatar: FalseBool(),
@ -884,7 +884,7 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
"local_account_1_avatar": {
ID: "01F8MH58A357CV5K7R7TJMSH6S",
StatusID: "", // this attachment isn't connected to a status
URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg",
URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg",
RemoteURL: "",
CreatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
@ -913,17 +913,17 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
Blurhash: "LKK9MT,p|YSNDkJ-5rsmvnwcOoe:",
Processing: 2,
File: gtsmodel.File{
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg",
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg",
ContentType: "image/jpeg",
FileSize: 457680,
UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
},
Thumbnail: gtsmodel.Thumbnail{
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg",
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg",
ContentType: "image/jpeg",
FileSize: 15374,
UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg",
URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg",
RemoteURL: "",
},
Avatar: TrueBool(),
@ -933,7 +933,7 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
"local_account_1_header": {
ID: "01PFPMWK2FF0D9WMHEJHR07C3Q",
StatusID: "",
URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg",
URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg",
RemoteURL: "",
CreatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
@ -962,17 +962,17 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
Blurhash: "L26j{^WCs+R-N}jsxWj@4;WWxDoK",
Processing: 2,
File: gtsmodel.File{
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg",
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg",
ContentType: "image/jpeg",
FileSize: 517226,
UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
},
Thumbnail: gtsmodel.Thumbnail{
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg",
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg",
ContentType: "image/jpeg",
FileSize: 42308,
UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg",
URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg",
RemoteURL: "",
},
Avatar: FalseBool(),
@ -982,8 +982,8 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
"remote_account_1_status_1_attachment_1": {
ID: "01FVW7RXPQ8YJHTEXYPE7Q8ZY0",
StatusID: "01FVW7JHQFSFK166WWKR8CBA6M",
URL: "http://localhost:8080/fileserver/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
RemoteURL: "http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpeg",
URL: "http://localhost:8080/fileserver/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg",
RemoteURL: "http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpg",
CreatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"),
UpdatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"),
Type: gtsmodel.FileTypeImage,
@ -1011,18 +1011,18 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
Blurhash: "LARysgM_IU_3~pD%M_Rj_39FIAt6",
Processing: 2,
File: gtsmodel.File{
Path: "01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
Path: "01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg",
ContentType: "image/jpeg",
FileSize: 19310,
UpdatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"),
},
Thumbnail: gtsmodel.Thumbnail{
Path: "01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
Path: "01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg",
ContentType: "image/jpeg",
FileSize: 20395,
FileSize: 19312,
UpdatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"),
URL: "http://localhost:8080/fileserver/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpeg",
URL: "http://localhost:8080/fileserver/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg",
RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpg",
},
Avatar: FalseBool(),
Header: FalseBool(),
@ -1031,8 +1031,8 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
"remote_account_1_status_1_attachment_2": {
ID: "01FVW7RXPQ8YJHTEXYPE7Q8ZY1",
StatusID: "01FVW7JHQFSFK166WWKR8CBA6M",
URL: "http://localhost:8080/fileserver/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
RemoteURL: "http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpeg",
URL: "http://localhost:8080/fileserver/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg",
RemoteURL: "http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpg",
CreatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"),
UpdatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"),
Type: gtsmodel.FileTypeImage,
@ -1060,18 +1060,18 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
Blurhash: "LARysgM_IU_3~pD%M_Rj_39FIAt6",
Processing: 2,
File: gtsmodel.File{
Path: "01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
Path: "01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg",
ContentType: "image/jpeg",
FileSize: 19310,
UpdatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"),
},
Thumbnail: gtsmodel.Thumbnail{
Path: "01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
Path: "01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg",
ContentType: "image/jpeg",
FileSize: 20395,
UpdatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"),
URL: "http://localhost:8080/fileserver/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpeg",
URL: "http://localhost:8080/fileserver/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg",
RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpg",
},
Avatar: FalseBool(),
Header: FalseBool(),
@ -1080,8 +1080,8 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
"remote_account_3_header": {
ID: "01PFPMWK2FF0D9WMHEJHR07C3R",
StatusID: "",
URL: "http://localhost:8080/fileserver/062G5WYKY35KKD12EMSM3F8PJ8/header/original/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg",
RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpeg",
URL: "http://localhost:8080/fileserver/062G5WYKY35KKD12EMSM3F8PJ8/header/original/01PFPMWK2FF0D9WMHEJHR07C3R.jpg",
RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpg",
CreatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
Type: gtsmodel.FileTypeImage,
@ -1109,18 +1109,18 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
Blurhash: "LARysgM_IU_3~pD%M_Rj_39FIAt6",
Processing: 2,
File: gtsmodel.File{
Path: "062G5WYKY35KKD12EMSM3F8PJ8/attachment/original/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg",
Path: "062G5WYKY35KKD12EMSM3F8PJ8/attachment/original/01PFPMWK2FF0D9WMHEJHR07C3R.jpg",
ContentType: "image/jpeg",
FileSize: 19310,
UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
},
Thumbnail: gtsmodel.Thumbnail{
Path: "062G5WYKY35KKD12EMSM3F8PJ8/attachment/small/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg",
Path: "062G5WYKY35KKD12EMSM3F8PJ8/attachment/small/01PFPMWK2FF0D9WMHEJHR07C3R.jpg",
ContentType: "image/jpeg",
FileSize: 20395,
UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
URL: "http://localhost:8080/fileserver/062G5WYKY35KKD12EMSM3F8PJ8/header/small/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg",
RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpeg",
URL: "http://localhost:8080/fileserver/062G5WYKY35KKD12EMSM3F8PJ8/header/small/01PFPMWK2FF0D9WMHEJHR07C3R.jpg",
RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpg",
},
Avatar: FalseBool(),
Header: TrueBool(),
@ -1262,32 +1262,32 @@ type filenames struct {
func newTestStoredAttachments() map[string]filenames {
return map[string]filenames{
"admin_account_status_1_attachment_1": {
Original: "welcome-original.jpeg",
Small: "welcome-small.jpeg",
Original: "welcome-original.jpg",
Small: "welcome-small.jpg",
},
"local_account_1_status_4_attachment_1": {
Original: "trent-original.gif",
Small: "trent-small.jpeg",
Small: "trent-small.jpg",
},
"local_account_1_status_4_attachment_2": {
Original: "cowlick-original.mp4",
Small: "cowlick-small.jpeg",
},
"local_account_1_unattached_1": {
Original: "ohyou-original.jpeg",
Small: "ohyou-small.jpeg",
Original: "ohyou-original.jpg",
Small: "ohyou-small.jpg",
},
"local_account_1_avatar": {
Original: "zork-original.jpeg",
Small: "zork-small.jpeg",
Original: "zork-original.jpg",
Small: "zork-small.jpg",
},
"local_account_1_header": {
Original: "team-fortress-original.jpeg",
Small: "team-fortress-small.jpeg",
Original: "team-fortress-original.jpg",
Small: "team-fortress-small.jpg",
},
"remote_account_1_status_1_attachment_1": {
Original: "thoughtsofdog-original.jpeg",
Small: "thoughtsofdog-small.jpeg",
Original: "thoughtsofdog-original.jpg",
Small: "thoughtsofdog-small.jpg",
},
}
}
@ -2070,7 +2070,7 @@ func NewTestActivities(accounts map[string]*gtsmodel.Account) map[string]Activit
[]vocab.ActivityStreamsMention{},
[]vocab.ActivityStreamsImage{
newAPImage(
URLMustParse("http://example.org/users/Some_User/statuses/afaba698-5740-4e32-a702-af61aa543bc1/attachment1.jpeg"),
URLMustParse("http://example.org/users/Some_User/statuses/afaba698-5740-4e32-a702-af61aa543bc1/attachment1.jpg"),
"image/jpeg",
"trent reznor looking handsome as balls",
"LEDara58O=t5EMSOENEN9]}?aK%0"),
@ -2322,7 +2322,7 @@ func NewTestFediAttachments(relativePath string) map[string]RemoteAttachmentFile
panic(err)
}
thoughtsOfDogBytes, err := os.ReadFile(fmt.Sprintf("%s/thoughtsofdog-original.jpeg", relativePath))
thoughtsOfDogBytes, err := os.ReadFile(fmt.Sprintf("%s/thoughtsofdog-original.jpg", relativePath))
if err != nil {
panic(err)
}
@ -2352,7 +2352,7 @@ func NewTestFediAttachments(relativePath string) map[string]RemoteAttachmentFile
Data: beeBytes,
ContentType: "image/jpeg",
},
"http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpeg": {
"http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpg": {
Data: thoughtsOfDogBytes,
ContentType: "image/jpeg",
},
@ -2390,7 +2390,7 @@ func NewTestFediStatuses() map[string]vocab.ActivityStreamsNote {
[]vocab.ActivityStreamsMention{},
[]vocab.ActivityStreamsImage{
newAPImage(
URLMustParse("http://example.org/users/Some_User/statuses/afaba698-5740-4e32-a702-af61aa543bc1/attachment1.jpeg"),
URLMustParse("http://example.org/users/Some_User/statuses/afaba698-5740-4e32-a702-af61aa543bc1/attachment1.jpg"),
"image/jpeg",
"trent reznor looking handsome as balls",
"LEDara58O=t5EMSOENEN9]}?aK%0"),

View file

@ -78,16 +78,16 @@ func (cp *CopyPool) Copy(dst io.Writer, src io.Reader) (int64, error) {
var buf []byte
if b, ok := cp.pool.Get().([]byte); ok {
if b, ok := cp.pool.Get().(*[]byte); ok {
// Acquired buf from pool
buf = b
buf = *b
} else {
// Allocate new buffer of size
buf = make([]byte, cp.Buffer(0))
}
// Defer release to pool
defer cp.pool.Put(buf)
defer cp.pool.Put(&buf)
var n int64
for {

9
vendor/codeberg.org/gruf/go-iotools/LICENSE generated vendored Normal file
View file

@ -0,0 +1,9 @@
MIT License
Copyright (c) 2022 gruf
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

35
vendor/codeberg.org/gruf/go-iotools/close.go generated vendored Normal file
View file

@ -0,0 +1,35 @@
package iotools
import "io"
// CloserFunc is a function signature which allows
// a function to implement the io.Closer type.
type CloserFunc func() error
func (c CloserFunc) Close() error {
return c()
}
func CloserCallback(c io.Closer, cb func()) io.Closer {
return CloserFunc(func() error {
defer cb()
return c.Close()
})
}
// CloseOnce wraps an io.Closer to ensure it only performs the close logic once.
func CloseOnce(c io.Closer) io.Closer {
return CloserFunc(func() error {
if c == nil {
// already run.
return nil
}
// Acquire.
cptr := c
c = nil
// Call the closer.
return cptr.Close()
})
}

28
vendor/codeberg.org/gruf/go-iotools/read.go generated vendored Normal file
View file

@ -0,0 +1,28 @@
package iotools
import (
"io"
)
// ReaderFunc is a function signature which allows
// a function to implement the io.Reader type.
type ReaderFunc func([]byte) (int, error)
func (r ReaderFunc) Read(b []byte) (int, error) {
return r(b)
}
// ReadCloser wraps an io.Reader and io.Closer in order to implement io.ReadCloser.
func ReadCloser(r io.Reader, c io.Closer) io.ReadCloser {
return &struct {
io.Reader
io.Closer
}{r, c}
}
// NopReadCloser wraps an io.Reader to implement io.ReadCloser with empty io.Closer implementation.
func NopReadCloser(r io.Reader) io.ReadCloser {
return ReadCloser(r, CloserFunc(func() error {
return nil
}))
}

26
vendor/codeberg.org/gruf/go-iotools/write.go generated vendored Normal file
View file

@ -0,0 +1,26 @@
package iotools
import "io"
// WriterFunc is a function signature which allows
// a function to implement the io.Writer type.
type WriterFunc func([]byte) (int, error)
func (w WriterFunc) Write(b []byte) (int, error) {
return w(b)
}
// WriteCloser wraps an io.Writer and io.Closer in order to implement io.WriteCloser.
func WriteCloser(w io.Writer, c io.Closer) io.WriteCloser {
return &struct {
io.Writer
io.Closer
}{w, c}
}
// NopWriteCloser wraps an io.Writer to implement io.WriteCloser with empty io.Closer implementation.
func NopWriteCloser(w io.Writer) io.WriteCloser {
return WriteCloser(w, CloserFunc(func() error {
return nil
}))
}

View file

@ -454,7 +454,9 @@ func (mu *rwmutex) Unlock() {
if mu.rcnt > 0 {
// RUnlock
mu.rcnt--
} else {
}
if mu.rcnt == 0 {
// Total unlock
mu.lock = 0
}

View file

@ -77,17 +77,17 @@ func (st *StateRW) GetStream(ctx context.Context, key string) (io.ReadCloser, er
}
// Put: see KVStore.Put(). Returns error if state already closed.
func (st *StateRW) Put(ctx context.Context, key string, value []byte) error {
func (st *StateRW) Put(ctx context.Context, key string, value []byte) (int, error) {
if st.store == nil {
return ErrStateClosed
return 0, ErrStateClosed
}
return st.store.put(st.state.Lock, ctx, key, value)
}
// PutStream: see KVStore.PutStream(). Returns error if state already closed.
func (st *StateRW) PutStream(ctx context.Context, key string, r io.Reader) error {
func (st *StateRW) PutStream(ctx context.Context, key string, r io.Reader) (int64, error) {
if st.store == nil {
return ErrStateClosed
return 0, ErrStateClosed
}
return st.store.putStream(st.state.Lock, ctx, key, r)
}

View file

@ -4,9 +4,9 @@
"context"
"io"
"codeberg.org/gruf/go-iotools"
"codeberg.org/gruf/go-mutexes"
"codeberg.org/gruf/go-store/v2/storage"
"codeberg.org/gruf/go-store/v2/util"
)
// KVStore is a very simple, yet performant key-value store
@ -117,17 +117,25 @@ func (st *KVStore) getStream(rlock func(string) func(), ctx context.Context, key
return nil, err
}
// Wrap readcloser in our own callback closer
return util.ReadCloserWithCallback(rd, runlock), nil
var unlocked bool
// Wrap readcloser to call our own callback
return iotools.ReadCloser(rd, iotools.CloserFunc(func() error {
if !unlocked {
unlocked = true
defer runlock()
}
return rd.Close()
})), nil
}
// Put places the bytes at the supplied key in the store.
func (st *KVStore) Put(ctx context.Context, key string, value []byte) error {
func (st *KVStore) Put(ctx context.Context, key string, value []byte) (int, error) {
return st.put(st.Lock, ctx, key, value)
}
// put performs the underlying logic for KVStore.Put(), using supplied lock func to allow use with states.
func (st *KVStore) put(lock func(string) func(), ctx context.Context, key string, value []byte) error {
func (st *KVStore) put(lock func(string) func(), ctx context.Context, key string, value []byte) (int, error) {
// Acquire write lock for key
unlock := lock(key)
defer unlock()
@ -137,12 +145,12 @@ func (st *KVStore) put(lock func(string) func(), ctx context.Context, key string
}
// PutStream writes the bytes from the supplied Reader at the supplied key in the store.
func (st *KVStore) PutStream(ctx context.Context, key string, r io.Reader) error {
func (st *KVStore) PutStream(ctx context.Context, key string, r io.Reader) (int64, error) {
return st.putStream(st.Lock, ctx, key, r)
}
// putStream performs the underlying logic for KVStore.PutStream(), using supplied lock func to allow use with states.
func (st *KVStore) putStream(lock func(string) func(), ctx context.Context, key string, r io.Reader) error {
func (st *KVStore) putStream(lock func(string) func(), ctx context.Context, key string, r io.Reader) (int64, error) {
// Acquire write lock for key
unlock := lock(key)
defer unlock()

View file

@ -10,12 +10,14 @@
"os"
"strings"
"sync"
"sync/atomic"
"syscall"
"codeberg.org/gruf/go-byteutil"
"codeberg.org/gruf/go-errors/v2"
"codeberg.org/gruf/go-fastcopy"
"codeberg.org/gruf/go-hashenc"
"codeberg.org/gruf/go-iotools"
"codeberg.org/gruf/go-pools"
"codeberg.org/gruf/go-store/v2/util"
)
@ -354,7 +356,7 @@ func (st *BlockStorage) ReadStream(ctx context.Context, key string) (io.ReadClos
}
// Prepare block reader and return
return util.NopReadCloser(&blockReader{
return iotools.NopReadCloser(&blockReader{
storage: st,
node: &node,
}), nil
@ -384,52 +386,54 @@ func (st *BlockStorage) readBlock(key string) ([]byte, error) {
}
// WriteBytes implements Storage.WriteBytes().
func (st *BlockStorage) WriteBytes(ctx context.Context, key string, value []byte) error {
return st.WriteStream(ctx, key, bytes.NewReader(value))
func (st *BlockStorage) WriteBytes(ctx context.Context, key string, value []byte) (int, error) {
n, err := st.WriteStream(ctx, key, bytes.NewReader(value))
return int(n), err
}
// WriteStream implements Storage.WriteStream().
func (st *BlockStorage) WriteStream(ctx context.Context, key string, r io.Reader) error {
func (st *BlockStorage) WriteStream(ctx context.Context, key string, r io.Reader) (int64, error) {
// Get node file path for key
npath, err := st.nodePathForKey(key)
if err != nil {
return err
return 0, err
}
// Check if open
if st.lock.Closed() {
return ErrClosed
return 0, ErrClosed
}
// Check context still valid
if err := ctx.Err(); err != nil {
return err
return 0, err
}
// Check if this exists
ok, err := stat(key)
if err != nil {
return err
return 0, err
}
// Check if we allow overwrites
if ok && !st.config.Overwrite {
return ErrAlreadyExists
return 0, ErrAlreadyExists
}
// Ensure nodes dir (and any leading up to) exists
err = os.MkdirAll(st.nodePath, defaultDirPerms)
if err != nil {
return err
return 0, err
}
// Ensure blocks dir (and any leading up to) exists
err = os.MkdirAll(st.blockPath, defaultDirPerms)
if err != nil {
return err
return 0, err
}
var node node
var total atomic.Int64
// Acquire HashEncoder
hc := st.hashPool.Get().(*hashEncoder)
@ -456,7 +460,7 @@ func (st *BlockStorage) WriteStream(ctx context.Context, key string, r io.Reader
break loop
default:
st.bufpool.Put(buf)
return err
return 0, err
}
// Hash the encoded data
@ -469,7 +473,7 @@ func (st *BlockStorage) WriteStream(ctx context.Context, key string, r io.Reader
has, err := st.statBlock(sum)
if err != nil {
st.bufpool.Put(buf)
return err
return 0, err
} else if has {
st.bufpool.Put(buf)
continue loop
@ -490,11 +494,14 @@ func (st *BlockStorage) WriteStream(ctx context.Context, key string, r io.Reader
}()
// Write block to store at hash
err = st.writeBlock(sum, buf.B[:n])
n, err := st.writeBlock(sum, buf.B[:n])
if err != nil {
onceErr.Store(err)
return
}
// Increment total.
total.Add(int64(n))
}()
// Break at end
@ -506,12 +513,12 @@ func (st *BlockStorage) WriteStream(ctx context.Context, key string, r io.Reader
// Wait, check errors
wg.Wait()
if onceErr.IsSet() {
return onceErr.Load()
return 0, onceErr.Load()
}
// If no hashes created, return
if len(node.hashes) < 1 {
return new_error("no hashes written")
return 0, new_error("no hashes written")
}
// Prepare to swap error if need-be
@ -535,7 +542,7 @@ func (st *BlockStorage) WriteStream(ctx context.Context, key string, r io.Reader
// Attempt to open RW file
file, err := open(npath, flags)
if err != nil {
return errSwap(err)
return 0, errSwap(err)
}
defer file.Close()
@ -546,11 +553,11 @@ func (st *BlockStorage) WriteStream(ctx context.Context, key string, r io.Reader
// Finally, write data to file
_, err = io.CopyBuffer(file, &nodeReader{node: node}, buf.B)
return err
return total.Load(), err
}
// writeBlock writes the block with hash and supplied value to the filesystem.
func (st *BlockStorage) writeBlock(hash string, value []byte) error {
func (st *BlockStorage) writeBlock(hash string, value []byte) (int, error) {
// Get block file path for key
bpath := st.blockPathForKey(hash)
@ -560,20 +567,19 @@ func (st *BlockStorage) writeBlock(hash string, value []byte) error {
if err == syscall.EEXIST {
err = nil /* race issue describe in struct NOTE */
}
return err
return 0, err
}
defer file.Close()
// Wrap the file in a compressor
cFile, err := st.config.Compression.Writer(file)
if err != nil {
return err
return 0, err
}
defer cFile.Close()
// Write value to file
_, err = cFile.Write(value)
return err
return cFile.Write(value)
}
// statBlock checks for existence of supplied block hash.

View file

@ -5,7 +5,7 @@
"io"
"sync"
"codeberg.org/gruf/go-store/v2/util"
"codeberg.org/gruf/go-iotools"
"github.com/klauspost/compress/gzip"
"github.com/klauspost/compress/snappy"
@ -15,10 +15,10 @@
// Compressor defines a means of compressing/decompressing values going into a key-value store
type Compressor interface {
// Reader returns a new decompressing io.ReadCloser based on supplied (compressed) io.Reader
Reader(io.Reader) (io.ReadCloser, error)
Reader(io.ReadCloser) (io.ReadCloser, error)
// Writer returns a new compressing io.WriteCloser based on supplied (uncompressed) io.Writer
Writer(io.Writer) (io.WriteCloser, error)
Writer(io.WriteCloser) (io.WriteCloser, error)
}
type gzipCompressor struct {
@ -47,8 +47,8 @@ func GZipCompressorLevel(level int) Compressor {
// Write empty data to ensure gzip
// header data is in byte buffer.
gw.Write([]byte{})
gw.Close()
_, _ = gw.Write([]byte{})
_ = gw.Close()
return &gzipCompressor{
rpool: sync.Pool{
@ -67,23 +67,61 @@ func GZipCompressorLevel(level int) Compressor {
}
}
func (c *gzipCompressor) Reader(r io.Reader) (io.ReadCloser, error) {
func (c *gzipCompressor) Reader(rc io.ReadCloser) (io.ReadCloser, error) {
var released bool
// Acquire from pool.
gr := c.rpool.Get().(*gzip.Reader)
if err := gr.Reset(r); err != nil {
if err := gr.Reset(rc); err != nil {
c.rpool.Put(gr)
return nil, err
}
return util.ReadCloserWithCallback(gr, func() {
c.rpool.Put(gr)
}), nil
return iotools.ReadCloser(gr, iotools.CloserFunc(func() error {
if !released {
released = true
defer c.rpool.Put(gr)
}
// Close compressor
err1 := gr.Close()
// Close original stream.
err2 := rc.Close()
// Return err1 or 2
if err1 != nil {
return err1
}
return err2
})), nil
}
func (c *gzipCompressor) Writer(w io.Writer) (io.WriteCloser, error) {
func (c *gzipCompressor) Writer(wc io.WriteCloser) (io.WriteCloser, error) {
var released bool
// Acquire from pool.
gw := c.wpool.Get().(*gzip.Writer)
gw.Reset(w)
return util.WriteCloserWithCallback(gw, func() {
c.wpool.Put(gw)
}), nil
gw.Reset(wc)
return iotools.WriteCloser(gw, iotools.CloserFunc(func() error {
if !released {
released = true
c.wpool.Put(gw)
}
// Close compressor
err1 := gw.Close()
// Close original stream.
err2 := wc.Close()
// Return err1 or 2
if err1 != nil {
return err1
}
return err2
})), nil
}
type zlibCompressor struct {
@ -139,26 +177,61 @@ func ZLibCompressorLevelDict(level int, dict []byte) Compressor {
}
}
func (c *zlibCompressor) Reader(r io.Reader) (io.ReadCloser, error) {
func (c *zlibCompressor) Reader(rc io.ReadCloser) (io.ReadCloser, error) {
var released bool
zr := c.rpool.Get().(interface {
io.ReadCloser
zlib.Resetter
})
if err := zr.Reset(r, c.dict); err != nil {
if err := zr.Reset(rc, c.dict); err != nil {
c.rpool.Put(zr)
return nil, err
}
return util.ReadCloserWithCallback(zr, func() {
c.rpool.Put(zr)
}), nil
return iotools.ReadCloser(zr, iotools.CloserFunc(func() error {
if !released {
released = true
defer c.rpool.Put(zr)
}
// Close compressor
err1 := zr.Close()
// Close original stream.
err2 := rc.Close()
// Return err1 or 2
if err1 != nil {
return err1
}
return err2
})), nil
}
func (c *zlibCompressor) Writer(w io.Writer) (io.WriteCloser, error) {
func (c *zlibCompressor) Writer(wc io.WriteCloser) (io.WriteCloser, error) {
var released bool
// Acquire from pool.
zw := c.wpool.Get().(*zlib.Writer)
zw.Reset(w)
return util.WriteCloserWithCallback(zw, func() {
c.wpool.Put(zw)
}), nil
zw.Reset(wc)
return iotools.WriteCloser(zw, iotools.CloserFunc(func() error {
if !released {
released = true
c.wpool.Put(zw)
}
// Close compressor
err1 := zw.Close()
// Close original stream.
err2 := wc.Close()
// Return err1 or 2
if err1 != nil {
return err1
}
return err2
})), nil
}
type snappyCompressor struct {
@ -178,22 +251,40 @@ func SnappyCompressor() Compressor {
}
}
func (c *snappyCompressor) Reader(r io.Reader) (io.ReadCloser, error) {
func (c *snappyCompressor) Reader(rc io.ReadCloser) (io.ReadCloser, error) {
var released bool
// Acquire from pool.
sr := c.rpool.Get().(*snappy.Reader)
sr.Reset(r)
return util.ReadCloserWithCallback(
util.NopReadCloser(sr),
func() { c.rpool.Put(sr) },
), nil
sr.Reset(rc)
return iotools.ReadCloser(sr, iotools.CloserFunc(func() error {
if !released {
released = true
defer c.rpool.Put(sr)
}
// Close original stream.
return rc.Close()
})), nil
}
func (c *snappyCompressor) Writer(w io.Writer) (io.WriteCloser, error) {
func (c *snappyCompressor) Writer(wc io.WriteCloser) (io.WriteCloser, error) {
var released bool
// Acquire from pool.
sw := c.wpool.Get().(*snappy.Writer)
sw.Reset(w)
return util.WriteCloserWithCallback(
util.NopWriteCloser(sw),
func() { c.wpool.Put(sw) },
), nil
sw.Reset(wc)
return iotools.WriteCloser(sw, iotools.CloserFunc(func() error {
if !released {
released = true
c.wpool.Put(sw)
}
// Close original stream.
return wc.Close()
})), nil
}
type nopCompressor struct{}
@ -203,10 +294,10 @@ func NoCompression() Compressor {
return &nopCompressor{}
}
func (c *nopCompressor) Reader(r io.Reader) (io.ReadCloser, error) {
return util.NopReadCloser(r), nil
func (c *nopCompressor) Reader(rc io.ReadCloser) (io.ReadCloser, error) {
return rc, nil
}
func (c *nopCompressor) Writer(w io.Writer) (io.WriteCloser, error) {
return util.NopWriteCloser(w), nil
func (c *nopCompressor) Writer(wc io.WriteCloser) (io.WriteCloser, error) {
return wc, nil
}

View file

@ -219,43 +219,41 @@ func (st *DiskStorage) ReadStream(ctx context.Context, key string) (io.ReadClose
// Wrap the file in a compressor
cFile, err := st.config.Compression.Reader(file)
if err != nil {
file.Close() // close this here, ignore error
_ = file.Close()
return nil, err
}
// Wrap compressor to ensure file close
return util.ReadCloserWithCallback(cFile, func() {
file.Close()
}), nil
return cFile, nil
}
// WriteBytes implements Storage.WriteBytes().
func (st *DiskStorage) WriteBytes(ctx context.Context, key string, value []byte) error {
return st.WriteStream(ctx, key, bytes.NewReader(value))
func (st *DiskStorage) WriteBytes(ctx context.Context, key string, value []byte) (int, error) {
n, err := st.WriteStream(ctx, key, bytes.NewReader(value))
return int(n), err
}
// WriteStream implements Storage.WriteStream().
func (st *DiskStorage) WriteStream(ctx context.Context, key string, r io.Reader) error {
func (st *DiskStorage) WriteStream(ctx context.Context, key string, r io.Reader) (int64, error) {
// Get file path for key
kpath, err := st.filepath(key)
if err != nil {
return err
return 0, err
}
// Check if open
if st.lock.Closed() {
return ErrClosed
return 0, ErrClosed
}
// Check context still valid
if err := ctx.Err(); err != nil {
return err
return 0, err
}
// Ensure dirs leading up to file exist
err = os.MkdirAll(path.Dir(kpath), defaultDirPerms)
if err != nil {
return err
return 0, err
}
// Prepare to swap error if need-be
@ -273,20 +271,21 @@ func (st *DiskStorage) WriteStream(ctx context.Context, key string, r io.Reader)
// Attempt to open file
file, err := open(kpath, flags)
if err != nil {
return errSwap(err)
return 0, errSwap(err)
}
defer file.Close()
// Wrap the file in a compressor
cFile, err := st.config.Compression.Writer(file)
if err != nil {
return err
_ = file.Close()
return 0, err
}
// Wraps file.Close().
defer cFile.Close()
// Copy provided reader to file
_, err = st.cppool.Copy(cFile, r)
return err
return st.cppool.Copy(cFile, r)
}
// Stat implements Storage.Stat().

View file

@ -1,6 +1,7 @@
package storage
import (
"fmt"
"io/fs"
"os"
"syscall"
@ -102,46 +103,32 @@ type frame struct {
// cleanDirs traverses the dir tree of the supplied path, removing any folders with zero children
func cleanDirs(path string) error {
// Acquire path builder
pb := util.GetPathBuilder()
defer util.PutPathBuilder(pb)
// Get top-level dir entries
entries, err := readDir(path)
if err != nil {
return err
}
for _, entry := range entries {
if entry.IsDir() {
// Recursively clean sub-directory entries
if err := cleanDir(pb, pb.Join(path, entry.Name())); err != nil {
return err
}
}
}
return nil
return cleanDir(pb, path, true)
}
// cleanDir performs the actual dir cleaning logic for the above top-level version.
func cleanDir(pb *fastpath.Builder, path string) error {
// Get dir entries
func cleanDir(pb *fastpath.Builder, path string, top bool) error {
// Get dir entries at path.
entries, err := readDir(path)
if err != nil {
return err
}
// If no entries, delete
if len(entries) < 1 {
// If no entries, delete dir.
if !top && len(entries) == 0 {
return rmdir(path)
}
for _, entry := range entries {
if entry.IsDir() {
// Recursively clean sub-directory entries
if err := cleanDir(pb, pb.Join(path, entry.Name())); err != nil {
return err
// Calculate directory path.
dirPath := pb.Join(path, entry.Name())
// Recursively clean sub-directory entries.
if err := cleanDir(pb, dirPath, false); err != nil {
fmt.Fprintf(os.Stderr, "[go-store/storage] error cleaning %s: %v", dirPath, err)
}
}
}

View file

@ -6,7 +6,7 @@
"sync/atomic"
"codeberg.org/gruf/go-bytes"
"codeberg.org/gruf/go-store/v2/util"
"codeberg.org/gruf/go-iotools"
"github.com/cornelk/hashmap"
)
@ -86,57 +86,57 @@ func (st *MemoryStorage) ReadStream(ctx context.Context, key string) (io.ReadClo
// Create io.ReadCloser from 'b' copy
r := bytes.NewReader(copyb(b))
return util.NopReadCloser(r), nil
return iotools.NopReadCloser(r), nil
}
// WriteBytes implements Storage.WriteBytes().
func (st *MemoryStorage) WriteBytes(ctx context.Context, key string, b []byte) error {
func (st *MemoryStorage) WriteBytes(ctx context.Context, key string, b []byte) (int, error) {
// Check store open
if st.closed() {
return ErrClosed
return 0, ErrClosed
}
// Check context still valid
if err := ctx.Err(); err != nil {
return err
return 0, err
}
// Check for key that already exists
if _, ok := st.fs.Get(key); ok && !st.ow {
return ErrAlreadyExists
return 0, ErrAlreadyExists
}
// Write key copy to store
st.fs.Set(key, copyb(b))
return nil
return len(b), nil
}
// WriteStream implements Storage.WriteStream().
func (st *MemoryStorage) WriteStream(ctx context.Context, key string, r io.Reader) error {
func (st *MemoryStorage) WriteStream(ctx context.Context, key string, r io.Reader) (int64, error) {
// Check store open
if st.closed() {
return ErrClosed
return 0, ErrClosed
}
// Check context still valid
if err := ctx.Err(); err != nil {
return err
return 0, err
}
// Check for key that already exists
if _, ok := st.fs.Get(key); ok && !st.ow {
return ErrAlreadyExists
return 0, ErrAlreadyExists
}
// Read all from reader
b, err := io.ReadAll(r)
if err != nil {
return err
return 0, err
}
// Write key to store
st.fs.Set(key, b)
return nil
return int64(len(b)), nil
}
// Stat implements Storage.Stat().

View file

@ -160,22 +160,23 @@ func (st *S3Storage) ReadStream(ctx context.Context, key string) (io.ReadCloser,
}
// WriteBytes implements Storage.WriteBytes().
func (st *S3Storage) WriteBytes(ctx context.Context, key string, value []byte) error {
return st.WriteStream(ctx, key, util.NewByteReaderSize(value))
func (st *S3Storage) WriteBytes(ctx context.Context, key string, value []byte) (int, error) {
n, err := st.WriteStream(ctx, key, util.NewByteReaderSize(value))
return int(n), err
}
// WriteStream implements Storage.WriteStream().
func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) error {
func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) (int64, error) {
// Check storage open
if st.closed() {
return ErrClosed
return 0, ErrClosed
}
if rs, ok := r.(util.ReaderSize); ok {
// This reader supports providing us the size of
// the encompassed data, allowing us to perform
// a singular .PutObject() call with length.
_, err := st.client.PutObject(
info, err := st.client.PutObject(
ctx,
st.bucket,
key,
@ -186,9 +187,9 @@ func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) e
st.config.PutOpts,
)
if err != nil {
return transformS3Error(err)
err = transformS3Error(err)
}
return nil
return info.Size, err
}
// Start a new multipart upload to get ID
@ -199,14 +200,15 @@ func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) e
st.config.PutOpts,
)
if err != nil {
return transformS3Error(err)
return 0, transformS3Error(err)
}
var (
count = 1
index = int(1) // parts index
total = int64(0)
parts []minio.CompletePart
chunk = make([]byte, st.config.PutChunkSize)
rdr = bytes.NewReader(nil)
rbuf = bytes.NewReader(nil)
)
// Note that we do not perform any kind of
@ -234,11 +236,11 @@ func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) e
// All other errors
default:
return err
return 0, err
}
// Reset byte reader
rdr.Reset(chunk[:n])
rbuf.Reset(chunk[:n])
// Put this object chunk in S3 store
pt, err := st.client.PutObjectPart(
@ -246,15 +248,15 @@ func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) e
st.bucket,
key,
uploadID,
count,
rdr,
index,
rbuf,
int64(n),
"",
"",
nil,
)
if err != nil {
return err
return 0, err
}
// Append completed part to slice
@ -267,8 +269,11 @@ func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) e
ChecksumSHA256: pt.ChecksumSHA256,
})
// Iterate part count
count++
// Iterate idx
index++
// Update total size
total += pt.Size
}
// Complete this multi-part upload operation
@ -281,10 +286,10 @@ func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) e
st.config.PutOpts,
)
if err != nil {
return err
return 0, err
}
return nil
return total, nil
}
// Stat implements Storage.Stat().

View file

@ -14,10 +14,10 @@ type Storage interface {
ReadStream(ctx context.Context, key string) (io.ReadCloser, error)
// WriteBytes writes the supplied value bytes at key in the storage
WriteBytes(ctx context.Context, key string, value []byte) error
WriteBytes(ctx context.Context, key string, value []byte) (int, error)
// WriteStream writes the bytes from supplied reader at key in the storage
WriteStream(ctx context.Context, key string, r io.Reader) error
WriteStream(ctx context.Context, key string, r io.Reader) (int64, error)
// Stat checks if the supplied key is in the storage
Stat(ctx context.Context, key string) (bool, error)

View file

@ -5,102 +5,37 @@
"io"
)
// ReaderSize ...
// ReaderSize defines a reader of known size in bytes.
type ReaderSize interface {
io.Reader
// Size ...
Size() int64
}
// ByteReaderSize ...
// ByteReaderSize implements ReaderSize for an in-memory byte-slice.
type ByteReaderSize struct {
bytes.Reader
br bytes.Reader
sz int64
}
// NewByteReaderSize ...
// NewByteReaderSize returns a new ByteReaderSize instance reset to slice b.
func NewByteReaderSize(b []byte) *ByteReaderSize {
rs := ByteReaderSize{}
rs := new(ByteReaderSize)
rs.Reset(b)
return &rs
return rs
}
// Size implements ReaderSize.Size().
func (rs ByteReaderSize) Size() int64 {
// Read implements io.Reader.
func (rs *ByteReaderSize) Read(b []byte) (int, error) {
return rs.br.Read(b)
}
// Size implements ReaderSize.
func (rs *ByteReaderSize) Size() int64 {
return rs.sz
}
// Reset resets the ReaderSize to be reading from b.
func (rs *ByteReaderSize) Reset(b []byte) {
rs.Reader.Reset(b)
rs.br.Reset(b)
rs.sz = int64(len(b))
}
// NopReadCloser turns a supplied io.Reader into io.ReadCloser with a nop Close() implementation.
func NopReadCloser(r io.Reader) io.ReadCloser {
return &nopReadCloser{r}
}
// NopWriteCloser turns a supplied io.Writer into io.WriteCloser with a nop Close() implementation.
func NopWriteCloser(w io.Writer) io.WriteCloser {
return &nopWriteCloser{w}
}
// ReadCloserWithCallback adds a customizable callback to be called upon Close() of a supplied io.ReadCloser.
// Note that the callback will never be called more than once, after execution this will remove the func reference.
func ReadCloserWithCallback(rc io.ReadCloser, cb func()) io.ReadCloser {
return &callbackReadCloser{
ReadCloser: rc,
callback: cb,
}
}
// WriteCloserWithCallback adds a customizable callback to be called upon Close() of a supplied io.WriteCloser.
// Note that the callback will never be called more than once, after execution this will remove the func reference.
func WriteCloserWithCallback(wc io.WriteCloser, cb func()) io.WriteCloser {
return &callbackWriteCloser{
WriteCloser: wc,
callback: cb,
}
}
// nopReadCloser turns an io.Reader -> io.ReadCloser with a nop Close().
type nopReadCloser struct{ io.Reader }
func (r *nopReadCloser) Close() error { return nil }
// nopWriteCloser turns an io.Writer -> io.WriteCloser with a nop Close().
type nopWriteCloser struct{ io.Writer }
func (w nopWriteCloser) Close() error { return nil }
// callbackReadCloser allows adding our own custom callback to an io.ReadCloser.
type callbackReadCloser struct {
io.ReadCloser
callback func()
}
func (c *callbackReadCloser) Close() error {
if c.callback != nil {
cb := c.callback
c.callback = nil
defer cb()
}
return c.ReadCloser.Close()
}
// callbackWriteCloser allows adding our own custom callback to an io.WriteCloser.
type callbackWriteCloser struct {
io.WriteCloser
callback func()
}
func (c *callbackWriteCloser) Close() error {
if c.callback != nil {
cb := c.callback
c.callback = nil
defer cb()
}
return c.WriteCloser.Close()
}

9
vendor/modules.txt vendored
View file

@ -24,7 +24,7 @@ codeberg.org/gruf/go-debug
# codeberg.org/gruf/go-errors/v2 v2.0.2
## explicit; go 1.16
codeberg.org/gruf/go-errors/v2
# codeberg.org/gruf/go-fastcopy v1.1.1
# codeberg.org/gruf/go-fastcopy v1.1.2
## explicit; go 1.17
codeberg.org/gruf/go-fastcopy
# codeberg.org/gruf/go-fastpath v1.0.3
@ -36,6 +36,9 @@ codeberg.org/gruf/go-fastpath/v2
# codeberg.org/gruf/go-hashenc v1.0.2
## explicit; go 1.16
codeberg.org/gruf/go-hashenc
# codeberg.org/gruf/go-iotools v0.0.0-20221224124424-3386841cb225
## explicit; go 1.19
codeberg.org/gruf/go-iotools
# codeberg.org/gruf/go-kv v1.5.2
## explicit; go 1.19
codeberg.org/gruf/go-kv
@ -49,7 +52,7 @@ codeberg.org/gruf/go-mangler
# codeberg.org/gruf/go-maps v1.0.3
## explicit; go 1.19
codeberg.org/gruf/go-maps
# codeberg.org/gruf/go-mutexes v1.1.4
# codeberg.org/gruf/go-mutexes v1.1.5
## explicit; go 1.14
codeberg.org/gruf/go-mutexes
# codeberg.org/gruf/go-pools v1.1.0
@ -61,7 +64,7 @@ codeberg.org/gruf/go-runners
# codeberg.org/gruf/go-sched v1.2.0
## explicit; go 1.19
codeberg.org/gruf/go-sched
# codeberg.org/gruf/go-store/v2 v2.0.10
# codeberg.org/gruf/go-store/v2 v2.2.1
## explicit; go 1.19
codeberg.org/gruf/go-store/v2/kv
codeberg.org/gruf/go-store/v2/storage