Compare commits

..

5 commits

15 changed files with 379 additions and 57 deletions

View file

@ -858,7 +858,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch8() {
"static_url": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/attachment/small/`+instanceAccount.AvatarMediaAttachment.ID+`.webp",`+`
"thumbnail_static_type": "image/webp",
"thumbnail_description": "A bouncing little green peglin.",
"blurhash": "LE9801Rl4Yt5%dWCV]t5Dmoex?WC"
"blurhash": "LF9Hm*Rl4Yt5.4RlRSt5IXkBxsj["
}`, string(instanceV2ThumbnailJson))
// double extra special bonus: now update the image description without changing the image

View file

@ -35,6 +35,8 @@
"github.com/gorilla/websocket"
)
var pingMsg = []byte("ping!")
// StreamGETHandler swagger:operation GET /api/v1/streaming streamGet
//
// Initiate a websocket connection for live streaming of statuses and notifications.
@ -389,40 +391,57 @@ func (m *Module) writeToWSConn(
) {
for {
// Wrap context with timeout to send a ping.
pingctx, cncl := context.WithTimeout(ctx, ping)
pingCtx, cncl := context.WithTimeout(ctx, ping)
// Block on receipt of msg.
msg, ok := stream.Recv(pingctx)
// Block and wait for
// one of the following:
//
// - receipt of msg
// - timeout of pingCtx
// - stream closed.
msg, haveMsg := stream.Recv(pingCtx)
// Check if cancel because ping.
pinged := (pingctx.Err() != nil)
// If ping context has timed
// out, we should send a ping.
//
// In any case cancel pingCtx
// as we're done with it.
shouldPing := (pingCtx.Err() != nil)
cncl()
switch {
case !ok && pinged:
// The ping context timed out!
l.Trace("writing websocket ping")
case !haveMsg && !shouldPing:
// We have no message and we shouldn't
// send a ping; this means the stream
// has been closed from the client's end,
// so there's nothing further to do here.
l.Trace("no message and we shouldn't ping, returning...")
return
// Wrapped context time-out, send a keep-alive "ping".
if err := wsConn.WriteControl(websocket.PingMessage, nil, time.Time{}); err != nil {
l.Debugf("error writing websocket ping: %v", err)
break
case haveMsg:
// We have a message to stream.
l.Tracef("writing websocket message: %+v", msg)
if err := wsConn.WriteJSON(msg); err != nil {
// If there's an error writing then drop the
// connection, as client may have disappeared
// suddenly; they can reconnect if necessary.
l.Debugf("error writing websocket message: %v", err)
return
}
case !ok:
// Stream was
// closed.
return
}
case shouldPing:
// We have no message but we do
// need to send a keep-alive ping.
l.Trace("writing websocket ping")
l.Trace("writing websocket message: %+v", msg)
// Received a new message from the processor.
if err := wsConn.WriteJSON(msg); err != nil {
l.Debugf("error writing websocket message: %v", err)
break
if err := wsConn.WriteControl(websocket.PingMessage, pingMsg, time.Time{}); err != nil {
// If there's an error writing then drop the
// connection, as client may have disappeared
// suddenly; they can reconnect if necessary.
l.Debugf("error writing websocket ping: %v", err)
return
}
}
}
l.Debug("finished websocket write")
}

View file

@ -247,6 +247,54 @@ func (suite *FilterTestSuite) TestFilterCRUD() {
}
}
func (suite *FilterTestSuite) TestFilterTitleOverlap() {
var (
ctx = context.Background()
account1 = "01HNEJXCPRTJVJY9MV0VVHGD47"
account2 = "01JAG5BRJPJYA0FSA5HR2MMFJH"
)
// Create an empty filter for account 1.
account1filter1 := &gtsmodel.Filter{
ID: "01HNEJNVZZVXJTRB3FX3K2B1YF",
AccountID: account1,
Title: "my filter",
Action: gtsmodel.FilterActionWarn,
ContextHome: util.Ptr(true),
}
if err := suite.db.PutFilter(ctx, account1filter1); err != nil {
suite.FailNow("", "error putting account1filter1: %s", err)
}
// Create a filter for account 2 with
// the same title, should be no issue.
account2filter1 := &gtsmodel.Filter{
ID: "01JAG5GPXG7H5Y4ZP78GV1F2ET",
AccountID: account2,
Title: "my filter",
Action: gtsmodel.FilterActionWarn,
ContextHome: util.Ptr(true),
}
if err := suite.db.PutFilter(ctx, account2filter1); err != nil {
suite.FailNow("", "error putting account2filter1: %s", err)
}
// Try to create another filter for
// account 1 with the same name as
// an existing filter of theirs.
account1filter2 := &gtsmodel.Filter{
ID: "01JAG5J8NYKQE2KYCD28Y4P05V",
AccountID: account1,
Title: "my filter",
Action: gtsmodel.FilterActionWarn,
ContextHome: util.Ptr(true),
}
err := suite.db.PutFilter(ctx, account1filter2)
if !errors.Is(err, db.ErrAlreadyExists) {
suite.FailNow("", "wanted ErrAlreadyExists, got %s", err)
}
}
func TestFilterTestSuite(t *testing.T) {
suite.Run(t, new(FilterTestSuite))
}

View file

@ -20,7 +20,7 @@
import (
"context"
gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
gtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20240126064004_add_filters"
"github.com/uptrace/bun"
)

View file

@ -0,0 +1,65 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package gtsmodel
import (
"regexp"
"time"
)
// Filter stores a filter created by a local account.
type Filter struct {
ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
ExpiresAt time.Time `bun:"type:timestamptz,nullzero"` // Time filter should expire. If null, should not expire.
AccountID string `bun:"type:CHAR(26),notnull,nullzero"` // ID of the local account that created the filter.
Title string `bun:",nullzero,notnull,unique"` // The name of the filter.
Action string `bun:",nullzero,notnull"` // The action to take.
Keywords []*FilterKeyword `bun:"-"` // Keywords for this filter.
Statuses []*FilterStatus `bun:"-"` // Statuses for this filter.
ContextHome *bool `bun:",nullzero,notnull,default:false"` // Apply filter to home timeline and lists.
ContextNotifications *bool `bun:",nullzero,notnull,default:false"` // Apply filter to notifications.
ContextPublic *bool `bun:",nullzero,notnull,default:false"` // Apply filter to home timeline and lists.
ContextThread *bool `bun:",nullzero,notnull,default:false"` // Apply filter when viewing a status's associated thread.
ContextAccount *bool `bun:",nullzero,notnull,default:false"` // Apply filter when viewing an account profile.
}
// FilterKeyword stores a single keyword to filter statuses against.
type FilterKeyword struct {
ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
AccountID string `bun:"type:CHAR(26),notnull,nullzero"` // ID of the local account that created the filter keyword.
FilterID string `bun:"type:CHAR(26),notnull,nullzero,unique:filter_keywords_filter_id_keyword_uniq"` // ID of the filter that this keyword belongs to.
Filter *Filter `bun:"-"` // Filter corresponding to FilterID
Keyword string `bun:",nullzero,notnull,unique:filter_keywords_filter_id_keyword_uniq"` // The keyword or phrase to filter against.
WholeWord *bool `bun:",nullzero,notnull,default:false"` // Should the filter consider word boundaries?
Regexp *regexp.Regexp `bun:"-"` // pre-prepared regular expression
}
// FilterStatus stores a single status to filter.
type FilterStatus struct {
ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
AccountID string `bun:"type:CHAR(26),notnull,nullzero"` // ID of the local account that created the filter keyword.
FilterID string `bun:"type:CHAR(26),notnull,nullzero,unique:filter_statuses_filter_id_status_id_uniq"` // ID of the filter that this keyword belongs to.
Filter *Filter `bun:"-"` // Filter corresponding to FilterID
StatusID string `bun:"type:CHAR(26),notnull,nullzero,unique:filter_statuses_filter_id_status_id_uniq"` // ID of the status to filter.
}

View file

@ -0,0 +1,131 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package migrations
import (
"context"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/uptrace/bun"
"github.com/uptrace/bun/dialect"
)
func init() {
up := func(ctx context.Context, db *bun.DB) error {
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
// Create the new filters table
// with the unique constraint
// set on AccountID + Title.
if _, err := tx.
NewCreateTable().
ModelTableExpr("new_filters").
Model((*gtsmodel.Filter)(nil)).
Exec(ctx); err != nil {
return err
}
// Explicitly specify columns to bring
// from old table to new, to avoid any
// potential Postgres shenanigans.
columns := []string{
"id",
"created_at",
"updated_at",
"expires_at",
"account_id",
"title",
"action",
"context_home",
"context_notifications",
"context_public",
"context_thread",
"context_account",
}
// Copy all data for existing
// filters to the new table.
if _, err := tx.
NewInsert().
Table("new_filters").
Table("filters").
Column(columns...).
Exec(ctx); err != nil {
return err
}
// Drop the old table.
if _, err := tx.
NewDropTable().
Table("filters").
Exec(ctx); err != nil {
return err
}
// Rename new table to old table.
if _, err := tx.
ExecContext(
ctx,
"ALTER TABLE ? RENAME TO ?",
bun.Ident("new_filters"),
bun.Ident("filters"),
); err != nil {
return err
}
// Index the new version
// of the filters table.
if _, err := tx.
NewCreateIndex().
Table("filters").
Index("filters_account_id_idx").
Column("account_id").
IfNotExists().
Exec(ctx); err != nil {
return err
}
if db.Dialect().Name() == dialect.PG {
// Rename "new_filters_pkey" from the
// new table to just "filters_pkey".
// This is only necessary on Postgres.
if _, err := tx.ExecContext(
ctx,
"ALTER TABLE ? RENAME CONSTRAINT ? TO ?",
bun.Ident("public.filters"),
bun.Safe("new_filters_pkey"),
bun.Safe("filters_pkey"),
); err != nil {
return err
}
}
return nil
})
}
down := func(ctx context.Context, db *bun.DB) error {
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
return nil
})
}
if err := Migrations.Register(up, down); err != nil {
panic(err)
}
}

View file

@ -26,20 +26,20 @@
// Filter stores a filter created by a local account.
type Filter struct {
ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
ExpiresAt time.Time `bun:"type:timestamptz,nullzero"` // Time filter should expire. If null, should not expire.
AccountID string `bun:"type:CHAR(26),notnull,nullzero"` // ID of the local account that created the filter.
Title string `bun:",nullzero,notnull,unique"` // The name of the filter.
Action FilterAction `bun:",nullzero,notnull"` // The action to take.
Keywords []*FilterKeyword `bun:"-"` // Keywords for this filter.
Statuses []*FilterStatus `bun:"-"` // Statuses for this filter.
ContextHome *bool `bun:",nullzero,notnull,default:false"` // Apply filter to home timeline and lists.
ContextNotifications *bool `bun:",nullzero,notnull,default:false"` // Apply filter to notifications.
ContextPublic *bool `bun:",nullzero,notnull,default:false"` // Apply filter to home timeline and lists.
ContextThread *bool `bun:",nullzero,notnull,default:false"` // Apply filter when viewing a status's associated thread.
ContextAccount *bool `bun:",nullzero,notnull,default:false"` // Apply filter when viewing an account profile.
ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
ExpiresAt time.Time `bun:"type:timestamptz,nullzero"` // Time filter should expire. If null, should not expire.
AccountID string `bun:"type:CHAR(26),notnull,nullzero,unique:filters_account_id_title_uniq"` // ID of the local account that created the filter.
Title string `bun:",nullzero,notnull,unique:filters_account_id_title_uniq"` // The name of the filter.
Action FilterAction `bun:",nullzero,notnull"` // The action to take.
Keywords []*FilterKeyword `bun:"-"` // Keywords for this filter.
Statuses []*FilterStatus `bun:"-"` // Statuses for this filter.
ContextHome *bool `bun:",nullzero,notnull,default:false"` // Apply filter to home timeline and lists.
ContextNotifications *bool `bun:",nullzero,notnull,default:false"` // Apply filter to notifications.
ContextPublic *bool `bun:",nullzero,notnull,default:false"` // Apply filter to home timeline and lists.
ContextThread *bool `bun:",nullzero,notnull,default:false"` // Apply filter when viewing a status's associated thread.
ContextAccount *bool `bun:",nullzero,notnull,default:false"` // Apply filter when viewing an account profile.
}
// Expired returns whether the filter has expired at a given time.

View file

@ -78,23 +78,17 @@ func ffmpegGenerateWebpThumb(ctx context.Context, inpath, outpath string, width,
// (NOT as libwebp_anim).
"-codec:v", "libwebp",
// Select thumb from first 7 frames.
// (in particular <= 7 reduced memory usage, marginally)
// (thumb filter: https://ffmpeg.org/ffmpeg-filters.html#thumbnail)
"-filter:v", "thumbnail=n=7,"+
// Only one frame
"-frames:v", "1",
// Scale to dimensions
// (scale filter: https://ffmpeg.org/ffmpeg-filters.html#scale)
"scale="+strconv.Itoa(width)+
":"+strconv.Itoa(height)+","+
// Scale to dimensions
// (scale filter: https://ffmpeg.org/ffmpeg-filters.html#scale)
"-filter:v", "scale="+strconv.Itoa(width)+":"+strconv.Itoa(height)+","+
// Attempt to use original pixel format
// (format filter: https://ffmpeg.org/ffmpeg-filters.html#format)
"format=pix_fmts="+pixfmt,
// Only one frame
"-frames:v", "1",
// Quality not specified,
// i.e. use default which
// should be 75% webp quality.

View file

@ -428,7 +428,7 @@ func (suite *ManagerTestSuite) TestSlothVineProcess() {
suite.Equal("video/mp4", attachment.File.ContentType)
suite.Equal("image/webp", attachment.Thumbnail.ContentType)
suite.Equal(312453, attachment.File.FileSize)
suite.Equal(5648, attachment.Thumbnail.FileSize)
suite.Equal(5598, attachment.Thumbnail.FileSize)
suite.Equal("LgIYH}xtNsofxtfPW.j[_4axn+of", attachment.Blurhash)
// now make sure the attachment is in the database
@ -441,6 +441,71 @@ func (suite *ManagerTestSuite) TestSlothVineProcess() {
equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/test-mp4-thumbnail.webp")
}
func (suite *ManagerTestSuite) TestAnimatedGifProcess() {
ctx := context.Background()
data := func(_ context.Context) (io.ReadCloser, error) {
// load bytes from a test image
b, err := os.ReadFile("./test/clock-original.gif")
if err != nil {
panic(err)
}
return io.NopCloser(bytes.NewBuffer(b)), nil
}
accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
// process the media with no additional info provided
processing, err := suite.manager.CreateMedia(ctx,
accountID,
data,
media.AdditionalMediaInfo{},
)
suite.NoError(err)
suite.NotNil(processing)
// do a blocking call to fetch the attachment
attachment, err := processing.Load(ctx)
suite.NoError(err)
suite.NotNil(attachment)
// make sure it's got the stuff set on it that we expect
// the attachment ID and accountID we expect
suite.Equal(processing.ID(), attachment.ID)
suite.Equal(accountID, attachment.AccountID)
// file meta should be correctly derived from the image
suite.EqualValues(gtsmodel.Original{
Width: 528,
Height: 528,
Size: 278784,
Aspect: 1,
Duration: util.Ptr(float32(8.58)),
Framerate: util.Ptr(float32(16)),
Bitrate: util.Ptr(uint64(114092)),
}, attachment.FileMeta.Original)
suite.EqualValues(gtsmodel.Small{
Width: 512,
Height: 512,
Size: 262144,
Aspect: 1,
}, attachment.FileMeta.Small)
suite.Equal("image/gif", attachment.File.ContentType)
suite.Equal("image/webp", attachment.Thumbnail.ContentType)
suite.Equal(122364, attachment.File.FileSize)
suite.Equal(12962, attachment.Thumbnail.FileSize)
suite.Equal("LmKUZkt700ofoffQofj[00WBj[WB", attachment.Blurhash)
// now make sure the attachment is in the database
dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID)
suite.NoError(err)
suite.NotNil(dbAttachment)
// ensure the files contain the expected data.
equalFiles(suite.T(), suite.state.Storage, dbAttachment.File.Path, "./test/clock-processed.gif")
equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/clock-thumbnail.webp")
}
func (suite *ManagerTestSuite) TestLongerMp4Process() {
ctx := context.Background()
@ -488,8 +553,8 @@ func (suite *ManagerTestSuite) TestLongerMp4Process() {
suite.Equal("video/mp4", attachment.File.ContentType)
suite.Equal("image/webp", attachment.Thumbnail.ContentType)
suite.Equal(109569, attachment.File.FileSize)
suite.Equal(2976, attachment.Thumbnail.FileSize)
suite.Equal("LIQJfl_3IU?b~qM{ofayWBWVofRj", attachment.Blurhash)
suite.Equal(2958, attachment.Thumbnail.FileSize)
suite.Equal("LIQ9}}_3IU?b~qM{ofayWBWVofRj", attachment.Blurhash)
// now make sure the attachment is in the database
dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID)
@ -548,8 +613,8 @@ func (suite *ManagerTestSuite) TestBirdnestMp4Process() {
suite.Equal("video/mp4", attachment.File.ContentType)
suite.Equal("image/webp", attachment.Thumbnail.ContentType)
suite.Equal(1409625, attachment.File.FileSize)
suite.Equal(14478, attachment.Thumbnail.FileSize)
suite.Equal("LLF$qyaeRO.9DgM_RPaetkV@WCMw", attachment.Blurhash)
suite.Equal(15056, attachment.Thumbnail.FileSize)
suite.Equal("LLF$nqafRO.9DgM_RPadtkV@WCMx", attachment.Blurhash)
// now make sure the attachment is in the database
dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID)

Binary file not shown.

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 120 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 120 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 13 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.9 KiB

After

Width:  |  Height:  |  Size: 2.9 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.5 KiB

After

Width:  |  Height:  |  Size: 5.5 KiB