From de45c0be60e453e69263f5b32ab2ce2661dc74ca Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Mon, 15 Jul 2024 14:24:53 +0000 Subject: [PATCH] [feature] more filetype support! (#3107) * add more supported file types to our media processor that ffmpeg supports, update supported mime type lists * add code comments to the supported mime types slice * don't check for zero value string, just parse * remove some unneeded consts which make the code a bit harder to read * fix test expected instance media mime types, use compact ffprobe json, simple media processing by type * final tweaks to media processing code * don't use safe divide where we don't need to --- cmd/process-emoji/main.go | 9 + cmd/process-media/main.go | 12 + .../api/client/instance/instancepatch_test.go | 102 ++++- internal/media/ffmpeg.go | 383 +++++++++++------- internal/media/manager.go | 65 ++- internal/media/manager_test.go | 2 +- internal/media/processingemoji.go | 20 +- internal/media/processingmedia.go | 155 ++----- internal/media/types.go | 21 - internal/typeutils/internaltofrontend_test.go | 34 +- internal/util/math.go | 34 ++ internal/util/ptr.go | 9 + 12 files changed, 495 insertions(+), 351 deletions(-) create mode 100644 internal/util/math.go diff --git a/cmd/process-emoji/main.go b/cmd/process-emoji/main.go index 62253bbdf..b06eb84f8 100644 --- a/cmd/process-emoji/main.go +++ b/cmd/process-emoji/main.go @@ -29,6 +29,7 @@ "github.com/superseriousbusiness/gotosocial/internal/db/bundb" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/media/ffmpeg" "github.com/superseriousbusiness/gotosocial/internal/state" "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/util" @@ -43,6 +44,14 @@ func main() { log.Panic(ctx, "Usage: go run ./cmd/process-emoji ") } + if err := ffmpeg.InitFfprobe(ctx, 1); err != nil { + log.Panic(ctx, err) + } + + if err := ffmpeg.InitFfmpeg(ctx, 1); err != nil { + log.Panic(ctx, err) + } + var st storage.Driver st.Storage = memory.Open(10, true) diff --git a/cmd/process-media/main.go b/cmd/process-media/main.go index 2f5a43f31..096d718f9 100644 --- a/cmd/process-media/main.go +++ b/cmd/process-media/main.go @@ -29,6 +29,7 @@ "github.com/superseriousbusiness/gotosocial/internal/db/bundb" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/media/ffmpeg" "github.com/superseriousbusiness/gotosocial/internal/state" "github.com/superseriousbusiness/gotosocial/internal/storage" ) @@ -42,6 +43,14 @@ func main() { log.Panic(ctx, "Usage: go run ./cmd/process-media ") } + if err := ffmpeg.InitFfprobe(ctx, 1); err != nil { + log.Panic(ctx, err) + } + + if err := ffmpeg.InitFfmpeg(ctx, 1); err != nil { + log.Panic(ctx, err) + } + var st storage.Driver st.Storage = memory.Open(10, true) @@ -105,6 +114,9 @@ func(ctx context.Context) (reader io.ReadCloser, err error) { func copyFile(ctx context.Context, st *storage.Driver, key string, path string) { rc, err := st.GetStream(ctx, key) if err != nil { + if storage.IsNotFound(err) { + return + } log.Panic(ctx, err) } defer rc.Close() diff --git a/internal/api/client/instance/instancepatch_test.go b/internal/api/client/instance/instancepatch_test.go index 5113e4c57..42833c23e 100644 --- a/internal/api/client/instance/instancepatch_test.go +++ b/internal/api/client/instance/instancepatch_test.go @@ -105,9 +105,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch1() { "supported_mime_types": [ "image/jpeg", "image/gif", - "image/png", "image/webp", - "video/mp4" + "audio/mp2", + "audio/mp3", + "video/x-msvideo", + "image/png", + "image/apng", + "audio/ogg", + "video/ogg", + "audio/x-m4a", + "video/mp4", + "video/quicktime", + "audio/x-ms-wma", + "video/x-ms-wmv", + "video/webm", + "audio/x-matroska", + "video/x-matroska" ], "image_size_limit": 41943040, "image_matrix_limit": 16777216, @@ -226,9 +239,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch2() { "supported_mime_types": [ "image/jpeg", "image/gif", - "image/png", "image/webp", - "video/mp4" + "audio/mp2", + "audio/mp3", + "video/x-msvideo", + "image/png", + "image/apng", + "audio/ogg", + "video/ogg", + "audio/x-m4a", + "video/mp4", + "video/quicktime", + "audio/x-ms-wma", + "video/x-ms-wmv", + "video/webm", + "audio/x-matroska", + "video/x-matroska" ], "image_size_limit": 41943040, "image_matrix_limit": 16777216, @@ -347,9 +373,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch3() { "supported_mime_types": [ "image/jpeg", "image/gif", - "image/png", "image/webp", - "video/mp4" + "audio/mp2", + "audio/mp3", + "video/x-msvideo", + "image/png", + "image/apng", + "audio/ogg", + "video/ogg", + "audio/x-m4a", + "video/mp4", + "video/quicktime", + "audio/x-ms-wma", + "video/x-ms-wmv", + "video/webm", + "audio/x-matroska", + "video/x-matroska" ], "image_size_limit": 41943040, "image_matrix_limit": 16777216, @@ -519,9 +558,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch6() { "supported_mime_types": [ "image/jpeg", "image/gif", - "image/png", "image/webp", - "video/mp4" + "audio/mp2", + "audio/mp3", + "video/x-msvideo", + "image/png", + "image/apng", + "audio/ogg", + "video/ogg", + "audio/x-m4a", + "video/mp4", + "video/quicktime", + "audio/x-ms-wma", + "video/x-ms-wmv", + "video/webm", + "audio/x-matroska", + "video/x-matroska" ], "image_size_limit": 41943040, "image_matrix_limit": 16777216, @@ -662,9 +714,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch8() { "supported_mime_types": [ "image/jpeg", "image/gif", - "image/png", "image/webp", - "video/mp4" + "audio/mp2", + "audio/mp3", + "video/x-msvideo", + "image/png", + "image/apng", + "audio/ogg", + "video/ogg", + "audio/x-m4a", + "video/mp4", + "video/quicktime", + "audio/x-ms-wma", + "video/x-ms-wmv", + "video/webm", + "audio/x-matroska", + "video/x-matroska" ], "image_size_limit": 41943040, "image_matrix_limit": 16777216, @@ -820,9 +885,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch9() { "supported_mime_types": [ "image/jpeg", "image/gif", - "image/png", "image/webp", - "video/mp4" + "audio/mp2", + "audio/mp3", + "video/x-msvideo", + "image/png", + "image/apng", + "audio/ogg", + "video/ogg", + "audio/x-m4a", + "video/mp4", + "video/quicktime", + "audio/x-ms-wma", + "video/x-ms-wmv", + "video/webm", + "audio/x-matroska", + "video/x-matroska" ], "image_size_limit": 41943040, "image_matrix_limit": 16777216, diff --git a/internal/media/ffmpeg.go b/internal/media/ffmpeg.go index b97c8413f..53facd15b 100644 --- a/internal/media/ffmpeg.go +++ b/internal/media/ffmpeg.go @@ -18,7 +18,6 @@ package media import ( - "cmp" "context" "encoding/json" "errors" @@ -135,7 +134,7 @@ func ffmpeg(ctx context.Context, dirpath string, args ...string) error { } // ffprobe calls `ffprobe` (WASM) on filepath, returning parsed JSON output. -func ffprobe(ctx context.Context, filepath string) (*ffprobeResult, error) { +func ffprobe(ctx context.Context, filepath string) (*result, error) { var stdout byteutil.Buffer // Get directory from filepath. @@ -148,7 +147,7 @@ func ffprobe(ctx context.Context, filepath string) (*ffprobeResult, error) { Args: []string{ "-i", filepath, "-loglevel", "quiet", - "-print_format", "json", + "-print_format", "json=compact=1", "-show_streams", "-show_format", "-show_error", @@ -172,7 +171,219 @@ func ffprobe(ctx context.Context, filepath string) (*ffprobeResult, error) { return nil, gtserror.Newf("error unmarshaling json: %w", err) } - return &result, nil + // Convert raw result data. + res, err := result.Process() + if err != nil { + return nil, err + } + + return res, nil +} + +// result contains parsed ffprobe result +// data in a more useful data format. +type result struct { + format string + audio []audioStream + video []videoStream + bitrate uint64 + duration float64 +} + +type stream struct { + codec string +} + +type audioStream struct { + stream +} + +type videoStream struct { + stream + width int + height int + framerate float32 +} + +// GetFileType determines file type and extension to use for media data. This +// function helps to abstract away the horrible complexities that are possible +// media container (i.e. the file) types and and possible sub-types within that. +// +// Note the checks for (len(res.video) > 0) may catch some audio files with embedded +// album art as video, but i blame that on the hellscape that is media filetypes. +// +// TODO: we can update this code to also return a mimetype and avoid later parsing! +func (res *result) GetFileType() (gtsmodel.FileType, string) { + switch res.format { + case "mpeg": + return gtsmodel.FileTypeVideo, "mpeg" + case "mjpeg": + return gtsmodel.FileTypeVideo, "mjpeg" + case "mov,mp4,m4a,3gp,3g2,mj2": + switch { + case len(res.video) > 0: + return gtsmodel.FileTypeVideo, "mp4" + case len(res.audio) > 0 && + res.audio[0].codec == "aac": + // m4a only supports [aac] audio. + return gtsmodel.FileTypeAudio, "m4a" + } + case "apng": + return gtsmodel.FileTypeImage, "apng" + case "png_pipe": + return gtsmodel.FileTypeImage, "png" + case "image2", "image2pipe", "jpeg_pipe": + return gtsmodel.FileTypeImage, "jpeg" + case "webp", "webp_pipe": + return gtsmodel.FileTypeImage, "webp" + case "gif": + return gtsmodel.FileTypeImage, "gif" + case "mp3": + if len(res.audio) > 0 { + switch res.audio[0].codec { + case "mp2": + return gtsmodel.FileTypeAudio, "mp2" + case "mp3": + return gtsmodel.FileTypeAudio, "mp3" + } + } + case "asf": + switch { + case len(res.video) > 0: + return gtsmodel.FileTypeVideo, "wmv" + case len(res.audio) > 0: + return gtsmodel.FileTypeAudio, "wma" + } + case "ogg": + switch { + case len(res.video) > 0: + return gtsmodel.FileTypeVideo, "ogv" + case len(res.audio) > 0: + return gtsmodel.FileTypeAudio, "ogg" + } + case "matroska,webm": + switch { + case len(res.video) > 0: + switch res.video[0].codec { + case "vp8", "vp9", "av1": + default: + return gtsmodel.FileTypeVideo, "mkv" + } + if len(res.audio) > 0 { + switch res.audio[0].codec { + case "vorbis", "opus", "libopus": + // webm only supports [VP8/VP9/AV1]+[vorbis/opus] + return gtsmodel.FileTypeVideo, "webm" + } + } + case len(res.audio) > 0: + return gtsmodel.FileTypeAudio, "mka" + } + case "avi": + return gtsmodel.FileTypeVideo, "avi" + } + return gtsmodel.FileTypeUnknown, res.format +} + +// ImageMeta extracts image metadata contained within ffprobe'd media result streams. +func (res *result) ImageMeta() (width int, height int, framerate float32) { + for _, stream := range res.video { + if stream.width > width { + width = stream.width + } + if stream.height > height { + height = stream.height + } + if fr := float32(stream.framerate); fr > 0 { + if framerate == 0 || fr < framerate { + framerate = fr + } + } + } + return +} + +// Process converts raw ffprobe result data into our more usable result{} type. +func (res *ffprobeResult) Process() (*result, error) { + if res.Error != nil { + return nil, res.Error + } + + if res.Format == nil { + return nil, errors.New("missing format data") + } + + var r result + var err error + + // Copy over container format. + r.format = res.Format.FormatName + + // Parsed media bitrate (if it was set). + if str := res.Format.BitRate; str != "" { + r.bitrate, err = strconv.ParseUint(str, 10, 64) + if err != nil { + return nil, gtserror.Newf("invalid bitrate %s: %w", str, err) + } + } + + // Parse media duration (if it was set). + if str := res.Format.Duration; str != "" { + r.duration, err = strconv.ParseFloat(str, 32) + if err != nil { + return nil, gtserror.Newf("invalid duration %s: %w", str, err) + } + } + + // Preallocate streams to max possible lengths. + r.audio = make([]audioStream, 0, len(res.Streams)) + r.video = make([]videoStream, 0, len(res.Streams)) + + // Convert streams to separate types. + for _, s := range res.Streams { + switch s.CodecType { + case "audio": + // Append audio stream data to result. + r.audio = append(r.audio, audioStream{ + stream: stream{codec: s.CodecName}, + }) + case "video": + var framerate float32 + + // Parse stream framerate, bearing in + // mind that some static container formats + // (e.g. jpeg) still return a framerate, so + // we also check for a non-1 timebase (dts). + if str := s.RFrameRate; str != "" && + s.DurationTS > 1 { + var num, den uint32 + den = 1 + + // Check for inequality (numerator / denominator). + if p := strings.SplitN(str, "/", 2); len(p) == 2 { + n, _ := strconv.ParseUint(p[0], 10, 32) + d, _ := strconv.ParseUint(p[1], 10, 32) + num, den = uint32(n), uint32(d) + } else { + n, _ := strconv.ParseUint(p[0], 10, 32) + num = uint32(n) + } + + // Set final divised framerate. + framerate = float32(num / den) + } + + // Append video stream data to result. + r.video = append(r.video, videoStream{ + stream: stream{codec: s.CodecName}, + width: s.Width, + height: s.Height, + framerate: framerate, + }) + } + } + + return &r, nil } // ffprobeResult contains parsed JSON data from @@ -183,175 +394,33 @@ type ffprobeResult struct { Error *ffprobeError `json:"error"` } -// ImageMeta extracts image metadata contained within ffprobe'd media result streams. -func (res *ffprobeResult) ImageMeta() (width int, height int, err error) { - for _, stream := range res.Streams { - if stream.Width > width { - width = stream.Width - } - if stream.Height > height { - height = stream.Height - } - } - if width == 0 || height == 0 { - err = errors.New("invalid image stream(s)") - } - return -} - -// EmbeddedImageMeta extracts embedded image metadata contained within ffprobe'd media result -// streams, should be used for pulling album image (can be animated image) from audio files. -func (res *ffprobeResult) EmbeddedImageMeta() (width int, height int, framerate float32, err error) { - for _, stream := range res.Streams { - if stream.Width > width { - width = stream.Width - } - if stream.Height > height { - height = stream.Height - } - if fr := stream.GetFrameRate(); fr > 0 { - if framerate == 0 || fr < framerate { - framerate = fr - } - } - } - // Need width + height but - // no framerate is fine. - if width == 0 || height == 0 { - err = errors.New("invalid image stream(s)") - } - return -} - -// VideoMeta extracts video metadata contained within ffprobe'd media result streams. -func (res *ffprobeResult) VideoMeta() (width, height int, framerate float32, err error) { - for _, stream := range res.Streams { - if stream.Width > width { - width = stream.Width - } - if stream.Height > height { - height = stream.Height - } - if fr := stream.GetFrameRate(); fr > 0 { - if framerate == 0 || fr < framerate { - framerate = fr - } - } - } - if width == 0 || height == 0 || framerate == 0 { - err = errors.New("invalid video stream(s)") - } - return -} - type ffprobeStream struct { - CodecName string `json:"codec_name"` - AvgFrameRate string `json:"avg_frame_rate"` - RFrameRate string `json:"r_frame_rate"` - Width int `json:"width"` - Height int `json:"height"` + CodecName string `json:"codec_name"` + CodecType string `json:"codec_type"` + RFrameRate string `json:"r_frame_rate"` + DurationTS uint `json:"duration_ts"` + Width int `json:"width"` + Height int `json:"height"` // + unused fields. } -// GetFrameRate calculates float32 framerate value from stream json string. -func (str *ffprobeStream) GetFrameRate() float32 { - numDen := func(strFR string) (float32, float32) { - var ( - // numerator - num float32 - - // denominator - den float32 - ) - - // Check for a provided inequality, i.e. numerator / denominator. - if p := strings.SplitN(strFR, "/", 2); len(p) == 2 { - n, _ := strconv.ParseFloat(p[0], 32) - d, _ := strconv.ParseFloat(p[1], 32) - num, den = float32(n), float32(d) - } else { - n, _ := strconv.ParseFloat(p[0], 32) - num = float32(n) - } - - return num, den - } - - var num, den float32 - if str.AvgFrameRate != "" { - // Check if we have avg_frame_rate. - num, den = numDen(str.AvgFrameRate) - } - - if num == 0 && str.RFrameRate != "" { - // Check if we have r_frame_rate. - num, den = numDen(str.RFrameRate) - } - - if num != 0 { - // Found it. - // Avoid divide by zero. - return num / cmp.Or(den, 1) - } - - return 0 -} - type ffprobeFormat struct { - Filename string `json:"filename"` FormatName string `json:"format_name"` Duration string `json:"duration"` BitRate string `json:"bit_rate"` // + unused fields } -// GetFileType determines file type and extension to use for media data. -func (fmt *ffprobeFormat) GetFileType() (gtsmodel.FileType, string) { - switch fmt.FormatName { - case "mov,mp4,m4a,3gp,3g2,mj2": - return gtsmodel.FileTypeVideo, "mp4" - case "apng": - return gtsmodel.FileTypeImage, "apng" - case "png_pipe": - return gtsmodel.FileTypeImage, "png" - case "image2", "jpeg_pipe": - return gtsmodel.FileTypeImage, "jpeg" - case "webp_pipe": - return gtsmodel.FileTypeImage, "webp" - case "gif": - return gtsmodel.FileTypeImage, "gif" - case "mp3": - return gtsmodel.FileTypeAudio, "mp3" - case "ogg": - return gtsmodel.FileTypeAudio, "ogg" - default: - return gtsmodel.FileTypeUnknown, fmt.FormatName - } -} - -// GetDuration calculates float32 framerate value from format json string. -func (fmt *ffprobeFormat) GetDuration() float32 { - if fmt.Duration != "" { - dur, _ := strconv.ParseFloat(fmt.Duration, 32) - return float32(dur) - } - return 0 -} - -// GetBitRate calculates uint64 bitrate value from format json string. -func (fmt *ffprobeFormat) GetBitRate() uint64 { - if fmt.BitRate != "" { - r, _ := strconv.ParseUint(fmt.BitRate, 10, 64) - return r - } - return 0 -} - type ffprobeError struct { Code int `json:"code"` String string `json:"string"` } +func isUnsupportedTypeErr(err error) bool { + ffprobeErr, ok := err.(*ffprobeError) + return ok && ffprobeErr.Code == -1094995529 +} + func (err *ffprobeError) Error() string { return err.String + " (" + strconv.Itoa(err.Code) + ")" } diff --git a/internal/media/manager.go b/internal/media/manager.go index aaf9448b8..82b066edc 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -34,17 +34,46 @@ ) var SupportedMIMETypes = []string{ - mimeImageJpeg, - mimeImageGif, - mimeImagePng, - mimeImageWebp, - mimeVideoMp4, + "image/jpeg", // .jpeg + "image/gif", // .gif + "image/webp", // .webp + + "audio/mp2", // .mp2 + "audio/mp3", // .mp3 + + "video/x-msvideo", // .avi + + // png types + "image/png", // .png + "image/apng", // .apng + + // ogg types + "audio/ogg", // .ogg + "video/ogg", // .ogv + + // mpeg4 types + "audio/x-m4a", // .m4a + "video/mp4", // .mp4 + "video/quicktime", // .mov + + // asf types + "audio/x-ms-wma", // .wma + "video/x-ms-wmv", // .wmv + + // matroska types + "video/webm", // .webm + "audio/x-matroska", // .mka + "video/x-matroska", // .mkv } var SupportedEmojiMIMETypes = []string{ - mimeImageGif, - mimeImagePng, - mimeImageWebp, + "image/jpeg", // .jpeg + "image/gif", // .gif + "image/webp", // .webp + + // png types + "image/png", // .png + "image/apng", // .apng } type Manager struct { @@ -102,8 +131,8 @@ func (m *Manager) CreateMedia( id, // Always encode attachment - // thumbnails as jpg. - "jpg", + // thumbnails as jpeg. + "jpeg", ) // Calculate attachment thumbnail URL. @@ -114,8 +143,8 @@ func (m *Manager) CreateMedia( id, // Always encode attachment - // thumbnails as jpg. - "jpg", + // thumbnails as jpeg. + "jpeg", ) // Populate initial fields on the new media, @@ -134,7 +163,7 @@ func (m *Manager) CreateMedia( Path: path, }, Thumbnail: gtsmodel.Thumbnail{ - ContentType: mimeImageJpeg, // thumbs always jpg. + ContentType: "image/jpeg", Path: thumbPath, URL: thumbURL, }, @@ -244,7 +273,7 @@ func (m *Manager) CreateEmoji( // All static emojis // are encoded as png. - mimePng, + "png", ) // Generate static image path for attachment. @@ -256,7 +285,7 @@ func (m *Manager) CreateEmoji( // All static emojis // are encoded as png. - mimePng, + "png", ) // Populate initial fields on the new emoji, @@ -268,7 +297,7 @@ func (m *Manager) CreateEmoji( Domain: domain, ImageStaticURL: staticURL, ImageStaticPath: staticPath, - ImageStaticContentType: mimeImagePng, + ImageStaticContentType: "image/png", Disabled: util.Ptr(false), VisibleInPicker: util.Ptr(true), CreatedAt: now, @@ -368,7 +397,7 @@ func (m *Manager) RefreshEmoji( // All static emojis // are encoded as png. - mimePng, + "png", ) // Generate new static image storage path for emoji. @@ -380,7 +409,7 @@ func (m *Manager) RefreshEmoji( // All static emojis // are encoded as png. - mimePng, + "png", ) // Finally, create new emoji in database. diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index a099d2b95..24e0ddd1e 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -421,7 +421,7 @@ func (suite *ManagerTestSuite) TestSlothVineProcess() { suite.Equal(81120, attachment.FileMeta.Original.Size) suite.EqualValues(float32(1.4083333), attachment.FileMeta.Original.Aspect) suite.EqualValues(float32(6.641), *attachment.FileMeta.Original.Duration) - suite.EqualValues(float32(29.00003), *attachment.FileMeta.Original.Framerate) + suite.EqualValues(float32(29), *attachment.FileMeta.Original.Framerate) suite.EqualValues(0x5be18, *attachment.FileMeta.Original.Bitrate) suite.EqualValues(gtsmodel.Small{ Width: 338, Height: 240, Size: 81120, Aspect: 1.4083333333333334, diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index cca456837..996a3aa03 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -160,27 +160,17 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { // Pass input file through ffprobe to // parse further metadata information. result, err := ffprobe(ctx, temppath) - if err != nil { - return gtserror.Newf("error ffprobing data: %w", err) - } - - switch { - // No errors parsing data. - case result.Error == nil: - - // Data type unhandleable by ffprobe. - case result.Error.Code == -1094995529: + if err != nil && !isUnsupportedTypeErr(err) { + return gtserror.Newf("ffprobe error: %w", err) + } else if result == nil { log.Warn(ctx, "unsupported data type") return nil - - default: - return gtserror.Newf("ffprobe error: %w", err) } var ext string - // Set media type from ffprobe format data. - fileType, ext := result.Format.GetFileType() + // Get type from ffprobe format data. + fileType, ext := result.GetFileType() if fileType != gtsmodel.FileTypeImage { return gtserror.Newf("unsupported emoji filetype: %s (%s)", fileType, ext) } diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index 8ee242749..e5af46a2f 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -180,36 +180,33 @@ func (p *ProcessingMedia) store(ctx context.Context) error { // Pass input file through ffprobe to // parse further metadata information. result, err := ffprobe(ctx, temppath) - if err != nil { - return gtserror.Newf("error ffprobing data: %w", err) - } - - switch { - // No errors parsing data. - case result.Error == nil: - - // Data type unhandleable by ffprobe. - case result.Error.Code == -1094995529: + if err != nil && !isUnsupportedTypeErr(err) { + return gtserror.Newf("ffprobe error: %w", err) + } else if result == nil { log.Warn(ctx, "unsupported data type") return nil - - default: - return gtserror.Newf("ffprobe error: %w", err) } var ext string - // Set the media type from ffprobe format data. - p.media.Type, ext = result.Format.GetFileType() - if p.media.Type == gtsmodel.FileTypeUnknown { - - // Return early (deleting file) - // for unhandled file types. - return nil - } + // Extract any video stream metadata from media. + // This will always be used regardless of type, + // as even audio files may contain embedded album art. + width, height, framerate := result.ImageMeta() + p.media.FileMeta.Original.Width = width + p.media.FileMeta.Original.Height = height + p.media.FileMeta.Original.Size = (width * height) + p.media.FileMeta.Original.Aspect = util.Div(float32(width), float32(height)) + p.media.FileMeta.Original.Framerate = util.PtrIf(framerate) + p.media.FileMeta.Original.Duration = util.PtrIf(float32(result.duration)) + p.media.FileMeta.Original.Bitrate = util.PtrIf(result.bitrate) + // Set media type from ffprobe format data. + p.media.Type, ext = result.GetFileType() switch p.media.Type { - case gtsmodel.FileTypeImage: + + case gtsmodel.FileTypeImage, + gtsmodel.FileTypeVideo: // Pass file through ffmpeg clearing // any excess metadata (e.g. EXIF). if err := ffmpegClearMetadata(ctx, @@ -218,16 +215,16 @@ func (p *ProcessingMedia) store(ctx context.Context) error { return gtserror.Newf("error cleaning metadata: %w", err) } - // Extract image metadata from streams. - width, height, err := result.ImageMeta() - if err != nil { - return err - } - p.media.FileMeta.Original.Width = width - p.media.FileMeta.Original.Height = height - p.media.FileMeta.Original.Size = (width * height) - p.media.FileMeta.Original.Aspect = float32(width) / float32(height) + case gtsmodel.FileTypeAudio: + // NOTE: we do not clean audio file + // metadata, in order to keep tags. + default: + log.Warn(ctx, "unsupported data type: %s", result.format) + return nil + } + + if width > 0 && height > 0 { // Determine thumbnail dimensions to use. thumbWidth, thumbHeight := thumbSize(width, height) p.media.FileMeta.Small.Width = thumbWidth @@ -244,90 +241,13 @@ func (p *ProcessingMedia) store(ctx context.Context) error { return gtserror.Newf("error generating image thumb: %w", err) } - case gtsmodel.FileTypeVideo: - // Pass file through ffmpeg clearing - // any excess metadata (e.g. EXIF). - if err := ffmpegClearMetadata(ctx, - temppath, ext, - ); err != nil { - return gtserror.Newf("error cleaning metadata: %w", err) - } - - // Extract video metadata we can from streams. - width, height, framerate, err := result.VideoMeta() - if err != nil { - return err - } - p.media.FileMeta.Original.Width = width - p.media.FileMeta.Original.Height = height - p.media.FileMeta.Original.Size = (width * height) - p.media.FileMeta.Original.Aspect = float32(width) / float32(height) - p.media.FileMeta.Original.Framerate = &framerate - - // Extract total duration from format. - duration := result.Format.GetDuration() - p.media.FileMeta.Original.Duration = &duration - - // Extract total bitrate from format. - bitrate := result.Format.GetBitRate() - p.media.FileMeta.Original.Bitrate = &bitrate - - // Determine thumbnail dimensions to use. - thumbWidth, thumbHeight := thumbSize(width, height) - p.media.FileMeta.Small.Width = thumbWidth - p.media.FileMeta.Small.Height = thumbHeight - p.media.FileMeta.Small.Size = (thumbWidth * thumbHeight) - p.media.FileMeta.Small.Aspect = float32(thumbWidth) / float32(thumbHeight) - - // Extract a thumbnail frame from input video path. - thumbpath, err = ffmpegGenerateThumb(ctx, temppath, - thumbWidth, - thumbHeight, - ) - if err != nil { - return gtserror.Newf("error extracting video frame: %w", err) - } - - case gtsmodel.FileTypeAudio: - // Extract total duration from format. - duration := result.Format.GetDuration() - p.media.FileMeta.Original.Duration = &duration - - // Extract total bitrate from format. - bitrate := result.Format.GetBitRate() - p.media.FileMeta.Original.Bitrate = &bitrate - - // Extract image metadata from streams (if any), - // this will only exist for embedded album art. - width, height, framerate, _ := result.EmbeddedImageMeta() - if width > 0 && height > 0 { - // Unlikely to need these but masto API includes them. - p.media.FileMeta.Original.Width = width - p.media.FileMeta.Original.Height = height - if framerate != 0 { - p.media.FileMeta.Original.Framerate = &framerate - } - - // Determine thumbnail dimensions to use. - thumbWidth, thumbHeight := thumbSize(width, height) - p.media.FileMeta.Small.Width = thumbWidth - p.media.FileMeta.Small.Height = thumbHeight - p.media.FileMeta.Small.Size = (thumbWidth * thumbHeight) - p.media.FileMeta.Small.Aspect = float32(thumbWidth) / float32(thumbHeight) - - // Generate a thumbnail image from input image path. - thumbpath, err = ffmpegGenerateThumb(ctx, temppath, - thumbWidth, - thumbHeight, - ) + if p.media.Blurhash == "" { + // Generate blurhash (if not already) from thumbnail. + p.media.Blurhash, err = generateBlurhash(thumbpath) if err != nil { - return gtserror.Newf("error generating image thumb: %w", err) + return gtserror.Newf("error generating thumb blurhash: %w", err) } } - - default: - log.Warnf(ctx, "unsupported type: %s (%s)", p.media.Type, result.Format.FormatName) - return nil } // Calculate final media attachment file path. @@ -352,17 +272,6 @@ func (p *ProcessingMedia) store(ctx context.Context) error { p.media.File.FileSize = int(filesz) if thumbpath != "" { - // Note that neither thumbnail storage - // nor a blurhash are needed for audio. - - if p.media.Blurhash == "" { - // Generate blurhash (if not already) from thumbnail. - p.media.Blurhash, err = generateBlurhash(thumbpath) - if err != nil { - return gtserror.Newf("error generating thumb blurhash: %w", err) - } - } - // Copy thumbnail file into storage at path. thumbsz, err := p.mgr.state.Storage.PutFile(ctx, p.media.Thumbnail.Path, diff --git a/internal/media/types.go b/internal/media/types.go index 2d19b84cc..9631a15bd 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -23,27 +23,6 @@ "time" ) -// mime consts -const ( - mimeImage = "image" - mimeVideo = "video" - - mimeJpeg = "jpeg" - mimeImageJpeg = mimeImage + "/" + mimeJpeg - - mimeGif = "gif" - mimeImageGif = mimeImage + "/" + mimeGif - - mimePng = "png" - mimeImagePng = mimeImage + "/" + mimePng - - mimeWebp = "webp" - mimeImageWebp = mimeImage + "/" + mimeWebp - - mimeMp4 = "mp4" - mimeVideoMp4 = mimeVideo + "/" + mimeMp4 -) - type Size string const ( diff --git a/internal/typeutils/internaltofrontend_test.go b/internal/typeutils/internaltofrontend_test.go index 6429df4fa..c4da0d57c 100644 --- a/internal/typeutils/internaltofrontend_test.go +++ b/internal/typeutils/internaltofrontend_test.go @@ -1225,9 +1225,22 @@ func (suite *InternalToFrontendTestSuite) TestInstanceV1ToFrontend() { "supported_mime_types": [ "image/jpeg", "image/gif", - "image/png", "image/webp", - "video/mp4" + "audio/mp2", + "audio/mp3", + "video/x-msvideo", + "image/png", + "image/apng", + "audio/ogg", + "video/ogg", + "audio/x-m4a", + "video/mp4", + "video/quicktime", + "audio/x-ms-wma", + "video/x-ms-wmv", + "video/webm", + "audio/x-matroska", + "video/x-matroska" ], "image_size_limit": 41943040, "image_matrix_limit": 16777216, @@ -1350,9 +1363,22 @@ func (suite *InternalToFrontendTestSuite) TestInstanceV2ToFrontend() { "supported_mime_types": [ "image/jpeg", "image/gif", - "image/png", "image/webp", - "video/mp4" + "audio/mp2", + "audio/mp3", + "video/x-msvideo", + "image/png", + "image/apng", + "audio/ogg", + "video/ogg", + "audio/x-m4a", + "video/mp4", + "video/quicktime", + "audio/x-ms-wma", + "video/x-ms-wmv", + "video/webm", + "audio/x-matroska", + "video/x-matroska" ], "image_size_limit": 41943040, "image_matrix_limit": 16777216, diff --git a/internal/util/math.go b/internal/util/math.go new file mode 100644 index 000000000..e1850f772 --- /dev/null +++ b/internal/util/math.go @@ -0,0 +1,34 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package util + +type Number interface { + ~int | ~int8 | ~int16 | ~int32 | ~int64 | + ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | + ~uintptr | ~float32 | ~float64 +} + +// Div performs a safe division of +// n1 and n2, checking for zero n2. In the +// case of zero n2, zero is returned. +func Div[N Number](n1, n2 N) N { + if n2 == 0 { + return 0 + } + return n1 / n2 +} diff --git a/internal/util/ptr.go b/internal/util/ptr.go index 0ad207617..d7c30da85 100644 --- a/internal/util/ptr.go +++ b/internal/util/ptr.go @@ -34,6 +34,15 @@ func Ptr[T any](t T) *T { return &t } +// PtrIf returns ptr value only if 't' non-zero. +func PtrIf[T comparable](t T) *T { + var z T + if t == z { + return nil + } + return &t +} + // PtrValueOr returns either value of ptr, or default. func PtrValueOr[T any](t *T, _default T) T { if t != nil {