[feature] more filetype support! (#3107)

* add more supported file types to our media processor that ffmpeg supports, update supported mime type lists

* add code comments to the supported mime types slice

* don't check for zero value string, just parse

* remove some unneeded consts which make the code a bit harder to read

* fix test expected instance media mime types, use compact ffprobe json, simple media processing by type

* final tweaks to media processing code

* don't use safe divide where we don't need to

cmd/process-emoji/main.go

@@ -29,6 +29,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/media/ffmpeg"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
@@ -43,6 +44,14 @@ func main() {
 		log.Panic(ctx, "Usage: go run ./cmd/process-emoji <input-file> <output-static>")
 	}
 
+	if err := ffmpeg.InitFfprobe(ctx, 1); err != nil {
+		log.Panic(ctx, err)
+	}
+
+	if err := ffmpeg.InitFfmpeg(ctx, 1); err != nil {
+		log.Panic(ctx, err)
+	}
+
 	var st storage.Driver
 	st.Storage = memory.Open(10, true)
 

cmd/process-media/main.go

@@ -29,6 +29,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/media/ffmpeg"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 )
@@ -42,6 +43,14 @@ func main() {
 		log.Panic(ctx, "Usage: go run ./cmd/process-media <input-file> <output-processed> <output-thumbnail>")
 	}
 
+	if err := ffmpeg.InitFfprobe(ctx, 1); err != nil {
+		log.Panic(ctx, err)
+	}
+
+	if err := ffmpeg.InitFfmpeg(ctx, 1); err != nil {
+		log.Panic(ctx, err)
+	}
+
 	var st storage.Driver
 	st.Storage = memory.Open(10, true)
 
@@ -105,6 +114,9 @@ func(ctx context.Context) (reader io.ReadCloser, err error) {
 func copyFile(ctx context.Context, st *storage.Driver, key string, path string) {
 	rc, err := st.GetStream(ctx, key)
 	if err != nil {
+		if storage.IsNotFound(err) {
+			return
+		}
 		log.Panic(ctx, err)
 	}
 	defer rc.Close()

internal/api/client/instance/instancepatch_test.go

@@ -105,9 +105,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch1() {
       "supported_mime_types": [
         "image/jpeg",
         "image/gif",
-        "image/png",
         "image/webp",
-        "video/mp4"
+        "audio/mp2",
+        "audio/mp3",
+        "video/x-msvideo",
+        "image/png",
+        "image/apng",
+        "audio/ogg",
+        "video/ogg",
+        "audio/x-m4a",
+        "video/mp4",
+        "video/quicktime",
+        "audio/x-ms-wma",
+        "video/x-ms-wmv",
+        "video/webm",
+        "audio/x-matroska",
+        "video/x-matroska"
       ],
       "image_size_limit": 41943040,
       "image_matrix_limit": 16777216,
@@ -226,9 +239,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch2() {
       "supported_mime_types": [
         "image/jpeg",
         "image/gif",
-        "image/png",
         "image/webp",
-        "video/mp4"
+        "audio/mp2",
+        "audio/mp3",
+        "video/x-msvideo",
+        "image/png",
+        "image/apng",
+        "audio/ogg",
+        "video/ogg",
+        "audio/x-m4a",
+        "video/mp4",
+        "video/quicktime",
+        "audio/x-ms-wma",
+        "video/x-ms-wmv",
+        "video/webm",
+        "audio/x-matroska",
+        "video/x-matroska"
       ],
       "image_size_limit": 41943040,
       "image_matrix_limit": 16777216,
@@ -347,9 +373,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch3() {
       "supported_mime_types": [
         "image/jpeg",
         "image/gif",
-        "image/png",
         "image/webp",
-        "video/mp4"
+        "audio/mp2",
+        "audio/mp3",
+        "video/x-msvideo",
+        "image/png",
+        "image/apng",
+        "audio/ogg",
+        "video/ogg",
+        "audio/x-m4a",
+        "video/mp4",
+        "video/quicktime",
+        "audio/x-ms-wma",
+        "video/x-ms-wmv",
+        "video/webm",
+        "audio/x-matroska",
+        "video/x-matroska"
       ],
       "image_size_limit": 41943040,
       "image_matrix_limit": 16777216,
@@ -519,9 +558,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch6() {
       "supported_mime_types": [
         "image/jpeg",
         "image/gif",
-        "image/png",
         "image/webp",
-        "video/mp4"
+        "audio/mp2",
+        "audio/mp3",
+        "video/x-msvideo",
+        "image/png",
+        "image/apng",
+        "audio/ogg",
+        "video/ogg",
+        "audio/x-m4a",
+        "video/mp4",
+        "video/quicktime",
+        "audio/x-ms-wma",
+        "video/x-ms-wmv",
+        "video/webm",
+        "audio/x-matroska",
+        "video/x-matroska"
       ],
       "image_size_limit": 41943040,
       "image_matrix_limit": 16777216,
@@ -662,9 +714,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch8() {
       "supported_mime_types": [
         "image/jpeg",
         "image/gif",
-        "image/png",
         "image/webp",
-        "video/mp4"
+        "audio/mp2",
+        "audio/mp3",
+        "video/x-msvideo",
+        "image/png",
+        "image/apng",
+        "audio/ogg",
+        "video/ogg",
+        "audio/x-m4a",
+        "video/mp4",
+        "video/quicktime",
+        "audio/x-ms-wma",
+        "video/x-ms-wmv",
+        "video/webm",
+        "audio/x-matroska",
+        "video/x-matroska"
       ],
       "image_size_limit": 41943040,
       "image_matrix_limit": 16777216,
@@ -820,9 +885,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch9() {
       "supported_mime_types": [
         "image/jpeg",
         "image/gif",
-        "image/png",
         "image/webp",
-        "video/mp4"
+        "audio/mp2",
+        "audio/mp3",
+        "video/x-msvideo",
+        "image/png",
+        "image/apng",
+        "audio/ogg",
+        "video/ogg",
+        "audio/x-m4a",
+        "video/mp4",
+        "video/quicktime",
+        "audio/x-ms-wma",
+        "video/x-ms-wmv",
+        "video/webm",
+        "audio/x-matroska",
+        "video/x-matroska"
       ],
       "image_size_limit": 41943040,
       "image_matrix_limit": 16777216,

internal/media/ffmpeg.go

@@ -18,7 +18,6 @@
 package media
 
 import (
-	"cmp"
 	"context"
 	"encoding/json"
 	"errors"
@@ -135,7 +134,7 @@ func ffmpeg(ctx context.Context, dirpath string, args ...string) error {
 }
 
 // ffprobe calls `ffprobe` (WASM) on filepath, returning parsed JSON output.
-func ffprobe(ctx context.Context, filepath string) (*ffprobeResult, error) {
+func ffprobe(ctx context.Context, filepath string) (*result, error) {
 	var stdout byteutil.Buffer
 
 	// Get directory from filepath.
@@ -148,7 +147,7 @@ func ffprobe(ctx context.Context, filepath string) (*ffprobeResult, error) {
 		Args: []string{
 			"-i", filepath,
 			"-loglevel", "quiet",
-			"-print_format", "json",
+			"-print_format", "json=compact=1",
 			"-show_streams",
 			"-show_format",
 			"-show_error",
@@ -172,7 +171,219 @@ func ffprobe(ctx context.Context, filepath string) (*ffprobeResult, error) {
 		return nil, gtserror.Newf("error unmarshaling json: %w", err)
 	}
 
-	return &result, nil
+	// Convert raw result data.
+	res, err := result.Process()
+	if err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}
+
+// result contains parsed ffprobe result
+// data in a more useful data format.
+type result struct {
+	format   string
+	audio    []audioStream
+	video    []videoStream
+	bitrate  uint64
+	duration float64
+}
+
+type stream struct {
+	codec string
+}
+
+type audioStream struct {
+	stream
+}
+
+type videoStream struct {
+	stream
+	width     int
+	height    int
+	framerate float32
+}
+
+// GetFileType determines file type and extension to use for media data. This
+// function helps to abstract away the horrible complexities that are possible
+// media container (i.e. the file) types and and possible sub-types within that.
+//
+// Note the checks for (len(res.video) > 0) may catch some audio files with embedded
+// album art as video, but i blame that on the hellscape that is media filetypes.
+//
+// TODO: we can update this code to also return a mimetype and avoid later parsing!
+func (res *result) GetFileType() (gtsmodel.FileType, string) {
+	switch res.format {
+	case "mpeg":
+		return gtsmodel.FileTypeVideo, "mpeg"
+	case "mjpeg":
+		return gtsmodel.FileTypeVideo, "mjpeg"
+	case "mov,mp4,m4a,3gp,3g2,mj2":
+		switch {
+		case len(res.video) > 0:
+			return gtsmodel.FileTypeVideo, "mp4"
+		case len(res.audio) > 0 &&
+			res.audio[0].codec == "aac":
+			// m4a only supports [aac] audio.
+			return gtsmodel.FileTypeAudio, "m4a"
+		}
+	case "apng":
+		return gtsmodel.FileTypeImage, "apng"
+	case "png_pipe":
+		return gtsmodel.FileTypeImage, "png"
+	case "image2", "image2pipe", "jpeg_pipe":
+		return gtsmodel.FileTypeImage, "jpeg"
+	case "webp", "webp_pipe":
+		return gtsmodel.FileTypeImage, "webp"
+	case "gif":
+		return gtsmodel.FileTypeImage, "gif"
+	case "mp3":
+		if len(res.audio) > 0 {
+			switch res.audio[0].codec {
+			case "mp2":
+				return gtsmodel.FileTypeAudio, "mp2"
+			case "mp3":
+				return gtsmodel.FileTypeAudio, "mp3"
+			}
+		}
+	case "asf":
+		switch {
+		case len(res.video) > 0:
+			return gtsmodel.FileTypeVideo, "wmv"
+		case len(res.audio) > 0:
+			return gtsmodel.FileTypeAudio, "wma"
+		}
+	case "ogg":
+		switch {
+		case len(res.video) > 0:
+			return gtsmodel.FileTypeVideo, "ogv"
+		case len(res.audio) > 0:
+			return gtsmodel.FileTypeAudio, "ogg"
+		}
+	case "matroska,webm":
+		switch {
+		case len(res.video) > 0:
+			switch res.video[0].codec {
+			case "vp8", "vp9", "av1":
+			default:
+				return gtsmodel.FileTypeVideo, "mkv"
+			}
+			if len(res.audio) > 0 {
+				switch res.audio[0].codec {
+				case "vorbis", "opus", "libopus":
+					// webm only supports [VP8/VP9/AV1]+[vorbis/opus]
+					return gtsmodel.FileTypeVideo, "webm"
+				}
+			}
+		case len(res.audio) > 0:
+			return gtsmodel.FileTypeAudio, "mka"
+		}
+	case "avi":
+		return gtsmodel.FileTypeVideo, "avi"
+	}
+	return gtsmodel.FileTypeUnknown, res.format
+}
+
+// ImageMeta extracts image metadata contained within ffprobe'd media result streams.
+func (res *result) ImageMeta() (width int, height int, framerate float32) {
+	for _, stream := range res.video {
+		if stream.width > width {
+			width = stream.width
+		}
+		if stream.height > height {
+			height = stream.height
+		}
+		if fr := float32(stream.framerate); fr > 0 {
+			if framerate == 0 || fr < framerate {
+				framerate = fr
+			}
+		}
+	}
+	return
+}
+
+// Process converts raw ffprobe result data into our more usable result{} type.
+func (res *ffprobeResult) Process() (*result, error) {
+	if res.Error != nil {
+		return nil, res.Error
+	}
+
+	if res.Format == nil {
+		return nil, errors.New("missing format data")
+	}
+
+	var r result
+	var err error
+
+	// Copy over container format.
+	r.format = res.Format.FormatName
+
+	// Parsed media bitrate (if it was set).
+	if str := res.Format.BitRate; str != "" {
+		r.bitrate, err = strconv.ParseUint(str, 10, 64)
+		if err != nil {
+			return nil, gtserror.Newf("invalid bitrate %s: %w", str, err)
+		}
+	}
+
+	// Parse media duration (if it was set).
+	if str := res.Format.Duration; str != "" {
+		r.duration, err = strconv.ParseFloat(str, 32)
+		if err != nil {
+			return nil, gtserror.Newf("invalid duration %s: %w", str, err)
+		}
+	}
+
+	// Preallocate streams to max possible lengths.
+	r.audio = make([]audioStream, 0, len(res.Streams))
+	r.video = make([]videoStream, 0, len(res.Streams))
+
+	// Convert streams to separate types.
+	for _, s := range res.Streams {
+		switch s.CodecType {
+		case "audio":
+			// Append audio stream data to result.
+			r.audio = append(r.audio, audioStream{
+				stream: stream{codec: s.CodecName},
+			})
+		case "video":
+			var framerate float32
+
+			// Parse stream framerate, bearing in
+			// mind that some static container formats
+			// (e.g. jpeg) still return a framerate, so
+			// we also check for a non-1 timebase (dts).
+			if str := s.RFrameRate; str != "" &&
+				s.DurationTS > 1 {
+				var num, den uint32
+				den = 1
+
+				// Check for inequality (numerator / denominator).
+				if p := strings.SplitN(str, "/", 2); len(p) == 2 {
+					n, _ := strconv.ParseUint(p[0], 10, 32)
+					d, _ := strconv.ParseUint(p[1], 10, 32)
+					num, den = uint32(n), uint32(d)
+				} else {
+					n, _ := strconv.ParseUint(p[0], 10, 32)
+					num = uint32(n)
+				}
+
+				// Set final divised framerate.
+				framerate = float32(num / den)
+			}
+
+			// Append video stream data to result.
+			r.video = append(r.video, videoStream{
+				stream:    stream{codec: s.CodecName},
+				width:     s.Width,
+				height:    s.Height,
+				framerate: framerate,
+			})
+		}
+	}
+
+	return &r, nil
 }
 
 // ffprobeResult contains parsed JSON data from
@@ -183,175 +394,33 @@ type ffprobeResult struct {
 	Error   *ffprobeError   `json:"error"`
 }
 
-// ImageMeta extracts image metadata contained within ffprobe'd media result streams.
-func (res *ffprobeResult) ImageMeta() (width int, height int, err error) {
-	for _, stream := range res.Streams {
-		if stream.Width > width {
-			width = stream.Width
-		}
-		if stream.Height > height {
-			height = stream.Height
-		}
-	}
-	if width == 0 || height == 0 {
-		err = errors.New("invalid image stream(s)")
-	}
-	return
-}
-
-// EmbeddedImageMeta extracts embedded image metadata contained within ffprobe'd media result
-// streams, should be used for pulling album image (can be animated image) from audio files.
-func (res *ffprobeResult) EmbeddedImageMeta() (width int, height int, framerate float32, err error) {
-	for _, stream := range res.Streams {
-		if stream.Width > width {
-			width = stream.Width
-		}
-		if stream.Height > height {
-			height = stream.Height
-		}
-		if fr := stream.GetFrameRate(); fr > 0 {
-			if framerate == 0 || fr < framerate {
-				framerate = fr
-			}
-		}
-	}
-	// Need width + height but
-	// no framerate is fine.
-	if width == 0 || height == 0 {
-		err = errors.New("invalid image stream(s)")
-	}
-	return
-}
-
-// VideoMeta extracts video metadata contained within ffprobe'd media result streams.
-func (res *ffprobeResult) VideoMeta() (width, height int, framerate float32, err error) {
-	for _, stream := range res.Streams {
-		if stream.Width > width {
-			width = stream.Width
-		}
-		if stream.Height > height {
-			height = stream.Height
-		}
-		if fr := stream.GetFrameRate(); fr > 0 {
-			if framerate == 0 || fr < framerate {
-				framerate = fr
-			}
-		}
-	}
-	if width == 0 || height == 0 || framerate == 0 {
-		err = errors.New("invalid video stream(s)")
-	}
-	return
-}
-
 type ffprobeStream struct {
 	CodecName  string `json:"codec_name"`
-	AvgFrameRate string `json:"avg_frame_rate"`
+	CodecType  string `json:"codec_type"`
 	RFrameRate string `json:"r_frame_rate"`
+	DurationTS uint   `json:"duration_ts"`
 	Width      int    `json:"width"`
 	Height     int    `json:"height"`
 	// + unused fields.
 }
 
-// GetFrameRate calculates float32 framerate value from stream json string.
-func (str *ffprobeStream) GetFrameRate() float32 {
-	numDen := func(strFR string) (float32, float32) {
-		var (
-			// numerator
-			num float32
-
-			// denominator
-			den float32
-		)
-
-		// Check for a provided inequality, i.e. numerator / denominator.
-		if p := strings.SplitN(strFR, "/", 2); len(p) == 2 {
-			n, _ := strconv.ParseFloat(p[0], 32)
-			d, _ := strconv.ParseFloat(p[1], 32)
-			num, den = float32(n), float32(d)
-		} else {
-			n, _ := strconv.ParseFloat(p[0], 32)
-			num = float32(n)
-		}
-
-		return num, den
-	}
-
-	var num, den float32
-	if str.AvgFrameRate != "" {
-		// Check if we have avg_frame_rate.
-		num, den = numDen(str.AvgFrameRate)
-	}
-
-	if num == 0 && str.RFrameRate != "" {
-		// Check if we have r_frame_rate.
-		num, den = numDen(str.RFrameRate)
-	}
-
-	if num != 0 {
-		// Found it.
-		// Avoid divide by zero.
-		return num / cmp.Or(den, 1)
-	}
-
-	return 0
-}
-
 type ffprobeFormat struct {
-	Filename   string `json:"filename"`
 	FormatName string `json:"format_name"`
 	Duration   string `json:"duration"`
 	BitRate    string `json:"bit_rate"`
 	// + unused fields
 }
 
-// GetFileType determines file type and extension to use for media data.
-func (fmt *ffprobeFormat) GetFileType() (gtsmodel.FileType, string) {
-	switch fmt.FormatName {
-	case "mov,mp4,m4a,3gp,3g2,mj2":
-		return gtsmodel.FileTypeVideo, "mp4"
-	case "apng":
-		return gtsmodel.FileTypeImage, "apng"
-	case "png_pipe":
-		return gtsmodel.FileTypeImage, "png"
-	case "image2", "jpeg_pipe":
-		return gtsmodel.FileTypeImage, "jpeg"
-	case "webp_pipe":
-		return gtsmodel.FileTypeImage, "webp"
-	case "gif":
-		return gtsmodel.FileTypeImage, "gif"
-	case "mp3":
-		return gtsmodel.FileTypeAudio, "mp3"
-	case "ogg":
-		return gtsmodel.FileTypeAudio, "ogg"
-	default:
-		return gtsmodel.FileTypeUnknown, fmt.FormatName
-	}
-}
-
-// GetDuration calculates float32 framerate value from format json string.
-func (fmt *ffprobeFormat) GetDuration() float32 {
-	if fmt.Duration != "" {
-		dur, _ := strconv.ParseFloat(fmt.Duration, 32)
-		return float32(dur)
-	}
-	return 0
-}
-
-// GetBitRate calculates uint64 bitrate value from format json string.
-func (fmt *ffprobeFormat) GetBitRate() uint64 {
-	if fmt.BitRate != "" {
-		r, _ := strconv.ParseUint(fmt.BitRate, 10, 64)
-		return r
-	}
-	return 0
-}
-
 type ffprobeError struct {
 	Code   int    `json:"code"`
 	String string `json:"string"`
 }
 
+func isUnsupportedTypeErr(err error) bool {
+	ffprobeErr, ok := err.(*ffprobeError)
+	return ok && ffprobeErr.Code == -1094995529
+}
+
 func (err *ffprobeError) Error() string {
 	return err.String + " (" + strconv.Itoa(err.Code) + ")"
 }

internal/media/manager.go

@@ -34,17 +34,46 @@
 )
 
 var SupportedMIMETypes = []string{
-	mimeImageJpeg,
+	"image/jpeg", // .jpeg
-	mimeImageGif,
+	"image/gif",  // .gif
-	mimeImagePng,
+	"image/webp", // .webp
-	mimeImageWebp,
+
-	mimeVideoMp4,
+	"audio/mp2", // .mp2
+	"audio/mp3", // .mp3
+
+	"video/x-msvideo", // .avi
+
+	// png types
+	"image/png",  // .png
+	"image/apng", // .apng
+
+	// ogg types
+	"audio/ogg", // .ogg
+	"video/ogg", // .ogv
+
+	// mpeg4 types
+	"audio/x-m4a",     // .m4a
+	"video/mp4",       // .mp4
+	"video/quicktime", // .mov
+
+	// asf types
+	"audio/x-ms-wma", // .wma
+	"video/x-ms-wmv", // .wmv
+
+	// matroska types
+	"video/webm",       // .webm
+	"audio/x-matroska", // .mka
+	"video/x-matroska", // .mkv
 }
 
 var SupportedEmojiMIMETypes = []string{
-	mimeImageGif,
+	"image/jpeg", // .jpeg
-	mimeImagePng,
+	"image/gif",  // .gif
-	mimeImageWebp,
+	"image/webp", // .webp
+
+	// png types
+	"image/png",  // .png
+	"image/apng", // .apng
 }
 
 type Manager struct {
@@ -102,8 +131,8 @@ func (m *Manager) CreateMedia(
 		id,
 
 		// Always encode attachment
-		// thumbnails as jpg.
+		// thumbnails as jpeg.
-		"jpg",
+		"jpeg",
 	)
 
 	// Calculate attachment thumbnail URL.
@@ -114,8 +143,8 @@ func (m *Manager) CreateMedia(
 		id,
 
 		// Always encode attachment
-		// thumbnails as jpg.
+		// thumbnails as jpeg.
-		"jpg",
+		"jpeg",
 	)
 
 	// Populate initial fields on the new media,
@@ -134,7 +163,7 @@ func (m *Manager) CreateMedia(
 			Path:        path,
 		},
 		Thumbnail: gtsmodel.Thumbnail{
-			ContentType: mimeImageJpeg, // thumbs always jpg.
+			ContentType: "image/jpeg",
 			Path:        thumbPath,
 			URL:         thumbURL,
 		},
@@ -244,7 +273,7 @@ func (m *Manager) CreateEmoji(
 
 		// All static emojis
 		// are encoded as png.
-		mimePng,
+		"png",
 	)
 
 	// Generate static image path for attachment.
@@ -256,7 +285,7 @@ func (m *Manager) CreateEmoji(
 
 		// All static emojis
 		// are encoded as png.
-		mimePng,
+		"png",
 	)
 
 	// Populate initial fields on the new emoji,
@@ -268,7 +297,7 @@ func (m *Manager) CreateEmoji(
 		Domain:                 domain,
 		ImageStaticURL:         staticURL,
 		ImageStaticPath:        staticPath,
-		ImageStaticContentType: mimeImagePng,
+		ImageStaticContentType: "image/png",
 		Disabled:               util.Ptr(false),
 		VisibleInPicker:        util.Ptr(true),
 		CreatedAt:              now,
@@ -368,7 +397,7 @@ func (m *Manager) RefreshEmoji(
 
 		// All static emojis
 		// are encoded as png.
-		mimePng,
+		"png",
 	)
 
 	// Generate new static image storage path for emoji.
@@ -380,7 +409,7 @@ func (m *Manager) RefreshEmoji(
 
 		// All static emojis
 		// are encoded as png.
-		mimePng,
+		"png",
 	)
 
 	// Finally, create new emoji in database.

internal/media/manager_test.go

@@ -421,7 +421,7 @@ func (suite *ManagerTestSuite) TestSlothVineProcess() {
 	suite.Equal(81120, attachment.FileMeta.Original.Size)
 	suite.EqualValues(float32(1.4083333), attachment.FileMeta.Original.Aspect)
 	suite.EqualValues(float32(6.641), *attachment.FileMeta.Original.Duration)
-	suite.EqualValues(float32(29.00003), *attachment.FileMeta.Original.Framerate)
+	suite.EqualValues(float32(29), *attachment.FileMeta.Original.Framerate)
 	suite.EqualValues(0x5be18, *attachment.FileMeta.Original.Bitrate)
 	suite.EqualValues(gtsmodel.Small{
 		Width: 338, Height: 240, Size: 81120, Aspect: 1.4083333333333334,

internal/media/processingemoji.go

@@ -160,27 +160,17 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
 	// Pass input file through ffprobe to
 	// parse further metadata information.
 	result, err := ffprobe(ctx, temppath)
-	if err != nil {
+	if err != nil && !isUnsupportedTypeErr(err) {
-		return gtserror.Newf("error ffprobing data: %w", err)
+		return gtserror.Newf("ffprobe error: %w", err)
-	}
+	} else if result == nil {
-
-	switch {
-	// No errors parsing data.
-	case result.Error == nil:
-
-	// Data type unhandleable by ffprobe.
-	case result.Error.Code == -1094995529:
 		log.Warn(ctx, "unsupported data type")
 		return nil
-
-	default:
-		return gtserror.Newf("ffprobe error: %w", err)
 	}
 
 	var ext string
 
-	// Set media type from ffprobe format data.
+	// Get type from ffprobe format data.
-	fileType, ext := result.Format.GetFileType()
+	fileType, ext := result.GetFileType()
 	if fileType != gtsmodel.FileTypeImage {
 		return gtserror.Newf("unsupported emoji filetype: %s (%s)", fileType, ext)
 	}

internal/media/processingmedia.go

@@ -180,36 +180,33 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 	// Pass input file through ffprobe to
 	// parse further metadata information.
 	result, err := ffprobe(ctx, temppath)
-	if err != nil {
+	if err != nil && !isUnsupportedTypeErr(err) {
-		return gtserror.Newf("error ffprobing data: %w", err)
+		return gtserror.Newf("ffprobe error: %w", err)
-	}
+	} else if result == nil {
-
-	switch {
-	// No errors parsing data.
-	case result.Error == nil:
-
-	// Data type unhandleable by ffprobe.
-	case result.Error.Code == -1094995529:
 		log.Warn(ctx, "unsupported data type")
 		return nil
-
-	default:
-		return gtserror.Newf("ffprobe error: %w", err)
 	}
 
 	var ext string
 
-	// Set the media type from ffprobe format data.
+	// Extract any video stream metadata from media.
-	p.media.Type, ext = result.Format.GetFileType()
+	// This will always be used regardless of type,
-	if p.media.Type == gtsmodel.FileTypeUnknown {
+	// as even audio files may contain embedded album art.
-
+	width, height, framerate := result.ImageMeta()
-		// Return early (deleting file)
+	p.media.FileMeta.Original.Width = width
-		// for unhandled file types.
+	p.media.FileMeta.Original.Height = height
-		return nil
+	p.media.FileMeta.Original.Size = (width * height)
-	}
+	p.media.FileMeta.Original.Aspect = util.Div(float32(width), float32(height))
+	p.media.FileMeta.Original.Framerate = util.PtrIf(framerate)
+	p.media.FileMeta.Original.Duration = util.PtrIf(float32(result.duration))
+	p.media.FileMeta.Original.Bitrate = util.PtrIf(result.bitrate)
 
+	// Set media type from ffprobe format data.
+	p.media.Type, ext = result.GetFileType()
 	switch p.media.Type {
-	case gtsmodel.FileTypeImage:
+
+	case gtsmodel.FileTypeImage,
+		gtsmodel.FileTypeVideo:
 		// Pass file through ffmpeg clearing
 		// any excess metadata (e.g. EXIF).
 		if err := ffmpegClearMetadata(ctx,
@@ -218,96 +215,16 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 			return gtserror.Newf("error cleaning metadata: %w", err)
 		}
 
-		// Extract image metadata from streams.
-		width, height, err := result.ImageMeta()
-		if err != nil {
-			return err
-		}
-		p.media.FileMeta.Original.Width = width
-		p.media.FileMeta.Original.Height = height
-		p.media.FileMeta.Original.Size = (width * height)
-		p.media.FileMeta.Original.Aspect = float32(width) / float32(height)
-
-		// Determine thumbnail dimensions to use.
-		thumbWidth, thumbHeight := thumbSize(width, height)
-		p.media.FileMeta.Small.Width = thumbWidth
-		p.media.FileMeta.Small.Height = thumbHeight
-		p.media.FileMeta.Small.Size = (thumbWidth * thumbHeight)
-		p.media.FileMeta.Small.Aspect = float32(thumbWidth) / float32(thumbHeight)
-
-		// Generate a thumbnail image from input image path.
-		thumbpath, err = ffmpegGenerateThumb(ctx, temppath,
-			thumbWidth,
-			thumbHeight,
-		)
-		if err != nil {
-			return gtserror.Newf("error generating image thumb: %w", err)
-		}
-
-	case gtsmodel.FileTypeVideo:
-		// Pass file through ffmpeg clearing
-		// any excess metadata (e.g. EXIF).
-		if err := ffmpegClearMetadata(ctx,
-			temppath, ext,
-		); err != nil {
-			return gtserror.Newf("error cleaning metadata: %w", err)
-		}
-
-		// Extract video metadata we can from streams.
-		width, height, framerate, err := result.VideoMeta()
-		if err != nil {
-			return err
-		}
-		p.media.FileMeta.Original.Width = width
-		p.media.FileMeta.Original.Height = height
-		p.media.FileMeta.Original.Size = (width * height)
-		p.media.FileMeta.Original.Aspect = float32(width) / float32(height)
-		p.media.FileMeta.Original.Framerate = &framerate
-
-		// Extract total duration from format.
-		duration := result.Format.GetDuration()
-		p.media.FileMeta.Original.Duration = &duration
-
-		// Extract total bitrate from format.
-		bitrate := result.Format.GetBitRate()
-		p.media.FileMeta.Original.Bitrate = &bitrate
-
-		// Determine thumbnail dimensions to use.
-		thumbWidth, thumbHeight := thumbSize(width, height)
-		p.media.FileMeta.Small.Width = thumbWidth
-		p.media.FileMeta.Small.Height = thumbHeight
-		p.media.FileMeta.Small.Size = (thumbWidth * thumbHeight)
-		p.media.FileMeta.Small.Aspect = float32(thumbWidth) / float32(thumbHeight)
-
-		// Extract a thumbnail frame from input video path.
-		thumbpath, err = ffmpegGenerateThumb(ctx, temppath,
-			thumbWidth,
-			thumbHeight,
-		)
-		if err != nil {
-			return gtserror.Newf("error extracting video frame: %w", err)
-		}
-
 	case gtsmodel.FileTypeAudio:
-		// Extract total duration from format.
+		// NOTE: we do not clean audio file
-		duration := result.Format.GetDuration()
+		// metadata, in order to keep tags.
-		p.media.FileMeta.Original.Duration = &duration
 
-		// Extract total bitrate from format.
+	default:
-		bitrate := result.Format.GetBitRate()
+		log.Warn(ctx, "unsupported data type: %s", result.format)
-		p.media.FileMeta.Original.Bitrate = &bitrate
+		return nil
-
-		// Extract image metadata from streams (if any),
-		// this will only exist for embedded album art.
-		width, height, framerate, _ := result.EmbeddedImageMeta()
-		if width > 0 && height > 0 {
-			// Unlikely to need these but masto API includes them.
-			p.media.FileMeta.Original.Width = width
-			p.media.FileMeta.Original.Height = height
-			if framerate != 0 {
-				p.media.FileMeta.Original.Framerate = &framerate
 	}
 
+	if width > 0 && height > 0 {
 		// Determine thumbnail dimensions to use.
 		thumbWidth, thumbHeight := thumbSize(width, height)
 		p.media.FileMeta.Small.Width = thumbWidth
@@ -323,11 +240,14 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 		if err != nil {
 			return gtserror.Newf("error generating image thumb: %w", err)
 		}
-		}
 
-	default:
+		if p.media.Blurhash == "" {
-		log.Warnf(ctx, "unsupported type: %s (%s)", p.media.Type, result.Format.FormatName)
+			// Generate blurhash (if not already) from thumbnail.
-		return nil
+			p.media.Blurhash, err = generateBlurhash(thumbpath)
+			if err != nil {
+				return gtserror.Newf("error generating thumb blurhash: %w", err)
+			}
+		}
 	}
 
 	// Calculate final media attachment file path.
@@ -352,17 +272,6 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 	p.media.File.FileSize = int(filesz)
 
 	if thumbpath != "" {
-		// Note that neither thumbnail storage
-		// nor a blurhash are needed for audio.
-
-		if p.media.Blurhash == "" {
-			// Generate blurhash (if not already) from thumbnail.
-			p.media.Blurhash, err = generateBlurhash(thumbpath)
-			if err != nil {
-				return gtserror.Newf("error generating thumb blurhash: %w", err)
-			}
-		}
-
 		// Copy thumbnail file into storage at path.
 		thumbsz, err := p.mgr.state.Storage.PutFile(ctx,
 			p.media.Thumbnail.Path,

internal/media/types.go

@@ -23,27 +23,6 @@
 	"time"
 )
 
-// mime consts
-const (
-	mimeImage = "image"
-	mimeVideo = "video"
-
-	mimeJpeg      = "jpeg"
-	mimeImageJpeg = mimeImage + "/" + mimeJpeg
-
-	mimeGif      = "gif"
-	mimeImageGif = mimeImage + "/" + mimeGif
-
-	mimePng      = "png"
-	mimeImagePng = mimeImage + "/" + mimePng
-
-	mimeWebp      = "webp"
-	mimeImageWebp = mimeImage + "/" + mimeWebp
-
-	mimeMp4      = "mp4"
-	mimeVideoMp4 = mimeVideo + "/" + mimeMp4
-)
-
 type Size string
 
 const (

internal/typeutils/internaltofrontend_test.go

@@ -1225,9 +1225,22 @@ func (suite *InternalToFrontendTestSuite) TestInstanceV1ToFrontend() {
       "supported_mime_types": [
         "image/jpeg",
         "image/gif",
-        "image/png",
         "image/webp",
-        "video/mp4"
+        "audio/mp2",
+        "audio/mp3",
+        "video/x-msvideo",
+        "image/png",
+        "image/apng",
+        "audio/ogg",
+        "video/ogg",
+        "audio/x-m4a",
+        "video/mp4",
+        "video/quicktime",
+        "audio/x-ms-wma",
+        "video/x-ms-wmv",
+        "video/webm",
+        "audio/x-matroska",
+        "video/x-matroska"
       ],
       "image_size_limit": 41943040,
       "image_matrix_limit": 16777216,
@@ -1350,9 +1363,22 @@ func (suite *InternalToFrontendTestSuite) TestInstanceV2ToFrontend() {
       "supported_mime_types": [
         "image/jpeg",
         "image/gif",
-        "image/png",
         "image/webp",
-        "video/mp4"
+        "audio/mp2",
+        "audio/mp3",
+        "video/x-msvideo",
+        "image/png",
+        "image/apng",
+        "audio/ogg",
+        "video/ogg",
+        "audio/x-m4a",
+        "video/mp4",
+        "video/quicktime",
+        "audio/x-ms-wma",
+        "video/x-ms-wmv",
+        "video/webm",
+        "audio/x-matroska",
+        "video/x-matroska"
       ],
       "image_size_limit": 41943040,
       "image_matrix_limit": 16777216,

internal/util/math.go

@@ -0,0 +1,34 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package util
+
+type Number interface {
+	~int | ~int8 | ~int16 | ~int32 | ~int64 |
+		~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 |
+		~uintptr | ~float32 | ~float64
+}
+
+// Div performs a safe division of
+// n1 and n2, checking for zero n2. In the
+// case of zero n2, zero is returned.
+func Div[N Number](n1, n2 N) N {
+	if n2 == 0 {
+		return 0
+	}
+	return n1 / n2
+}

internal/util/ptr.go

@@ -34,6 +34,15 @@ func Ptr[T any](t T) *T {
 	return &t
 }
 
+// PtrIf returns ptr value only if 't' non-zero.
+func PtrIf[T comparable](t T) *T {
+	var z T
+	if t == z {
+		return nil
+	}
+	return &t
+}
+
 // PtrValueOr returns either value of ptr, or default.
 func PtrValueOr[T any](t *T, _default T) T {
 	if t != nil {