Compare commits

..

4 commits

Author SHA1 Message Date
CDN 8d05895119
Merge d8041be23f into f3b2eca8b8 2024-11-04 14:17:44 +00:00
kim f3b2eca8b8
[feature] add support for hinting via api/v_/instance preferred image / video max sizes (#3505)
* add support for hinting via api/v_/instance endpoints a preferred image / video size limit

* fix tests expecting old default values
2024-11-04 15:00:10 +01:00
kim 8f288f1689
[bugfix] determine mime-type to use during ffprobe evaluation stage, don't bother checking against file extension (#3506)
* determine mime-type to use during ffprobe evaluation stage, don't bother rechecking by file extension

* set mjpeg content-type

* fix up tests expecting differing default values
2024-11-04 14:58:15 +01:00
dependabot[bot] d2820a1470
[chore]: Bump github.com/minio/minio-go/v7 from 7.0.79 to 7.0.80 (#3511)
Bumps [github.com/minio/minio-go/v7](https://github.com/minio/minio-go) from 7.0.79 to 7.0.80.
- [Release notes](https://github.com/minio/minio-go/releases)
- [Commits](https://github.com/minio/minio-go/compare/v7.0.79...v7.0.80)

---
updated-dependencies:
- dependency-name: github.com/minio/minio-go/v7
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2024-11-04 12:03:54 +01:00
24 changed files with 290 additions and 1430 deletions

View file

@ -18,6 +18,24 @@
# Default: 40MiB (41943040 bytes)
media-local-max-size: 40MiB
# Size. Size in bytes of max image size referred to on /api/v_/instance endpoints,
# used by applications like Tusky to automatically scale locally uploaded media.
#
# Leaving this unset will default to media-local-max-size.
#
# Examples: [64, 500, 5MiB, 5MB, 50M]
# Default: unset
media-image-size-hint: 5MiB
# Size. Size in bytes of max video size referred to on /api/v_/instance endpoints,
# used by applications like Tusky to automatically scale locally uploaded media.
#
# Leaving this unset will default to media-local-max-size.
#
# Examples: [64, 4096, 4MiB, 4MB, 40M]
# Default: unset
media-video-size-hint: 40MiB
# Size. Max size in bytes of media to download from other instances.
#
# Lowering this limit may cause your instance not to fetch post media.

View file

@ -471,6 +471,24 @@ accounts-custom-css-length: 10000
# Default: 40MiB (41943040 bytes)
media-local-max-size: 40MiB
# Size. Size in bytes of max image size referred to on /api/v_/instance endpoints,
# used by applications like Tusky to automatically scale locally uploaded media.
#
# Leaving this unset will default to media-local-max-size.
#
# Examples: [64, 500, 5MiB, 5MB, 50M]
# Default: unset
media-image-size-hint: 5MiB
# Size. Size in bytes of max video size referred to on /api/v_/instance endpoints,
# used by applications like Tusky to automatically scale locally uploaded media.
#
# Leaving this unset will default to media-local-max-size.
#
# Examples: [64, 4096, 4MiB, 4MB, 40M]
# Default: unset
media-video-size-hint: 40MiB
# Size. Max size in bytes of media to download from other instances.
#
# Lowering this limit may cause your instance not to fetch post media.

3
go.mod
View file

@ -17,7 +17,6 @@ require (
codeberg.org/gruf/go-kv v1.6.5
codeberg.org/gruf/go-list v0.0.0-20240425093752-494db03d641f
codeberg.org/gruf/go-mempool v0.0.0-20240507125005-cef10d64a760
codeberg.org/gruf/go-mimetypes v1.2.0
codeberg.org/gruf/go-mutexes v1.5.1
codeberg.org/gruf/go-runners v1.6.3
codeberg.org/gruf/go-sched v1.2.4
@ -42,7 +41,7 @@ require (
github.com/k3a/html2text v1.2.1
github.com/microcosm-cc/bluemonday v1.0.27
github.com/miekg/dns v1.1.62
github.com/minio/minio-go/v7 v7.0.79
github.com/minio/minio-go/v7 v7.0.80
github.com/mitchellh/mapstructure v1.5.0
github.com/ncruces/go-sqlite3 v0.20.0
github.com/oklog/ulid v1.3.1

6
go.sum generated
View file

@ -62,8 +62,6 @@ codeberg.org/gruf/go-maps v1.0.4 h1:K+Ww4vvR3TZqm5jqrKVirmguZwa3v1VUvmig2SE8uxY=
codeberg.org/gruf/go-maps v1.0.4/go.mod h1:ASX7osM7kFwt5O8GfGflcFjrwYGD8eIuRLl/oMjhEi8=
codeberg.org/gruf/go-mempool v0.0.0-20240507125005-cef10d64a760 h1:m2/UCRXhjDwAg4vyji6iKCpomKw6P4PmBOUi5DvAMH4=
codeberg.org/gruf/go-mempool v0.0.0-20240507125005-cef10d64a760/go.mod h1:E3RcaCFNq4zXpvaJb8lfpPqdUAmSkP5F1VmMiEUYTEk=
codeberg.org/gruf/go-mimetypes v1.2.0 h1:3rZGXY/SkNYbamiddWXs2gETXIBkGIeWYnbWpp2OEbc=
codeberg.org/gruf/go-mimetypes v1.2.0/go.mod h1:YiUWRj/nAdJQc+UFRvcsL6xXZsbc6b6Ic739ycEO8Yg=
codeberg.org/gruf/go-mutexes v1.5.1 h1:xICU0WXhWr6wf+Iror4eE3xT+xnXNPrO6o77D/G6QuY=
codeberg.org/gruf/go-mutexes v1.5.1/go.mod h1:rPEqQ/y6CmGITaZ3GPTMQVsoZAOzbsAHyIaLsJcOqVE=
codeberg.org/gruf/go-runners v1.6.3 h1:To/AX7eTrWuXrTkA3RA01YTP5zha1VZ68LQ+0D4RY7E=
@ -413,8 +411,8 @@ github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ=
github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.79 h1:SvJZpj3hT0RN+4KiuX/FxLfPZdsuegy6d/2PiemM/bM=
github.com/minio/minio-go/v7 v7.0.79/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
github.com/minio/minio-go/v7 v7.0.80 h1:2mdUHXEykRdY/BigLt3Iuu1otL0JTogT0Nmltg0wujk=
github.com/minio/minio-go/v7 v7.0.80/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=

View file

@ -120,7 +120,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch1() {
"image/apng",
"audio/ogg",
"video/ogg",
"audio/x-m4a",
"audio/mp4",
"video/mp4",
"video/quicktime",
"audio/x-ms-wma",
@ -130,10 +130,10 @@ func (suite *InstancePatchTestSuite) TestInstancePatch1() {
"video/x-matroska"
],
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
"image_matrix_limit": 9223372036854775807,
"video_size_limit": 41943040,
"video_frame_rate_limit": 60,
"video_matrix_limit": 16777216
"video_frame_rate_limit": 9223372036854775807,
"video_matrix_limit": 9223372036854775807
},
"polls": {
"max_options": 6,
@ -261,7 +261,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch2() {
"image/apng",
"audio/ogg",
"video/ogg",
"audio/x-m4a",
"audio/mp4",
"video/mp4",
"video/quicktime",
"audio/x-ms-wma",
@ -271,10 +271,10 @@ func (suite *InstancePatchTestSuite) TestInstancePatch2() {
"video/x-matroska"
],
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
"image_matrix_limit": 9223372036854775807,
"video_size_limit": 41943040,
"video_frame_rate_limit": 60,
"video_matrix_limit": 16777216
"video_frame_rate_limit": 9223372036854775807,
"video_matrix_limit": 9223372036854775807
},
"polls": {
"max_options": 6,
@ -402,7 +402,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch3() {
"image/apng",
"audio/ogg",
"video/ogg",
"audio/x-m4a",
"audio/mp4",
"video/mp4",
"video/quicktime",
"audio/x-ms-wma",
@ -412,10 +412,10 @@ func (suite *InstancePatchTestSuite) TestInstancePatch3() {
"video/x-matroska"
],
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
"image_matrix_limit": 9223372036854775807,
"video_size_limit": 41943040,
"video_frame_rate_limit": 60,
"video_matrix_limit": 16777216
"video_frame_rate_limit": 9223372036854775807,
"video_matrix_limit": 9223372036854775807
},
"polls": {
"max_options": 6,
@ -594,7 +594,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch6() {
"image/apng",
"audio/ogg",
"video/ogg",
"audio/x-m4a",
"audio/mp4",
"video/mp4",
"video/quicktime",
"audio/x-ms-wma",
@ -604,10 +604,10 @@ func (suite *InstancePatchTestSuite) TestInstancePatch6() {
"video/x-matroska"
],
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
"image_matrix_limit": 9223372036854775807,
"video_size_limit": 41943040,
"video_frame_rate_limit": 60,
"video_matrix_limit": 16777216
"video_frame_rate_limit": 9223372036854775807,
"video_matrix_limit": 9223372036854775807
},
"polls": {
"max_options": 6,
@ -757,7 +757,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch8() {
"image/apng",
"audio/ogg",
"video/ogg",
"audio/x-m4a",
"audio/mp4",
"video/mp4",
"video/quicktime",
"audio/x-ms-wma",
@ -767,10 +767,10 @@ func (suite *InstancePatchTestSuite) TestInstancePatch8() {
"video/x-matroska"
],
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
"image_matrix_limit": 9223372036854775807,
"video_size_limit": 41943040,
"video_frame_rate_limit": 60,
"video_matrix_limit": 16777216
"video_frame_rate_limit": 9223372036854775807,
"video_matrix_limit": 9223372036854775807
},
"polls": {
"max_options": 6,
@ -939,7 +939,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch9() {
"image/apng",
"audio/ogg",
"video/ogg",
"audio/x-m4a",
"audio/mp4",
"video/mp4",
"video/quicktime",
"audio/x-ms-wma",
@ -949,10 +949,10 @@ func (suite *InstancePatchTestSuite) TestInstancePatch9() {
"video/x-matroska"
],
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
"image_matrix_limit": 9223372036854775807,
"video_size_limit": 41943040,
"video_frame_rate_limit": 60,
"video_matrix_limit": 16777216
"video_frame_rate_limit": 9223372036854775807,
"video_matrix_limit": 9223372036854775807
},
"polls": {
"max_options": 6,

View file

@ -98,6 +98,8 @@ type Configuration struct {
MediaRemoteCacheDays int `name:"media-remote-cache-days" usage:"Number of days to locally cache media from remote instances. If set to 0, remote media will be kept indefinitely."`
MediaEmojiLocalMaxSize bytesize.Size `name:"media-emoji-local-max-size" usage:"Max size in bytes of emojis uploaded to this instance via the admin API."`
MediaEmojiRemoteMaxSize bytesize.Size `name:"media-emoji-remote-max-size" usage:"Max size in bytes of emojis to download from other instances."`
MediaImageSizeHint bytesize.Size `name:"media-image-size-hint" usage:"Size in bytes of max image size referred to on /api/v_/instance endpoints (else, local max size)"`
MediaVideoSizeHint bytesize.Size `name:"media-video-size-hint" usage:"Size in bytes of max video size referred to on /api/v_/instance endpoints (else, local max size)"`
MediaLocalMaxSize bytesize.Size `name:"media-local-max-size" usage:"Max size in bytes of media uploaded to this instance via API"`
MediaRemoteMaxSize bytesize.Size `name:"media-remote-max-size" usage:"Max size in bytes of media to download from other instances"`
MediaCleanupFrom string `name:"media-cleanup-from" usage:"Time of day from which to start running media cleanup/prune jobs. Should be in the format 'hh:mm:ss', eg., '15:04:05'."`

View file

@ -1225,6 +1225,56 @@ func GetMediaEmojiRemoteMaxSize() bytesize.Size { return global.GetMediaEmojiRem
// SetMediaEmojiRemoteMaxSize safely sets the value for global configuration 'MediaEmojiRemoteMaxSize' field
func SetMediaEmojiRemoteMaxSize(v bytesize.Size) { global.SetMediaEmojiRemoteMaxSize(v) }
// GetMediaImageSizeHint safely fetches the Configuration value for state's 'MediaImageSizeHint' field
func (st *ConfigState) GetMediaImageSizeHint() (v bytesize.Size) {
st.mutex.RLock()
v = st.config.MediaImageSizeHint
st.mutex.RUnlock()
return
}
// SetMediaImageSizeHint safely sets the Configuration value for state's 'MediaImageSizeHint' field
func (st *ConfigState) SetMediaImageSizeHint(v bytesize.Size) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.MediaImageSizeHint = v
st.reloadToViper()
}
// MediaImageSizeHintFlag returns the flag name for the 'MediaImageSizeHint' field
func MediaImageSizeHintFlag() string { return "media-image-size-hint" }
// GetMediaImageSizeHint safely fetches the value for global configuration 'MediaImageSizeHint' field
func GetMediaImageSizeHint() bytesize.Size { return global.GetMediaImageSizeHint() }
// SetMediaImageSizeHint safely sets the value for global configuration 'MediaImageSizeHint' field
func SetMediaImageSizeHint(v bytesize.Size) { global.SetMediaImageSizeHint(v) }
// GetMediaVideoSizeHint safely fetches the Configuration value for state's 'MediaVideoSizeHint' field
func (st *ConfigState) GetMediaVideoSizeHint() (v bytesize.Size) {
st.mutex.RLock()
v = st.config.MediaVideoSizeHint
st.mutex.RUnlock()
return
}
// SetMediaVideoSizeHint safely sets the Configuration value for state's 'MediaVideoSizeHint' field
func (st *ConfigState) SetMediaVideoSizeHint(v bytesize.Size) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.MediaVideoSizeHint = v
st.reloadToViper()
}
// MediaVideoSizeHintFlag returns the flag name for the 'MediaVideoSizeHint' field
func MediaVideoSizeHintFlag() string { return "media-video-size-hint" }
// GetMediaVideoSizeHint safely fetches the value for global configuration 'MediaVideoSizeHint' field
func GetMediaVideoSizeHint() bytesize.Size { return global.GetMediaVideoSizeHint() }
// SetMediaVideoSizeHint safely sets the value for global configuration 'MediaVideoSizeHint' field
func SetMediaVideoSizeHint(v bytesize.Size) { global.SetMediaVideoSizeHint(v) }
// GetMediaLocalMaxSize safely fetches the Configuration value for state's 'MediaLocalMaxSize' field
func (st *ConfigState) GetMediaLocalMaxSize() (v bytesize.Size) {
st.mutex.RLock()

View file

@ -323,14 +323,14 @@ type videoStream struct {
//
// Note the checks for (len(res.video) > 0) may catch some audio files with embedded
// album art as video, but i blame that on the hellscape that is media filetypes.
//
// TODO: we can update this code to also return a mimetype and avoid later parsing!
func (res *result) GetFileType() (gtsmodel.FileType, string) {
func (res *result) GetFileType() (gtsmodel.FileType, string, string) {
switch res.format {
case "mpeg":
return gtsmodel.FileTypeVideo, "mpeg"
return gtsmodel.FileTypeVideo,
"video/mpeg", "mpeg"
case "mjpeg":
return gtsmodel.FileTypeVideo, "mjpeg"
return gtsmodel.FileTypeVideo,
"video/x-motion-jpeg", "mjpeg"
case "mov,mp4,m4a,3gp,3g2,mj2":
switch {
case len(res.video) > 0:
@ -338,55 +338,70 @@ func (res *result) GetFileType() (gtsmodel.FileType, string) {
res.duration <= 30 {
// Short, soundless
// video file aka gifv.
return gtsmodel.FileTypeGifv, "mp4"
return gtsmodel.FileTypeGifv,
"video/mp4", "mp4"
} else {
// Video file (with or without audio).
return gtsmodel.FileTypeVideo, "mp4"
return gtsmodel.FileTypeVideo,
"video/mp4", "mp4"
}
case len(res.audio) > 0 &&
res.audio[0].codec == "aac":
// m4a only supports [aac] audio.
return gtsmodel.FileTypeAudio, "m4a"
return gtsmodel.FileTypeAudio,
"audio/mp4", "m4a"
}
case "apng":
return gtsmodel.FileTypeImage, "apng"
return gtsmodel.FileTypeImage,
"image/apng", "apng"
case "png_pipe":
return gtsmodel.FileTypeImage, "png"
return gtsmodel.FileTypeImage,
"image/png", "png"
case "image2", "image2pipe", "jpeg_pipe":
return gtsmodel.FileTypeImage, "jpeg"
return gtsmodel.FileTypeImage,
"image/jpeg", "jpeg"
case "webp", "webp_pipe":
return gtsmodel.FileTypeImage, "webp"
return gtsmodel.FileTypeImage,
"image/webp", "webp"
case "gif":
return gtsmodel.FileTypeImage, "gif"
return gtsmodel.FileTypeImage,
"image/gif", "gif"
case "mp3":
if len(res.audio) > 0 {
switch res.audio[0].codec {
case "mp2":
return gtsmodel.FileTypeAudio, "mp2"
return gtsmodel.FileTypeAudio,
"audio/mp2", "mp2"
case "mp3":
return gtsmodel.FileTypeAudio, "mp3"
return gtsmodel.FileTypeAudio,
"audio/mp3", "mp3"
}
}
case "asf":
switch {
case len(res.video) > 0:
return gtsmodel.FileTypeVideo, "wmv"
return gtsmodel.FileTypeVideo,
"video/x-ms-wmv", "wmv"
case len(res.audio) > 0:
return gtsmodel.FileTypeAudio, "wma"
return gtsmodel.FileTypeAudio,
"audio/x-ms-wma", "wma"
}
case "ogg":
if len(res.video) > 0 {
switch res.video[0].codec {
case "theora", "dirac": // daala, tarkin
return gtsmodel.FileTypeVideo, "ogv"
return gtsmodel.FileTypeVideo,
"video/ogg", "ogv"
}
}
if len(res.audio) > 0 {
switch res.audio[0].codec {
case "opus", "libopus":
return gtsmodel.FileTypeAudio, "opus"
return gtsmodel.FileTypeAudio,
"audio/opus", "opus"
default:
return gtsmodel.FileTypeAudio, "ogg"
return gtsmodel.FileTypeAudio,
"audio/ogg", "ogg"
}
}
case "matroska,webm":
@ -411,21 +426,27 @@ func (res *result) GetFileType() (gtsmodel.FileType, string) {
}
if isWebm {
// Check for valid webm codec config.
return gtsmodel.FileTypeVideo, "webm"
// Check valid webm codec config.
return gtsmodel.FileTypeVideo,
"video/webm", "webm"
}
// All else falls under generic mkv.
return gtsmodel.FileTypeVideo, "mkv"
return gtsmodel.FileTypeVideo,
"video/x-matroska", "mkv"
case len(res.audio) > 0:
return gtsmodel.FileTypeAudio, "mka"
return gtsmodel.FileTypeAudio,
"audio/x-matroska", "mka"
}
case "avi":
return gtsmodel.FileTypeVideo, "avi"
return gtsmodel.FileTypeVideo,
"video/x-msvideo", "avi"
case "flac":
return gtsmodel.FileTypeAudio, "flac"
return gtsmodel.FileTypeAudio,
"audio/flac", "flac"
}
return gtsmodel.FileTypeUnknown, res.format
return gtsmodel.FileTypeUnknown,
"", res.format
}
// ImageMeta extracts image metadata contained within ffprobe'd media result streams.

View file

@ -56,7 +56,7 @@
"video/ogg", // .ogv
// mpeg4 types
"audio/x-m4a", // .m4a
"audio/mp4", // .m4a
"video/mp4", // .mp4
"video/quicktime", // .mov

View file

@ -664,7 +664,7 @@ func (suite *ManagerTestSuite) TestOpusProcess() {
Duration: util.Ptr(float32(122.10006)),
Bitrate: util.Ptr(uint64(116426)),
}, attachment.FileMeta.Original)
suite.Equal("audio/ogg", attachment.File.ContentType)
suite.Equal("audio/opus", attachment.File.ContentType)
suite.Equal(1776956, attachment.File.FileSize)
suite.Empty(attachment.Blurhash)

View file

@ -163,9 +163,10 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
}
var ext string
var fileType gtsmodel.FileType
// Get type from ffprobe format data.
fileType, ext := result.GetFileType()
// Get abstract file type, mimetype and ext from ffprobe data.
fileType, p.emoji.ImageContentType, ext = result.GetFileType()
if fileType != gtsmodel.FileTypeImage {
return gtserror.Newf("unsupported emoji filetype: %s (%s)", fileType, ext)
}
@ -216,10 +217,6 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
"png",
)
// Get mimetype for the file container
// type, falling back to generic data.
p.emoji.ImageContentType = getMimeType(ext)
// Set the known emoji static content type.
p.emoji.ImageStaticContentType = "image/png"

View file

@ -186,8 +186,8 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
p.media.FileMeta.Original.Duration = util.PtrIf(float32(result.duration))
p.media.FileMeta.Original.Bitrate = util.PtrIf(result.bitrate)
// Set media type from ffprobe format data.
p.media.Type, ext = result.GetFileType()
// Set generic media type and mimetype from ffprobe format data.
p.media.Type, p.media.File.ContentType, ext = result.GetFileType()
// Add file extension to path.
newpath := temppath + "." + ext
@ -236,10 +236,10 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
// Determine if blurhash needs generating.
needBlurhash := (p.media.Blurhash == "")
var newBlurhash string
var newBlurhash, mimeType string
// Generate thumbnail, and new blurhash if need from media.
thumbpath, newBlurhash, err = generateThumb(ctx, temppath,
// Generate thumbnail, and new blurhash if needed from temp media.
thumbpath, mimeType, newBlurhash, err = generateThumb(ctx, temppath,
thumbWidth,
thumbHeight,
result.orientation,
@ -250,6 +250,9 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
return gtserror.Newf("error generating image thumb: %w", err)
}
// Set generated thumbnail's mimetype.
p.media.Thumbnail.ContentType = mimeType
if needBlurhash {
// Set newly determined blurhash.
p.media.Blurhash = newBlurhash
@ -265,10 +268,6 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
ext,
)
// Get mimetype for the file container
// type, falling back to generic data.
p.media.File.ContentType = getMimeType(ext)
// Copy temporary file into storage at path.
filesz, err := p.mgr.state.Storage.PutFile(ctx,
p.media.File.Path,
@ -295,9 +294,6 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
thumbExt,
)
// Determine thumbnail content-type from thumb ext.
p.media.Thumbnail.ContentType = getMimeType(thumbExt)
// Copy thumbnail file into storage at path.
thumbsz, err := p.mgr.state.Storage.PutFile(ctx,
p.media.Thumbnail.Path,

View file

@ -84,17 +84,21 @@ func generateThumb(
needBlurhash bool,
) (
outpath string,
mimeType string,
blurhash string,
err error,
) {
var ext string
// Default type is webp.
mimeType = "image/webp"
// Generate thumb output path REPLACING extension.
if i := strings.IndexByte(filepath, '.'); i != -1 {
outpath = filepath[:i] + "_thumb.webp"
ext = filepath[i+1:] // old extension
} else {
return "", "", gtserror.New("input file missing extension")
return "", "", "", gtserror.New("input file missing extension")
}
// Check for the few media types we
@ -106,6 +110,7 @@ func generateThumb(
// Replace the "webp" with "jpeg", as we'll
// use our native Go thumbnailing generation.
outpath = outpath[:len(outpath)-4] + "jpeg"
mimeType = "image/jpeg"
log.Debug(ctx, "generating thumb from jpeg")
blurhash, err := generateNativeThumb(
@ -117,7 +122,7 @@ func generateThumb(
jpeg.Decode,
needBlurhash,
)
return outpath, blurhash, err
return outpath, mimeType, blurhash, err
// We specifically only allow generating native
// thumbnails from gif IF it doesn't contain an
@ -128,6 +133,7 @@ func generateThumb(
// Replace the "webp" with "jpeg", as we'll
// use our native Go thumbnailing generation.
outpath = outpath[:len(outpath)-4] + "jpeg"
mimeType = "image/jpeg"
log.Debug(ctx, "generating thumb from gif")
blurhash, err := generateNativeThumb(
@ -139,7 +145,7 @@ func generateThumb(
gif.Decode,
needBlurhash,
)
return outpath, blurhash, err
return outpath, mimeType, blurhash, err
// We specifically only allow generating native
// thumbnails from png IF it doesn't contain an
@ -150,6 +156,7 @@ func generateThumb(
// Replace the "webp" with "jpeg", as we'll
// use our native Go thumbnailing generation.
outpath = outpath[:len(outpath)-4] + "jpeg"
mimeType = "image/jpeg"
log.Debug(ctx, "generating thumb from png")
blurhash, err := generateNativeThumb(
@ -161,7 +168,7 @@ func generateThumb(
png.Decode,
needBlurhash,
)
return outpath, blurhash, err
return outpath, mimeType, blurhash, err
// We specifically only allow generating native
// thumbnails from webp IF it doesn't contain an
@ -172,6 +179,7 @@ func generateThumb(
// Replace the "webp" with "jpeg", as we'll
// use our native Go thumbnailing generation.
outpath = outpath[:len(outpath)-4] + "jpeg"
mimeType = "image/jpeg"
log.Debug(ctx, "generating thumb from webp")
blurhash, err := generateNativeThumb(
@ -183,7 +191,7 @@ func generateThumb(
webp.Decode,
needBlurhash,
)
return outpath, blurhash, err
return outpath, mimeType, blurhash, err
}
// The fallback for thumbnail generation, which
@ -196,18 +204,18 @@ func generateThumb(
height,
pixfmt,
); err != nil {
return outpath, "", err
return outpath, "", "", err
}
if needBlurhash {
// Generate new blurhash from webp output thumb.
blurhash, err = generateWebpBlurhash(outpath)
if err != nil {
return outpath, "", gtserror.Newf("error generating blurhash: %w", err)
return outpath, "", "", gtserror.Newf("error generating blurhash: %w", err)
}
}
return outpath, blurhash, err
return outpath, mimeType, blurhash, nil
}
// generateNativeThumb generates a thumbnail

View file

@ -18,7 +18,6 @@
package media
import (
"cmp"
"errors"
"fmt"
"io"
@ -28,7 +27,6 @@
"codeberg.org/gruf/go-bytesize"
"codeberg.org/gruf/go-iotools"
"codeberg.org/gruf/go-mimetypes"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
)
@ -87,12 +85,6 @@ func getExtension(path string) string {
return ""
}
// getMimeType returns a suitable mimetype for file extension.
func getMimeType(ext string) string {
const defaultType = "application/octet-stream"
return cmp.Or(mimetypes.MimeTypes[ext], defaultType)
}
// drainToTmp drains data from given reader into a new temp file
// and closes it, returning the path of the resulting temp file.
//

View file

@ -18,9 +18,11 @@
package typeutils
import (
"cmp"
"context"
"errors"
"fmt"
"math"
"slices"
"strconv"
"strings"
@ -42,16 +44,13 @@
)
const (
instanceStatusesCharactersReservedPerURL = 25
instanceMediaAttachmentsImageMatrixLimit = 16777216 // width * height
instanceMediaAttachmentsVideoMatrixLimit = 16777216 // width * height
instanceMediaAttachmentsVideoFrameRateLimit = 60
instancePollsMinExpiration = 300 // seconds
instancePollsMaxExpiration = 2629746 // seconds
instanceAccountsMaxFeaturedTags = 10
instanceAccountsMaxProfileFields = 6 // FIXME: https://github.com/superseriousbusiness/gotosocial/issues/1876
instanceSourceURL = "https://github.com/superseriousbusiness/gotosocial"
instanceMastodonVersion = "3.5.3"
instanceStatusesCharactersReservedPerURL = 25
instancePollsMinExpiration = 300 // seconds
instancePollsMaxExpiration = 2629746 // seconds
instanceAccountsMaxFeaturedTags = 10
instanceAccountsMaxProfileFields = 6 // FIXME: https://github.com/superseriousbusiness/gotosocial/issues/1876
instanceSourceURL = "https://github.com/superseriousbusiness/gotosocial"
instanceMastodonVersion = "3.5.3"
)
var instanceStatusesSupportedMimeTypes = []string{
@ -1563,11 +1562,24 @@ func (c *Converter) InstanceToAPIV1Instance(ctx context.Context, i *gtsmodel.Ins
instance.Configuration.Statuses.CharactersReservedPerURL = instanceStatusesCharactersReservedPerURL
instance.Configuration.Statuses.SupportedMimeTypes = instanceStatusesSupportedMimeTypes
instance.Configuration.MediaAttachments.SupportedMimeTypes = media.SupportedMIMETypes
instance.Configuration.MediaAttachments.ImageSizeLimit = int(config.GetMediaRemoteMaxSize()) // #nosec G115 -- Already validated.
instance.Configuration.MediaAttachments.ImageMatrixLimit = instanceMediaAttachmentsImageMatrixLimit
instance.Configuration.MediaAttachments.VideoSizeLimit = int(config.GetMediaRemoteMaxSize()) // #nosec G115 -- Already validated.
instance.Configuration.MediaAttachments.VideoFrameRateLimit = instanceMediaAttachmentsVideoFrameRateLimit
instance.Configuration.MediaAttachments.VideoMatrixLimit = instanceMediaAttachmentsVideoMatrixLimit
// NOTE: we use the local max sizes here
// as it hints to apps like Tusky for image
// compression of locally uploaded media.
//
// TODO: return local / remote depending
// on authorized endpoint user (if any)?
localMax := config.GetMediaLocalMaxSize()
imageSz := cmp.Or(config.GetMediaImageSizeHint(), localMax)
videoSz := cmp.Or(config.GetMediaVideoSizeHint(), localMax)
instance.Configuration.MediaAttachments.ImageSizeLimit = int(imageSz) // #nosec G115 -- Already validated.
instance.Configuration.MediaAttachments.VideoSizeLimit = int(videoSz) // #nosec G115 -- Already validated.
// we don't actually set any limits on these. set to max possible.
instance.Configuration.MediaAttachments.ImageMatrixLimit = math.MaxInt
instance.Configuration.MediaAttachments.VideoFrameRateLimit = math.MaxInt
instance.Configuration.MediaAttachments.VideoMatrixLimit = math.MaxInt
instance.Configuration.Polls.MaxOptions = config.GetStatusesPollMaxOptions()
instance.Configuration.Polls.MaxCharactersPerOption = config.GetStatusesPollOptionMaxChars()
instance.Configuration.Polls.MinExpiration = instancePollsMinExpiration
@ -1713,11 +1725,24 @@ func (c *Converter) InstanceToAPIV2Instance(ctx context.Context, i *gtsmodel.Ins
instance.Configuration.Statuses.CharactersReservedPerURL = instanceStatusesCharactersReservedPerURL
instance.Configuration.Statuses.SupportedMimeTypes = instanceStatusesSupportedMimeTypes
instance.Configuration.MediaAttachments.SupportedMimeTypes = media.SupportedMIMETypes
instance.Configuration.MediaAttachments.ImageSizeLimit = int(config.GetMediaRemoteMaxSize()) // #nosec G115 -- Already validated.
instance.Configuration.MediaAttachments.ImageMatrixLimit = instanceMediaAttachmentsImageMatrixLimit
instance.Configuration.MediaAttachments.VideoSizeLimit = int(config.GetMediaRemoteMaxSize()) // #nosec G115 -- Already validated.
instance.Configuration.MediaAttachments.VideoFrameRateLimit = instanceMediaAttachmentsVideoFrameRateLimit
instance.Configuration.MediaAttachments.VideoMatrixLimit = instanceMediaAttachmentsVideoMatrixLimit
// NOTE: we use the local max sizes here
// as it hints to apps like Tusky for image
// compression of locally uploaded media.
//
// TODO: return local / remote depending
// on authorized endpoint user (if any)?
localMax := config.GetMediaLocalMaxSize()
imageSz := cmp.Or(config.GetMediaImageSizeHint(), localMax)
videoSz := cmp.Or(config.GetMediaVideoSizeHint(), localMax)
instance.Configuration.MediaAttachments.ImageSizeLimit = int(imageSz) // #nosec G115 -- Already validated.
instance.Configuration.MediaAttachments.VideoSizeLimit = int(videoSz) // #nosec G115 -- Already validated.
// we don't actually set any limits on these. set to max possible.
instance.Configuration.MediaAttachments.ImageMatrixLimit = math.MaxInt
instance.Configuration.MediaAttachments.VideoFrameRateLimit = math.MaxInt
instance.Configuration.MediaAttachments.VideoMatrixLimit = math.MaxInt
instance.Configuration.Polls.MaxOptions = config.GetStatusesPollMaxOptions()
instance.Configuration.Polls.MaxCharactersPerOption = config.GetStatusesPollOptionMaxChars()
instance.Configuration.Polls.MinExpiration = instancePollsMinExpiration

View file

@ -1958,7 +1958,7 @@ func (suite *InternalToFrontendTestSuite) TestInstanceV1ToFrontend() {
"image/apng",
"audio/ogg",
"video/ogg",
"audio/x-m4a",
"audio/mp4",
"video/mp4",
"video/quicktime",
"audio/x-ms-wma",
@ -1968,10 +1968,10 @@ func (suite *InternalToFrontendTestSuite) TestInstanceV1ToFrontend() {
"video/x-matroska"
],
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
"image_matrix_limit": 9223372036854775807,
"video_size_limit": 41943040,
"video_frame_rate_limit": 60,
"video_matrix_limit": 16777216
"video_frame_rate_limit": 9223372036854775807,
"video_matrix_limit": 9223372036854775807
},
"polls": {
"max_options": 6,
@ -2103,7 +2103,7 @@ func (suite *InternalToFrontendTestSuite) TestInstanceV2ToFrontend() {
"image/apng",
"audio/ogg",
"video/ogg",
"audio/x-m4a",
"audio/mp4",
"video/mp4",
"video/quicktime",
"audio/x-ms-wma",
@ -2113,10 +2113,10 @@ func (suite *InternalToFrontendTestSuite) TestInstanceV2ToFrontend() {
"video/x-matroska"
],
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
"image_matrix_limit": 9223372036854775807,
"video_size_limit": 41943040,
"video_frame_rate_limit": 60,
"video_matrix_limit": 16777216
"video_frame_rate_limit": 9223372036854775807,
"video_matrix_limit": 9223372036854775807
},
"polls": {
"max_options": 6,

View file

@ -129,9 +129,11 @@ EXPECT=$(cat << "EOF"
"media-emoji-local-max-size": 420,
"media-emoji-remote-max-size": 420,
"media-ffmpeg-pool-size": 8,
"media-image-size-hint": 5242880,
"media-local-max-size": 420,
"media-remote-cache-days": 30,
"media-remote-max-size": 420,
"media-video-size-hint": 41943040,
"metrics-auth-enabled": false,
"metrics-auth-password": "",
"metrics-auth-username": "",
@ -244,12 +246,14 @@ GTS_ACCOUNTS_REGISTRATION_OPEN=true \
GTS_ACCOUNTS_REASON_REQUIRED=false \
GTS_MEDIA_DESCRIPTION_MIN_CHARS=69 \
GTS_MEDIA_DESCRIPTION_MAX_CHARS=5000 \
GTS_MEDIA_IMAGE_SIZE_HINT='5MiB' \
GTS_MEDIA_LOCAL_MAX_SIZE=420 \
GTS_MEDIA_REMOTE_MAX_SIZE=420 \
GTS_MEDIA_REMOTE_CACHE_DAYS=30 \
GTS_MEDIA_EMOJI_LOCAL_MAX_SIZE=420 \
GTS_MEDIA_EMOJI_REMOTE_MAX_SIZE=420 \
GTS_MEDIA_FFMPEG_POOL_SIZE=8 \
GTS_MEDIA_VIDEO_SIZE_HINT='40MiB' \
GTS_METRICS_AUTH_ENABLED=false \
GTS_METRICS_ENABLED=false \
GTS_STORAGE_BACKEND='local' \

View file

@ -1,5 +0,0 @@
# go-mimetypes
A generated lookup map of file extensions to mimetypes, from data provided at: https://raw.githubusercontent.com/micnic/mime.json/master/index.json
This allows determining mimetype without relying on OS mimetype lookups.

View file

@ -1,42 +0,0 @@
#!/bin/sh
# Mime types JSON source
URL='https://raw.githubusercontent.com/micnic/mime.json/master/index.json'
# Define intro to file
FILE='
// This is an automatically generated file, do not edit
package mimetypes
// MimeTypes is a map of file extensions to mime types.
var MimeTypes = map[string]string{
'
# Set break on new-line
IFS='
'
for line in $(curl -fL "$URL" | grep -E '".+"\s*:\s*".+"'); do
# Trim final whitespace
line=$(echo "$line" | sed -e 's|\s*$||')
# Ensure it ends in a comma
[ "${line%,}" = "$line" ] && line="${line},"
# Add to file
FILE="${FILE}${line}
"
done
# Add final statement to file
FILE="${FILE}
}
"
# Write to file
echo "$FILE" > 'mime.gen.go'
# Check for valid go
gofumpt -w 'mime.gen.go'

File diff suppressed because it is too large Load diff

View file

@ -1,47 +0,0 @@
package mimetypes
import "path"
// PreferredExts defines preferred file
// extensions for input mime types (as there
// can be multiple extensions per mime type).
var PreferredExts = map[string]string{
MimeTypes["mp3"]: "mp3", // audio/mpeg
MimeTypes["mpeg"]: "mpeg", // video/mpeg
}
// GetForFilename returns mimetype for given filename.
func GetForFilename(filename string) (string, bool) {
ext := path.Ext(filename)
if len(ext) < 1 {
return "", false
}
mime, ok := MimeTypes[ext[1:]]
return mime, ok
}
// GetFileExt returns the file extension to use for mimetype. Relying first upon
// the 'PreferredExts' map. It simply returns the first match there may multiple.
func GetFileExt(mimeType string) (string, bool) {
ext, ok := PreferredExts[mimeType]
if ok {
return ext, true
}
for ext, mime := range MimeTypes {
if mime == mimeType {
return ext, true
}
}
return "", false
}
// GetFileExts returns known file extensions used for mimetype.
func GetFileExts(mimeType string) []string {
var exts []string
for ext, mime := range MimeTypes {
if mime == mimeType {
exts = append(exts, ext)
}
}
return exts
}

View file

@ -99,6 +99,7 @@ type Client struct {
healthStatus int32
trailingHeaderSupport bool
maxRetries int
}
// Options for New method
@ -123,12 +124,16 @@ type Options struct {
// Custom hash routines. Leave nil to use standard.
CustomMD5 func() md5simd.Hasher
CustomSHA256 func() md5simd.Hasher
// Number of times a request is retried. Defaults to 10 retries if this option is not configured.
// Set to 1 to disable retries.
MaxRetries int
}
// Global constants.
const (
libraryName = "minio-go"
libraryVersion = "v7.0.79"
libraryVersion = "v7.0.80"
)
// User Agent should always following the below style.
@ -278,6 +283,11 @@ func privateNew(endpoint string, opts *Options) (*Client, error) {
// healthcheck is not initialized
clnt.healthStatus = unknown
clnt.maxRetries = MaxRetry
if opts.MaxRetries > 0 {
clnt.maxRetries = opts.MaxRetries
}
// Return.
return clnt, nil
}
@ -590,9 +600,9 @@ func (c *Client) executeMethod(ctx context.Context, method string, metadata requ
return nil, errors.New(c.endpointURL.String() + " is offline.")
}
var retryable bool // Indicates if request can be retried.
var bodySeeker io.Seeker // Extracted seeker from io.Reader.
reqRetry := MaxRetry // Indicates how many times we can retry the request
var retryable bool // Indicates if request can be retried.
var bodySeeker io.Seeker // Extracted seeker from io.Reader.
var reqRetry = c.maxRetries // Indicates how many times we can retry the request
if metadata.contentBody != nil {
// Check if body is seekable then it is retryable.

View file

@ -434,12 +434,34 @@ func (de DelMarkerExpiration) MarshalXML(enc *xml.Encoder, start xml.StartElemen
return enc.EncodeElement(delMarkerExp(de), start)
}
// AllVersionsExpiration represents AllVersionsExpiration actions element in an ILM policy
type AllVersionsExpiration struct {
XMLName xml.Name `xml:"AllVersionsExpiration" json:"-"`
Days int `xml:"Days,omitempty" json:"Days,omitempty"`
DeleteMarker ExpireDeleteMarker `xml:"DeleteMarker,omitempty" json:"DeleteMarker,omitempty"`
}
// IsNull returns true if days field is 0
func (e AllVersionsExpiration) IsNull() bool {
return e.Days == 0
}
// MarshalXML satisfies xml.Marshaler to provide custom encoding
func (e AllVersionsExpiration) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
if e.IsNull() {
return nil
}
type allVersionsExp AllVersionsExpiration
return enc.EncodeElement(allVersionsExp(e), start)
}
// MarshalJSON customizes json encoding by omitting empty values
func (r Rule) MarshalJSON() ([]byte, error) {
type rule struct {
AbortIncompleteMultipartUpload *AbortIncompleteMultipartUpload `json:"AbortIncompleteMultipartUpload,omitempty"`
Expiration *Expiration `json:"Expiration,omitempty"`
DelMarkerExpiration *DelMarkerExpiration `json:"DelMarkerExpiration,omitempty"`
AllVersionsExpiration *AllVersionsExpiration `json:"AllVersionsExpiration,omitempty"`
ID string `json:"ID"`
RuleFilter *Filter `json:"Filter,omitempty"`
NoncurrentVersionExpiration *NoncurrentVersionExpiration `json:"NoncurrentVersionExpiration,omitempty"`
@ -475,6 +497,9 @@ type rule struct {
if !r.NoncurrentVersionTransition.isNull() {
newr.NoncurrentVersionTransition = &r.NoncurrentVersionTransition
}
if !r.AllVersionsExpiration.IsNull() {
newr.AllVersionsExpiration = &r.AllVersionsExpiration
}
return json.Marshal(newr)
}
@ -485,6 +510,7 @@ type Rule struct {
AbortIncompleteMultipartUpload AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty" json:"AbortIncompleteMultipartUpload,omitempty"`
Expiration Expiration `xml:"Expiration,omitempty" json:"Expiration,omitempty"`
DelMarkerExpiration DelMarkerExpiration `xml:"DelMarkerExpiration,omitempty" json:"DelMarkerExpiration,omitempty"`
AllVersionsExpiration AllVersionsExpiration `xml:"AllVersionsExpiration,omitempty" json:"AllVersionsExpiration,omitempty"`
ID string `xml:"ID" json:"ID"`
RuleFilter Filter `xml:"Filter,omitempty" json:"Filter,omitempty"`
NoncurrentVersionExpiration NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty" json:"NoncurrentVersionExpiration,omitempty"`

5
vendor/modules.txt vendored
View file

@ -48,9 +48,6 @@ codeberg.org/gruf/go-maps
# codeberg.org/gruf/go-mempool v0.0.0-20240507125005-cef10d64a760
## explicit; go 1.22.2
codeberg.org/gruf/go-mempool
# codeberg.org/gruf/go-mimetypes v1.2.0
## explicit; go 1.17
codeberg.org/gruf/go-mimetypes
# codeberg.org/gruf/go-mutexes v1.5.1
## explicit; go 1.22.2
codeberg.org/gruf/go-mutexes
@ -486,7 +483,7 @@ github.com/miekg/dns
# github.com/minio/md5-simd v1.1.2
## explicit; go 1.14
github.com/minio/md5-simd
# github.com/minio/minio-go/v7 v7.0.79
# github.com/minio/minio-go/v7 v7.0.80
## explicit; go 1.22
github.com/minio/minio-go/v7
github.com/minio/minio-go/v7/pkg/cors