[chore] Add media-ffmpeg-pool-size config var (#3164)

This commit is contained in:
tobi 2024-08-03 16:40:26 +02:00 committed by GitHub
parent 09f239d7e3
commit fa59c3713c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 77 additions and 4 deletions

View file

@ -487,16 +487,24 @@ func setLimits(ctx context.Context) {
}
func precompileWASM(ctx context.Context) error {
// TODO: make max number instances configurable
maxprocs := runtime.GOMAXPROCS(0)
if err := sqlite3.Initialize(); err != nil {
return gtserror.Newf("error compiling sqlite3: %w", err)
}
if err := ffmpeg.InitFfmpeg(ctx, maxprocs); err != nil {
// Use admin-set ffmpeg pool size, and fall
// back to GOMAXPROCS if number 0 or less.
ffPoolSize := config.GetMediaFfmpegPoolSize()
if ffPoolSize <= 0 {
ffPoolSize = runtime.GOMAXPROCS(0)
}
if err := ffmpeg.InitFfmpeg(ctx, ffPoolSize); err != nil {
return gtserror.Newf("error compiling ffmpeg: %w", err)
}
if err := ffmpeg.InitFfprobe(ctx, maxprocs); err != nil {
if err := ffmpeg.InitFfprobe(ctx, ffPoolSize); err != nil {
return gtserror.Newf("error compiling ffprobe: %w", err)
}
return nil
}

View file

@ -56,6 +56,24 @@ media-emoji-local-max-size: 50KiB
# Default: 100KiB (102400 bytes)
media-emoji-remote-max-size: 100KiB
# Int. Number of instances of ffmpeg+ffprobe to add to the media processing pool.
#
# Increasing this number will lead to faster concurrent media processing,
# but at the cost of up to about 250MB of (spiking) memory usage per increment.
#
# You'll want to increase this number if you have RAM to spare, and/or if you're
# hosting an instance for more than 50 or so people who post/view lots of media,
# but you should leave it at 1 for single-user instances or when running GoToSocial
# in a constrained (low-memory) environment.
#
# If you set this number to 0 or less, then instead of a fixed number of instances,
# it will scale with GOMAXPROCS x 1, yielding (usually) one ffmpeg instance and one
# ffprobe instance per CPU core on the host machine.
#
# Examples: [1, 2, -1, 8]
# Default: 1
media-ffmpeg-pool-size: 1
# The below media cleanup settings allow admins to customize when and
# how often media cleanup + prune jobs run, while being set to a fairly
# sensible default (every night @ midnight). For more information on exactly

View file

@ -493,6 +493,24 @@ media-emoji-local-max-size: 50KiB
# Default: 100KiB (102400 bytes)
media-emoji-remote-max-size: 100KiB
# Int. Number of instances of ffmpeg+ffprobe to add to the media processing pool.
#
# Increasing this number will lead to faster concurrent media processing,
# but at the cost of up to about 250MB of (spiking) memory usage per increment.
#
# You'll want to increase this number if you have RAM to spare, and/or if you're
# hosting an instance for more than 50 or so people who post/view lots of media,
# but you should leave it at 1 for single-user instances or when running GoToSocial
# in a constrained (low-memory) environment.
#
# If you set this number to 0 or less, then instead of a fixed number of instances,
# it will scale with GOMAXPROCS x 1, yielding (usually) one ffmpeg instance and one
# ffprobe instance per CPU core on the host machine.
#
# Examples: [1, 2, -1, 8]
# Default: 1
media-ffmpeg-pool-size: 1
# The below media cleanup settings allow admins to customize when and
# how often media cleanup + prune jobs run, while being set to a fairly
# sensible default (every night @ midnight). For more information on exactly

View file

@ -101,6 +101,7 @@ type Configuration struct {
MediaRemoteMaxSize bytesize.Size `name:"media-remote-max-size" usage:"Max size in bytes of media to download from other instances"`
MediaCleanupFrom string `name:"media-cleanup-from" usage:"Time of day from which to start running media cleanup/prune jobs. Should be in the format 'hh:mm:ss', eg., '15:04:05'."`
MediaCleanupEvery time.Duration `name:"media-cleanup-every" usage:"Period to elapse between cleanups, starting from media-cleanup-at."`
MediaFfmpegPoolSize int `name:"media-ffmpeg-pool-size" usage:"Number of instances of the embedded ffmpeg WASM binary to add to the media processing pool. 0 or less uses GOMAXPROCS."`
StorageBackend string `name:"storage-backend" usage:"Storage backend to use for media attachments"`
StorageLocalBasePath string `name:"storage-local-base-path" usage:"Full path to an already-created directory where gts should store/retrieve media files. Subfolders will be created within this dir."`

View file

@ -80,6 +80,7 @@
MediaEmojiRemoteMaxSize: 100 * bytesize.KiB,
MediaCleanupFrom: "00:00", // Midnight.
MediaCleanupEvery: 24 * time.Hour, // 1/day.
MediaFfmpegPoolSize: 1,
StorageBackend: "local",
StorageLocalBasePath: "/gotosocial/storage",

View file

@ -1300,6 +1300,31 @@ func GetMediaCleanupEvery() time.Duration { return global.GetMediaCleanupEvery()
// SetMediaCleanupEvery safely sets the value for global configuration 'MediaCleanupEvery' field
func SetMediaCleanupEvery(v time.Duration) { global.SetMediaCleanupEvery(v) }
// GetMediaFfmpegPoolSize safely fetches the Configuration value for state's 'MediaFfmpegPoolSize' field
func (st *ConfigState) GetMediaFfmpegPoolSize() (v int) {
st.mutex.RLock()
v = st.config.MediaFfmpegPoolSize
st.mutex.RUnlock()
return
}
// SetMediaFfmpegPoolSize safely sets the Configuration value for state's 'MediaFfmpegPoolSize' field
func (st *ConfigState) SetMediaFfmpegPoolSize(v int) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.MediaFfmpegPoolSize = v
st.reloadToViper()
}
// MediaFfmpegPoolSizeFlag returns the flag name for the 'MediaFfmpegPoolSize' field
func MediaFfmpegPoolSizeFlag() string { return "media-ffmpeg-pool-size" }
// GetMediaFfmpegPoolSize safely fetches the value for global configuration 'MediaFfmpegPoolSize' field
func GetMediaFfmpegPoolSize() int { return global.GetMediaFfmpegPoolSize() }
// SetMediaFfmpegPoolSize safely sets the value for global configuration 'MediaFfmpegPoolSize' field
func SetMediaFfmpegPoolSize(v int) { global.SetMediaFfmpegPoolSize(v) }
// GetStorageBackend safely fetches the Configuration value for state's 'StorageBackend' field
func (st *ConfigState) GetStorageBackend() (v string) {
st.mutex.RLock()

View file

@ -126,6 +126,7 @@ EXPECT=$(cat << "EOF"
"media-description-min-chars": 69,
"media-emoji-local-max-size": 420,
"media-emoji-remote-max-size": 420,
"media-ffmpeg-pool-size": 8,
"media-local-max-size": 420,
"media-remote-cache-days": 30,
"media-remote-max-size": 420,
@ -245,6 +246,7 @@ GTS_MEDIA_REMOTE_MAX_SIZE=420 \
GTS_MEDIA_REMOTE_CACHE_DAYS=30 \
GTS_MEDIA_EMOJI_LOCAL_MAX_SIZE=420 \
GTS_MEDIA_EMOJI_REMOTE_MAX_SIZE=420 \
GTS_MEDIA_FFMPEG_POOL_SIZE=8 \
GTS_METRICS_AUTH_ENABLED=false \
GTS_METRICS_ENABLED=false \
GTS_STORAGE_BACKEND='local' \