mirror of
https://github.com/superseriousbusiness/gotosocial.git
synced 2024-11-25 13:16:40 +00:00
[feature] Object store custom URL (S3) (#3046)
* tweaks * boobs * fix variable name + typo --------- Co-authored-by: tobi <tobi.smethurst@protonmail.com>
This commit is contained in:
parent
26022c2733
commit
43519324b3
|
@ -30,11 +30,42 @@ storage-local-base-path: "/gotosocial/storage"
|
||||||
# Default: ""
|
# Default: ""
|
||||||
storage-s3-endpoint: ""
|
storage-s3-endpoint: ""
|
||||||
|
|
||||||
# Bool. If data stored in S3 should be proxied through GoToSocial instead of redirecting to a presigned URL.
|
# Bool. Set this to true if data stored in S3 should be proxied through
|
||||||
|
# GoToSocial instead of forwarding the request to a presigned URL.
|
||||||
|
#
|
||||||
|
# In most cases you won't need to touch this setting, but it might be useful
|
||||||
|
# if it's not possible for your bucket provider to generate presigned URLs,
|
||||||
|
# or if your bucket is not able to exposed to the wider internet.
|
||||||
#
|
#
|
||||||
# Default: false
|
# Default: false
|
||||||
storage-s3-proxy: false
|
storage-s3-proxy: false
|
||||||
|
|
||||||
|
# String. URL to use a base for redirecting incoming media requests to.
|
||||||
|
#
|
||||||
|
# Must start with "http://" or "https://" and end without a trailing slash.
|
||||||
|
#
|
||||||
|
# DON'T SET THIS VALUE UNLESS YOU HAVE GOOD REASON TO! It's not necessary for
|
||||||
|
# "normal" s3 usage, and most admins can happily just ignore this setting.
|
||||||
|
#
|
||||||
|
# If set, then media fileserver requests to your instance will be redirected
|
||||||
|
# to this URL instead of your bucket URL, preserving relevant path parts.
|
||||||
|
#
|
||||||
|
# This is useful if you are using a CDN proxy in front of your S3 bucket, and you
|
||||||
|
# want to serve media from the CDN rather than serving from your S3 bucket directly.
|
||||||
|
#
|
||||||
|
# For example, if you have your storage-s3-endpoint value set to "s3.my-storage.example.org",
|
||||||
|
# and you have a CDN set up to proxy your bucket, serving from "cdn.some-fancy-host.org",
|
||||||
|
# then you should set storage-s3-redirect-url to "https://cdn.some-fancy-host.org".
|
||||||
|
#
|
||||||
|
# This will allow your GoToSocial instance to *upload* data to "s3.my-storage.example.org",
|
||||||
|
# but direct callers to *download* that data from "https://cdn.some-fancy-host.org".
|
||||||
|
#
|
||||||
|
# This value is ignored if storage-backend is not s3, or if storage-s3-proxy is true.
|
||||||
|
#
|
||||||
|
# Examples: ["https://cdn.some-fancy-host.org"]
|
||||||
|
# Default: ""
|
||||||
|
storage-s3-redirect-url: ""
|
||||||
|
|
||||||
# Bool. Use SSL for S3 connections.
|
# Bool. Use SSL for S3 connections.
|
||||||
#
|
#
|
||||||
# Only set this to 'false' when testing locally.
|
# Only set this to 'false' when testing locally.
|
||||||
|
@ -76,7 +107,7 @@ storage-s3-bucket: ""
|
||||||
GoToSocial by default creates signed URL's which means we don't need to change anything major on the policies of the bucket.
|
GoToSocial by default creates signed URL's which means we don't need to change anything major on the policies of the bucket.
|
||||||
|
|
||||||
1. Login to AWS -> select S3 as service.
|
1. Login to AWS -> select S3 as service.
|
||||||
2. click Create Bucket
|
2. Click Create Bucket
|
||||||
3. Provide a unique name and avoid adding "." in the name
|
3. Provide a unique name and avoid adding "." in the name
|
||||||
4. Do not change the public access settings (Let them be on "block public access" mode)
|
4. Do not change the public access settings (Let them be on "block public access" mode)
|
||||||
|
|
||||||
|
@ -110,6 +141,14 @@ GoToSocial by default creates signed URL's which means we don't need to change a
|
||||||
* `storage-s3-secret-key` -> Secret key you obtained for the user created above
|
* `storage-s3-secret-key` -> Secret key you obtained for the user created above
|
||||||
* `storage-s3-bucket` -> The `<bucketname>` that you created just now
|
* `storage-s3-bucket` -> The `<bucketname>` that you created just now
|
||||||
|
|
||||||
|
### `storage-s3-redirect-url`
|
||||||
|
|
||||||
|
If you are using a CDN in front of your S3 bucket, and you want to serve media from the CDN rather than serving from your S3 bucket directly, you should set the `storage-s3-redirect-url` to the CDN URL.
|
||||||
|
|
||||||
|
For example, if you have your `storage-s3-endpoint` value set to "s3.my-storage.example.org", and you have a CDN set up to proxy your bucket, serving from "cdn.some-fancy-host.org", then you should set `storage-s3-redirect-url` to "https://cdn.some-fancy-host.org".
|
||||||
|
|
||||||
|
This will allow your GoToSocial instance to *upload* data to "s3.my-storage.example.org", but direct callers to *download* that data from "https://cdn.some-fancy-host.org".
|
||||||
|
|
||||||
## Storage migration
|
## Storage migration
|
||||||
|
|
||||||
Migration between backends is freely possible. To do so, you only have to move the directories (and their contents) between the different implementations.
|
Migration between backends is freely possible. To do so, you only have to move the directories (and their contents) between the different implementations.
|
||||||
|
|
|
@ -551,11 +551,42 @@ storage-local-base-path: "/gotosocial/storage"
|
||||||
# Default: ""
|
# Default: ""
|
||||||
storage-s3-endpoint: ""
|
storage-s3-endpoint: ""
|
||||||
|
|
||||||
# Bool. If data stored in S3 should be proxied through GoToSocial instead of redirecting to a presigned URL.
|
# Bool. Set this to true if data stored in S3 should be proxied through
|
||||||
|
# GoToSocial instead of forwarding the request to a presigned URL.
|
||||||
|
#
|
||||||
|
# In most cases you won't need to touch this setting, but it might be useful
|
||||||
|
# if it's not possible for your bucket provider to generate presigned URLs,
|
||||||
|
# or if your bucket is not able to exposed to the wider internet.
|
||||||
#
|
#
|
||||||
# Default: false
|
# Default: false
|
||||||
storage-s3-proxy: false
|
storage-s3-proxy: false
|
||||||
|
|
||||||
|
# String. URL to use a base for redirecting incoming media requests to.
|
||||||
|
#
|
||||||
|
# Must start with "http://" or "https://" and end without a trailing slash.
|
||||||
|
#
|
||||||
|
# DON'T SET THIS VALUE UNLESS YOU HAVE GOOD REASON TO! It's not necessary for
|
||||||
|
# "normal" s3 usage, and most admins can happily just ignore this setting.
|
||||||
|
#
|
||||||
|
# If set, then media fileserver requests to your instance will be redirected
|
||||||
|
# to this URL instead of your bucket URL, preserving relevant path parts.
|
||||||
|
#
|
||||||
|
# This is useful if you are using a CDN proxy in front of your S3 bucket, and you
|
||||||
|
# want to serve media from the CDN rather than serving from your S3 bucket directly.
|
||||||
|
#
|
||||||
|
# For example, if you have your storage-s3-endpoint value set to "s3.my-storage.example.org",
|
||||||
|
# and you have a CDN set up to proxy your bucket, serving from "cdn.some-fancy-host.org",
|
||||||
|
# then you should set storage-s3-redirect-url to "https://cdn.some-fancy-host.org".
|
||||||
|
#
|
||||||
|
# This will allow your GoToSocial instance to *upload* data to "s3.my-storage.example.org",
|
||||||
|
# but direct callers to *download* that data from "https://cdn.some-fancy-host.org".
|
||||||
|
#
|
||||||
|
# This value is ignored if storage-backend is not s3, or if storage-s3-proxy is true.
|
||||||
|
#
|
||||||
|
# Examples: ["https://cdn.some-fancy-host.org"]
|
||||||
|
# Default: ""
|
||||||
|
storage-s3-redirect-url: ""
|
||||||
|
|
||||||
# Bool. Use SSL for S3 connections.
|
# Bool. Use SSL for S3 connections.
|
||||||
#
|
#
|
||||||
# Only set this to 'false' when testing locally.
|
# Only set this to 'false' when testing locally.
|
||||||
|
|
|
@ -110,6 +110,7 @@ type Configuration struct {
|
||||||
StorageS3UseSSL bool `name:"storage-s3-use-ssl" usage:"Use SSL for S3 connections. Only set this to 'false' when testing locally"`
|
StorageS3UseSSL bool `name:"storage-s3-use-ssl" usage:"Use SSL for S3 connections. Only set this to 'false' when testing locally"`
|
||||||
StorageS3BucketName string `name:"storage-s3-bucket" usage:"Place blobs in this bucket"`
|
StorageS3BucketName string `name:"storage-s3-bucket" usage:"Place blobs in this bucket"`
|
||||||
StorageS3Proxy bool `name:"storage-s3-proxy" usage:"Proxy S3 contents through GoToSocial instead of redirecting to a presigned URL"`
|
StorageS3Proxy bool `name:"storage-s3-proxy" usage:"Proxy S3 contents through GoToSocial instead of redirecting to a presigned URL"`
|
||||||
|
StorageS3RedirectURL string `name:"storage-s3-redirect-url" usage:"Custom URL to use for redirecting S3 media links. If set, this will be used instead of the S3 bucket URL."`
|
||||||
|
|
||||||
StatusesMaxChars int `name:"statuses-max-chars" usage:"Max permitted characters for posted statuses, including content warning"`
|
StatusesMaxChars int `name:"statuses-max-chars" usage:"Max permitted characters for posted statuses, including content warning"`
|
||||||
StatusesPollMaxOptions int `name:"statuses-poll-max-options" usage:"Max amount of options permitted on a poll"`
|
StatusesPollMaxOptions int `name:"statuses-poll-max-options" usage:"Max amount of options permitted on a poll"`
|
||||||
|
|
|
@ -85,6 +85,7 @@
|
||||||
StorageLocalBasePath: "/gotosocial/storage",
|
StorageLocalBasePath: "/gotosocial/storage",
|
||||||
StorageS3UseSSL: true,
|
StorageS3UseSSL: true,
|
||||||
StorageS3Proxy: false,
|
StorageS3Proxy: false,
|
||||||
|
StorageS3RedirectURL: "",
|
||||||
|
|
||||||
StatusesMaxChars: 5000,
|
StatusesMaxChars: 5000,
|
||||||
StatusesPollMaxOptions: 6,
|
StatusesPollMaxOptions: 6,
|
||||||
|
|
|
@ -1500,6 +1500,31 @@ func GetStorageS3Proxy() bool { return global.GetStorageS3Proxy() }
|
||||||
// SetStorageS3Proxy safely sets the value for global configuration 'StorageS3Proxy' field
|
// SetStorageS3Proxy safely sets the value for global configuration 'StorageS3Proxy' field
|
||||||
func SetStorageS3Proxy(v bool) { global.SetStorageS3Proxy(v) }
|
func SetStorageS3Proxy(v bool) { global.SetStorageS3Proxy(v) }
|
||||||
|
|
||||||
|
// GetStorageS3RedirectURL safely fetches the Configuration value for state's 'StorageS3RedirectURL' field
|
||||||
|
func (st *ConfigState) GetStorageS3RedirectURL() (v string) {
|
||||||
|
st.mutex.RLock()
|
||||||
|
v = st.config.StorageS3RedirectURL
|
||||||
|
st.mutex.RUnlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetStorageS3RedirectURL safely sets the Configuration value for state's 'StorageS3RedirectURL' field
|
||||||
|
func (st *ConfigState) SetStorageS3RedirectURL(v string) {
|
||||||
|
st.mutex.Lock()
|
||||||
|
defer st.mutex.Unlock()
|
||||||
|
st.config.StorageS3RedirectURL = v
|
||||||
|
st.reloadToViper()
|
||||||
|
}
|
||||||
|
|
||||||
|
// StorageS3RedirectURLFlag returns the flag name for the 'StorageS3RedirectURL' field
|
||||||
|
func StorageS3RedirectURLFlag() string { return "storage-s3-redirect-url" }
|
||||||
|
|
||||||
|
// GetStorageS3RedirectURL safely fetches the value for global configuration 'StorageS3RedirectURL' field
|
||||||
|
func GetStorageS3RedirectURL() string { return global.GetStorageS3RedirectURL() }
|
||||||
|
|
||||||
|
// SetStorageS3RedirectURL safely sets the value for global configuration 'StorageS3RedirectURL' field
|
||||||
|
func SetStorageS3RedirectURL(v string) { global.SetStorageS3RedirectURL(v) }
|
||||||
|
|
||||||
// GetStatusesMaxChars safely fetches the Configuration value for state's 'StatusesMaxChars' field
|
// GetStatusesMaxChars safely fetches the Configuration value for state's 'StatusesMaxChars' field
|
||||||
func (st *ConfigState) GetStatusesMaxChars() (v int) {
|
func (st *ConfigState) GetStatusesMaxChars() (v int) {
|
||||||
st.mutex.RLock()
|
st.mutex.RLock()
|
||||||
|
|
|
@ -19,6 +19,8 @@
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/miekg/dns"
|
"github.com/miekg/dns"
|
||||||
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
|
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
|
||||||
|
@ -118,6 +120,28 @@ func Validate() error {
|
||||||
errf("%s must be set", WebAssetBaseDirFlag())
|
errf("%s must be set", WebAssetBaseDirFlag())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// `storage-s3-redirect-url`
|
||||||
|
if s3RedirectURL := GetStorageS3RedirectURL(); s3RedirectURL != "" {
|
||||||
|
if strings.HasSuffix(s3RedirectURL, "/") {
|
||||||
|
errf(
|
||||||
|
"%s must not end with a trailing slash",
|
||||||
|
StorageS3RedirectURLFlag(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if url, err := url.Parse(s3RedirectURL); err != nil {
|
||||||
|
errf(
|
||||||
|
"%s invalid: %w",
|
||||||
|
StorageS3RedirectURLFlag(), err,
|
||||||
|
)
|
||||||
|
} else if url.Scheme != "https" && url.Scheme != "http" {
|
||||||
|
errf(
|
||||||
|
"%s scheme must be https or http",
|
||||||
|
StorageS3RedirectURLFlag(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Custom / LE TLS settings.
|
// Custom / LE TLS settings.
|
||||||
//
|
//
|
||||||
// Only one of custom certs or LE can be set,
|
// Only one of custom certs or LE can be set,
|
||||||
|
|
|
@ -79,6 +79,7 @@ type Driver struct {
|
||||||
Proxy bool
|
Proxy bool
|
||||||
Bucket string
|
Bucket string
|
||||||
PresignedCache *ttl.Cache[string, PresignedURL]
|
PresignedCache *ttl.Cache[string, PresignedURL]
|
||||||
|
RedirectURL string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get returns the byte value for key in storage.
|
// Get returns the byte value for key in storage.
|
||||||
|
@ -163,13 +164,28 @@ func (d *Driver) URL(ctx context.Context, key string) *PresignedURL {
|
||||||
return &e.Value
|
return &e.Value
|
||||||
}
|
}
|
||||||
|
|
||||||
u, err := s3.Client().PresignedGetObject(ctx, d.Bucket, key, urlCacheTTL, url.Values{
|
var (
|
||||||
|
u *url.URL
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
|
||||||
|
if d.RedirectURL != "" {
|
||||||
|
u, err = url.Parse(d.RedirectURL + "/" + key)
|
||||||
|
if err != nil {
|
||||||
|
// If URL parsing fails, fallback is to
|
||||||
|
// fetch the file. So ignore the error here
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
u, err = s3.Client().PresignedGetObject(ctx, d.Bucket, key, urlCacheTTL, url.Values{
|
||||||
"response-content-type": []string{mime.TypeByExtension(path.Ext(key))},
|
"response-content-type": []string{mime.TypeByExtension(path.Ext(key))},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// If URL request fails, fallback is to fetch the file. So ignore the error here
|
// If URL request fails, fallback is to
|
||||||
|
// fetch the file. So ignore the error here
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
psu := PresignedURL{
|
psu := PresignedURL{
|
||||||
URL: u,
|
URL: u,
|
||||||
|
@ -204,6 +220,14 @@ func (d *Driver) ProbeCSPUri(ctx context.Context) (string, error) {
|
||||||
return "", nil
|
return "", nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If an S3 redirect URL is set, just
|
||||||
|
// return this URL without probing; we
|
||||||
|
// likely don't have write access on it
|
||||||
|
// anyway since it's probs a CDN bucket.
|
||||||
|
if d.RedirectURL != "" {
|
||||||
|
return d.RedirectURL + "/", nil
|
||||||
|
}
|
||||||
|
|
||||||
const cspKey = "gotosocial-csp-probe"
|
const cspKey = "gotosocial-csp-probe"
|
||||||
|
|
||||||
// Create an empty file in S3 storage.
|
// Create an empty file in S3 storage.
|
||||||
|
@ -273,6 +297,7 @@ func NewS3Storage() (*Driver, error) {
|
||||||
secret := config.GetStorageS3SecretKey()
|
secret := config.GetStorageS3SecretKey()
|
||||||
secure := config.GetStorageS3UseSSL()
|
secure := config.GetStorageS3UseSSL()
|
||||||
bucket := config.GetStorageS3BucketName()
|
bucket := config.GetStorageS3BucketName()
|
||||||
|
redirectURL := config.GetStorageS3RedirectURL()
|
||||||
|
|
||||||
// Open the s3 storage implementation
|
// Open the s3 storage implementation
|
||||||
s3, err := s3.Open(endpoint, bucket, &s3.Config{
|
s3, err := s3.Open(endpoint, bucket, &s3.Config{
|
||||||
|
@ -300,5 +325,6 @@ func NewS3Storage() (*Driver, error) {
|
||||||
Bucket: config.GetStorageS3BucketName(),
|
Bucket: config.GetStorageS3BucketName(),
|
||||||
Storage: s3,
|
Storage: s3,
|
||||||
PresignedCache: presignedCache,
|
PresignedCache: presignedCache,
|
||||||
|
RedirectURL: redirectURL,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -173,6 +173,7 @@ EXPECT=$(cat << "EOF"
|
||||||
"storage-s3-bucket": "gts",
|
"storage-s3-bucket": "gts",
|
||||||
"storage-s3-endpoint": "localhost:9000",
|
"storage-s3-endpoint": "localhost:9000",
|
||||||
"storage-s3-proxy": true,
|
"storage-s3-proxy": true,
|
||||||
|
"storage-s3-redirect-url": "",
|
||||||
"storage-s3-secret-key": "miniostorage",
|
"storage-s3-secret-key": "miniostorage",
|
||||||
"storage-s3-use-ssl": false,
|
"storage-s3-use-ssl": false,
|
||||||
"syslog-address": "127.0.0.1:6969",
|
"syslog-address": "127.0.0.1:6969",
|
||||||
|
@ -253,6 +254,7 @@ GTS_STORAGE_S3_SECRET_KEY='miniostorage' \
|
||||||
GTS_STORAGE_S3_ENDPOINT='localhost:9000' \
|
GTS_STORAGE_S3_ENDPOINT='localhost:9000' \
|
||||||
GTS_STORAGE_S3_USE_SSL='false' \
|
GTS_STORAGE_S3_USE_SSL='false' \
|
||||||
GTS_STORAGE_S3_PROXY='true' \
|
GTS_STORAGE_S3_PROXY='true' \
|
||||||
|
GTS_STORAGE_S3_REDIRECT_URL='' \
|
||||||
GTS_STORAGE_S3_BUCKET='gts' \
|
GTS_STORAGE_S3_BUCKET='gts' \
|
||||||
GTS_STATUSES_MAX_CHARS=69 \
|
GTS_STATUSES_MAX_CHARS=69 \
|
||||||
GTS_STATUSES_CW_MAX_CHARS=420 \
|
GTS_STATUSES_CW_MAX_CHARS=420 \
|
||||||
|
|
Loading…
Reference in a new issue