gotosocial/test/envparsing.sh

298 lines
9.7 KiB
Bash
Raw Normal View History

#!/bin/sh
set -eu
EXPECT=$(cat <<"EOF"
{
"account-domain": "peepee",
"accounts-allow-custom-css": true,
"accounts-approval-required": false,
"accounts-custom-css-length": 5000,
"accounts-reason-required": false,
"accounts-registration-open": true,
"advanced-cookies-samesite": "strict",
"advanced-rate-limit-requests": 6969,
"advanced-sender-multiplier": -1,
"advanced-throttling-multiplier": -1,
"advanced-throttling-retry-after": 10000000000,
"application-name": "gts",
"bind-address": "127.0.0.1",
"cache": {
"gts": {
"account-max-size": 99,
"account-note-max-size": 1000,
"account-note-sweep-freq": 60000000000,
"account-note-ttl": 1800000000000,
"account-sweep-freq": 1000000000,
"account-ttl": 10800000000000,
[performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
"block-max-size": 1000,
"block-sweep-freq": 60000000000,
"block-ttl": 1800000000000,
"domain-block-max-size": 2000,
"domain-block-sweep-freq": 60000000000,
"domain-block-ttl": 86400000000000,
"emoji-category-max-size": 100,
[performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
"emoji-category-sweep-freq": 60000000000,
"emoji-category-ttl": 1800000000000,
"emoji-max-size": 2000,
"emoji-sweep-freq": 60000000000,
"emoji-ttl": 1800000000000,
"follow-max-size": 2000,
"follow-request-max-size": 2000,
"follow-request-sweep-freq": 60000000000,
"follow-request-ttl": 1800000000000,
"follow-sweep-freq": 60000000000,
"follow-ttl": 1800000000000,
"instance-max-size": 2000,
"instance-sweep-freq": 60000000000,
"instance-ttl": 1800000000000,
"list-entry-max-size": 2000,
"list-entry-sweep-freq": 60000000000,
"list-entry-ttl": 1800000000000,
"list-max-size": 2000,
"list-sweep-freq": 60000000000,
"list-ttl": 1800000000000,
[performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
"media-max-size": 1000,
"media-sweep-freq": 60000000000,
"media-ttl": 1800000000000,
"mention-max-size": 2000,
"mention-sweep-freq": 60000000000,
"mention-ttl": 1800000000000,
"notification-max-size": 1000,
"notification-sweep-freq": 60000000000,
"notification-ttl": 1800000000000,
"report-max-size": 100,
[performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
"report-sweep-freq": 60000000000,
"report-ttl": 1800000000000,
"status-fave-max-size": 2000,
"status-fave-sweep-freq": 60000000000,
"status-fave-ttl": 1800000000000,
"status-max-size": 2000,
"status-sweep-freq": 60000000000,
"status-ttl": 1800000000000,
"tombstone-max-size": 500,
"tombstone-sweep-freq": 60000000000,
"tombstone-ttl": 1800000000000,
"user-max-size": 500,
"user-sweep-freq": 60000000000,
"user-ttl": 1800000000000,
"webfinger-max-size": 250,
"webfinger-sweep-freq": 900000000000,
"webfinger-ttl": 86400000000000
[performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
2023-03-28 13:03:14 +00:00
},
"visibility-max-size": 2000,
"visibility-sweep-freq": 60000000000,
"visibility-ttl": 1800000000000
},
"config-path": "internal/config/testdata/test.yaml",
"db-address": ":memory:",
"db-database": "gotosocial_prod",
"db-max-open-conns-multiplier": 3,
"db-password": "hunter2",
"db-port": 6969,
"db-sqlite-busy-timeout": 1000000000,
"db-sqlite-cache-size": 0,
"db-sqlite-journal-mode": "DELETE",
"db-sqlite-synchronous": "FULL",
"db-tls-ca-cert": "",
"db-tls-mode": "disable",
"db-type": "sqlite",
"db-user": "sex-haver",
"dry-run": true,
"email": "",
"host": "example.com",
"http-client": {
"allow-ips": [],
"block-ips": [],
"timeout": 10000000000
},
"instance-deliver-to-shared-inboxes": false,
"instance-expose-peers": true,
"instance-expose-public-timeline": true,
"instance-expose-suspended": true,
"instance-expose-suspended-web": true,
"instance-inject-mastodon-version": true,
"landing-page-user": "admin",
"letsencrypt-cert-dir": "/gotosocial/storage/certs",
"letsencrypt-email-address": "",
"letsencrypt-enabled": true,
"letsencrypt-port": 80,
"log-client-ip": false,
"log-db-queries": true,
"log-level": "info",
"media-description-max-chars": 5000,
"media-description-min-chars": 69,
"media-emoji-local-max-size": 420,
"media-emoji-remote-max-size": 420,
"media-image-max-size": 420,
"media-remote-cache-days": 30,
"media-video-max-size": 420,
"oidc-admin-groups": [
"steamy"
],
"oidc-client-id": "1234",
"oidc-client-secret": "shhhh its a secret",
"oidc-enabled": true,
"oidc-idp-name": "sex-haver",
"oidc-issuer": "whoknows",
"oidc-link-existing": true,
"oidc-scopes": [
"read",
"write"
],
"oidc-skip-verification": true,
"password": "",
"path": "",
"port": 6969,
"protocol": "http",
"request-id-header": "X-Trace-Id",
"smtp-disclose-recipients": true,
"smtp-from": "queen.rip.in.piss@terfisland.org",
"smtp-host": "example.com",
"smtp-password": "hunter2",
"smtp-port": 4269,
"smtp-username": "sex-haver",
"software-version": "",
"statuses-cw-max-chars": 420,
"statuses-max-chars": 69,
"statuses-media-max-files": 1,
"statuses-poll-max-options": 1,
"statuses-poll-option-max-chars": 50,
"storage-backend": "local",
"storage-local-base-path": "/root/store",
"storage-s3-access-key": "minio",
"storage-s3-bucket": "gts",
"storage-s3-endpoint": "localhost:9000",
"storage-s3-proxy": true,
"storage-s3-secret-key": "miniostorage",
"storage-s3-use-ssl": false,
"syslog-address": "127.0.0.1:6969",
"syslog-enabled": true,
"syslog-protocol": "udp",
"tls-certificate-chain": "",
"tls-certificate-key": "",
2023-05-09 17:19:48 +00:00
"tracing-enabled": false,
"tracing-endpoint": "localhost:4317",
"tracing-insecure": false,
"tracing-transport": "grpc",
"trusted-proxies": [
"127.0.0.1/32",
"docker.host.local"
],
"username": "",
"web-asset-base-dir": "/root",
"web-template-base-dir": "/root"
}
EOF
)
# Set all the environment variables to
# ensure that these are parsed without panic
OUTPUT=$(GTS_LOG_LEVEL='info' \
GTS_LOG_DB_QUERIES=true \
GTS_LOG_CLIENT_IP=false \
GTS_APPLICATION_NAME=gts \
GTS_LANDING_PAGE_USER=admin \
GTS_HOST=example.com \
GTS_ACCOUNT_DOMAIN='peepee' \
GTS_PROTOCOL=http \
GTS_BIND_ADDRESS='127.0.0.1' \
GTS_PORT=6969 \
GTS_TRUSTED_PROXIES='127.0.0.1/32,docker.host.local' \
GTS_DB_TYPE='sqlite' \
GTS_DB_ADDRESS=':memory:' \
GTS_DB_PORT=6969 \
GTS_DB_USER='sex-haver' \
GTS_DB_PASSWORD='hunter2' \
GTS_DB_DATABASE='gotosocial_prod' \
GTS_DB_MAX_OPEN_CONNS_MULTIPLIER=3 \
GTS_DB_SQLITE_JOURNAL_MODE='DELETE' \
GTS_DB_SQLITE_SYNCHRONOUS='FULL' \
GTS_DB_SQLITE_CACHE_SIZE=0 \
GTS_DB_SQLITE_BUSY_TIMEOUT='1s' \
GTS_TLS_MODE='' \
GTS_DB_TLS_CA_CERT='' \
GTS_WEB_TEMPLATE_BASE_DIR='/root' \
GTS_WEB_ASSET_BASE_DIR='/root' \
GTS_INSTANCE_EXPOSE_PEERS=true \
GTS_INSTANCE_EXPOSE_SUSPENDED=true \
GTS_INSTANCE_EXPOSE_SUSPENDED_WEB=true \
GTS_INSTANCE_EXPOSE_PUBLIC_TIMELINE=true \
GTS_INSTANCE_DELIVER_TO_SHARED_INBOXES=false \
GTS_INSTANCE_INJECT_MASTODON_VERSION=true \
GTS_ACCOUNTS_ALLOW_CUSTOM_CSS=true \
GTS_ACCOUNTS_CUSTOM_CSS_LENGTH=5000 \
GTS_ACCOUNTS_REGISTRATION_OPEN=true \
GTS_ACCOUNTS_APPROVAL_REQUIRED=false \
GTS_ACCOUNTS_REASON_REQUIRED=false \
GTS_MEDIA_IMAGE_MAX_SIZE=420 \
GTS_MEDIA_VIDEO_MAX_SIZE=420 \
GTS_MEDIA_DESCRIPTION_MIN_CHARS=69 \
GTS_MEDIA_DESCRIPTION_MAX_CHARS=5000 \
GTS_MEDIA_REMOTE_CACHE_DAYS=30 \
GTS_MEDIA_EMOJI_LOCAL_MAX_SIZE=420 \
GTS_MEDIA_EMOJI_REMOTE_MAX_SIZE=420 \
GTS_STORAGE_BACKEND='local' \
GTS_STORAGE_LOCAL_BASE_PATH='/root/store' \
GTS_STORAGE_S3_ACCESS_KEY='minio' \
GTS_STORAGE_S3_SECRET_KEY='miniostorage' \
GTS_STORAGE_S3_ENDPOINT='localhost:9000' \
GTS_STORAGE_S3_USE_SSL='false' \
GTS_STORAGE_S3_PROXY='true' \
GTS_STORAGE_S3_BUCKET='gts' \
GTS_STATUSES_MAX_CHARS=69 \
GTS_STATUSES_CW_MAX_CHARS=420 \
GTS_STATUSES_POLL_MAX_OPTIONS=1 \
GTS_STATUSES_POLL_OPTIONS_MAX_CHARS=69 \
GTS_STATUSES_MEDIA_MAX_FILES=1 \
GTS_LETS_ENCRYPT_ENABLED=false \
GTS_LETS_ENCRYPT_PORT=8080 \
GTS_LETS_ENCRYPT_CERT_DIR='/root/certs' \
GTS_LETS_ENCRYPT_EMAIL_ADDRESS='le@example.com' \
GTS_OIDC_ENABLED=true \
GTS_OIDC_IDP_NAME='sex-haver' \
GTS_OIDC_SKIP_VERIFICATION=true \
GTS_OIDC_ISSUER='whoknows' \
GTS_OIDC_CLIENT_ID='1234' \
GTS_OIDC_CLIENT_SECRET='shhhh its a secret' \
GTS_OIDC_SCOPES='read,write' \
GTS_OIDC_LINK_EXISTING=true \
GTS_OIDC_ADMIN_GROUPS='steamy' \
GTS_SMTP_HOST='example.com' \
GTS_SMTP_PORT=4269 \
GTS_SMTP_USERNAME='sex-haver' \
GTS_SMTP_PASSWORD='hunter2' \
GTS_SMTP_FROM='queen.rip.in.piss@terfisland.org' \
GTS_SMTP_DISCLOSE_RECIPIENTS=true \
GTS_SYSLOG_ENABLED=true \
GTS_SYSLOG_PROTOCOL='udp' \
GTS_SYSLOG_ADDRESS='127.0.0.1:6969' \
2023-05-09 17:19:48 +00:00
GTS_TRACING_ENDPOINT='localhost:4317' \
GTS_ADVANCED_COOKIES_SAMESITE='strict' \
GTS_ADVANCED_RATE_LIMIT_REQUESTS=6969 \
GTS_ADVANCED_SENDER_MULTIPLIER=-1 \
GTS_ADVANCED_THROTTLING_MULTIPLIER=-1 \
GTS_ADVANCED_THROTTLING_RETRY_AFTER='10s' \
GTS_REQUEST_ID_HEADER='X-Trace-Id' \
go run ./cmd/gotosocial/... --config-path internal/config/testdata/test.yaml debug config)
OUTPUT_OUT=$(mktemp)
echo "$OUTPUT" > "$OUTPUT_OUT"
EXPECT_OUT=$(mktemp)
echo "$EXPECT" > "$EXPECT_OUT"
DIFFCMD=$(command -v diff 2>&1)
if command -v jd >/dev/null 2>&1; then
DIFFCMD=$(command -v jd 2>&1)
fi
if ! DIFF=$("$DIFFCMD" "$OUTPUT_OUT" "$EXPECT_OUT"); then
echo "OUTPUT not equal EXPECTED"
echo "$DIFF"
exit 1
else
echo "OK"
exit 0
fi