Merge 9acf9ddc75 into 400fd8c62b

* [docs] Document experimental, unsupported `nowasm` tag

* change some awkward wording

.goreleaser.yml

@@ -39,7 +39,6 @@ builds:
     goos:
       - linux
       - freebsd
-      - openbsd
     goarch:
       - 386
       - amd64
@@ -49,19 +48,14 @@ builds:
       - 6
       - 7
     ignore:
-      # build freebsd + openbsd only for amd64
+      # Build FreeBSD
+      # only for amd64.
       - goos: freebsd
         goarch: arm64
       - goos: freebsd
         goarch: arm
       - goos: freebsd
         goarch: 386
-      - goos: openbsd
-        goarch: arm64
-      - goos: openbsd
-        goarch: arm
-      - goos: openbsd
-        goarch: 386
     mod_timestamp: "{{ .CommitTimestamp }}"
   # MODERNC SQLITE BINARY BUILDS
   -
@@ -88,7 +82,6 @@ builds:
     goos:
       - linux
       - freebsd
-      - openbsd
     goarch:
       - 386
       - amd64
@@ -98,19 +91,14 @@ builds:
       - 6
       - 7
     ignore:
-      # build freebsd + openbsd only for amd64
+      # Build FreeBSD
+      # only for amd64.
       - goos: freebsd
         goarch: arm64
       - goos: freebsd
         goarch: arm
       - goos: freebsd
         goarch: 386
-      - goos: openbsd
-        goarch: arm64
-      - goos: openbsd
-        goarch: arm
-      - goos: openbsd
-        goarch: 386
     mod_timestamp: "{{ .CommitTimestamp }}"
 
 # https://goreleaser.com/customization/docker/
@@ -474,17 +462,42 @@ release:
     
     Tl;dr: Regardless of whether you're using SQLite or Postgres as your DB driver, you most likely you want the regular version without `moderncsqlite` in the name.
     
-    However, if you're on FreeBSD or OpenBSD, use the `moderncsqlite` version instead. See the table below:
+    However, if you're on FreeBSD, 32-bit Linux or 32-bit ARM, we recommend using the `moderncsqlite` version instead.
     
-    | OS      | Architecture            | Binary archive                                                                                                                                                                     | Docker                                                         |
+    You may need to change some configuration options too. See the table below:
-    | ------- | ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- |
+    
-    | Linux   | x86-64/AMD64 (64-bit)   | [linux_amd64.tar.gz](https://github.com/superseriousbusiness/gotosocial/releases/download/{{ .Tag }}/gotosocial_{{ .Version }}_linux_amd64.tar.gz)                                 | `superseriousbusiness/gotosocial:{{ .Version }}`               |
+    | OS      | Architecture            | Support level                      | Binary archive                                                                                                                                                                     | Docker                                                         |
-    | Linux   | x86-32/i386 (32-bit)    | [linux_386.tar.gz](https://github.com/superseriousbusiness/gotosocial/releases/download/{{ .Tag }}/gotosocial_{{ .Version }}_linux_386.tar.gz)                                     | `superseriousbusiness/gotosocial:{{ .Version }}`               |
+    | ------- | ----------------------- | ---------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- |
-    | Linux   | Armv8/ARM64 (64-bit)    | [linux_arm64.tar.gz](https://github.com/superseriousbusiness/gotosocial/releases/download/{{ .Tag }}/gotosocial_{{ .Version }}_linux_arm64.tar.gz)                                 | `superseriousbusiness/gotosocial:{{ .Version }}`               |
+    | Linux   | x86-64/AMD64 (64-bit)   | 🟢 Full                            | [linux_amd64.tar.gz](https://github.com/superseriousbusiness/gotosocial/releases/download/{{ .Tag }}/gotosocial_{{ .Version }}_linux_amd64.tar.gz)                                 | `superseriousbusiness/gotosocial:{{ .Version }}`               |
-    | Linux   | Armv7/ARM32 (32-bit)    | [linux_armv7.tar.gz](https://github.com/superseriousbusiness/gotosocial/releases/download/{{ .Tag }}/gotosocial_{{ .Version }}_linux_armv7.tar.gz)                                 | `superseriousbusiness/gotosocial:{{ .Version }}`               |
+    | Linux   | Armv8/ARM64 (64-bit)    | 🟢 Full                            | [linux_arm64.tar.gz](https://github.com/superseriousbusiness/gotosocial/releases/download/{{ .Tag }}/gotosocial_{{ .Version }}_linux_arm64.tar.gz)                                 | `superseriousbusiness/gotosocial:{{ .Version }}`               |
-    | Linux   | Armv6/ARM32 (32-bit)    | [linux_armv6.tar.gz](https://github.com/superseriousbusiness/gotosocial/releases/download/{{ .Tag }}/gotosocial_{{ .Version }}_linux_armv6.tar.gz)                                 | `superseriousbusiness/gotosocial:{{ .Version }}`               |
+    | FreeBSD | x86-64/AMD64 (64-bit)   | 🟢 Full<sup>[1](#freebsd)</sup>    | [freebsd_amd64_moderncsqlite.tar.gz](https://github.com/superseriousbusiness/gotosocial/releases/download/{{ .Tag }}/gotosocial_{{ .Version }}_freebsd_amd64_moderncsqlite.tar.gz) | None provided                                                  |
-    | OpenBSD | x86-64/AMD64 (64-bit)   | [openbsd_amd64_moderncsqlite.tar.gz](https://github.com/superseriousbusiness/gotosocial/releases/download/{{ .Tag }}/gotosocial_{{ .Version }}_openbsd_amd64_moderncsqlite.tar.gz) | `superseriousbusiness/gotosocial:{{ .Version }}-moderncsqlite` |
+    | Linux   | x86-32/i386 (32-bit)    | 🟡 Partial<sup>[2](#32-bit)</sup>  | [linux_386_moderncsqlite.tar.gz](https://github.com/superseriousbusiness/gotosocial/releases/download/{{ .Tag }}/gotosocial_{{ .Version }}_linux_386_moderncsqlite.tar.gz)         | `superseriousbusiness/gotosocial:{{ .Version }}-moderncsqlite` |
-    | FreeBSD | x86-64/AMD64 (64-bit)   | [freebsd_amd64_moderncsqlite.tar.gz](https://github.com/superseriousbusiness/gotosocial/releases/download/{{ .Tag }}/gotosocial_{{ .Version }}_freebsd_amd64_moderncsqlite.tar.gz) | `superseriousbusiness/gotosocial:{{ .Version }}-moderncsqlite` |
+    | Linux   | Armv7/ARM32 (32-bit)    | 🟡 Partial<sup>[2](#32-bit)</sup>  | [linux_armv7_moderncsqlite.tar.gz](https://github.com/superseriousbusiness/gotosocial/releases/download/{{ .Tag }}/gotosocial_{{ .Version }}_linux_armv7_moderncsqlite.tar.gz)     | `superseriousbusiness/gotosocial:{{ .Version }}-moderncsqlite` |
+    | Linux   | Armv6/ARM32 (32-bit)    | 🟡 Partial<sup>[2](#32-bit)</sup>  | [linux_armv6_moderncsqlite.tar.gz](https://github.com/superseriousbusiness/gotosocial/releases/download/{{ .Tag }}/gotosocial_{{ .Version }}_linux_armv7_moderncsqlite.tar.gz)     | `superseriousbusiness/gotosocial:{{ .Version }}-moderncsqlite` |
+    | Linux   | x86-32/i386 (32-bit)    | 🟡 Partial<sup>[2](#32-bit)</sup>  | [linux_386_moderncsqlite.tar.gz](https://github.com/superseriousbusiness/gotosocial/releases/download/{{ .Tag }}/gotosocial_{{ .Version }}_linux_386_moderncsqlite.tar.gz)         | `superseriousbusiness/gotosocial:{{ .Version }}-moderncsqlite` |
+    
+    #### FreeBSD
+    
+    `moderncsqlite` version currently recommended, though you might have success with the regular WASM SQLite version.
+    
+    If running with regular WASM SQLite and having instability or memory issues, the following settings *may* help:
+    
+    ```yaml
+    db-max-open-conns-multiplier: 0
+    db-sqlite-journal-mode: "TRUNCATE"
+    db-sqlite-synchronous: "FULL"
+    ```
+    
+    #### 32-bit
+    
+    `moderncsqlite` version is needed, as performance with regular WASM SQLite is not guaranteed when running on 32-bit.
+    
+    Remote media processing will likely not work with reasonable performance, so you should set the following config variables to prevent download of remote media onto your instance:
+    
+    ```yaml
+    media-remote-max-size: 0
+    media-emoji-remote-max-size: 0
+    ``` 
 
 # https://goreleaser.com/customization/changelog/
 changelog:

Dockerfile

@@ -39,9 +39,10 @@ USER 1000:1000
 #
 # See https://docs.docker.com/engine/reference/builder/#workdir
 #
-# First make sure storage exists + is owned by 1000:1000, then go back
+# First make sure storage + cache exist and are owned by 1000:1000,
-# to just /gotosocial, where we'll run from
+# then go back to just /gotosocial, where we'll actually run from.
 WORKDIR "/gotosocial/storage"
+WORKDIR "/gotosocial/.cache"
 WORKDIR "/gotosocial"
 
 # copy the dist binary created by goreleaser or build.sh
@@ -51,5 +52,5 @@ COPY --chown=1000:1000 gotosocial /gotosocial/gotosocial
 COPY --chown=1000:1000 --from=bundler web /gotosocial/web
 COPY --chown=1000:1000 --from=swagger /go/src/github.com/superseriousbusiness/gotosocial/swagger.yaml web/assets/swagger.yaml
 
-VOLUME [ "/gotosocial/storage" ]
+VOLUME [ "/gotosocial/storage", "/gotosocial/.cache" ]
 ENTRYPOINT [ "/gotosocial/gotosocial", "server", "start" ]

README.md

@@ -41,7 +41,11 @@ Here's a screenshot of the instance landing page!
   - [OIDC integration](#oidc-integration)
   - [Backend-first design](#backend-first-design)
 - [Known Issues](#known-issues)
-- [Getting Started](#getting-started)
+- [Installing GoToSocial](#installing-gotosocial)
+  - [Supported Platforms](#supported-platforms)
+    - [FreeBSD](#freebsd)
+    - [32-bit](#32-bit)
+    - [OpenBSD](#openbsd)
   - [Stable Releases](#stable-releases)
   - [Snapshot Releases](#snapshot-releases)
     - [Docker](#docker)
@@ -258,9 +262,47 @@ Since every ActivityPub server implementation has a slightly different interpret
 
 ---
 
-## Getting Started
+## Installing GoToSocial
 
-Check our [releases page](https://github.com/superseriousbusiness/gotosocial/releases) and our [getting started](https://docs.gotosocial.org/en/latest/getting_started/) documentation.
+Check our [getting started](https://docs.gotosocial.org/en/latest/getting_started/) documentation! And have a peruse of our [releases page](https://github.com/superseriousbusiness/gotosocial/releases).
+
+<!--releases-start-->
+### Supported Platforms
+
+While we try to support a reasonable number of architectures and operating systems, it's not always possible to support a given platform due to library constraints or performance issues.
+
+Platforms that we don't officially support *may* still work, but we can't test or guarantee performance or stability.
+
+This is the current status of support offered by GoToSocial for different platforms (if something is unlisted it means we haven't checked yet so we don't know):
+
+| OS      | Architecture            | Support level                      | Binary archive | Docker container |
+| ------- | ----------------------- | ---------------------------------- | -------------- | ---------------- |
+| Linux   | x86-64/AMD64 (64-bit)   | 🟢 Full                            | Yes            | Yes              |
+| Linux   | Armv8/ARM64 (64-bit)    | 🟢 Full                            | Yes            | Yes              |
+| FreeBSD | x86-64/AMD64 (64-bit)   | 🟢 Full<sup>[1](#freebsd)</sup>    | Yes            | No               |
+| Linux   | x86-32/i386 (32-bit)    | 🟡 Partial<sup>[2](#32-bit)</sup>  | Yes            | Yes              |
+| Linux   | Armv7/ARM32 (32-bit)    | 🟡 Partial<sup>[2](#32-bit)</sup>  | Yes            | Yes              |
+| Linux   | Armv6/ARM32 (32-bit)    | 🟡 Partial<sup>[2](#32-bit)</sup>  | Yes            | Yes              |
+| Linux   | x86-32/i386 (32-bit)    | 🟡 Partial<sup>[2](#32-bit)</sup>  | Yes            | Yes              |
+| OpenBSD | Any                     | 🔴 None<sup>[3](#openbsd)</sup>    | No             | No               |
+
+#### FreeBSD
+
+Mostly works, just a few issues with WASM SQLite; check release notes carefully when installing on FreeBSD. If running with Postgres you should have no issues.
+
+#### 32-bit
+
+GtS doesn't work well on 32-bit systems like i386, or Armv6/v7, mainly due to performance of media decoding.
+
+We don't recommend running GtS on 32-bit, but you may have some success either turning off remote media processing altogether, or building a binary yourself with the totally **unsupported, experimental** [nowasm](https://docs.gotosocial.org/en/latest/advanced/builds/nowasm/) tag.
+
+For more guidance, check release notes when trying to install on 32-bit. 
+
+#### OpenBSD
+
+Marked as unsupported due to performance issues (high memory usage when idle, crashes while processing media).
+
+While we don't support running GtS on OpenBSD, you may have some success building a binary yourself with the totally **unsupported, experimental** [nowasm](https://docs.gotosocial.org/en/latest/advanced/builds/nowasm/) tag.
 
 ### Stable Releases
 
@@ -302,6 +344,7 @@ You can also deploy your own instance of GoToSocial with the help of:
 - [Ansible Playbook (MASH)](https://github.com/mother-of-all-self-hosting/mash-playbook): The playbook supports a many services, including GoToSocial. [Documentation](https://github.com/mother-of-all-self-hosting/mash-playbook/blob/main/docs/services/gotosocial.md)
 - [GoToSocial Helm Chart](https://github.com/fSocietySocial/charts/tree/main/charts/gotosocial) by [0hlov3](https://github.com/0hlov3).
 
+<!--releases-end-->
 ---
 
 ## Contributing

docs/advanced/builds/nowasm.md

@@ -0,0 +1,27 @@
+# Build without Wazero / WASM
+
+!!! Danger "This is unsupported"
+    We do not offer any kind of support for deployments of GoToSocial built with the `nowasm` tag described in this section. Such builds should be considered strictly experimental, and any issues that come when running them are none of our business! Please don't open issues on the repo looking for help debugging deployments of `nowasm` builds.
+
+On [supported platforms](../../getting_started/releases.md#supported-platforms), GoToSocial uses the WebAssembly runtime [Wazero](https://wazero.io/) to sandbox `ffmpeg`, `ffprobe`, and `sqlite3` WebAssembly binaries, allowing these applications to be packaged and run inside the GoToSocial binary, without requiring admins to install + manage any external dependencies.
+
+This has the advantage of making it easier for admins to maintain their GoToSocial instance, as their GtS binary is completely isolated from any changes to their system-installed `ffmpeg`, `ffprobe`, and `sqlite`. It's also a bit safer to run `ffmpeg` in this way, as GoToSocial wraps the `ffmpeg` binary in a very constrained file system that doesn't permit the `ffmpeg` binary to access any files other than the ones it's decoding + reencoding. In other words, GoToSocial on supported platforms offers most of the functionality of `ffmpeg` and so on, without some of the headaches.
+
+However, not all platforms are capable of running Wazero in the much-faster "compiler" mode, and have to fall back to the very slow (and resource-heavy) "interpreter" mode. See [this table](https://github.com/tetratelabs/wazero?tab=readme-ov-file#conformance) from Wazero for conformance.
+
+"Interpreter" mode runs so poorly for GoToSocial's use case that it's simply not feasible to run a GoToSocial instance in a stable manner on platforms that aren't 64-bit Linux or 64-bit FreeBSD, as all the memory and CPU get gobbled up by media processing.
+
+However! To enable folks to run **experimental, unsupported deployments of GoToSocial**, we expose the `nowasm` build tag, which can be used to compile a build of GoToSocial that does not use Wazero or WASM at all.
+
+A GoToSocial binary built with `nowasm` will use the [modernc version of SQLite](https://pkg.go.dev/modernc.org/sqlite) instead of the WASM one, and will use on-system `ffmpeg` and `ffprobe` binaries for media processing.
+
+To build GoToSocial with the `nowasm` tag, you can pass the tag into our convenience `build.sh` script like so:
+
+```bash
+GO_BUILDTAGS=nowasm ./scripts/build.sh
+```
+
+In order to run a version of GoToSocial built in this way, you must ensure that `ffmpeg` and `ffprobe` are installed on the host. This is usually as simple as running a command like `doas -u root pkg_add ffmpeg` (OpenBSD), or `sudo apt install ffmpeg` (Debian etc.). 
+
+!!! Danger "No really though, it's unsupported"
+    Again, if running builds of GoToSocial with `nowasm` works for your OS/Arch combination, that's great, but we do not support such builds and we won't be able to help debugging why something doesn't work.

docs/advanced/index.md

@@ -1,6 +1,6 @@
 # Advanced
 
-In this section we touch on a number of more advanced topics, primarily related around deploying, operating and tuning GoToSocial.
+In this section we touch on a number of more advanced topics, primarily related around building, deploying, operating and tuning GoToSocial.
 
 We consider these topics advanced because applying them incorrectly does have the possibility of causing client and federation issues. Applying any of these configuration changes may also make it harder for you to debug an issue with your GoToSocial instance if you don't understand the changes that you're making.
 

docs/configuration/index.md

@@ -126,3 +126,19 @@ This means in cases where you want to just try changing one thing, but don't wan
 Reasonable default values are provided for *most* of the configuration parameters, except in cases where a custom value is absolutely required.
 
 See the [example config file](https://github.com/superseriousbusiness/gotosocial/blob/main/example/config.yaml) for the default values, or run `gotosocial --help`.
+
+## `GTS_WAZERO_COMPILATION_CACHE`
+
+On startup, GoToSocial compiles embedded WebAssembly `ffmpeg` and `ffprobe` binaries into [Wazero](https://wazero.io/)-compatible modules, which are used for media processing without requiring any external dependencies.
+
+To speed up startup time of GoToSocial, you can cache the compiled modules between restarts so that GoToSocial doesn't have to compile them on every startup from scratch.
+
+You can instruct GoToSocial on where to store the Wazero artifacts by setting the environment variable `GTS_WAZERO_COMPILATION_CACHE` to a directory, which will be used by GtS to store two smallish artifacts of ~50MiB or so each (~100MiB total).
+
+For an example of this in action, see the [docker-compose.yaml](https://raw.githubusercontent.com/superseriousbusiness/gotosocial/main/example/docker-compose/docker-compose.yaml), and the [gotosocial.service](https://raw.githubusercontent.com/superseriousbusiness/gotosocial/main/example/gotosocial.service) example files.
+
+If you want to provide this value to GtS outside of systemd or Docker, you can do so in the following manner when starting up your GtS server:
+
+```bash
+GTS_WAZERO_COMPILATION_CACHE=~/gotosocial/.cache ./gotosocial --config-path ./config.yaml server start
+```

docs/federation/http_signatures.md

@@ -6,7 +6,7 @@ GoToSocial will also sign all outgoing `GET` and `POST` requests that it makes t
 
 This behavior is the equivalent of Mastodon's [AUTHORIZED_FETCH / "secure mode"](https://docs.joinmastodon.org/admin/config/#authorized_fetch).
 
-GoToSocial uses the [superseriousbusiness/httpsig](https://github.com/superseriousbusiness/httpsign) library (forked from go-fed) for signing outgoing requests, and for parsing and validating the signatures of incoming requests. This library strictly follows the [Cavage http signature RFC](https://datatracker.ietf.org/doc/html/draft-cavage-http-signatures-12), which is the same RFC used by other implementations like Mastodon, Pixelfed, Akkoma/Pleroma, etc. (This RFC has since been superceded by the [httpbis http signature RFC](https://datatracker.ietf.org/doc/html/draft-ietf-httpbis-message-signatures), but this is not yet widely implemented.)
+GoToSocial uses the [superseriousbusiness/httpsig](https://github.com/superseriousbusiness/httpsig) library (forked from go-fed) for signing outgoing requests, and for parsing and validating the signatures of incoming requests. This library strictly follows the [Cavage http signature RFC](https://datatracker.ietf.org/doc/html/draft-cavage-http-signatures-12), which is the same RFC used by other implementations like Mastodon, Pixelfed, Akkoma/Pleroma, etc. (This RFC has since been superceded by the [httpbis http signature RFC](https://datatracker.ietf.org/doc/html/draft-ietf-httpbis-message-signatures), but this is not yet widely implemented.)
 
 ## Query Parameters
 

docs/getting_started/installation/container.md

@@ -91,6 +91,32 @@ If you want to use [LetsEncrypt](../../configuration/tls.md) for TLS certificate
     
     For help translating variable names from the config.yaml file to environment variables, refer to the [configuration section](../../configuration/index.md#environment-variables).
 
+### Wazero Compilation Cache (optional)
+
+On startup, GoToSocial compiles embedded WebAssembly `ffmpeg` and `ffprobe` binaries into [Wazero](https://wazero.io/)-compatible modules, which are used for media processing without requiring any external dependencies.
+
+To speed up startup time of GoToSocial, you can cache the compiled modules between restarts so that GoToSocial doesn't have to compile them on every startup from scratch.
+
+If you'd like to do this in your Docker container, first create a `.cache` directory in your working folder to store the modules:
+
+```bash
+mkdir -p ~/gotosocial/.cache
+```
+
+Then, uncomment the second volume in the docker-compose.yaml file by removing the leading `#` symbol, so that instead of
+
+```yaml
+#- ~/gotosocial/.cache:/gotosocial/.cache
+```
+
+it reads
+
+```yaml
+- ~/gotosocial/.cache:/gotosocial/.cache
+```
+
+This will instruct Docker to mount the `~/gotosocial/.cache` directory at `/gotosocial/.cache` inside the Docker container.
+
 ## Start GoToSocial
 
 With those small changes out of the way, you can now start GoToSocial with the following command:

docs/getting_started/releases.md

@@ -1,47 +1,11 @@
 # Releases
 
-GoToSocial can be installed in a number of different ways. We publish official binary releases as well as container images. A number of third-party packages are maintained by different distributions and some people have created additional deployment tooling to make it easy to deploy GoToSocial yourself.
+GoToSocial can be installed in a number of different ways. We publish official binary releases as well as container images.
 
-## Binary releases
+A number of third-party packages are maintained by different distributions, and some people have created additional deployment tooling to make it easy to deploy GoToSocial yourself.
 
-We publish binary builds for Linux to [our GitHub project](https://github.com/superseriousbusiness/gotosocial/releases):
+{%
-
+  include "../../README.md"
-* 32-bit Intel/AMD (i386/x86)
+  start='<!--releases-start-->'
-* 64-bit Intel/AMD (amd64/x86_64)
+  end='<!--releases-end-->'
-* 32-bit ARM (v6 and v7)
+%}
-* 64-bit ARM64
-
-For FreeBSD we publish:
-
-* 64-bit Intel/AMD (amd64/x86_64)
-
-## Containers
-
-We also publish container images [on the Docker Hub](https://hub.docker.com/r/superseriousbusiness/gotosocial).
-
-Containers are released for the same Linux platforms as our binary releases, with the exception of 32-bit Intel/AMD.
-
-## Snapshots
-
-We publish snapshot binary builds and Docker images of whatever is currently on main.
-
-We always recommend using a stable release instead, but if you want to live on the edge (at your own risk!) then see the [snapshots](https://github.com/superseriousbusiness/gotosocial#snapshots) section on our GitHub repo for more information.
-
-## Third-party
-
-Some folks have created distribution packages for GoToSocial or additional tooling to aid in installing GoToSocial.
-
-### Distribution packages
-
-These packages are not maintained by GoToSocial, so please direct questions and issues to the repository maintainers (and donate to them!).
-
-[![Packaging status](https://repology.org/badge/vertical-allrepos/gotosocial.svg)](https://repology.org/project/gotosocial/versions)
-
-### Deployment tools
-
-You can deploy your own instance of GoToSocial with the help of:
-
-- [YunoHost GoToSocial Packaging](https://github.com/YunoHost-Apps/gotosocial_ynh) by [OniriCorpe](https://github.com/OniriCorpe).
-- [Ansible Playbook (MASH)](https://github.com/mother-of-all-self-hosting/mash-playbook): The playbook supports a many services, including GoToSocial. [Documentation](https://github.com/mother-of-all-self-hosting/mash-playbook/blob/main/docs/services/gotosocial.md)
-- GoToSocial Helm Charts:
-  - [GoToSocial Helm Chart](https://github.com/fSocietySocial/charts/tree/main/charts/gotosocial) by [0hlov3](https://github.com/0hlov3).

example/docker-compose/docker-compose.yaml

@@ -8,11 +8,21 @@ services:
     networks:
       - gotosocial
     environment:
+      # Change this to your actual host value.
       GTS_HOST: example.org
       GTS_DB_TYPE: sqlite
+      # Path in the GtS Docker container where
+      # the sqlite.db file will be stored.
       GTS_DB_ADDRESS: /gotosocial/storage/sqlite.db
+      # Change this to true if you're not running
+      # GoToSocial behind a reverse proxy.
       GTS_LETSENCRYPT_ENABLED: "false"
+      # Set your email address here if you
+      # want to receive letsencrypt notices.
       GTS_LETSENCRYPT_EMAIL_ADDRESS: ""
+      # Path in the GtS Docker container where the
+      # Wazero compilation cache will be stored.
+      GTS_WAZERO_COMPILATION_CACHE: /gotosocial/.cache
       ## For reverse proxy setups:
       # GTS_TRUSTED_PROXIES: "172.x.x.x"
       ## Set the timezone of your server:
@@ -24,7 +34,13 @@ services:
       ## For reverse proxy setups:
       #- "127.0.0.1:8080:8080"
     volumes:
+      # Your data volume, for your
+      # sqlite.db file and media files.
       - ~/gotosocial/data:/gotosocial/storage
+      # OPTIONAL: To mount volume for the WAZERO
+      # compilation cache, for speedier restart
+      # times, uncomment the below line:
+      #- ~/gotosocial/.cache:/gotosocial/.cache
     restart: "always"
 
 networks:

example/gotosocial.service

@@ -13,6 +13,15 @@ Group=gotosocial
 Type=exec
 Restart=on-failure
 
+# For speedier restart times, you can uncomment the following Environment line to have GoToSocial cache compiled
+# Wazero artifacts in the given directory between restarts, so that it doesn't need to compile on startup every time.
+#
+# You may need to change the exact path depending on where you've got GoToSocial installed, for example if you've
+# installed at "~/gotosocial" then change the value to "GTS_WAZERO_COMPILATION_CACHE=~/gotosocial/.cache".
+#
+# Whatever you do, make sure the dir exists and that the gotosocial user has permission to write + read from it.
+#Environment="GTS_WAZERO_COMPILATION_CACHE=/gotosocial/.cache"
+
 # change if your path to the GoToSocial binary is different
 ExecStart=/gotosocial/gotosocial --config-path config.yaml server start
 WorkingDirectory=/gotosocial

go.mod

@@ -42,7 +42,7 @@ require (
 	github.com/k3a/html2text v1.2.1
 	github.com/microcosm-cc/bluemonday v1.0.27
 	github.com/miekg/dns v1.1.62
-	github.com/minio/minio-go/v7 v7.0.77
+	github.com/minio/minio-go/v7 v7.0.78
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/ncruces/go-sqlite3 v0.19.0
 	github.com/oklog/ulid v1.3.1
@@ -63,7 +63,7 @@ require (
 	github.com/uptrace/bun/dialect/sqlitedialect v1.2.1
 	github.com/uptrace/bun/extra/bunotel v1.2.1
 	github.com/wagslane/go-password-validator v0.3.0
-	github.com/yuin/goldmark v1.7.4
+	github.com/yuin/goldmark v1.7.6
 	go.opentelemetry.io/otel v1.29.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.29.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0
@@ -158,7 +158,7 @@ require (
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect

go.sum

@@ -381,8 +381,8 @@ github.com/k3a/html2text v1.2.1/go.mod h1:ieEXykM67iT8lTvEWBh6fhpH4B23kB9OMKPdIB
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.10.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
-github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
@@ -413,8 +413,8 @@ github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ=
 github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.77 h1:GaGghJRg9nwDVlNbwYjSDJT1rqltQkBFDsypWX1v3Bw=
+github.com/minio/minio-go/v7 v7.0.78 h1:LqW2zy52fxnI4gg8C2oZviTaKHcBV36scS+RzJnxUFs=
-github.com/minio/minio-go/v7 v7.0.77/go.mod h1:AVM3IUN6WwKzmwBxVdjzhH8xq+f57JSbbvzqvUzR6eg=
+github.com/minio/minio-go/v7 v7.0.78/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
 github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
@@ -621,8 +621,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg=
+github.com/yuin/goldmark v1.7.6 h1:cZgJxVh5mL5cu8KOnwxvFJy5TFB0BHUskZZyq7TYbDg=
-github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
+github.com/yuin/goldmark v1.7.6/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
 gitlab.com/NyaaaWhatsUpDoc/sqlite v1.33.1-concurrency-workaround h1:pFMJnlc1PuH+jcVz4vz53vcpnoZG+NqFBr3qikDmEB4=
 gitlab.com/NyaaaWhatsUpDoc/sqlite v1.33.1-concurrency-workaround/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k=
 go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80=

internal/db/bundb/bundb.go

@@ -401,6 +401,18 @@ func maxOpenConns() int {
 	if multiplier < 1 {
 		return 1
 	}
+
+	// Specifically for SQLite databases with
+	// a journal mode of anything EXCEPT "wal",
+	// only 1 concurrent connection is supported.
+	if strings.ToLower(config.GetDbType()) == "sqlite" {
+		journalMode := config.GetDbSqliteJournalMode()
+		journalMode = strings.ToLower(journalMode)
+		if journalMode != "wal" {
+			return 1
+		}
+	}
+
 	return multiplier * runtime.GOMAXPROCS(0)
 }
 

internal/db/sqlite/driver.go

@@ -15,7 +15,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-//go:build !moderncsqlite3
+//go:build !moderncsqlite3 && !nowasm
 
 package sqlite
 

internal/db/sqlite/driver_moderncsqlite3.go

@@ -15,7 +15,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-//go:build moderncsqlite3
+//go:build moderncsqlite3 || nowasm
 
 package sqlite
 

internal/db/sqlite/errors.go

@@ -15,7 +15,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-//go:build !moderncsqlite3
+//go:build !moderncsqlite3 && !nowasm
 
 package sqlite
 

internal/db/sqlite/errors_moderncsqlite3.go

@@ -15,7 +15,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-//go:build moderncsqlite3
+//go:build moderncsqlite3 || nowasm
 
 package sqlite
 

internal/media/ffmpeg/args.go

@@ -0,0 +1,28 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ffmpeg
+
+import (
+	"codeberg.org/gruf/go-ffmpreg/wasm"
+)
+
+// Args encapsulates the passing of common
+// configuration options to run an instance
+// of a compiled WebAssembly module that is
+// run in a typical CLI manner.
+type Args = wasm.Args

internal/media/ffmpeg/exec_nowasm.go

@@ -0,0 +1,142 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build nowasm
+
+package ffmpeg
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/fs"
+	"os/exec"
+
+	"codeberg.org/gruf/go-ffmpreg/wasm"
+	"github.com/tetratelabs/wazero"
+	"github.com/tetratelabs/wazero/sys"
+)
+
+func init() {
+	fmt.Println("!! you are using an unsupported build configuration of gotosocial with WebAssembly disabled !!")
+	fmt.Println("!! please do not file bug reports regarding media processing with this configuration !!")
+	fmt.Println("!! it is also less secure; this does not enforce version checks on ffmpeg / ffprobe versions !!")
+}
+
+// runCmd will run 'name' with the given arguments, returning exit code or error.
+func runCmd(ctx context.Context, name string, args wasm.Args) (uint32, error) {
+	cmd := exec.CommandContext(ctx, name, args.Args...) //nolint:gosec
+
+	// Set provided std files.
+	cmd.Stdin = args.Stdin
+	cmd.Stdout = args.Stdout
+	cmd.Stderr = args.Stderr
+
+	if args.Config != nil {
+		// Gather some information
+		// from module config func.
+		var cfg falseModuleConfig
+		_ = args.Config(&cfg)
+
+		// Extract from conf.
+		cmd.Env = cfg.env
+	}
+
+	// Run prepared command, catching err type.
+	switch err := cmd.Run(); err := err.(type) {
+
+	// Extract code from
+	// any exit error type.
+	case *exec.ExitError:
+		rc := err.ExitCode()
+		return uint32(rc), err
+
+	default:
+		return 0, err
+	}
+}
+
+type falseModuleConfig struct{ env []string }
+
+func (cfg *falseModuleConfig) WithArgs(...string) wazero.ModuleConfig {
+	return cfg // noop
+}
+
+func (cfg *falseModuleConfig) WithEnv(key string, value string) wazero.ModuleConfig {
+	cfg.env = append(cfg.env, key+"="+value)
+	return cfg // noop
+}
+
+func (cfg *falseModuleConfig) WithFS(fs.FS) wazero.ModuleConfig {
+	return cfg // noop
+}
+
+func (cfg *falseModuleConfig) WithFSConfig(wazero.FSConfig) wazero.ModuleConfig {
+	return cfg // noop
+}
+
+func (cfg *falseModuleConfig) WithName(string) wazero.ModuleConfig {
+	return cfg // noop
+}
+
+func (cfg *falseModuleConfig) WithStartFunctions(...string) wazero.ModuleConfig {
+	return cfg // noop
+}
+
+func (cfg *falseModuleConfig) WithStderr(io.Writer) wazero.ModuleConfig {
+	return cfg // noop
+}
+
+func (cfg *falseModuleConfig) WithStdin(io.Reader) wazero.ModuleConfig {
+	return cfg // noop
+}
+
+func (cfg *falseModuleConfig) WithStdout(io.Writer) wazero.ModuleConfig {
+	return cfg // noop
+}
+
+func (cfg *falseModuleConfig) WithWalltime(sys.Walltime, sys.ClockResolution) wazero.ModuleConfig {
+	return cfg // noop
+}
+
+func (cfg *falseModuleConfig) WithSysWalltime() wazero.ModuleConfig {
+	return cfg // noop
+}
+
+func (cfg *falseModuleConfig) WithNanotime(sys.Nanotime, sys.ClockResolution) wazero.ModuleConfig {
+	return cfg // noop
+}
+
+func (cfg *falseModuleConfig) WithSysNanotime() wazero.ModuleConfig {
+	return cfg // noop
+}
+
+func (cfg *falseModuleConfig) WithNanosleep(sys.Nanosleep) wazero.ModuleConfig {
+	return cfg // noop
+}
+
+func (cfg *falseModuleConfig) WithOsyield(sys.Osyield) wazero.ModuleConfig {
+	return cfg // noop
+}
+
+func (cfg *falseModuleConfig) WithSysNanosleep() wazero.ModuleConfig {
+	return cfg // noop
+}
+
+func (cfg *falseModuleConfig) WithRandSource(io.Reader) wazero.ModuleConfig {
+	return cfg // noop
+}

internal/media/ffmpeg/ffmpeg.go

@@ -15,10 +15,14 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+//go:build !nowasm
+
 package ffmpeg
 
 import (
 	"context"
+
+	"codeberg.org/gruf/go-ffmpreg/wasm"
 )
 
 // ffmpegRunner limits the number of
@@ -36,5 +40,7 @@ func InitFfmpeg(ctx context.Context, max int) error {
 
 // Ffmpeg runs the given arguments with an instance of ffmpeg.
 func Ffmpeg(ctx context.Context, args Args) (uint32, error) {
-	return ffmpegRunner.Run(ctx, ffmpeg, args)
+	return ffmpegRunner.Run(ctx, func() (uint32, error) {
+		return wasm.Run(ctx, runtime, ffmpeg, args)
+	})
 }

internal/media/ffmpeg/ffmpeg_nowasm.go

@@ -0,0 +1,49 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build nowasm
+
+package ffmpeg
+
+import (
+	"context"
+	"os/exec"
+)
+
+// ffmpegRunner limits the number of
+// ffmpeg WebAssembly instances that
+// may be concurrently running, in
+// order to reduce memory usage.
+var ffmpegRunner runner
+
+// InitFfmpeg looks for a local copy of ffmpeg in path, and prepares
+// the runner to only allow max given concurrent running instances.
+func InitFfmpeg(ctx context.Context, max int) error {
+	_, err := exec.LookPath("ffmpeg")
+	if err != nil {
+		return err
+	}
+	ffmpegRunner.Init(max)
+	return nil
+}
+
+// Ffmpeg runs the given arguments with an instance of ffmpeg.
+func Ffmpeg(ctx context.Context, args Args) (uint32, error) {
+	return ffmpegRunner.Run(ctx, func() (uint32, error) {
+		return runCmd(ctx, "ffmpeg", args)
+	})
+}

internal/media/ffmpeg/ffprobe.go

@@ -15,10 +15,14 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+//go:build !nowasm
+
 package ffmpeg
 
 import (
 	"context"
+
+	"codeberg.org/gruf/go-ffmpreg/wasm"
 )
 
 // ffprobeRunner limits the number of
@@ -36,5 +40,7 @@ func InitFfprobe(ctx context.Context, max int) error {
 
 // Ffprobe runs the given arguments with an instance of ffprobe.
 func Ffprobe(ctx context.Context, args Args) (uint32, error) {
-	return ffprobeRunner.Run(ctx, ffprobe, args)
+	return ffmpegRunner.Run(ctx, func() (uint32, error) {
+		return wasm.Run(ctx, runtime, ffprobe, args)
+	})
 }

internal/media/ffmpeg/ffprobe_nowasm.go

@@ -0,0 +1,49 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build nowasm
+
+package ffmpeg
+
+import (
+	"context"
+	"os/exec"
+)
+
+// ffprobeRunner limits the number of
+// ffprobe WebAssembly instances that
+// may be concurrently running, in
+// order to reduce memory usage.
+var ffprobeRunner runner
+
+// InitFfprobe looks for a local copy of ffprobe in path, and prepares
+// the runner to only allow max given concurrent running instances.
+func InitFfprobe(ctx context.Context, max int) error {
+	_, err := exec.LookPath("ffprobe")
+	if err != nil {
+		return err
+	}
+	ffprobeRunner.Init(max)
+	return nil
+}
+
+// Ffprobe runs the given arguments with an instance of ffprobe.
+func Ffprobe(ctx context.Context, args Args) (uint32, error) {
+	return ffprobeRunner.Run(ctx, func() (uint32, error) {
+		return runCmd(ctx, "ffprobe", args)
+	})
+}

internal/media/ffmpeg/runner.go

@@ -19,9 +19,6 @@
 
 import (
 	"context"
-
-	"codeberg.org/gruf/go-ffmpreg/wasm"
-	"github.com/tetratelabs/wazero"
 )
 
 // runner simply abstracts away the complexities
@@ -53,7 +50,7 @@ func (r *runner) Init(n int) {
 
 // Run will attempt to pass the given compiled WebAssembly module with args to run(), waiting on
 // the receiving runner until a free slot is available to run an instance, (if a limit is enabled).
-func (r *runner) Run(ctx context.Context, cmod wazero.CompiledModule, args Args) (uint32, error) {
+func (r *runner) Run(ctx context.Context, run func() (uint32, error)) (uint32, error) {
 	select {
 	// Context canceled.
 	case <-ctx.Done():
@@ -66,6 +63,6 @@ func (r *runner) Run(ctx context.Context, cmod wazero.CompiledModule, args Args)
 	// Release slot back to pool on end.
 	defer func() { r.pool <- struct{}{} }()
 
-	// Pass to main module runner function.
+	// Call run.
-	return wasm.Run(ctx, runtime, cmod, args)
+	return run()
 }

internal/media/ffmpeg/wasm.go

@@ -15,6 +15,8 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+//go:build !nowasm
+
 package ffmpeg
 
 import (
@@ -41,12 +43,6 @@
 	ffprobe wazero.CompiledModule
 )
 
-// Args encapsulates the passing of common
-// configuration options to run an instance
-// of a compiled WebAssembly module that is
-// run in a typical CLI manner.
-type Args = wasm.Args
-
 // compileFfmpeg ensures the ffmpeg WebAssembly has been
 // pre-compiled into memory. If already compiled is a no-op.
 func compileFfmpeg(ctx context.Context) error {

mkdocs.yml

@@ -100,7 +100,7 @@ nav:
       - "configuration/advanced.md"
       - "configuration/observability.md"
   - "Advanced":
-    - "advanced/index.md"
+    - "Overview": "advanced/index.md"
     - "advanced/host-account-domain.md"
     - "advanced/outgoing-proxy.md"
     - "Caching":
@@ -117,6 +117,8 @@ nav:
     - "advanced/metrics.md"
     - "advanced/replicating-sqlite.md"
     - "advanced/sqlite-networked-storage.md"
+    - "Advanced builds":
+      - "advanced/builds/nowasm.md"
 
   - "Admin":
       - "admin/settings.md"

scripts/build.sh

@@ -23,6 +23,8 @@ GO_GCFLAGS=${GO_GCFLAGS-}
 # - debug:          enables /debug/pprof endpoint                                  (adds debug, at performance cost)
 # - debugenv:       enables /debug/pprof endpoint if DEBUG=1 env during runtime    (adds debug, at performance cost)
 # - moderncsqlite3: reverts to using the C-to-Go transpiled SQLite driver          (disables the WASM-based SQLite driver)
+# - nowasm:         [UNSUPPORTED] removes all WebAssembly from builds including 
+#                   ffmpeg, ffprobe and SQLite (instead falling back to modernc).
 log_exec env CGO_ENABLED=0 go build -trimpath -v \
                        -tags "${GO_BUILDTAGS}" \
                        -ldflags="${GO_LDFLAGS}" \

vendor/github.com/klauspost/compress/.goreleaser.yml

@@ -1,5 +1,5 @@
-# This is an example goreleaser.yaml file with some sane defaults.
+version: 2
-# Make sure to check the documentation at http://goreleaser.com
+
 before:
   hooks:
     - ./gen.sh
@@ -99,7 +99,7 @@ archives:
 checksum:
   name_template: 'checksums.txt'
 snapshot:
-  name_template: "{{ .Tag }}-next"
+  version_template: "{{ .Tag }}-next"
 changelog:
   sort: asc
   filters:

vendor/github.com/klauspost/compress/README.md

@@ -16,6 +16,27 @@ This package provides various compression algorithms.
 
 # changelog
 
+* Sep 23rd, 2024 - [1.17.10](https://github.com/klauspost/compress/releases/tag/v1.17.10)
+	* gzhttp: Add TransportAlwaysDecompress option. https://github.com/klauspost/compress/pull/978
+	* gzhttp: Add supported decompress request body by @mirecl in https://github.com/klauspost/compress/pull/1002
+	* s2: Add EncodeBuffer buffer recycling callback https://github.com/klauspost/compress/pull/982
+	* zstd: Improve memory usage on small streaming encodes https://github.com/klauspost/compress/pull/1007
+	* flate: read data written with partial flush by @vajexal in https://github.com/klauspost/compress/pull/996
+
+* Jun 12th, 2024 - [1.17.9](https://github.com/klauspost/compress/releases/tag/v1.17.9)
+	* s2: Reduce ReadFrom temporary allocations https://github.com/klauspost/compress/pull/949
+	* flate, zstd: Shave some bytes off amd64 matchLen by @greatroar in https://github.com/klauspost/compress/pull/963
+	* Upgrade zip/zlib to 1.22.4 upstream https://github.com/klauspost/compress/pull/970 https://github.com/klauspost/compress/pull/971
+	* zstd: BuildDict fails with RLE table https://github.com/klauspost/compress/pull/951
+
+* Apr 9th, 2024 - [1.17.8](https://github.com/klauspost/compress/releases/tag/v1.17.8)
+	* zstd: Reject blocks where reserved values are not 0 https://github.com/klauspost/compress/pull/885
+	* zstd: Add RLE detection+encoding https://github.com/klauspost/compress/pull/938
+
+* Feb 21st, 2024 - [1.17.7](https://github.com/klauspost/compress/releases/tag/v1.17.7)
+	* s2: Add AsyncFlush method: Complete the block without flushing by @Jille in https://github.com/klauspost/compress/pull/927
+	* s2: Fix literal+repeat exceeds dst crash https://github.com/klauspost/compress/pull/930
+  
 * Feb 5th, 2024 - [1.17.6](https://github.com/klauspost/compress/releases/tag/v1.17.6)
 	* zstd: Fix incorrect repeat coding in best mode https://github.com/klauspost/compress/pull/923
 	* s2: Fix DecodeConcurrent deadlock on errors https://github.com/klauspost/compress/pull/925
@@ -81,7 +102,7 @@ https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/comp
 	* zstd: Various minor improvements by @greatroar in https://github.com/klauspost/compress/pull/788 https://github.com/klauspost/compress/pull/794 https://github.com/klauspost/compress/pull/795
 	* s2: Fix huge block overflow https://github.com/klauspost/compress/pull/779
 	* s2: Allow CustomEncoder fallback https://github.com/klauspost/compress/pull/780
-	* gzhttp: Suppport ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799
+	* gzhttp: Support ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799
 
 * Mar 13, 2023 - [v1.16.1](https://github.com/klauspost/compress/releases/tag/v1.16.1)
 	* zstd: Speed up + improve best encoder by @greatroar in https://github.com/klauspost/compress/pull/776
@@ -136,7 +157,7 @@ https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/comp
 	* zstd: Add [WithDecodeAllCapLimit](https://pkg.go.dev/github.com/klauspost/compress@v1.15.10/zstd#WithDecodeAllCapLimit) https://github.com/klauspost/compress/pull/649
 	* Add Go 1.19 - deprecate Go 1.16  https://github.com/klauspost/compress/pull/651
 	* flate: Improve level 5+6 compression https://github.com/klauspost/compress/pull/656
-	* zstd: Improve "better" compresssion  https://github.com/klauspost/compress/pull/657
+	* zstd: Improve "better" compression  https://github.com/klauspost/compress/pull/657
 	* s2: Improve "best" compression https://github.com/klauspost/compress/pull/658
 	* s2: Improve "better" compression. https://github.com/klauspost/compress/pull/635
 	* s2: Slightly faster non-assembly decompression https://github.com/klauspost/compress/pull/646
@@ -339,7 +360,7 @@ While the release has been extensively tested, it is recommended to testing when
 	* s2: Fix binaries.
 
 * Feb 25, 2021 (v1.11.8)
-	* s2: Fixed occational out-of-bounds write on amd64. Upgrade recommended.
+	* s2: Fixed occasional out-of-bounds write on amd64. Upgrade recommended.
 	* s2: Add AMD64 assembly for better mode. 25-50% faster. [#315](https://github.com/klauspost/compress/pull/315)
 	* s2: Less upfront decoder allocation. [#322](https://github.com/klauspost/compress/pull/322)
 	* zstd: Faster "compression" of incompressible data. [#314](https://github.com/klauspost/compress/pull/314)
@@ -518,7 +539,7 @@ While the release has been extensively tested, it is recommended to testing when
 * Feb 19, 2016: Faster bit writer, level -2 is 15% faster, level 1 is 4% faster.
 * Feb 19, 2016: Handle small payloads faster in level 1-3.
 * Feb 19, 2016: Added faster level 2 + 3 compression modes.
-* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progresssion in terms of compression. New default level is 5.
+* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progression in terms of compression. New default level is 5.
 * Feb 14, 2016: Snappy: Merge upstream changes. 
 * Feb 14, 2016: Snappy: Fix aggressive skipping.
 * Feb 14, 2016: Snappy: Update benchmark.

vendor/github.com/klauspost/compress/fse/decompress.go

@@ -15,7 +15,7 @@
 // It is possible, but by no way guaranteed that corrupt data will
 // return an error.
 // It is up to the caller to verify integrity of the returned data.
-// Use a predefined Scrach to set maximum acceptable output size.
+// Use a predefined Scratch to set maximum acceptable output size.
 func Decompress(b []byte, s *Scratch) ([]byte, error) {
 	s, err := s.prepare(b)
 	if err != nil {

vendor/github.com/klauspost/compress/huff0/decompress.go

@@ -1136,7 +1136,7 @@ func (s *Scratch) matches(ct cTable, w io.Writer) {
 			errs++
 		}
 		if errs > 0 {
-			fmt.Fprintf(w, "%d errros in base, stopping\n", errs)
+			fmt.Fprintf(w, "%d errors in base, stopping\n", errs)
 			continue
 		}
 		// Ensure that all combinations are covered.
@@ -1152,7 +1152,7 @@ func (s *Scratch) matches(ct cTable, w io.Writer) {
 				errs++
 			}
 			if errs > 20 {
-				fmt.Fprintf(w, "%d errros, stopping\n", errs)
+				fmt.Fprintf(w, "%d errors, stopping\n", errs)
 				break
 			}
 		}

vendor/github.com/klauspost/compress/s2/encode.go

@@ -9,6 +9,9 @@
 	"encoding/binary"
 	"math"
 	"math/bits"
+	"sync"
+
+	"github.com/klauspost/compress/internal/race"
 )
 
 // Encode returns the encoded form of src. The returned slice may be a sub-
@@ -52,6 +55,8 @@ func Encode(dst, src []byte) []byte {
 	return dst[:d]
 }
 
+var estblockPool [2]sync.Pool
+
 // EstimateBlockSize will perform a very fast compression
 // without outputting the result and return the compressed output size.
 // The function returns -1 if no improvement could be achieved.
@@ -61,9 +66,25 @@ func EstimateBlockSize(src []byte) (d int) {
 		return -1
 	}
 	if len(src) <= 1024 {
-		d = calcBlockSizeSmall(src)
+		const sz, pool = 2048, 0
+		tmp, ok := estblockPool[pool].Get().(*[sz]byte)
+		if !ok {
+			tmp = &[sz]byte{}
+		}
+		race.WriteSlice(tmp[:])
+		defer estblockPool[pool].Put(tmp)
+
+		d = calcBlockSizeSmall(src, tmp)
 	} else {
-		d = calcBlockSize(src)
+		const sz, pool = 32768, 1
+		tmp, ok := estblockPool[pool].Get().(*[sz]byte)
+		if !ok {
+			tmp = &[sz]byte{}
+		}
+		race.WriteSlice(tmp[:])
+		defer estblockPool[pool].Put(tmp)
+
+		d = calcBlockSize(src, tmp)
 	}
 
 	if d == 0 {

vendor/github.com/klauspost/compress/s2/encode_amd64.go

@@ -3,10 +3,16 @@
 
 package s2
 
-import "github.com/klauspost/compress/internal/race"
+import (
+	"sync"
+
+	"github.com/klauspost/compress/internal/race"
+)
 
 const hasAmd64Asm = true
 
+var encPools [4]sync.Pool
+
 // encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It
 // assumes that the varint-encoded length of the decompressed bytes has already
 // been written.
@@ -29,23 +35,60 @@ func encodeBlock(dst, src []byte) (d int) {
 	)
 
 	if len(src) >= 4<<20 {
-		return encodeBlockAsm(dst, src)
+		const sz, pool = 65536, 0
+		tmp, ok := encPools[pool].Get().(*[sz]byte)
+		if !ok {
+			tmp = &[sz]byte{}
+		}
+		race.WriteSlice(tmp[:])
+		defer encPools[pool].Put(tmp)
+		return encodeBlockAsm(dst, src, tmp)
 	}
 	if len(src) >= limit12B {
-		return encodeBlockAsm4MB(dst, src)
+		const sz, pool = 65536, 0
+		tmp, ok := encPools[pool].Get().(*[sz]byte)
+		if !ok {
+			tmp = &[sz]byte{}
+		}
+		race.WriteSlice(tmp[:])
+		defer encPools[pool].Put(tmp)
+		return encodeBlockAsm4MB(dst, src, tmp)
 	}
 	if len(src) >= limit10B {
-		return encodeBlockAsm12B(dst, src)
+		const sz, pool = 16384, 1
+		tmp, ok := encPools[pool].Get().(*[sz]byte)
+		if !ok {
+			tmp = &[sz]byte{}
+		}
+		race.WriteSlice(tmp[:])
+		defer encPools[pool].Put(tmp)
+		return encodeBlockAsm12B(dst, src, tmp)
 	}
 	if len(src) >= limit8B {
-		return encodeBlockAsm10B(dst, src)
+		const sz, pool = 4096, 2
+		tmp, ok := encPools[pool].Get().(*[sz]byte)
+		if !ok {
+			tmp = &[sz]byte{}
+		}
+		race.WriteSlice(tmp[:])
+		defer encPools[pool].Put(tmp)
+		return encodeBlockAsm10B(dst, src, tmp)
 	}
 	if len(src) < minNonLiteralBlockSize {
 		return 0
 	}
-	return encodeBlockAsm8B(dst, src)
+	const sz, pool = 1024, 3
+	tmp, ok := encPools[pool].Get().(*[sz]byte)
+	if !ok {
+		tmp = &[sz]byte{}
+	}
+	race.WriteSlice(tmp[:])
+	defer encPools[pool].Put(tmp)
+	return encodeBlockAsm8B(dst, src, tmp)
 }
 
+var encBetterPools [5]sync.Pool
+
 // encodeBlockBetter encodes a non-empty src to a guaranteed-large-enough dst. It
 // assumes that the varint-encoded length of the decompressed bytes has already
 // been written.
@@ -68,21 +111,59 @@ func encodeBlockBetter(dst, src []byte) (d int) {
 	)
 
 	if len(src) > 4<<20 {
-		return encodeBetterBlockAsm(dst, src)
+		const sz, pool = 589824, 0
+		tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
+		if !ok {
+			tmp = &[sz]byte{}
+		}
+		race.WriteSlice(tmp[:])
+		defer encBetterPools[pool].Put(tmp)
+		return encodeBetterBlockAsm(dst, src, tmp)
 	}
 	if len(src) >= limit12B {
-		return encodeBetterBlockAsm4MB(dst, src)
+		const sz, pool = 589824, 0
+		tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
+		if !ok {
+			tmp = &[sz]byte{}
+		}
+		race.WriteSlice(tmp[:])
+		defer encBetterPools[pool].Put(tmp)
+
+		return encodeBetterBlockAsm4MB(dst, src, tmp)
 	}
 	if len(src) >= limit10B {
-		return encodeBetterBlockAsm12B(dst, src)
+		const sz, pool = 81920, 0
+		tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
+		if !ok {
+			tmp = &[sz]byte{}
+		}
+		race.WriteSlice(tmp[:])
+		defer encBetterPools[pool].Put(tmp)
+
+		return encodeBetterBlockAsm12B(dst, src, tmp)
 	}
 	if len(src) >= limit8B {
-		return encodeBetterBlockAsm10B(dst, src)
+		const sz, pool = 20480, 1
+		tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
+		if !ok {
+			tmp = &[sz]byte{}
+		}
+		race.WriteSlice(tmp[:])
+		defer encBetterPools[pool].Put(tmp)
+		return encodeBetterBlockAsm10B(dst, src, tmp)
 	}
 	if len(src) < minNonLiteralBlockSize {
 		return 0
 	}
-	return encodeBetterBlockAsm8B(dst, src)
+
+	const sz, pool = 5120, 2
+	tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
+	if !ok {
+		tmp = &[sz]byte{}
+	}
+	race.WriteSlice(tmp[:])
+	defer encBetterPools[pool].Put(tmp)
+	return encodeBetterBlockAsm8B(dst, src, tmp)
 }
 
 // encodeBlockSnappy encodes a non-empty src to a guaranteed-large-enough dst. It
@@ -105,22 +186,57 @@ func encodeBlockSnappy(dst, src []byte) (d int) {
 		// Use 8 bit table when less than...
 		limit8B = 512
 	)
-	if len(src) >= 64<<10 {
+	if len(src) > 65536 {
-		return encodeSnappyBlockAsm(dst, src)
+		const sz, pool = 65536, 0
+		tmp, ok := encPools[pool].Get().(*[sz]byte)
+		if !ok {
+			tmp = &[sz]byte{}
+		}
+		race.WriteSlice(tmp[:])
+		defer encPools[pool].Put(tmp)
+		return encodeSnappyBlockAsm(dst, src, tmp)
 	}
 	if len(src) >= limit12B {
-		return encodeSnappyBlockAsm64K(dst, src)
+		const sz, pool = 65536, 0
+		tmp, ok := encPools[pool].Get().(*[sz]byte)
+		if !ok {
+			tmp = &[sz]byte{}
+		}
+		race.WriteSlice(tmp[:])
+		defer encPools[pool].Put(tmp)
+		return encodeSnappyBlockAsm64K(dst, src, tmp)
 	}
 	if len(src) >= limit10B {
-		return encodeSnappyBlockAsm12B(dst, src)
+		const sz, pool = 16384, 1
+		tmp, ok := encPools[pool].Get().(*[sz]byte)
+		if !ok {
+			tmp = &[sz]byte{}
+		}
+		race.WriteSlice(tmp[:])
+		defer encPools[pool].Put(tmp)
+		return encodeSnappyBlockAsm12B(dst, src, tmp)
 	}
 	if len(src) >= limit8B {
-		return encodeSnappyBlockAsm10B(dst, src)
+		const sz, pool = 4096, 2
+		tmp, ok := encPools[pool].Get().(*[sz]byte)
+		if !ok {
+			tmp = &[sz]byte{}
+		}
+		race.WriteSlice(tmp[:])
+		defer encPools[pool].Put(tmp)
+		return encodeSnappyBlockAsm10B(dst, src, tmp)
 	}
 	if len(src) < minNonLiteralBlockSize {
 		return 0
 	}
-	return encodeSnappyBlockAsm8B(dst, src)
+	const sz, pool = 1024, 3
+	tmp, ok := encPools[pool].Get().(*[sz]byte)
+	if !ok {
+		tmp = &[sz]byte{}
+	}
+	race.WriteSlice(tmp[:])
+	defer encPools[pool].Put(tmp)
+	return encodeSnappyBlockAsm8B(dst, src, tmp)
 }
 
 // encodeBlockSnappy encodes a non-empty src to a guaranteed-large-enough dst. It
@@ -143,20 +259,59 @@ func encodeBlockBetterSnappy(dst, src []byte) (d int) {
 		// Use 8 bit table when less than...
 		limit8B = 512
 	)
-	if len(src) >= 64<<10 {
+	if len(src) > 65536 {
-		return encodeSnappyBetterBlockAsm(dst, src)
+		const sz, pool = 589824, 0
+		tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
+		if !ok {
+			tmp = &[sz]byte{}
 		}
+		race.WriteSlice(tmp[:])
+		defer encBetterPools[pool].Put(tmp)
+		return encodeSnappyBetterBlockAsm(dst, src, tmp)
+	}
+
 	if len(src) >= limit12B {
-		return encodeSnappyBetterBlockAsm64K(dst, src)
+		const sz, pool = 294912, 4
+		tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
+		if !ok {
+			tmp = &[sz]byte{}
+		}
+		race.WriteSlice(tmp[:])
+		defer encBetterPools[pool].Put(tmp)
+
+		return encodeSnappyBetterBlockAsm64K(dst, src, tmp)
 	}
 	if len(src) >= limit10B {
-		return encodeSnappyBetterBlockAsm12B(dst, src)
+		const sz, pool = 81920, 0
+		tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
+		if !ok {
+			tmp = &[sz]byte{}
+		}
+		race.WriteSlice(tmp[:])
+		defer encBetterPools[pool].Put(tmp)
+
+		return encodeSnappyBetterBlockAsm12B(dst, src, tmp)
 	}
 	if len(src) >= limit8B {
-		return encodeSnappyBetterBlockAsm10B(dst, src)
+		const sz, pool = 20480, 1
+		tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
+		if !ok {
+			tmp = &[sz]byte{}
+		}
+		race.WriteSlice(tmp[:])
+		defer encBetterPools[pool].Put(tmp)
+		return encodeSnappyBetterBlockAsm10B(dst, src, tmp)
 	}
 	if len(src) < minNonLiteralBlockSize {
 		return 0
 	}
-	return encodeSnappyBetterBlockAsm8B(dst, src)
+
+	const sz, pool = 5120, 2
+	tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
+	if !ok {
+		tmp = &[sz]byte{}
+	}
+	race.WriteSlice(tmp[:])
+	defer encBetterPools[pool].Put(tmp)
+	return encodeSnappyBetterBlockAsm8B(dst, src, tmp)
 }

vendor/github.com/klauspost/compress/s2/encode_go.go

@@ -317,7 +317,7 @@ func matchLen(a []byte, b []byte) int {
 }
 
 // input must be > inputMargin
-func calcBlockSize(src []byte) (d int) {
+func calcBlockSize(src []byte, _ *[32768]byte) (d int) {
 	// Initialize the hash table.
 	const (
 		tableBits    = 13
@@ -503,7 +503,7 @@ func calcBlockSize(src []byte) (d int) {
 }
 
 // length must be > inputMargin.
-func calcBlockSizeSmall(src []byte) (d int) {
+func calcBlockSizeSmall(src []byte, _ *[2048]byte) (d int) {
 	// Initialize the hash table.
 	const (
 		tableBits    = 9

vendor/github.com/klauspost/compress/s2/encodeblock_amd64.go

@@ -11,154 +11,154 @@ func _dummy_()
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeBlockAsm(dst []byte, src []byte) int
+func encodeBlockAsm(dst []byte, src []byte, tmp *[65536]byte) int
 
 // encodeBlockAsm4MB encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 4194304 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeBlockAsm4MB(dst []byte, src []byte) int
+func encodeBlockAsm4MB(dst []byte, src []byte, tmp *[65536]byte) int
 
 // encodeBlockAsm12B encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 16383 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeBlockAsm12B(dst []byte, src []byte) int
+func encodeBlockAsm12B(dst []byte, src []byte, tmp *[16384]byte) int
 
 // encodeBlockAsm10B encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 4095 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeBlockAsm10B(dst []byte, src []byte) int
+func encodeBlockAsm10B(dst []byte, src []byte, tmp *[4096]byte) int
 
 // encodeBlockAsm8B encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 511 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeBlockAsm8B(dst []byte, src []byte) int
+func encodeBlockAsm8B(dst []byte, src []byte, tmp *[1024]byte) int
 
 // encodeBetterBlockAsm encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 4294967295 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeBetterBlockAsm(dst []byte, src []byte) int
+func encodeBetterBlockAsm(dst []byte, src []byte, tmp *[589824]byte) int
 
 // encodeBetterBlockAsm4MB encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 4194304 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeBetterBlockAsm4MB(dst []byte, src []byte) int
+func encodeBetterBlockAsm4MB(dst []byte, src []byte, tmp *[589824]byte) int
 
 // encodeBetterBlockAsm12B encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 16383 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeBetterBlockAsm12B(dst []byte, src []byte) int
+func encodeBetterBlockAsm12B(dst []byte, src []byte, tmp *[81920]byte) int
 
 // encodeBetterBlockAsm10B encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 4095 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeBetterBlockAsm10B(dst []byte, src []byte) int
+func encodeBetterBlockAsm10B(dst []byte, src []byte, tmp *[20480]byte) int
 
 // encodeBetterBlockAsm8B encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 511 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeBetterBlockAsm8B(dst []byte, src []byte) int
+func encodeBetterBlockAsm8B(dst []byte, src []byte, tmp *[5120]byte) int
 
 // encodeSnappyBlockAsm encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 4294967295 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeSnappyBlockAsm(dst []byte, src []byte) int
+func encodeSnappyBlockAsm(dst []byte, src []byte, tmp *[65536]byte) int
 
 // encodeSnappyBlockAsm64K encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 65535 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeSnappyBlockAsm64K(dst []byte, src []byte) int
+func encodeSnappyBlockAsm64K(dst []byte, src []byte, tmp *[65536]byte) int
 
 // encodeSnappyBlockAsm12B encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 16383 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeSnappyBlockAsm12B(dst []byte, src []byte) int
+func encodeSnappyBlockAsm12B(dst []byte, src []byte, tmp *[16384]byte) int
 
 // encodeSnappyBlockAsm10B encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 4095 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeSnappyBlockAsm10B(dst []byte, src []byte) int
+func encodeSnappyBlockAsm10B(dst []byte, src []byte, tmp *[4096]byte) int
 
 // encodeSnappyBlockAsm8B encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 511 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeSnappyBlockAsm8B(dst []byte, src []byte) int
+func encodeSnappyBlockAsm8B(dst []byte, src []byte, tmp *[1024]byte) int
 
 // encodeSnappyBetterBlockAsm encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 4294967295 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeSnappyBetterBlockAsm(dst []byte, src []byte) int
+func encodeSnappyBetterBlockAsm(dst []byte, src []byte, tmp *[589824]byte) int
 
 // encodeSnappyBetterBlockAsm64K encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 65535 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeSnappyBetterBlockAsm64K(dst []byte, src []byte) int
+func encodeSnappyBetterBlockAsm64K(dst []byte, src []byte, tmp *[294912]byte) int
 
 // encodeSnappyBetterBlockAsm12B encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 16383 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeSnappyBetterBlockAsm12B(dst []byte, src []byte) int
+func encodeSnappyBetterBlockAsm12B(dst []byte, src []byte, tmp *[81920]byte) int
 
 // encodeSnappyBetterBlockAsm10B encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 4095 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeSnappyBetterBlockAsm10B(dst []byte, src []byte) int
+func encodeSnappyBetterBlockAsm10B(dst []byte, src []byte, tmp *[20480]byte) int
 
 // encodeSnappyBetterBlockAsm8B encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 511 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func encodeSnappyBetterBlockAsm8B(dst []byte, src []byte) int
+func encodeSnappyBetterBlockAsm8B(dst []byte, src []byte, tmp *[5120]byte) int
 
 // calcBlockSize encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 4294967295 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func calcBlockSize(src []byte) int
+func calcBlockSize(src []byte, tmp *[32768]byte) int
 
 // calcBlockSizeSmall encodes a non-empty src to a guaranteed-large-enough dst.
 // Maximum input 1024 bytes.
 // It assumes that the varint-encoded length of the decompressed bytes has already been written.
 //
 //go:noescape
-func calcBlockSizeSmall(src []byte) int
+func calcBlockSizeSmall(src []byte, tmp *[2048]byte) int
 
 // emitLiteral writes a literal chunk and returns the number of bytes written.
 //

vendor/github.com/klauspost/compress/s2/writer.go

@@ -83,11 +83,14 @@ type Writer struct {
 	snappy            bool
 	flushOnWrite      bool
 	appendIndex       bool
+	bufferCB          func([]byte)
 	level             uint8
 }
 
 type result struct {
 	b []byte
+	// return when writing
+	ret []byte
 	// Uncompressed start offset
 	startOffset int64
 }
@@ -146,6 +149,10 @@ func (w *Writer) Reset(writer io.Writer) {
 		for write := range toWrite {
 			// Wait for the data to be available.
 			input := <-write
+			if input.ret != nil && w.bufferCB != nil {
+				w.bufferCB(input.ret)
+				input.ret = nil
+			}
 			in := input.b
 			if len(in) > 0 {
 				if w.err(nil) == nil {
@@ -341,7 +348,8 @@ func (w *Writer) AddSkippableBlock(id uint8, data []byte) (err error) {
 // but the input buffer cannot be written to by the caller
 // until Flush or Close has been called when concurrency != 1.
 //
-// If you cannot control that, use the regular Write function.
+// Use the WriterBufferDone to receive a callback when the buffer is done
+// Processing.
 //
 // Note that input is not buffered.
 // This means that each write will result in discrete blocks being created.
@@ -364,6 +372,9 @@ func (w *Writer) EncodeBuffer(buf []byte) (err error) {
 	}
 	if w.concurrency == 1 {
 		_, err := w.writeSync(buf)
+		if w.bufferCB != nil {
+			w.bufferCB(buf)
+		}
 		return err
 	}
 
@@ -378,7 +389,7 @@ func (w *Writer) EncodeBuffer(buf []byte) (err error) {
 			hWriter <- result{startOffset: w.uncompWritten, b: magicChunkBytes}
 		}
 	}
-
+	orgBuf := buf
 	for len(buf) > 0 {
 		// Cut input.
 		uncompressed := buf
@@ -397,6 +408,9 @@ func (w *Writer) EncodeBuffer(buf []byte) (err error) {
 			startOffset: w.uncompWritten,
 		}
 		w.uncompWritten += int64(len(uncompressed))
+		if len(buf) == 0 && w.bufferCB != nil {
+			res.ret = orgBuf
+		}
 		go func() {
 			race.ReadSlice(uncompressed)
 
@@ -922,7 +936,7 @@ func WriterBetterCompression() WriterOption {
 }
 
 // WriterBestCompression will enable better compression.
-// EncodeBetter compresses better than Encode but typically with a
+// EncodeBest compresses better than Encode but typically with a
 // big speed decrease on compression.
 func WriterBestCompression() WriterOption {
 	return func(w *Writer) error {
@@ -941,6 +955,17 @@ func WriterUncompressed() WriterOption {
 	}
 }
 
+// WriterBufferDone will perform a callback when EncodeBuffer has finished
+// writing a buffer to the output and the buffer can safely be reused.
+// If the buffer was split into several blocks, it will be sent after the last block.
+// Callbacks will not be done concurrently.
+func WriterBufferDone(fn func(b []byte)) WriterOption {
+	return func(w *Writer) error {
+		w.bufferCB = fn
+		return nil
+	}
+}
+
 // WriterBlockSize allows to override the default block size.
 // Blocks will be this size or smaller.
 // Minimum size is 4KB and maximum size is 4MB.

vendor/github.com/klauspost/compress/zstd/blockdec.go

@@ -598,7 +598,9 @@ func (b *blockDec) prepareSequences(in []byte, hist *history) (err error) {
 					printf("RLE set to 0x%x, code: %v", symb, v)
 				}
 			case compModeFSE:
+				if debugDecoder {
 					println("Reading table for", tableIndex(i))
+				}
 				if seq.fse == nil || seq.fse.preDefined {
 					seq.fse = fseDecoderPool.Get().(*fseDecoder)
 				}

vendor/github.com/klauspost/compress/zstd/enc_better.go

@@ -179,9 +179,9 @@ func (e *betterFastEncoder) Encode(blk *blockEnc, src []byte) {
 				if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) {
 					// Consider history as well.
 					var seq seq
-					lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src)
+					length := 4 + e.matchlen(s+4+repOff, repIndex+4, src)
 
-					seq.matchLen = uint32(lenght - zstdMinMatch)
+					seq.matchLen = uint32(length - zstdMinMatch)
 
 					// We might be able to match backwards.
 					// Extend as long as we can.
@@ -210,12 +210,12 @@ func (e *betterFastEncoder) Encode(blk *blockEnc, src []byte) {
 
 					// Index match start+1 (long) -> s - 1
 					index0 := s + repOff
-					s += lenght + repOff
+					s += length + repOff
 
 					nextEmit = s
 					if s >= sLimit {
 						if debugEncoder {
-							println("repeat ended", s, lenght)
+							println("repeat ended", s, length)
 
 						}
 						break encodeLoop
@@ -241,9 +241,9 @@ func (e *betterFastEncoder) Encode(blk *blockEnc, src []byte) {
 				if false && repIndex >= 0 && load6432(src, repIndex) == load6432(src, s+repOff) {
 					// Consider history as well.
 					var seq seq
-					lenght := 8 + e.matchlen(s+8+repOff2, repIndex+8, src)
+					length := 8 + e.matchlen(s+8+repOff2, repIndex+8, src)
 
-					seq.matchLen = uint32(lenght - zstdMinMatch)
+					seq.matchLen = uint32(length - zstdMinMatch)
 
 					// We might be able to match backwards.
 					// Extend as long as we can.
@@ -270,11 +270,11 @@ func (e *betterFastEncoder) Encode(blk *blockEnc, src []byte) {
 					}
 					blk.sequences = append(blk.sequences, seq)
 
-					s += lenght + repOff2
+					s += length + repOff2
 					nextEmit = s
 					if s >= sLimit {
 						if debugEncoder {
-							println("repeat ended", s, lenght)
+							println("repeat ended", s, length)
 
 						}
 						break encodeLoop
@@ -708,9 +708,9 @@ func (e *betterFastEncoderDict) Encode(blk *blockEnc, src []byte) {
 				if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) {
 					// Consider history as well.
 					var seq seq
-					lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src)
+					length := 4 + e.matchlen(s+4+repOff, repIndex+4, src)
 
-					seq.matchLen = uint32(lenght - zstdMinMatch)
+					seq.matchLen = uint32(length - zstdMinMatch)
 
 					// We might be able to match backwards.
 					// Extend as long as we can.
@@ -738,12 +738,12 @@ func (e *betterFastEncoderDict) Encode(blk *blockEnc, src []byte) {
 					blk.sequences = append(blk.sequences, seq)
 
 					// Index match start+1 (long) -> s - 1
-					s += lenght + repOff
+					s += length + repOff
 
 					nextEmit = s
 					if s >= sLimit {
 						if debugEncoder {
-							println("repeat ended", s, lenght)
+							println("repeat ended", s, length)
 
 						}
 						break encodeLoop
@@ -772,9 +772,9 @@ func (e *betterFastEncoderDict) Encode(blk *blockEnc, src []byte) {
 				if false && repIndex >= 0 && load6432(src, repIndex) == load6432(src, s+repOff) {
 					// Consider history as well.
 					var seq seq
-					lenght := 8 + e.matchlen(s+8+repOff2, repIndex+8, src)
+					length := 8 + e.matchlen(s+8+repOff2, repIndex+8, src)
 
-					seq.matchLen = uint32(lenght - zstdMinMatch)
+					seq.matchLen = uint32(length - zstdMinMatch)
 
 					// We might be able to match backwards.
 					// Extend as long as we can.
@@ -801,11 +801,11 @@ func (e *betterFastEncoderDict) Encode(blk *blockEnc, src []byte) {
 					}
 					blk.sequences = append(blk.sequences, seq)
 
-					s += lenght + repOff2
+					s += length + repOff2
 					nextEmit = s
 					if s >= sLimit {
 						if debugEncoder {
-							println("repeat ended", s, lenght)
+							println("repeat ended", s, length)
 
 						}
 						break encodeLoop

vendor/github.com/klauspost/compress/zstd/enc_dfast.go

@@ -138,9 +138,9 @@ func (e *doubleFastEncoder) Encode(blk *blockEnc, src []byte) {
 				if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) {
 					// Consider history as well.
 					var seq seq
-					lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src)
+					length := 4 + e.matchlen(s+4+repOff, repIndex+4, src)
 
-					seq.matchLen = uint32(lenght - zstdMinMatch)
+					seq.matchLen = uint32(length - zstdMinMatch)
 
 					// We might be able to match backwards.
 					// Extend as long as we can.
@@ -166,11 +166,11 @@ func (e *doubleFastEncoder) Encode(blk *blockEnc, src []byte) {
 						println("repeat sequence", seq, "next s:", s)
 					}
 					blk.sequences = append(blk.sequences, seq)
-					s += lenght + repOff
+					s += length + repOff
 					nextEmit = s
 					if s >= sLimit {
 						if debugEncoder {
-							println("repeat ended", s, lenght)
+							println("repeat ended", s, length)
 
 						}
 						break encodeLoop
@@ -798,9 +798,9 @@ func (e *doubleFastEncoderDict) Encode(blk *blockEnc, src []byte) {
 				if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) {
 					// Consider history as well.
 					var seq seq
-					lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src)
+					length := 4 + e.matchlen(s+4+repOff, repIndex+4, src)
 
-					seq.matchLen = uint32(lenght - zstdMinMatch)
+					seq.matchLen = uint32(length - zstdMinMatch)
 
 					// We might be able to match backwards.
 					// Extend as long as we can.
@@ -826,11 +826,11 @@ func (e *doubleFastEncoderDict) Encode(blk *blockEnc, src []byte) {
 						println("repeat sequence", seq, "next s:", s)
 					}
 					blk.sequences = append(blk.sequences, seq)
-					s += lenght + repOff
+					s += length + repOff
 					nextEmit = s
 					if s >= sLimit {
 						if debugEncoder {
-							println("repeat ended", s, lenght)
+							println("repeat ended", s, length)
 
 						}
 						break encodeLoop

vendor/github.com/klauspost/compress/zstd/encoder.go

@@ -6,6 +6,7 @@
 
 import (
 	"crypto/rand"
+	"errors"
 	"fmt"
 	"io"
 	"math"
@@ -149,6 +150,9 @@ func (e *Encoder) ResetContentSize(w io.Writer, size int64) {
 // and write CRC if requested.
 func (e *Encoder) Write(p []byte) (n int, err error) {
 	s := &e.state
+	if s.eofWritten {
+		return 0, ErrEncoderClosed
+	}
 	for len(p) > 0 {
 		if len(p)+len(s.filling) < e.o.blockSize {
 			if e.o.crc {
@@ -202,7 +206,7 @@ func (e *Encoder) nextBlock(final bool) error {
 			return nil
 		}
 		if final && len(s.filling) > 0 {
-			s.current = e.EncodeAll(s.filling, s.current[:0])
+			s.current = e.encodeAll(s.encoder, s.filling, s.current[:0])
 			var n2 int
 			n2, s.err = s.w.Write(s.current)
 			if s.err != nil {
@@ -288,6 +292,9 @@ func (e *Encoder) nextBlock(final bool) error {
 	s.filling, s.current, s.previous = s.previous[:0], s.filling, s.current
 	s.nInput += int64(len(s.current))
 	s.wg.Add(1)
+	if final {
+		s.eofWritten = true
+	}
 	go func(src []byte) {
 		if debugEncoder {
 			println("Adding block,", len(src), "bytes, final:", final)
@@ -303,9 +310,6 @@ func (e *Encoder) nextBlock(final bool) error {
 		blk := enc.Block()
 		enc.Encode(blk, src)
 		blk.last = final
-		if final {
-			s.eofWritten = true
-		}
 		// Wait for pending writes.
 		s.wWg.Wait()
 		if s.writeErr != nil {
@@ -401,12 +405,20 @@ func (e *Encoder) Flush() error {
 	if len(s.filling) > 0 {
 		err := e.nextBlock(false)
 		if err != nil {
+			// Ignore Flush after Close.
+			if errors.Is(s.err, ErrEncoderClosed) {
+				return nil
+			}
 			return err
 		}
 	}
 	s.wg.Wait()
 	s.wWg.Wait()
 	if s.err != nil {
+		// Ignore Flush after Close.
+		if errors.Is(s.err, ErrEncoderClosed) {
+			return nil
+		}
 		return s.err
 	}
 	return s.writeErr
@@ -422,6 +434,9 @@ func (e *Encoder) Close() error {
 	}
 	err := e.nextBlock(true)
 	if err != nil {
+		if errors.Is(s.err, ErrEncoderClosed) {
+			return nil
+		}
 		return err
 	}
 	if s.frameContentSize > 0 {
@@ -459,6 +474,11 @@ func (e *Encoder) Close() error {
 		}
 		_, s.err = s.w.Write(frame)
 	}
+	if s.err == nil {
+		s.err = ErrEncoderClosed
+		return nil
+	}
+
 	return s.err
 }
 
@@ -469,6 +489,15 @@ func (e *Encoder) Close() error {
 // Data compressed with EncodeAll can be decoded with the Decoder,
 // using either a stream or DecodeAll.
 func (e *Encoder) EncodeAll(src, dst []byte) []byte {
+	e.init.Do(e.initialize)
+	enc := <-e.encoders
+	defer func() {
+		e.encoders <- enc
+	}()
+	return e.encodeAll(enc, src, dst)
+}
+
+func (e *Encoder) encodeAll(enc encoder, src, dst []byte) []byte {
 	if len(src) == 0 {
 		if e.o.fullZero {
 			// Add frame header.
@@ -491,13 +520,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte {
 		}
 		return dst
 	}
-	e.init.Do(e.initialize)
+
-	enc := <-e.encoders
-	defer func() {
-		// Release encoder reference to last block.
-		// If a non-single block is needed the encoder will reset again.
-		e.encoders <- enc
-	}()
 	// Use single segments when above minimum window and below window size.
 	single := len(src) <= e.o.windowSize && len(src) > MinWindowSize
 	if e.o.single != nil {

vendor/github.com/klauspost/compress/zstd/framedec.go

@@ -146,7 +146,9 @@ func (d *frameDec) reset(br byteBuffer) error {
 			}
 			return err
 		}
+		if debugDecoder {
 			printf("raw: %x, mantissa: %d, exponent: %d\n", wd, wd&7, wd>>3)
+		}
 		windowLog := 10 + (wd >> 3)
 		windowBase := uint64(1) << windowLog
 		windowAdd := (windowBase / 8) * uint64(wd&0x7)

vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go

@@ -146,7 +146,7 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) {
 		return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize)
 
 	default:
-		return true, fmt.Errorf("sequenceDecs_decode returned erronous code %d", errCode)
+		return true, fmt.Errorf("sequenceDecs_decode returned erroneous code %d", errCode)
 	}
 
 	s.seqSize += ctx.litRemain
@@ -292,7 +292,7 @@ func (s *sequenceDecs) decode(seqs []seqVals) error {
 			return io.ErrUnexpectedEOF
 		}
 
-		return fmt.Errorf("sequenceDecs_decode_amd64 returned erronous code %d", errCode)
+		return fmt.Errorf("sequenceDecs_decode_amd64 returned erroneous code %d", errCode)
 	}
 
 	if ctx.litRemain < 0 {

vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s

@@ -1814,7 +1814,7 @@ TEXT ·sequenceDecs_decodeSync_amd64(SB), $64-32
 	MOVQ    40(SP), AX
 	ADDQ    AX, 48(SP)
 
-	// Calculate poiter to s.out[cap(s.out)] (a past-end pointer)
+	// Calculate pointer to s.out[cap(s.out)] (a past-end pointer)
 	ADDQ R10, 32(SP)
 
 	// outBase += outPosition
@@ -2376,7 +2376,7 @@ TEXT ·sequenceDecs_decodeSync_bmi2(SB), $64-32
 	MOVQ    40(SP), CX
 	ADDQ    CX, 48(SP)
 
-	// Calculate poiter to s.out[cap(s.out)] (a past-end pointer)
+	// Calculate pointer to s.out[cap(s.out)] (a past-end pointer)
 	ADDQ R9, 32(SP)
 
 	// outBase += outPosition
@@ -2896,7 +2896,7 @@ TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $64-32
 	MOVQ    40(SP), AX
 	ADDQ    AX, 48(SP)
 
-	// Calculate poiter to s.out[cap(s.out)] (a past-end pointer)
+	// Calculate pointer to s.out[cap(s.out)] (a past-end pointer)
 	ADDQ R10, 32(SP)
 
 	// outBase += outPosition
@@ -3560,7 +3560,7 @@ TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $64-32
 	MOVQ    40(SP), CX
 	ADDQ    CX, 48(SP)
 
-	// Calculate poiter to s.out[cap(s.out)] (a past-end pointer)
+	// Calculate pointer to s.out[cap(s.out)] (a past-end pointer)
 	ADDQ R9, 32(SP)
 
 	// outBase += outPosition

vendor/github.com/klauspost/compress/zstd/zstd.go

@@ -88,6 +88,10 @@
 	// Close has been called.
 	ErrDecoderClosed = errors.New("decoder used after Close")
 
+	// ErrEncoderClosed will be returned if the Encoder was used after
+	// Close has been called.
+	ErrEncoderClosed = errors.New("encoder used after Close")
+
 	// ErrDecoderNilInput is returned when a nil Reader was provided
 	// and an operation other than Reset/DecodeAll/Close was attempted.
 	ErrDecoderNilInput = errors.New("nil input provided as reader")

vendor/github.com/minio/minio-go/v7/Makefile

@@ -32,6 +32,10 @@ functional-test:
 	@GO111MODULE=on go build -race functional_tests.go
 	@SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minioadmin SECRET_KEY=minioadmin ENABLE_HTTPS=1 MINT_MODE=full ./functional_tests
 
+functional-test-notls:
+	@GO111MODULE=on go build -race functional_tests.go
+	@SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minioadmin SECRET_KEY=minioadmin ENABLE_HTTPS=0 MINT_MODE=full ./functional_tests
+
 clean:
 	@echo "Cleaning up all the generated files"
 	@find . -name '*.test' | xargs rm -fv

vendor/github.com/minio/minio-go/v7/api-put-object.go

@@ -45,6 +45,8 @@
 	ReplicationStatusFailed ReplicationStatus = "FAILED"
 	// ReplicationStatusReplica indicates object is a replica of a source
 	ReplicationStatusReplica ReplicationStatus = "REPLICA"
+	// ReplicationStatusReplicaEdge indicates object is a replica of a edge source
+	ReplicationStatusReplicaEdge ReplicationStatus = "REPLICA-EDGE"
 )
 
 // Empty returns true if no replication status set.

vendor/github.com/minio/minio-go/v7/api.go

@@ -128,7 +128,7 @@ type Options struct {
 // Global constants.
 const (
 	libraryName    = "minio-go"
-	libraryVersion = "v7.0.77"
+	libraryVersion = "v7.0.78"
 )
 
 // User Agent should always following the below style.

vendor/github.com/minio/minio-go/v7/functional_tests.go

@@ -3565,16 +3565,10 @@ func validateObjectAttributeRequest(OA *minio.ObjectAttributes, opts *minio.Obje
 		}
 	}
 
-	hasFullObjectChecksum := true
+	hasFullObjectChecksum := (OA.Checksum.ChecksumCRC32 != "" ||
-	if OA.Checksum.ChecksumCRC32 == "" {
+		OA.Checksum.ChecksumCRC32C != "" ||
-		if OA.Checksum.ChecksumCRC32C == "" {
+		OA.Checksum.ChecksumSHA1 != "" ||
-			if OA.Checksum.ChecksumSHA1 == "" {
+		OA.Checksum.ChecksumSHA256 != "")
-				if OA.Checksum.ChecksumSHA256 == "" {
-					hasFullObjectChecksum = false
-				}
-			}
-		}
-	}
 
 	if test.HasFullChecksum {
 		if !hasFullObjectChecksum {

vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go

@@ -25,6 +25,7 @@
 	"io"
 	"net/http"
 	"net/url"
+	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -85,29 +86,59 @@ type STSWebIdentity struct {
 	// assuming.
 	RoleARN string
 
+	// Policy is the policy where the credentials should be limited too.
+	Policy string
+
 	// roleSessionName is the identifier for the assumed role session.
 	roleSessionName string
 }
 
 // NewSTSWebIdentity returns a pointer to a new
 // Credentials object wrapping the STSWebIdentity.
-func NewSTSWebIdentity(stsEndpoint string, getWebIDTokenExpiry func() (*WebIdentityToken, error)) (*Credentials, error) {
+func NewSTSWebIdentity(stsEndpoint string, getWebIDTokenExpiry func() (*WebIdentityToken, error), opts ...func(*STSWebIdentity)) (*Credentials, error) {
 	if stsEndpoint == "" {
 		return nil, errors.New("STS endpoint cannot be empty")
 	}
 	if getWebIDTokenExpiry == nil {
 		return nil, errors.New("Web ID token and expiry retrieval function should be defined")
 	}
-	return New(&STSWebIdentity{
+	i := &STSWebIdentity{
 		Client: &http.Client{
 			Transport: http.DefaultTransport,
 		},
 		STSEndpoint:         stsEndpoint,
 		GetWebIDTokenExpiry: getWebIDTokenExpiry,
-	}), nil
+	}
+	for _, o := range opts {
+		o(i)
+	}
+	return New(i), nil
 }
 
-func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSessionName string,
+// NewKubernetesIdentity returns a pointer to a new
+// Credentials object using the Kubernetes service account
+func NewKubernetesIdentity(stsEndpoint string, opts ...func(*STSWebIdentity)) (*Credentials, error) {
+	return NewSTSWebIdentity(stsEndpoint, func() (*WebIdentityToken, error) {
+		token, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")
+		if err != nil {
+			return nil, err
+		}
+
+		return &WebIdentityToken{
+			Token: string(token),
+		}, nil
+	}, opts...)
+}
+
+// WithPolicy option will enforce that the returned credentials
+// will be scoped down to the specified policy
+func WithPolicy(policy string) func(*STSWebIdentity) {
+	return func(i *STSWebIdentity) {
+		i.Policy = policy
+	}
+}
+
+func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSessionName string, policy string,
 	getWebIDTokenExpiry func() (*WebIdentityToken, error),
 ) (AssumeRoleWithWebIdentityResponse, error) {
 	idToken, err := getWebIDTokenExpiry()
@@ -133,6 +164,9 @@ func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSession
 	if idToken.Expiry > 0 {
 		v.Set("DurationSeconds", fmt.Sprintf("%d", idToken.Expiry))
 	}
+	if policy != "" {
+		v.Set("Policy", policy)
+	}
 	v.Set("Version", STSVersion)
 
 	u, err := url.Parse(endpoint)
@@ -183,7 +217,7 @@ func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSession
 // Retrieve retrieves credentials from the MinIO service.
 // Error will be returned if the request fails.
 func (m *STSWebIdentity) Retrieve() (Value, error) {
-	a, err := getWebIdentityCredentials(m.Client, m.STSEndpoint, m.RoleARN, m.roleSessionName, m.GetWebIDTokenExpiry)
+	a, err := getWebIdentityCredentials(m.Client, m.STSEndpoint, m.RoleARN, m.roleSessionName, m.Policy, m.GetWebIDTokenExpiry)
 	if err != nil {
 		return Value{}, err
 	}

vendor/github.com/minio/minio-go/v7/pkg/tags/tags.go

@@ -69,7 +69,7 @@ func (err errTag) Error() string {
 // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions
 // borrowed from this article and also testing various ASCII characters following regex
 // is supported by AWS S3 for both tags and values.
-var validTagKeyValue = regexp.MustCompile(`^[a-zA-Z0-9-+\-._:/@ ]+$`)
+var validTagKeyValue = regexp.MustCompile(`^[a-zA-Z0-9-+\-._:/@ =]+$`)
 
 func checkKey(key string) error {
 	if len(key) == 0 {

vendor/github.com/yuin/goldmark/ast/block.go

@@ -1,6 +1,7 @@
 package ast
 
 import (
+	"bytes"
 	"fmt"
 	"strings"
 
@@ -47,6 +48,15 @@ func (b *BaseBlock) SetLines(v *textm.Segments) {
 	b.lines = v
 }
 
+// Text implements Node.Text.
+func (b *BaseBlock) Text(source []byte) []byte {
+	var buf bytes.Buffer
+	for _, line := range b.Lines().Sliced(0, b.Lines().Len()) {
+		buf.Write(line.Value(source))
+	}
+	return buf.Bytes()
+}
+
 // A Document struct is a root node of Markdown text.
 type Document struct {
 	BaseBlock

vendor/github.com/yuin/goldmark/extension/table.go

@@ -184,11 +184,11 @@ func (b *tableParagraphTransformer) Transform(node *gast.Paragraph, reader text.
 func (b *tableParagraphTransformer) parseRow(segment text.Segment,
 	alignments []ast.Alignment, isHeader bool, reader text.Reader, pc parser.Context) *ast.TableRow {
 	source := reader.Source()
+	segment = segment.TrimLeftSpace(source)
+	segment = segment.TrimRightSpace(source)
 	line := segment.Value(source)
 	pos := 0
-	pos += util.TrimLeftSpaceLength(line)
 	limit := len(line)
-	limit -= util.TrimRightSpaceLength(line)
 	row := ast.NewTableRow(alignments)
 	if len(line) > 0 && line[pos] == '|' {
 		pos++

vendor/github.com/yuin/goldmark/parser/blockquote.go

@@ -28,12 +28,13 @@ func (b *blockquoteParser) process(reader text.Reader) bool {
 		reader.Advance(pos)
 		return true
 	}
-	if line[pos] == ' ' || line[pos] == '\t' {
-		pos++
-	}
 	reader.Advance(pos)
-	if line[pos-1] == '\t' {
+	if line[pos] == ' ' || line[pos] == '\t' {
-		reader.SetPadding(2)
+		padding := 0
+		if line[pos] == '\t' {
+			padding = util.TabWidth(reader.LineOffset()) - 1
+		}
+		reader.AdvanceAndSetPadding(1, padding)
 	}
 	return true
 }

vendor/github.com/yuin/goldmark/parser/list_item.go

@@ -58,7 +58,7 @@ func (b *listItemParser) Continue(node ast.Node, reader text.Reader, pc Context)
 	}
 
 	offset := lastOffset(node.Parent())
-	isEmpty := node.ChildCount() == 0
+	isEmpty := node.ChildCount() == 0 && pc.Get(emptyListItemWithBlankLines) != nil
 	indent, _ := util.IndentWidth(line, reader.LineOffset())
 	if (isEmpty || indent < offset) && indent < 4 {
 		_, typ := matchesListItem(line, true)

vendor/github.com/yuin/goldmark/parser/parser.go

@@ -878,10 +878,17 @@ func (p *parser) Parse(reader text.Reader, opts ...ParseOption) ast.Node {
 	blockReader := text.NewBlockReader(reader.Source(), nil)
 	p.walkBlock(root, func(node ast.Node) {
 		p.parseBlock(blockReader, node, pc)
+		lines := node.Lines()
+		if lines != nil && lines.Len() != 0 {
+			s := lines.At(lines.Len() - 1)
+			s.EOB = true
+			lines.Set(lines.Len()-1, s)
+		}
 	})
 	for _, at := range p.astTransformers {
 		at.Transform(root, reader, pc)
 	}
+
 	// root.Dump(reader.Source(), 0)
 	return root
 }
@@ -1256,4 +1263,5 @@ func (p *parser) parseBlock(block text.BlockReader, parent ast.Node, pc Context)
 	for _, ip := range p.closeBlockers {
 		ip.CloseBlock(parent, block, pc)
 	}
+
 }

vendor/github.com/yuin/goldmark/text/segment.go

@@ -2,6 +2,7 @@
 
 import (
 	"bytes"
+
 	"github.com/yuin/goldmark/util"
 )
 
@@ -18,6 +19,9 @@ type Segment struct {
 
 	// Padding is a padding length of the segment.
 	Padding int
+
+	// EOB is true if the segment is end of the block.
+	EOB bool
 }
 
 // NewSegment return a new Segment.
@@ -40,12 +44,18 @@ func NewSegmentPadding(start, stop, n int) Segment {
 
 // Value returns a value of the segment.
 func (t *Segment) Value(buffer []byte) []byte {
+	var result []byte
 	if t.Padding == 0 {
-		return buffer[t.Start:t.Stop]
+		result = buffer[t.Start:t.Stop]
-	}
+	} else {
-	result := make([]byte, 0, t.Padding+t.Stop-t.Start+1)
+		result = make([]byte, 0, t.Padding+t.Stop-t.Start+1)
 		result = append(result, bytes.Repeat(space, t.Padding)...)
-	return append(result, buffer[t.Start:t.Stop]...)
+		result = append(result, buffer[t.Start:t.Stop]...)
+	}
+	if t.EOB && len(result) > 0 && result[len(result)-1] != '\n' {
+		result = append(result, '\n')
+	}
+	return result
 }
 
 // Len returns a length of the segment.

vendor/github.com/yuin/goldmark/util/html5entities.go

@@ -1,6 +1,8 @@
 //nolint:golint,lll,misspell
 package util
 
+import "sync"
+
 // An HTML5Entity struct represents HTML5 entitites.
 type HTML5Entity struct {
 	Name       string
@@ -11,11 +13,18 @@ type HTML5Entity struct {
 // LookUpHTML5EntityByName returns (an HTML5Entity, true) if an entity named
 // given name is found, otherwise (nil, false).
 func LookUpHTML5EntityByName(name string) (*HTML5Entity, bool) {
-	v, ok := html5entities[name]
+	v, ok := html5entities()[name]
 	return v, ok
 }
 
-var html5entities = map[string]*HTML5Entity{
+var html5entitiesOnce sync.Once // TODO: uses sync.OnceValue for future
+
+var _html5entities map[string]*HTML5Entity
+
+func html5entities() map[string]*HTML5Entity {
+	html5entitiesOnce.Do(func() {
+		_html5entities =
+			map[string]*HTML5Entity{
 				"AElig":                           {Name: "AElig", CodePoints: []int{198}, Characters: []byte{0xc3, 0x86}},
 				"AMP":                             {Name: "AMP", CodePoints: []int{38}, Characters: []byte{0x26}},
 				"Aacute":                          {Name: "Aacute", CodePoints: []int{193}, Characters: []byte{0xc3, 0x81}},
@@ -2140,4 +2149,7 @@ func LookUpHTML5EntityByName(name string) (*HTML5Entity, bool) {
 				"zscr":                            {Name: "zscr", CodePoints: []int{120015}, Characters: []byte{0xf0, 0x9d, 0x93, 0x8f}},
 				"zwj":                             {Name: "zwj", CodePoints: []int{8205}, Characters: []byte{0xe2, 0x80, 0x8d}},
 				"zwnj":                            {Name: "zwnj", CodePoints: []int{8204}, Characters: []byte{0xe2, 0x80, 0x8c}},
+			}
+	})
+	return _html5entities
 }

vendor/github.com/yuin/goldmark/util/util.go

@@ -166,7 +166,13 @@ func IndentPositionPadding(bs []byte, currentPos, paddingv, width int) (pos, pad
 	w := 0
 	i := 0
 	l := len(bs)
+	p := paddingv
 	for ; i < l; i++ {
+		if p > 0 {
+			p--
+			w++
+			continue
+		}
 		if bs[i] == '\t' && w < width {
 			w += TabWidth(currentPos + w)
 		} else if bs[i] == ' ' && w < width {

vendor/github.com/yuin/goldmark/util/util_unsafe_go120.go

@@ -1,5 +1,5 @@
-//go:build !appengine && !js
+//go:build !appengine && !js && !go1.21
-// +build !appengine,!js
+// +build !appengine,!js,!go1.21
 
 package util
 

vendor/github.com/yuin/goldmark/util/util_unsafe_go121.go

@@ -0,0 +1,18 @@
+//go:build !appengine && !js && go1.21
+// +build !appengine,!js,go1.21
+
+package util
+
+import (
+	"unsafe"
+)
+
+// BytesToReadOnlyString returns a string converted from given bytes.
+func BytesToReadOnlyString(b []byte) string {
+	return unsafe.String(unsafe.SliceData(b), len(b))
+}
+
+// StringToReadOnlyBytes returns bytes converted from given string.
+func StringToReadOnlyBytes(s string) []byte {
+	return unsafe.Slice(unsafe.StringData(s), len(s))
+}

vendor/modules.txt

@@ -441,8 +441,8 @@ github.com/json-iterator/go
 # github.com/k3a/html2text v1.2.1
 ## explicit; go 1.16
 github.com/k3a/html2text
-# github.com/klauspost/compress v1.17.9
+# github.com/klauspost/compress v1.17.11
-## explicit; go 1.20
+## explicit; go 1.21
 github.com/klauspost/compress
 github.com/klauspost/compress/fse
 github.com/klauspost/compress/huff0
@@ -486,8 +486,8 @@ github.com/miekg/dns
 # github.com/minio/md5-simd v1.1.2
 ## explicit; go 1.14
 github.com/minio/md5-simd
-# github.com/minio/minio-go/v7 v7.0.77
+# github.com/minio/minio-go/v7 v7.0.78
-## explicit; go 1.21
+## explicit; go 1.22
 github.com/minio/minio-go/v7
 github.com/minio/minio-go/v7/pkg/cors
 github.com/minio/minio-go/v7/pkg/credentials
@@ -954,7 +954,7 @@ github.com/vmihailenco/tagparser/v2/internal/parser
 # github.com/wagslane/go-password-validator v0.3.0
 ## explicit; go 1.16
 github.com/wagslane/go-password-validator
-# github.com/yuin/goldmark v1.7.4
+# github.com/yuin/goldmark v1.7.6
 ## explicit; go 1.19
 github.com/yuin/goldmark
 github.com/yuin/goldmark/ast