Merge pull request 'fix reflected xss when returning errors on web endpoint' (#1472) from escape-errors-v1 into main-v1

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1472

.woodpecker/debug.yaml

@@ -1,3 +1,6 @@
+labels:
+  nix: "enabled"
+
 when:
   event:
   - push
@@ -9,27 +12,32 @@ when:
 
 steps:
   - name: check formatting
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
-      - nix-shell --attr devShell --run "cargo fmt -- --check"
+      - nix-build -j4 --attr flakePackages.fmt
 
   - name: build
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-build -j4 --attr flakePackages.dev
 
   - name: unit + func tests (lmdb)
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-build -j4 --attr flakePackages.tests-lmdb
 
   - name: unit + func tests (sqlite)
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-build -j4 --attr flakePackages.tests-sqlite
 
+  - name: unit + func tests (fjall)
+    image: nixpkgs/nix:nixos-24.05
+    commands:
+      - nix-build -j4 --attr flakePackages.tests-fjall
+
   - name: integration tests
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-build -j4 --attr flakePackages.dev
       - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)

.woodpecker/publish.yaml

@@ -1,3 +1,6 @@
+labels:
+  nix: "enabled"
+
 when:
   event:
   - deployment
@@ -8,7 +11,7 @@ depends_on:
 
 steps:
   - name: refresh-index
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     environment:
       AWS_ACCESS_KEY_ID:
         from_secret: garagehq_aws_access_key_id
@@ -19,7 +22,7 @@ steps:
       - nix-shell --attr ci --run "refresh_index"
 
   - name: multiarch-docker
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     environment:
       DOCKER_AUTH:
         from_secret: docker_auth

.woodpecker/release.yaml

@@ -1,3 +1,6 @@
+labels:
+  nix: "enabled"
+
 when:
   event:
   - deployment
@@ -16,17 +19,17 @@ matrix:
 
 steps:
   - name: build
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-build --attr releasePackages.${ARCH} --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
 
   - name: check is static binary
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-shell --attr ci --run "./script/not-dynamic.sh result/bin/garage"
 
   - name: integration tests
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
     when:
@@ -36,7 +39,7 @@ steps:
           ARCH: i386
 
   - name: upgrade tests
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
     when:
@@ -44,7 +47,7 @@ steps:
           ARCH: amd64
 
   - name: push static binary
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     environment:
       TARGET: "${TARGET}"
       AWS_ACCESS_KEY_ID:
@@ -55,7 +58,7 @@ steps:
       - nix-shell --attr ci --run "to_s3"
 
   - name: docker build and publish
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     environment:
       DOCKER_PLATFORM: "linux/${ARCH}"
       CONTAINER_NAME: "dxflrs/${ARCH}_garage"

Cargo.toml

@@ -24,18 +24,18 @@ default-members = ["src/garage"]
 
 # Internal Garage crates
 format_table = { version = "0.1.1", path = "src/format-table" }
-garage_api_common = { version = "1.1.0", path = "src/api/common" }
+garage_api_common = { version = "1.3.1", path = "src/api/common" }
-garage_api_admin = { version = "1.1.0", path = "src/api/admin" }
+garage_api_admin = { version = "1.3.1", path = "src/api/admin" }
-garage_api_s3 = { version = "1.1.0", path = "src/api/s3" }
+garage_api_s3 = { version = "1.3.1", path = "src/api/s3" }
-garage_api_k2v = { version = "1.1.0", path = "src/api/k2v" }
+garage_api_k2v = { version = "1.3.1", path = "src/api/k2v" }
-garage_block = { version = "1.1.0", path = "src/block" }
+garage_block = { version = "1.3.1", path = "src/block" }
-garage_db = { version = "1.1.0", path = "src/db", default-features = false }
+garage_db = { version = "1.3.1", path = "src/db", default-features = false }
-garage_model = { version = "1.1.0", path = "src/model", default-features = false }
+garage_model = { version = "1.3.1", path = "src/model", default-features = false }
-garage_net = { version = "1.1.0", path = "src/net" }
+garage_net = { version = "1.3.1", path = "src/net" }
-garage_rpc = { version = "1.1.0", path = "src/rpc" }
+garage_rpc = { version = "1.3.1", path = "src/rpc" }
-garage_table = { version = "1.1.0", path = "src/table" }
+garage_table = { version = "1.3.1", path = "src/table" }
-garage_util = { version = "1.1.0", path = "src/util" }
+garage_util = { version = "1.3.1", path = "src/util" }
-garage_web = { version = "1.1.0", path = "src/web" }
+garage_web = { version = "1.3.1", path = "src/web" }
 k2v-client = { version = "0.0.4", path = "src/k2v-client" }
 
 # External crates from crates.io
@@ -52,11 +52,11 @@ chrono = "0.4"
 crc32fast = "1.4"
 crc32c = "0.6"
 crypto-common = "0.1"
-err-derive = "0.3"
 gethostname = "0.4"
 git-version = "0.3.4"
 hex = "0.4"
 hexdump = "0.1"
+html-escape = "0.2.13"
 hmac = "0.12"
 itertools = "0.12"
 ipnet = "2.9.0"
@@ -65,6 +65,7 @@ md-5 = "0.10"
 mktemp = "0.5"
 nix = { version = "0.29", default-features = false, features = ["fs"] }
 nom = "7.1"
+parking_lot = "0.12"
 parse_duration = "2.1"
 pin-project = "1.0.12"
 pnet_datalink = "0.34"
@@ -83,12 +84,14 @@ pretty_env_logger = "0.5"
 structopt = { version = "0.3", default-features = false }
 syslog-tracing = "0.3"
 tracing = "0.1"
+tracing-journald = "0.3.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 heed = { version = "0.11", default-features = false, features = ["lmdb"] }
-rusqlite = "0.31.0"
+rusqlite = "0.37"
 r2d2 = "0.8"
-r2d2_sqlite = "0.24"
+r2d2_sqlite = "0.31"
+fjall = "2.4"
 
 async-compression = { version = "0.4", features = ["tokio", "zstd"] }
 zstd = { version = "0.13", default-features = false }
@@ -134,7 +137,7 @@ prometheus = "0.13"
 aws-sigv4 = { version = "1.1", default-features = false }
 hyper-rustls = { version = "0.26", default-features = false, features = ["http1", "http2", "ring", "rustls-native-certs"] }
 log = "0.4"
-thiserror = "1.0"
+thiserror = "2.0"
 
 # ---- used only as build / dev dependencies ----
 assert-json-diff = "2.0"
@@ -144,12 +147,8 @@ aws-smithy-runtime = { version = "1.8", default-features = false, features = ["t
 aws-sdk-config = { version = "1.62", default-features = false }
 aws-sdk-s3 = { version = "1.79", default-features = false, features = ["rt-tokio"] }
 
-[profile.dev]
-#lto = "thin"	# disabled for now, adds 2-4 min to each CI build
-lto = "off"
-
 [profile.release]
-lto = true
+lto = "thin"
-codegen-units = 1
+codegen-units = 16
-opt-level = "s"
+opt-level = 3
-strip = true
+strip = "debuginfo"

doc/book/connect/apps/index.md

@@ -12,7 +12,7 @@ In this section, we cover the following web applications:
 | [Mastodon](#mastodon)     | ✅       | Natively supported    |
 | [Matrix](#matrix)     | ✅       |  Tested with `synapse-s3-storage-provider`    |
 | [ejabberd](#ejabberd)     | ✅       |  `mod_s3_upload`    |
-| [Pixelfed](#pixelfed)     | ❓       |  Not yet tested    |
+| [Pixelfed](#pixelfed)     | ✅       |  Natively supported    |
 | [Pleroma](#pleroma)     | ❓       |  Not yet tested    |
 | [Lemmy](#lemmy)     | ✅        |  Supported with pict-rs    |
 | [Funkwhale](#funkwhale)     | ❓       | Not yet tested     |
@@ -191,10 +191,10 @@ garage key create peertube-key
 
 Keep the Key ID and the Secret key in a pad, they will be needed later.  
 
-We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist).
+We need two buckets, one for normal videos (named peertube-videos) and one for webtorrent videos (named peertube-playlists).
 ```bash
 garage bucket create peertube-videos
-garage bucket create peertube-playlist
+garage bucket create peertube-playlists
 ```
 
 Now we allow our key to read and write on these buckets:
@@ -253,7 +253,7 @@ object_storage:
     proxify_private_files: false
 
   streaming_playlists:
-    bucket_name: 'peertube-playlist'
+    bucket_name: 'peertube-playlists'
 
     # Keep it empty for our example
     prefix: ''

doc/book/connect/backup.md

@@ -161,3 +161,49 @@ kopia repository validate-provider
 
 You can then run all the standard kopia commands: `kopia snapshot create`, `kopia mount`...
 Everything should work out-of-the-box.
+
+## Plakar
+
+Create your key and bucket on Garage server:
+
+```bash
+garage key create my-plakar-key
+garage bucket create plakar-backups
+garage bucket allow plakar-backups --read --write --key my-plakar-key
+…
+```
+
+On Plakar server, add your Garage as a storage location:
+```bash
+plakar store add garageS3 s3://my-garage.tld/plakar-backups \
+region=garage # Or as you've specified in garage.toml \
+access_key=<Key ID from "garage key info my-plakar-key"> \
+secret_access_key=<Secret key from "garage key info my-plakar-key">
+```
+
+Then create the repository.
+```bash
+plakar at @garageS3 create -plaintext # Unencrypted
+# or
+plakar at @garageS3 create #encrypted
+```
+
+If you encrypt your backups (Plakar default), you will need to define a strong passphrase. Do not forget to save your password safely. It will be needed to decrypt your backups.
+
+
+After the repository has been created, check that everything works as expected (that might give an empty result as no file has been added yet, but no error message):
+```bash
+plakar at @garageS3 check
+```
+
+Now that everything is configure, you can use Garage as your backups storage. For instance sync it with a local backup storage:
+```bash
+$ plakar at ~/backups sync to @garageS3
+```
+
+Or list the S3 storage content:
+```bash
+$ plakar at @garageS3 ls
+```
+
+More information in Plakar documentation: https://www.plakar.io/docs/main/quickstart/

doc/book/cookbook/ansible.md

@@ -8,18 +8,18 @@ have published Ansible roles.  We list them and compare them below.
 
 ## Comparison of Ansible roles
 
-| Feature                            | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) |
+| Feature                            | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | [eddster ansible-role-garage](#eddster-ansible-role-garage) |
-|------------------------------------|---------------------------------------------|---------------------------------------------------------------|
+|------------------------------------|---------------------------------------------|---------------------------------------------------------------|---------------------------------|
-| **Runtime**                        | Systemd                                     | Docker                                                        |
+| **Runtime**                        | Systemd                                     | Docker                                                        | Systemd                         |
-| **Target OS**                      | Any Linux                                   | Any Linux                                                     |
+| **Target OS**                      | Any Linux                                   | Any Linux                                                     | Any Linux                       |
-| **Architecture**                   | amd64, arm64, i686                          | amd64, arm64                                                  |
+| **Architecture**                   | amd64, arm64, i686                          | amd64, arm64                                                  | arm64, arm, 386, amd64                            |
-| **Additional software**            | None                                        | Traefik                                                       |
+| **Additional software**            | None                                        | Traefik                                                       | Ngnix and Keepalived (optional) |
-| **Automatic node connection**      | ❌                                          | ✅                                                            |
+| **Automatic node connection**      | ❌                                          | ✅                                                            | ✅                              |
-| **Layout management**              | ❌                                          | ✅                                                            |
+| **Layout management**              | ❌                                          | ✅                                                            | ✅                              |
-| **Manage buckets & keys**          | ❌                                          | ✅ (basic)                                                    |
+| **Manage buckets & keys**          | ❌                                          | ✅ (basic)                                                    | ✅                              |
-| **Allow custom Garage config**     | ✅                                          | ❌                                                            |
+| **Allow custom Garage config**     | ✅                                          | ❌                                                            | ❌                              |
-| **Facilitate Garage upgrades**     | ✅                                          | ❌                                                            |
+| **Facilitate Garage upgrades**     | ✅                                          | ❌                                                            | ✅                              |
-| **Multiple instances on one host** | ✅                                          | ✅                                                            |
+| **Multiple instances on one host** | ✅                                          | ✅                                                            | ❌                              |
 
 
 ## zorun/ansible-role-garage
@@ -49,3 +49,15 @@ structured DNS names, etc).
 
 As a result, this role makes it easier to start with Garage on Ansible,
 but is less flexible.
+
+## eddster2309/ansible-role-garage
+
+[Source code](https://github.com/eddster2309/ansible-role-garage), [Ansible galaxy](https://galaxy.ansible.com/ui/standalone/roles/eddster2309/garage/)
+
+This role is a opinionated but customisable role using the official Garage
+static binaries and only requires Systemd. As such it should work on any
+Linux based host. It includes all the nesscary configuration to
+automatically setup a clustered Garage deployment. Most Garage
+configuration options are exposed through Ansible variables so while you
+can't provide a custom config you can get very close. It can optionally
+installed a HA nginx deployment with Keepalived.

doc/book/cookbook/binary-packages.md

@@ -15,9 +15,10 @@ Alpine Linux repositories (available since v3.17):
 apk add garage
 ```
 
-The default configuration file is installed to `/etc/garage.toml`. You can run
+The default configuration file is installed to `/etc/garage/garage.toml`. You can run
-Garage using: `rc-service garage start`. If you don't specify `rpc_secret`, it
+Garage using: `rc-service garage start`.
-will be automatically replaced with a random string on the first start.
+
+If you don't specify `rpc_secret`, it will be automatically replaced with a random string on the first start.
 
 Please note that this package is built without Consul discovery, Kubernetes
 discovery, OpenTelemetry exporter, and K2V features (K2V will be enabled once
@@ -26,7 +27,7 @@ it's stable).
 
 ## Arch Linux
 
-Garage is available in the [AUR](https://aur.archlinux.org/packages/garage).
+Garage is available in the official repositories under [extra](https://archlinux.org/packages/extra/x86_64/garage).
 
 ## FreeBSD
 

doc/book/cookbook/kubernetes.md

@@ -11,7 +11,7 @@ Firstly clone the repository:
 
 ```bash
 git clone https://git.deuxfleurs.fr/Deuxfleurs/garage
-cd garage/scripts/helm
+cd garage/script/helm
 ```
 
 Deploy with default options:

doc/book/cookbook/real-world.md

@@ -96,14 +96,14 @@ to store 2 TB of data in total.
 ## Get a Docker image
 
 Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
-We encourage you to use a fixed tag (eg. `v1.1.0`) and not the `latest` tag.
+We encourage you to use a fixed tag (eg. `v1.3.0`) and not the `latest` tag.
-For this example, we will use the latest published version at the time of the writing which is `v1.1.0` but it's up to you
+For this example, we will use the latest published version at the time of the writing which is `v1.3.0` but it's up to you
 to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
 
 For example:
 
 ```
-sudo docker pull dxflrs/garage:v1.1.0
+sudo docker pull dxflrs/garage:v1.3.0
 ```
 
 ## Deploying and configuring Garage
@@ -171,7 +171,7 @@ docker run \
   -v /etc/garage.toml:/etc/garage.toml \
   -v /var/lib/garage/meta:/var/lib/garage/meta \
   -v /var/lib/garage/data:/var/lib/garage/data \
-  dxflrs/garage:v1.1.0
+  dxflrs/garage:v1.3.0
 ```
 
 With this command line, Garage should be started automatically at each boot.
@@ -185,7 +185,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
 version: "3"
 services:
   garage:
-    image: dxflrs/garage:v1.1.0
+    image: dxflrs/garage:v1.3.0
     network_mode: "host"
     restart: unless-stopped
     volumes:

doc/book/quick-start/_index.md

@@ -132,7 +132,7 @@ docker run \
   -v /path/to/garage.toml:/etc/garage.toml \
   -v /path/to/garage/meta:/var/lib/garage/meta \
   -v /path/to/garage/data:/var/lib/garage/data \
-  dxflrs/garage:v1.1.0
+  dxflrs/garage:v1.3.0
 ```
 
 Under Linux, you can substitute `--network host` for `-p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903`
@@ -182,11 +182,12 @@ ID                 Hostname  Address         Tag                   Zone  Capacit
 ## Creating a cluster layout
 
 Creating a cluster layout for a Garage deployment means informing Garage
-of the disk space available on each node of the cluster
+of the disk space available on each node of the cluster, `-c`,
-as well as the zone (e.g. datacenter) each machine is located in.
+as well as the name of the zone (e.g. datacenter), `-z`, each machine is located in.
 
-For our test deployment, we are using only one node. The way in which we configure
+For our test deployment, we are have only one node with zone named `dc1` and a
-it does not matter, you can simply write:
+capacity of `1G`, though the capacity is ignored for a single node deployment
+and can be changed later when adding new nodes.
 
 ```bash
 garage layout assign -z dc1 -c 1G <node_id>

doc/book/reference-manual/configuration.md

@@ -24,7 +24,8 @@ db_engine = "lmdb"
 
 block_size = "1M"
 block_ram_buffer_max = "256MiB"
-
+block_max_concurrent_reads = 16
+block_max_concurrent_writes_per_request =10
 lmdb_map_size = "1T"
 
 compression_level = 1
@@ -93,30 +94,32 @@ The following gives details about each available configuration option.
 
 [Environment variables](#env_variables).
 
-Top-level configuration options:
+Top-level configuration options, in alphabetical order:
+[`allow_punycode`](#allow_punycode),
 [`allow_world_readable_secrets`](#allow_world_readable_secrets),
+[`block_max_concurrent_reads`](`block_max_concurrent_reads),
 [`block_ram_buffer_max`](#block_ram_buffer_max),
+[`block_max_concurrent_writes_per_request`](#block_max_concurrent_writes_per_request),
 [`block_size`](#block_size),
 [`bootstrap_peers`](#bootstrap_peers),
 [`compression_level`](#compression_level),
+[`consistency_mode`](#consistency_mode),
 [`data_dir`](#data_dir),
 [`data_fsync`](#data_fsync),
 [`db_engine`](#db_engine),
 [`disable_scrub`](#disable_scrub),
-[`use_local_tz`](#use_local_tz),
 [`lmdb_map_size`](#lmdb_map_size),
 [`metadata_auto_snapshot_interval`](#metadata_auto_snapshot_interval),
 [`metadata_dir`](#metadata_dir),
 [`metadata_fsync`](#metadata_fsync),
 [`metadata_snapshots_dir`](#metadata_snapshots_dir),
 [`replication_factor`](#replication_factor),
-[`consistency_mode`](#consistency_mode),
 [`rpc_bind_addr`](#rpc_bind_addr),
 [`rpc_bind_outgoing`](#rpc_bind_outgoing),
 [`rpc_public_addr`](#rpc_public_addr),
 [`rpc_public_addr_subnet`](#rpc_public_addr_subnet)
-[`rpc_secret`/`rpc_secret_file`](#rpc_secret).
+[`rpc_secret`/`rpc_secret_file`](#rpc_secret),
-[`allow_punycode`](#allow_punycode).
+[`use_local_tz`](#use_local_tz).
 
 The `[consul_discovery]` section:
 [`api`](#consul_api),
@@ -153,13 +156,17 @@ The `[admin]` section:
 
 ### Environment variables {#env_variables}
 
-The following configuration parameter must be specified as an environment
+The following configuration parameters must be specified as environment variables,
-variable, it does not exist in the configuration file:
+they do not exist in the configuration file:
 
 - `GARAGE_LOG_TO_SYSLOG` (since `v0.9.4`): set this to `1` or `true` to make the
   Garage daemon send its logs to `syslog` (using the libc `syslog` function)
   instead of printing to stderr.
 
+- `GARAGE_LOG_TO_JOURNALD` (since `v1.2.0`): set this to `1` or `true` to make the
+  Garage daemon send its logs to `journald` (using the native protocol of `systemd-journald`)
+  instead of printing to stderr.
+
 The following environment variables can be used to override the corresponding
 values in the configuration file:
 
@@ -171,7 +178,7 @@ values in the configuration file:
 
 ### Top-level configuration options
 
-#### `replication_factor` {#replication_factor}
+#### `replication_factor` (since `v1.0.0`) {#replication_factor}
 
 The replication factor can be any positive integer smaller or equal the node count in your cluster.
 The chosen replication factor has a big impact on the cluster's failure tolerancy and performance characteristics.
@@ -219,7 +226,7 @@ is in progress.  In theory, no data should be lost as rebalancing is a
 routine operation for Garage, although we cannot guarantee you that everything
  will go right in such an extreme scenario.
 
-#### `consistency_mode` {#consistency_mode}
+#### `consistency_mode` (since `v1.0.0`) {#consistency_mode}
 
 The consistency mode setting determines the read and write behaviour of your cluster.
 
@@ -329,6 +336,7 @@ Since `v0.8.0`, Garage can use alternative storage backends as follows:
 | --------- | ----------------- | ------------- |
 | [LMDB](https://www.symas.com/lmdb) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` |
 | [Sqlite](https://sqlite.org) (since `v0.8.0`) | `"sqlite"` | `<metadata_dir>/db.sqlite` |
+| [Fjall](https://github.com/fjall-rs/fjall) (**experimental support** since `v1.3.0`) | `"fjall"` | `<metadata_dir>/db.fjall/` |
 | [Sled](https://sled.rs) (old default, removed since `v1.0`) | `"sled"` | `<metadata_dir>/db/` |
 
 Sled was supported until Garage v0.9.x, and was removed in Garage v1.0.
@@ -365,6 +373,14 @@ LMDB works very well, but is known to have the following limitations:
   so it is not the best choice for high-performance storage clusters,
   but it should work fine in many cases.
 
+- Fjall: a storage engine based on LSM trees, which theoretically allow for
+  higher write throughput than other storage engines that are based on B-trees.
+  Using Fjall could potentially improve Garage's performance significantly in
+  write-heavy workloads. **Support for Fjall is experimental at this point**,
+  we have added it to Garage for evaluation purposes only. **Do not use it for
+  production-critical workloads.**
+
+
 It is possible to convert Garage's metadata directory from one format to another
 using the `garage convert-db` command, which should be used as follows:
 
@@ -402,6 +418,7 @@ Here is how this option impacts the different database engines:
 |----------|------------------------------------|-------------------------------|
 | Sqlite   | `PRAGMA synchronous = OFF`         | `PRAGMA synchronous = NORMAL` |
 | LMDB     | `MDB_NOMETASYNC` + `MDB_NOSYNC`    | `MDB_NOMETASYNC`              |
+| Fjall    | default options                    | not supported                 |
 
 Note that the Sqlite database is always ran in `WAL` mode (`PRAGMA journal_mode = WAL`).
 
@@ -508,6 +525,37 @@ node.
 
 The default value is 256MiB.
 
+#### `block_max_concurrent_reads` (since `v1.3.0` / `v2.1.0`) {#block_max_concurrent_reads}
+
+The maximum number of blocks (individual files in the data directory) open
+simultaneously for reading.
+
+Reducing this number does not limit the number of data blocks that can be
+transferred through the network simultaneously. This mechanism was just added
+as a backpressure mechanism for HDD read speed: it helps avoid a situation
+where too many requests are coming in and Garage is reading too many block
+files simultaneously, thus not making timely progress on any of the reads.
+
+When a request to read a data block comes in through the network, the requests
+awaits for one of the `block_max_concurrent_reads` slots to be available
+(internally implemented using a Semaphore object). Once it acquired a read
+slot, it reads the entire block file to RAM and frees the slot as soon as the
+block file is finished reading. Only after the slot is released will the
+block's data start being transferred over the network.  If the request fails to
+acquire a reading slot wihtin 15 seconds, it fails with a timeout error.
+Timeout events can be monitored through the `block_read_semaphore_timeouts`
+metric in Prometheus: a non-zero number of such events indicates an I/O
+bottleneck on HDD read speed.
+
+
+#### `block_max_concurrent_writes_per_request` (since  `v2.1.0`) {#block_max_concurrent_writes_per_request}
+
+This parameter is designed to adapt to the concurrent write performance of 
+different storage media.Maximum number of parallel block writes per put request 
+Higher values improve throughput but increase memory usage.
+
+Default: 3, Recommended: 10-30 for NVMe, 3-10 for HDD
+
 #### `lmdb_map_size` {#lmdb_map_size}
 
 This parameters can be used to set the map size used by LMDB,

doc/book/reference-manual/s3-compatibility.md

@@ -23,17 +23,17 @@ Feel free to open a PR to suggest fixes this table. Minio is missing because the
 - 2022-05-25 - Many Ceph S3 endpoints are not documented but implemented. Following a notification from the Ceph community, we added them.
 
 
-
 ## High-level features
 
 | Feature                      | Garage                           | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) |
 |------------------------------|----------------------------------|-----------------|---------------|---------|-----|
-| [signature v2](https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html) (deprecated) | ❌ Missing | ✅ |  ✅ | ✅ | ✅ |
+| [signature v2](https://docs.aws.amazon.com/AmazonS3/latest/API/Appendix-Sigv2.html) (deprecated) | ❌ Missing | ✅ |  ✅ | ✅ | ✅ |
 | [signature v4](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) |  ✅ Implemented |  ✅ |  ✅ | ❌ | ✅ |
 | [URL path-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access) (eg. `host.tld/bucket/key`) |  ✅ Implemented | ✅ |  ✅ | ❓| ✅ |
 | [URL vhost-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access) URL (eg. `bucket.host.tld/key`) |  ✅ Implemented | ❌| ✅| ✅ | ✅ |
 | [Presigned URLs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ShareObjectPreSignedURL.html) |  ✅ Implemented | ❌|  ✅ | ✅ |  ✅(❓) |
 | [SSE-C encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html) |  ✅ Implemented | ❓ |  ✅ | ❌ |  ✅ |
+| [Bucket versioning](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html) | ❌ Missing | ✅ |  ✅ | ❌ | ✅ |
 
 *Note:* OpenIO does not says if it supports presigned URLs. Because it is part
 of signature v4 and they claim they support it without additional precisions,

doc/drafts/admin-api.md

@@ -70,7 +70,7 @@ Example response body:
 ```json
 {
   "node": "b10c110e4e854e5aa3f4637681befac755154b20059ec163254ddbfae86b09df",
-  "garageVersion": "v1.1.0",
+  "garageVersion": "v1.3.0",
   "garageFeatures": [
     "k2v",
     "lmdb",

flake.lock

@@ -50,17 +50,17 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1736692550,
+        "lastModified": 1763977559,
-        "narHash": "sha256-7tk8xH+g0sJkKLTJFOxphJxxOjMDFMWv24nXslaU2ro=",
+        "narHash": "sha256-g4MKqsIRy5yJwEsI+fYODqLUnAqIY4kZai0nldAP6EM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "7c4869c47090dd7f9f1bdfb49a22aea026996815",
+        "rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "7c4869c47090dd7f9f1bdfb49a22aea026996815",
+        "rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632",
         "type": "github"
       }
     },
@@ -80,17 +80,17 @@
         ]
       },
       "locked": {
-        "lastModified": 1738549608,
+        "lastModified": 1763952169,
-        "narHash": "sha256-GdyT9QEUSx5k/n8kILuNy83vxxdyUfJ8jL5mMpQZWfw=",
+        "narHash": "sha256-+PeDBD8P+NKauH+w7eO/QWCIp8Cx4mCfWnh9sJmy9CM=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "35c6f8c4352f995ecd53896200769f80a3e8f22d",
+        "rev": "ab726555a9a72e6dc80649809147823a813fa95b",
         "type": "github"
       },
       "original": {
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "35c6f8c4352f995ecd53896200769f80a3e8f22d",
+        "rev": "ab726555a9a72e6dc80649809147823a813fa95b",
         "type": "github"
       }
     },

flake.nix

@@ -2,13 +2,13 @@
   description =
     "Garage, an S3-compatible distributed object store for self-hosted deployments";
 
-  # Nixpkgs 24.11 as of 2025-01-12
+  # Nixpkgs 25.05 as of 2025-11-24
   inputs.nixpkgs.url =
-    "github:NixOS/nixpkgs/7c4869c47090dd7f9f1bdfb49a22aea026996815";
+    "github:NixOS/nixpkgs/cfe2c7d5b5d3032862254e68c37a6576b633d632";
 
-  # Rust overlay as of 2025-02-03
+  # Rust overlay as of 2025-11-24
   inputs.rust-overlay.url =
-  "github:oxalica/rust-overlay/35c6f8c4352f995ecd53896200769f80a3e8f22d";
+  "github:oxalica/rust-overlay/ab726555a9a72e6dc80649809147823a813fa95b";
   inputs.rust-overlay.inputs.nixpkgs.follows = "nixpkgs";
 
   inputs.crane.url = "github:ipetkov/crane";
@@ -30,6 +30,10 @@
           inherit system nixpkgs crane rust-overlay extraTestEnv;
           release = false;
         }).garage-test;
+        lints = (compile {
+          inherit system nixpkgs crane rust-overlay;
+          release = false;
+        });
       in
       {
         packages = {
@@ -53,6 +57,13 @@
           tests-sqlite = testWith {
             GARAGE_TEST_INTEGRATION_DB_ENGINE = "sqlite";
           };
+          tests-fjall = testWith {
+            GARAGE_TEST_INTEGRATION_DB_ENGINE = "fjall";
+          };
+
+          # lints (fmt, clippy)
+          fmt = lints.garage-cargo-fmt;
+          clippy = lints.garage-cargo-clippy;
         };
 
         # ---- developpment shell, for making native builds only ----

nix/compile.nix

@@ -48,7 +48,7 @@ let
 
   inherit (pkgs) lib stdenv;
 
-  toolchainFn = (p: p.rust-bin.stable."1.82.0".default.override {
+  toolchainFn = (p: p.rust-bin.stable."1.91.0".default.override {
     targets = lib.optionals (target != null) [ rustTarget ];
     extensions = [
       "rust-src"
@@ -68,12 +68,13 @@ let
   rootFeatures = if features != null then
     features
   else
-    ([ "bundled-libs" "lmdb" "sqlite" "k2v" ] ++ (lib.optionals release [
+    ([ "bundled-libs" "lmdb" "sqlite" "fjall" "k2v" ] ++ (lib.optionals release [
       "consul-discovery"
       "kubernetes-discovery"
       "metrics"
       "telemetry-otlp"
       "syslog"
+      "journald"
     ]));
 
   featuresStr = lib.concatStringsSep "," rootFeatures;
@@ -189,4 +190,15 @@ in rec {
       pkgs.cacert
     ];
   } // extraTestEnv);
+
+  # ---- source code linting ----
+
+  garage-cargo-fmt = craneLib.cargoFmt (commonArgs // {
+    cargoExtraArgs = "";
+  });
+
+  garage-cargo-clippy = craneLib.cargoClippy (commonArgs // {
+    cargoArtifacts = garage-deps;
+    cargoClippyExtraArgs = "--all-targets -- -D warnings";
+  });
 }

script/dev-env-aws.sh

@@ -1,6 +1,7 @@
 export AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
 export AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
 export AWS_DEFAULT_REGION='garage'
+export AWS_REQUEST_CHECKSUM_CALCULATION='when_required'
 # FUTUREWORK: set AWS_ENDPOINT_URL instead, once nixpkgs bumps awscli to >=2.13.0.
 function aws { command aws --endpoint-url http://127.0.0.1:3911 $@ ; }
 

script/helm/garage/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: garage
 description: S3-compatible object store for small self-hosted geo-distributed deployments
 type: application
-version: 0.7.0
+version: 0.7.3
-appVersion: "v1.1.0"
+appVersion: "v1.3.1"
 home: https://garagehq.deuxfleurs.fr/
 icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg
 
@@ -15,4 +15,4 @@ keywords:
 sources:
 - https://git.deuxfleurs.fr/Deuxfleurs/garage.git
 
-maintainers: []
+maintainers: []