Merge pull request 'fix reflected xss when returning errors on web endpoint' (#1472) from escape-errors-v1 into main-v1

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1472

.woodpecker/debug.yaml

@@ -1,3 +1,6 @@
+labels:
+  nix: "enabled"
+
 when:
   event:
   - push
@@ -9,27 +12,32 @@ when:
 
 steps:
   - name: check formatting
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
-      - nix-shell --attr devShell --run "cargo fmt -- --check"
+      - nix-build -j4 --attr flakePackages.fmt
 
   - name: build
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-build -j4 --attr flakePackages.dev
 
   - name: unit + func tests (lmdb)
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-build -j4 --attr flakePackages.tests-lmdb
 
   - name: unit + func tests (sqlite)
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-build -j4 --attr flakePackages.tests-sqlite
 
+  - name: unit + func tests (fjall)
+    image: nixpkgs/nix:nixos-24.05
+    commands:
+      - nix-build -j4 --attr flakePackages.tests-fjall
+
   - name: integration tests
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-build -j4 --attr flakePackages.dev
       - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)

.woodpecker/publish.yaml

@@ -1,3 +1,6 @@
+labels:
+  nix: "enabled"
+
 when:
   event:
   - deployment
@@ -8,7 +11,7 @@ depends_on:
 
 steps:
   - name: refresh-index
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     environment:
       AWS_ACCESS_KEY_ID:
         from_secret: garagehq_aws_access_key_id
@@ -19,7 +22,7 @@ steps:
       - nix-shell --attr ci --run "refresh_index"
 
   - name: multiarch-docker
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     environment:
       DOCKER_AUTH:
         from_secret: docker_auth

.woodpecker/release.yaml

@@ -1,3 +1,6 @@
+labels:
+  nix: "enabled"
+
 when:
   event:
   - deployment
@@ -16,17 +19,17 @@ matrix:
 
 steps:
   - name: build
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-build --attr releasePackages.${ARCH} --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
 
   - name: check is static binary
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-shell --attr ci --run "./script/not-dynamic.sh result/bin/garage"
 
   - name: integration tests
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
     when:
@@ -36,7 +39,7 @@ steps:
           ARCH: i386
 
   - name: upgrade tests
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
     when:
@@ -44,7 +47,7 @@ steps:
           ARCH: amd64
 
   - name: push static binary
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     environment:
       TARGET: "${TARGET}"
       AWS_ACCESS_KEY_ID:
@@ -55,7 +58,7 @@ steps:
       - nix-shell --attr ci --run "to_s3"
 
   - name: docker build and publish
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     environment:
       DOCKER_PLATFORM: "linux/${ARCH}"
       CONTAINER_NAME: "dxflrs/${ARCH}_garage"

Cargo.toml

@@ -24,18 +24,18 @@ default-members = ["src/garage"]
 
 # Internal Garage crates
 format_table = { version = "0.1.1", path = "src/format-table" }
-garage_api_common = { version = "1.1.0", path = "src/api/common" }
-garage_api_admin = { version = "1.1.0", path = "src/api/admin" }
-garage_api_s3 = { version = "1.1.0", path = "src/api/s3" }
-garage_api_k2v = { version = "1.1.0", path = "src/api/k2v" }
-garage_block = { version = "1.1.0", path = "src/block" }
-garage_db = { version = "1.1.0", path = "src/db", default-features = false }
-garage_model = { version = "1.1.0", path = "src/model", default-features = false }
-garage_net = { version = "1.1.0", path = "src/net" }
-garage_rpc = { version = "1.1.0", path = "src/rpc" }
-garage_table = { version = "1.1.0", path = "src/table" }
-garage_util = { version = "1.1.0", path = "src/util" }
-garage_web = { version = "1.1.0", path = "src/web" }
+garage_api_common = { version = "1.3.1", path = "src/api/common" }
+garage_api_admin = { version = "1.3.1", path = "src/api/admin" }
+garage_api_s3 = { version = "1.3.1", path = "src/api/s3" }
+garage_api_k2v = { version = "1.3.1", path = "src/api/k2v" }
+garage_block = { version = "1.3.1", path = "src/block" }
+garage_db = { version = "1.3.1", path = "src/db", default-features = false }
+garage_model = { version = "1.3.1", path = "src/model", default-features = false }
+garage_net = { version = "1.3.1", path = "src/net" }
+garage_rpc = { version = "1.3.1", path = "src/rpc" }
+garage_table = { version = "1.3.1", path = "src/table" }
+garage_util = { version = "1.3.1", path = "src/util" }
+garage_web = { version = "1.3.1", path = "src/web" }
 k2v-client = { version = "0.0.4", path = "src/k2v-client" }
 
 # External crates from crates.io
@@ -52,13 +52,12 @@ chrono = "0.4"
 crc32fast = "1.4"
 crc32c = "0.6"
 crypto-common = "0.1"
-err-derive = "0.3"
 gethostname = "0.4"
 git-version = "0.3.4"
 hex = "0.4"
 hexdump = "0.1"
+html-escape = "0.2.13"
 hmac = "0.12"
-idna = "0.5"
 itertools = "0.12"
 ipnet = "2.9.0"
 lazy_static = "1.4"
@@ -66,6 +65,7 @@ md-5 = "0.10"
 mktemp = "0.5"
 nix = { version = "0.29", default-features = false, features = ["fs"] }
 nom = "7.1"
+parking_lot = "0.12"
 parse_duration = "2.1"
 pin-project = "1.0.12"
 pnet_datalink = "0.34"
@@ -84,12 +84,14 @@ pretty_env_logger = "0.5"
 structopt = { version = "0.3", default-features = false }
 syslog-tracing = "0.3"
 tracing = "0.1"
+tracing-journald = "0.3.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 heed = { version = "0.11", default-features = false, features = ["lmdb"] }
-rusqlite = "0.31.0"
+rusqlite = "0.37"
 r2d2 = "0.8"
-r2d2_sqlite = "0.24"
+r2d2_sqlite = "0.31"
+fjall = "2.4"
 
 async-compression = { version = "0.4", features = ["tokio", "zstd"] }
 zstd = { version = "0.13", default-features = false }
@@ -135,7 +137,7 @@ prometheus = "0.13"
 aws-sigv4 = { version = "1.1", default-features = false }
 hyper-rustls = { version = "0.26", default-features = false, features = ["http1", "http2", "ring", "rustls-native-certs"] }
 log = "0.4"
-thiserror = "1.0"
+thiserror = "2.0"
 
 # ---- used only as build / dev dependencies ----
 assert-json-diff = "2.0"
@@ -145,12 +147,8 @@ aws-smithy-runtime = { version = "1.8", default-features = false, features = ["t
 aws-sdk-config = { version = "1.62", default-features = false }
 aws-sdk-s3 = { version = "1.79", default-features = false, features = ["rt-tokio"] }
 
-[profile.dev]
-#lto = "thin"	# disabled for now, adds 2-4 min to each CI build
-lto = "off"
-
 [profile.release]
-lto = true
-codegen-units = 1
-opt-level = "s"
-strip = true
+lto = "thin"
+codegen-units = 16
+opt-level = 3
+strip = "debuginfo"

doc/book/connect/apps/index.md

@@ -12,7 +12,7 @@ In this section, we cover the following web applications:
 | [Mastodon](#mastodon)     | ✅       | Natively supported    |
 | [Matrix](#matrix)     | ✅       |  Tested with `synapse-s3-storage-provider`    |
 | [ejabberd](#ejabberd)     | ✅       |  `mod_s3_upload`    |
-| [Pixelfed](#pixelfed)     | ❓       |  Not yet tested    |
+| [Pixelfed](#pixelfed)     | ✅       |  Natively supported    |
 | [Pleroma](#pleroma)     | ❓       |  Not yet tested    |
 | [Lemmy](#lemmy)     | ✅        |  Supported with pict-rs    |
 | [Funkwhale](#funkwhale)     | ❓       | Not yet tested     |
@@ -69,7 +69,7 @@ $CONFIG = array(
         'hostname' => '127.0.0.1', // Can also be a domain name, eg. garage.example.com
         'port' => 3900,            // Put your reverse proxy port or your S3 API port
         'use_ssl' => false,        // Set it to true if you have a TLS enabled reverse proxy
-        'region' => 'garage',      // Garage has only one region named "garage"
+        'region' => 'garage',      // Garage default region is named "garage", edit according to your cluster config
         'use_path_style' => true   // Garage supports only path style, must be set to true
     ],
 ],
@@ -135,7 +135,7 @@ bucket but doesn't also know the secret encryption key.
 *Click on the picture to zoom*
 
 Add a new external storage. Put what you want in "folder name" (eg. "shared"). Select "Amazon S3". Keep "Access Key" for the Authentication field.
-In Configuration, put your bucket name (eg. nextcloud), the host (eg. 127.0.0.1), the port (eg. 3900 or 443), the region (garage). Tick the SSL box if you have put an HTTPS proxy in front of garage. You must tick the "Path access" box and you must leave the "Legacy authentication (v2)" box empty. Put your Key ID (eg. GK...) and your Secret Key in the last two input boxes. Finally click on the tick symbol on the right of your screen.
+In Configuration, put your bucket name (eg. nextcloud), the host (eg. 127.0.0.1), the port (eg. 3900 or 443), the region ("garage" if you use the default, or the one your configured in your `garage.toml`). Tick the SSL box if you have put an HTTPS proxy in front of garage. You must tick the "Path access" box and you must leave the "Legacy authentication (v2)" box empty. Put your Key ID (eg. GK...) and your Secret Key in the last two input boxes. Finally click on the tick symbol on the right of your screen.
 
 Now go to your "Files" app and a new "linked folder" has appeared with the name you chose earlier (eg. "shared").
 
@@ -191,10 +191,10 @@ garage key create peertube-key
 
 Keep the Key ID and the Secret key in a pad, they will be needed later.  
 
-We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist).
+We need two buckets, one for normal videos (named peertube-videos) and one for webtorrent videos (named peertube-playlists).
 ```bash
 garage bucket create peertube-videos
-garage bucket create peertube-playlist
+garage bucket create peertube-playlists
 ```
 
 Now we allow our key to read and write on these buckets:
@@ -238,7 +238,7 @@ object_storage:
   # Put localhost only if you have a garage instance running on that node
   endpoint: 'http://localhost:3900' # or "garage.example.com" if you have TLS on port 443
 
-  # Garage supports only one region for now, named garage
+  # Garage default region is named "garage", edit according to your config
   region: 'garage'
 
   credentials:
@@ -253,7 +253,7 @@ object_storage:
     proxify_private_files: false
 
   streaming_playlists:
-    bucket_name: 'peertube-playlist'
+    bucket_name: 'peertube-playlists'
 
     # Keep it empty for our example
     prefix: ''
@@ -441,7 +441,7 @@ media_storage_providers:
   store_synchronous: True  # do we want to wait that the file has been written before returning?
   config:
     bucket: matrix       # the name of our bucket, we chose matrix earlier
-    region_name: garage  # only "garage" is supported for the region field
+    region_name: garage  # "garage" by default, edit according to your cluster config
     endpoint_url: http://localhost:3900 # the path to the S3 endpoint
     access_key_id: "GKxxx" # your Key ID
     secret_access_key: "xxxx" # your Secret Key

doc/book/connect/backup.md

@@ -161,3 +161,49 @@ kopia repository validate-provider
 
 You can then run all the standard kopia commands: `kopia snapshot create`, `kopia mount`...
 Everything should work out-of-the-box.
+
+## Plakar
+
+Create your key and bucket on Garage server:
+
+```bash
+garage key create my-plakar-key
+garage bucket create plakar-backups
+garage bucket allow plakar-backups --read --write --key my-plakar-key
+…
+```
+
+On Plakar server, add your Garage as a storage location:
+```bash
+plakar store add garageS3 s3://my-garage.tld/plakar-backups \
+region=garage # Or as you've specified in garage.toml \
+access_key=<Key ID from "garage key info my-plakar-key"> \
+secret_access_key=<Secret key from "garage key info my-plakar-key">
+```
+
+Then create the repository.
+```bash
+plakar at @garageS3 create -plaintext # Unencrypted
+# or
+plakar at @garageS3 create #encrypted
+```
+
+If you encrypt your backups (Plakar default), you will need to define a strong passphrase. Do not forget to save your password safely. It will be needed to decrypt your backups.
+
+
+After the repository has been created, check that everything works as expected (that might give an empty result as no file has been added yet, but no error message):
+```bash
+plakar at @garageS3 check
+```
+
+Now that everything is configure, you can use Garage as your backups storage. For instance sync it with a local backup storage:
+```bash
+$ plakar at ~/backups sync to @garageS3
+```
+
+Or list the S3 storage content:
+```bash
+$ plakar at @garageS3 ls
+```
+
+More information in Plakar documentation: https://www.plakar.io/docs/main/quickstart/

doc/book/cookbook/ansible.md

@@ -8,18 +8,18 @@ have published Ansible roles.  We list them and compare them below.
 
 ## Comparison of Ansible roles
 
-| Feature                            | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) |
-|------------------------------------|---------------------------------------------|---------------------------------------------------------------|
-| **Runtime**                        | Systemd                                     | Docker                                                        |
-| **Target OS**                      | Any Linux                                   | Any Linux                                                     |
-| **Architecture**                   | amd64, arm64, i686                          | amd64, arm64                                                  |
-| **Additional software**            | None                                        | Traefik                                                       |
-| **Automatic node connection**      | ❌                                          | ✅                                                            |
-| **Layout management**              | ❌                                          | ✅                                                            |
-| **Manage buckets & keys**          | ❌                                          | ✅ (basic)                                                    |
-| **Allow custom Garage config**     | ✅                                          | ❌                                                            |
-| **Facilitate Garage upgrades**     | ✅                                          | ❌                                                            |
-| **Multiple instances on one host** | ✅                                          | ✅                                                            |
+| Feature                            | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | [eddster ansible-role-garage](#eddster-ansible-role-garage) |
+|------------------------------------|---------------------------------------------|---------------------------------------------------------------|---------------------------------|
+| **Runtime**                        | Systemd                                     | Docker                                                        | Systemd                         |
+| **Target OS**                      | Any Linux                                   | Any Linux                                                     | Any Linux                       |
+| **Architecture**                   | amd64, arm64, i686                          | amd64, arm64                                                  | arm64, arm, 386, amd64                            |
+| **Additional software**            | None                                        | Traefik                                                       | Ngnix and Keepalived (optional) |
+| **Automatic node connection**      | ❌                                          | ✅                                                            | ✅                              |
+| **Layout management**              | ❌                                          | ✅                                                            | ✅                              |
+| **Manage buckets & keys**          | ❌                                          | ✅ (basic)                                                    | ✅                              |
+| **Allow custom Garage config**     | ✅                                          | ❌                                                            | ❌                              |
+| **Facilitate Garage upgrades**     | ✅                                          | ❌                                                            | ✅                              |
+| **Multiple instances on one host** | ✅                                          | ✅                                                            | ❌                              |
 
 
 ## zorun/ansible-role-garage
@@ -49,3 +49,15 @@ structured DNS names, etc).
 
 As a result, this role makes it easier to start with Garage on Ansible,
 but is less flexible.
+
+## eddster2309/ansible-role-garage
+
+[Source code](https://github.com/eddster2309/ansible-role-garage), [Ansible galaxy](https://galaxy.ansible.com/ui/standalone/roles/eddster2309/garage/)
+
+This role is a opinionated but customisable role using the official Garage
+static binaries and only requires Systemd. As such it should work on any
+Linux based host. It includes all the nesscary configuration to
+automatically setup a clustered Garage deployment. Most Garage
+configuration options are exposed through Ansible variables so while you
+can't provide a custom config you can get very close. It can optionally
+installed a HA nginx deployment with Keepalived.

doc/book/cookbook/binary-packages.md

@@ -15,9 +15,10 @@ Alpine Linux repositories (available since v3.17):
 apk add garage
 ```
 
-The default configuration file is installed to `/etc/garage.toml`. You can run
-Garage using: `rc-service garage start`. If you don't specify `rpc_secret`, it
-will be automatically replaced with a random string on the first start.
+The default configuration file is installed to `/etc/garage/garage.toml`. You can run
+Garage using: `rc-service garage start`.
+
+If you don't specify `rpc_secret`, it will be automatically replaced with a random string on the first start.
 
 Please note that this package is built without Consul discovery, Kubernetes
 discovery, OpenTelemetry exporter, and K2V features (K2V will be enabled once
@@ -26,7 +27,7 @@ it's stable).
 
 ## Arch Linux
 
-Garage is available in the [AUR](https://aur.archlinux.org/packages/garage).
+Garage is available in the official repositories under [extra](https://archlinux.org/packages/extra/x86_64/garage).
 
 ## FreeBSD
 

doc/book/cookbook/kubernetes.md

@@ -11,7 +11,7 @@ Firstly clone the repository:
 
 ```bash
 git clone https://git.deuxfleurs.fr/Deuxfleurs/garage
-cd garage/scripts/helm
+cd garage/script/helm
 ```
 
 Deploy with default options:
@@ -26,6 +26,13 @@ Or deploy with custom values:
 helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml
 ```
 
+If you want to manage the CustomRessourceDefinition used by garage for its `kubernetes_discovery` outside of the helm chart, add `garage.kubernetesSkipCrd: true` to your custom values and use the kustomization before deploying the helm chart:
+
+```bash
+kubectl apply -k ../k8s/crd
+helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml
+```
+
 After deploying, cluster layout must be configured manually as described in [Creating a cluster layout](@/documentation/quick-start/_index.md#creating-a-cluster-layout). Use the following command to access garage CLI:
 
 ```bash

doc/book/cookbook/real-world.md

@@ -96,14 +96,14 @@ to store 2 TB of data in total.
 ## Get a Docker image
 
 Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
-We encourage you to use a fixed tag (eg. `v1.1.0`) and not the `latest` tag.
-For this example, we will use the latest published version at the time of the writing which is `v1.1.0` but it's up to you
+We encourage you to use a fixed tag (eg. `v1.3.0`) and not the `latest` tag.
+For this example, we will use the latest published version at the time of the writing which is `v1.3.0` but it's up to you
 to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
 
 For example:
 
 ```
-sudo docker pull dxflrs/garage:v1.1.0
+sudo docker pull dxflrs/garage:v1.3.0
 ```
 
 ## Deploying and configuring Garage
@@ -171,7 +171,7 @@ docker run \
   -v /etc/garage.toml:/etc/garage.toml \
   -v /var/lib/garage/meta:/var/lib/garage/meta \
   -v /var/lib/garage/data:/var/lib/garage/data \
-  dxflrs/garage:v1.1.0
+  dxflrs/garage:v1.3.0
 ```
 
 With this command line, Garage should be started automatically at each boot.
@@ -185,7 +185,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
 version: "3"
 services:
   garage:
-    image: dxflrs/garage:v1.1.0
+    image: dxflrs/garage:v1.3.0
     network_mode: "host"
     restart: unless-stopped
     volumes:

doc/book/cookbook/systemd.md

@@ -28,6 +28,7 @@ StateDirectory=garage
 DynamicUser=true
 ProtectHome=true
 NoNewPrivileges=true
+LimitNOFILE=42000
 
 [Install]
 WantedBy=multi-user.target

doc/book/quick-start/_index.md

@@ -129,10 +129,10 @@ docker run \
   -d \
   --name garaged \
   -p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903 \
-  -v /etc/garage.toml:/path/to/garage.toml \
-  -v /var/lib/garage/meta:/path/to/garage/meta \
-  -v /var/lib/garage/data:/path/to/garage/data \
-  dxflrs/garage:v1.1.0
+  -v /path/to/garage.toml:/etc/garage.toml \
+  -v /path/to/garage/meta:/var/lib/garage/meta \
+  -v /path/to/garage/data:/var/lib/garage/data \
+  dxflrs/garage:v1.3.0
 ```
 
 Under Linux, you can substitute `--network host` for `-p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903`
@@ -182,11 +182,12 @@ ID                 Hostname  Address         Tag                   Zone  Capacit
 ## Creating a cluster layout
 
 Creating a cluster layout for a Garage deployment means informing Garage
-of the disk space available on each node of the cluster
-as well as the zone (e.g. datacenter) each machine is located in.
+of the disk space available on each node of the cluster, `-c`,
+as well as the name of the zone (e.g. datacenter), `-z`, each machine is located in.
 
-For our test deployment, we are using only one node. The way in which we configure
-it does not matter, you can simply write:
+For our test deployment, we are have only one node with zone named `dc1` and a
+capacity of `1G`, though the capacity is ignored for a single node deployment
+and can be changed later when adding new nodes.
 
 ```bash
 garage layout assign -z dc1 -c 1G <node_id>

doc/book/reference-manual/configuration.md

@@ -24,7 +24,8 @@ db_engine = "lmdb"
 
 block_size = "1M"
 block_ram_buffer_max = "256MiB"
-
+block_max_concurrent_reads = 16
+block_max_concurrent_writes_per_request =10
 lmdb_map_size = "1T"
 
 compression_level = 1
@@ -46,6 +47,7 @@ bootstrap_peers = [
     "212fd62eeaca72c122b45a7f4fa0f55e012aa5e24ac384a72a3016413fa724ff@[fc00:F::1]:3901",
 ]
 
+allow_punycode = false
 
 [consul_discovery]
 api = "catalog"
@@ -92,29 +94,32 @@ The following gives details about each available configuration option.
 
 [Environment variables](#env_variables).
 
-Top-level configuration options:
+Top-level configuration options, in alphabetical order:
+[`allow_punycode`](#allow_punycode),
 [`allow_world_readable_secrets`](#allow_world_readable_secrets),
+[`block_max_concurrent_reads`](`block_max_concurrent_reads),
 [`block_ram_buffer_max`](#block_ram_buffer_max),
+[`block_max_concurrent_writes_per_request`](#block_max_concurrent_writes_per_request),
 [`block_size`](#block_size),
 [`bootstrap_peers`](#bootstrap_peers),
 [`compression_level`](#compression_level),
+[`consistency_mode`](#consistency_mode),
 [`data_dir`](#data_dir),
 [`data_fsync`](#data_fsync),
 [`db_engine`](#db_engine),
 [`disable_scrub`](#disable_scrub),
-[`use_local_tz`](#use_local_tz),
 [`lmdb_map_size`](#lmdb_map_size),
 [`metadata_auto_snapshot_interval`](#metadata_auto_snapshot_interval),
 [`metadata_dir`](#metadata_dir),
 [`metadata_fsync`](#metadata_fsync),
 [`metadata_snapshots_dir`](#metadata_snapshots_dir),
 [`replication_factor`](#replication_factor),
-[`consistency_mode`](#consistency_mode),
 [`rpc_bind_addr`](#rpc_bind_addr),
 [`rpc_bind_outgoing`](#rpc_bind_outgoing),
 [`rpc_public_addr`](#rpc_public_addr),
 [`rpc_public_addr_subnet`](#rpc_public_addr_subnet)
-[`rpc_secret`/`rpc_secret_file`](#rpc_secret).
+[`rpc_secret`/`rpc_secret_file`](#rpc_secret),
+[`use_local_tz`](#use_local_tz).
 
 The `[consul_discovery]` section:
 [`api`](#consul_api),
@@ -151,13 +156,17 @@ The `[admin]` section:
 
 ### Environment variables {#env_variables}
 
-The following configuration parameter must be specified as an environment
-variable, it does not exist in the configuration file:
+The following configuration parameters must be specified as environment variables,
+they do not exist in the configuration file:
 
 - `GARAGE_LOG_TO_SYSLOG` (since `v0.9.4`): set this to `1` or `true` to make the
   Garage daemon send its logs to `syslog` (using the libc `syslog` function)
   instead of printing to stderr.
 
+- `GARAGE_LOG_TO_JOURNALD` (since `v1.2.0`): set this to `1` or `true` to make the
+  Garage daemon send its logs to `journald` (using the native protocol of `systemd-journald`)
+  instead of printing to stderr.
+
 The following environment variables can be used to override the corresponding
 values in the configuration file:
 
@@ -169,7 +178,7 @@ values in the configuration file:
 
 ### Top-level configuration options
 
-#### `replication_factor` {#replication_factor}
+#### `replication_factor` (since `v1.0.0`) {#replication_factor}
 
 The replication factor can be any positive integer smaller or equal the node count in your cluster.
 The chosen replication factor has a big impact on the cluster's failure tolerancy and performance characteristics.
@@ -217,7 +226,7 @@ is in progress.  In theory, no data should be lost as rebalancing is a
 routine operation for Garage, although we cannot guarantee you that everything
  will go right in such an extreme scenario.
 
-#### `consistency_mode` {#consistency_mode}
+#### `consistency_mode` (since `v1.0.0`) {#consistency_mode}
 
 The consistency mode setting determines the read and write behaviour of your cluster.
 
@@ -327,6 +336,7 @@ Since `v0.8.0`, Garage can use alternative storage backends as follows:
 | --------- | ----------------- | ------------- |
 | [LMDB](https://www.symas.com/lmdb) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` |
 | [Sqlite](https://sqlite.org) (since `v0.8.0`) | `"sqlite"` | `<metadata_dir>/db.sqlite` |
+| [Fjall](https://github.com/fjall-rs/fjall) (**experimental support** since `v1.3.0`) | `"fjall"` | `<metadata_dir>/db.fjall/` |
 | [Sled](https://sled.rs) (old default, removed since `v1.0`) | `"sled"` | `<metadata_dir>/db/` |
 
 Sled was supported until Garage v0.9.x, and was removed in Garage v1.0.
@@ -363,6 +373,14 @@ LMDB works very well, but is known to have the following limitations:
   so it is not the best choice for high-performance storage clusters,
   but it should work fine in many cases.
 
+- Fjall: a storage engine based on LSM trees, which theoretically allow for
+  higher write throughput than other storage engines that are based on B-trees.
+  Using Fjall could potentially improve Garage's performance significantly in
+  write-heavy workloads. **Support for Fjall is experimental at this point**,
+  we have added it to Garage for evaluation purposes only. **Do not use it for
+  production-critical workloads.**
+
+
 It is possible to convert Garage's metadata directory from one format to another
 using the `garage convert-db` command, which should be used as follows:
 
@@ -400,6 +418,7 @@ Here is how this option impacts the different database engines:
 |----------|------------------------------------|-------------------------------|
 | Sqlite   | `PRAGMA synchronous = OFF`         | `PRAGMA synchronous = NORMAL` |
 | LMDB     | `MDB_NOMETASYNC` + `MDB_NOSYNC`    | `MDB_NOMETASYNC`              |
+| Fjall    | default options                    | not supported                 |
 
 Note that the Sqlite database is always ran in `WAL` mode (`PRAGMA journal_mode = WAL`).
 
@@ -506,6 +525,37 @@ node.
 
 The default value is 256MiB.
 
+#### `block_max_concurrent_reads` (since `v1.3.0` / `v2.1.0`) {#block_max_concurrent_reads}
+
+The maximum number of blocks (individual files in the data directory) open
+simultaneously for reading.
+
+Reducing this number does not limit the number of data blocks that can be
+transferred through the network simultaneously. This mechanism was just added
+as a backpressure mechanism for HDD read speed: it helps avoid a situation
+where too many requests are coming in and Garage is reading too many block
+files simultaneously, thus not making timely progress on any of the reads.
+
+When a request to read a data block comes in through the network, the requests
+awaits for one of the `block_max_concurrent_reads` slots to be available
+(internally implemented using a Semaphore object). Once it acquired a read
+slot, it reads the entire block file to RAM and frees the slot as soon as the
+block file is finished reading. Only after the slot is released will the
+block's data start being transferred over the network.  If the request fails to
+acquire a reading slot wihtin 15 seconds, it fails with a timeout error.
+Timeout events can be monitored through the `block_read_semaphore_timeouts`
+metric in Prometheus: a non-zero number of such events indicates an I/O
+bottleneck on HDD read speed.
+
+
+#### `block_max_concurrent_writes_per_request` (since  `v2.1.0`) {#block_max_concurrent_writes_per_request}
+
+This parameter is designed to adapt to the concurrent write performance of 
+different storage media.Maximum number of parallel block writes per put request 
+Higher values improve throughput but increase memory usage.
+
+Default: 3, Recommended: 10-30 for NVMe, 3-10 for HDD
+
 #### `lmdb_map_size` {#lmdb_map_size}
 
 This parameters can be used to set the map size used by LMDB,
@@ -604,7 +654,7 @@ be obtained by running `garage node id` and then included directly in the
 key will be returned by `garage node id` and you will have to add the IP
 yourself.
 
-### `allow_world_readable_secrets` or `GARAGE_ALLOW_WORLD_READABLE_SECRETS` (env) {#allow_world_readable_secrets}
+#### `allow_world_readable_secrets` or `GARAGE_ALLOW_WORLD_READABLE_SECRETS` (env) {#allow_world_readable_secrets}
 
 Garage checks the permissions of your secret files to make sure they're not
 world-readable. In some cases, the check might fail and consider your files as
@@ -616,6 +666,13 @@ permission verification.
 Alternatively, you can set the `GARAGE_ALLOW_WORLD_READABLE_SECRETS`
 environment variable to `true` to bypass the permissions check.
 
+#### `allow_punycode` {#allow_punycode}
+
+Allow creating buckets with names containing punycode. When used for buckets served
+as websites, this allows using almost any unicode character in the domain name.
+
+Default to `false`.
+
 ### The `[consul_discovery]` section
 
 Garage supports discovering other nodes of the cluster using Consul.  For this

doc/book/reference-manual/s3-compatibility.md

@@ -23,17 +23,17 @@ Feel free to open a PR to suggest fixes this table. Minio is missing because the
 - 2022-05-25 - Many Ceph S3 endpoints are not documented but implemented. Following a notification from the Ceph community, we added them.
 
 
-
 ## High-level features
 
 | Feature                      | Garage                           | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) |
 |------------------------------|----------------------------------|-----------------|---------------|---------|-----|
-| [signature v2](https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html) (deprecated) | ❌ Missing | ✅ |  ✅ | ✅ | ✅ |
+| [signature v2](https://docs.aws.amazon.com/AmazonS3/latest/API/Appendix-Sigv2.html) (deprecated) | ❌ Missing | ✅ |  ✅ | ✅ | ✅ |
 | [signature v4](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) |  ✅ Implemented |  ✅ |  ✅ | ❌ | ✅ |
 | [URL path-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access) (eg. `host.tld/bucket/key`) |  ✅ Implemented | ✅ |  ✅ | ❓| ✅ |
 | [URL vhost-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access) URL (eg. `bucket.host.tld/key`) |  ✅ Implemented | ❌| ✅| ✅ | ✅ |
 | [Presigned URLs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ShareObjectPreSignedURL.html) |  ✅ Implemented | ❌|  ✅ | ✅ |  ✅(❓) |
 | [SSE-C encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html) |  ✅ Implemented | ❓ |  ✅ | ❌ |  ✅ |
+| [Bucket versioning](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html) | ❌ Missing | ✅ |  ✅ | ❌ | ✅ |
 
 *Note:* OpenIO does not says if it supports presigned URLs. Because it is part
 of signature v4 and they claim they support it without additional precisions,

doc/drafts/admin-api.md

@@ -70,7 +70,7 @@ Example response body:
 ```json
 {
   "node": "b10c110e4e854e5aa3f4637681befac755154b20059ec163254ddbfae86b09df",
-  "garageVersion": "v1.1.0",
+  "garageVersion": "v1.3.0",
   "garageFeatures": [
     "k2v",
     "lmdb",

flake.lock

@@ -50,17 +50,17 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1736692550,
-        "narHash": "sha256-7tk8xH+g0sJkKLTJFOxphJxxOjMDFMWv24nXslaU2ro=",
+        "lastModified": 1763977559,
+        "narHash": "sha256-g4MKqsIRy5yJwEsI+fYODqLUnAqIY4kZai0nldAP6EM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "7c4869c47090dd7f9f1bdfb49a22aea026996815",
+        "rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "7c4869c47090dd7f9f1bdfb49a22aea026996815",
+        "rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632",
         "type": "github"
       }
     },
@@ -80,17 +80,17 @@
         ]
       },
       "locked": {
-        "lastModified": 1738549608,
-        "narHash": "sha256-GdyT9QEUSx5k/n8kILuNy83vxxdyUfJ8jL5mMpQZWfw=",
+        "lastModified": 1763952169,
+        "narHash": "sha256-+PeDBD8P+NKauH+w7eO/QWCIp8Cx4mCfWnh9sJmy9CM=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "35c6f8c4352f995ecd53896200769f80a3e8f22d",
+        "rev": "ab726555a9a72e6dc80649809147823a813fa95b",
         "type": "github"
       },
       "original": {
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "35c6f8c4352f995ecd53896200769f80a3e8f22d",
+        "rev": "ab726555a9a72e6dc80649809147823a813fa95b",
         "type": "github"
       }
     },

flake.nix

@@ -2,13 +2,13 @@
   description =
     "Garage, an S3-compatible distributed object store for self-hosted deployments";
 
-  # Nixpkgs 24.11 as of 2025-01-12
+  # Nixpkgs 25.05 as of 2025-11-24
   inputs.nixpkgs.url =
-    "github:NixOS/nixpkgs/7c4869c47090dd7f9f1bdfb49a22aea026996815";
+    "github:NixOS/nixpkgs/cfe2c7d5b5d3032862254e68c37a6576b633d632";
 
-  # Rust overlay as of 2025-02-03
+  # Rust overlay as of 2025-11-24
   inputs.rust-overlay.url =
-  "github:oxalica/rust-overlay/35c6f8c4352f995ecd53896200769f80a3e8f22d";
+  "github:oxalica/rust-overlay/ab726555a9a72e6dc80649809147823a813fa95b";
   inputs.rust-overlay.inputs.nixpkgs.follows = "nixpkgs";
 
   inputs.crane.url = "github:ipetkov/crane";
@@ -30,6 +30,10 @@
           inherit system nixpkgs crane rust-overlay extraTestEnv;
           release = false;
         }).garage-test;
+        lints = (compile {
+          inherit system nixpkgs crane rust-overlay;
+          release = false;
+        });
       in
       {
         packages = {
@@ -53,6 +57,13 @@
           tests-sqlite = testWith {
             GARAGE_TEST_INTEGRATION_DB_ENGINE = "sqlite";
           };
+          tests-fjall = testWith {
+            GARAGE_TEST_INTEGRATION_DB_ENGINE = "fjall";
+          };
+
+          # lints (fmt, clippy)
+          fmt = lints.garage-cargo-fmt;
+          clippy = lints.garage-cargo-clippy;
         };
 
         # ---- developpment shell, for making native builds only ----

nix/compile.nix

@@ -48,7 +48,7 @@ let
 
   inherit (pkgs) lib stdenv;
 
-  toolchainFn = (p: p.rust-bin.stable."1.82.0".default.override {
+  toolchainFn = (p: p.rust-bin.stable."1.91.0".default.override {
     targets = lib.optionals (target != null) [ rustTarget ];
     extensions = [
       "rust-src"
@@ -68,12 +68,13 @@ let
   rootFeatures = if features != null then
     features
   else
-    ([ "bundled-libs" "lmdb" "sqlite" "k2v" ] ++ (lib.optionals release [
+    ([ "bundled-libs" "lmdb" "sqlite" "fjall" "k2v" ] ++ (lib.optionals release [
       "consul-discovery"
       "kubernetes-discovery"
       "metrics"
       "telemetry-otlp"
       "syslog"
+      "journald"
     ]));
 
   featuresStr = lib.concatStringsSep "," rootFeatures;
@@ -189,4 +190,15 @@ in rec {
       pkgs.cacert
     ];
   } // extraTestEnv);
+
+  # ---- source code linting ----
+
+  garage-cargo-fmt = craneLib.cargoFmt (commonArgs // {
+    cargoExtraArgs = "";
+  });
+
+  garage-cargo-clippy = craneLib.cargoClippy (commonArgs // {
+    cargoArtifacts = garage-deps;
+    cargoClippyExtraArgs = "--all-targets -- -D warnings";
+  });
 }

script/dev-env-aws.sh

@@ -1,6 +1,7 @@
 export AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
 export AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
 export AWS_DEFAULT_REGION='garage'
+export AWS_REQUEST_CHECKSUM_CALCULATION='when_required'
 # FUTUREWORK: set AWS_ENDPOINT_URL instead, once nixpkgs bumps awscli to >=2.13.0.
 function aws { command aws --endpoint-url http://127.0.0.1:3911 $@ ; }