Merge pull request 'fix reflected xss when returning errors on web endpoint' (#1472) from escape-errors-v1 into main-v1

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1472

.drone.yml

@@ -1,288 +0,0 @@
----
-kind: pipeline
-name: default
-
-node:
-  nix-daemon: 1
-
-steps:
-  - name: check formatting
-    image: nixpkgs/nix:nixos-22.05
-    commands:
-      - nix-shell --attr rust --run "cargo fmt -- --check"
-
-  - name: build
-    image: nixpkgs/nix:nixos-22.05
-    commands:
-      - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
-
-  - name: unit + func tests
-    image: nixpkgs/nix:nixos-22.05
-    environment:
-      GARAGE_TEST_INTEGRATION_EXE: result-bin/bin/garage
-    commands:
-      - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-build --no-build-output --attr test.amd64
-      - ./result/bin/garage_db-*
-      - ./result/bin/garage_api-*
-      - ./result/bin/garage_model-*
-      - ./result/bin/garage_rpc-*
-      - ./result/bin/garage_table-*
-      - ./result/bin/garage_util-*
-      - ./result/bin/garage_web-*
-      - ./result/bin/garage-*
-      - ./result/bin/integration-*
-      - rm result
-
-  - name: integration tests
-    image: nixpkgs/nix:nixos-22.05
-    commands:
-      - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
-
-trigger:
-  event:
-  - custom
-  - push
-  - pull_request
-  - tag
-  - cron
-
----
-kind: pipeline
-type: docker
-name: release-linux-amd64
-
-node:
-  nix-daemon: 1
-
-steps:
-  - name: build
-    image: nixpkgs/nix:nixos-22.05
-    commands:
-      - nix-build --no-build-output --attr pkgs.amd64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
-
-  - name: integration
-    image: nixpkgs/nix:nixos-22.05
-    commands:
-      - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
-
-  - name: push static binary
-    image: nixpkgs/nix:nixos-22.05
-    environment:
-      AWS_ACCESS_KEY_ID:
-        from_secret: garagehq_aws_access_key_id
-      AWS_SECRET_ACCESS_KEY:
-        from_secret: garagehq_aws_secret_access_key
-      TARGET: "x86_64-unknown-linux-musl"
-    commands:
-      - nix-shell --attr release --run "to_s3"
-
-  - name: docker build and publish
-    image: nixpkgs/nix:nixos-22.05
-    environment:
-      DOCKER_AUTH:
-        from_secret: docker_auth
-      DOCKER_PLATFORM: "linux/amd64"
-      CONTAINER_NAME: "dxflrs/amd64_garage"
-      HOME: "/kaniko"
-    commands:
-      - mkdir -p /kaniko/.docker
-      - echo $DOCKER_AUTH > /kaniko/.docker/config.json
-      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr release --run "to_docker"
-
-
-trigger:
-  event:
-  - promote
-  - cron
-
----
-kind: pipeline
-type: docker
-name: release-linux-i386
-
-node:
-  nix-daemon: 1
-
-steps:
-  - name: build
-    image: nixpkgs/nix:nixos-22.05
-    commands:
-      - nix-build --no-build-output --attr pkgs.i386.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
-
-  - name: integration
-    image: nixpkgs/nix:nixos-22.05
-    commands:
-      - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
-
-  - name: push static binary
-    image: nixpkgs/nix:nixos-22.05
-    environment:
-      AWS_ACCESS_KEY_ID:
-        from_secret: garagehq_aws_access_key_id
-      AWS_SECRET_ACCESS_KEY:
-        from_secret: garagehq_aws_secret_access_key
-      TARGET: "i686-unknown-linux-musl"
-    commands:
-      - nix-shell --attr release --run "to_s3"
-
-  - name: docker build and publish
-    image: nixpkgs/nix:nixos-22.05
-    environment:
-      DOCKER_AUTH:
-        from_secret: docker_auth
-      DOCKER_PLATFORM: "linux/386"
-      CONTAINER_NAME: "dxflrs/386_garage"
-      HOME: "/kaniko"
-    commands:
-      - mkdir -p /kaniko/.docker
-      - echo $DOCKER_AUTH > /kaniko/.docker/config.json
-      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr release --run "to_docker"
-
-trigger:
-  event:
-  - promote
-  - cron
-
----
-kind: pipeline
-type: docker
-name: release-linux-arm64
-
-node:
-  nix-daemon: 1
-
-steps:
-  - name: build
-    image: nixpkgs/nix:nixos-22.05
-    commands:
-      - nix-build --no-build-output --attr pkgs.arm64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
-
-  - name: push static binary
-    image: nixpkgs/nix:nixos-22.05
-    environment:
-      AWS_ACCESS_KEY_ID:
-        from_secret: garagehq_aws_access_key_id
-      AWS_SECRET_ACCESS_KEY:
-        from_secret: garagehq_aws_secret_access_key
-      TARGET: "aarch64-unknown-linux-musl"
-    commands:
-      - nix-shell --attr release --run "to_s3"
-
-  - name: docker build and publish
-    image: nixpkgs/nix:nixos-22.05
-    environment:
-      DOCKER_AUTH:
-        from_secret: docker_auth
-      DOCKER_PLATFORM: "linux/arm64"
-      CONTAINER_NAME: "dxflrs/arm64_garage"
-      HOME: "/kaniko"
-    commands:
-      - mkdir -p /kaniko/.docker
-      - echo $DOCKER_AUTH > /kaniko/.docker/config.json
-      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr release --run "to_docker"
-
-trigger:
-  event:
-  - promote
-  - cron
-
----
-kind: pipeline
-type: docker
-name: release-linux-arm
-
-node:
-  nix-daemon: 1
-
-steps:
-  - name: build
-    image: nixpkgs/nix:nixos-22.05
-    commands:
-      - nix-build --no-build-output --attr pkgs.arm.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
-
-  - name: push static binary
-    image: nixpkgs/nix:nixos-22.05
-    environment:
-      AWS_ACCESS_KEY_ID:
-        from_secret: garagehq_aws_access_key_id
-      AWS_SECRET_ACCESS_KEY:
-        from_secret: garagehq_aws_secret_access_key
-      TARGET: "armv6l-unknown-linux-musleabihf"
-    commands:
-      - nix-shell --attr release --run "to_s3"
-
-  - name: docker build and publish
-    image: nixpkgs/nix:nixos-22.05
-    environment:
-      DOCKER_AUTH:
-        from_secret: docker_auth
-      DOCKER_PLATFORM: "linux/arm"
-      CONTAINER_NAME: "dxflrs/arm_garage"
-      HOME: "/kaniko"
-    commands:
-      - mkdir -p /kaniko/.docker
-      - echo $DOCKER_AUTH > /kaniko/.docker/config.json
-      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr release --run "to_docker"
-
-trigger:
-  event:
-  - promote
-  - cron
-
----
-kind: pipeline
-type: docker
-name: refresh-release-page
-
-node:
-  nix-daemon: 1
-
-steps:
-  - name: multiarch-docker
-    image: nixpkgs/nix:nixos-22.05
-    environment:
-      DOCKER_AUTH:
-        from_secret: docker_auth
-      HOME: "/root"
-    commands:
-      - mkdir -p /root/.docker
-      - echo $DOCKER_AUTH > /root/.docker/config.json
-      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr release --run "multiarch_docker"
-  - name: refresh-index
-    image: nixpkgs/nix:nixos-22.05
-    environment:
-      AWS_ACCESS_KEY_ID:
-        from_secret: garagehq_aws_access_key_id
-      AWS_SECRET_ACCESS_KEY:
-        from_secret: garagehq_aws_secret_access_key
-    commands:
-      - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
-      - nix-shell --attr release --run "refresh_index"
-
-depends_on:
-  - release-linux-amd64
-  - release-linux-i386
-  - release-linux-arm64
-  - release-linux-arm
-
-trigger:
-  event:
-  - promote
-  - cron
-
----
-kind: signature
-hmac: ac09a5a8c82502f67271f93afa1e1e21ce66383b8e24a6deb26b285cc1c378ba
-
-...

.woodpecker/debug.yaml

@@ -0,0 +1,44 @@
+labels:
+  nix: "enabled"
+
+when:
+  event:
+  - push
+  - tag
+  - pull_request
+  - deployment
+  - cron
+  - manual
+
+steps:
+  - name: check formatting
+    image: nixpkgs/nix:nixos-24.05
+    commands:
+      - nix-build -j4 --attr flakePackages.fmt
+
+  - name: build
+    image: nixpkgs/nix:nixos-24.05
+    commands:
+      - nix-build -j4 --attr flakePackages.dev
+
+  - name: unit + func tests (lmdb)
+    image: nixpkgs/nix:nixos-24.05
+    commands:
+      - nix-build -j4 --attr flakePackages.tests-lmdb
+
+  - name: unit + func tests (sqlite)
+    image: nixpkgs/nix:nixos-24.05
+    commands:
+      - nix-build -j4 --attr flakePackages.tests-sqlite
+
+  - name: unit + func tests (fjall)
+    image: nixpkgs/nix:nixos-24.05
+    commands:
+      - nix-build -j4 --attr flakePackages.tests-fjall
+
+  - name: integration tests
+    image: nixpkgs/nix:nixos-24.05
+    commands:
+      - nix-build -j4 --attr flakePackages.dev
+      - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+    depends_on: [ build ]

.woodpecker/publish.yaml

@@ -0,0 +1,33 @@
+labels:
+  nix: "enabled"
+
+when:
+  event:
+  - deployment
+  - cron
+
+depends_on:
+  - release
+
+steps:
+  - name: refresh-index
+    image: nixpkgs/nix:nixos-24.05
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: garagehq_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: garagehq_aws_secret_access_key
+    commands:
+      - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
+      - nix-shell --attr ci --run "refresh_index"
+
+  - name: multiarch-docker
+    image: nixpkgs/nix:nixos-24.05
+    environment:
+      DOCKER_AUTH:
+        from_secret: docker_auth
+    commands:
+      - mkdir -p /root/.docker
+      - echo $DOCKER_AUTH > /root/.docker/config.json
+      - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
+      - nix-shell --attr ci --run "multiarch_docker"

.woodpecker/release.yaml

@@ -0,0 +1,71 @@
+labels:
+  nix: "enabled"
+
+when:
+  event:
+  - deployment
+  - cron
+
+matrix:
+  include:
+    - ARCH: amd64
+      TARGET: x86_64-unknown-linux-musl
+    - ARCH: i386
+      TARGET: i686-unknown-linux-musl
+    - ARCH: arm64
+      TARGET: aarch64-unknown-linux-musl
+    - ARCH: arm
+      TARGET: armv6l-unknown-linux-musleabihf
+
+steps:
+  - name: build
+    image: nixpkgs/nix:nixos-24.05
+    commands:
+      - nix-build --attr releasePackages.${ARCH} --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
+
+  - name: check is static binary
+    image: nixpkgs/nix:nixos-24.05
+    commands:
+      - nix-shell --attr ci --run "./script/not-dynamic.sh result/bin/garage"
+
+  - name: integration tests
+    image: nixpkgs/nix:nixos-24.05
+    commands:
+      - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+    when:
+      - matrix:
+          ARCH: amd64
+      - matrix:
+          ARCH: i386
+
+  - name: upgrade tests
+    image: nixpkgs/nix:nixos-24.05
+    commands:
+      - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
+    when:
+      - matrix:
+          ARCH: amd64
+
+  - name: push static binary
+    image: nixpkgs/nix:nixos-24.05
+    environment:
+      TARGET: "${TARGET}"
+      AWS_ACCESS_KEY_ID:
+        from_secret: garagehq_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: garagehq_aws_secret_access_key
+    commands:
+      - nix-shell --attr ci --run "to_s3"
+
+  - name: docker build and publish
+    image: nixpkgs/nix:nixos-24.05
+    environment:
+      DOCKER_PLATFORM: "linux/${ARCH}"
+      CONTAINER_NAME: "dxflrs/${ARCH}_garage"
+      DOCKER_AUTH:
+        from_secret: docker_auth
+    commands:
+      - mkdir -p /root/.docker
+      - echo $DOCKER_AUTH > /root/.docker/config.json
+      - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
+      - nix-shell --attr ci --run "to_docker"

Cargo.toml

@@ -3,20 +3,152 @@ resolver = "2"
 members = [
 	"src/db",
 	"src/util",
+	"src/net",
 	"src/rpc",
 	"src/table",
 	"src/block",
 	"src/model",
-	"src/api",
+	"src/api/common",
+	"src/api/s3",
+	"src/api/k2v",
+	"src/api/admin",
 	"src/web",
 	"src/garage",
 	"src/k2v-client",
+	"src/format-table",
 ]
 
 default-members = ["src/garage"]
 
-[profile.dev]
+[workspace.dependencies]
-lto = "off"
+
+# Internal Garage crates
+format_table = { version = "0.1.1", path = "src/format-table" }
+garage_api_common = { version = "1.3.1", path = "src/api/common" }
+garage_api_admin = { version = "1.3.1", path = "src/api/admin" }
+garage_api_s3 = { version = "1.3.1", path = "src/api/s3" }
+garage_api_k2v = { version = "1.3.1", path = "src/api/k2v" }
+garage_block = { version = "1.3.1", path = "src/block" }
+garage_db = { version = "1.3.1", path = "src/db", default-features = false }
+garage_model = { version = "1.3.1", path = "src/model", default-features = false }
+garage_net = { version = "1.3.1", path = "src/net" }
+garage_rpc = { version = "1.3.1", path = "src/rpc" }
+garage_table = { version = "1.3.1", path = "src/table" }
+garage_util = { version = "1.3.1", path = "src/util" }
+garage_web = { version = "1.3.1", path = "src/web" }
+k2v-client = { version = "0.0.4", path = "src/k2v-client" }
+
+# External crates from crates.io
+arc-swap = "1.0"
+argon2 = "0.5"
+async-trait = "0.1.7"
+backtrace = "0.3"
+base64 = "0.21"
+blake2 = "0.10"
+bytes = "1.0"
+bytesize = "1.1"
+cfg-if = "1.0"
+chrono = "0.4"
+crc32fast = "1.4"
+crc32c = "0.6"
+crypto-common = "0.1"
+gethostname = "0.4"
+git-version = "0.3.4"
+hex = "0.4"
+hexdump = "0.1"
+html-escape = "0.2.13"
+hmac = "0.12"
+itertools = "0.12"
+ipnet = "2.9.0"
+lazy_static = "1.4"
+md-5 = "0.10"
+mktemp = "0.5"
+nix = { version = "0.29", default-features = false, features = ["fs"] }
+nom = "7.1"
+parking_lot = "0.12"
+parse_duration = "2.1"
+pin-project = "1.0.12"
+pnet_datalink = "0.34"
+rand = "0.8"
+sha1 = "0.10"
+sha2 = "0.10"
+timeago = { version = "0.4", default-features = false }
+xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] }
+
+aes-gcm = { version = "0.10", features = ["aes", "stream"] }
+sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
+kuska-handshake = { version = "0.2.0", features = ["default", "async_std"] }
+
+clap = { version = "4.1", features = ["derive", "env"] }
+pretty_env_logger = "0.5"
+structopt = { version = "0.3", default-features = false }
+syslog-tracing = "0.3"
+tracing = "0.1"
+tracing-journald = "0.3.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+
+heed = { version = "0.11", default-features = false, features = ["lmdb"] }
+rusqlite = "0.37"
+r2d2 = "0.8"
+r2d2_sqlite = "0.31"
+fjall = "2.4"
+
+async-compression = { version = "0.4", features = ["tokio", "zstd"] }
+zstd = { version = "0.13", default-features = false }
+
+quick-xml = { version = "0.26", features = [ "serialize" ] }
+rmp-serde = "1.1.2"
+serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
+serde_bytes = "0.11"
+serde_json = "1.0"
+toml = { version = "0.8", default-features = false, features = ["parse"] }
+
+# newer version requires rust edition 2021
+k8s-openapi = { version = "0.21", features = ["v1_24"] }
+kube = { version = "0.88", default-features = false, features = ["runtime", "derive", "client", "rustls-tls"] }
+schemars = "0.8"
+reqwest = { version = "0.11", default-features = false, features = ["rustls-tls-manual-roots", "json"] }
+
+form_urlencoded = "1.0.0"
+http = "1.0"
+httpdate = "1.0"
+http-range = "0.1"
+http-body-util = "0.1"
+hyper = { version = "1.0", default-features = false }
+hyper-util = { version = "0.1", features = [ "full" ] }
+multer = "3.0"
+percent-encoding = "2.2"
+roxmltree = "0.19"
+url = "2.3"
+
+futures = "0.3"
+futures-util = "0.3"
+tokio = { version = "1.0", default-features = false, features = ["net", "rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
+tokio-util = { version = "0.7", features = ["compat", "io"] }
+tokio-stream = { version = "0.1", features = ["net"] }
+
+opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] }
+opentelemetry-prometheus = "0.10"
+opentelemetry-otlp = "0.10"
+opentelemetry-contrib = "0.9"
+prometheus = "0.13"
+
+# used by the k2v-client crate only
+aws-sigv4 = { version = "1.1", default-features = false }
+hyper-rustls = { version = "0.26", default-features = false, features = ["http1", "http2", "ring", "rustls-native-certs"] }
+log = "0.4"
+thiserror = "2.0"
+
+# ---- used only as build / dev dependencies ----
+assert-json-diff = "2.0"
+rustc_version = "0.4.0"
+static_init = "1.0"
+aws-smithy-runtime = { version = "1.8", default-features = false, features = ["tls-rustls"] }
+aws-sdk-config = { version = "1.62", default-features = false }
+aws-sdk-s3 = { version = "1.79", default-features = false, features = ["rt-tokio"] }
 
 [profile.release]
-debug = true
+lto = "thin"
+codegen-units = 16
+opt-level = 3
+strip = "debuginfo"

Dockerfile

@@ -3,5 +3,5 @@ FROM scratch
 ENV RUST_BACKTRACE=1
 ENV RUST_LOG=garage=info
 
-COPY result-bin/bin/garage /
+COPY result/bin/garage /
 CMD [ "/garage", "server"]

Makefile

@@ -1,13 +1,8 @@
-.PHONY: doc all release shell run1 run2 run3
+.PHONY: doc all run1 run2 run3
 
 all:
-	clear; cargo build
+	clear
-
+	cargo build
-release:
-	nix-build --attr pkgs.amd64.release --no-build-output
-
-shell:
-	nix-shell
 
 # ----
 

README.md

@@ -1,4 +1,4 @@
-Garage [![Build Status](https://drone.deuxfleurs.fr/api/badges/Deuxfleurs/garage/status.svg?ref=refs/heads/main)](https://drone.deuxfleurs.fr/Deuxfleurs/garage)
+Garage [![status-badge](https://woodpecker.deuxfleurs.fr/api/badges/1/status.svg)](https://woodpecker.deuxfleurs.fr/repos/1)
 ===
 
 <p align="center" style="text-align:center;">

default.nix

@@ -3,54 +3,22 @@
 with import ./nix/common.nix;
 
 let
-  pkgs = import pkgsSrc { };
+  pkgs = import nixpkgs { };
   compile = import ./nix/compile.nix;
 
-  build_debug_and_release = (target: {
+  build_release = target: (compile {
-    debug = (compile {
+    inherit target system git_version nixpkgs;
-      inherit system target git_version pkgsSrc cargo2nixOverlay;
+    crane = flake.inputs.crane;
-      release = false;
+    rust-overlay = flake.inputs.rust-overlay;
-    }).workspace.garage { compileMode = "build"; };
-
-    release = (compile {
-      inherit system target git_version pkgsSrc cargo2nixOverlay;
     release = true;
-    }).workspace.garage { compileMode = "build"; };
+  }).garage;
-  });
-
-  test = (rustPkgs:
-    pkgs.symlinkJoin {
-      name = "garage-tests";
-      paths =
-        builtins.map (key: rustPkgs.workspace.${key} { compileMode = "test"; })
-        (builtins.attrNames rustPkgs.workspace);
-    });
 
 in {
-  pkgs = {
+  releasePackages = {
-    amd64 = build_debug_and_release "x86_64-unknown-linux-musl";
+    amd64 = build_release "x86_64-unknown-linux-musl";
-    i386 = build_debug_and_release "i686-unknown-linux-musl";
+    i386 = build_release "i686-unknown-linux-musl";
-    arm64 = build_debug_and_release "aarch64-unknown-linux-musl";
+    arm64 = build_release "aarch64-unknown-linux-musl";
-    arm = build_debug_and_release "armv6l-unknown-linux-musleabihf";
+    arm = build_release "armv6l-unknown-linux-musleabihf";
-  };
-  test = {
-    amd64 = test (compile {
-      inherit system git_version pkgsSrc cargo2nixOverlay;
-      target = "x86_64-unknown-linux-musl";
-      features = [
-        "garage/bundled-libs"
-        "garage/k2v"
-        "garage/sled"
-        "garage/lmdb"
-        "garage/sqlite"
-      ];
-    });
-  };
-  clippy = {
-    amd64 = (compile {
-      inherit system git_version pkgsSrc cargo2nixOverlay;
-      target = "x86_64-unknown-linux-musl";
-      compiler = "clippy";
-    }).workspace.garage { compileMode = "build"; };
   };
+  flakePackages = flake.packages.${system};
 }

doc/api/garage-admin-v0.yml

@@ -632,7 +632,7 @@ paths:
       operationId: "UpdateBucket"
       summary: "Update a bucket"
       description: |
-        All fields (`websiteAccess` and `quotas`) are optionnal.
+        All fields (`websiteAccess` and `quotas`) are optional.
         If they are present, the corresponding modifications are applied to the bucket, otherwise nothing is changed.
 
         In `websiteAccess`: if `enabled` is `true`, `indexDocument` must be specified.

doc/api/garage-admin-v1.html

@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Garage Adminstration API v0</title>
+    <!-- needed for adaptive design -->
+    <meta charset="utf-8"/>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link href="./css/redoc.css" rel="stylesheet">
+
+    <!--
+    Redoc doesn't change outer page styles
+    -->
+    <style>
+      body {
+        margin: 0;
+        padding: 0;
+      }
+    </style>
+  </head>
+  <body>
+    <redoc spec-url='./garage-admin-v1.yml'></redoc>
+    <script src="./redoc.standalone.js"> </script>
+  </body>
+</html>

doc/book/build/_index.md

@@ -1,6 +1,6 @@
 +++
 title = "Build your own app"
-weight = 4
+weight = 40
 sort_by = "weight"
 template = "documentation.html"
 +++

doc/book/build/golang.md

@@ -37,30 +37,84 @@ import (
     "context"
     "fmt"
     "os"
+    "strings"
     garage "git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-golang"
 )
 
 func main() {
-    // Set Host and other parameters
+    // Initialization
     configuration := garage.NewConfiguration()
     configuration.Host = "127.0.0.1:3903"
-
-
-    // We can now generate a client
     client := garage.NewAPIClient(configuration)
-
-    // Authentication is handled through the context pattern
     ctx := context.WithValue(context.Background(), garage.ContextAccessToken, "s3cr3t")
 
-    // Send a request
+    // Nodes
-    resp, r, err := client.NodesApi.GetNodes(ctx).Execute()
+    fmt.Println("--- nodes ---")
-    if err != nil {
+    nodes, _, _ := client.NodesApi.GetNodes(ctx).Execute()
-        fmt.Fprintf(os.Stderr, "Error when calling `NodesApi.GetNodes``: %v\n", err)
+    fmt.Fprintf(os.Stdout, "First hostname: %v\n", nodes.KnownNodes[0].Hostname)
-        fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r)
+    capa := int64(1000000000)
+    change := []garage.NodeRoleChange{
+	    garage.NodeRoleChange{NodeRoleUpdate: &garage.NodeRoleUpdate {
+	        Id: *nodes.KnownNodes[0].Id,
+	        Zone: "dc1",
+	        Capacity: *garage.NewNullableInt64(&capa),
+	        Tags: []string{ "fast", "amd64" },
+	    }},
     }
+    staged, _, _ := client.LayoutApi.AddLayout(ctx).NodeRoleChange(change).Execute()
+    msg, _, _ := client.LayoutApi.ApplyLayout(ctx).LayoutVersion(*garage.NewLayoutVersion(staged.Version + 1)).Execute()
+    fmt.Printf(strings.Join(msg.Message, "\n")) // Layout configured
 
-    // Process the response
+    health, _, _ := client.NodesApi.GetHealth(ctx).Execute()
-    fmt.Fprintf(os.Stdout, "Target hostname: %v\n", resp.KnownNodes[resp.Node].Hostname)
+    fmt.Printf("Status: %s, nodes: %v/%v, storage: %v/%v, partitions: %v/%v\n", health.Status, health.ConnectedNodes, health.KnownNodes, health.StorageNodesOk, health.StorageNodes, health.PartitionsAllOk, health.Partitions)
+
+    // Key
+    fmt.Println("\n--- key ---")
+    key := "openapi-key"
+    keyInfo, _, _ := client.KeyApi.AddKey(ctx).AddKeyRequest(garage.AddKeyRequest{Name: *garage.NewNullableString(&key) }).Execute()
+    defer client.KeyApi.DeleteKey(ctx).Id(*keyInfo.AccessKeyId).Execute()
+    fmt.Printf("AWS_ACCESS_KEY_ID=%s\nAWS_SECRET_ACCESS_KEY=%s\n", *keyInfo.AccessKeyId, *keyInfo.SecretAccessKey.Get())
+
+    id := *keyInfo.AccessKeyId
+    canCreateBucket := true
+    updateKeyRequest := *garage.NewUpdateKeyRequest()
+    updateKeyRequest.SetName("openapi-key-updated")
+    updateKeyRequest.SetAllow(garage.UpdateKeyRequestAllow { CreateBucket: &canCreateBucket })
+    update, _, _ := client.KeyApi.UpdateKey(ctx).Id(id).UpdateKeyRequest(updateKeyRequest).Execute()
+    fmt.Printf("Updated %v with key name %v\n", *update.AccessKeyId, *update.Name)
+
+    keyList, _, _ := client.KeyApi.ListKeys(ctx).Execute()
+    fmt.Printf("Keys count: %v\n", len(keyList))
+
+    // Bucket
+    fmt.Println("\n--- bucket ---")
+    global_name := "global-ns-openapi-bucket"
+    local_name := "local-ns-openapi-bucket"
+    bucketInfo, _, _ := client.BucketApi.CreateBucket(ctx).CreateBucketRequest(garage.CreateBucketRequest{
+        GlobalAlias: &global_name,
+        LocalAlias: &garage.CreateBucketRequestLocalAlias {
+            AccessKeyId: keyInfo.AccessKeyId,
+	    Alias: &local_name,
+        },
+    }).Execute()
+    defer client.BucketApi.DeleteBucket(ctx).Id(*bucketInfo.Id).Execute()
+    fmt.Printf("Bucket id: %s\n", *bucketInfo.Id)
+
+    updateBucketRequest := *garage.NewUpdateBucketRequest()
+    website := garage.NewUpdateBucketRequestWebsiteAccess()
+    website.SetEnabled(true)
+    website.SetIndexDocument("index.html")
+    website.SetErrorDocument("errors/4xx.html")
+    updateBucketRequest.SetWebsiteAccess(*website)
+    quotas := garage.NewUpdateBucketRequestQuotas()
+    quotas.SetMaxSize(1000000000)
+    quotas.SetMaxObjects(999999999)
+    updateBucketRequest.SetQuotas(*quotas)
+    updatedBucket, _, _ := client.BucketApi.UpdateBucket(ctx).Id(*bucketInfo.Id).UpdateBucketRequest(updateBucketRequest).Execute()
+    fmt.Printf("Bucket %v website activation: %v\n", *updatedBucket.Id, *updatedBucket.WebsiteAccess)
+
+    bucketList, _, _ := client.BucketApi.ListBuckets(ctx).Execute()
+    fmt.Printf("Bucket count: %v\n", len(bucketList))
 }
 ```
 

doc/book/build/javascript.md

@@ -31,9 +31,9 @@ npm install --save git+https://git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-js.
 A short example:
 
 ```javascript
-const garage = require('garage_administration_api_v0garage_v0_8_0');
+const garage = require('garage_administration_api_v1garage_v0_9_0');
 
-const api = new garage.ApiClient("http://127.0.0.1:3903/v0");
+const api = new garage.ApiClient("http://127.0.0.1:3903/v1");
 api.authentications['bearerAuth'].accessToken = "s3cr3t";
 
 const [node, layout, key, bucket] = [

doc/book/build/python.md

@@ -23,7 +23,7 @@ client = minio.Minio(
   "GKyourapikey",
   "abcd[...]1234",
   # Force the region, this is specific to garage
-  region="region",
+  region="garage",
 )
 ```
 
@@ -80,7 +80,7 @@ from garage_admin_sdk.apis import *
 from garage_admin_sdk.models import *
 
 configuration = garage_admin_sdk.Configuration(
-  host = "http://localhost:3903/v0",
+  host = "http://localhost:3903/v1",
   access_token = "s3cr3t"
 )
 
@@ -94,13 +94,14 @@ print(f"running garage {status.garage_version}, node_id {status.node}")
 
 # Change layout of this node
 current = layout.get_layout()
-layout.add_layout({
+layout.add_layout([
-  status.node: NodeClusterInfo(
+  NodeRoleChange(
+    id = status.node,
     zone = "dc1",
-    capacity = 1,
+    capacity = 1000000000,
     tags = [ "dev" ],
   )
-})
+])
 layout.apply_layout(LayoutVersion(
   version = current.version + 1
 ))

doc/book/connect/_index.md

@@ -1,6 +1,6 @@
 +++
 title = "Existing integrations"
-weight = 3
+weight = 30
 sort_by = "weight"
 template = "documentation.html"
 +++

doc/book/connect/apps/index.md

@@ -11,7 +11,8 @@ In this section, we cover the following web applications:
 | [Peertube](#peertube)     | ✅       | Supported with the website endpoint, proxifying private videos unsupported     |
 | [Mastodon](#mastodon)     | ✅       | Natively supported    |
 | [Matrix](#matrix)     | ✅       |  Tested with `synapse-s3-storage-provider`    |
-| [Pixelfed](#pixelfed)     | ❓       |  Not yet tested    |
+| [ejabberd](#ejabberd)     | ✅       |  `mod_s3_upload`    |
+| [Pixelfed](#pixelfed)     | ✅       |  Natively supported    |
 | [Pleroma](#pleroma)     | ❓       |  Not yet tested    |
 | [Lemmy](#lemmy)     | ✅        |  Supported with pict-rs    |
 | [Funkwhale](#funkwhale)     | ❓       | Not yet tested     |
@@ -36,7 +37,7 @@ Second, we suppose you have created a key and a bucket.
 As a reminder, you can create a key for your nextcloud instance as follow:
 
 ```bash
-garage key new --name nextcloud-key
+garage key create nextcloud-key
 ```
 
 Keep the Key ID and the Secret key in a pad, they will be needed later.  
@@ -68,7 +69,7 @@ $CONFIG = array(
         'hostname' => '127.0.0.1', // Can also be a domain name, eg. garage.example.com
         'port' => 3900,            // Put your reverse proxy port or your S3 API port
         'use_ssl' => false,        // Set it to true if you have a TLS enabled reverse proxy
-        'region' => 'garage',      // Garage has only one region named "garage"
+        'region' => 'garage',      // Garage default region is named "garage", edit according to your cluster config
         'use_path_style' => true   // Garage supports only path style, must be set to true
     ],
 ],
@@ -79,6 +80,53 @@ To test your new configuration, just reload your Nextcloud webpage and start sen
 
 *External link:* [Nextcloud Documentation > Primary Storage](https://docs.nextcloud.com/server/latest/admin_manual/configuration_files/primary_storage.html)
 
+#### SSE-C encryption (since Garage v1.0)
+
+Since version 1.0, Garage supports server-side encryption with customer keys
+(SSE-C).  In this mode, Garage is responsible for encrypting and decrypting
+objects, but it does not store the encryption key itself. The encryption key
+should be provided by Nextcloud upon each request.  This mode of operation is
+supported by Nextcloud and it has successfully been tested together with
+Garage.
+
+To enable SSE-C encryption:
+
+1. Make sure your Garage server is accessible via SSL through a reverse proxy
+   such as Nginx, and that it is using a valid public certificate (Nextcloud
+   might be able to connect to an S3 server that is using a self-signed
+   certificate, but you will lose many hours while trying, so don't).
+   Configure values for `use_ssl` and `port` accordingly in your `config.php`
+   file.
+
+2. Generate an encryption key using the following command:
+
+    ```
+    openssl rand -base64 32
+    ```
+
+    Make sure to keep this key **secret**!
+
+3. Add the encryption key in your `config.php` file as follows:
+
+
+    ```php
+    <?php
+    $CONFIG = array(
+    'objectstore' => [
+        'class' => '\\OC\\Files\\ObjectStore\\S3',
+        'arguments' => [
+            ...
+            'sse_c_key' => 'exampleencryptionkeyLbU+5fKYQcVoqnn+RaIOXgo=',
+            ...
+        ],
+    ],
+    ```
+
+Nextcloud will now make Garage encrypt files at rest in the storage bucket.
+These files will not be readable by an S3 client that has credentials to the
+bucket but doesn't also know the secret encryption key.
+
+
 ### External Storage
 
 **From the GUI.** Activate the "External storage support" app from the "Applications" page (click on your account icon on the top right corner of your screen to display the menu). Go to your parameters page (also located below your account icon). Click on external storage (or the corresponding translation in your language).
@@ -87,7 +135,7 @@ To test your new configuration, just reload your Nextcloud webpage and start sen
 *Click on the picture to zoom*
 
 Add a new external storage. Put what you want in "folder name" (eg. "shared"). Select "Amazon S3". Keep "Access Key" for the Authentication field.
-In Configuration, put your bucket name (eg. nextcloud), the host (eg. 127.0.0.1), the port (eg. 3900 or 443), the region (garage). Tick the SSL box if you have put an HTTPS proxy in front of garage. You must tick the "Path access" box and you must leave the "Legacy authentication (v2)" box empty. Put your Key ID (eg. GK...) and your Secret Key in the last two input boxes. Finally click on the tick symbol on the right of your screen.
+In Configuration, put your bucket name (eg. nextcloud), the host (eg. 127.0.0.1), the port (eg. 3900 or 443), the region ("garage" if you use the default, or the one your configured in your `garage.toml`). Tick the SSL box if you have put an HTTPS proxy in front of garage. You must tick the "Path access" box and you must leave the "Legacy authentication (v2)" box empty. Put your Key ID (eg. GK...) and your Secret Key in the last two input boxes. Finally click on the tick symbol on the right of your screen.
 
 Now go to your "Files" app and a new "linked folder" has appeared with the name you chose earlier (eg. "shared").
 
@@ -138,15 +186,15 @@ a reasonable trade-off for some instances.
 Create a key for Peertube:
 
 ```bash
-garage key new --name peertube-key
+garage key create peertube-key
 ```
 
 Keep the Key ID and the Secret key in a pad, they will be needed later.  
 
-We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist).
+We need two buckets, one for normal videos (named peertube-videos) and one for webtorrent videos (named peertube-playlists).
 ```bash
-garage bucket create peertube-video
+garage bucket create peertube-videos
-garage bucket create peertube-playlist
+garage bucket create peertube-playlists
 ```
 
 Now we allow our key to read and write on these buckets:
@@ -190,7 +238,7 @@ object_storage:
   # Put localhost only if you have a garage instance running on that node
   endpoint: 'http://localhost:3900' # or "garage.example.com" if you have TLS on port 443
 
-  # Garage supports only one region for now, named garage
+  # Garage default region is named "garage", edit according to your config
   region: 'garage'
 
   credentials:
@@ -205,7 +253,7 @@ object_storage:
     proxify_private_files: false
 
   streaming_playlists:
-    bucket_name: 'peertube-playlist'
+    bucket_name: 'peertube-playlists'
 
     # Keep it empty for our example
     prefix: ''
@@ -215,7 +263,7 @@ object_storage:
 
   # Same settings but for webtorrent videos
   videos:
-    bucket_name: 'peertube-video'
+    bucket_name: 'peertube-videos'
     prefix: ''
     # You must fill this field to make Peertube use our reverse proxy/website logic
     base_url: 'http://peertube-videos.web.garage.localhost'
@@ -244,7 +292,7 @@ with average object size ranging from 50 KB to 150 KB.
 As such, your Garage cluster should be configured appropriately for good performance:
 
 - use Garage v0.8.0 or higher with the [LMDB database engine](@documentation/reference-manual/configuration.md#db-engine-since-v0-8-0).
-  With the default Sled database engine, your database could quickly end up taking tens of GB of disk space.
+  Older versions of Garage used the Sled database engine which had issues, such as databases quickly ending up taking tens of GB of disk space.
 - the Garage database should be stored on a SSD
 
 ### Creating your bucket
@@ -252,7 +300,7 @@ As such, your Garage cluster should be configured appropriately for good perform
 This is the usual Garage setup:
 
 ```bash
-garage key new --name mastodon-key
+garage key create mastodon-key
 garage bucket create mastodon-data
 garage bucket allow mastodon-data --read --write --key mastodon-key
 ```
@@ -287,6 +335,7 @@ From the [official Mastodon documentation](https://docs.joinmastodon.org/admin/t
 
 ```bash
 $ RAILS_ENV=production bin/tootctl media remove --days 3
+$ RAILS_ENV=production bin/tootctl media remove --days 15 --prune-profiles
 $ RAILS_ENV=production bin/tootctl media remove-orphans
 $ RAILS_ENV=production bin/tootctl preview_cards remove --days 15
 ```
@@ -305,8 +354,6 @@ Imports:	1.7 KB
 Settings:	0 Bytes
 ```
 
-Unfortunately, [old avatars and headers cannot currently be cleaned up](https://github.com/mastodon/mastodon/issues/9567).
-
 ### Migrating your data
 
 Data migration should be done with an efficient S3 client.
@@ -378,7 +425,7 @@ Supposing you have a working synapse installation, you can add the module with p
 Now create a bucket and a key for your matrix instance (note your Key ID and Secret Key somewhere, they will be needed later):
 
 ```bash
-garage key new --name matrix-key
+garage key create matrix-key
 garage bucket create matrix
 garage bucket allow matrix --read --write --key matrix-key
 ```
@@ -394,7 +441,7 @@ media_storage_providers:
   store_synchronous: True  # do we want to wait that the file has been written before returning?
   config:
     bucket: matrix       # the name of our bucket, we chose matrix earlier
-    region_name: garage  # only "garage" is supported for the region field
+    region_name: garage  # "garage" by default, edit according to your cluster config
     endpoint_url: http://localhost:3900 # the path to the S3 endpoint
     access_key_id: "GKxxx" # your Key ID
     secret_access_key: "xxxx" # your Secret Key
@@ -420,7 +467,7 @@ Now we can write a simple script (eg `~/.local/bin/matrix-cache-gc`):
 ## CONFIGURATION ##
 AWS_ACCESS_KEY_ID=GKxxx
 AWS_SECRET_ACCESS_KEY=xxxx
-S3_ENDPOINT=http://localhost:3900
+AWS_ENDPOINT_URL=http://localhost:3900
 S3_BUCKET=matrix
 MEDIA_STORE=/var/lib/matrix-synapse/media
 PG_USER=matrix
@@ -441,7 +488,7 @@ EOF
 
 s3_media_upload update-db 1d
 s3_media_upload --no-progress check-deleted $MEDIA_STORE
-s3_media_upload --no-progress upload $MEDIA_STORE $S3_BUCKET --delete --endpoint-url $S3_ENDPOINT
+s3_media_upload --no-progress upload $MEDIA_STORE $S3_BUCKET --delete --endpoint-url $AWS_ENDPOINT_URL
 ```
 
 This script will list all the medias that were not accessed in the 24 hours according to your database.
@@ -474,6 +521,52 @@ And add a new line. For example, to run it every 10 minutes:
 
 *External link:* [matrix-media-repo Documentation > S3](https://docs.t2bot.io/matrix-media-repo/configuration/s3-datastore.html)
 
+## ejabberd
+
+ejabberd is an XMPP server implementation which, with the `mod_s3_upload`
+module in the [ejabberd-contrib](https://github.com/processone/ejabberd-contrib)
+repository, can be integrated to store chat media files in Garage.
+
+For uploads, this module leverages presigned URLs - this allows XMPP clients to
+directly send media to Garage. Receiving clients then retrieve this media
+through the [static website](@/documentation/cookbook/exposing-websites.md)
+functionality.
+
+As the data itself is publicly accessible to someone with knowledge of the
+object URL - users are recommended to use
+[E2EE](@/documentation/cookbook/encryption.md) to protect this data-at-rest
+from unauthorized access.
+
+Install the module with:
+
+```bash
+ejabberdctl module_install mod_s3_upload
+```
+
+Create the required key and bucket with:
+
+```bash
+garage key new --name ejabberd
+garage bucket create objects.xmpp-server.fr
+garage bucket allow objects.xmpp-server.fr --read --write --key ejabberd
+garage bucket website --allow objects.xmpp-server.fr
+```
+
+The module can then be configured with:
+
+```
+  mod_s3_upload:
+    #bucket_url: https://objects.xmpp-server.fr.my-garage-instance.mydomain.tld
+    bucket_url: https://my-garage-instance.mydomain.tld/objects.xmpp-server.fr
+    access_key_id: GK...
+    access_key_secret: ...
+    region: garage
+    download_url: https://objects.xmpp-server.fr
+```
+
+Other configuration options can be found in the
+[configuration YAML file](https://github.com/processone/ejabberd-contrib/blob/master/mod_s3_upload/conf/mod_s3_upload.yml).
+
 ## Pixelfed
 
 [Pixelfed Technical Documentation > Configuration](https://docs.pixelfed.org/technical-documentation/env.html#filesystem)
@@ -539,7 +632,7 @@ secret_key = 'abcdef0123456789...'
 
 ```
 PICTRS__STORE__TYPE=object_storage
-PICTRS__STORE__ENDPOINT=http:/my-garage-instance.mydomain.tld:3900
+PICTRS__STORE__ENDPOINT=http://my-garage-instance.mydomain.tld:3900
 PICTRS__STORE__BUCKET_NAME=pictrs-data
 PICTRS__STORE__REGION=garage
 PICTRS__STORE__ACCESS_KEY=GK...